diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh index 9f72f2d6e64..740d626beea 100755 --- a/.github/e2e-stack/down.sh +++ b/.github/e2e-stack/down.sh @@ -10,7 +10,7 @@ for pid_file in "${STACK_DIR}"/pids/*.pid; do rm -f "${pid_file}" done -for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do +for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do docker rm -f "${container}" >/dev/null 2>&1 done diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index a62358f81ff..238818a0d36 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile( ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" + r"|^tests/e2e/idp_realm\.json$" r"|^tests/e2e/gateway/" r"|^\.github/e2e-stack/" r"|^\.github/workflows/test-e2e-changed\.yml$" diff --git a/.github/e2e-stack/start-idp.sh b/.github/e2e-stack/start-idp.sh new file mode 100644 index 00000000000..e59a7ade34c --- /dev/null +++ b/.github/e2e-stack/start-idp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEYCLOAK_IMAGE="${E2E_KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:ff4257d0d64efbe99ed1ddfaf07765cc3c36dc7518bf8324d41961327f441c54}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" +POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" +: "${DATABASE_HOST:?}" "${DATABASE_PORT:?}" "${DATABASE_USER:?}" "${DATABASE_PASSWORD:?}" "${DATABASE_NAME:?}" + +DB_HOST="${DATABASE_HOST}" +DB_NETWORK_ARGS=(--network bridge) +IDP_NETWORK_ARGS=(-p "127.0.0.1:${KEYCLOAK_PORT}:${KEYCLOAK_PORT}") +if [[ "$(uname)" == "Linux" ]]; then + DB_NETWORK_ARGS=(--network host) + IDP_NETWORK_ARGS=(--network host) +elif [[ "${DB_HOST}" == "127.0.0.1" || "${DB_HOST}" == "localhost" ]]; then + DB_HOST=host.docker.internal +fi + +docker run --rm "${DB_NETWORK_ARGS[@]}" -e "PGPASSWORD=${DATABASE_PASSWORD}" \ + "${POSTGRES_IMAGE}" psql -h "${DB_HOST}" -p "${DATABASE_PORT}" \ + -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -v ON_ERROR_STOP=1 \ + -c 'CREATE SCHEMA IF NOT EXISTS keycloak' >/dev/null + +docker rm -f e2e-keycloak >/dev/null 2>&1 || true +docker run -d --name e2e-keycloak "${IDP_NETWORK_ARGS[@]}" --memory 1536m \ + -v "${REPO_ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + -e KC_DB=postgres -e "KC_DB_URL_HOST=${DB_HOST}" -e "KC_DB_URL_PORT=${DATABASE_PORT}" \ + -e "KC_DB_URL_DATABASE=${DATABASE_NAME}" -e KC_DB_SCHEMA=keycloak \ + -e "KC_DB_USERNAME=${DATABASE_USER}" -e "KC_DB_PASSWORD=${DATABASE_PASSWORD}" \ + -e KC_DB_POOL_INITIAL_SIZE=2 -e KC_DB_POOL_MIN_SIZE=2 -e KC_DB_POOL_MAX_SIZE=10 \ + -e "KC_HTTP_PORT=${KEYCLOAK_PORT}" -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + -e KC_BOOTSTRAP_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret \ + "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null + +deadline=$((SECONDS + ${E2E_KEYCLOAK_STARTUP_TIMEOUT:-300})) +until curl -fsS --connect-timeout 2 --max-time 3 \ + "http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/.well-known/openid-configuration" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo 'e2e-stack: timed out waiting for the Keycloak realm' >&2 + exit 1 + fi + sleep 2 +done +echo 'e2e-stack: Keycloak realm is up' diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 2f2e6c6f9a8..a789a570483 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -25,6 +25,7 @@ DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" @@ -124,6 +125,9 @@ SERVER_ENV=( "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" + "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" + "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" + "JWT_AUDIENCE=litellm-e2e" ) if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" @@ -132,6 +136,8 @@ fi cd "${REPO_ROOT}" +env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh + log "running migrations" env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 @@ -200,6 +206,9 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} +E2E_KEYCLOAK_ADMIN_USER=admin +E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME} EOF diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e85a397cbd2..1a2c81d1f92 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -127,6 +127,8 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac - Low: anything else worth noting: naming, cleanup, an edge case nobody hits Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a human reader + If you assumed something instead of testing it, e.g. "only reproduces with X on" or "no + user-observable behavior difference", list it here too with what breaks if it is wrong Leave this section empty if there are none --> ## QA runbook diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 899e2a211c0..4fb8f068eb0 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -205,7 +205,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 20_000_000 + native_size_limit: Final = 25_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), @@ -222,7 +222,7 @@ def main( ("Python extension entry point is present", extension_entry_point_present), ("Native module loads", native_module_loads), ("Production module omits the panic test hook", panic_test_hook_absent), - ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Native extension does not exceed 25 MB", native_size_within_limit), ("Wheel contents are valid", not unexpected_members), ) diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index e6d2264fbf0..9c7e0db7065 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -81,7 +81,7 @@ jobs: run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py - name: test_e2e_changed_gate - run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 23ab6dfcfe4..1db597ff673 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -27,6 +27,8 @@ jobs: sparse-checkout: | .github/e2e-stack tests/e2e/access_control + tests/e2e/management/test_jwt_management_e2e.py + tests/e2e/other/test_jwt_auth_e2e.py persist-credentials: false ref: ${{ github.sha }} @@ -45,7 +47,8 @@ jobs: --jq '.[] | select(.status != "removed") | .filename')" gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" tests="$(printf '%s\n' "${files}" \ - | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)" + | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \ + tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)" echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml new file mode 100644 index 00000000000..c7412a63334 --- /dev/null +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -0,0 +1,103 @@ +name: "Redis Chaos E2E" + +on: + workflow_dispatch: + workflow_call: + inputs: + ref: + description: "Commit SHA or ref to test. Defaults to the ref the workflow was triggered on" + required: false + type: string + +permissions: + contents: read + +jobs: + redis-chaos-e2e: + runs-on: ubuntu-latest-16-cores + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U llmproxy" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + valkey: + image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + LITELLM_MASTER_KEY: sk-redis-chaos-e2e + LITELLM_LOG: WARNING + JSON_LOGS: "true" + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + ref: ${{ inputs.ref || github.sha }} + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Start a multi-worker proxy on the chaos config + run: | + nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 & + echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV" + echo "E2E_PROXY_LOG=$(pwd)/proxy.log" >> "$GITHUB_ENV" + for _ in $(seq 1 90); do + if curl -fs http://localhost:4000/health/liveliness > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "proxy never became live" + tail -n 100 proxy.log + exit 1 + + - name: Run the Redis chaos load test + env: + E2E_REDIS_CHAOS: "1" + LITELLM_PROXY_URL: http://localhost:4000 + REDIS_HOST: 127.0.0.1 + REDIS_PORT: "6379" + run: | + uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s + + - name: Show proxy log on failure + if: failure() + run: tail -n 300 proxy.log diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py index bfbfd7bfb15..f0f85178672 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py @@ -11,10 +11,14 @@ import sys sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import functools +import configparser +import contextlib +import itertools +import re import tempfile +from collections.abc import Generator, Iterator, Sequence from contextvars import ContextVar -from typing import TYPE_CHECKING, ClassVar, Literal, Optional +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -433,12 +437,101 @@ _default_detect_secrets_config = { "name": "ZendeskSecretKeyDetector", "path": _custom_plugins_path + "/zendesk_secret_key.py", }, + { + "name": "CredentialKeywordDetector", + "path": _custom_plugins_path + "/credential_keyword.py", + }, {"name": "Base64HighEntropyString", "limit": 4.5}, {"name": "HexHighEntropyString", "limit": 3.0}, ], } +_CONFIG_SECTION: Final = "litellm-prompt" + +_ASSIGNMENT_LINE: Final = re.compile(r"[^\s\[#;:=][^:=]*[:=]") + +_SHELL_ASSIGNMENT: Final = re.compile(r"(?P[^\s\[#;:=](?:[^:=]*[^\s:=])?)=(?P\S+)") + +_SHELL_OPERATORS: Final = ";&|" + +_SHELL_TRAILER: Final = re.compile(r"\\|#.*|-*\w[\w.-]*=\S*") + +_SCAN_SUFFIX: Final = ".py" + + +@contextlib.contextmanager +def _temp_file(text: str) -> Generator[str, None, None]: + temp_file: Final = tempfile.NamedTemporaryFile(suffix=_SCAN_SUFFIX, delete=False) + try: + temp_file.write(text.encode("utf-8")) + temp_file.close() + yield temp_file.name + finally: + temp_file.close() + os.remove(temp_file.name) + + +def _scan_lines(lines: Sequence[str]) -> frozenset[tuple[str, str]]: + from detect_secrets import SecretsCollection + + secrets: Final = SecretsCollection() + with _temp_file("\n".join(lines)) as path: + secrets.scan_file(path) + + return frozenset( + (found_secret.secret_value, found_secret.type) + for file in secrets.files + for found_secret in secrets[file] + if found_secret.secret_value is not None + ) + + +def _classify_line(state: tuple[bool, str | None], numbered: tuple[int, str]) -> tuple[bool, str | None]: + open_option: Final = state[0] + number, line = numbered + stripped: Final = line.strip() + if not stripped or stripped[0] in "#;": + return open_option, None + shell_assignment: Final = _SHELL_ASSIGNMENT.match(stripped) + if shell_assignment is not None: + return True, f"{shell_assignment['key']}_{number}={shell_assignment['value']}" + assignment: Final = _ASSIGNMENT_LINE.match(stripped) + if assignment is not None: + return True, f"{assignment.group()[:-1].strip()}_{number}{stripped[assignment.end() - 1 :]}" + if line[0].isspace() and open_option: + return True, line + return False, None + + +def _parseable_lines(text: str) -> Iterator[str]: + states: Final = itertools.accumulate(enumerate(text.splitlines()), _classify_line, initial=(False, None)) + return (line for _, line in states if line is not None) + + +def _lone_value(line: str) -> str | None: + tokens: Final = line.split() + if not tokens or '"' in tokens[0]: + return None + value: Final = tokens[0].rstrip(_SHELL_OPERATORS) + if len(tokens) == 1 or value != tokens[0] or _SHELL_TRAILER.fullmatch(tokens[1]) is not None: + return value + return None + + +def _quoted_assignments(text: str) -> tuple[str, ...]: + parser: Final = configparser.ConfigParser(interpolation=None) + parser.optionxform = str # pyright: ignore[reportAttributeAccessIssue] # configparser types optionxform as a method + parser.read_string(f"[{_CONFIG_SECTION}]\n" + "\n".join(_parseable_lines(text))) + return tuple( + f'{key} = "{value}"' + for section in parser + for key, values in parser.items(section) + for line in values.splitlines() + if (value := _lone_value(line)) is not None + ) + + class _ENTERPRISE_SecretDetection(CustomGuardrail): # Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail # path skips should_run_check and never sees data["prompt"]). @@ -449,35 +542,21 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail): super().__init__(**kwargs) def scan_message_for_secrets(self, message_content: str): - from detect_secrets import SecretsCollection from detect_secrets.settings import transient_settings - temp_file = tempfile.NamedTemporaryFile(delete=False) - temp_file.write(message_content.encode("utf-8")) - temp_file.close() - - secrets = SecretsCollection() - detect_secrets_config = ( self.user_defined_detect_secrets_config or _default_detect_secrets_config ) with transient_settings(detect_secrets_config): - secrets.scan_file(temp_file.name) - - os.remove(temp_file.name) + found: Final = _scan_lines( + (*message_content.splitlines(), *_quoted_assignments(message_content)) + ) return [ - {"type": found_secret.type, "value": found_secret.secret_value} - for file in sorted(secrets.files) - for found_secret in sorted( - secrets[file], - key=lambda secret: ( - -len(secret.secret_value or ""), - secret.type, - secret.secret_value or "", - ), + {"type": secret_type, "value": value} + for value, secret_type in sorted( + found, key=lambda pair: (-len(pair[0]), pair[1], pair[0]) ) - if found_secret.secret_value is not None ] def redact_text(self, text: str, source: str = "message") -> str: @@ -490,15 +569,16 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail): if counts is not None: for secret in detected_secrets: counts[secret["type"]] = counts.get(secret["type"], 0) + 1 - secret_types = [secret["type"] for secret in detected_secrets] + secret_types: Final = sorted( + dict.fromkeys(secret["type"] for secret in detected_secrets) + ) verbose_proxy_logger.warning( - f"Detected and redacted secrets in {source}: {secret_types}" + "Detected and redacted secrets in %s: %s", source, secret_types ) - return functools.reduce( - lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"), - detected_secrets, - text, + pattern: Final = re.compile( + "|".join(re.escape(secret["value"]) for secret in detected_secrets) ) + return pattern.sub("[REDACTED]", text) async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool: if user_api_key_dict.permissions is not None: diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/credential_keyword.py b/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/credential_keyword.py new file mode 100644 index 00000000000..b69e347ded5 --- /dev/null +++ b/enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/credential_keyword.py @@ -0,0 +1,63 @@ +import re +from collections.abc import Generator, Mapping +from string import punctuation +from typing import Final + +from detect_secrets.plugins.keyword import ( + QUOTES_REQUIRED_DENYLIST_REGEX_TO_GROUP, + KeywordDetector, +) + +_CREDENTIAL_VALUE: Final = re.compile(r"[^\s()\[\]]+") +_ENVIRONMENT_REFERENCE: Final = re.compile(r"os\.environ/\w+", re.IGNORECASE) +_ENVIRONMENT_VARIABLE_NAME: Final = re.compile(r"[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+") +_LOWERCASE_WORD_SEQUENCE: Final = re.compile(r"[a-z]+(?:[-._/][a-z]+)+") +_ISO_8601_TIMESTAMP: Final = re.compile( + r"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?" +) +_URL_WITHOUT_USERINFO_OR_QUERY: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://[^\s@?]*") +_BENIGN_VALUES: Final = ( + _ENVIRONMENT_REFERENCE, + _ENVIRONMENT_VARIABLE_NAME, + _LOWERCASE_WORD_SEQUENCE, + _ISO_8601_TIMESTAMP, + _URL_WITHOUT_USERINFO_OR_QUERY, +) + + +class CredentialKeywordDetector(KeywordDetector): # pyright: ignore[reportUntypedBaseClass] # detect_secrets ships no type information + secret_type = "Credential Keyword" + + def __init__(self, minimum_length: int = 12, keyword_exclude: str | None = None) -> None: + if ( + not isinstance(minimum_length, int) # pyright: ignore[reportUnnecessaryIsInstance] # the value comes from an operator's YAML + or minimum_length < 1 + ): + raise ValueError(f"minimum_length must be a positive integer, got {minimum_length!r}") + super().__init__(keyword_exclude=keyword_exclude) + self.minimum_length = minimum_length + + def _is_credential(self, value: str) -> bool: + core: Final = value.strip(punctuation) + return ( + len(value) >= self.minimum_length + and _CREDENTIAL_VALUE.fullmatch(value) is not None + and all(benign.fullmatch(core) is None for benign in _BENIGN_VALUES) + ) + + def analyze_string( + self, + string: str, + denylist_regex_to_group: Mapping[re.Pattern[str], int] | None = None, + ) -> Generator[str, None, None]: + if self.keyword_exclude is not None and self.keyword_exclude.search(string): + return + regex_to_group: Final = ( + QUOTES_REQUIRED_DENYLIST_REGEX_TO_GROUP if denylist_regex_to_group is None else denylist_regex_to_group + ) + yield from ( + match.group(group) + for regex, group in regex_to_group.items() + for match in regex.finditer(string) + if self._is_credential(match.group(group)) + ) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 92b73867e67..3733072a948 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -3,7 +3,8 @@ The gateway exposes the LLM data-plane surface: chat/completions, embeddings, audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image, responses, vector stores, passthrough providers, realtime websockets, MCP -tool-call endpoints, and operational endpoints (/health, /metrics). +tool-call endpoints, and operational endpoints (/health, /metrics, and the +/debug/memory/summary read of the serving worker's RSS). Any path not listed here is dropped from the gateway process so management/UI endpoints don't ride on the same pods. @@ -121,6 +122,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/docs/oauth2-redirect", "/redoc", "/test", + "/debug/memory/summary", } ) diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 692a799e783..20fd1a722dc 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -257,6 +257,14 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets. - name: DATABASE_SCHEMA value: {{ .schema | quote }} {{- end }} +{{- if .sslMode }} +- name: DATABASE_SSLMODE + value: {{ .sslMode | quote }} +{{- end }} +{{- if .sslRootCert }} +- name: DATABASE_SSLROOTCERT + value: {{ .sslRootCert | quote }} +{{- end }} {{- if and .useIAMAuth .useAzureEntraAuth }} {{- fail "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" }} {{- end }} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index 732564b280f..d42558b9396 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -89,7 +89,7 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}} +{{- $builtinPathKeys := list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix" -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -129,6 +129,8 @@ spec: # --- Gateway data plane --- # Exact /test only (see the $gatewayPrefixes comment above); # /test/* MCP management endpoints fall to the backend catch-all. + # Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory + # gate); the rest of /debug/* stays on the backend. - path: /test pathType: Exact backend: @@ -136,6 +138,13 @@ spec: name: {{ $gatewayName }} port: number: {{ $gatewayPort }} + - path: /debug/memory/summary + pathType: Exact + backend: + service: + name: {{ $gatewayName }} + port: + number: {{ $gatewayPort }} {{- range $gatewayPrefixes }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }} {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }} diff --git a/helm/litellm/tests/database_auth_tests.yaml b/helm/litellm/tests/database_auth_tests.yaml index adbe14c59c2..add531f68bb 100644 --- a/helm/litellm/tests/database_auth_tests.yaml +++ b/helm/litellm/tests/database_auth_tests.yaml @@ -4,6 +4,7 @@ templates: - gateway/configmap.yaml - backend/deployment.yaml - backend/configmap.yaml + - migrations-job.yaml values: - ./values/required.yaml tests: @@ -67,6 +68,82 @@ tests: value: "true" any: true + - it: emits no TLS env by default + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLMODE + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLROOTCERT + any: true + + - it: writer sslMode and sslRootCert reach gateway and backend as DATABASE_SSLMODE and DATABASE_SSLROOTCERT + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + set: + database.writer.useIAMAuth: true + database.writer.sslMode: verify-full + database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLMODE + value: verify-full + any: true + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLROOTCERT + value: /etc/ssl/certs/ca-certificates.crt + any: true + + - it: writer sslMode and sslRootCert reach the collector sidecar and the migrations job, which dial Postgres themselves + set: + gateway.collector.enabled: true + database.connectionPool.enabled: true + database.writer.sslMode: verify-full + database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt + asserts: + - equal: + path: spec.template.spec.containers[1].name + value: collector + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_SSLMODE + value: verify-full + any: true + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_SSLROOTCERT + value: /etc/ssl/certs/ca-certificates.crt + any: true + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLMODE + value: verify-full + any: true + template: migrations-job.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_SSLROOTCERT + value: /etc/ssl/certs/ca-certificates.crt + any: true + template: migrations-job.yaml + - it: writer rejects both token sources at once template: gateway/deployment.yaml set: diff --git a/helm/litellm/tests/ingress_controller_tests.yaml b/helm/litellm/tests/ingress_controller_tests.yaml index 40790ba674a..aa30db3c9c1 100644 --- a/helm/litellm/tests/ingress_controller_tests.yaml +++ b/helm/litellm/tests/ingress_controller_tests.yaml @@ -97,6 +97,16 @@ tests: name: RELEASE-NAME-litellm-gateway port: number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /debug/memory/summary + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 - equal: path: spec.rules[0].http.paths[-1] value: diff --git a/helm/litellm/tests/ingress_extra_paths_tests.yaml b/helm/litellm/tests/ingress_extra_paths_tests.yaml index fc7d5943278..1305af15ae2 100644 --- a/helm/litellm/tests/ingress_extra_paths_tests.yaml +++ b/helm/litellm/tests/ingress_extra_paths_tests.yaml @@ -288,6 +288,17 @@ tests: - failedTemplate: errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it" + - it: rejects an entry that would take over the exact /debug/memory/summary route + set: + ingress.enabled: true + ingress.extraPaths: + - path: /debug/memory/summary + pathType: Exact + service: backend + asserts: + - failedTemplate: + errorMessage: "ingress.extraPaths[0]: path /debug/memory/summary with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it" + - it: allows a built-in path under a different pathType, which is a distinct rule set: ingress.enabled: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 1873219d1ea..4ca54131d6a 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -208,6 +208,11 @@ database: name: litellm-writer-secret usernameKey: username passwordKey: password + # libpq sslmode / sslrootcert applied to the writer and reader URLs (Prisma and the + # in-container PgBouncer); e.g. verify-full with /etc/ssl/certs/ca-certificates.crt for AWS RDS. + # sslRootCert on its own implies sslMode verify-full + sslMode: "" + sslRootCert: "" # Optional read-replica routing. When `reader.host` is set, the proxy routes # reads (find_*, count, group_by, query_raw/_first) to this endpoint while diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index a7c514270da..7b0b593b70f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -34,6 +40,15 @@ dependencies = [ "cc", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -55,6 +70,29 @@ dependencies = [ "rustversion", ] +[[package]] +name = "async-compression" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.91" @@ -482,6 +520,58 @@ dependencies = [ "tracing", ] +[[package]] +name = "azure_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e41cbd819986ba41904c207d8ffc4106f8f8352a548d773e9554906379bb2fb" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "tokio", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "tracing", +] + +[[package]] +name = "azure_identity" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32edf96b356ca7c51d7590c4925cc36efc3947a5da4468e8e0b25c56ecbb3de5" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "pin-project", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "url", +] + [[package]] name = "base64" version = "0.13.1" @@ -494,6 +584,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -606,6 +702,20 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -673,6 +783,16 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.9.1" @@ -688,6 +808,23 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compression-codecs" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" + [[package]] name = "const-oid" version = "0.10.2" @@ -728,6 +865,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.8.2" @@ -763,6 +909,15 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -889,6 +1044,9 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] [[package]] name = "derive_builder" @@ -960,6 +1118,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.16.0" @@ -972,12 +1136,42 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "esaxx-rs" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -990,6 +1184,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1011,6 +1216,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.33" @@ -1027,6 +1247,17 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.33" @@ -1068,6 +1299,7 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1078,6 +1310,33 @@ dependencies = [ "slab", ] +[[package]] +name = "gcp_auth" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d27dbcc645b60b8e7f6e2868a9d7102ece97d1bb49c1288b5321fcc67f7260" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "http 1.4.2", + "http-body-util", + "hyper 1.10.1", + "hyper-rustls 0.27.9", + "hyper-util", + "ring", + "rustls 0.23.42", + "rustls-pki-types", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", + "tracing-futures", + "url", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1386,6 +1645,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1537,6 +1820,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.19", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -1580,7 +1912,7 @@ dependencies = [ "futures-util", "litellm-config", "litellm-core", - "reqwest", + "reqwest 0.12.28", "rustls 0.23.42", "rustls-native-certs", "serde", @@ -1613,20 +1945,27 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "azure_core", + "azure_identity", "base64 0.22.1", "data-url", + "gcp_auth", + "moka", "rand 0.8.7", - "reqwest", + "reqwest 0.12.28", "rstest", "serde", "serde_json", "serde_path_to_error", "sha2 0.10.9", + "strum", + "subtle", "thiserror 2.0.19", "tokio", "tracing", "tracing-subscriber", "url", + "veil", ] [[package]] @@ -1663,11 +2002,13 @@ dependencies = [ name = "litellm-token-counter" version = "0.1.0" dependencies = [ + "base64 0.22.1", "criterion", "indexmap", "itoa", "rand 0.8.7", "rstest", + "rustc-hash", "serde", "serde_json", "thiserror 2.0.19", @@ -1681,6 +2022,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -1743,6 +2093,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1754,6 +2114,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "monostate" version = "0.1.18" @@ -1866,6 +2246,35 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" @@ -1884,6 +2293,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2085,6 +2514,7 @@ version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", @@ -2252,6 +2682,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -2332,11 +2771,49 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "hyper 1.10.1", + "hyper-rustls 0.27.9", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.42", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -2444,6 +2921,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.42", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.13", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2496,6 +3000,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "sct" version = "0.7.1" @@ -2649,6 +3159,38 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -2711,6 +3253,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2759,6 +3322,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "target-lexicon" version = "0.13.5" @@ -2922,6 +3491,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", @@ -3039,12 +3609,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http 1.4.2", "http-body 1.1.0", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3095,6 +3670,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -3138,6 +3723,57 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typespec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0373af0f9d4f580b3a1a9d9639cedaabe015ed262b35bfbe13941bfb14fe1ea6" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.2", + "reqwest 0.13.5", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c608f4427943f8adb211abc95c87672b1b98847152783507d54e3246e502f60" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "unicase" version = "2.9.0" @@ -3213,10 +3849,32 @@ version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] +[[package]] +name = "veil" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7352f0bbf3ab98911b0c0277065094c1b1ec79bbc85fa3b7d16bf1859c3d96f" +dependencies = [ + "once_cell", + "veil-macros", +] + +[[package]] +name = "veil-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47a3f4f06d904eb789b935253752ba6bcc1dfa61349f8d5341c66abe070b44e5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "version_check" version = "0.9.5" @@ -3331,6 +3989,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -3351,6 +4022,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.9" @@ -3391,12 +4071,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3609,6 +4342,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f3e54e5b2aa..5f25e69a1f8 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -41,8 +41,14 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +gcp_auth = "0.12.7" +azure_core = "1.0.0" +azure_identity = { version = "1.0.0", features = ["tokio"] } +moka = { version = "0.12.16", features = ["future"] } +strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" criterion = "0.8.2" +veil = "0.3.0" [profile.release] opt-level = 3 diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 6f4a573e6ff..6f48f38c9f6 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -272,7 +272,8 @@ fn core_error_kind(error: &Error) -> &'static str { Error::Auth(_) | Error::MissingApiKey { .. } | Error::MissingAzureAiCredentials - | Error::MissingAzureAiCredentialsOrAdToken => "AuthError", + | Error::MissingAzureDocumentIntelligenceCredentials + | Error::MissingReductoApiKey => "AuthError", Error::InvalidProvider(_) => "InvalidProvider", Error::InvalidRequest(_) => "InvalidRequest", Error::InvalidType { .. } => "InvalidType", diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 207c31dffa0..1aa31adcc38 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -15,6 +15,8 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::AuthError; +use litellm_core::auth::error::MissingCredential; use litellm_core::error::Error; use litellm_core::realtime::transformation::RealtimeProviderConfig; use litellm_core::realtime::types::RealtimeEvent; @@ -32,8 +34,6 @@ use crate::io::tls::connect_upstream; /// Environment variable holding the OpenAI API key (last-resort fallback). const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; -const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable"; - /// Default **idle** timeout: if neither side sends a frame for this long, the /// session is reaped. It resets on any activity, so it does not cap a healthy /// (continuously streaming) session — it only frees a stalled one (e.g. a @@ -59,7 +59,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { .ok() .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiRealtimeApiKey))) } /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9df3d0c6cc5..7f3b6b0650f 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -4,7 +4,9 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::AuthError; use litellm_core::Error; +use litellm_core::auth::error::MissingCredential; use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; use litellm_core::responses::types::ResponsesWsEvent; use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; @@ -23,8 +25,6 @@ use crate::constants::{ }; const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; -const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a Responses WebSocket call is being made but no key was passed via params or the OPENAI_API_KEY environment variable"; - pub type ResponsesUpstreamWs = WebSocketStream>; type UpstreamTx = SplitSink; type UpstreamRx = SplitStream; @@ -120,7 +120,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { .ok() .filter(|value| !value.trim().is_empty()) }) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiResponsesApiKey))) } async fn dial_upstream( diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs deleted file mode 100644 index d2be17260a3..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ /dev/null @@ -1,529 +0,0 @@ -use std::net::IpAddr; -use std::time::{Duration, Instant}; - -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::error::Error; -use litellm_core::ocr::transformation::OcrProviderConfig; -use reqwest::Url; -use serde_json::{Map, Value}; - -use litellm_core::providers::azure_ai::ocr::transformation::{ - AZURE_AI_OCR_CONFIG, AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG, -}; -use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; -use litellm_core::providers::reducto::ocr::transformation as reducto; -use litellm_core::providers::vertex_ai::ocr::transformation as vertex_ai; -use litellm_core::providers::vertex_ai::ocr::transformation::{ - VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG, -}; - -use crate::client::http_client; - -const ERROR_BODY_MAX_CHARS: usize = 256; -const AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS: u64 = 120; -const DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: f64 = 50.0; -const MAX_SAFE_FETCH_REDIRECTS: usize = 10; - -pub(super) fn truncate_error_body(body: &str) -> String { - if body.chars().count() <= ERROR_BODY_MAX_CHARS { - return body.to_string(); - } - let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect(); - format!("{truncated}... (truncated)") -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(super) fn ocr_provider_config( - provider: &str, - model: &str, -) -> Option<&'static dyn OcrProviderConfig> { - match provider { - "mistral" => Some(&MISTRAL_OCR_CONFIG), - "reducto" => reducto::config_for_model(model), - "azure_ai" if is_azure_document_intelligence_model(model) => { - Some(&AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG) - } - "azure_ai" => Some(&AZURE_AI_OCR_CONFIG), - "vertex_ai" if vertex_ai::is_deepseek_model(model) => Some(&VERTEX_AI_DEEPSEEK_OCR_CONFIG), - "vertex_ai" => Some(&VERTEX_AI_OCR_CONFIG), - _ => None, - } -} - -fn is_azure_document_intelligence_model(model: &str) -> bool { - let model = model.to_ascii_lowercase(); - model.contains("doc-intelligence") || model.contains("documentintelligence") -} - -pub(super) fn string_headers( - extra_headers: Option>, -) -> Result, Error> { - extra_headers - .unwrap_or_default() - .into_iter() - .map(|(key, value)| { - value - .as_str() - .map(|value| (key.clone(), value.to_string())) - .ok_or_else(|| { - Error::InvalidRequest(format!( - "OCR extra_headers.{key} must be a string, got {}", - litellm_core::error::json_type_name(&value) - )) - }) - }) - .collect() -} - -fn document_url_field(document: &Value) -> Result, Error> { - let Some(object) = document.as_object() else { - return Ok(None); - }; - let Some(doc_type) = object.get("type").and_then(Value::as_str) else { - return Ok(None); - }; - let field = match doc_type { - "document_url" => "document_url", - "image_url" => "image_url", - _ => return Ok(None), - }; - let Some(url) = object.get(field).and_then(Value::as_str) else { - return Ok(None); - }; - Ok(Some((field, url))) -} - -fn is_url_requiring_fetch(url: &str) -> bool { - !url.starts_with("data:") && (url.starts_with("http://") || url.starts_with("https://")) -} - -fn max_document_download_bytes() -> u64 { - let max_size_mb = std::env::var("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB); - (max_size_mb.max(0.0) * 1024.0 * 1024.0) as u64 -} - -fn is_blocked_ip(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(ip) => { - ip.is_private() - || ip.is_loopback() - || ip.is_link_local() - || ip.is_broadcast() - || ip.is_multicast() - || ip.is_unspecified() - } - IpAddr::V6(ip) => { - let first_segment = ip.segments()[0]; - let is_unique_local = (first_segment & 0xfe00) == 0xfc00; - let is_link_local = (first_segment & 0xffc0) == 0xfe80; - ip.is_loopback() - || ip.is_unspecified() - || ip.is_multicast() - || is_unique_local - || is_link_local - || ip - .to_ipv4_mapped() - .or_else(|| ip.to_ipv4()) - .map(|v4| is_blocked_ip(IpAddr::V4(v4))) - .unwrap_or(false) - } - } -} - -fn blocked_url_error(url: &Url) -> Error { - Error::InvalidRequest(format!( - "OCR document URL rejected by SSRF protection: {url}" - )) -} - -async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { - if !matches!(url.scheme(), "http" | "https") { - return Err(blocked_url_error(url)); - } - - let host = url.host_str().ok_or_else(|| blocked_url_error(url))?; - if let Ok(ip) = host.parse::() { - if is_blocked_ip(ip) { - return Err(blocked_url_error(url)); - } - return Ok(()); - } - - let port = url - .port_or_known_default() - .ok_or_else(|| blocked_url_error(url))?; - let addresses = tokio::net::lookup_host((host, port)) - .await - .map_err(|err| Error::Network(err.to_string()))?; - let mut saw_address = false; - for address in addresses { - saw_address = true; - if is_blocked_ip(address.ip()) { - return Err(blocked_url_error(url)); - } - } - if !saw_address { - return Err(blocked_url_error(url)); - } - Ok(()) -} - -fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { - let location = response - .headers() - .get(reqwest::header::LOCATION) - .and_then(|value| value.to_str().ok()) - .ok_or_else(|| { - Error::InvalidResponse("OCR document redirect missing Location header".to_string()) - })?; - url.join(location) - .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) -} - -async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|err| Error::Network(err.to_string()))?; - let mut current_url = Url::parse(url) - .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; - - for _ in 0..MAX_SAFE_FETCH_REDIRECTS { - validate_safe_fetch_url(¤t_url).await?; - let response = client - .get(current_url.clone()) - .send() - .await - .map_err(|err| Error::Network(err.to_string()))?; - if !response.status().is_redirection() { - return Ok((current_url, response)); - } - current_url = redirect_location(&response, ¤t_url)?; - } - - Err(Error::InvalidRequest( - "Too many redirects while fetching OCR document URL".to_string(), - )) -} - -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { - if max_bytes == 0 { - return Err(Error::InvalidRequest(format!( - "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" - ))); - } - if content_length > max_bytes { - let size_mb = content_length as f64 / (1024.0 * 1024.0); - let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(Error::InvalidRequest(format!( - "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" - ))); - } - Ok(()) -} - -async fn read_response_with_limit( - mut response: reqwest::Response, - url: &Url, -) -> Result, Error> { - let max_bytes = max_document_download_bytes(); - if let Some(content_length) = response.content_length() { - enforce_download_size(content_length, max_bytes, url)?; - } else { - enforce_download_size(0, max_bytes, url)?; - } - - let mut bytes = Vec::new(); - let mut bytes_downloaded: u64 = 0; - while let Some(chunk) = response - .chunk() - .await - .map_err(|err| Error::Network(err.to_string()))? - { - bytes_downloaded += chunk.len() as u64; - enforce_download_size(bytes_downloaded, max_bytes, url)?; - bytes.extend_from_slice(&chunk); - } - Ok(bytes) -} - -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { - let Some((field, url)) = document_url_field(&document)? else { - return Ok(document); - }; - if !is_url_requiring_fetch(url) { - return Ok(document); - } - - let (final_url, response) = safe_get_document_url(url).await?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&body), - }); - } - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.split(';').next()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or("application/octet-stream") - .to_string(); - let bytes = read_response_with_limit(response, &final_url).await?; - let data_uri = format!( - "data:{content_type};base64,{}", - BASE64_STANDARD.encode(bytes) - ); - - let mut transformed = document - .as_object() - .cloned() - .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; - transformed.insert(field.to_string(), Value::String(data_uri)); - Ok(Value::Object(transformed)) -} - -fn same_origin(left: &str, right: &str) -> bool { - let Ok(left) = reqwest::Url::parse(left) else { - return false; - }; - let Ok(right) = reqwest::Url::parse(right) else { - return false; - }; - left.scheme() == right.scheme() - && left.host_str() == right.host_str() - && left.port_or_known_default() == right.port_or_known_default() -} - -fn retry_after_secs(response: &reqwest::Response) -> u64 { - response - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .unwrap_or(2) -} - -fn operation_status(response_json: &Value) -> Result<&str, Error> { - let status = response_json - .get("status") - .and_then(Value::as_str) - .ok_or(Error::MissingField("status"))?; - match status { - "succeeded" => Ok("succeeded"), - "running" | "notStarted" => Ok("running"), - "failed" => { - let message = response_json - .get("error") - .and_then(|error| error.get("message")) - .and_then(Value::as_str) - .unwrap_or("Unknown error"); - Err(Error::InvalidResponse(format!( - "Azure Document Intelligence analysis failed: {message}" - ))) - } - other => Err(Error::InvalidResponse(format!( - "Unknown operation status: {other}" - ))), - } -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(super) async fn poll_document_intelligence( - operation_url: &str, - original_url: &str, - headers: &[(String, String)], - timeout: Option, -) -> Result { - if !same_origin(operation_url, original_url) { - return Err(Error::InvalidResponse( - "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), - )); - } - - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs( - AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS, - )); - loop { - if start.elapsed() > timeout { - return Err(Error::Network(format!( - "Azure Document Intelligence operation polling timed out after {} seconds", - timeout.as_secs() - ))); - } - - let mut request_builder = http_client().get(operation_url); - for (key, value) in headers { - if key.eq_ignore_ascii_case("ocp-apim-subscription-key") { - request_builder = request_builder.header(key, value); - } - } - let response = request_builder - .send() - .await - .map_err(|err| Error::Network(err.to_string()))?; - let retry_after = retry_after_secs(&response); - let status = response.status(); - let text = response - .text() - .await - .map_err(|err| Error::Network(err.to_string()))?; - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - let response_json: Value = serde_json::from_str(&text).map_err(|err| { - Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) - })?; - if operation_status(&response_json)? == "succeeded" { - return Ok(response_json); - } - tokio::time::sleep(Duration::from_secs(retry_after)).await; - } -} - -#[cfg(test)] -mod tests { - use litellm_core::ocr::transformation::OcrResponseHandling; - use serde_json::json; - - use super::*; - - #[test] - fn blocks_private_and_metadata_ips() { - assert!(is_blocked_ip("127.0.0.1".parse().unwrap())); - assert!(is_blocked_ip("10.0.0.1".parse().unwrap())); - assert!(is_blocked_ip("169.254.169.254".parse().unwrap())); - assert!(is_blocked_ip("::1".parse().unwrap())); - assert!(is_blocked_ip("fd00::1".parse().unwrap())); - assert!(is_blocked_ip("fe80::1".parse().unwrap())); - assert!(is_blocked_ip("::ffff:169.254.169.254".parse().unwrap())); - assert!(is_blocked_ip("::ffff:10.0.0.1".parse().unwrap())); - assert!(!is_blocked_ip("8.8.8.8".parse().unwrap())); - assert!(!is_blocked_ip("::ffff:8.8.8.8".parse().unwrap())); - } - - #[tokio::test] - async fn convert_document_url_rejects_loopback_fetch() { - let error = convert_document_url_to_data_uri(json!({ - "type": "image_url", - "image_url": "http://127.0.0.1/image.png" - })) - .await - .unwrap_err(); - - assert!(matches!( - error, - Error::InvalidRequest(message) - if message.contains("SSRF protection") - )); - } - - #[tokio::test] - async fn convert_document_url_leaves_data_uri_untouched() { - let document = json!({ - "type": "image_url", - "image_url": "data:image/png;base64,abcd" - }); - - let transformed = convert_document_url_to_data_uri(document.clone()) - .await - .unwrap(); - - assert_eq!(transformed, document); - } - - #[test] - fn truncate_error_body_passes_short_strings_through() { - let body = "Unauthorized"; - assert_eq!(truncate_error_body(body), "Unauthorized"); - } - - #[test] - fn truncate_error_body_caps_long_payloads() { - let body = "x".repeat(306); - let truncated = truncate_error_body(&body); - - assert!(truncated.ends_with("... (truncated)")); - let prefix_chars = truncated - .strip_suffix("... (truncated)") - .expect("truncated marker present") - .chars() - .count(); - assert_eq!(prefix_chars, 256); - } - - #[test] - fn truncate_error_body_does_not_split_multibyte_chars() { - let body = "é".repeat(266); - let truncated = truncate_error_body(&body); - assert!(truncated.is_char_boundary(truncated.len())); - } - - #[test] - fn ocr_dispatch_supports_migrated_providers() { - assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some()); - assert!( - ocr_provider_config("azure_ai", "pixtral-12b-2409") - .expect("azure ai config resolves") - .requires_data_uri_document() - ); - assert_eq!( - ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read") - .expect("document intelligence config resolves") - .response_handling(), - OcrResponseHandling::AzureDocumentIntelligencePoll - ); - assert!( - ocr_provider_config("vertex_ai", "deepseek-ocr-maas") - .expect("vertex deepseek config resolves") - .supported_ocr_params() - .contains(&"temperature") - ); - assert!(ocr_provider_config("openai", "gpt-4o").is_none()); - } - - #[test] - fn string_headers_accepts_string_values() { - let headers = json!({ - "x-trace-id": "trace-1" - }) - .as_object() - .unwrap() - .clone(); - - assert_eq!( - string_headers(Some(headers)).expect("string headers accepted"), - vec![("x-trace-id".to_string(), "trace-1".to_string())] - ); - } - - #[test] - fn string_headers_rejects_non_string_values() { - let headers = json!({ - "x-retry-count": 3 - }) - .as_object() - .unwrap() - .clone(); - - let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert_eq!( - err, - Error::InvalidRequest( - "OCR extra_headers.x-retry-count must be a string, got number".to_string() - ) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs deleted file mode 100644 index 6c6e12724cd..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ /dev/null @@ -1,84 +0,0 @@ -use litellm_core::error::Error; -use litellm_core::http_utils::http_request; -use litellm_core::ocr::transformation::OcrResponseHandling; -use serde_json::Value; - -use super::common_utils::{poll_document_intelligence, truncate_error_body}; -use super::hooks::OcrLifecycleHooks; -use super::types::PreparedOcrRequest; -use crate::client::http_client; - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(crate) async fn execute_ocr_provider_call( - request: PreparedOcrRequest, - hooks: &OcrLifecycleHooks, -) -> Result { - let request = hooks.prepare_provider_request(request).await?; - let mut request_builder = http_client().post(&request.url).json(&request.body); - for (key, value) in &request.upstream_headers { - request_builder = request_builder.header(key, value); - } - if let Some(duration) = request.timeout { - request_builder = request_builder.timeout(duration); - } - - let response = http_request(request_builder) - .await - .map_err(|err| Error::Network(err.to_string()))?; - - let status = response.status(); - if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll - && status.as_u16() == 202 - { - let operation_url = response - .headers() - .get("operation-location") - .and_then(|value| value.to_str().ok()) - .map(str::to_string) - .ok_or_else(|| { - Error::InvalidResponse( - "Azure Document Intelligence returned 202 but no Operation-Location header found" - .to_string(), - ) - })?; - let response_json = poll_document_intelligence( - &operation_url, - &request.url, - &request.upstream_headers, - request.timeout, - ) - .await?; - return Ok(request - .config - .transform_ocr_response_with_params( - &request.model, - response_json, - &request.optional_params, - )? - .into_json()); - } - - let text = response - .text() - .await - .map_err(|err| Error::Network(err.to_string()))?; - - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - - let response_json: Value = serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; - - Ok(request - .config - .transform_ocr_response_with_params( - &request.model, - response_json, - &request.optional_params, - )? - .into_json()) -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs deleted file mode 100644 index c7e8344aafc..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ /dev/null @@ -1,404 +0,0 @@ -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::Error; -use litellm_core::providers::reducto::ocr::transformation::{ - build_upload_request, extract_document_source, extract_upload_file_id, -}; -use serde_json::{Map, Value, json}; -use std::future::Future; -use std::pin::Pin; - -use super::common_utils::{convert_document_url_to_data_uri, string_headers, truncate_error_body}; -use super::types::{PreparedOcrRequest, ProviderOcrRequest}; -use crate::client::http_client; -use crate::integrations::custom_guardrail::{ - CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, -}; -use crate::integrations::custom_logger::{ - CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::{ - RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, -}; - -pub(crate) struct OcrLifecycleHooks { - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, -} - -type OcrFuture<'a, T> = Pin> + Send + 'a>>; -type OcrLogFuture<'a> = Pin + Send + 'a>>; - -impl OcrLifecycleHooks { - pub(crate) fn new( - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, - ) -> Self { - Self { - logger_runner, - guardrail_runner, - request_metadata, - } - } - - async fn run_pre_call_guardrails( - &self, - request: PreparedOcrRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - - let context = guardrail_context(&self.request_metadata); - let guardrail_request = GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": request.custom_llm_provider, - "document": request.document, - "optional_params": request.optional_params, - })); - let (guardrail_request, _) = self - .guardrail_runner - .run_pre_call(&context, guardrail_request) - .await - .map_err(guardrail_error_to_core_error)?; - let (document, optional_params) = parse_ocr_pre_call_guardrail_request(guardrail_request)?; - let optional_params = match &request.config { - Ok(config) => config.map_ocr_params(&optional_params), - Err(_) => optional_params, - }; - Ok(PreparedOcrRequest { - document, - optional_params, - ..request - }) - } - - pub(crate) async fn prepare_provider_request( - &self, - request: PreparedOcrRequest, - ) -> Result { - let config = request.config?; - let env_lookup = |key: &str| std::env::var(key).ok(); - let upstream_headers = config.validate_environment( - string_headers(request.extra_headers)?, - request.api_key.as_deref(), - &env_lookup, - )?; - let url = config.complete_url( - request.api_base.as_deref(), - &request.model, - &request.optional_params, - &env_lookup, - )?; - let model = request.model.clone(); - let custom_llm_provider = request.custom_llm_provider.clone(); - let is_reducto = custom_llm_provider == "reducto"; - let document = if is_reducto { - let guarded_document = self - .run_during_call_guardrails(&model, &custom_llm_provider, &url, request.document) - .await?; - upload_reducto_document( - &guarded_document, - request.api_base.as_deref(), - request.timeout, - &upstream_headers, - ) - .await? - } else if config.requires_data_uri_document() { - convert_document_url_to_data_uri(request.document).await? - } else { - request.document - }; - let optional_params = request.optional_params; - let body = config - .transform_ocr_request(&request.model, document, optional_params.clone())? - .data; - let body = if is_reducto { - body - } else { - self.run_during_call_guardrails(&model, &custom_llm_provider, &url, body) - .await? - }; - Ok(ProviderOcrRequest { - model, - config, - url, - body, - optional_params, - upstream_headers, - timeout: request.timeout, - }) - } - - async fn run_during_call_guardrails( - &self, - model: &str, - custom_llm_provider: &str, - url: &str, - body: Value, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(body); - } - - let context = guardrail_context(&self.request_metadata); - let guardrail_request = GuardrailRequest::new(json!({ - "model": model, - "custom_llm_provider": custom_llm_provider, - "url": url, - "body": body, - })); - let (guardrail_request, _) = self - .guardrail_runner - .run_during_call(&context, guardrail_request) - .await - .map_err(guardrail_error_to_core_error)?; - parse_ocr_during_call_guardrail_request(guardrail_request) - } - - fn standard_logging_payload( - &self, - context: &CallLifecycleContext, - timing: &CallLifecycleTiming, - ) -> StandardLoggingPayload { - StandardLoggingPayload { - id: context.litellm_call_id.clone(), - litellm_call_id: context.litellm_call_id.clone(), - call_type: context.call_type.clone(), - model: context.model.clone(), - custom_llm_provider: context.custom_llm_provider.clone(), - response_cost: 0.0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - start_time: timing.start_time, - end_time: timing.end_time, - stream: false, - metadata: StandardLoggingMetadata { - user_api_key_hash: self.request_metadata.user_api_key_hash.clone(), - user_api_key_user_id: self.request_metadata.user_api_key_user_id.clone(), - user_api_key_team_id: self.request_metadata.user_api_key_team_id.clone(), - ..Default::default() - }, - messages: None, - } - } -} - -async fn upload_reducto_document( - document: &Value, - api_base: Option<&str>, - timeout: Option, - upstream_headers: &[(String, String)], -) -> Result { - let source = extract_document_source(document)?; - let Some(authorization) = upstream_headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("authorization")) - .map(|(_, value)| value.as_str()) - else { - return Err(Error::Auth( - "Reducto upload requires an Authorization header".to_string(), - )); - }; - let Some(upload) = build_upload_request(source, authorization, api_base) else { - return Ok(document.clone()); - }; - let part = reqwest::multipart::Part::bytes(upload.bytes) - .file_name(upload.file_name) - .mime_str(&upload.mime_type) - .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let form = reqwest::multipart::Form::new().part("file", part); - let mut request_builder = http_client().post(upload.url).multipart(form); - for (name, value) in upstream_headers { - if !name.eq_ignore_ascii_case("content-type") - && !name.eq_ignore_ascii_case("content-length") - { - request_builder = request_builder.header(name, value); - } - } - if let Some(timeout) = timeout { - request_builder = request_builder.timeout(timeout); - } - let response = request_builder - .send() - .await - .map_err(|error| Error::Network(error.to_string()))?; - let status = response.status(); - let body = response - .text() - .await - .map_err(|error| Error::Network(error.to_string()))?; - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&body), - }); - } - let response_json: Value = serde_json::from_str(&body).map_err(|error| { - Error::InvalidResponse(format!("invalid Reducto upload response JSON: {error}")) - })?; - let file_id = extract_upload_file_id(&response_json)?; - Ok(json!({"type": "document_url", "document_url": file_id})) -} - -impl CallLifecycleHooks for OcrLifecycleHooks { - type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; - type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; - type SuccessFuture<'a> = OcrLogFuture<'a>; - type FailureFuture<'a> = OcrLogFuture<'a>; - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedOcrRequest, - ) -> Self::PreCallFuture<'a> { - Box::pin(async move { self.run_pre_call_guardrails(request).await }) - } - - fn async_during_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedOcrRequest, - ) -> Self::DuringCallFuture<'a> { - Box::pin(async move { Ok(request) }) - } - - #[tracing::instrument( - name = "success_callback", - target = "litellm::function_trace", - level = "trace", - skip_all - )] - fn async_log_success_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - response: &'a Value, - timing: &'a CallLifecycleTiming, - ) -> Self::SuccessFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - let response_obj = CallbackValue::new("ocr", response.clone()); - self.logger_runner - .async_log_success_event( - &ModelCallDetails::from_standard_logging_payload( - self.standard_logging_payload(context, timing), - ), - &response_obj, - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } - - #[tracing::instrument( - name = "failure_callback", - target = "litellm::function_trace", - level = "trace", - skip_all - )] - fn async_log_failure_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - error: &'a Error, - timing: &'a CallLifecycleTiming, - ) -> Self::FailureFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - let logging_error = LoggingError { - message: error.to_string(), - kind: core_error_kind(error).to_string(), - }; - let response_obj = CallbackValue::new( - "error", - json!({ - "message": logging_error.message, - "kind": logging_error.kind, - }), - ); - self.logger_runner - .async_log_failure_event( - &ModelCallDetails::from_standard_logging_payload( - self.standard_logging_payload(context, timing), - ) - .with_failure_error(logging_error), - Some(&response_obj), - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } -} - -fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { - GuardrailContext { - call_type: CallType::Ocr, - selected_guardrails: Vec::new(), - metadata: std::collections::HashMap::new(), - user_api_key_hash: metadata.user_api_key_hash.clone(), - user_api_key_user_id: metadata.user_api_key_user_id.clone(), - user_api_key_team_id: metadata.user_api_key_team_id.clone(), - trace_parent: None, - } -} - -fn parse_ocr_pre_call_guardrail_request( - request: GuardrailRequest, -) -> Result<(Value, Map), Error> { - let Value::Object(mut data) = request.data else { - return Err(Error::InvalidRequest( - "OCR pre_call guardrail must return an object".to_string(), - )); - }; - let document = data.remove("document").ok_or_else(|| { - Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) - })?; - let optional_params = match data.remove("optional_params") { - Some(Value::Object(params)) => params, - Some(_) => { - return Err(Error::InvalidRequest( - "OCR pre_call guardrail optional_params must be an object".to_string(), - )); - } - None => Map::new(), - }; - Ok((document, optional_params)) -} - -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { - let Value::Object(mut data) = request.data else { - return Err(Error::InvalidRequest( - "OCR during_call guardrail must return an object".to_string(), - )); - }; - data.remove("body") - .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) -} - -fn guardrail_error_to_core_error(error: GuardrailError) -> Error { - Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) -} - -fn core_error_kind(error: &Error) -> &'static str { - match error { - Error::Auth(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureAiCredentialsOrAdToken => "AuthError", - Error::InvalidProvider(_) => "InvalidProvider", - Error::InvalidRequest(_) => "InvalidRequest", - Error::InvalidType { .. } => "InvalidType", - Error::MissingField(_) => "MissingField", - Error::Http { .. } => "HttpError", - Error::InvalidResponse(_) => "InvalidResponse", - Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", - Error::Routing(_) => "RoutingError", - Error::Unsupported(_) => "UnsupportedRequest", - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index 9ed93d779f1..fb63a02f7ad 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,186 +1,127 @@ use litellm_core::Error; -use litellm_core::call_lifecycle::CallLifecycle; +use litellm_core::ocr::{ + OcrClient, + wire::{OcrWireRequest, decode_request}, +}; use serde_json::Value; -mod common_utils; -mod handler; -mod hooks; -mod prepare; mod types; pub use types::OcrRequest; -use handler::execute_ocr_provider_call; -use prepare::{PreparedOcrCall, prepare_ocr_call}; - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn ocr(request: OcrRequest<'_>) -> Result { - let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); - CallLifecycle::default() - .run_request(request, &hooks, |request| { - execute_ocr_provider_call(request, &hooks) - }) + core_ocr(request).await +} + +async fn core_ocr(request: OcrRequest<'_>) -> Result { + validate_host_hooks(&request)?; + let client = OcrClient::new(crate::client::http_client().clone())?; + let core_request = decode_request(OcrWireRequest { + model: request.model.to_string(), + document: request.document, + api_key: request.api_key.map(str::to_string), + api_base: request.api_base.map(str::to_string), + custom_llm_provider: request.custom_llm_provider.map(str::to_string), + extra_headers: request.extra_headers, + optional_params: request.optional_params, + input_sources: Default::default(), + timeout_seconds: request.timeout.map(|timeout| timeout.as_secs_f64()), + })?; + client + .perform(core_request) .await + .map(|response| response.into_json()) +} + +fn validate_host_hooks(request: &OcrRequest<'_>) -> Result<(), Error> { + if !request.guardrails.is_empty() { + return Err(Error::Unsupported( + "OCR host guardrails are not wired to the core path", + )); + } + if !request.callbacks.is_empty() { + return Err(Error::Unsupported( + "OCR host callbacks are not wired to the core path", + )); + } + Ok(()) } #[cfg(test)] mod tests { - use serde_json::{Map, json}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::net::{TcpListener, TcpStream}; + use std::sync::Arc; - use super::{OcrRequest, ocr}; - use crate::integrations::types::RequestMetadata; use litellm_core::ocr::wire::is_supported_request; + use serde_json::{Map, json}; - #[test] - fn core_activation_excludes_unmigrated_azure_document_intelligence() { - assert!(is_supported_request("model", Some("mistral"))); - assert!(is_supported_request("pixtral-12b", Some("azure_ai"))); - assert!(!is_supported_request( - "doc-intelligence/prebuilt-layout", - Some("azure_ai") - )); - assert!(!is_supported_request("parse-v3", Some("reducto"))); - } + use super::{OcrRequest, validate_host_hooks}; + use crate::integrations::custom_guardrail::{CustomGuardrail, GuardrailEventHook}; + use crate::integrations::custom_logger::CustomLogger; - async fn read_http_request(socket: &mut TcpStream) -> String { - let mut request = Vec::new(); - let mut buffer = [0_u8; 1024]; - let header_end = loop { - let n = socket.read(&mut buffer).await.expect("reads request"); - if n == 0 { - break request.len(); - } - request.extend_from_slice(&buffer[..n]); - if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") { - break position + 4; - } - }; - let headers = String::from_utf8_lossy(&request[..header_end]); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - while request.len().saturating_sub(header_end) < content_length { - let n = socket.read(&mut buffer).await.expect("reads body"); - if n == 0 { - break; - } - request.extend_from_slice(&buffer[..n]); + struct TestGuardrail; + + impl CustomGuardrail for TestGuardrail { + fn guardrail_name(&self) -> &str { + "test" + } + + fn supported_event_hooks(&self) -> &[GuardrailEventHook] { + &[] } - String::from_utf8(request).expect("request is utf8") } - fn base_ocr_request(model: &str) -> OcrRequest<'_> { + struct TestLogger; + + impl CustomLogger for TestLogger {} + + fn request() -> OcrRequest<'static> { OcrRequest { - model, - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), + model: "model", + document: json!({"type":"image_url","image_url":"data:image/png;base64,YQ=="}), + api_key: None, api_base: None, - custom_llm_provider: None, + custom_llm_provider: Some("mistral"), extra_headers: None, optional_params: Map::new(), timeout: None, callbacks: Vec::new(), guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), + request_metadata: Default::default(), litellm_call_id: None, } } - #[tokio::test] - async fn reducto_file_upload_then_parse_maps_response() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let address = listener.local_addr().expect("listener has local address"); - let server = tokio::spawn(async move { - let (mut upload_socket, _) = listener.accept().await.expect("accepts upload request"); - let upload_request = read_http_request(&mut upload_socket).await; - let upload_body = r#"{"file_id":"reducto://uploaded.pdf"}"#; - let upload_response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - upload_body.len(), - upload_body - ); - upload_socket - .write_all(upload_response.as_bytes()) - .await - .expect("writes upload response"); + #[test] + fn core_activation_includes_migrated_providers() { + assert!(is_supported_request("model", Some("mistral"))); + assert!(is_supported_request("pixtral-12b", Some("azure_ai"))); + assert!(is_supported_request( + "doc-intelligence/prebuilt-layout", + Some("azure_ai") + )); + assert!(is_supported_request("parse-v3", Some("reducto"))); + assert!(is_supported_request("mistral-ocr", Some("vertex_ai"))); + assert!(is_supported_request("deepseek-ocr", Some("vertex_ai"))); + } - let (mut parse_socket, _) = listener.accept().await.expect("accepts parse request"); - let parse_request = read_http_request(&mut parse_socket).await; - let parse_body = r#"{"job_id":"job_123","usage":{"num_pages":3,"credits":3},"result":{"chunks":[{"content":"Page 1 block A","blocks":[{"content":"Page 1 block A","bbox":{"page":1},"kind":"text"}]},{"content":"Page 2 block A","blocks":[{"content":"Page 2 block A","bbox":{"page":2},"kind":"table"}]},{"content":"Page 1 block B","blocks":[{"content":"Page 1 block B","bbox":{"page":1},"kind":"text"}]},{"content":"Page 3 block A","blocks":[{"content":"Page 3 block A","bbox":{"page":3},"kind":"figure"}]}]}}"#; - let parse_response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - parse_body.len(), - parse_body - ); - parse_socket - .write_all(parse_response.as_bytes()) - .await - .expect("writes parse response"); - (upload_request, parse_request) - }); - let api_base = format!("http://{address}"); - let mut request = base_ocr_request("reducto/parse-v3"); - request.api_base = Some(&api_base); - request.api_key = None; - request.extra_headers = Some(Map::from_iter([ - ("Authorization".to_string(), json!("Bearer test-key")), - ("x-trace-id".to_string(), json!("trace-1")), - ])); - request.document = json!({ - "type": "document_url", - "document_url": "data:application/pdf;base64,JVBERi0xLjQ=" - }); - request.optional_params = Map::from_iter([ - ( - "formatting".to_string(), - json!({"table_output_format": "html"}), - ), - ("retrieval".to_string(), json!({"chunk_mode": "section"})), - ("settings".to_string(), json!({"ocr_system": "standard"})), - ]); + #[test] + fn core_path_rejects_unwired_guardrails() { + let request = OcrRequest { + guardrails: vec![Arc::new(TestGuardrail)], + ..request() + }; + let error = validate_host_hooks(&request).unwrap_err(); + assert!(error.to_string().contains("guardrails are not wired")); + } - let response = ocr(request).await.expect("Reducto OCR succeeds"); - - assert_eq!(response["pages"].as_array().map(Vec::len), Some(3)); - assert_eq!( - response["pages"][0]["markdown"], - "Page 1 block A\n\nPage 1 block B" - ); - assert_eq!(response["pages"][1]["markdown"], "Page 2 block A"); - assert_eq!(response["pages"][2]["markdown"], "Page 3 block A"); - assert_eq!(response["usage_info"]["pages_processed"], 3); - assert_eq!(response["usage_info"]["credits"], 3); - assert_eq!(response["provider_native_response"]["job_id"], "job_123"); - let (upload_request, parse_request) = server.await.expect("server task completes"); - assert!( - upload_request - .to_ascii_lowercase() - .contains("authorization: bearer test-key") - ); - assert!(upload_request.contains("application/pdf")); - assert!(upload_request.contains("%PDF-1.4")); - assert!(upload_request.contains("x-trace-id: trace-1")); - assert!( - parse_request - .to_ascii_lowercase() - .contains("authorization: bearer test-key") - ); - assert!(parse_request.contains(r#""input":"reducto://uploaded.pdf""#)); - assert!(parse_request.contains(r#""table_output_format":"html""#)); - assert!(parse_request.contains(r#""chunk_mode":"section""#)); - assert!(parse_request.contains(r#""ocr_system":"standard""#)); + #[test] + fn core_path_rejects_unwired_callbacks() { + let request = OcrRequest { + callbacks: vec![Arc::new(TestLogger)], + ..request() + }; + let error = validate_host_hooks(&request).unwrap_err(); + assert!(error.to_string().contains("callbacks are not wired")); } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs deleted file mode 100644 index fa9ca1a193e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs +++ /dev/null @@ -1,163 +0,0 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; -use serde_json::{Map, Value}; - -use super::common_utils::ocr_provider_config; -use super::hooks::OcrLifecycleHooks; -use super::types::{OcrRequest, PreparedOcrRequest}; -use crate::integrations::custom_guardrail::CustomGuardrailRunner; -use crate::integrations::custom_logger::CustomLoggerRunner; - -pub(crate) struct PreparedOcrCall { - pub(crate) request: PreparedOcrRequest, - pub(crate) hooks: OcrLifecycleHooks, -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall { - let call_id = request - .litellm_call_id - .map(str::to_string) - .unwrap_or_else(new_ocr_call_id); - let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) - .unwrap_or(CustomLlmProvider { - model: request.model, - custom_llm_provider: "mistral", - }); - let model = provider_info.model.to_string(); - let custom_llm_provider = provider_info.custom_llm_provider.to_string(); - let config = ocr_provider_config(&custom_llm_provider, &model) - .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone())) - .and_then(|config| { - validate_request_format(config, &request.optional_params, &custom_llm_provider)?; - Ok(config) - }); - let optional_params = match &config { - Ok(config) => { - let supported = config.supported_ocr_params(); - let mut mapped = config.map_ocr_params( - &request - .optional_params - .iter() - .filter(|(name, _)| supported.contains(&name.as_str())) - .map(|(name, value)| (name.clone(), value.clone())) - .collect(), - ); - for name in [ - "vertex_project", - "vertex_ai_project", - "vertex_location", - "vertex_ai_location", - ] { - if let Some(value) = request.optional_params.get(name) { - mapped.insert(name.to_string(), value.clone()); - } - } - mapped - } - Err(_) => request.optional_params, - }; - - PreparedOcrCall { - request: PreparedOcrRequest { - config, - model, - custom_llm_provider, - litellm_call_id: call_id, - document: request.document, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - extra_headers: request.extra_headers, - optional_params, - timeout: request.timeout, - }, - hooks: OcrLifecycleHooks::new( - CustomLoggerRunner::new(request.callbacks), - CustomGuardrailRunner::new(request.guardrails), - request.request_metadata, - ), - } -} - -fn validate_request_format( - config: &'static dyn litellm_core::ocr::transformation::OcrProviderConfig, - optional_params: &Map, - provider: &str, -) -> Result<(), litellm_core::Error> { - let Some(format) = optional_params.get("req_format") else { - return Ok(()); - }; - match format.as_str() { - Some("litellm") => Ok(()), - Some("native") if config.supported_ocr_params().contains(&"req_format") => Ok(()), - Some("native") => Err(litellm_core::Error::InvalidRequest(format!( - "`req_format=native` is not supported for provider {provider}" - ))), - _ => Err(litellm_core::Error::InvalidRequest(format!( - "Invalid `req_format`: {format}. Expected `litellm` or `native`" - ))), - } -} - -fn new_ocr_call_id() -> String { - static COUNTER: AtomicU64 = AtomicU64::new(1); - let sequence = COUNTER.fetch_add(1, Ordering::Relaxed); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - format!("ocr-{timestamp}-{sequence}") -} - -#[cfg(test)] -mod tests { - use litellm_core::error::Error; - use serde_json::{Map, json}; - - use super::{OcrRequest, prepare_ocr_call}; - use crate::integrations::types::RequestMetadata; - - fn base_ocr_request(model: &str) -> OcrRequest<'_> { - OcrRequest { - model, - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: None, - custom_llm_provider: None, - extra_headers: None, - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - } - } - - fn request_with_format(format: &str) -> OcrRequest<'_> { - let mut request = base_ocr_request("mistral/mistral-ocr-latest"); - request.optional_params = Map::from_iter([("req_format".to_string(), json!(format))]); - request - } - - #[test] - fn native_format_rejected_for_provider_without_support_as_bad_request() { - let prepared = prepare_ocr_call(request_with_format("native")); - assert!( - matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("not supported for provider")) - ); - } - - #[test] - fn unknown_format_rejected_for_provider_without_support_as_bad_request() { - let prepared = prepare_ocr_call(request_with_format("raw")); - assert!( - matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("Invalid `req_format`")) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs index 75a8e61ddbf..e96d2df1adb 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/types.rs @@ -1,8 +1,6 @@ use std::sync::Arc; use std::time::Duration; -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; -use litellm_core::ocr::transformation::OcrProviderConfig; use serde_json::{Map, Value}; use crate::integrations::custom_guardrail::CustomGuardrail; @@ -23,37 +21,3 @@ pub struct OcrRequest<'a> { pub request_metadata: RequestMetadata, pub litellm_call_id: Option<&'a str>, } - -pub(crate) struct PreparedOcrRequest { - pub(crate) config: Result<&'static dyn OcrProviderConfig, litellm_core::Error>, - pub(crate) model: String, - pub(crate) custom_llm_provider: String, - pub(crate) litellm_call_id: String, - pub(crate) document: Value, - pub(crate) api_key: Option, - pub(crate) api_base: Option, - pub(crate) extra_headers: Option>, - pub(crate) optional_params: Map, - pub(crate) timeout: Option, -} - -impl CallLifecycleRequest for PreparedOcrRequest { - fn lifecycle_context(&self) -> CallLifecycleContext { - CallLifecycleContext::new( - "ocr", - self.model.clone(), - self.custom_llm_provider.clone(), - self.litellm_call_id.clone(), - ) - } -} - -pub(crate) struct ProviderOcrRequest { - pub(crate) model: String, - pub(crate) config: &'static dyn OcrProviderConfig, - pub(crate) url: String, - pub(crate) body: Value, - pub(crate) optional_params: Map, - pub(crate) upstream_headers: Vec<(String, String)>, - pub(crate) timeout: Option, -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index adfbf2b5910..39465e28e84 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -105,7 +105,11 @@ impl IntoResponse for MessagesRouteError { StatusCode::NOT_FOUND, "no messages deployment is configured for this model".to_string(), ), - Error::Auth(_) => ( + Error::Auth(_) + | Error::MissingApiKey { .. } + | Error::MissingAzureAiCredentials + | Error::MissingAzureDocumentIntelligenceCredentials + | Error::MissingReductoApiKey => ( StatusCode::BAD_GATEWAY, "messages provider authentication failed".to_string(), ), @@ -114,10 +118,7 @@ impl IntoResponse for MessagesRouteError { | Error::Connect(_) | Error::InvalidResponse(_) | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureAiCredentialsOrAdToken => ( + | Error::MissingField(_) => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), diff --git a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs deleted file mode 100644 index 5502a467511..00000000000 --- a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs +++ /dev/null @@ -1,692 +0,0 @@ -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use litellm_ai_gateway::integrations::custom_guardrail::{ - CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook, - GuardrailFuture, GuardrailRequest, -}; -use litellm_ai_gateway::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, -}; -use litellm_ai_gateway::integrations::types::RequestMetadata; -use litellm_ai_gateway::ocr::{OcrRequest, ocr}; -use litellm_core::error::Error; -#[cfg(feature = "trace-parity")] -use litellm_core::observability::FunctionTrace; -use serde_json::{Map, Value, json}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -#[cfg(feature = "trace-parity")] -use tracing::instrument::WithSubscriber; - -async fn read_http_headers(socket: &mut TcpStream) -> String { - let mut request = Vec::new(); - let mut buffer = [0_u8; 1024]; - loop { - let n = socket.read(&mut buffer).await.expect("reads request"); - if n == 0 { - break; - } - request.extend_from_slice(&buffer[..n]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - String::from_utf8(request).expect("request is utf8") -} - -async fn read_http_request(socket: &mut TcpStream) -> String { - let mut request = Vec::new(); - let mut buffer = [0_u8; 1024]; - let header_end = loop { - let n = socket.read(&mut buffer).await.expect("reads request"); - if n == 0 { - break request.len(); - } - request.extend_from_slice(&buffer[..n]); - if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") { - break position + 4; - } - }; - let headers = String::from_utf8_lossy(&request[..header_end]); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - while request.len().saturating_sub(header_end) < content_length { - let n = socket.read(&mut buffer).await.expect("reads body"); - if n == 0 { - break; - } - request.extend_from_slice(&buffer[..n]); - } - String::from_utf8(request).expect("request is utf8") -} - -#[derive(Clone, Debug, PartialEq)] -struct RecordedLogEvent { - hook: &'static str, - model: String, - call_type: String, - user_id: Option, - response_object: Option, - error_kind: Option, -} - -#[derive(Default)] -struct RecordingOcrLogger { - events: Mutex>, -} - -impl RecordingOcrLogger { - fn events(&self) -> Vec { - self.events.lock().unwrap().clone() - } -} - -impl CustomLogger for RecordingOcrLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedLogEvent { - hook: "async_log_success_event", - model: model_call_details.model.clone(), - call_type: model_call_details.call_type.to_string(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: Some(response_obj.object.clone()), - error_kind: None, - }); - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedLogEvent { - hook: "async_log_failure_event", - model: model_call_details.model.clone(), - call_type: model_call_details.call_type.to_string(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: response_obj.map(|value| value.object.clone()), - error_kind: model_call_details - .failure_error - .as_ref() - .map(|error| error.kind.clone()), - }); - Ok(()) - }) - } -} - -struct RecordingOcrGuardrail { - hooks: Vec, - events: Mutex>, - block_pre_call: bool, - block_during_call: bool, -} - -impl RecordingOcrGuardrail { - fn new(hooks: Vec) -> Self { - Self { - hooks, - events: Mutex::new(Vec::new()), - block_pre_call: false, - block_during_call: false, - } - } - - fn blocking_pre_call() -> Self { - Self { - hooks: vec![GuardrailEventHook::PreCall], - events: Mutex::new(Vec::new()), - block_pre_call: true, - block_during_call: false, - } - } - - fn blocking_during_call() -> Self { - Self { - hooks: vec![GuardrailEventHook::DuringCall], - events: Mutex::new(Vec::new()), - block_pre_call: false, - block_during_call: true, - } - } - - fn events(&self) -> Vec<&'static str> { - self.events.lock().unwrap().clone() - } -} - -impl CustomGuardrail for RecordingOcrGuardrail { - fn guardrail_name(&self) -> &str { - "recording-ocr-guardrail" - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &self.hooks - } - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - mut request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push("async_pre_call_hook"); - if self.block_pre_call { - return Ok(GuardrailDecision::Block(GuardrailError::blocked( - "blocked before provider", - ))); - } - request.data["document"]["guarded_pre"] = json!(true); - Ok(GuardrailDecision::Mask(request)) - }) - } - - fn async_moderation_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - mut request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push("async_moderation_hook"); - if self.block_during_call { - return Ok(GuardrailDecision::Block(GuardrailError::blocked( - "blocked before provider", - ))); - } - request.data["body"]["guarded_during"] = json!(true); - Ok(GuardrailDecision::Mask(request)) - }) - } -} - -fn base_ocr_request(model: &str) -> OcrRequest<'_> { - OcrRequest { - model, - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: None, - custom_llm_provider: None, - extra_headers: None, - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - } -} - -#[tokio::test] -async fn azure_mistral_uses_prepared_authorization_through_gateway() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let api_base = format!("http://{}", listener.local_addr().unwrap()); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let request = read_http_request(&mut socket).await; - let body = br#"{"pages":[]}"#; - socket - .write_all( - format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", - body.len() - ) - .as_bytes(), - ) - .await - .unwrap(); - socket.write_all(body).await.unwrap(); - request - }); - let request = OcrRequest { - model: "mistral-ocr-2505", - document: json!({ - "type":"document_url", - "document_url":"data:application/pdf;base64,YWJj" - }), - api_key: None, - api_base: Some(&api_base), - custom_llm_provider: Some("azure_ai"), - extra_headers: Some(Map::from_iter([( - "Authorization".into(), - json!("Bearer python-prepared-token"), - )])), - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - }; - - ocr(request).await.unwrap(); - let sent = server.await.unwrap(); - assert!(sent.starts_with("POST /providers/mistral/azure/ocr ")); - assert!( - sent.to_ascii_lowercase() - .contains("authorization: bearer python-prepared-token\r\n") - ); -} - -#[tokio::test] -async fn reducto_during_call_guardrail_blocks_before_upload() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let address = listener.local_addr().expect("listener has local address"); - let api_base = format!("http://{address}"); - let guardrail = Arc::new(RecordingOcrGuardrail::blocking_during_call()); - let mut request = base_ocr_request("reducto/parse-v3"); - request.api_base = Some(&api_base); - request.document = json!({ - "type": "document_url", - "document_url": "data:application/pdf;base64,JVBERi0xLjQ=" - }); - request.guardrails = vec![guardrail.clone()]; - - let error = ocr(request).await.expect_err("guardrail blocks upload"); - - assert!(matches!(error, Error::InvalidRequest(_))); - assert_eq!(guardrail.events(), vec!["async_moderation_hook"]); - let accepted = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await; - assert!(accepted.is_err(), "upload socket should not be touched"); -} - -#[tokio::test] -async fn reducto_upload_error_body_is_truncated() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let address = listener.local_addr().expect("listener has local address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts upload request"); - let _request = read_http_request(&mut socket).await; - let body = "x".repeat(300); - let response = format!( - "HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - body.len(), - body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes upload response"); - }); - let api_base = format!("http://{address}"); - let mut request = base_ocr_request("reducto/parse-v3"); - request.api_base = Some(&api_base); - request.document = json!({ - "type": "document_url", - "document_url": "data:application/pdf;base64,JVBERi0xLjQ=" - }); - - let error = ocr(request).await.expect_err("upload should fail"); - - assert!( - matches!(error, Error::Http { status: 500, body } if body.chars().count() < 300 && body.ends_with("... (truncated)")) - ); - server.await.expect("server task completes"); -} - -#[tokio::test] -async fn ocr_lifecycle_runs_pre_during_and_success_hooks() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let addr = listener.local_addr().expect("listener has local addr"); - - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts one request"); - let request = read_http_request(&mut socket).await; - let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-latest","usage_info":{"pages_processed":1}}"#; - let response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response_body.len(), - response_body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - request - }); - - let logger = Arc::new(RecordingOcrLogger::default()); - let guardrail = Arc::new(RecordingOcrGuardrail::new(vec![ - GuardrailEventHook::PreCall, - GuardrailEventHook::DuringCall, - ])); - #[cfg(feature = "trace-parity")] - let trace = FunctionTrace::default(); - let api_base = format!("http://{addr}"); - let call = ocr(OcrRequest { - model: "mistral-ocr-latest", - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: Some(&api_base), - custom_llm_provider: Some("mistral"), - extra_headers: None, - optional_params: Map::new(), - timeout: Some(Duration::from_secs(5)), - callbacks: vec![logger.clone()], - guardrails: vec![guardrail.clone()], - request_metadata: RequestMetadata { - user_api_key_user_id: Some("user-1".to_string()), - ..Default::default() - }, - litellm_call_id: Some("ocr-call-1"), - }); - #[cfg(feature = "trace-parity")] - let call = call.with_subscriber(trace.dispatcher()); - let response = call.await.expect("ocr request succeeds"); - - assert_eq!(response["pages"][0]["markdown"], "ok"); - assert_eq!( - guardrail.events(), - vec!["async_pre_call_hook", "async_moderation_hook"] - ); - assert_eq!( - logger.events(), - vec![RecordedLogEvent { - hook: "async_log_success_event", - model: "mistral-ocr-latest".to_string(), - call_type: "ocr".to_string(), - user_id: Some("user-1".to_string()), - response_object: Some("ocr".to_string()), - error_kind: None, - }] - ); - #[cfg(feature = "trace-parity")] - assert_eq!( - trace - .events() - .iter() - .filter(|event| event.function.ends_with("_callback")) - .map(|event| event.function) - .collect::>(), - vec!["success_callback"] - ); - - let request = server.await.expect("server task completes"); - assert!(request.contains(r#""guarded_pre":true"#), "{request}"); - assert!(request.contains(r#""guarded_during":true"#), "{request}"); -} - -#[tokio::test] -async fn ocr_lifecycle_runs_failure_hook_on_provider_error() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let addr = listener.local_addr().expect("listener has local addr"); - - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts one request"); - let _request = read_http_request(&mut socket).await; - let response_body = "provider failed"; - let response = format!( - "HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response_body.len(), - response_body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - }); - - let logger = Arc::new(RecordingOcrLogger::default()); - #[cfg(feature = "trace-parity")] - let trace = FunctionTrace::default(); - let api_base = format!("http://{addr}"); - let call = ocr(OcrRequest { - model: "mistral-ocr-latest", - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: Some(&api_base), - custom_llm_provider: Some("mistral"), - extra_headers: None, - optional_params: Map::new(), - timeout: Some(Duration::from_secs(5)), - callbacks: vec![logger.clone()], - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: Some("ocr-call-2"), - }); - #[cfg(feature = "trace-parity")] - let call = call.with_subscriber(trace.dispatcher()); - let err = call.await.expect_err("provider error propagates"); - - assert!(matches!(err, Error::Http { status: 500, .. })); - server.await.expect("server task completes"); - assert_eq!( - logger.events(), - vec![RecordedLogEvent { - hook: "async_log_failure_event", - model: "mistral-ocr-latest".to_string(), - call_type: "ocr".to_string(), - user_id: None, - response_object: Some("error".to_string()), - error_kind: Some("HttpError".to_string()), - }] - ); - #[cfg(feature = "trace-parity")] - assert_eq!( - trace - .events() - .iter() - .filter(|event| event.function.ends_with("_callback")) - .map(|event| event.function) - .collect::>(), - vec!["failure_callback"] - ); -} - -#[tokio::test] -async fn ocr_lifecycle_pre_call_block_skips_provider_socket() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let addr = listener.local_addr().expect("listener has local addr"); - let logger = Arc::new(RecordingOcrLogger::default()); - let guardrail = Arc::new(RecordingOcrGuardrail::blocking_pre_call()); - - let err = ocr(OcrRequest { - model: "mistral-ocr-latest", - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: Some(&format!("http://{addr}")), - custom_llm_provider: Some("mistral"), - extra_headers: None, - optional_params: Map::new(), - timeout: Some(Duration::from_millis(100)), - callbacks: vec![logger.clone()], - guardrails: vec![guardrail.clone()], - request_metadata: RequestMetadata::default(), - litellm_call_id: Some("ocr-call-3"), - }) - .await - .expect_err("guardrail blocks request"); - - assert!(matches!(err, Error::InvalidRequest(_))); - assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); - assert_eq!( - logger.events(), - vec![RecordedLogEvent { - hook: "async_log_failure_event", - model: "mistral-ocr-latest".to_string(), - call_type: "ocr".to_string(), - user_id: None, - response_object: Some("error".to_string()), - error_kind: Some("InvalidRequest".to_string()), - }] - ); - let accepted = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await; - assert!(accepted.is_err(), "provider socket should not be touched"); -} - -#[tokio::test] -async fn ocr_does_not_duplicate_authorization_header_when_header_is_supplied() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let addr = listener.local_addr().expect("listener has local addr"); - - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts one request"); - let request = read_http_headers(&mut socket).await; - let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-latest","usage_info":{"pages_processed":1}}"#; - let response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response_body.len(), - response_body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - request - }); - - let mut headers = Map::new(); - headers.insert( - "Authorization".to_string(), - Value::String("Bearer sk-from-python".to_string()), - ); - headers.insert( - "x-trace-id".to_string(), - Value::String("trace-1".to_string()), - ); - - let response = ocr(OcrRequest { - model: "mistral-ocr-latest", - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-for-rust-fallback"), - api_base: Some(&format!("http://{addr}")), - custom_llm_provider: Some("mistral"), - extra_headers: Some(headers), - optional_params: Map::new(), - timeout: Some(Duration::from_secs(5)), - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - }) - .await - .expect("ocr request succeeds"); - - assert_eq!(response["pages"][0]["markdown"], "ok"); - - let request = server.await.expect("server task completes"); - let authorization_count = request - .lines() - .filter(|line| line.to_ascii_lowercase().starts_with("authorization:")) - .count(); - assert_eq!(authorization_count, 1, "{request}"); - assert!( - request.contains("authorization: Bearer sk-from-python") - || request.contains("Authorization: Bearer sk-from-python"), - "{request}" - ); -} - -#[tokio::test] -async fn document_intelligence_poll_uses_resolved_subscription_key() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener binds"); - let addr = listener.local_addr().expect("listener has local addr"); - let operation_url = format!("http://{addr}/operations/1"); - - let server = tokio::spawn(async move { - let (mut post_socket, _) = listener.accept().await.expect("accepts post request"); - let post_request = read_http_headers(&mut post_socket).await; - let post_response = format!( - "HTTP/1.1 202 Accepted\r\noperation-location: {operation_url}\r\ncontent-length: 0\r\nconnection: close\r\n\r\n" - ); - post_socket - .write_all(post_response.as_bytes()) - .await - .expect("writes post response"); - - let (mut poll_socket, _) = listener.accept().await.expect("accepts poll request"); - let poll_request = read_http_headers(&mut poll_socket).await; - let response_body = r#"{"status":"succeeded","analyzeResult":{"pages":[{"pageNumber":1,"lines":[{"content":"ok"}]}]}}"#; - let poll_response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response_body.len(), - response_body - ); - poll_socket - .write_all(poll_response.as_bytes()) - .await - .expect("writes poll response"); - (post_request, poll_request) - }); - - let response = ocr(OcrRequest { - model: "doc-intelligence/prebuilt-read", - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("di-key"), - api_base: Some(&format!("http://{addr}")), - custom_llm_provider: Some("azure_ai"), - extra_headers: None, - optional_params: Map::new(), - timeout: Some(Duration::from_secs(5)), - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - }) - .await - .expect("document intelligence request succeeds"); - - assert_eq!(response["pages"][0]["markdown"], "ok"); - - let (post_request, poll_request) = server.await.expect("server task completes"); - assert!( - post_request - .to_ascii_lowercase() - .contains("ocp-apim-subscription-key: di-key"), - "{post_request}" - ); - assert!( - poll_request - .to_ascii_lowercase() - .contains("ocp-apim-subscription-key: di-key"), - "{poll_request}" - ); -} diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 6d0a2fa775b..a2433435e34 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -12,18 +12,25 @@ path = "tests/workspace_crate_allowlist.rs" [dependencies] base64.workspace = true +azure_core.workspace = true +azure_identity.workspace = true data-url = "0.3.2" +gcp_auth.workspace = true +moka.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true serde_path_to_error = "0.1" +strum.workspace = true +subtle.workspace = true tokio.workspace = true thiserror.workspace = true tracing.workspace = true tracing-subscriber = { workspace = true, optional = true } sha2.workspace = true url.workspace = true +veil.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } diff --git a/litellm-rust/crates/core/src/auth/credential.rs b/litellm-rust/crates/core/src/auth/credential.rs new file mode 100644 index 00000000000..b5235b6780c --- /dev/null +++ b/litellm-rust/crates/core/src/auth/credential.rs @@ -0,0 +1,168 @@ +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +use veil::Redact; + +use crate::AuthError; + +use super::{ResolvedCredential, SecretValue, TokenProviderHandle}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CredentialFileRef { + Path(PathBuf), + EnvironmentVariable(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CredentialRef { + Explicit(SecretValue), + Env(String), + File(CredentialFileRef), + Request(String), + Host(String), + None, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CredentialLookup { + Found(SecretValue), + Missing, + Declined, +} + +pub type CredentialLookupFuture<'a> = + Pin> + Send + 'a>>; + +pub trait CredentialResolver: std::fmt::Debug + Send + Sync { + fn resolve<'a>(&'a self, reference: &'a CredentialRef) -> CredentialLookupFuture<'a>; +} + +#[derive(Clone, Redact)] +pub struct CredentialResolverHandle(#[redact(with = "[REDACTED]")] Arc); + +impl CredentialResolverHandle { + pub fn new(resolver: Arc) -> Self { + Self(resolver) + } + + pub async fn resolve(&self, reference: &CredentialRef) -> Result { + self.0.resolve(reference).await + } +} + +#[derive(Clone, Debug)] +pub enum CredentialPlan { + Static(CredentialRef), + Caller(TokenProviderHandle), + None, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CredentialPlanResolution { + Resolved(ResolvedCredential), + Unavailable, +} + +impl CredentialPlan { + pub async fn resolve( + &self, + resolver: &CredentialResolverHandle, + ) -> Result { + match self { + Self::Static(CredentialRef::Explicit(secret)) => Ok( + CredentialPlanResolution::Resolved(ResolvedCredential::Static(secret.clone())), + ), + Self::Static(CredentialRef::None) | Self::None => { + Ok(CredentialPlanResolution::Unavailable) + } + Self::Static(reference) => match resolver.resolve(reference).await? { + CredentialLookup::Found(secret) => Ok(CredentialPlanResolution::Resolved( + ResolvedCredential::Static(secret), + )), + CredentialLookup::Missing | CredentialLookup::Declined => { + Ok(CredentialPlanResolution::Unavailable) + } + }, + Self::Caller(caller) => { + let credential = caller.acquire().await?; + if credential.secret().expose().is_empty() { + return Err(AuthError::EmptyCallerCredential); + } + Ok(CredentialPlanResolution::Resolved(credential)) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::{ + CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, + CredentialRef, CredentialResolver, CredentialResolverHandle, + }; + use crate::AuthError; + use crate::auth::SecretValue; + + #[derive(Debug)] + struct HostResolver; + + impl CredentialResolver for HostResolver { + fn resolve<'a>(&'a self, reference: &'a CredentialRef) -> CredentialLookupFuture<'a> { + Box::pin(async move { + Ok(match reference { + CredentialRef::Host(name) if name == "rotating-token" => { + CredentialLookup::Found(SecretValue::new("resolved")) + } + _ => CredentialLookup::Declined, + }) + }) + } + } + + #[tokio::test] + async fn static_host_reference_resolves_at_acquisition_time() { + let resolver = CredentialResolverHandle::new(Arc::new(HostResolver)); + let plan = CredentialPlan::Static(CredentialRef::Host("rotating-token".to_string())); + + let resolved = plan.resolve(&resolver).await.unwrap(); + + assert!(matches!(resolved, CredentialPlanResolution::Resolved(_))); + } + + #[tokio::test] + async fn declined_reference_is_available_for_pre_acquisition_fallback() { + let resolver = CredentialResolverHandle::new(Arc::new(HostResolver)); + let plan = CredentialPlan::Static(CredentialRef::Request("api-key".to_string())); + + assert_eq!( + plan.resolve(&resolver).await.unwrap(), + CredentialPlanResolution::Unavailable + ); + } + + #[derive(Debug)] + struct FailingResolver; + + impl CredentialResolver for FailingResolver { + fn resolve<'a>(&'a self, _reference: &'a CredentialRef) -> CredentialLookupFuture<'a> { + Box::pin(async { Err(AuthError::UnresolvedOidcReference) }) + } + } + + #[tokio::test] + async fn acquisition_failure_is_terminal() { + let resolver = CredentialResolverHandle::new(Arc::new(FailingResolver)); + let plan = CredentialPlan::Static(CredentialRef::Host("token".to_string())); + + let error = plan + .resolve(&resolver) + .await + .expect_err("acquisition errors cannot become fallback"); + + assert_eq!(error, AuthError::UnresolvedOidcReference); + } +} diff --git a/litellm-rust/crates/core/src/auth/error.rs b/litellm-rust/crates/core/src/auth/error.rs new file mode 100644 index 00000000000..e7027c0df10 --- /dev/null +++ b/litellm-rust/crates/core/src/auth/error.rs @@ -0,0 +1,128 @@ +use thiserror::Error; + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum AuthError { + #[error("invalid authentication configuration: {0}")] + Configuration(#[from] AuthConfigurationError), + #[error("credential acquisition failed: {0}")] + AzureTokenAcquisition(String), + #[error("credential acquisition failed: Vertex AI credentials: {0}")] + VertexTokenAcquisition(String), + #[error("credential acquisition failed: {}", .0.iter().map(ToString::to_string).collect::>().join("; "))] + CredentialChain(Vec), + #[error("credential caller failed: credential caller returned an empty credential")] + EmptyCallerCredential, + #[error("credential caller failed: Azure AD token provider returned an empty token")] + EmptyAzureToken, + #[error("credential acquisition failed: Azure OIDC reference did not resolve to a value")] + UnresolvedOidcReference, + #[error( + "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" + )] + MissingApiKey { provider: &'static str }, + #[error( + "Missing {provider} API Base - Set {environment_variable} environment variable or pass api_base parameter" + )] + MissingApiBase { + provider: &'static str, + environment_variable: &'static str, + }, + #[error("{0}")] + MissingCredential(#[from] MissingCredential), + #[error("{0}")] + Aws(#[from] AwsAuthError), + #[error("invalid authentication header")] + InvalidHeader, +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum AuthConfigurationError { + #[error("credential header already exists")] + ExistingCredentialHeader, + #[error("credential plan is not allowed by the provider auth policy")] + DisallowedCredentialPlan, + #[error("credential cannot be empty")] + EmptyCredential, + #[error("invalid Azure credential selector")] + InvalidAzureSelector, + #[error("ClientSecretCredential requires tenant_id, client_id, and client_secret")] + MissingClientSecretFields, + #[error("WorkloadIdentityCredential requires tenant_id")] + MissingWorkloadTenant, + #[error("WorkloadIdentityCredential requires client_id")] + MissingWorkloadClient, + #[error("WorkloadIdentityCredential requires azure_federated_token_file")] + MissingWorkloadTokenFile, + #[error("credential reference requires a host credential resolver")] + MissingHostResolver, + #[error("caller credential plan requires provider-specific inputs")] + MissingCallerInputs, + #[error("credential header {0} already exists")] + DuplicateHeader(&'static str), + #[error("{0} must be a string or null")] + InvalidFieldType(String), + #[error("unsupported OIDC reference")] + UnsupportedOidcReference, + #[error("{0} cannot be empty")] + EmptyReference(String), + #[error("Azure credential initialization failed: {0}")] + AzureCredentialInitialization(String), + #[error("Azure authority must be an HTTPS origin without credentials, query, or fragment")] + InvalidAzureAuthority, + #[error("request-controlled Azure auth inputs cannot be combined with host credentials")] + MixedAzureCredentialSources, + #[error("request-controlled Azure credential references are not allowed")] + RequestAzureCredentialReference, + #[error("host credentials cannot be sent to a request-controlled Azure endpoint")] + RequestAzureCredentialDestination, + #[error("credentials cannot be sent to a request-controlled Vertex AI endpoint")] + RequestVertexCredentialDestination, + #[error( + "request-controlled Vertex credentials must use the canonical Google OAuth token endpoint" + )] + RequestVertexTokenEndpoint, +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum MissingCredential { + #[error( + "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable" + )] + AnthropicApiKey, + #[error("Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable")] + AzureApiKey, + #[error( + "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://.services.ai.azure.com/anthropic" + )] + AzureApiBase, + #[error( + "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable" + )] + OpenAiRealtimeApiKey, + #[error( + "Missing OpenAI API Key - a Responses WebSocket call is being made but no key was passed via params or the OPENAI_API_KEY environment variable" + )] + OpenAiResponsesApiKey, +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum AwsAuthError { + #[error("AWS profile credentials failed: {0}")] + Profile(String), + #[error("AWS default credentials failed: {0}")] + DefaultChain(String), + #[error("AWS role credentials failed: {0}")] + AssumeRole(String), + #[error("AWS web identity credentials failed: {0}")] + WebIdentity(String), + #[error("AWS web identity expiration was invalid: {0}")] + WebIdentityExpiration(String), + #[error("AWS signing parameters failed: {0}")] + SigningParameters(String), + #[error("AWS signable request failed: {0}")] + SignableRequest(String), + #[error("AWS request signing failed: {0}")] + Signing(String), + #[error("AWS web identity response had no credentials")] + MissingWebIdentityCredentials, +} diff --git a/litellm-rust/crates/core/src/auth/http.rs b/litellm-rust/crates/core/src/auth/http.rs new file mode 100644 index 00000000000..83931311550 --- /dev/null +++ b/litellm-rust/crates/core/src/auth/http.rs @@ -0,0 +1,86 @@ +use crate::AuthError; +use crate::auth::error::AuthConfigurationError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CredentialPlacement { + Bearer, + Header(&'static str), +} + +impl CredentialPlacement { + pub fn header_name(self) -> &'static str { + match self { + Self::Bearer => "Authorization", + Self::Header(name) => name, + } + } +} + +pub(crate) fn apply_credential( + headers: Vec<(String, String)>, + credential: &str, + placement: CredentialPlacement, +) -> Result, AuthError> { + if credential.trim().is_empty() { + return Err(AuthError::Configuration( + AuthConfigurationError::EmptyCredential, + )); + } + if headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case(placement.header_name())) + { + return Err(AuthError::Configuration( + AuthConfigurationError::DuplicateHeader(placement.header_name()), + )); + } + let value = match placement { + CredentialPlacement::Bearer => format!("Bearer {credential}"), + CredentialPlacement::Header(_) => credential.to_string(), + }; + Ok( + std::iter::once((placement.header_name().to_string(), value)) + .chain(headers) + .collect(), + ) +} + +/// How the upstream call is authenticated. API-key strategies are resolved in +/// `prepare`; SigV4 needs the serialized body, so the handler signs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RequestAuth { + Header { name: &'static str, value: String }, + Bearer { token: String }, + AwsSigV4 { region: String }, +} + +#[cfg(test)] +mod tests { + use super::{CredentialPlacement, apply_credential}; + + #[test] + fn bearer_uses_authorization_header() { + let headers = apply_credential(Vec::new(), "key", CredentialPlacement::Bearer) + .expect("credential applies"); + + assert_eq!( + headers, + vec![("Authorization".to_string(), "Bearer key".to_string())] + ); + } + + #[test] + fn named_header_rejects_existing_value() { + let error = apply_credential( + vec![( + "ocp-apim-subscription-key".to_string(), + "caller-key".to_string(), + )], + "configured-key", + CredentialPlacement::Header("Ocp-Apim-Subscription-Key"), + ) + .expect_err("provider policy must handle existing credentials"); + + assert!(error.to_string().contains("already exists")); + } +} diff --git a/litellm-rust/crates/core/src/auth/mod.rs b/litellm-rust/crates/core/src/auth/mod.rs new file mode 100644 index 00000000000..35d9c676f65 --- /dev/null +++ b/litellm-rust/crates/core/src/auth/mod.rs @@ -0,0 +1,56 @@ +mod credential; +pub mod error; +pub(crate) mod vertex; +pub use error::AuthError; +pub(crate) mod http; +mod policy; +mod secret; +mod token; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum InputSource { + Request, + #[default] + Deployment, + Environment, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Sourced { + value: T, + source: InputSource, +} + +impl Sourced { + pub fn new(value: T, source: InputSource) -> Self { + Self { value, source } + } + + pub fn value(&self) -> &T { + &self.value + } + + pub fn source(&self) -> InputSource { + self.source + } + + pub fn into_value(self) -> T { + self.value + } + + pub fn map(self, map: impl FnOnce(T) -> U) -> Sourced { + Sourced::new(map(self.value), self.source) + } +} + +pub use credential::{ + CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan, + CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, +}; +pub use http::{CredentialPlacement, RequestAuth}; +pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; +pub use secret::SecretValue; +pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm-rust/crates/core/src/auth/policy.rs b/litellm-rust/crates/core/src/auth/policy.rs new file mode 100644 index 00000000000..b796dedf0d8 --- /dev/null +++ b/litellm-rust/crates/core/src/auth/policy.rs @@ -0,0 +1,114 @@ +use crate::AuthError; +use crate::auth::error::AuthConfigurationError; + +use super::http::apply_credential; +use super::{CredentialPlacement, ResolvedCredential}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CredentialPlanKind { + Static, + Entra, + Caller, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CredentialRule { + pub kind: CredentialPlanKind, + pub placement: CredentialPlacement, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExistingHeaderBehavior { + Preserve, + Reject, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProviderAuthPolicy { + pub rules: &'static [CredentialRule], + pub accepted_existing_headers: &'static [&'static str], + pub existing_header_behavior: ExistingHeaderBehavior, + pub scope: Option<&'static str>, + pub audience: Option<&'static str>, +} + +impl ProviderAuthPolicy { + pub fn has_existing_credential(&self, headers: &[(String, String)]) -> bool { + headers.iter().any(|(name, _)| { + self.accepted_existing_headers + .iter() + .any(|accepted| name.eq_ignore_ascii_case(accepted)) + }) + } + + pub fn apply( + &self, + headers: Vec<(String, String)>, + kind: CredentialPlanKind, + credential: &ResolvedCredential, + ) -> Result, AuthError> { + if self.has_existing_credential(&headers) { + return match self.existing_header_behavior { + ExistingHeaderBehavior::Preserve => Ok(headers), + ExistingHeaderBehavior::Reject => Err(AuthError::Configuration( + AuthConfigurationError::ExistingCredentialHeader, + )), + }; + } + let rule = + self.rules + .iter() + .find(|rule| rule.kind == kind) + .ok_or(AuthError::Configuration( + AuthConfigurationError::DisallowedCredentialPlan, + ))?; + apply_credential(headers, credential.secret().expose(), rule.placement) + } +} + +#[cfg(test)] +mod tests { + use super::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; + use crate::auth::{CredentialPlacement, ResolvedCredential, SecretValue}; + + const RULES: &[CredentialRule] = &[CredentialRule { + kind: CredentialPlanKind::Static, + placement: CredentialPlacement::Header("x-api-key"), + }]; + const POLICY: ProviderAuthPolicy = ProviderAuthPolicy { + rules: RULES, + accepted_existing_headers: &["x-api-key"], + existing_header_behavior: ExistingHeaderBehavior::Preserve, + scope: None, + audience: None, + }; + + #[test] + fn rules_define_allowed_plans_and_credential_placement() { + let headers = POLICY + .apply( + Vec::new(), + CredentialPlanKind::Static, + &ResolvedCredential::Static(SecretValue::new("secret")), + ) + .unwrap(); + + assert_eq!( + headers, + vec![("x-api-key".to_string(), "secret".to_string())] + ); + } + + #[test] + fn unsupported_plan_is_rejected() { + let error = POLICY + .apply( + Vec::new(), + CredentialPlanKind::Entra, + &ResolvedCredential::Static(SecretValue::new("secret")), + ) + .unwrap_err(); + + assert!(error.to_string().contains("not allowed")); + } +} diff --git a/litellm-rust/crates/core/src/auth/secret.rs b/litellm-rust/crates/core/src/auth/secret.rs new file mode 100644 index 00000000000..3ecb0a835ee --- /dev/null +++ b/litellm-rust/crates/core/src/auth/secret.rs @@ -0,0 +1,41 @@ +use veil::Redact; + +#[derive(Redact, Clone)] +pub struct SecretValue(#[redact(with = "[REDACTED]")] String); + +impl SecretValue { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl PartialEq for SecretValue { + fn eq(&self, other: &Self) -> bool { + subtle::ConstantTimeEq::ct_eq(self.0.as_bytes(), other.0.as_bytes()).into() + } +} + +impl Eq for SecretValue {} + +#[cfg(test)] +mod tests { + use super::SecretValue; + + #[test] + fn debug_redacts_plaintext() { + let debug = format!("{:?}", SecretValue::new("credential-value")); + + assert!(!debug.contains("credential-value")); + assert!(debug.contains("REDACTED")); + } + + #[test] + fn equality_compares_plaintext_values() { + assert_eq!(SecretValue::new("same"), SecretValue::new("same")); + assert_ne!(SecretValue::new("same"), SecretValue::new("different")); + } +} diff --git a/litellm-rust/crates/core/src/auth/token.rs b/litellm-rust/crates/core/src/auth/token.rs new file mode 100644 index 00000000000..cfc6b8f0d6b --- /dev/null +++ b/litellm-rust/crates/core/src/auth/token.rs @@ -0,0 +1,47 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::SystemTime; + +use veil::Redact; + +use crate::AuthError; + +use super::secret::SecretValue; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ResolvedCredential { + Static(SecretValue), + AccessToken { + token: SecretValue, + expires_on: Option, + }, +} + +impl ResolvedCredential { + pub fn secret(&self) -> &SecretValue { + match self { + Self::Static(secret) | Self::AccessToken { token: secret, .. } => secret, + } + } +} + +pub type TokenFuture<'a> = + Pin> + Send + 'a>>; + +pub trait TokenProvider: std::fmt::Debug + Send + Sync { + fn acquire(&self) -> TokenFuture<'_>; +} + +#[derive(Clone, Redact)] +pub struct TokenProviderHandle(#[redact(with = "[REDACTED]")] Arc); + +impl TokenProviderHandle { + pub fn new(caller: Arc) -> Self { + Self(caller) + } + + pub async fn acquire(&self) -> Result { + self.0.acquire().await + } +} diff --git a/litellm-rust/crates/core/src/auth/vertex.rs b/litellm-rust/crates/core/src/auth/vertex.rs new file mode 100644 index 00000000000..00a0a7ea7ee --- /dev/null +++ b/litellm-rust/crates/core/src/auth/vertex.rs @@ -0,0 +1,592 @@ +use std::collections::BTreeMap; +use std::future::Future; +use std::path::Path; +use std::pin::Pin; +use std::sync::Arc; + +use gcp_auth::{CustomServiceAccount, TokenProvider}; +use moka::future::Cache; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::auth::error::AuthConfigurationError; +use crate::auth::http::apply_credential; +use crate::auth::{AuthError, CredentialPlacement, InputSource, SecretValue, Sourced}; + +const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform"; +const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token"; +const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS"; +const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY"; +const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY"; +const VERTEXAI_CREDENTIALS_ENV: &str = "VERTEXAI_CREDENTIALS"; +const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT"; +const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION"; +const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION"; + +#[derive(Clone, Debug, Default)] +pub(crate) struct VertexConfig { + credentials: Option>, + project_id: Option, + location: Option, +} + +impl VertexConfig { + pub(crate) fn from_sourced_optional_params( + params: &Map, + sources: &BTreeMap, + ) -> Result { + Ok(Self { + credentials: optional_credentials( + params, + sources, + &["vertex_credentials", "vertex_ai_credentials"], + )?, + project_id: optional_string(params, &["vertex_project", "vertex_ai_project"])?, + location: optional_string(params, &["vertex_location", "vertex_ai_location"])?, + }) + } + + pub(crate) fn project_id(&self) -> Option<&str> { + self.project_id.as_deref() + } + + pub(crate) fn location(&self) -> Option<&str> { + self.location.as_deref() + } +} + +pub(crate) struct VertexEnvironment { + pub headers: Vec<(String, String)>, + pub project_id: String, +} + +struct VertexAccessToken { + token: String, + project_id: String, +} + +pub(crate) fn get_vertex_ai_project( + config: &VertexConfig, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option { + config + .project_id() + .map(str::to_string) + .or_else(|| non_empty_env(env_lookup, VERTEXAI_PROJECT_ENV)) +} + +pub(crate) fn get_vertex_ai_location( + config: &VertexConfig, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option { + config + .location() + .map(str::to_string) + .or_else(|| non_empty_env(env_lookup, VERTEXAI_LOCATION_ENV)) + .or_else(|| non_empty_env(env_lookup, VERTEX_LOCATION_ENV)) +} + +#[derive(Clone)] +pub(crate) struct VertexAuth { + providers: Cache>, + loader: Arc, +} + +impl Default for VertexAuth { + fn default() -> Self { + Self::new(Arc::new(GcpProviderLoader)) + } +} + +impl VertexAuth { + fn new(loader: Arc) -> Self { + Self { + providers: Cache::builder().max_capacity(64).build(), + loader, + } + } + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + pub(crate) async fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + config: &VertexConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let has_authorization = headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("Authorization")); + let static_token = api_key + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| non_empty_env(env_lookup, VERTEX_AI_API_KEY_ENV)) + .or_else(|| non_empty_env(env_lookup, VERTEXAI_API_KEY_ENV)); + let project_id = get_vertex_ai_project(config, env_lookup); + + if !has_authorization && static_token.is_none() { + let access = self.get_access_token(config, env_lookup).await?; + return Ok(VertexEnvironment { + headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?, + project_id: project_id.unwrap_or(access.project_id), + }); + } + + let project_id = match project_id { + Some(project_id) => project_id, + None => { + self.load_provider(config, env_lookup) + .await? + .project_id() + .await? + } + }; + let headers = if has_authorization { + headers + } else { + apply_credential( + headers, + static_token.as_deref().expect("static token was checked"), + CredentialPlacement::Bearer, + )? + }; + Ok(VertexEnvironment { + headers, + project_id, + }) + } + + async fn get_access_token( + &self, + config: &VertexConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let provider = self.load_provider(config, env_lookup).await?; + let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?; + Ok(VertexAccessToken { token, project_id }) + } + + async fn load_provider( + &self, + config: &VertexConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result, AuthError> { + let source = credential_source(config, env_lookup); + let key = source.cache_key(); + self.providers + .try_get_with(key, self.loader.load(source)) + .await + .map_err(|error| (*error).clone()) + } +} + +trait VertexTokenSource: Send + Sync { + fn project_id(&self) -> VertexAuthFuture<'_, String>; + fn token(&self) -> VertexAuthFuture<'_, String>; +} + +trait VertexProviderLoader: Send + Sync { + fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc>; +} + +type VertexAuthFuture<'a, T> = Pin> + Send + 'a>>; + +struct GcpTokenSource(Arc); + +impl VertexTokenSource for GcpTokenSource { + fn project_id(&self) -> VertexAuthFuture<'_, String> { + Box::pin(async move { + self.0 + .project_id() + .await + .map(|project| project.to_string()) + .map_err(auth_acquisition_error) + }) + } + + fn token(&self) -> VertexAuthFuture<'_, String> { + Box::pin(async move { + self.0 + .token(&[CLOUD_PLATFORM_SCOPE]) + .await + .map(|token| token.as_str().to_string()) + .map_err(auth_acquisition_error) + }) + } +} + +struct GcpProviderLoader; + +impl VertexProviderLoader for GcpProviderLoader { + fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc> { + Box::pin(async move { + let provider: Arc = match source { + CredentialSource::Inline(configured) => Arc::new( + CustomServiceAccount::from_json(validate_request_credentials( + configured.expose(), + )?) + .map_err(auth_acquisition_error)?, + ), + CredentialSource::Trusted(configured) => { + let configured = configured.expose(); + let service_account = if Path::new(configured).is_file() { + CustomServiceAccount::from_file(configured) + } else { + CustomServiceAccount::from_json(configured) + } + .map_err(auth_acquisition_error)?; + Arc::new(service_account) + } + CredentialSource::ApplicationCredentials(path) => { + Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?) + } + CredentialSource::Adc => { + gcp_auth::provider().await.map_err(auth_acquisition_error)? + } + }; + Ok(Arc::new(GcpTokenSource(provider)) as Arc) + }) + } +} + +fn validate_request_credentials(configured: &str) -> Result<&str, AuthError> { + let token_uri = serde_json::from_str::(configured) + .ok() + .and_then(|credentials| { + credentials + .get("token_uri") + .and_then(Value::as_str) + .map(str::to_string) + }); + if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { + return Err(AuthConfigurationError::RequestVertexTokenEndpoint.into()); + } + Ok(configured) +} + +#[derive(Clone, Debug)] +enum CredentialSource { + Inline(SecretValue), + Trusted(SecretValue), + ApplicationCredentials(String), + Adc, +} + +impl CredentialSource { + fn cache_key(&self) -> CredentialCacheKey { + match self { + Self::Inline(configured) => { + CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into()) + } + Self::Trusted(configured) => { + CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into()) + } + Self::ApplicationCredentials(path) => { + CredentialCacheKey::ApplicationCredentials(path.clone()) + } + Self::Adc => CredentialCacheKey::Adc, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +enum CredentialCacheKey { + Inline([u8; 32]), + Trusted([u8; 32]), + ApplicationCredentials(String), + Adc, +} + +fn credential_source( + config: &VertexConfig, + env_lookup: &dyn Fn(&str) -> Option, +) -> CredentialSource { + if let Some(configured) = config.credentials.clone() { + return match configured.source() { + InputSource::Request => CredentialSource::Inline(configured.into_value()), + InputSource::Deployment | InputSource::Environment => { + CredentialSource::Trusted(configured.into_value()) + } + }; + } + if let Some(configured) = non_empty_env(env_lookup, VERTEXAI_CREDENTIALS_ENV) { + return CredentialSource::Trusted(SecretValue::new(configured)); + } + non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) + .map(CredentialSource::ApplicationCredentials) + .unwrap_or(CredentialSource::Adc) +} + +fn optional_credentials( + params: &Map, + sources: &BTreeMap, + names: &[&str], +) -> Result>, AuthError> { + for name in names { + let source = source_for(sources, name); + match params.get(*name) { + None | Some(Value::Null) => continue, + Some(Value::String(value)) if value.trim().is_empty() => continue, + Some(Value::String(value)) => { + return Ok(Some(Sourced::new(SecretValue::new(value), source))); + } + Some(Value::Object(value)) if value.is_empty() => continue, + Some(Value::Object(value)) => { + return serde_json::to_string(value) + .map(SecretValue::new) + .map(|value| Sourced::new(value, source)) + .map(Some) + .map_err(|error| { + AuthError::Configuration(AuthConfigurationError::InvalidFieldType(format!( + "{}: {error}", + names[0] + ))) + }); + } + Some(_) => { + return Err(AuthError::Configuration( + AuthConfigurationError::InvalidFieldType(names[0].to_string()), + )); + } + } + } + Ok(None) +} + +fn source_for(sources: &BTreeMap, name: &str) -> InputSource { + sources.get(name).copied().unwrap_or_default() +} + +fn optional_string( + params: &Map, + names: &[&str], +) -> Result, AuthError> { + for name in names { + match params.get(*name) { + None | Some(Value::Null) => continue, + Some(Value::String(value)) if value.trim().is_empty() => continue, + Some(Value::String(value)) => return Ok(Some(value.clone())), + Some(_) => { + return Err(AuthError::Configuration( + AuthConfigurationError::InvalidFieldType(names[0].to_string()), + )); + } + } + } + Ok(None) +} + +fn non_empty_env(env_lookup: &dyn Fn(&str) -> Option, name: &str) -> Option { + env_lookup(name) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn auth_acquisition_error(error: gcp_auth::Error) -> AuthError { + AuthError::VertexTokenAcquisition(error.to_string()) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use serde_json::json; + + use super::*; + + struct FakeProvider { + calls: Arc, + } + + impl VertexTokenSource for FakeProvider { + fn project_id(&self) -> VertexAuthFuture<'_, String> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok("adc-project".into()) }) + } + + fn token(&self) -> VertexAuthFuture<'_, String> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok("adc-token".into()) }) + } + } + + struct FakeLoader { + loads: Arc, + provider: Arc, + } + + impl VertexProviderLoader for FakeLoader { + fn load( + &self, + _source: CredentialSource, + ) -> VertexAuthFuture<'_, Arc> { + let loads = self.loads.clone(); + let provider = self.provider.clone(); + Box::pin(async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(provider) + }) + } + } + + fn config(value: Value) -> VertexConfig { + VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new()) + .unwrap() + } + + fn auth(calls: Arc, loads: Arc) -> VertexAuth { + let provider: Arc = Arc::new(FakeProvider { calls }); + VertexAuth::new(Arc::new(FakeLoader { loads, provider })) + } + + #[test] + fn config_is_typed_and_secrets_are_redacted() { + let config = config(json!({ + "vertex_credentials":{"private_key":"secret-key"}, + "vertex_project":"project-1", + "vertex_location":"europe-west4" + })); + assert_eq!(config.project_id(), Some("project-1")); + assert_eq!(config.location(), Some("europe-west4")); + assert!(!format!("{config:?}").contains("secret-key")); + assert!( + VertexConfig::from_sourced_optional_params( + json!({"vertex_credentials":true}).as_object().unwrap(), + &BTreeMap::new() + ) + .is_err() + ); + } + + #[test] + fn empty_primary_values_fall_back_to_python_aliases() { + let config = config(json!({ + "vertex_credentials": null, + "vertex_ai_credentials": "alias-credentials", + "vertex_project": " ", + "vertex_ai_project": "alias-project", + "vertex_location": null, + "vertex_ai_location": "alias-location" + })); + assert_eq!( + config.credentials.as_ref().unwrap().value().expose(), + "alias-credentials" + ); + assert_eq!(config.project_id(), Some("alias-project")); + assert_eq!(config.location(), Some("alias-location")); + } + + #[test] + fn project_and_location_prefer_input_then_environment() { + let configured = + config(json!({"vertex_project":"input-project","vertex_location":"input-location"})); + let env = |name: &str| Some(format!("env-{name}")); + assert_eq!( + get_vertex_ai_project(&configured, &env).as_deref(), + Some("input-project") + ); + assert_eq!( + get_vertex_ai_location(&configured, &env).as_deref(), + Some("input-location") + ); + let empty = VertexConfig::default(); + assert_eq!( + get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(), + Some("env-project") + ); + assert_eq!( + get_vertex_ai_location(&empty, &|name| (name == VERTEX_LOCATION_ENV) + .then(|| "fallback-location".into())) + .as_deref(), + Some("fallback-location") + ); + } + + #[test] + fn credential_discovery_prefers_input_then_environment_then_adc() { + let params = json!({"vertex_credentials":"input-json"}); + let sources = BTreeMap::from([("vertex_credentials".to_string(), InputSource::Request)]); + let configured = + VertexConfig::from_sourced_optional_params(params.as_object().unwrap(), &sources) + .unwrap(); + assert!( + matches!(credential_source(&configured, &|_| Some("environment-value".into())), CredentialSource::Inline(value) if value.expose() == "input-json") + ); + let empty = VertexConfig::default(); + assert!( + matches!(credential_source(&empty, &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into())), CredentialSource::Trusted(value) if value.expose() == "environment-json") + ); + assert!( + matches!(credential_source(&empty, &|name| (name == GOOGLE_APPLICATION_CREDENTIALS_ENV).then(|| "adc.json".into())), CredentialSource::ApplicationCredentials(path) if path == "adc.json") + ); + assert!(matches!( + credential_source(&empty, &|_| None), + CredentialSource::Adc + )); + assert_ne!( + CredentialSource::Inline(SecretValue::new("same-value")).cache_key(), + CredentialSource::Trusted(SecretValue::new("same-value")).cache_key() + ); + } + + #[test] + fn request_credentials_require_canonical_token_endpoint() { + assert!( + validate_request_credentials(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#) + .is_ok() + ); + assert!(matches!( + validate_request_credentials(r#"{"token_uri":"http://127.0.0.1/token"}"#), + Err(AuthError::Configuration( + AuthConfigurationError::RequestVertexTokenEndpoint + )) + )); + assert!(matches!( + validate_request_credentials("{}"), + Err(AuthError::Configuration( + AuthConfigurationError::RequestVertexTokenEndpoint + )) + )); + } + + #[tokio::test] + async fn explicit_token_and_header_do_not_acquire_adc() { + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone()); + let configured = config(json!({"vertex_project":"project-1"})); + let explicit = auth + .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None) + .await + .unwrap(); + assert_eq!(explicit.headers[0].1, "Bearer access-token"); + let existing = auth + .validate_environment( + vec![("authorization".into(), "Bearer existing".into())], + None, + &configured, + &|_| None, + ) + .await + .unwrap(); + assert_eq!(existing.headers[0].1, "Bearer existing"); + assert_eq!(loads.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn provider_is_reused_across_authentication_calls() { + let calls = Arc::new(AtomicUsize::new(0)); + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(calls.clone(), loads.clone()); + for _ in 0..2 { + let environment = auth + .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None) + .await + .unwrap(); + assert_eq!(environment.project_id, "adc-project"); + assert_eq!(environment.headers[0].1, "Bearer adc-token"); + } + assert_eq!(loads.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 4); + } +} diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 8a12e186197..9469d379462 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -51,5 +51,15 @@ pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; pub(crate) const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; pub(crate) const OCR_MAX_FETCH_REDIRECTS: usize = 10; +pub(crate) const OCR_POLL_TIMEOUT_SECS: u64 = 120; +pub(crate) const OCR_POLL_RETRY_SECS: u64 = 2; +pub(crate) const AZURE_DI_API_VERSION: &str = "2024-11-30"; +pub(crate) const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key"; +pub(crate) const AZURE_DI_DEFAULT_DPI: i64 = 96; +pub(crate) const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5; +pub(crate) const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0; +pub(crate) const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; +pub(crate) const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; +pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://"; pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr"; pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index eefa7d606d8..fa4a9d36e03 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -22,11 +22,17 @@ pub enum Error { )] MissingApiKey { provider: &'static str }, #[error( - "Missing Azure AI credentials - set AZURE_AI_API_KEY or provide an Authorization header" + "invalid authentication configuration: Missing Azure AI credentials - set AZURE_AI_API_KEY or configure Entra ID" )] MissingAzureAiCredentials, - #[error("Missing Azure AI credentials - set AZURE_AI_API_KEY or provide azure_ad_token")] - MissingAzureAiCredentialsOrAdToken, + #[error( + "invalid authentication configuration: Missing Azure Document Intelligence credentials - set AZURE_DOCUMENT_INTELLIGENCE_API_KEY or configure Entra ID" + )] + MissingAzureDocumentIntelligenceCredentials, + #[error( + "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()" + )] + MissingReductoApiKey, #[error("upstream request failed with status {status}: {body}")] Http { status: u16, body: String }, #[error("upstream network error: {0}")] @@ -121,6 +127,15 @@ impl From for Error { } } +impl From for Error { + fn from(error: crate::AuthError) -> Self { + match error { + crate::AuthError::MissingApiKey { provider } => Self::MissingApiKey { provider }, + error => Self::Auth(error.to_string()), + } + } +} + pub fn json_type_name(value: &serde_json::Value) -> &'static str { match value { serde_json::Value::Null => "null", @@ -136,6 +151,14 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str { mod transport_tests { use super::*; + #[test] + fn missing_auth_key_preserves_provider_in_public_error() { + assert_eq!( + Error::from(crate::AuthError::MissingApiKey { provider: "Vertex" }), + Error::MissingApiKey { provider: "Vertex" } + ); + } + #[tokio::test] async fn transport_errors_remove_urls_and_keep_dispatch_context() { let error = reqwest::Client::builder() diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 7e81b292441..0b3573deab2 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,4 +1,5 @@ pub mod audio_transcription; +pub mod auth; pub mod caching; pub mod call_lifecycle; pub mod chat_completions; @@ -17,4 +18,5 @@ pub mod router; pub mod routing_utils; mod url_utils; +pub use auth::AuthError; pub use error::Error; diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs new file mode 100644 index 00000000000..71ca69ddc58 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs @@ -0,0 +1,214 @@ +use super::super::OcrAdapter; +use crate::Error; +use crate::auth::{InputSource, Sourced}; +use crate::constants::{AZURE_DI_API_VERSION, AZURE_DI_SUBSCRIPTION_HEADER}; +use crate::ocr::OcrClient; +use crate::ocr::codecs::document_intelligence::{ + self, AzureDocumentIntelligenceOperation, DocumentIntelligenceParams, +}; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::prepare::{credential_env, transform_request_body}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrResponseFormat}; +use crate::ocr::wire::DecodedOcrResponse; +use crate::providers::azure_ai::auth::AzureAuthInputs; +use crate::url_utils::ApiUrl; + +mod polling; + +const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; +const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; + +#[derive(Clone, Debug)] +pub(crate) struct AzureDocumentIntelligenceAdapter; + +impl OcrAdapter for AzureDocumentIntelligenceAdapter { + type ProviderResponse = AzureDocumentIntelligenceOperation; + const PROVIDER: OcrProvider = OcrProvider::AzureAi; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + let params = map_ocr_params(request)?; + let config = AzureAuthInputs::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + ) + .map_err(Error::from)?; + let headers = validate_environment(&request.connection, &config, &credential_env).await?; + let endpoint = nonblank(request.connection.api_base.clone()) + .or_else(|| nonblank(credential_env(AZURE_DI_ENDPOINT_ENV))) + .ok_or_else(|| Error::Auth("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into()))?; + let url = get_complete_url(&endpoint, &request.model, ¶ms)?; + let body = document_intelligence::transform_ocr_request(request.document.clone())?; + transform_request_body(client, request, &url, &headers, body, |_| Ok(())).await + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + document_intelligence::transform_ocr_response(&request.model, response) + } + + async fn read_response( + &self, + client: &OcrClient, + response: reqwest::Response, + url: &str, + headers: &[(String, String)], + request: &LiteLLMOcrRequest, + ) -> Result, OcrError> { + polling::read_operation_response( + client.polling_http(), + response, + url, + headers, + &request.connection, + request.response_format()? == OcrResponseFormat::Native, + ) + .await + } +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +fn map_ocr_params( + request: &LiteLLMOcrRequest, +) -> Result { + let params = document_intelligence::decode_input_params( + request.optional_params.clone(), + "optional_params", + )?; + let crate::ocr::prepare::ParsedProviderParams { + known: params, + extra_params: _extra_params, + } = params; + document_intelligence::map_ocr_params(params) +} + +fn get_complete_url( + endpoint: &str, + model: &str, + params: &DocumentIntelligenceParams, +) -> Result { + let model = format!("{}:analyze", model_id(model)?); + ApiUrl::parse(endpoint) + .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model])) + .map(|url| { + url.append_query_pairs( + [("api-version", AZURE_DI_API_VERSION)] + .into_iter() + .chain(params.pages.iter().map(|pages| ("pages", pages.as_str()))) + .chain( + params + .features + .iter() + .map(|features| ("features", features.as_str())), + ), + ) + .into_string() + }) + .map_err(|_| OcrRequestError::RequestField { + path: "api_base".into(), + }) + .map_err(OcrError::from) +} + +async fn validate_environment( + connection: &OcrConnection, + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + if crate::http_utils::has_header(&connection.extra_headers, "authorization") + || crate::http_utils::has_header(&connection.extra_headers, AZURE_DI_SUBSCRIPTION_HEADER) + { + super::validate_destination(connection, connection.extra_headers_source)?; + return Ok(connection.extra_headers.clone()); + } + let key = nonblank(connection.api_key.clone()) + .map(|value| Sourced::new(value, connection.api_key_source)) + .or_else(|| { + nonblank(env_lookup(AZURE_DI_API_KEY_ENV)) + .map(|value| Sourced::new(value, InputSource::Environment)) + }); + if let Some(key) = key { + super::validate_destination(connection, key.source())?; + return Ok( + std::iter::once((AZURE_DI_SUBSCRIPTION_HEADER.into(), key.into_value())) + .chain(connection.extra_headers.clone()) + .collect(), + ); + } + let token = super::resolve_entra(config, env_lookup) + .await? + .ok_or(Error::MissingAzureDocumentIntelligenceCredentials)?; + super::validate_destination(connection, token.source())?; + Ok( + std::iter::once(("Authorization".into(), format!("Bearer {}", token.value()))) + .chain(connection.extra_headers.clone()) + .collect(), + ) +} + +fn model_id(model: &str) -> Result<&str, OcrRequestError> { + let model = model.rsplit('/').next().unwrap_or(model); + if matches!(model, "." | "..") { + return Err(OcrRequestError::DotModel); + } + Ok(model) +} + +fn nonblank(value: Option) -> Option { + value + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn request_endpoint_cannot_receive_environment_key() { + let connection = OcrConnection { + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let error = validate_environment(&connection, &Default::default(), &|name| { + (name == AZURE_DI_API_KEY_ENV).then(|| "environment-key".into()) + }) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("request-controlled Azure endpoint") + ); + } + + #[tokio::test] + async fn request_endpoint_accepts_request_owned_key() { + let connection = OcrConnection { + api_key: Some("request-key".into()), + api_key_source: InputSource::Request, + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let headers = validate_environment(&connection, &Default::default(), &|_| None) + .await + .unwrap(); + + assert_eq!( + headers[0], + (AZURE_DI_SUBSCRIPTION_HEADER.into(), "request-key".into()) + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs new file mode 100644 index 00000000000..1bddea0da4f --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs @@ -0,0 +1,100 @@ +use std::time::Duration; + +use reqwest::Url; +use tokio::time::Instant; + +use crate::constants::{AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS}; +use crate::ocr::client::read_json_response; +use crate::ocr::codecs::document_intelligence::{ + AzureDocumentIntelligenceOperation, OperationStatus, +}; +use crate::ocr::error::{OcrError, OcrPollingError, OcrResponseError}; +use crate::ocr::types::OcrConnection; +use crate::ocr::wire::DecodedOcrResponse; + +pub(super) async fn read_operation_response( + http_client: &reqwest::Client, + response: reqwest::Response, + original_url: &str, + headers: &[(String, String)], + connection: &OcrConnection, + native: bool, +) -> Result, OcrError> { + if response.status() != reqwest::StatusCode::ACCEPTED { + return read_json_response(response, native).await; + } + let location = response + .headers() + .get("operation-location") + .and_then(|value| value.to_str().ok()) + .ok_or(OcrPollingError::PollLocation)?; + let original = Url::parse(original_url).map_err(|_| OcrPollingError::PollOrigin)?; + let operation = Url::parse(location).map_err(|_| OcrPollingError::PollOrigin)?; + if original.origin() != operation.origin() + || !operation.username().is_empty() + || operation.password().is_some() + { + return Err(OcrPollingError::PollOrigin.into()); + } + poll_operation(http_client, operation, headers, connection, native).await +} + +async fn poll_operation( + http_client: &reqwest::Client, + url: Url, + headers: &[(String, String)], + connection: &OcrConnection, + native: bool, +) -> Result, OcrError> { + let deadline = Instant::now() + .checked_add(connection.poll_timeout) + .ok_or(OcrPollingError::PollTimeout)?; + loop { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(OcrPollingError::PollTimeout)?; + let builder = http_client + .get(url.clone()) + .timeout(remaining.min(connection.timeout)); + let builder = crate::http_utils::with_headers( + builder, + headers, + crate::http_utils::HeaderPolicy::Only(&[AZURE_DI_SUBSCRIPTION_HEADER, "authorization"]), + ); + let response = tokio::time::timeout_at(deadline, crate::http_utils::http_request(builder)) + .await + .map_err(|_| OcrPollingError::PollTimeout)? + .map_err(crate::error::TransportError::from)?; + let retry = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(OCR_POLL_RETRY_SECS) + .max(1); + let decoded = tokio::time::timeout_at( + deadline, + read_json_response::(response, native), + ) + .await + .map_err(|_| OcrPollingError::PollTimeout)??; + match &decoded.data.status { + Some(OperationStatus::Succeeded) => return Ok(decoded), + Some(OperationStatus::Running | OperationStatus::NotStarted) => { + tokio::time::timeout_at(deadline, tokio::time::sleep(Duration::from_secs(retry))) + .await + .map_err(|_| OcrPollingError::PollTimeout)?; + } + status => { + return Err(OcrResponseError::OperationStatus( + status + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "None".into()), + ) + .into()); + } + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure_mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs similarity index 57% rename from litellm-rust/crates/core/src/ocr/adapters/azure_mistral.rs rename to litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs index 468c883a1dd..3107494d39e 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure_mistral.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs @@ -1,5 +1,6 @@ -use super::OcrAdapter; +use super::super::OcrAdapter; use crate::Error; +use crate::auth::{InputSource, Sourced}; use crate::constants::AZURE_AI_OCR_PATH; use crate::ocr::OcrClient; use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; @@ -10,6 +11,7 @@ use crate::ocr::prepare::{ }; use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection}; +use crate::providers::azure_ai::auth::AzureAuthInputs; use crate::url_utils::ApiUrl; const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; @@ -31,7 +33,12 @@ impl OcrAdapter for AzureMistralAdapter { known: params, extra_params: _extra_params, } = _prepare_ocr_request::(request)?; - let headers = authenticate(&request.connection, &credential_env)?; + let config = AzureAuthInputs::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + ) + .map_err(Error::from)?; + let headers = validate_environment(&request.connection, &config, &credential_env).await?; let url = get_complete_url(request.connection.api_base.as_deref(), &credential_env)?; let document = inline_remote_document( client.document_fetcher(), @@ -76,21 +83,36 @@ fn get_complete_url( }) } -fn authenticate( +async fn validate_environment( connection: &OcrConnection, - env_lookup: &dyn Fn(&str) -> Option, + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result, OcrError> { if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + super::validate_destination(connection, connection.extra_headers_source)?; return Ok(connection.extra_headers.clone()); } let key = nonblank(connection.api_key.clone()) - .or_else(|| nonblank(env_lookup(AZURE_AI_API_KEY_ENV))) + .map(|value| Sourced::new(value, connection.api_key_source)) + .or_else(|| { + nonblank(env_lookup(AZURE_AI_API_KEY_ENV)) + .map(|value| Sourced::new(value, InputSource::Environment)) + }); + if let Some(key) = key { + super::validate_destination(connection, key.source())?; + return Ok(bearer_headers(connection, key.value())); + } + let key = super::resolve_entra(config, env_lookup) + .await? .ok_or(Error::MissingAzureAiCredentials)?; - Ok( - std::iter::once(("Authorization".into(), format!("Bearer {key}"))) - .chain(connection.extra_headers.clone()) - .collect(), - ) + super::validate_destination(connection, key.source())?; + Ok(bearer_headers(connection, key.value())) +} + +fn bearer_headers(connection: &OcrConnection, key: &str) -> Vec<(String, String)> { + std::iter::once(("Authorization".into(), format!("Bearer {key}"))) + .chain(connection.extra_headers.clone()) + .collect() } fn nonblank(value: Option) -> Option { @@ -119,27 +141,76 @@ mod tests { ); } - #[test] - fn supplied_authorization_precedes_keys() { + #[tokio::test] + async fn supplied_authorization_precedes_keys() { let connection = OcrConnection { api_key: Some("request-key".into()), extra_headers: vec![("authorization".into(), "Bearer prepared".into())], ..Default::default() }; assert_eq!( - authenticate(&connection, &|_| Some("environment-key".into())).unwrap(), + validate_environment(&connection, &Default::default(), &|_| { + Some("environment-key".into()) + }) + .await + .unwrap(), connection.extra_headers ); } - #[test] - fn request_key_precedes_environment_key() { + #[tokio::test] + async fn request_key_precedes_environment_key() { let connection = OcrConnection { api_key: Some("request-key".into()), ..Default::default() }; assert_eq!( - authenticate(&connection, &|_| Some("environment-key".into())).unwrap()[0], + validate_environment(&connection, &Default::default(), &|_| { + Some("environment-key".into()) + }) + .await + .unwrap()[0], + ("Authorization".into(), "Bearer request-key".into()) + ); + } + + #[tokio::test] + async fn request_endpoint_cannot_receive_environment_key() { + let connection = OcrConnection { + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let error = validate_environment(&connection, &Default::default(), &|name| { + (name == AZURE_AI_API_KEY_ENV).then(|| "environment-key".into()) + }) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("request-controlled Azure endpoint") + ); + } + + #[tokio::test] + async fn request_endpoint_accepts_request_owned_key() { + let connection = OcrConnection { + api_key: Some("request-key".into()), + api_key_source: InputSource::Request, + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let headers = validate_environment(&connection, &Default::default(), &|_| None) + .await + .unwrap(); + + assert_eq!( + headers[0], ("Authorization".into(), "Bearer request-key".into()) ); } diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs new file mode 100644 index 00000000000..9c02a7471c9 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs @@ -0,0 +1,49 @@ +mod document_intelligence; +mod mistral; + +use std::sync::OnceLock; + +use crate::Error; +use crate::auth::error::AuthConfigurationError; +use crate::auth::{InputSource, Sourced}; +use crate::ocr::error::OcrError; +use crate::ocr::types::OcrConnection; +use crate::providers::azure_ai::auth::{AzureAuthInputs, AzureAuthService}; + +pub(crate) use document_intelligence::AzureDocumentIntelligenceAdapter; +pub(crate) use mistral::AzureMistralAdapter; + +async fn resolve_entra( + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result>, Error> { + static SERVICE: OnceLock = OnceLock::new(); + SERVICE + .get_or_init(AzureAuthService::default) + .get_azure_ad_token(config, env_lookup) + .await + .map(|credential| { + credential.map(|credential| { + let source = credential.source(); + let value = credential.value().secret().expose().to_string(); + Sourced::new(value, source) + }) + }) + .map_err(Error::from) +} + +fn validate_destination( + connection: &OcrConnection, + credential_source: InputSource, +) -> Result<(), OcrError> { + if connection.api_base.is_some() + && connection.api_base_source == InputSource::Request + && credential_source != InputSource::Request + { + return Err(Error::from(crate::AuthError::Configuration( + AuthConfigurationError::RequestAzureCredentialDestination, + )) + .into()); + } + Ok(()) +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/mod.rs index bbd6feb6c7b..9171d11836c 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/mod.rs @@ -8,11 +8,15 @@ use super::registry::OcrProvider; use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrResponseFormat}; use super::wire::DecodedOcrResponse; -mod azure_mistral; +mod azure; mod mistral; +mod reducto; +mod vertex; -pub(crate) use azure_mistral::AzureMistralAdapter; +pub(crate) use azure::{AzureDocumentIntelligenceAdapter, AzureMistralAdapter}; pub(crate) use mistral::MistralAdapter; +pub(crate) use reducto::{ReductoLegacyAdapter, ReductoV3Adapter}; +pub(crate) use vertex::{VertexDeepSeekAdapter, VertexMistralAdapter}; /// Converts a complete LiteLLM OCR call to provider HTTP and normalizes its response. pub(crate) trait OcrAdapter: Send + Sync + Sized + 'static { @@ -65,6 +69,11 @@ macro_rules! for_each_ocr_adapter { $callback! { Mistral, $crate::ocr::adapters::MistralAdapter, $crate::ocr::adapters::MistralAdapter, Mistral; AzureMistral, $crate::ocr::adapters::AzureMistralAdapter, $crate::ocr::adapters::AzureMistralAdapter, AzureAi; + AzureDocumentIntelligence, $crate::ocr::adapters::AzureDocumentIntelligenceAdapter, $crate::ocr::adapters::AzureDocumentIntelligenceAdapter, AzureAi; + ReductoLegacy, $crate::ocr::adapters::ReductoLegacyAdapter, $crate::ocr::adapters::ReductoLegacyAdapter, Reducto; + ReductoV3, $crate::ocr::adapters::ReductoV3Adapter, $crate::ocr::adapters::ReductoV3Adapter, Reducto; + VertexMistral, $crate::ocr::adapters::VertexMistralAdapter, $crate::ocr::adapters::VertexMistralAdapter, VertexAi; + VertexDeepSeek, $crate::ocr::adapters::VertexDeepSeekAdapter, $crate::ocr::adapters::VertexDeepSeekAdapter, VertexAi; } }; } diff --git a/litellm-rust/crates/core/src/ocr/adapters/reducto/legacy.rs b/litellm-rust/crates/core/src/ocr/adapters/reducto/legacy.rs new file mode 100644 index 00000000000..062a0071a34 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/reducto/legacy.rs @@ -0,0 +1,45 @@ +use super::super::OcrAdapter; +use crate::ocr::OcrClient; +use crate::ocr::codecs::reducto::{self, ReductoLegacyParams, ReductoResponse}; +use crate::ocr::error::{OcrError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, build_http_request, credential_env, + guardrail_document, merge_extra_params, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; + +#[derive(Clone, Debug)] +pub(crate) struct ReductoLegacyAdapter; + +impl OcrAdapter for ReductoLegacyAdapter { + type ProviderResponse = ReductoResponse; + const PROVIDER: OcrProvider = OcrProvider::Reducto; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + let ParsedProviderParams { + known: params, + extra_params, + } = _prepare_ocr_request::(request)?; + let headers = super::validate_environment(&request.connection, &credential_env)?; + let url = super::get_complete_url(request.connection.api_base.as_deref(), "parse")?; + let document = guardrail_document(request, &url).await?; + let document = + super::prepare_document(client, document, &request.connection, &headers).await?; + let body = reducto::transform_legacy_ocr_request(&request.model, document, ¶ms)?; + let body = merge_extra_params(&body, extra_params)?; + build_http_request(client, request, &url, &headers, &body) + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + reducto::transform_ocr_response(&request.model, response) + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs new file mode 100644 index 00000000000..7621d0d326a --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs @@ -0,0 +1,148 @@ +mod legacy; +mod v3; + +use crate::Error; +use crate::constants::{REDUCTO_API_BASE, REDUCTO_API_KEY_ENV, REDUCTO_ID_PREFIX}; +use crate::ocr::document::InlineDocument; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::types::{OcrConnection, OcrDocument}; +use crate::url_utils::ApiUrl; + +pub(crate) use legacy::ReductoLegacyAdapter; +pub(crate) use v3::ReductoV3Adapter; + +pub(super) fn get_complete_url(api_base: Option<&str>, path: &str) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(REDUCTO_API_BASE); + ApiUrl::parse(base) + .and_then(|url| url.complete_path(&[path])) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +pub(super) fn validate_environment( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + return Ok(connection.extra_headers.clone()); + } + let api_key = connection + .api_key + .as_deref() + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| { + env_lookup(REDUCTO_API_KEY_ENV) + .map(|key| key.trim().to_string()) + .filter(|key| !key.is_empty()) + }) + .ok_or(Error::MissingReductoApiKey)?; + Ok( + std::iter::once(("Authorization".into(), format!("Bearer {api_key}"))) + .chain(connection.extra_headers.clone()) + .collect(), + ) +} + +pub(super) async fn prepare_document( + client: &crate::ocr::OcrClient, + document: OcrDocument, + connection: &OcrConnection, + headers: &[(String, String)], +) -> Result { + if document.source().starts_with(REDUCTO_ID_PREFIX) { + if document.source()[REDUCTO_ID_PREFIX.len()..] + .trim() + .is_empty() + { + return Err(OcrRequestError::RequestField { + path: "document file id".into(), + } + .into()); + } + return Ok(document); + } + let inline = InlineDocument::parse(document.source())?.ok_or(OcrRequestError::ReductoSource)?; + let mime = inline.mime_type().to_string(); + let bytes = inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + let part = reqwest::multipart::Part::bytes(bytes) + .file_name("document") + .mime_str(&mime) + .map_err(|_| OcrRequestError::InvalidDataUri)?; + let builder = client + .provider_http() + .post(get_complete_url(connection.api_base.as_deref(), "upload")?) + .multipart(reqwest::multipart::Form::new().part("file", part)) + .timeout(connection.timeout); + let builder = crate::http_utils::with_headers( + builder, + headers, + crate::http_utils::HeaderPolicy::Except(&["content-type", "content-length"]), + ); + let response = crate::http_utils::http_request(builder) + .await + .map_err(crate::error::TransportError::from)?; + let uploaded = crate::ocr::client::read_json_response::< + crate::ocr::codecs::reducto::ReductoUploadResponse, + >(response, false) + .await? + .data; + let file_id = uploaded + .file_id + .as_deref() + .map(str::trim) + .filter(|id| !id.is_empty()); + let Some(file_id) = file_id else { + return Err(OcrResponseError::ResponseField { + path: "file_id".into(), + } + .into()); + }; + Ok(document.with_source(file_id.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn explicit_key_precedes_environment_key() { + let connection = OcrConnection { + api_key: Some("passed-key".into()), + ..Default::default() + }; + let headers = validate_environment(&connection, &|_| Some("env-key".into())).unwrap(); + assert_eq!(headers[0].1, "Bearer passed-key"); + } + + #[test] + fn blank_explicit_key_uses_environment_key() { + let connection = OcrConnection { + api_key: Some(" ".into()), + ..Default::default() + }; + let headers = validate_environment(&connection, &|_| Some(" env-key ".into())).unwrap(); + assert_eq!(headers[0].1, "Bearer env-key"); + } + + #[test] + fn existing_authorization_skips_key_lookup() { + let connection = OcrConnection { + extra_headers: vec![("authorization".into(), "Bearer existing".into())], + ..Default::default() + }; + assert_eq!( + validate_environment(&connection, &|_| None).unwrap(), + connection.extra_headers + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/reducto/v3.rs b/litellm-rust/crates/core/src/ocr/adapters/reducto/v3.rs new file mode 100644 index 00000000000..a49f8105e26 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/reducto/v3.rs @@ -0,0 +1,45 @@ +use super::super::OcrAdapter; +use crate::ocr::OcrClient; +use crate::ocr::codecs::reducto::{self, ReductoResponse, ReductoV3Params}; +use crate::ocr::error::{OcrError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, build_http_request, credential_env, + guardrail_document, merge_extra_params, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; + +#[derive(Clone, Debug)] +pub(crate) struct ReductoV3Adapter; + +impl OcrAdapter for ReductoV3Adapter { + type ProviderResponse = ReductoResponse; + const PROVIDER: OcrProvider = OcrProvider::Reducto; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + let ParsedProviderParams { + known: params, + extra_params, + } = _prepare_ocr_request::(request)?; + let headers = super::validate_environment(&request.connection, &credential_env)?; + let url = super::get_complete_url(request.connection.api_base.as_deref(), "parse")?; + let document = guardrail_document(request, &url).await?; + let document = + super::prepare_document(client, document, &request.connection, &headers).await?; + let body = reducto::transform_v3_ocr_request(&request.model, document, ¶ms)?; + let body = merge_extra_params(&body, extra_params)?; + build_http_request(client, request, &url, &headers, &body) + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + reducto::transform_ocr_response(&request.model, response) + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs new file mode 100644 index 00000000000..ef188f8b9ac --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs @@ -0,0 +1,134 @@ +use super::super::OcrAdapter; +use super::validate_destination; +use crate::Error; +use crate::auth::vertex::{self, VertexConfig}; +use crate::ocr::OcrClient; +use crate::ocr::codecs::deepseek::{self, DeepSeekOcrParams, DeepSeekOcrResponse}; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, credential_env, transform_request_body, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use crate::url_utils::ApiUrl; +const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com"; +const MODEL_NAMESPACE: &str = "deepseek-ai"; +const DEFAULT_LOCATION: &str = "us-central1"; + +#[derive(Clone, Debug)] +pub(crate) struct VertexDeepSeekAdapter; + +impl OcrAdapter for VertexDeepSeekAdapter { + type ProviderResponse = DeepSeekOcrResponse; + const PROVIDER: OcrProvider = OcrProvider::VertexAi; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + validate_destination(&request.connection)?; + let ParsedProviderParams { + known: params, + extra_params: _extra_params, + } = _prepare_ocr_request::(request)?; + let config = VertexConfig::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + ) + .map_err(Error::from)?; + let authentication = client + .vertex_auth() + .validate_environment( + request.connection.extra_headers.clone(), + request.connection.api_key.as_deref(), + &config, + &credential_env, + ) + .await + .map_err(Error::from)?; + let location = vertex::get_vertex_ai_location(&config, &credential_env) + .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); + let url = get_complete_url( + request.connection.api_base.as_deref(), + &authentication.project_id, + &location, + )?; + let document = request.document.clone(); + let body = + deepseek::transform_ocr_request(&provider_model(&request.model), document, ¶ms)?; + transform_request_body(client, request, &url, &authentication.headers, body, |_| { + Ok(()) + }) + .await + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + deepseek::transform_ocr_response(&request.model, response) + } +} + +fn provider_model(model: &str) -> String { + if model.starts_with(&format!("{MODEL_NAMESPACE}/")) { + model.to_string() + } else { + format!("{MODEL_NAMESPACE}/{model}") + } +} + +fn get_complete_url( + api_base: Option<&str>, + project: &str, + location: &str, +) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(DEFAULT_API_BASE); + ApiUrl::parse(base) + .and_then(|url| { + url.complete_path(&[ + "v1", + "projects", + project, + "locations", + location, + "endpoints", + "openapi", + "chat", + "completions", + ]) + }) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +#[cfg(test)] +mod tests { + use super::{get_complete_url, provider_model}; + + #[test] + fn adapter_owns_model_namespace_and_endpoint() { + assert_eq!( + provider_model("deepseek-ocr-maas"), + "deepseek-ai/deepseek-ocr-maas" + ); + assert_eq!( + provider_model("deepseek-ai/deepseek-ocr-maas"), + "deepseek-ai/deepseek-ocr-maas" + ); + assert_eq!( + get_complete_url(None, "proj-1", "europe-west4").unwrap(), + "https://aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/endpoints/openapi/chat/completions" + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs new file mode 100644 index 00000000000..f3335bf497c --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs @@ -0,0 +1,154 @@ +use super::super::OcrAdapter; +use super::validate_destination; +use crate::Error; +use crate::auth::vertex::{self, VertexConfig}; +use crate::ocr::OcrClient; +use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; +use crate::ocr::document::{inline_remote_document, validate_inline_document}; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, credential_env, transform_request_body, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use crate::url_utils::ApiUrl; +const DEFAULT_LOCATION: &str = "us-central1"; + +#[derive(Clone, Debug)] +pub(crate) struct VertexMistralAdapter; + +impl OcrAdapter for VertexMistralAdapter { + type ProviderResponse = MistralOcrResponse; + const PROVIDER: OcrProvider = OcrProvider::VertexAi; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + validate_destination(&request.connection)?; + let ParsedProviderParams { + known: params, + extra_params: _extra_params, + } = _prepare_ocr_request::(request)?; + let config = VertexConfig::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + ) + .map_err(Error::from)?; + let authentication = client + .vertex_auth() + .validate_environment( + request.connection.extra_headers.clone(), + request.connection.api_key.as_deref(), + &config, + &credential_env, + ) + .await + .map_err(Error::from)?; + let location = vertex::get_vertex_ai_location(&config, &credential_env) + .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); + let url = get_complete_url( + request.connection.api_base.as_deref(), + &authentication.project_id, + &location, + &request.model, + )?; + let document = inline_remote_document( + client.document_fetcher(), + request.document.clone(), + &request.connection, + ) + .await?; + let body = mistral::transform_ocr_request(&request.model, document, ¶ms)?; + transform_request_body( + client, + request, + &url, + &authentication.headers, + body, + |body| validate_inline_document(&body.document), + ) + .await + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + mistral::transform_ocr_response(&request.model, response) + } +} + +fn get_complete_url( + api_base: Option<&str>, + project: &str, + location: &str, + model: &str, +) -> Result { + validate_location(location)?; + let default_base = format!("https://{location}-aiplatform.googleapis.com"); + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(&default_base); + let prediction = format!("{model}:rawPredict"); + ApiUrl::parse(base) + .and_then(|url| { + url.complete_path(&[ + "v1", + "projects", + project, + "locations", + location, + "publishers", + "mistralai", + "models", + &prediction, + ]) + }) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +fn validate_location(location: &str) -> Result<(), OcrError> { + let valid = !location.is_empty() + && location + .bytes() + .all(|value| value.is_ascii_lowercase() || value.is_ascii_digit() || value == b'-') + && location + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && location + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric); + if valid { + return Ok(()); + } + Err(OcrRequestError::RequestField { + path: "vertex_location".into(), + } + .into()) +} + +#[cfg(test)] +mod tests { + use super::get_complete_url; + + #[test] + fn endpoint_uses_location_project_and_model() { + assert_eq!( + get_complete_url(None, "proj-1", "europe-west4", "mistral-ocr-maas").unwrap(), + "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); + assert!(get_complete_url(None, "proj-1", "attacker.example/path", "model").is_err()); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs new file mode 100644 index 00000000000..270c41e647d --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs @@ -0,0 +1,21 @@ +mod deepseek; +mod mistral; + +use crate::Error; +use crate::auth::InputSource; +use crate::auth::error::AuthConfigurationError; +use crate::ocr::error::OcrError; +use crate::ocr::types::OcrConnection; + +pub(crate) use deepseek::VertexDeepSeekAdapter; +pub(crate) use mistral::VertexMistralAdapter; + +fn validate_destination(connection: &OcrConnection) -> Result<(), OcrError> { + if connection.api_base.is_some() && connection.api_base_source == InputSource::Request { + return Err(Error::from(crate::AuthError::Configuration( + AuthConfigurationError::RequestVertexCredentialDestination, + )) + .into()); + } + Ok(()) +} diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 699e44412ac..ab2d098d0bb 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -8,6 +8,7 @@ use super::handler::perform_ocr_request; use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; use super::wire::{DecodedOcrResponse, decode_response}; use crate::Error; +use crate::auth::vertex::VertexAuth; use crate::constants::OCR_CONNECT_TIMEOUT_SECS; use crate::error::TransportError; use crate::media::MediaFetcher; @@ -15,7 +16,9 @@ use crate::media::MediaFetcher; #[derive(Clone)] pub struct OcrClient { provider_http: reqwest::Client, + polling_http: reqwest::Client, document_fetcher: MediaFetcher, + vertex_auth: VertexAuth, } impl OcrClient { @@ -23,7 +26,9 @@ impl OcrClient { let document_fetcher = MediaFetcher::new().map_err(TransportError::from)?; Ok(Self { provider_http, + polling_http: no_redirect_http()?, document_fetcher, + vertex_auth: VertexAuth::default(), }) } @@ -41,19 +46,37 @@ impl OcrClient { &self.provider_http } + pub(crate) fn polling_http(&self) -> &reqwest::Client { + &self.polling_http + } + pub(crate) fn document_fetcher(&self) -> &MediaFetcher { &self.document_fetcher } + pub(crate) fn vertex_auth(&self) -> &VertexAuth { + &self.vertex_auth + } + #[cfg(test)] pub(crate) fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { Self { provider_http, + polling_http: no_redirect_http().expect("test polling client builds"), document_fetcher: MediaFetcher::for_test(document_http), + vertex_auth: VertexAuth::default(), } } } +fn no_redirect_http() -> Result { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(TransportError::from) +} + pub async fn ocr(request: LiteLLMOcrRequest) -> Result { static CLIENT: OnceLock> = OnceLock::new(); let client = CLIENT diff --git a/litellm-rust/crates/core/src/ocr/codecs/deepseek/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/deepseek/mod.rs new file mode 100644 index 00000000000..682b3addde7 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/deepseek/mod.rs @@ -0,0 +1,5 @@ +mod transformation; +mod types; + +pub(crate) use transformation::{transform_ocr_request, transform_ocr_response}; +pub(crate) use types::{DeepSeekOcrParams, DeepSeekOcrResponse}; diff --git a/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs new file mode 100644 index 00000000000..98cfc0db78d --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs @@ -0,0 +1,98 @@ +use serde::de::IntoDeserializer; +use serde_json::{Value, json}; + +use super::types::*; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn transform_ocr_request( + provider_model: &str, + document: OcrDocument, + params: &DeepSeekOcrParams, +) -> Result { + if document.source().is_empty() { + return Err(OcrRequestError::MissingField("document URL")); + } + Ok(DeepSeekOcrRequest { + model: provider_model.to_string(), + messages: vec![DeepSeekOcrMessage { + role: UserRole::User, + content: vec![document], + }], + params: params.clone(), + }) +} + +pub(crate) fn transform_ocr_response( + model: &str, + response: DeepSeekOcrResponse, +) -> Result { + let content = response + .choices + .into_iter() + .next() + .and_then(|choice| choice.message.content) + .ok_or(OcrResponseError::EmptyContent)?; + let decoded = decode_content(content)?; + let pages = match decoded.result.pages { + Some(pages) if !pages.is_empty() => pages + .into_iter() + .map(|page| serde_json::to_value(page).expect("DeepSeek page serializes")) + .collect(), + _ => vec![json!({ + "index":0, + "markdown":decoded.fallback_markdown, + "images":null + })], + }; + Ok(LiteLLMOcrResponse { + pages, + model: decoded.result.model.unwrap_or_else(|| model.to_string()), + document_annotation: decoded.result.document_annotation, + usage_info: decoded.result.usage_info.or(response.usage), + object: "ocr".into(), + extra_fields: decoded.result.extra_fields, + provider_native_response: None, + }) +} + +struct DecodedContent { + result: DeepSeekOcrResult, + fallback_markdown: String, +} + +fn decode_content(content: DeepSeekContent) -> Result { + let (result, fallback_markdown) = match content { + DeepSeekContent::Text(text) if text.is_empty() => { + return Err(OcrResponseError::EmptyContent); + } + DeepSeekContent::Text(text) => (decode_json_content(&text)?, text), + DeepSeekContent::Object(object) => { + let fallback = + serde_json::to_string(&object).map_err(|_| OcrResponseError::ResponseField { + path: "choices[0].message.content".into(), + })?; + (Some(object), fallback) + } + }; + Ok(DecodedContent { + result: result.unwrap_or_default(), + fallback_markdown, + }) +} + +fn decode_json_content(text: &str) -> Result, OcrResponseError> { + if !text.trim_start().starts_with('{') { + return Ok(None); + } + let value = match serde_json::from_str::(text) { + Ok(value) => value, + Err(_) => return Ok(None), + }; + serde_path_to_error::deserialize(value.into_deserializer()) + .map(Some) + .map_err(|error| OcrResponseError::ResponseField { + path: format!("choices[0].message.content.{}", error.path()), + }) +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/deepseek/types.rs b/litellm-rust/crates/core/src/ocr/codecs/deepseek/types.rs new file mode 100644 index 00000000000..0ce2d9913f7 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/deepseek/types.rs @@ -0,0 +1,95 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct DeepSeekOcrParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub stream: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum StopSequences { + One(String), + Many(Vec), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct DeepSeekOcrRequest { + pub model: String, + pub messages: Vec, + #[serde(flatten)] + pub params: DeepSeekOcrParams, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct DeepSeekOcrMessage { + pub role: UserRole, + pub content: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum UserRole { + User, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DeepSeekOcrResponse { + #[serde(default)] + pub choices: Vec, + pub usage: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DeepSeekChoice { + pub message: DeepSeekResponseMessage, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DeepSeekResponseMessage { + pub content: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub(crate) enum DeepSeekContent { + Text(String), + Object(DeepSeekOcrResult), +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct DeepSeekOcrResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage_info: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct DeepSeekPage { + #[serde(default)] + pub index: i64, + #[serde(default)] + pub markdown: String, + pub images: Option, + pub dimensions: Option, + #[serde(flatten)] + pub extra_fields: Map, +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/mod.rs new file mode 100644 index 00000000000..8031f2124a3 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/mod.rs @@ -0,0 +1,9 @@ +mod params; +mod transformation; +mod types; + +pub(crate) use params::{decode_input_params, map_ocr_params}; +pub(crate) use transformation::{transform_ocr_request, transform_ocr_response}; +pub(crate) use types::{ + AzureDocumentIntelligenceOperation, DocumentIntelligenceParams, OperationStatus, +}; diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/params.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/params.rs new file mode 100644 index 00000000000..85d1dafa542 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/params.rs @@ -0,0 +1,195 @@ +use std::collections::BTreeSet; + +use serde_json::{Map, Value}; + +use super::types::{ + DocumentIntelligenceInputParams, DocumentIntelligenceParams, FeaturesInput, PagesInput, +}; +use crate::ocr::error::OcrRequestError; +use crate::ocr::prepare::ParsedProviderParams; + +pub(crate) fn decode_input_params( + params: Map, + prefix: &str, +) -> Result, OcrRequestError> { + if let Some(Value::Array(pages)) = params.get("pages") { + if pages.iter().any(Value::is_boolean) { + return Err(OcrRequestError::Pages("boolean page index".into())); + } + if pages + .iter() + .any(|page| page.is_number() && page.as_i64().is_none()) + { + return Err(OcrRequestError::Pages("page index is out of range".into())); + } + if !pages.iter().all(Value::is_i64) && !pages.iter().all(Value::is_string) { + return Err(OcrRequestError::Pages("mixed page element types".into())); + } + } + crate::ocr::wire::decode_request_value(Value::Object(params), prefix) +} + +pub(crate) fn map_ocr_params( + params: DocumentIntelligenceInputParams, +) -> Result { + Ok(DocumentIntelligenceParams { + pages: params.pages.map(normalize_pages).transpose()?.flatten(), + features: params + .features + .map(normalize_features) + .transpose()? + .flatten(), + }) +} + +fn normalize_pages(pages: PagesInput) -> Result, OcrRequestError> { + let normalized = match pages { + PagesInput::ZeroBasedIndices(indices) => { + if indices.is_empty() { + return Ok(None); + } + indices + .into_iter() + .map(|page| { + if page < 0 { + return Err(OcrRequestError::Pages("negative page index".into())); + } + page.checked_add(1) + .ok_or_else(|| OcrRequestError::Pages("page index is out of range".into())) + }) + .collect::, _>>()? + .into_iter() + .map(|page| page.to_string()) + .collect::>() + .join(",") + } + PagesInput::NativeTokens(tokens) => { + if tokens.is_empty() { + return Ok(None); + } + tokens + .iter() + .map(|token| token.trim()) + .collect::>() + .join(",") + } + PagesInput::NativeRange(range) => range + .split(',') + .map(str::trim) + .collect::>() + .join(","), + }; + if !normalized.split(',').all(valid_page_token) { + return Err(OcrRequestError::Pages("invalid native page range".into())); + } + Ok(Some(normalized)) +} + +fn valid_page_token(token: &str) -> bool { + let mut parts = token.split('-'); + let start = parts.next().unwrap_or_default(); + if start.is_empty() || !start.chars().all(|character| character.is_ascii_digit()) { + return false; + } + match parts.next() { + None => true, + Some(end) => { + !end.is_empty() + && end.chars().all(|character| character.is_ascii_digit()) + && parts.next().is_none() + } + } +} + +fn normalize_features(features: FeaturesInput) -> Result, OcrRequestError> { + let tokens = match features { + FeaturesInput::Names(names) => names, + FeaturesInput::CommaSeparated(names) => names.split(',').map(str::to_string).collect(), + }; + if tokens.is_empty() { + return Ok(None); + } + let normalized = tokens.iter().map(|token| token.trim()).collect::>(); + if !normalized.iter().all(|token| { + let Some((first, rest)) = token.as_bytes().split_first() else { + return false; + }; + first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) + }) { + return Err(OcrRequestError::Features); + } + Ok(Some(normalized.join(","))) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + fn map(value: Value) -> Result { + let fields = value.as_object().unwrap().clone(); + map_ocr_params(decode_input_params(fields, "optional_params")?.known) + } + + #[test] + fn input_params_retain_unknown_fields() { + let parsed = decode_input_params( + json!({ + "pages": [0], + "future_ocr_option": true, + "extra_body": {"provider_option": "value"} + }) + .as_object() + .unwrap() + .clone(), + "optional_params", + ) + .unwrap(); + + assert_eq!( + parsed.known.pages, + Some(PagesInput::ZeroBasedIndices(vec![0])) + ); + assert_eq!(parsed.extra_params["future_ocr_option"], true); + assert_eq!( + parsed.extra_params["extra_body"], + json!({"provider_option": "value"}) + ); + assert_eq!( + serde_json::to_value(map_ocr_params(parsed.known).unwrap()).unwrap(), + json!({"pages": "1", "features": null}) + ); + } + + #[rstest] + #[case(json!(["keyValuePairs"]), "keyValuePairs")] + #[case(json!(["keyValuePairs", "languages"]), "keyValuePairs,languages")] + #[case(json!("keyValuePairs"), "keyValuePairs")] + #[case(json!("keyValuePairs,languages"), "keyValuePairs,languages")] + #[case(json!("keyValuePairs, languages"), "keyValuePairs,languages")] + fn feature_mapping_matches_python(#[case] input: Value, #[case] expected: &str) { + assert_eq!( + map(json!({"features": input})).unwrap().features.as_deref(), + Some(expected) + ); + } + + #[rstest] + #[case(json!("keyValuePairs&pages=9"))] + #[case(json!("key value pairs"))] + #[case(json!(""))] + #[case(json!([1, 2]))] + #[case(json!([["keyValuePairs"]]))] + #[case(json!({"feature":"keyValuePairs"}))] + #[case(json!(5))] + fn invalid_feature_mapping_matches_python(#[case] input: Value) { + assert!(map(json!({"features": input})).is_err()); + } + + #[test] + fn empty_feature_list_is_omitted() { + assert_eq!(map(json!({"features": []})).unwrap().features, None); + } +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs new file mode 100644 index 00000000000..2b848fcfb7a --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs @@ -0,0 +1,111 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde_json::{Map, Value, json}; + +use super::types::*; +use crate::constants::{AZURE_DI_DEFAULT_DPI, AZURE_DI_DEFAULT_HEIGHT, AZURE_DI_DEFAULT_WIDTH}; +use crate::ocr::document::InlineDocument; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn transform_ocr_request( + document: OcrDocument, +) -> Result { + let source = document.source(); + if source.is_empty() { + return Err(OcrRequestError::MissingField("document URL")); + } + Ok(if let Some(document) = InlineDocument::parse(source)? { + DocumentIntelligenceRequest::Base64Source( + STANDARD.encode(document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?), + ) + } else { + DocumentIntelligenceRequest::UrlSource(source.to_string()) + }) +} + +pub(crate) fn transform_ocr_response( + model: &str, + response: AzureDocumentIntelligenceOperation, +) -> Result { + if response.status != Some(OperationStatus::Succeeded) { + return Err(OcrResponseError::OperationStatus( + response + .status + .map(|status| status.to_string()) + .unwrap_or_else(|| "None".into()), + )); + } + let result = response.analyze_result.unwrap_or_default(); + let pages = result + .pages + .into_iter() + .map(normalize_page) + .collect::, _>>()?; + let pages_processed = pages.len(); + let mut extra_fields = Map::new(); + extra_fields.insert("content".into(), option_value(result.content)); + extra_fields.insert("tables".into(), option_value(result.tables)); + extra_fields.insert( + "key_value_pairs".into(), + option_value(result.key_value_pairs), + ); + Ok(LiteLLMOcrResponse { + pages, + model: model.into(), + document_annotation: None, + usage_info: Some(json!({"pages_processed":pages_processed})), + object: "ocr".into(), + extra_fields, + provider_native_response: None, + }) +} + +fn normalize_page(page: AzureDocumentIntelligencePage) -> Result { + let index = page + .page_number + .unwrap_or(1) + .checked_sub(1) + .ok_or(OcrResponseError::NumericRange("page.pageNumber"))?; + let scale = if page.unit.as_deref().unwrap_or("inch") == "inch" { + AZURE_DI_DEFAULT_DPI as f64 + } else { + 1.0 + }; + let width = pixel_dimension( + page.width.unwrap_or(AZURE_DI_DEFAULT_WIDTH), + scale, + "page.width", + )?; + let height = pixel_dimension( + page.height.unwrap_or(AZURE_DI_DEFAULT_HEIGHT), + scale, + "page.height", + )?; + let markdown = page + .lines + .iter() + .map(|line| line.content.as_deref().unwrap_or_default()) + .collect::>() + .join("\n"); + Ok(json!({ + "index":index, + "markdown":markdown, + "images":null, + "dimensions":{"width":width,"height":height,"dpi":AZURE_DI_DEFAULT_DPI} + })) +} + +fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result { + let value = value * scale; + if !value.is_finite() || value < i64::MIN as f64 || value > i64::MAX as f64 { + return Err(OcrResponseError::NumericRange(field)); + } + Ok(value.trunc() as i64) +} + +fn option_value(value: Option) -> Value { + value + .and_then(|value| serde_json::to_value(value).ok()) + .unwrap_or(Value::Null) +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/types.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/types.rs new file mode 100644 index 00000000000..793f4547e99 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/types.rs @@ -0,0 +1,138 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PagesInput { + ZeroBasedIndices(Vec), + NativeTokens(Vec), + NativeRange(String), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum FeaturesInput { + Names(Vec), + CommaSeparated(String), +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub(crate) struct DocumentIntelligenceInputParams { + pub pages: Option, + pub features: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub(crate) struct DocumentIntelligenceParams { + pub pages: Option, + pub features: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) enum DocumentIntelligenceRequest { + #[serde(rename = "urlSource")] + UrlSource(String), + #[serde(rename = "base64Source")] + Base64Source(String), +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum OperationStatus { + Succeeded, + Running, + NotStarted, + Failed, + Unknown(String), +} + +impl<'de> Deserialize<'de> for OperationStatus { + fn deserialize>(deserializer: D) -> Result { + Ok(match String::deserialize(deserializer)?.as_str() { + "succeeded" => Self::Succeeded, + "running" => Self::Running, + "notStarted" => Self::NotStarted, + "failed" => Self::Failed, + value => Self::Unknown(value.to_string()), + }) + } +} + +impl std::fmt::Display for OperationStatus { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::Succeeded => "succeeded", + Self::Running => "running", + Self::NotStarted => "notStarted", + Self::Failed => "failed", + Self::Unknown(value) => value, + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct AzureDocumentIntelligenceOperation { + pub status: Option, + #[serde(rename = "analyzeResult")] + pub analyze_result: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub(crate) struct AzureDocumentIntelligenceAnalyzeResult { + pub content: Option, + #[serde(default)] + pub pages: Vec, + pub tables: Option>>, + #[serde(rename = "keyValuePairs")] + pub key_value_pairs: Option>>, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct AzureDocumentIntelligencePage { + #[serde(rename = "pageNumber", default, deserialize_with = "optional_i64")] + pub page_number: Option, + #[serde(default, deserialize_with = "optional_f64")] + pub width: Option, + #[serde(default, deserialize_with = "optional_f64")] + pub height: Option, + pub unit: Option, + #[serde(default)] + pub lines: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct AzureDocumentIntelligenceLine { + pub content: Option, +} + +fn optional_i64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(number)) => number + .as_i64() + .map(Some) + .ok_or_else(|| serde::de::Error::custom("expected an integer")), + Some(Value::String(value)) => value + .parse::() + .map(Some) + .map_err(|_| serde::de::Error::custom("expected an integer")), + Some(_) => Err(serde::de::Error::custom("expected an integer")), + } +} + +fn optional_f64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(number)) => number + .as_f64() + .filter(|value| value.is_finite()) + .map(Some) + .ok_or_else(|| serde::de::Error::custom("expected a finite number")), + Some(Value::String(value)) => value + .parse::() + .ok() + .filter(|value| value.is_finite()) + .map(Some) + .ok_or_else(|| serde::de::Error::custom("expected a finite number")), + Some(_) => Err(serde::de::Error::custom("expected a number")), + } +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs index cd0a1dc6b17..5bd7e555a1e 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs @@ -36,6 +36,101 @@ mod tests { use rstest::rstest; use serde_json::{Value, json}; + fn mapped_params(value: Value) -> Value { + serde_json::to_value(serde_json::from_value::(value).unwrap()).unwrap() + } + + fn document() -> OcrDocument { + serde_json::from_value( + json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + ) + .unwrap() + } + + #[rstest] + fn extract_header_is_a_supported_ocr_param() { + assert_eq!( + mapped_params(json!({"extract_header":true}))["extract_header"], + true + ); + } + + #[rstest] + fn extract_footer_is_a_supported_ocr_param() { + assert_eq!( + mapped_params(json!({"extract_footer":false}))["extract_footer"], + false + ); + } + + #[rstest] + fn existing_ocr_params_remain_supported() { + let mapped = mapped_params(json!({ + "pages":[0,2], + "include_image_base64":true, + "image_limit":2, + "image_min_size":100, + "bbox_annotation_format":{"type":"json_schema"}, + "document_annotation_format":{"type":"json_schema"} + })); + assert_eq!(mapped["pages"], json!([0, 2])); + assert_eq!(mapped["include_image_base64"], true); + assert_eq!(mapped["image_limit"], 2); + assert_eq!(mapped["image_min_size"], 100); + assert_eq!(mapped["bbox_annotation_format"]["type"], "json_schema"); + assert_eq!(mapped["document_annotation_format"]["type"], "json_schema"); + } + + #[rstest] + fn map_ocr_params_forwards_extract_header() { + assert_eq!( + mapped_params(json!({"extract_header":true}))["extract_header"], + true + ); + } + + #[rstest] + fn map_ocr_params_forwards_extract_footer() { + assert_eq!( + mapped_params(json!({"extract_footer":true}))["extract_footer"], + true + ); + } + + #[rstest] + fn map_ocr_params_forwards_extract_header_and_footer() { + let mapped = mapped_params(json!({"extract_header":true,"extract_footer":false})); + assert_eq!(mapped["extract_header"], true); + assert_eq!(mapped["extract_footer"], false); + } + + #[rstest] + fn map_ocr_params_drops_unknown_params() { + let mapped = mapped_params(json!({"extract_header":true,"unsupported_param":"value"})); + assert_eq!(mapped["extract_header"], true); + assert!(mapped.get("unsupported_param").is_none()); + } + + #[rstest] + #[case("table_format", json!("html"))] + #[case("confidence_scores_granularity", json!("word"))] + #[case("document_annotation_prompt", json!("extract"))] + #[case("include_blocks", json!(true))] + #[case("id", json!("req-123"))] + fn new_ocr_params_are_supported(#[case] name: &str, #[case] value: Value) { + assert_eq!(mapped_params(json!({name:value.clone()}))[name], value); + } + + #[rstest] + #[case("table_format", json!("html"))] + #[case("confidence_scores_granularity", json!("word"))] + #[case("document_annotation_prompt", json!("extract"))] + #[case("include_blocks", json!(true))] + #[case("id", json!("req-123"))] + fn map_ocr_params_forwards_new_ocr_params(#[case] name: &str, #[case] value: Value) { + assert_eq!(mapped_params(json!({name:value.clone()}))[name], value); + } + #[rstest] #[case("pages", json!([0, 2]))] #[case("include_image_base64", json!(true))] @@ -53,50 +148,81 @@ mod tests { fn request_mapping_matches_python(#[case] name: &str, #[case] value: Value) { let params: MistralOcrParams = serde_json::from_value(json!({name: value.clone()})).unwrap(); - let document: OcrDocument = serde_json::from_value( - json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), - ) - .unwrap(); let result = - serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) + serde_json::to_value(transform_ocr_request("model", document(), ¶ms).unwrap()) .unwrap(); assert_eq!(result["model"], "model"); assert_eq!(result[name], value); } - #[test] - fn request_mapping_filters_unknown_fields() { - let params: MistralOcrParams = serde_json::from_value(json!({"unknown": true})).unwrap(); - let document: OcrDocument = serde_json::from_value( - json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + #[rstest] + #[case("table_format", json!("html"))] + #[case("confidence_scores_granularity", json!("word"))] + #[case("document_annotation_prompt", json!("extract"))] + #[case("id", json!("req-123"))] + #[case("extract_header", json!(true))] + #[case("include_blocks", json!(true))] + #[case("pages", json!([0,1]))] + fn transform_ocr_request_includes_each_optional_param( + #[case] name: &str, + #[case] value: Value, + ) { + let params: MistralOcrParams = serde_json::from_value(json!({name:value.clone()})).unwrap(); + let result = serde_json::to_value( + transform_ocr_request("mistral-ocr-latest", document(), ¶ms).unwrap(), ) .unwrap(); - let result = - serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) - .unwrap(); - assert!(result.get("unknown").is_none()); + assert_eq!(result[name], value); + assert_eq!(result["model"], "mistral-ocr-latest"); } - #[test] - fn response_preserves_provider_fields() { + #[rstest] + fn transform_ocr_request_includes_multiple_new_params() { + let params: MistralOcrParams = serde_json::from_value(json!({ + "table_format":"html", + "confidence_scores_granularity":"page", + "extract_header":true + })) + .unwrap(); + let result = serde_json::to_value( + transform_ocr_request("mistral-ocr-latest", document(), ¶ms).unwrap(), + ) + .unwrap(); + assert_eq!(result["table_format"], "html"); + assert_eq!(result["confidence_scores_granularity"], "page"); + assert_eq!(result["extract_header"], true); + } + + #[rstest] + fn transform_ocr_response_preserves_blocks_and_confidence_scores() { let response: MistralOcrResponse = serde_json::from_value(json!({ - "pages":[{"index":0,"markdown":"hello","header":"head","confidence_scores":{"mean":0.99}}], + "pages":[{"index":0,"markdown":"hello","blocks":[{"type":"title"}],"confidence_scores":{"mean":0.99}}], "model":"returned-model", - "usage_info":{"pages_processed":1,"future_counter":5}, - "future_response_field":"kept" + "usage_info":{"pages_processed":1} })) .unwrap(); let result = transform_ocr_response("model", response) .unwrap() .into_json(); - assert_eq!(result["pages"][0]["header"], "head"); - assert_eq!(result["usage_info"]["future_counter"], 5); - assert_eq!(result["future_response_field"], "kept"); - assert_eq!(result["model"], "returned-model"); + assert_eq!(result["pages"][0]["blocks"][0]["type"], "title"); + assert_eq!(result["pages"][0]["confidence_scores"]["mean"], 0.99); } - #[test] - fn response_rejects_null_pages() { - assert!(serde_json::from_value::(json!({"pages":null})).is_err()); + #[rstest] + fn transform_ocr_response_preserves_ocr4_page_fields() { + let page = json!({ + "index":0, + "markdown":"table page", + "tables":[{"rows":2,"cols":3}], + "hyperlinks":["https://example.com"], + "header":"header", + "footer":"footer" + }); + let response: MistralOcrResponse = + serde_json::from_value(json!({"pages":[page.clone()]})).unwrap(); + let result = transform_ocr_response("model", response) + .unwrap() + .into_json(); + assert_eq!(result["pages"][0], page); } } diff --git a/litellm-rust/crates/core/src/ocr/codecs/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/mod.rs index 170ef5f68a7..7c752749901 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/mod.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/mod.rs @@ -1 +1,4 @@ +pub(crate) mod deepseek; +pub(crate) mod document_intelligence; pub(crate) mod mistral; +pub(crate) mod reducto; diff --git a/litellm-rust/crates/core/src/ocr/codecs/reducto/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/reducto/mod.rs new file mode 100644 index 00000000000..3fff40451c6 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/reducto/mod.rs @@ -0,0 +1,9 @@ +mod transformation; +mod types; + +pub(crate) use transformation::{ + transform_legacy_ocr_request, transform_ocr_response, transform_v3_ocr_request, +}; +pub(crate) use types::{ + ReductoLegacyParams, ReductoResponse, ReductoUploadResponse, ReductoV3Params, +}; diff --git a/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs new file mode 100644 index 00000000000..7073643f6b6 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs @@ -0,0 +1,115 @@ +use std::collections::BTreeMap; + +use serde_json::{Value, json}; + +use super::types::*; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; + +#[tracing::instrument( + name = "transform_ocr_request", + target = "litellm::function_trace", + level = "trace", + skip_all +)] +pub(crate) fn transform_v3_ocr_request( + _model: &str, + document: OcrDocument, + params: &ReductoV3Params, +) -> Result { + Ok(ReductoV3Request { + input: document.source().to_string(), + params: params.clone(), + }) +} + +#[tracing::instrument( + name = "transform_ocr_request", + target = "litellm::function_trace", + level = "trace", + skip_all +)] +pub(crate) fn transform_legacy_ocr_request( + _model: &str, + document: OcrDocument, + params: &ReductoLegacyParams, +) -> Result { + Ok(ReductoLegacyRequest { + document_url: document.source().to_string(), + options: params.enhance.as_ref().map(|_| params.clone()), + }) +} + +pub(crate) fn transform_ocr_response( + model: &str, + response: ReductoResponse, +) -> Result { + let result = match response.result { + Some(result) => result.unwrap_or_default(), + None => ReductoResult { + chunks: response.chunks, + }, + }; + let usage = response.usage.unwrap_or_default(); + Ok(LiteLLMOcrResponse { + pages: build_pages(result.chunks.unwrap_or_default()), + model: model.to_string(), + document_annotation: None, + usage_info: Some(json!({ + "pages_processed": usage.num_pages, + "credits": usage.credits, + })), + object: "ocr".to_string(), + extra_fields: serde_json::Map::new(), + provider_native_response: None, + }) +} + +fn build_pages(chunks: Vec) -> Vec { + let blocks_by_page = chunks + .iter() + .flat_map(|chunk| chunk.blocks.iter().flatten()) + .filter_map(|block| block.bbox.as_ref()?.page.map(|page| (page, block))) + .fold( + BTreeMap::>::new(), + |mut pages, (page, block)| { + pages.entry(page).or_default().push(block); + pages + }, + ); + if blocks_by_page.is_empty() { + let markdown = join_content(chunks.iter().map(|chunk| chunk.content.as_deref())); + return if markdown.is_empty() { + Vec::new() + } else { + vec![page(0, markdown, None)] + }; + } + blocks_by_page + .into_iter() + .map(|(index, blocks)| { + let markdown = join_content(blocks.iter().map(|block| block.content.as_deref())); + page( + index.saturating_sub(1).max(0), + markdown, + Some(json!(blocks)), + ) + }) + .collect() +} + +fn join_content<'a>(content: impl Iterator>) -> String { + content + .flatten() + .filter(|text| !text.is_empty()) + .collect::>() + .join("\n\n") +} + +fn page(index: i64, markdown: String, blocks: Option) -> Value { + let mut result = json!({"index":index,"markdown":markdown,"images":null}); + if let (Value::Object(fields), Some(blocks)) = (&mut result, blocks) { + fields.insert("blocks".into(), blocks); + } + result +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/reducto/types.rs b/litellm-rust/crates/core/src/ocr/codecs/reducto/types.rs new file mode 100644 index 00000000000..c03720cc8ae --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/reducto/types.rs @@ -0,0 +1,128 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct ReductoV3Params { + #[serde(skip_serializing_if = "Option::is_none")] + pub formatting: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retrieval: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub settings: Option>, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct ReductoLegacyParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub enhance: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ReductoV3Request { + pub input: String, + #[serde(flatten)] + pub params: ReductoV3Params, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ReductoLegacyRequest { + pub document_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub options: Option, +} + +#[derive(Deserialize)] +pub(crate) struct ReductoUploadResponse { + pub file_id: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ReductoResponse { + #[serde(default, deserialize_with = "present_nullable")] + pub result: Option>, + pub usage: Option, + #[serde(default)] + pub chunks: Option>, +} + +fn present_nullable<'de, D: Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Option::::deserialize(deserializer).map(Some) +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub(crate) struct ReductoResult { + pub chunks: Option>, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub(crate) struct ReductoUsage { + #[serde(default, deserialize_with = "optional_i64")] + pub num_pages: Option, + #[serde(default, deserialize_with = "optional_f64")] + pub credits: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ReductoChunk { + pub content: Option, + pub blocks: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ReductoBlock { + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ReductoBoundingBox { + #[serde(default, deserialize_with = "optional_i64")] + pub page: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +fn optional_i64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(number)) => number + .as_i64() + .or_else(|| number.as_f64().and_then(checked_truncated_i64)) + .map(Some) + .ok_or_else(|| serde::de::Error::custom("expected an integer")), + Some(Value::String(value)) => value + .trim() + .parse::() + .map(Some) + .map_err(|_| serde::de::Error::custom("expected an integer")), + Some(Value::Bool(value)) => Ok(Some(i64::from(value))), + Some(_) => Ok(None), + } +} + +fn optional_f64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(number)) => number + .as_f64() + .map(Some) + .ok_or_else(|| serde::de::Error::custom("expected a number")), + Some(Value::String(value)) => value + .trim() + .parse::() + .map(Some) + .map_err(|_| serde::de::Error::custom("expected a number")), + Some(_) => Ok(None), + } +} + +fn checked_truncated_i64(value: f64) -> Option { + (value.is_finite() && value >= i64::MIN as f64 && value <= i64::MAX as f64) + .then(|| value.trunc() as i64) +} diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs index 11a6e612a3c..e89b1c5c569 100644 --- a/litellm-rust/crates/core/src/ocr/document.rs +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -1,5 +1,4 @@ use base64::{Engine, engine::general_purpose::STANDARD}; -#[cfg(test)] use data_url::mime::Mime; use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError}; use reqwest::Url; @@ -21,7 +20,6 @@ impl<'a> InlineDocument<'a> { } } - #[cfg(test)] pub(crate) fn mime_type(&self) -> &Mime { self.0.mime_type() } diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs index 395bb60000c..522d059ec48 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -12,6 +12,8 @@ pub enum OcrRequestError { MissingField(&'static str), #[error("invalid OCR document data URI")] InvalidDataUri, + #[error("Reducto requires a reducto:// id or a data URI")] + ReductoSource, #[error("inline OCR document exceeds the size limit")] InlineDocumentTooLarge, #[error("OCR document URL is blocked by network policy")] @@ -22,16 +24,38 @@ pub enum OcrRequestError { DownloadTooLarge, #[error("OCR document download exceeded the redirect limit")] TooManyRedirects, + #[error("invalid OCR pages: {0}")] + Pages(String), + #[error("invalid OCR features")] + Features, + #[error("OCR model cannot be a dot segment")] + DotModel, } #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum OcrResponseError { #[error("invalid OCR response field: {path}")] ResponseField { path: String }, + #[error("OCR response is missing non-empty content")] + EmptyContent, #[error("OCR document redirect is missing a location")] MissingRedirectLocation, #[error("OCR document redirect location is invalid")] InvalidRedirect, + #[error("OCR operation ended with status {0}")] + OperationStatus(String), + #[error("OCR response numeric value is out of range: {0}")] + NumericRange(&'static str), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrPollingError { + #[error("OCR accepted response is missing a valid operation-location")] + PollLocation, + #[error("OCR operation-location must use the submission origin without credentials")] + PollOrigin, + #[error("OCR polling timed out")] + PollTimeout, } #[derive(Debug, Error)] @@ -43,6 +67,8 @@ pub enum OcrError { #[error("{0}")] Transport(#[from] TransportError), #[error("{0}")] + Polling(#[from] OcrPollingError), + #[error("{0}")] Public(#[from] crate::Error), } @@ -52,6 +78,7 @@ impl From for crate::Error { OcrError::Request(error) => error.into(), OcrError::Response(error) => error.into(), OcrError::Transport(error) => error.into(), + OcrError::Polling(error) => crate::Error::InvalidResponse(error.to_string()), OcrError::Public(error) => error, } } diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index aa11d0ab3cf..1e975c3f521 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -7,7 +7,6 @@ mod handler; pub mod hooks; mod prepare; mod registry; -pub mod transformation; pub mod types; pub mod wire; @@ -18,8 +17,23 @@ pub use types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrDocumen #[path = "../../tests/azure_ai_ocr.rs"] mod azure_ai_tests; #[cfg(test)] +#[path = "../../tests/azure_document_intelligence_ocr.rs"] +mod azure_document_intelligence_tests; +#[cfg(test)] +#[path = "../../tests/deepseek_ocr.rs"] +mod deepseek_tests; +#[cfg(test)] +#[path = "../../tests/reducto_ocr.rs"] +mod reducto_tests; +#[cfg(test)] #[path = "../../tests/ocr/support.rs"] pub(crate) mod test_support; #[cfg(test)] #[path = "../../tests/ocr.rs"] pub(crate) mod tests; +#[cfg(test)] +#[path = "../../tests/vertex_ai_deepseek_ocr.rs"] +mod vertex_ai_deepseek_tests; +#[cfg(test)] +#[path = "../../tests/vertex_ai_ocr.rs"] +mod vertex_ai_tests; diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index cff40b7de50..bf6f924088c 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value}; use super::OcrClient; use super::error::{OcrError, OcrRequestError}; use super::hooks::OcrDuringCallRequest; -use super::types::LiteLLMOcrRequest; +use super::types::{LiteLLMOcrRequest, OcrDocument}; #[derive(Debug, Deserialize)] pub(crate) struct ParsedProviderParams { @@ -24,6 +24,39 @@ pub(crate) fn _prepare_ocr_request( ) } +pub(crate) fn merge_extra_params( + body: &B, + extra_params: Map, +) -> Result { + let Value::Object(fields) = + serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField { + path: "body".into(), + })? + else { + return Err(OcrRequestError::RequestField { + path: "body".into(), + }); + }; + let extra_body = extra_params + .get("extra_body") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default() + .into_iter() + .collect::>(); + Ok(Value::Object( + fields + .into_iter() + .chain( + extra_params + .into_iter() + .filter(|(name, _)| name != "extra_body"), + ) + .chain(extra_body) + .collect(), + )) +} + pub(crate) async fn transform_request_body( client: &OcrClient, request: &LiteLLMOcrRequest, @@ -77,6 +110,29 @@ pub(crate) fn build_http_request( .map_err(OcrError::from) } +pub(crate) async fn guardrail_document( + request: &LiteLLMOcrRequest, + url: &str, +) -> Result { + if !request.hooks.has_guardrails() { + return Ok(request.document.clone()); + } + let changed = request + .hooks + .during_call(OcrDuringCallRequest { + model: request.model.clone(), + custom_llm_provider: request.adapter.provider().as_str().into(), + url: url.into(), + body: serde_json::to_value(&request.document).map_err(|_| { + OcrRequestError::RequestField { + path: "document".into(), + } + })?, + }) + .await?; + super::wire::decode_request_value(changed.body, "guardrail.document").map_err(OcrError::from) +} + #[derive(Serialize)] struct OcrWireBody { #[serde(flatten)] @@ -107,7 +163,6 @@ impl OcrWireBody { pub(crate) fn credential_env(name: &str) -> Option { std::env::var(name).ok() } - #[cfg(test)] mod tests { use serde_json::json; diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index e70bd7314c0..1b20a91143b 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -25,6 +25,8 @@ super::adapters::for_each_ocr_adapter!(define_adapter_types); pub(crate) enum OcrProvider { Mistral, AzureAi, + Reducto, + VertexAi, } impl OcrProvider { @@ -32,6 +34,8 @@ impl OcrProvider { match self { Self::Mistral => "mistral", Self::AzureAi => "azure_ai", + Self::Reducto => "reducto", + Self::VertexAi => "vertex_ai", } } } @@ -48,18 +52,77 @@ pub(crate) fn resolve_wire_adapter( let typed_provider = match provider.custom_llm_provider { "mistral" => OcrProvider::Mistral, "azure_ai" => OcrProvider::AzureAi, + "reducto" => OcrProvider::Reducto, + "vertex_ai" => OcrProvider::VertexAi, value => return Err(Error::InvalidProvider(value.to_string())), }; - match typed_provider { - OcrProvider::Mistral => Ok((provider.model.to_string(), OcrAdapterKind::Mistral)), + let adapter = match typed_provider { + OcrProvider::Mistral => OcrAdapterKind::Mistral, OcrProvider::AzureAi if is_document_intelligence_model(provider.model) => { - Err(Error::InvalidProvider("azure_ai".into())) + OcrAdapterKind::AzureDocumentIntelligence } - OcrProvider::AzureAi => Ok((provider.model.to_string(), OcrAdapterKind::AzureMistral)), - } + OcrProvider::AzureAi => OcrAdapterKind::AzureMistral, + OcrProvider::Reducto if provider.model.eq_ignore_ascii_case("parse-legacy") => { + OcrAdapterKind::ReductoLegacy + } + OcrProvider::Reducto if provider.model.eq_ignore_ascii_case("parse-v3") => { + OcrAdapterKind::ReductoV3 + } + OcrProvider::Reducto => { + return Err(Error::InvalidRequest(format!( + "unsupported Reducto OCR model: {}", + provider.model + ))); + } + OcrProvider::VertexAi if provider.model.to_ascii_lowercase().contains("deepseek") => { + OcrAdapterKind::VertexDeepSeek + } + OcrProvider::VertexAi => OcrAdapterKind::VertexMistral, + }; + Ok((provider.model.to_string(), adapter)) } fn is_document_intelligence_model(model: &str) -> bool { let model = model.to_ascii_lowercase(); model.contains("doc-intelligence") || model.contains("documentintelligence") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_models_are_preserved_without_a_local_allowlist() { + let cases = [ + ("mistral/future-ocr-model", OcrAdapterKind::Mistral), + ("azure_ai/future-ocr-model", OcrAdapterKind::AzureMistral), + ]; + + for (qualified_model, expected_adapter) in cases { + let expected_model = qualified_model.split_once('/').unwrap().1; + let (model, adapter) = resolve_wire_adapter(qualified_model, None).unwrap(); + assert_eq!(model, expected_model); + assert_eq!(adapter, expected_adapter); + } + } + + #[test] + fn unknown_reducto_models_are_rejected() { + assert!(matches!( + resolve_wire_adapter("reducto/future-parse-model", None), + Err(Error::InvalidRequest(_)) + )); + } + + #[test] + fn known_protocol_models_still_select_specialized_adapters() { + let (model, adapter) = resolve_wire_adapter("reducto/parse-legacy", None).unwrap(); + assert_eq!(model, "parse-legacy"); + assert_eq!(adapter, OcrAdapterKind::ReductoLegacy); + + let (model, adapter) = + resolve_wire_adapter("azure_ai/doc-intelligence/prebuilt-layout", None).unwrap(); + assert_eq!(model, "doc-intelligence/prebuilt-layout"); + assert_eq!(adapter, OcrAdapterKind::AzureDocumentIntelligence); + } +} diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs deleted file mode 100644 index ac4f10bf15b..00000000000 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ /dev/null @@ -1,107 +0,0 @@ -use crate::Error; -use serde_json::{Map, Value}; - -use super::types::{LiteLLMOcrResponse, OcrRequestData}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum OcrAuthStrategy { - Bearer, - Header(&'static str), -} - -impl OcrAuthStrategy { - pub fn header_name(self) -> &'static str { - match self { - Self::Bearer => "authorization", - Self::Header(header_name) => header_name, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum OcrResponseHandling { - Json, - AzureDocumentIntelligencePoll, -} - -pub trait OcrProviderConfig: Sync { - fn supported_ocr_params(&self) -> &'static [&'static str]; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params(&self, non_default_params: &Map) -> Map { - let mut mapped_params = Map::new(); - for (param, value) in non_default_params { - if self.supported_ocr_params().contains(¶m.as_str()) { - mapped_params.insert(param.clone(), value.clone()); - } - } - mapped_params - } - - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result; - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result; - - fn transform_ocr_response_with_params( - &self, - model: &str, - response_json: Value, - _optional_params: &Map, - ) -> Result { - self.transform_ocr_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn validate_environment( - &self, - headers: Vec<(String, String)>, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result, Error> { - let strategy = self.auth_strategy(); - if crate::http_utils::has_header(&headers, strategy.header_name()) { - return Ok(headers); - } - let api_key = self.resolve_api_key(api_key, env_lookup)?; - let auth_header = match strategy { - OcrAuthStrategy::Bearer => ("Authorization".to_string(), format!("Bearer {api_key}")), - OcrAuthStrategy::Header(name) => (name.to_string(), api_key), - }; - Ok(std::iter::once(auth_header).chain(headers).collect()) - } - - fn auth_strategy(&self) -> OcrAuthStrategy { - OcrAuthStrategy::Bearer - } - - fn requires_data_uri_document(&self) -> bool { - false - } - - fn response_handling(&self) -> OcrResponseHandling { - OcrResponseHandling::Json - } -} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index eeda94738a0..06519f86c91 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -7,14 +8,9 @@ use serde_json::{Map, Value}; use super::hooks::{NoopOcrHooks, OcrHooks}; use super::registry::{OcrAdapterKind, resolve_wire_adapter}; use crate::Error; +use crate::auth::InputSource; use crate::constants::OCR_HTTP_TIMEOUT_SECS; -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct OcrRequestData { - pub data: Value, - pub files: Option, -} - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] pub enum OcrDocument { @@ -65,20 +61,28 @@ pub enum OcrResponseFormat { #[derive(Clone)] pub struct OcrConnection { pub api_key: Option, + pub api_key_source: InputSource, pub api_base: Option, + pub api_base_source: InputSource, pub extra_headers: Vec<(String, String)>, + pub extra_headers_source: InputSource, pub timeout: Duration, pub max_download_bytes: u64, + pub poll_timeout: Duration, } impl Default for OcrConnection { fn default() -> Self { Self { api_key: None, + api_key_source: InputSource::Deployment, api_base: None, + api_base_source: InputSource::Deployment, extra_headers: Vec::new(), + extra_headers_source: InputSource::Deployment, timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), max_download_bytes: crate::constants::OCR_DOWNLOAD_MAX_BYTES, + poll_timeout: Duration::from_secs(crate::constants::OCR_POLL_TIMEOUT_SECS), } } } @@ -90,6 +94,7 @@ pub struct LiteLLMOcrRequest { pub hooks: Arc, pub litellm_call_id: Option, pub optional_params: Map, + pub input_sources: BTreeMap, pub(crate) adapter: OcrAdapterKind, } @@ -109,6 +114,7 @@ impl LiteLLMOcrRequest { hooks: Arc::new(NoopOcrHooks), litellm_call_id: None, optional_params, + input_sources: BTreeMap::new(), adapter: adapter_kind, }) } diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index db8d91905c4..34d0a7d7b86 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -1,10 +1,12 @@ use crate::ocr::error::OcrRequestError; use crate::ocr::error::OcrResponseError; +use std::collections::BTreeMap; use std::time::Duration; use super::hooks::{OcrDuringCallRequest, OcrPreCallRequest}; use super::types::{LiteLLMOcrRequest, OcrConnection, OcrDocument}; use crate::Error; +use crate::auth::InputSource; use serde::{ Deserialize, de::{DeserializeOwned, IntoDeserializer}, @@ -28,6 +30,8 @@ pub struct OcrWireRequest { pub extra_headers: Option>, #[serde(default)] pub optional_params: Map, + #[serde(default)] + pub input_sources: BTreeMap, pub timeout_seconds: Option, } @@ -36,6 +40,9 @@ pub fn is_supported_request(model: &str, custom_llm_provider: Option<&str>) -> b } pub fn decode_request(wire: OcrWireRequest) -> Result { + let api_key_source = source_for(&wire.input_sources, "api_key"); + let api_base_source = source_for(&wire.input_sources, "api_base"); + let extra_headers_source = source_for(&wire.input_sources, "extra_headers"); let document = decode_request_value(wire.document, "document")?; let headers = wire .extra_headers @@ -67,17 +74,26 @@ pub fn decode_request(wire: OcrWireRequest) -> Result )?; let connection = OcrConnection { api_key: nonblank(wire.api_key), + api_key_source, api_base: nonblank(wire.api_base), + api_base_source, extra_headers: headers, + extra_headers_source, timeout: timeout.unwrap_or(defaults.timeout), max_download_bytes: defaults.max_download_bytes, + poll_timeout: defaults.poll_timeout, }; Ok(LiteLLMOcrRequest { connection, + input_sources: wire.input_sources, ..request }) } +fn source_for(sources: &BTreeMap, name: &str) -> InputSource { + sources.get(name).copied().unwrap_or_default() +} + fn nonblank(value: Option) -> Option { value .map(|s| s.trim().to_string()) diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index f31b961e78a..3ed00b7cc5f 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,3 +1,4 @@ +use crate::auth::error::MissingCredential; use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; @@ -21,13 +22,7 @@ pub fn resolve_anthropic_api_key( non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| { - Error::Auth( - "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \ - environment variable" - .to_string(), - ) - }) + .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AnthropicApiKey))) } pub fn complete_anthropic_url( diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs new file mode 100644 index 00000000000..297e4cc6502 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs @@ -0,0 +1,43 @@ +use std::future::Future; +use std::sync::Arc; + +use azure_core::credentials::TokenCredential; +use moka::future::Cache; + +use crate::AuthError; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct AzureCredentialProviderCacheKey { + pub(crate) mechanism: &'static str, + pub(crate) authority: String, + pub(crate) tenant_id: String, + pub(crate) client_id: String, + pub(crate) scope: String, + pub(crate) secret_identity: String, +} + +pub(crate) struct AzureCredentialProviderCache { + entries: Cache>, +} + +impl AzureCredentialProviderCache { + pub(crate) fn new(capacity: u64) -> Self { + Self { + entries: Cache::builder().max_capacity(capacity).build(), + } + } + + pub(crate) async fn get_or_create( + &self, + key: AzureCredentialProviderCacheKey, + create: F, + ) -> Result, AuthError> + where + F: Future, AuthError>>, + { + self.entries + .try_get_with(key, create) + .await + .map_err(|error| (*error).clone()) + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs new file mode 100644 index 00000000000..33d007c1945 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs @@ -0,0 +1,7 @@ +mod credential_provider_cache; +mod native; +mod resolve; +mod types; + +pub(crate) use resolve::AzureAuthService; +pub(crate) use types::AzureAuthInputs; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs new file mode 100644 index 00000000000..b8f19818d16 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs @@ -0,0 +1,702 @@ +use crate::auth::error::AuthConfigurationError; +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use azure_core::cloud::{CloudConfiguration, CustomConfiguration}; +use azure_core::credentials::{Secret, TokenCredential}; +use azure_core::http::ClientOptions; +use azure_identity::{ + ClientAssertion, ClientAssertionCredential, ClientAssertionCredentialOptions, + ClientSecretCredential, ClientSecretCredentialOptions, DeveloperToolsCredential, + ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, + WorkloadIdentityCredential, WorkloadIdentityCredentialOptions, +}; +use sha2::{Digest, Sha256}; + +use crate::AuthError; +use crate::auth::{InputSource, ResolvedCredential, SecretValue, Sourced}; + +use super::credential_provider_cache::{ + AzureCredentialProviderCache, AzureCredentialProviderCacheKey, +}; + +#[derive(Clone, Debug)] +pub(crate) enum NativeAzureRequest { + ClientSecret { + tenant_id: Sourced, + client_id: Sourced, + client_secret: Sourced, + scope: Sourced, + authority: Option>, + }, + ClientAssertion { + tenant_id: Sourced, + client_id: Sourced, + assertion: Sourced, + assertion_identity: String, + scope: Sourced, + authority: Option>, + }, + WorkloadIdentity { + tenant_id: Sourced, + client_id: Sourced, + token_file_path: Sourced, + scope: Sourced, + authority: Option>, + }, + ManagedIdentity { + client_id: Option>, + scope: Sourced, + selection_source: InputSource, + }, + DeveloperTools { + scope: Sourced, + selection_source: InputSource, + }, +} + +#[derive(Clone, Debug)] +pub(crate) struct ValidatedAzureRequest { + request: NativeAzureRequest, + credential_source: InputSource, +} + +impl ValidatedAzureRequest { + pub(crate) fn new(request: NativeAzureRequest) -> Result { + validate_authority(&request)?; + let credential_source = validate_sources(&request)?; + Ok(Self { + request, + credential_source, + }) + } + + pub(crate) fn credential_source(&self) -> InputSource { + self.credential_source + } + + #[cfg(test)] + pub(super) fn kind(&self) -> &'static str { + match self.request { + NativeAzureRequest::ClientSecret { .. } => "client-secret", + NativeAzureRequest::ClientAssertion { .. } => "client-assertion", + NativeAzureRequest::WorkloadIdentity { .. } => "workload-identity", + NativeAzureRequest::ManagedIdentity { .. } => "managed-identity", + NativeAzureRequest::DeveloperTools { .. } => "developer-tools", + } + } +} + +pub(crate) struct NativeAzureTokenAcquirer { + cache: AzureCredentialProviderCache, + transport: Option, +} + +impl Default for NativeAzureTokenAcquirer { + fn default() -> Self { + Self::new(64) + } +} + +impl NativeAzureTokenAcquirer { + pub(crate) fn new(cache_capacity: u64) -> Self { + Self { + cache: AzureCredentialProviderCache::new(cache_capacity), + transport: None, + } + } + + #[cfg(test)] + pub(super) fn with_transport( + cache_capacity: u64, + transport: azure_core::http::Transport, + ) -> Self { + Self { + cache: AzureCredentialProviderCache::new(cache_capacity), + transport: Some(transport), + } + } + + pub(crate) async fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> Result { + let scope = request.request.scope().to_string(); + let key = request.request.cache_key(); + let transport = self.transport.clone(); + let credential = self + .cache + .get_or_create( + key, + async move { build_credential(request.request, transport) }, + ) + .await?; + let token = credential + .get_token(&[scope.as_str()], None) + .await + .map_err(|error| AuthError::AzureTokenAcquisition(error.to_string()))?; + let expires_on = u64::try_from(token.expires_on.unix_timestamp()) + .ok() + .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)); + + Ok(ResolvedCredential::AccessToken { + token: SecretValue::new(token.token.secret()), + expires_on, + }) + } +} + +impl NativeAzureRequest { + fn scope(&self) -> &str { + match self { + Self::ClientSecret { scope, .. } + | Self::ClientAssertion { scope, .. } + | Self::WorkloadIdentity { scope, .. } + | Self::ManagedIdentity { scope, .. } + | Self::DeveloperTools { scope, .. } => scope.value(), + } + } + + fn cache_key(&self) -> AzureCredentialProviderCacheKey { + match self { + Self::ClientSecret { + tenant_id, + client_id, + client_secret, + scope, + authority, + } => AzureCredentialProviderCacheKey { + mechanism: "client-secret", + authority: authority + .as_ref() + .map(|value| value.value().clone()) + .unwrap_or_default(), + tenant_id: tenant_id.value().clone(), + client_id: client_id.value().clone(), + scope: scope.value().clone(), + secret_identity: secret_digest(client_secret.value().expose()), + }, + Self::ClientAssertion { + tenant_id, + client_id, + assertion, + assertion_identity, + scope, + authority, + } => AzureCredentialProviderCacheKey { + mechanism: "client-assertion", + authority: authority + .as_ref() + .map(|value| value.value().clone()) + .unwrap_or_default(), + tenant_id: tenant_id.value().clone(), + client_id: client_id.value().clone(), + scope: scope.value().clone(), + secret_identity: format!( + "{assertion_identity}:{}", + secret_digest(assertion.value().expose()) + ), + }, + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + scope, + authority, + } => AzureCredentialProviderCacheKey { + mechanism: "workload-identity", + authority: authority + .as_ref() + .map(|value| value.value().clone()) + .unwrap_or_default(), + tenant_id: tenant_id.value().clone(), + client_id: client_id.value().clone(), + scope: scope.value().clone(), + secret_identity: token_file_path.value().clone(), + }, + Self::ManagedIdentity { + client_id, scope, .. + } => AzureCredentialProviderCacheKey { + mechanism: "managed-identity", + authority: String::new(), + tenant_id: String::new(), + client_id: client_id + .as_ref() + .map(|value| value.value().clone()) + .unwrap_or_default(), + scope: scope.value().clone(), + secret_identity: String::new(), + }, + Self::DeveloperTools { scope, .. } => AzureCredentialProviderCacheKey { + mechanism: "developer-tools", + authority: String::new(), + tenant_id: String::new(), + client_id: String::new(), + scope: scope.value().clone(), + secret_identity: String::new(), + }, + } + } +} + +fn validate_authority(request: &NativeAzureRequest) -> Result<(), AuthError> { + let authority = match request { + NativeAzureRequest::ClientSecret { authority, .. } + | NativeAzureRequest::ClientAssertion { authority, .. } + | NativeAzureRequest::WorkloadIdentity { authority, .. } => authority.as_ref(), + NativeAzureRequest::ManagedIdentity { .. } | NativeAzureRequest::DeveloperTools { .. } => { + None + } + }; + let Some(authority) = authority else { + return Ok(()); + }; + let url = url::Url::parse(authority.value()) + .map_err(|_| AuthError::Configuration(AuthConfigurationError::InvalidAzureAuthority))?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || !matches!(url.path(), "" | "/") + { + return Err(AuthError::Configuration( + AuthConfigurationError::InvalidAzureAuthority, + )); + } + Ok(()) +} + +fn validate_sources(request: &NativeAzureRequest) -> Result { + match request { + NativeAzureRequest::ClientSecret { + tenant_id, + client_id, + client_secret, + scope, + authority, + } => { + let identity_sources = [ + tenant_id.source(), + client_id.source(), + client_secret.source(), + ]; + let request_identity = identity_sources.contains(&InputSource::Request); + if request_identity + && !identity_sources + .iter() + .all(|source| *source == InputSource::Request) + { + return mixed_sources(); + } + if !request_identity && is_request_controlled(scope, authority.as_ref()) { + return mixed_sources(); + } + Ok(if request_identity { + InputSource::Request + } else { + trusted_source(&identity_sources) + }) + } + NativeAzureRequest::ClientAssertion { + tenant_id, + client_id, + assertion, + scope, + authority, + .. + } => trusted_only(&[ + tenant_id.source(), + client_id.source(), + assertion.source(), + scope.source(), + authority + .as_ref() + .map(Sourced::source) + .unwrap_or(InputSource::Environment), + ]), + NativeAzureRequest::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + scope, + authority, + } => trusted_only(&[ + tenant_id.source(), + client_id.source(), + token_file_path.source(), + scope.source(), + authority + .as_ref() + .map(Sourced::source) + .unwrap_or(InputSource::Environment), + ]), + NativeAzureRequest::ManagedIdentity { + client_id, + scope, + selection_source, + } => trusted_only(&[ + client_id + .as_ref() + .map(Sourced::source) + .unwrap_or(InputSource::Environment), + scope.source(), + *selection_source, + ]), + NativeAzureRequest::DeveloperTools { + scope, + selection_source, + } => trusted_only(&[scope.source(), *selection_source]), + } +} + +fn is_request_controlled(value: &Sourced, optional: Option<&Sourced>) -> bool { + value.source() == InputSource::Request + || optional.is_some_and(|value| value.source() == InputSource::Request) +} + +fn trusted_only(sources: &[InputSource]) -> Result { + if sources.contains(&InputSource::Request) { + return mixed_sources(); + } + Ok(trusted_source(sources)) +} + +fn trusted_source(sources: &[InputSource]) -> InputSource { + if sources.contains(&InputSource::Deployment) { + InputSource::Deployment + } else { + InputSource::Environment + } +} + +fn mixed_sources() -> Result { + Err(AuthError::Configuration( + AuthConfigurationError::MixedAzureCredentialSources, + )) +} + +fn build_credential( + request: NativeAzureRequest, + transport: Option, +) -> Result, AuthError> { + match request { + NativeAzureRequest::ClientSecret { + tenant_id, + client_id, + client_secret, + authority, + .. + } => ClientSecretCredential::new( + tenant_id.value(), + client_id.into_value(), + Secret::new(client_secret.value().expose().to_string()), + Some(ClientSecretCredentialOptions { + client_options: client_options(authority.map(Sourced::into_value), transport), + }), + ) + .map(|credential| credential as Arc), + NativeAzureRequest::ClientAssertion { + tenant_id, + client_id, + assertion, + authority, + .. + } => ClientAssertionCredential::new( + tenant_id.into_value(), + client_id.into_value(), + StaticAssertion(assertion.into_value()), + Some(ClientAssertionCredentialOptions { + client_options: client_options(authority.map(Sourced::into_value), transport), + }), + ) + .map(|credential| credential as Arc), + NativeAzureRequest::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + authority, + .. + } => WorkloadIdentityCredential::new(Some(WorkloadIdentityCredentialOptions { + credential_options: azure_identity::ClientAssertionCredentialOptions { + client_options: client_options(authority.map(Sourced::into_value), transport), + }, + client_id: Some(client_id.into_value()), + tenant_id: Some(tenant_id.into_value()), + token_file_path: Some(token_file_path.into_value().into()), + })) + .map(|credential| credential as Arc), + NativeAzureRequest::ManagedIdentity { client_id, .. } => { + ManagedIdentityCredential::new(Some(ManagedIdentityCredentialOptions { + user_assigned_id: client_id + .map(Sourced::into_value) + .map(UserAssignedId::ClientId), + client_options: client_options(None, transport), + })) + .map(|credential| credential as Arc) + } + NativeAzureRequest::DeveloperTools { .. } => DeveloperToolsCredential::new(None) + .map(|credential| credential as Arc), + } + .map_err(|error| { + AuthError::Configuration(AuthConfigurationError::AzureCredentialInitialization( + error.to_string(), + )) + }) +} + +fn client_options( + authority: Option, + transport: Option, +) -> ClientOptions { + let cloud = authority.map(|authority_host| { + let mut custom = CustomConfiguration::default(); + custom.authority_host = authority_host; + Arc::new(CloudConfiguration::from(custom)) + }); + ClientOptions { + cloud, + transport, + ..Default::default() + } +} + +fn secret_digest(secret: &str) -> String { + format!("{:x}", Sha256::digest(secret.as_bytes())) +} + +#[derive(Debug)] +struct StaticAssertion(SecretValue); + +impl ClientAssertion for StaticAssertion { + fn secret<'life0, 'life1, 'async_trait>( + &'life0 self, + _options: Option>, + ) -> std::pin::Pin< + Box> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { Ok(self.0.expose().to_string()) }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use azure_core::http::headers::Headers; + use azure_core::http::{AsyncRawResponse, HttpClient, Request, StatusCode, Transport}; + use azure_core::{Bytes, Result}; + + use super::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; + use crate::auth::{InputSource, SecretValue, Sourced}; + + fn deployment(value: T) -> Sourced { + Sourced::new(value, InputSource::Deployment) + } + + fn sourced_client_secret( + credential_source: InputSource, + authority_source: InputSource, + authority: &str, + ) -> NativeAzureRequest { + NativeAzureRequest::ClientSecret { + tenant_id: Sourced::new("tenant".to_string(), credential_source), + client_id: Sourced::new("client".to_string(), credential_source), + client_secret: Sourced::new(SecretValue::new("secret"), credential_source), + scope: Sourced::new("scope".to_string(), InputSource::Environment), + authority: Some(Sourced::new(authority.to_string(), authority_source)), + } + } + + fn client_secret_request( + tenant: &str, + client: &str, + secret: &str, + scope: &str, + authority: &str, + ) -> ValidatedAzureRequest { + ValidatedAzureRequest::new(NativeAzureRequest::ClientSecret { + tenant_id: deployment(tenant.to_string()), + client_id: deployment(client.to_string()), + client_secret: deployment(SecretValue::new(secret)), + scope: deployment(scope.to_string()), + authority: Some(deployment(authority.to_string())), + }) + .unwrap() + } + + #[derive(Debug, Default)] + struct RecordingTokenClient { + requests: Mutex>, + } + + impl HttpClient for RecordingTokenClient { + fn execute_request<'life0, 'life1, 'async_trait>( + &'life0 self, + request: &'life1 Request, + ) -> std::pin::Pin< + Box> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + let body = Bytes::from(request.body()); + self.requests.lock().unwrap().push(( + request.url().to_string(), + String::from_utf8(body.to_vec()).unwrap(), + )); + Ok(AsyncRawResponse::from_bytes( + StatusCode::Ok, + Headers::new(), + r#"{"token_type":"Bearer","expires_in":3600,"ext_expires_in":3600,"access_token":"native-token"}"#, + )) + }) + } + } + + #[tokio::test] + async fn client_secret_uses_sdk_protocol_and_reuses_cached_credential() { + let transport = Arc::new(RecordingTokenClient::default()); + let acquirer = + NativeAzureTokenAcquirer::with_transport(4, Transport::new(transport.clone())); + let request = client_secret_request( + "tenant", + "client", + "secret", + "https://service.test/.default", + "https://login.test", + ); + + let first = acquirer.acquire(request.clone()).await.unwrap(); + let second = acquirer.acquire(request).await.unwrap(); + + assert_eq!(first.secret().expose(), "native-token"); + assert_eq!(second.secret().expose(), "native-token"); + let requests = transport.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].0, "https://login.test/tenant/oauth2/v2.0/token"); + assert!(requests[0].1.contains("client_id=client")); + assert!(requests[0].1.contains("client_secret=secret")); + assert!( + requests[0] + .1 + .contains("scope=https%3A%2F%2Fservice.test%2F.default") + ); + } + + #[tokio::test] + async fn credential_provider_cache_isolates_every_client_secret_identity_field() { + let transport = Arc::new(RecordingTokenClient::default()); + let acquirer = + NativeAzureTokenAcquirer::with_transport(16, Transport::new(transport.clone())); + let request = client_secret_request; + let base = request("tenant", "client", "secret", "scope", "https://login.test"); + let variants = [ + base.clone(), + request( + "other-tenant", + "client", + "secret", + "scope", + "https://login.test", + ), + request( + "tenant", + "other-client", + "secret", + "scope", + "https://login.test", + ), + request( + "tenant", + "client", + "other-secret", + "scope", + "https://login.test", + ), + request( + "tenant", + "client", + "secret", + "other-scope", + "https://login.test", + ), + request( + "tenant", + "client", + "secret", + "scope", + "https://other-login.test", + ), + ]; + + acquirer.acquire(base.clone()).await.unwrap(); + acquirer.acquire(base).await.unwrap(); + for request in variants.into_iter().skip(1) { + acquirer.acquire(request).await.unwrap(); + } + + assert_eq!(transport.requests.lock().unwrap().len(), 6); + } + + #[test] + fn request_authority_requires_request_owned_client_secret_identity() { + let error = ValidatedAzureRequest::new(sourced_client_secret( + InputSource::Deployment, + InputSource::Request, + "https://login.example", + )) + .unwrap_err(); + + assert!(matches!( + error, + crate::AuthError::Configuration( + crate::auth::error::AuthConfigurationError::MixedAzureCredentialSources + ) + )); + } + + #[test] + fn request_owned_client_secret_identity_can_select_custom_authority() { + let request = ValidatedAzureRequest::new(sourced_client_secret( + InputSource::Request, + InputSource::Request, + "https://login.example", + )) + .unwrap(); + + assert_eq!(request.credential_source(), InputSource::Request); + } + + #[test] + fn authority_is_restricted_to_an_https_origin() { + for authority in [ + "http://login.example", + "https://user@login.example", + "https://login.example/tenant", + "https://login.example?target=other", + ] { + let error = ValidatedAzureRequest::new(sourced_client_secret( + InputSource::Deployment, + InputSource::Deployment, + authority, + )) + .unwrap_err(); + assert!(matches!( + error, + crate::AuthError::Configuration( + crate::auth::error::AuthConfigurationError::InvalidAzureAuthority + ) + )); + } + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs new file mode 100644 index 00000000000..025dd4f8740 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs @@ -0,0 +1,683 @@ +use crate::AuthError; +use crate::auth::error::AuthConfigurationError; +use crate::auth::{ + CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, + SecretValue, Sourced, TokenProviderHandle, +}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; +use super::types::{AzureAuthInputs, AzureCredentialType, ConfigValue, DEFAULT_AZURE_SCOPE}; + +const AZURE_AD_TOKEN_ENV: &str = "AZURE_AD_TOKEN"; +const AZURE_TENANT_ID_ENV: &str = "AZURE_TENANT_ID"; +const AZURE_CLIENT_ID_ENV: &str = "AZURE_CLIENT_ID"; +const AZURE_CLIENT_SECRET_ENV: &str = "AZURE_CLIENT_SECRET"; +const AZURE_SCOPE_ENV: &str = "AZURE_SCOPE"; +const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST"; +const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL"; +const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; + +#[derive(Clone, Debug)] +pub(crate) enum AzureCredentialPlan { + Supplied(Sourced), + Caller(TokenProviderHandle), + Oidc { + reference: Sourced, + tenant_id: Sourced, + client_id: Sourced, + scope: Sourced, + authority: Option>, + }, + Native(ValidatedAzureRequest), + Chain(Vec), + Missing, +} + +/// Rust counterpart to Python's `get_azure_ad_token`, not `BaseAzureLLM`. +pub(crate) struct AzureAuthService { + native: Arc, +} + +trait AzureTokenAcquirer: Send + Sync { + fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> Pin> + Send + '_>>; +} + +impl AzureTokenAcquirer for NativeAzureTokenAcquirer { + fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> Pin> + Send + '_>> { + Box::pin(NativeAzureTokenAcquirer::acquire(self, request)) + } +} + +impl Default for AzureAuthService { + fn default() -> Self { + Self { + native: Arc::new(NativeAzureTokenAcquirer::default()), + } + } +} + +impl AzureAuthService { + #[cfg(test)] + fn with_acquirer(native: Arc) -> Self { + Self { native } + } + + pub(crate) async fn get_azure_ad_token( + &self, + inputs: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result>, AuthError> { + match select_auth_plan(inputs, env_lookup)? { + AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)), + AzureCredentialPlan::Caller(caller) => { + let credential = caller.acquire().await?; + if credential.secret().expose().is_empty() { + return Err(AuthError::EmptyAzureToken); + } + Ok(Some(Sourced::new(credential, InputSource::Deployment))) + } + AzureCredentialPlan::Oidc { + reference, + tenant_id, + client_id, + scope, + authority, + } => { + let assertion = resolve_reference(inputs, env_lookup, reference.value()) + .await? + .ok_or(AuthError::UnresolvedOidcReference)?; + let request = ValidatedAzureRequest::new(NativeAzureRequest::ClientAssertion { + tenant_id, + client_id, + assertion: Sourced::new(assertion, reference.source()), + assertion_identity: format!("{:?}", reference.value()), + scope, + authority, + })?; + let source = request.credential_source(); + self.native + .acquire(request) + .await + .map(|credential| Sourced::new(credential, source)) + .map(Some) + } + AzureCredentialPlan::Native(request) => { + let source = request.credential_source(); + self.native + .acquire(request) + .await + .map(|credential| Some(Sourced::new(credential, source))) + } + AzureCredentialPlan::Chain(requests) => { + let mut failures = Vec::new(); + for request in requests { + let source = request.credential_source(); + match self.native.acquire(request).await { + Ok(credential) => return Ok(Some(Sourced::new(credential, source))), + Err(error) => failures.push(error), + } + } + Err(AuthError::CredentialChain(failures)) + } + AzureCredentialPlan::Missing => Ok(None), + } + } +} + +pub(crate) fn select_auth_plan( + inputs: &AzureAuthInputs, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup); + let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup); + let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup); + let client_secret = + configured_secret(&inputs.client_secret, AZURE_CLIENT_SECRET_ENV, env_lookup); + let scope = configured_string(&inputs.azure_scope, AZURE_SCOPE_ENV, env_lookup) + .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment)); + let authority = configured_string( + &inputs.azure_authority_host, + AZURE_AUTHORITY_HOST_ENV, + env_lookup, + ); + let selector = configured_string(&inputs.azure_credential, AZURE_CREDENTIAL_ENV, env_lookup) + .map(|value| { + value + .value() + .parse::() + .map(|selector| Sourced::new(selector, value.source())) + }) + .transpose() + .map_err(|_| AuthError::Configuration(AuthConfigurationError::InvalidAzureSelector))?; + let federated_token_file = configured_string( + &inputs.federated_token_file, + AZURE_FEDERATED_TOKEN_FILE_ENV, + env_lookup, + ); + + if inputs.azure_ad_token_provider.is_none() + && let (Some(tenant_id), Some(client_id), Some(client_secret)) = + (tenant_id.clone(), client_id.clone(), client_secret) + { + return Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( + NativeAzureRequest::ClientSecret { + tenant_id, + client_id, + client_secret, + scope, + authority, + }, + )?)); + } + + if let (Some(reference), Some(tenant_id), Some(client_id)) = ( + oidc_reference(&token)?, + tenant_id.clone(), + client_id.clone(), + ) { + return Ok(AzureCredentialPlan::Oidc { + reference, + tenant_id, + client_id, + scope, + authority, + }); + } + + if let Some(caller) = &inputs.azure_ad_token_provider { + return Ok(AzureCredentialPlan::Caller(caller.clone())); + } + + if let Some(token) = token { + return Ok(AzureCredentialPlan::Supplied(token.map(|token| { + ResolvedCredential::AccessToken { + token, + expires_on: None, + } + }))); + } + + if !*inputs.enable_azure_ad_token_refresh.value() && selector.is_none() { + return Ok(AzureCredentialPlan::Missing); + } + + select_native_plan( + selector, + tenant_id, + client_id, + federated_token_file, + scope, + authority, + inputs.enable_azure_ad_token_refresh.source(), + ) +} + +fn select_native_plan( + selector: Option>, + tenant_id: Option>, + client_id: Option>, + federated_token_file: Option>, + scope: Sourced, + authority: Option>, + refresh_source: InputSource, +) -> Result { + let selected = selector.unwrap_or_else(|| { + Sourced::new( + { + if federated_token_file.is_some() { + AzureCredentialType::DefaultAzureCredential + } else if client_id.is_some() { + AzureCredentialType::ManagedIdentityCredential + } else { + AzureCredentialType::DefaultAzureCredential + } + }, + refresh_source, + ) + }); + let selection_source = selected.source(); + + match selected.into_value() { + AzureCredentialType::ClientSecretCredential => Err(AuthError::Configuration( + AuthConfigurationError::MissingClientSecretFields, + )), + AzureCredentialType::WorkloadIdentityCredential => { + Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( + workload_request(tenant_id, client_id, federated_token_file, scope, authority)?, + )?)) + } + AzureCredentialType::ManagedIdentityCredential => Ok(AzureCredentialPlan::Native( + ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id, + scope, + selection_source, + })?, + )), + AzureCredentialType::DefaultAzureCredential => { + let workload = match (tenant_id, client_id.clone(), federated_token_file) { + (Some(tenant_id), Some(client_id), Some(token_file_path)) => { + Some(NativeAzureRequest::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + scope: scope.clone(), + authority, + }) + } + _ => None, + }; + Ok(AzureCredentialPlan::Chain( + workload + .into_iter() + .chain(std::iter::once(NativeAzureRequest::ManagedIdentity { + client_id, + scope: scope.clone(), + selection_source, + })) + .chain(std::iter::once(NativeAzureRequest::DeveloperTools { + scope, + selection_source, + })) + .map(ValidatedAzureRequest::new) + .collect::, _>>()?, + )) + } + AzureCredentialType::DeploymentIdentityCredential => { + let workload = match (tenant_id, client_id.clone(), federated_token_file) { + (Some(tenant_id), Some(client_id), Some(token_file_path)) => { + Some(NativeAzureRequest::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + scope: scope.clone(), + authority, + }) + } + _ => None, + }; + let user_assigned = client_id.map(|client_id| NativeAzureRequest::ManagedIdentity { + client_id: Some(client_id), + scope: scope.clone(), + selection_source, + }); + Ok(AzureCredentialPlan::Chain( + workload + .into_iter() + .chain(user_assigned) + .chain(std::iter::once(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope, + selection_source, + })) + .map(ValidatedAzureRequest::new) + .collect::, _>>()?, + )) + } + } +} + +fn workload_request( + tenant_id: Option>, + client_id: Option>, + token_file_path: Option>, + scope: Sourced, + authority: Option>, +) -> Result { + Ok(NativeAzureRequest::WorkloadIdentity { + tenant_id: tenant_id.ok_or(AuthError::Configuration( + AuthConfigurationError::MissingWorkloadTenant, + ))?, + client_id: client_id.ok_or(AuthError::Configuration( + AuthConfigurationError::MissingWorkloadClient, + ))?, + token_file_path: token_file_path.ok_or(AuthError::Configuration( + AuthConfigurationError::MissingWorkloadTokenFile, + ))?, + scope, + authority, + }) +} + +fn configured_string( + configured: &ConfigValue, + environment_name: &str, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option> { + configured + .as_value() + .filter(|value| !value.value().is_empty()) + .cloned() + .or_else(|| { + env_lookup(environment_name) + .filter(|value| !value.is_empty()) + .map(|value| Sourced::new(value, InputSource::Environment)) + }) +} + +fn configured_secret( + configured: &ConfigValue, + environment_name: &str, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option> { + configured + .as_value() + .filter(|value| !value.value().expose().is_empty()) + .cloned() + .or_else(|| { + env_lookup(environment_name) + .filter(|value| !value.is_empty()) + .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment)) + }) +} + +async fn resolve_reference( + inputs: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + reference: &CredentialRef, +) -> Result, AuthError> { + let lookup = match reference { + CredentialRef::Explicit(secret) => return Ok(Some(secret.clone())), + CredentialRef::Env(name) => env_lookup(name) + .filter(|value| !value.is_empty()) + .map(SecretValue::new) + .map_or(CredentialLookup::Missing, CredentialLookup::Found), + CredentialRef::None => return Ok(None), + CredentialRef::File(_) | CredentialRef::Request(_) | CredentialRef::Host(_) => { + let resolver = inputs + .credential_resolver + .as_ref() + .ok_or(AuthError::Configuration( + AuthConfigurationError::MissingHostResolver, + ))?; + resolver.resolve(reference).await? + } + }; + Ok(match lookup { + CredentialLookup::Found(secret) => Some(secret), + CredentialLookup::Missing | CredentialLookup::Declined => None, + }) +} + +fn oidc_reference( + token: &Option>, +) -> Result>, AuthError> { + let Some(token) = token.as_ref() else { + return Ok(None); + }; + let value = token.value().expose(); + if token.source() == InputSource::Request && value.starts_with("oidc/") { + return Err(AuthError::Configuration( + AuthConfigurationError::RequestAzureCredentialReference, + )); + } + if let Some(name) = value.strip_prefix("oidc/env/") { + return non_empty_reference(name, "OIDC environment reference") + .map(CredentialRef::Env) + .map(|reference| Sourced::new(reference, token.source())) + .map(Some); + } + if let Some(name) = value.strip_prefix("oidc/env_path/") { + return non_empty_reference(name, "OIDC environment path reference") + .map(|name| CredentialRef::File(CredentialFileRef::EnvironmentVariable(name))) + .map(|reference| Sourced::new(reference, token.source())) + .map(Some); + } + if let Some(path) = value.strip_prefix("oidc/file/") { + let path = non_empty_reference(path, "OIDC file reference")?; + return Ok(Some(Sourced::new( + CredentialRef::File(CredentialFileRef::Path(path.into())), + token.source(), + ))); + } + if value.starts_with("oidc/") { + return Err(AuthError::Configuration( + AuthConfigurationError::UnsupportedOidcReference, + )); + } + Ok(None) +} + +fn non_empty_reference(value: &str, kind: &str) -> Result { + if value.is_empty() { + return Err(AuthError::Configuration( + AuthConfigurationError::EmptyReference(kind.to_string()), + )); + } + Ok(value.to_string()) +} + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::sync::{Arc, Mutex}; + + use serde_json::json; + + use super::{ + AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference, + resolve_reference, select_auth_plan, + }; + use crate::AuthError; + use crate::auth::ResolvedCredential; + use crate::auth::{ + CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialRef, + CredentialResolver, CredentialResolverHandle, InputSource, SecretValue, Sourced, + }; + use crate::providers::azure_ai::auth::native::ValidatedAzureRequest; + use crate::providers::azure_ai::auth::types::AzureAuthInputs; + + #[derive(Debug)] + struct FileResolver; + + struct ChainAcquirer { + requests: Mutex>, + succeed_on: Option<&'static str>, + } + + impl AzureTokenAcquirer for ChainAcquirer { + fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> std::pin::Pin< + Box> + Send + '_>, + > { + let kind = request.kind(); + self.requests.lock().unwrap().push(kind); + Box::pin(async move { + if self.succeed_on == Some(kind) { + Ok(ResolvedCredential::AccessToken { + token: SecretValue::new("chain-token"), + expires_on: None, + }) + } else { + Err(AuthError::AzureTokenAcquisition(format!("{kind} failed"))) + } + }) + } + } + + impl CredentialResolver for FileResolver { + fn resolve<'a>(&'a self, reference: &'a CredentialRef) -> CredentialLookupFuture<'a> { + Box::pin(async move { + Ok(match reference { + CredentialRef::File(CredentialFileRef::Path(path)) + if path == std::path::Path::new("/run/secrets/assertion") => + { + CredentialLookup::Found(SecretValue::new("rotated-assertion")) + } + _ => CredentialLookup::Declined, + }) + }) + } + } + + #[test] + fn null_and_empty_values_fall_back_to_environment() { + let params = json!({"tenant_id": null, "client_id": "", "client_secret": null}); + let inputs = AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap(); + let plan = select_auth_plan(&inputs, &|name| match name { + "AZURE_TENANT_ID" => Some("tenant".to_string()), + "AZURE_CLIENT_ID" => Some("client".to_string()), + "AZURE_CLIENT_SECRET" => Some("secret".to_string()), + _ => None, + }) + .unwrap(); + + assert!(matches!(plan, AzureCredentialPlan::Native(_))); + } + + #[test] + fn supplied_token_does_not_require_refresh() { + let params = json!({"azure_ad_token": "token"}); + let inputs = AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap(); + + assert!(matches!( + select_auth_plan(&inputs, &|_| None).unwrap(), + AzureCredentialPlan::Supplied(_) + )); + } + + #[test] + fn oidc_reference_is_deferred() { + let params = json!({ + "azure_ad_token": "oidc/env/ASSERTION", + "tenant_id": "tenant", + "client_id": "client" + }); + let inputs = AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap(); + + assert!(matches!( + select_auth_plan(&inputs, &|_| None).unwrap(), + AzureCredentialPlan::Oidc { + reference, + .. + } if reference.value() == &CredentialRef::Env("ASSERTION".to_string()) + )); + } + + #[test] + fn oidc_file_location_is_typed_before_resolution() { + assert_eq!( + oidc_reference(&Some(Sourced::new( + SecretValue::new("oidc/file//run/secrets/assertion"), + InputSource::Deployment, + ))) + .unwrap() + .map(Sourced::into_value), + Some(CredentialRef::File(CredentialFileRef::Path( + "/run/secrets/assertion".into() + ))) + ); + } + + #[test] + fn unsupported_oidc_reference_is_rejected_during_plan_creation() { + let error = oidc_reference(&Some(Sourced::new( + SecretValue::new("oidc/vault/assertion"), + InputSource::Deployment, + ))) + .expect_err("unsupported backend must fail validation"); + + assert!(error.to_string().contains("unsupported OIDC reference")); + } + + #[test] + fn request_oidc_reference_is_rejected_before_lookup() { + let params = json!({ + "azure_ad_token": "oidc/env/ASSERTION", + "tenant_id": "tenant", + "client_id": "client" + }); + let sources = std::collections::BTreeMap::from([ + ("azure_ad_token".to_string(), InputSource::Request), + ("tenant_id".to_string(), InputSource::Request), + ("client_id".to_string(), InputSource::Request), + ]); + let inputs = + AzureAuthInputs::from_sourced_optional_params(params.as_object().unwrap(), &sources) + .unwrap(); + + let error = select_auth_plan(&inputs, &|name| { + assert_ne!(name, "ASSERTION"); + None + }) + .unwrap_err(); + + assert!(matches!( + error, + AuthError::Configuration( + crate::auth::error::AuthConfigurationError::RequestAzureCredentialReference + ) + )); + } + + #[tokio::test] + async fn host_resolver_owns_file_access() { + let inputs = AzureAuthInputs { + credential_resolver: Some(CredentialResolverHandle::new(Arc::new(FileResolver))), + ..AzureAuthInputs::default() + }; + let reference = + CredentialRef::File(CredentialFileRef::Path("/run/secrets/assertion".into())); + + let resolved = resolve_reference(&inputs, &|_| None, &reference) + .await + .unwrap(); + + assert_eq!(resolved, Some(SecretValue::new("rotated-assertion"))); + } + + #[tokio::test] + async fn default_chain_uses_declared_order_and_stops_after_success() { + let acquirer = Arc::new(ChainAcquirer { + requests: Mutex::new(Vec::new()), + succeed_on: Some("developer-tools"), + }); + let service = AzureAuthService::with_acquirer(acquirer.clone()); + let inputs = AzureAuthInputs { + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + + let credential = service + .get_azure_ad_token(&inputs, &|_| None) + .await + .unwrap() + .unwrap(); + + assert_eq!(credential.value().secret().expose(), "chain-token"); + assert_eq!( + *acquirer.requests.lock().unwrap(), + ["managed-identity", "developer-tools"] + ); + } + + #[tokio::test] + async fn chain_reports_each_acquisition_failure() { + let acquirer = Arc::new(ChainAcquirer { + requests: Mutex::new(Vec::new()), + succeed_on: None, + }); + let service = AzureAuthService::with_acquirer(acquirer); + let inputs = AzureAuthInputs { + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + + let error = service + .get_azure_ad_token(&inputs, &|_| None) + .await + .unwrap_err(); + + assert!(matches!(error, AuthError::CredentialChain(errors) if errors.len() == 2)); + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs new file mode 100644 index 00000000000..f15d526d945 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs @@ -0,0 +1,195 @@ +use crate::auth::error::AuthConfigurationError; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; +use strum::EnumString; + +use crate::AuthError; +use crate::auth::{ + CredentialResolverHandle, InputSource, SecretValue, Sourced, TokenProviderHandle, +}; + +pub const DEFAULT_AZURE_SCOPE: &str = "https://cognitiveservices.azure.com/.default"; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ConfigValue { + #[default] + Absent, + ExplicitNone(InputSource), + Value(Sourced), +} + +impl ConfigValue { + pub fn as_value(&self) -> Option<&Sourced> { + match self { + Self::Value(value) => Some(value), + Self::Absent | Self::ExplicitNone(_) => None, + } + } +} + +#[derive(Clone, Copy, Debug, EnumString, PartialEq, Eq, Hash)] +#[allow(clippy::enum_variant_names)] +pub enum AzureCredentialType { + ClientSecretCredential, + ManagedIdentityCredential, + DefaultAzureCredential, + DeploymentIdentityCredential, + WorkloadIdentityCredential, +} + +#[derive(Clone, Debug, Default)] +pub struct AzureAuthInputs { + pub azure_ad_token: ConfigValue, + pub azure_ad_token_provider: Option, + pub credential_resolver: Option, + pub tenant_id: ConfigValue, + pub client_id: ConfigValue, + pub client_secret: ConfigValue, + pub azure_scope: ConfigValue, + pub azure_authority_host: ConfigValue, + pub azure_credential: ConfigValue, + pub federated_token_file: ConfigValue, + pub enable_azure_ad_token_refresh: Sourced, +} + +impl AzureAuthInputs { + #[cfg(test)] + pub fn from_optional_params(params: &Map) -> Result { + Self::from_sourced_optional_params(params, &BTreeMap::new()) + } + + pub fn from_sourced_optional_params( + params: &Map, + sources: &BTreeMap, + ) -> Result { + Ok(Self { + azure_ad_token: secret_config(params, sources, "azure_ad_token")?, + azure_ad_token_provider: None, + credential_resolver: None, + tenant_id: string_config(params, sources, "tenant_id")?, + client_id: string_config(params, sources, "client_id")?, + client_secret: secret_config(params, sources, "client_secret")?, + azure_scope: string_config(params, sources, "azure_scope")?, + azure_authority_host: string_config(params, sources, "azure_authority_host")?, + azure_credential: string_config(params, sources, "azure_credential")?, + federated_token_file: string_config(params, sources, "azure_federated_token_file")?, + enable_azure_ad_token_refresh: Sourced::new( + params + .get("enable_azure_ad_token_refresh") + .and_then(Value::as_bool) + .unwrap_or(false), + source_for(sources, "enable_azure_ad_token_refresh"), + ), + }) + } +} + +fn string_config( + params: &Map, + sources: &BTreeMap, + name: &str, +) -> Result, AuthError> { + let source = source_for(sources, name); + match params.get(name) { + None => Ok(ConfigValue::Absent), + Some(Value::Null) => Ok(ConfigValue::ExplicitNone(source)), + Some(Value::String(value)) => Ok(ConfigValue::Value(Sourced::new(value.clone(), source))), + Some(_) => Err(AuthError::Configuration( + AuthConfigurationError::InvalidFieldType(name.to_string()), + )), + } +} + +fn secret_config( + params: &Map, + sources: &BTreeMap, + name: &str, +) -> Result, AuthError> { + Ok(match string_config(params, sources, name)? { + ConfigValue::Absent => ConfigValue::Absent, + ConfigValue::ExplicitNone(source) => ConfigValue::ExplicitNone(source), + ConfigValue::Value(value) => ConfigValue::Value(value.map(SecretValue::new)), + }) +} + +fn source_for(sources: &BTreeMap, name: &str) -> InputSource { + sources.get(name).copied().unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use std::collections::BTreeMap; + + use super::{AzureAuthInputs, AzureCredentialType, ConfigValue}; + use crate::auth::{InputSource, Sourced}; + + #[test] + fn selector_parsing_is_exact() { + assert_eq!( + "ClientSecretCredential".parse::(), + Ok(AzureCredentialType::ClientSecretCredential) + ); + assert!( + "clientsecretcredential" + .parse::() + .is_err() + ); + } + + #[test] + fn defaults_preserve_absence() { + let inputs = AzureAuthInputs::default(); + + assert_eq!(inputs.tenant_id, ConfigValue::Absent); + assert_eq!(inputs.azure_ad_token, ConfigValue::Absent); + } + + #[test] + fn parsing_distinguishes_null_empty_and_absent() { + let params = json!({"tenant_id": null, "client_id": ""}); + let inputs = AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap(); + + assert_eq!( + inputs.tenant_id, + ConfigValue::ExplicitNone(InputSource::Deployment) + ); + assert_eq!( + inputs.client_id, + ConfigValue::Value(Sourced::new(String::new(), InputSource::Deployment)) + ); + assert_eq!(inputs.client_secret, ConfigValue::Absent); + } + + #[test] + fn parsing_preserves_trusted_input_sources() { + let params = json!({"tenant_id": "tenant", "client_secret": null}); + let sources = BTreeMap::from([ + ("tenant_id".to_string(), InputSource::Request), + ("client_secret".to_string(), InputSource::Request), + ]); + let inputs = + AzureAuthInputs::from_sourced_optional_params(params.as_object().unwrap(), &sources) + .unwrap(); + + assert_eq!( + inputs.tenant_id, + ConfigValue::Value(Sourced::new("tenant".to_string(), InputSource::Request)) + ); + assert_eq!( + inputs.client_secret, + ConfigValue::ExplicitNone(InputSource::Request) + ); + } + + #[test] + fn debug_does_not_expose_secrets() { + let params = json!({"azure_ad_token": "token-value", "client_secret": "secret-value"}); + let inputs = AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap(); + let debug = format!("{inputs:?}"); + + assert!(!debug.contains("token-value")); + assert!(!debug.contains("secret-value")); + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index b8ca10461fb..585b34f393f 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,3 +1,4 @@ +use crate::auth::error::MissingCredential; use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ @@ -32,12 +33,7 @@ pub fn resolve_azure_api_key( non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| { - Error::Auth( - "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable" - .to_string(), - ) - }) + .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AzureApiKey))) } pub fn complete_azure_anthropic_url( @@ -47,13 +43,7 @@ pub fn complete_azure_anthropic_url( let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| { - Error::Auth( - "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \ - Expected format: https://.services.ai.azure.com/anthropic" - .to_string(), - ) - })?; + .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AzureApiBase)))?; let api_base = api_base.trim_end_matches('/'); diff --git a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs index 5d13fa93e00..4f41d1d6abb 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs @@ -1,2 +1,2 @@ +pub(crate) mod auth; pub mod messages; -pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs deleted file mode 100644 index 2dbec8e2187..00000000000 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ /dev/null @@ -1,1376 +0,0 @@ -use std::collections::BTreeSet; - -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; -use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; -use serde_json::{Map, Value, json}; - -use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; - -const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; -const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; -const AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; -const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; -const AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: &str = "2024-11-30"; -const AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: i64 = 96; - -const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = - &["pages", "features", "req_format"]; - -pub struct AzureAiOcrConfig; -pub struct AzureDocumentIntelligenceOcrConfig; - -pub const AZURE_AI_OCR_CONFIG: AzureAiOcrConfig = AzureAiOcrConfig; -pub const AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG: AzureDocumentIntelligenceOcrConfig = - AzureDocumentIntelligenceOcrConfig; - -fn non_empty(value: Option<&str>) -> Option<&str> { - value.map(str::trim).filter(|value| !value.is_empty()) -} - -fn resolve_value( - explicit: Option<&str>, - env_name: &str, - env_lookup: &dyn Fn(&str) -> Option, - missing_message: &str, -) -> Result { - non_empty(explicit) - .map(str::to_string) - .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| Error::Auth(missing_message.to_string())) -} - -pub fn resolve_azure_ai_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_key, - AZURE_AI_API_KEY_ENV, - env_lookup, - "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params", - ) -} - -pub fn resolve_azure_ai_api_base( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_base, - AZURE_AI_API_BASE_ENV, - env_lookup, - "Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter", - ) -} - -pub fn complete_azure_ai_url( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let base = resolve_azure_ai_api_base(api_base, env_lookup)?; - Ok(format!( - "{}/providers/mistral/azure/ocr", - base.trim_end_matches('/') - )) -} - -pub fn resolve_document_intelligence_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_key, - AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, - env_lookup, - "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter", - ) -} - -pub fn resolve_document_intelligence_endpoint( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_base, - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, - env_lookup, - "Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter", - ) -} - -fn prepend_auth_header( - headers: Vec<(String, String)>, - name: &str, - value: String, -) -> Vec<(String, String)> { - std::iter::once((name.to_string(), value)) - .chain(headers) - .collect() -} - -pub fn validate_azure_ai_environment( - headers: Vec<(String, String)>, - api_key: Option<&str>, - azure_ad_token: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result, Error> { - if crate::http_utils::has_header(&headers, "Authorization") - || crate::http_utils::has_header(&headers, "Api-Key") - { - return Ok(headers); - } - if let Ok(api_key) = resolve_azure_ai_api_key(api_key, env_lookup) { - return Ok(prepend_auth_header(headers, "Api-Key", api_key)); - } - non_empty(azure_ad_token) - .map(|token| prepend_auth_header(headers, "Authorization", format!("Bearer {token}"))) - .ok_or(Error::MissingAzureAiCredentialsOrAdToken) -} - -pub fn validate_document_intelligence_environment( - headers: Vec<(String, String)>, - api_key: Option<&str>, - azure_ad_token: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result, Error> { - if crate::http_utils::has_header(&headers, "Authorization") - || crate::http_utils::has_header(&headers, "Ocp-Apim-Subscription-Key") - { - return Ok(headers); - } - if let Ok(api_key) = resolve_document_intelligence_api_key(api_key, env_lookup) { - return Ok(prepend_auth_header( - headers, - "Ocp-Apim-Subscription-Key", - api_key, - )); - } - non_empty(azure_ad_token) - .map(|token| prepend_auth_header(headers, "Authorization", format!("Bearer {token}"))) - .ok_or_else(|| { - Error::Auth( - "Missing Azure Document Intelligence credentials - set AZURE_DOCUMENT_INTELLIGENCE_API_KEY or provide azure_ad_token" - .to_string(), - ) - }) -} - -fn encode_model_id(model: &str) -> Result { - let model_id = model.rsplit('/').next().unwrap_or(model); - if matches!(model_id, "." | "..") { - return Err(Error::InvalidRequest( - "model_id cannot be a dot path segment".to_string(), - )); - } - Ok(model_id - .bytes() - .flat_map(|byte| match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - vec![byte as char] - } - _ => format!("%{byte:02X}").chars().collect(), - }) - .collect()) -} - -fn pages_token_is_valid(token: &str) -> bool { - let mut parts = token.split('-'); - let Some(start) = parts.next() else { - return false; - }; - if start.is_empty() || !start.chars().all(|ch| ch.is_ascii_digit()) { - return false; - } - match parts.next() { - None => true, - Some(end) => { - !end.is_empty() && end.chars().all(|ch| ch.is_ascii_digit()) && parts.next().is_none() - } - } -} - -fn normalize_pages_param(pages: &Value) -> Result, Error> { - match pages { - Value::String(value) => { - let normalized = value - .split(',') - .map(str::trim) - .collect::>() - .join(","); - if normalized.split(',').all(pages_token_is_valid) { - Ok(Some(normalized)) - } else { - Err(Error::InvalidRequest(format!( - "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." - ))) - } - } - Value::Array(values) => { - if values.is_empty() { - return Ok(None); - } - if values.iter().any(Value::is_boolean) { - return Err(Error::InvalidRequest( - "`pages` must be integers, not booleans".to_string(), - )); - } - if values.iter().all(Value::is_i64) { - let mut pages = BTreeSet::new(); - for value in values { - let page = value.as_i64().expect("checked is_i64"); - if page < 0 { - return Err(Error::InvalidRequest( - "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), - )); - } - pages.insert(page + 1); - } - return Ok(Some( - pages - .into_iter() - .map(|page| page.to_string()) - .collect::>() - .join(","), - )); - } - if values.iter().all(Value::is_string) { - let normalized = values - .iter() - .filter_map(Value::as_str) - .map(str::trim) - .collect::>() - .join(","); - if normalized.split(',').all(pages_token_is_valid) { - return Ok(Some(normalized)); - } - return Err(Error::InvalidRequest(format!( - "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." - ))); - } - Err(Error::InvalidRequest( - "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." - .to_string(), - )) - } - _ => Err(Error::InvalidRequest( - "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." - .to_string(), - )), - } -} - -fn feature_token_is_valid(token: &str) -> bool { - let Some((first, rest)) = token.as_bytes().split_first() else { - return false; - }; - first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) -} - -fn invalid_features_error(features: &Value) -> Error { - Error::InvalidRequest(format!( - "Invalid `features` for Azure Document Intelligence: {features:?}. Expected a list of feature names or a comma-separated string like 'keyValuePairs' or 'keyValuePairs,languages'." - )) -} - -fn normalize_features_param(features: &Value) -> Result, Error> { - let normalized = match features { - Value::String(value) => value - .split(',') - .map(str::trim) - .collect::>() - .join(","), - Value::Array(values) if values.is_empty() => return Ok(None), - Value::Array(values) => values - .iter() - .map(Value::as_str) - .collect::>>() - .ok_or_else(|| invalid_features_error(features))? - .into_iter() - .map(str::trim) - .collect::>() - .join(","), - _ => return Err(invalid_features_error(features)), - }; - - if normalized.split(',').all(feature_token_is_valid) { - Ok(Some(normalized)) - } else { - Err(invalid_features_error(features)) - } -} - -fn normalize_req_format(req_format: &Value) -> Result { - match req_format.as_str() { - Some(value @ ("native" | "litellm")) => Ok(value.to_string()), - _ => Err(Error::InvalidRequest(format!( - "Invalid `req_format` for Azure Document Intelligence: {req_format:?}. Expected 'native' or 'litellm'." - ))), - } -} - -pub fn map_document_intelligence_ocr_params( - non_default_params: &Map, -) -> Result, Error> { - let mut mapped = Map::new(); - if let Some(pages) = non_default_params.get("pages") - && let Some(normalized) = normalize_pages_param(pages)? - { - mapped.insert("pages".to_string(), Value::String(normalized)); - } - if let Some(features) = non_default_params.get("features") - && let Some(normalized) = normalize_features_param(features)? - { - mapped.insert("features".to_string(), Value::String(normalized)); - } - if let Some(req_format) = non_default_params.get("req_format") { - mapped.insert( - "req_format".to_string(), - Value::String(normalize_req_format(req_format)?), - ); - } - Ok(mapped) -} - -pub fn complete_document_intelligence_url( - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; - let mut url = format!( - "{}/documentintelligence/documentModels/{}:analyze?api-version={}", - endpoint.trim_end_matches('/'), - encode_model_id(model)?, - AZURE_DOCUMENT_INTELLIGENCE_API_VERSION - ); - - if let Some(pages) = optional_params.get("pages") - && let Some(normalized) = normalize_pages_param(pages)? - { - url.push_str("&pages="); - url.push_str(&normalized); - } - - if let Some(features) = optional_params.get("features") - && let Some(normalized) = normalize_features_param(features)? - { - url.push_str("&features="); - url.push_str(&normalized); - } - - if let Some(req_format) = optional_params.get("req_format") { - normalize_req_format(req_format)?; - } - - Ok(url) -} - -fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { - let object = document.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(document), - })?; - let doc_type = object - .get("type") - .and_then(Value::as_str) - .ok_or(Error::MissingField("document.type"))?; - let field_name = match doc_type { - "document_url" => "document_url", - "image_url" => "image_url", - other => { - return Err(Error::InvalidRequest(format!( - "Invalid document type: {other}. Must be 'document_url' or 'image_url'" - ))); - } - }; - object - .get(field_name) - .and_then(Value::as_str) - .filter(|value| !value.is_empty()) - .ok_or(Error::MissingField(field_name)) -} - -fn extract_base64_from_data_uri(data_uri: &str) -> &str { - data_uri - .split_once(',') - .map(|(_, data)| data) - .unwrap_or(data_uri) -} - -fn page_markdown(page: &Map) -> String { - page.get("lines") - .and_then(Value::as_array) - .map(|lines| { - lines - .iter() - .filter_map(|line| line.get("content").and_then(Value::as_str)) - .collect::>() - .join("\n") - }) - .unwrap_or_default() -} - -fn page_dimensions(page: &Map) -> Value { - let width = page.get("width").and_then(Value::as_f64).unwrap_or(8.5); - let height = page.get("height").and_then(Value::as_f64).unwrap_or(11.0); - let unit = page.get("unit").and_then(Value::as_str).unwrap_or("inch"); - let (width, height) = if unit == "inch" { - ( - (width * AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI as f64) as i64, - (height * AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI as f64) as i64, - ) - } else { - (width as i64, height as i64) - }; - json!({ - "width": width, - "height": height, - "dpi": AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI, - }) -} - -fn transform_document_intelligence_response( - model: &str, - response_json: Value, - preserve_native_response: bool, -) -> Result { - let response = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - let status = response - .get("status") - .and_then(Value::as_str) - .ok_or(Error::MissingField("status"))?; - if status != "succeeded" { - return Err(Error::InvalidResponse(format!( - "Azure Document Intelligence analysis failed with status: {status}" - ))); - } - - let analyze_result = response.get("analyzeResult").and_then(Value::as_object); - let azure_pages = analyze_result - .and_then(|result| result.get("pages")) - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let pages = azure_pages - .iter() - .filter_map(Value::as_object) - .map(|page| { - let page_number = page.get("pageNumber").and_then(Value::as_i64).unwrap_or(1); - json!({ - "index": page_number - 1, - "markdown": page_markdown(page), - "dimensions": page_dimensions(page), - }) - }) - .collect::>(); - let extra_fields = ["content", "tables", "keyValuePairs"] - .into_iter() - .map(|field| { - ( - field.to_string(), - analyze_result - .and_then(|result| result.get(field)) - .cloned() - .unwrap_or(Value::Null), - ) - }) - .collect(); - - Ok(LiteLLMOcrResponse { - usage_info: Some(json!({ - "pages_processed": pages.len(), - "doc_size_bytes": null, - })), - pages, - model: model.to_string(), - document_annotation: None, - object: "ocr".to_string(), - extra_fields, - provider_native_response: preserve_native_response.then_some(response_json), - }) -} - -impl OcrProviderConfig for AzureAiOcrConfig { - fn supported_ocr_params(&self) -> &'static [&'static str] { - MISTRAL_OCR_CONFIG.supported_ocr_params() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_azure_ai_url(api_base, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_azure_ai_api_key(api_key, env_lookup) - } - - fn requires_data_uri_document(&self) -> bool { - true - } -} - -impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_ocr_params(&self) -> &'static [&'static str] { - AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params(&self, non_default_params: &Map) -> Map { - map_document_intelligence_ocr_params(non_default_params).unwrap_or_else(|_| { - non_default_params - .iter() - .filter(|(name, _)| { - AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS.contains(&name.as_str()) - }) - .map(|(name, value)| (name.clone(), value.clone())) - .collect() - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - _optional_params: Map, - ) -> Result { - let document_url = document_url_from_mistral_document(&document)?; - let mut data = Map::new(); - if document_url.starts_with("data:") { - data.insert( - "base64Source".to_string(), - Value::String(extract_base64_from_data_uri(document_url).to_string()), - ); - } else { - data.insert( - "urlSource".to_string(), - Value::String(document_url.to_string()), - ); - } - Ok(OcrRequestData { - data: Value::Object(data), - files: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_document_intelligence_response(model, response_json, false) - } - - fn transform_ocr_response_with_params( - &self, - model: &str, - response_json: Value, - optional_params: &Map, - ) -> Result { - transform_document_intelligence_response( - model, - response_json, - optional_params.get("req_format").and_then(Value::as_str) == Some("native"), - ) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_document_intelligence_url(api_base, model, optional_params, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_document_intelligence_api_key(api_key, env_lookup) - } - - fn auth_strategy(&self) -> OcrAuthStrategy { - OcrAuthStrategy::Header("Ocp-Apim-Subscription-Key") - } - - fn response_handling(&self) -> OcrResponseHandling { - OcrResponseHandling::AzureDocumentIntelligencePoll - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rstest::{fixture, rstest}; - - const ENDPOINT: &str = "https://example.cognitiveservices.azure.com"; - - #[fixture] - fn document_intelligence_config() -> AzureDocumentIntelligenceOcrConfig { - AzureDocumentIntelligenceOcrConfig - } - - fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) - } - - #[fixture] - fn native_operation() -> Value { - json!({ - "status": "succeeded", - "createdDateTime": "2026-07-02T00:00:00Z", - "lastUpdatedDateTime": "2026-07-02T00:00:05Z", - "analyzeResult": { - "content": "Invoice\nInvoice No: INV-12345\nTotal: $100.00", - "pages": [{ - "pageNumber": 1, - "width": 8.5, - "height": 11, - "unit": "inch", - "angle": 0.13, - "lines": [ - {"content": "Invoice"}, - {"content": "Invoice No: INV-12345"}, - {"content": "Total: $100.00"} - ], - "words": [{"content": "Invoice", "confidence": 0.994}] - }], - "tables": [ - { - "rowCount": 2, - "columnCount": 2, - "cells": [ - {"kind": "columnHeader", "rowIndex": 0, "columnIndex": 0, "content": "Item"}, - {"kind": "columnHeader", "rowIndex": 0, "columnIndex": 1, "content": "Price"}, - {"rowIndex": 1, "columnIndex": 0, "content": "Widget"}, - {"rowIndex": 1, "columnIndex": 1, "content": "$100.00"} - ] - }, - { - "rowCount": 1, - "columnCount": 1, - "cells": [{"rowIndex": 0, "columnIndex": 0, "content": "Totals"}] - } - ], - "keyValuePairs": [ - { - "key": {"content": "Invoice No"}, - "value": {"content": "INV-12345"}, - "confidence": 0.98 - }, - { - "key": {"content": "Total"}, - "value": {"content": "$100.00"}, - "confidence": 0.95 - } - ], - "paragraphs": [{"content": "Invoice"}] - } - }) - } - - fn assert_native_fields_preserved(response: &LiteLLMOcrResponse, operation: &Value) { - let analyze_result = &operation["analyzeResult"]; - - assert_eq!(response.extra_fields["content"], analyze_result["content"]); - assert_eq!(response.extra_fields["tables"], analyze_result["tables"]); - assert_eq!( - response.extra_fields["keyValuePairs"], - analyze_result["keyValuePairs"] - ); - assert_eq!(response.object, "ocr"); - assert_eq!( - response.usage_info, - Some(json!({"pages_processed": 1, "doc_size_bytes": null})) - ); - assert_eq!(response.pages[0]["index"], 0); - assert_eq!( - response.pages[0]["markdown"], - "Invoice\nInvoice No: INV-12345\nTotal: $100.00" - ); - assert_eq!( - response.pages[0]["dimensions"], - json!({"width": 816, "height": 1056, "dpi": 96}) - ); - } - - #[test] - fn azure_ai_reuses_mistral_body_transform() { - let body = AZURE_AI_OCR_CONFIG - .transform_ocr_request( - "pixtral-12b-2409", - json!({"type": "document_url", "document_url": "data:application/pdf;base64,abc"}), - serde_json::Map::from_iter([("include_image_base64".to_string(), json!(true))]), - ) - .expect("request transforms") - .data; - - assert_eq!(body["model"], "pixtral-12b-2409"); - assert_eq!(body["include_image_base64"], true); - assert_eq!( - body["document"]["document_url"], - "data:application/pdf;base64,abc" - ); - } - - #[test] - fn document_intelligence_url_normalizes_zero_based_pages() { - let params = serde_json::Map::from_iter([("pages".to_string(), json!([2, 0, 2]))]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com/"), - "azure_ai/doc-intelligence/prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,3" - ); - } - - #[test] - fn document_intelligence_url_normalizes_features() { - let params = serde_json::Map::from_iter([( - "features".to_string(), - json!("keyValuePairs, languages"), - )]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&features=keyValuePairs,languages" - ); - } - - #[test] - fn document_intelligence_url_combines_pages_and_feature_list() { - let params = serde_json::Map::from_iter([ - ("pages".to_string(), json!([0, 1, 2])), - ( - "features".to_string(), - json!([" keyValuePairs ", "languages"]), - ), - ]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,2,3&features=keyValuePairs,languages" - ); - } - - #[test] - fn document_intelligence_url_omits_empty_feature_list() { - let params = serde_json::Map::from_iter([("features".to_string(), json!([]))]); - assert!( - map_document_intelligence_ocr_params(¶ms) - .expect("empty features map") - .is_empty() - ); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30" - ); - } - - #[rstest] - #[case::query_injection(json!("keyValuePairs&pages=9"))] - #[case::spaces(json!("key value pairs"))] - #[case::empty_string(json!(""))] - #[case::integer_list(json!([1, 2]))] - #[case::nested_list(json!([["keyValuePairs"]]))] - #[case::object(json!({"feature": "keyValuePairs"}))] - #[case::number(json!(5))] - fn document_intelligence_mapping_rejects_invalid_features(#[case] features: Value) { - let params = serde_json::Map::from_iter([("features".to_string(), features)]); - let error = - map_document_intelligence_ocr_params(¶ms).expect_err("invalid features must fail"); - - assert!(matches!( - error, - Error::InvalidRequest(message) if message.contains("Invalid `features`") - )); - } - - #[rstest] - #[case::single_list(json!(["keyValuePairs"]), "keyValuePairs")] - #[case::multiple_list( - json!(["keyValuePairs", "languages"]), - "keyValuePairs,languages" - )] - #[case::single_string(json!("keyValuePairs"), "keyValuePairs")] - #[case::comma_separated(json!("keyValuePairs,languages"), "keyValuePairs,languages")] - #[case::spaces(json!("keyValuePairs, languages"), "keyValuePairs,languages")] - fn document_intelligence_maps_features(#[case] features: Value, #[case] expected: &str) { - let params = Map::from_iter([ - ("features".to_string(), features), - ("unsupported".to_string(), json!(true)), - ]); - - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.map_ocr_params(¶ms), - Map::from_iter([("features".to_string(), json!(expected))]) - ); - } - - #[test] - fn document_intelligence_request_uses_base64_source_for_data_uri() { - let body = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_request( - "prebuilt-read", - json!({"type": "document_url", "document_url": "data:application/pdf;base64,abc123"}), - Map::new(), - ) - .expect("request transforms") - .data; - - assert_eq!(body, json!({"base64Source": "abc123"})); - } - - #[rstest] - fn document_intelligence_response_normalizes_pages(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response("prebuilt-layout", native_operation.clone()) - .expect("response transforms"); - - assert_native_fields_preserved(&response, &native_operation); - } - - #[test] - fn azure_document_intelligence_model_id_is_encoded() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout?x=1#frag", - &Map::new(), - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout%3Fx%3D1%23frag:analyze?api-version=2024-11-30" - ); - } - - #[test] - fn azure_document_intelligence_dot_segment_model_id_is_rejected() { - let error = complete_document_intelligence_url( - Some(ENDPOINT), - "azure_ai/doc-intelligence/..", - &Map::new(), - &|_| None, - ) - .expect_err("dot segment must fail"); - - assert_eq!( - error, - Error::InvalidRequest("model_id cannot be a dot path segment".to_string()) - ); - } - - #[rstest] - fn document_intelligence_async_response_preserves_normalized_fields(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - ) - .expect("response transforms"); - - assert_native_fields_preserved(&response, &native_operation); - } - - #[test] - fn document_intelligence_response_tolerates_missing_native_fields() { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-read", - json!({ - "status": "succeeded", - "analyzeResult": { - "pages": [{ - "pageNumber": 1, - "width": 8.5, - "height": 11, - "unit": "inch", - "lines": [{"content": "hello"}] - }] - } - }), - ) - .expect("missing optional fields are allowed"); - - assert_eq!(response.pages[0]["markdown"], "hello"); - assert_eq!(response.extra_fields["content"], Value::Null); - assert_eq!(response.extra_fields["tables"], Value::Null); - assert_eq!(response.extra_fields["keyValuePairs"], Value::Null); - } - - #[test] - fn document_intelligence_non_succeeded_status_is_rejected() { - let error = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-layout", - json!({"status": "failed"}), - ) - .expect_err("failed status must fail"); - - assert_eq!( - error, - Error::InvalidResponse( - "Azure Document Intelligence analysis failed with status: failed".to_string() - ) - ); - } - - #[test] - fn document_intelligence_supported_params_include_features() { - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.supported_ocr_params(), - &["pages", "features", "req_format"] - ); - } - - #[rstest] - fn document_intelligence_native_format_carries_raw_operation(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &Map::from_iter([("req_format".to_string(), json!("native"))]), - ) - .expect("native response transforms"); - - assert_eq!( - response.provider_native_response, - Some(native_operation.clone()) - ); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - fn document_intelligence_async_native_format_carries_raw_operation(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &Map::from_iter([("req_format".to_string(), json!("native"))]), - ) - .expect("native response transforms"); - - assert_eq!( - response.provider_native_response, - Some(native_operation.clone()) - ); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - #[case::default(Map::new())] - #[case::litellm(Map::from_iter([("req_format".to_string(), json!("litellm"))]))] - fn document_intelligence_default_format_omits_raw_operation( - #[case] optional_params: Map, - native_operation: Value, - ) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &optional_params, - ) - .expect("response transforms"); - - assert_eq!(response.provider_native_response, None); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - #[case::native("native")] - #[case::litellm("litellm")] - fn document_intelligence_maps_req_format(#[case] req_format: &str) { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "req_format".to_string(), - json!(req_format), - )])) - .expect("req_format maps"); - - assert_eq!( - mapped, - Map::from_iter([("req_format".to_string(), json!(req_format))]) - ); - } - - #[test] - fn document_intelligence_rejects_unknown_req_format() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "req_format".to_string(), - json!("azure"), - )])) - .expect_err("unknown req_format must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `req_format`")) - ); - } - - #[test] - fn document_intelligence_url_omits_req_format() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout", - &Map::from_iter([("req_format".to_string(), json!("native"))]), - &|_| None, - ) - .expect("url builds"); - - assert!(!url.contains("req_format")); - } - - #[test] - fn document_intelligence_validate_environment_uses_subscription_key() { - let headers = - validate_document_intelligence_environment(Vec::new(), Some("my-key"), None, &|_| None) - .expect("api key authenticates"); - - assert_eq!( - header_value(&headers, "Ocp-Apim-Subscription-Key"), - Some("my-key") - ); - } - - #[test] - fn document_intelligence_validate_environment_falls_back_to_entra_token() { - let headers = validate_document_intelligence_environment( - Vec::new(), - None, - Some("entra-token"), - &|_| None, - ) - .expect("Entra token authenticates"); - - assert_eq!( - header_value(&headers, "Authorization"), - Some("Bearer entra-token") - ); - assert_eq!(header_value(&headers, "Ocp-Apim-Subscription-Key"), None); - } - - #[test] - fn document_intelligence_supported_params_include_pages_features_and_req_format() { - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.supported_ocr_params(), - &["pages", "features", "req_format"] - ); - } - - #[test] - fn document_intelligence_maps_zero_based_page_list() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([0, 1, 2]), - )])) - .expect("pages map"); - - assert_eq!( - mapped, - Map::from_iter([("pages".to_string(), json!("1,2,3"))]) - ); - } - - #[test] - fn document_intelligence_page_mapping_dedupes_and_sorts() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([2, 0, 0, 1]), - )])) - .expect("pages map"); - - assert_eq!(mapped["pages"], "1,2,3"); - } - - #[test] - fn document_intelligence_page_mapping_omits_empty_list() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([]), - )])) - .expect("empty pages map"); - - assert!(mapped.is_empty()); - } - - #[test] - fn document_intelligence_page_mapping_accepts_native_range() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("3-9"), - )])) - .expect("range maps"); - - assert_eq!(mapped["pages"], "3-9"); - } - - #[test] - fn document_intelligence_page_mapping_strips_spaces() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("1-3, 5"), - )])) - .expect("range maps"); - - assert_eq!(mapped["pages"], "1-3,5"); - } - - #[test] - fn document_intelligence_page_mapping_accepts_string_tokens() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!(["1", "3-5"]), - )])) - .expect("tokens map"); - - assert_eq!(mapped["pages"], "1,3-5"); - } - - #[test] - fn document_intelligence_page_mapping_rejects_invalid_string() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("a,b"), - )])) - .expect_err("invalid pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `pages` string")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_negative_index() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([-1]), - )])) - .expect_err("negative pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("must be >= 0")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_bool_list() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([true, false]), - )])) - .expect_err("boolean pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("integers, not booleans")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_unsupported_type() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!(5), - )])) - .expect_err("unsupported pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Mistral-style")) - ); - } - - #[test] - fn document_intelligence_url_appends_pages_query() { - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com/"), - "azure_ai/doc-intelligence/prebuilt-layout", - &Map::from_iter([("pages".to_string(), json!("1-3,5"))]), - &|_| None, - ) - .expect("url builds"); - - assert!(url.contains("api-version=2024-11-30")); - assert!(url.contains("pages=1-3,5")); - assert!(url.contains("/documentintelligence/documentModels/prebuilt-layout:analyze")); - } - - #[test] - fn document_intelligence_url_has_no_pages_when_params_are_empty() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout", - &Map::new(), - &|_| None, - ) - .expect("url builds"); - - assert!(!url.contains("pages=")); - } - - #[rstest] - fn document_intelligence_request_keeps_pages_out_of_body( - document_intelligence_config: AzureDocumentIntelligenceOcrConfig, - ) { - let request = document_intelligence_config - .transform_ocr_request( - "prebuilt-layout", - json!({"type": "document_url", "document_url": "https://example.com/x.pdf"}), - Map::from_iter([("pages".to_string(), json!("1,2,3"))]), - ) - .expect("request transforms"); - - assert_eq!( - request.data, - json!({"urlSource": "https://example.com/x.pdf"}) - ); - } - - #[test] - fn document_intelligence_mistral_pages_flow_to_query_only() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([2, 3, 4, 5, 6, 7, 8]), - )])) - .expect("pages map"); - let url = - complete_document_intelligence_url(Some(ENDPOINT), "prebuilt-layout", &mapped, &|_| { - None - }) - .expect("url builds"); - let request = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_request( - "prebuilt-layout", - json!({"type": "document_url", "document_url": "https://example.com/x.pdf"}), - mapped, - ) - .expect("request transforms"); - - assert!(url.contains("pages=3,4,5,6,7,8,9")); - assert_eq!( - request.data, - json!({"urlSource": "https://example.com/x.pdf"}) - ); - } - - #[test] - fn document_intelligence_endpoint_ignores_generic_azure_ai_base() { - let resolved = resolve_document_intelligence_endpoint(None, &|name| match name { - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - AZURE_AI_API_BASE_ENV => Some("https://generic.example.com".to_string()), - _ => None, - }) - .expect("endpoint resolves"); - - assert_eq!(resolved, ENDPOINT); - } - - #[test] - fn document_intelligence_endpoint_honors_explicit_api_base() { - let resolved = resolve_document_intelligence_endpoint( - Some("https://my-di.cognitiveservices.azure.com"), - &|name| match name { - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - AZURE_AI_API_BASE_ENV => Some("https://generic.example.com".to_string()), - _ => None, - }, - ) - .expect("endpoint resolves"); - - assert_eq!(resolved, "https://my-di.cognitiveservices.azure.com"); - } - - #[test] - fn azure_ai_mistral_ocr_uses_generic_api_base() { - let resolved = resolve_azure_ai_api_base(None, &|name| match name { - AZURE_AI_API_BASE_ENV => Some("https://generic-azure-ai.example.com".to_string()), - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - _ => None, - }) - .expect("api base resolves"); - - assert_eq!(resolved, "https://generic-azure-ai.example.com"); - } - - #[test] - fn azure_ai_ocr_authenticates_with_entra_token() { - let headers = - validate_azure_ai_environment(Vec::new(), None, Some("entra-token"), &|_| None) - .expect("Entra token authenticates"); - - assert_eq!( - header_value(&headers, "Authorization"), - Some("Bearer entra-token") - ); - } -} diff --git a/litellm-rust/crates/core/src/providers/mistral/mod.rs b/litellm-rust/crates/core/src/providers/mistral/mod.rs deleted file mode 100644 index 3621ff6a2fd..00000000000 --- a/litellm-rust/crates/core/src/providers/mistral/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs deleted file mode 100644 index 044fc587c22..00000000000 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ /dev/null @@ -1,436 +0,0 @@ -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; -use serde_json::{Map, Value}; - -const SUPPORTED_OCR_PARAMS: &[&str] = &[ - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - "document_annotation_prompt", - "extract_header", - "extract_footer", - "table_format", - "confidence_scores_granularity", - "include_blocks", - "id", -]; - -/// Default Mistral API base, used when the caller does not override `api_base`. -pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1"; - -/// Environment variable holding the Mistral API key. -pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; - -/// Error message raised when no Mistral API key can be resolved. -pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"; - -/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`. -/// -/// Blank/whitespace `api_base` is treated as absent (guard at resolution time). -pub fn complete_url(api_base: Option<&str>) -> String { - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(MISTRAL_DEFAULT_API_BASE) - .trim_end_matches('/'); - - if base.ends_with("/v1") { - format!("{base}/ocr") - } else { - format!("{base}/v1/ocr") - } -} - -/// Resolve the Mistral API key from the explicit param or the environment. -/// -/// Blank/whitespace values are treated as absent. Returns `Error::Auth` -/// when no usable key is available. -/// -/// Note: the env fallback only reads the process environment. Secret-manager -/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in -/// via `api_key`; this fallback is a last resort for direct/standalone use. -pub fn resolve_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) -} - -pub struct MistralOcrConfig; - -pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; - -impl OcrProviderConfig for MistralOcrConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_ocr_params(&self) -> &'static [&'static str] { - SUPPORTED_OCR_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - if !document.is_object() { - return Err(Error::InvalidType { - expected: "object", - actual: json_type_name(&document), - }); - } - - let mut data = Map::new(); - data.insert("model".to_string(), Value::String(model.to_string())); - data.insert("document".to_string(), document); - for (param, value) in optional_params { - data.insert(param, value); - } - - Ok(OcrRequestData { - data: Value::Object(data), - files: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - let response_object = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - - let pages = response_object - .get("pages") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let model = response_object - .get("model") - .and_then(Value::as_str) - .unwrap_or(model) - .to_string(); - let document_annotation = response_object.get("document_annotation").cloned(); - let usage_info = response_object.get("usage_info").cloned(); - - Ok(LiteLLMOcrResponse { - pages, - model, - document_annotation, - usage_info, - object: "ocr".to_string(), - extra_fields: Map::new(), - provider_native_response: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(complete_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn supported_ocr_params() -> &'static [&'static str] { - MISTRAL_OCR_CONFIG.supported_ocr_params() -} - -pub fn map_ocr_params(non_default_params: &Map) -> Map { - MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params) -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_request( - model: &str, - document: Value, - optional_params: Map, -) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_response( - model: &str, - response_json: Value, -) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_header_is_a_supported_ocr_param() { - assert!(supported_ocr_params().contains(&"extract_header")); - } - - #[test] - fn extract_footer_is_a_supported_ocr_param() { - assert!(supported_ocr_params().contains(&"extract_footer")); - } - - #[test] - fn existing_ocr_params_remain_supported() { - for param in [ - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - ] { - assert!(supported_ocr_params().contains(¶m)); - } - } - - #[test] - fn map_ocr_params_forwards_extract_header() { - let params = json!({"extract_header": true}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_forwards_extract_footer() { - let params = json!({"extract_footer": true}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_forwards_extract_header_and_footer() { - let params = json!({"extract_header": true, "extract_footer": false}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_drops_unknown_params() { - let params = json!({"extract_header": true, "unsupported_param": "value"}); - let mapped = map_ocr_params(params.as_object().unwrap()); - assert_eq!(mapped.get("extract_header"), Some(&json!(true))); - assert!(!mapped.contains_key("unsupported_param")); - } - - #[test] - fn new_ocr_params_are_supported() { - for param in [ - "table_format", - "confidence_scores_granularity", - "document_annotation_prompt", - "include_blocks", - "id", - ] { - assert!(supported_ocr_params().contains(¶m)); - } - } - - #[test] - fn map_ocr_params_forwards_new_ocr_params() { - for (param, value) in [ - ("table_format", json!("html")), - ("confidence_scores_granularity", json!("word")), - ( - "document_annotation_prompt", - json!("Extract all invoice line items"), - ), - ("include_blocks", json!(true)), - ("id", json!("req-123")), - ] { - let params = json!({param: value}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - } - - #[test] - fn transform_ocr_request_includes_each_optional_param() { - let document = json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }); - for (param, value) in [ - ("table_format", json!("html")), - ("confidence_scores_granularity", json!("word")), - ( - "document_annotation_prompt", - json!("Extract all invoice line items"), - ), - ("id", json!("req-123")), - ("extract_header", json!(true)), - ("include_blocks", json!(true)), - ("pages", json!([0, 1])), - ] { - let result = transform_ocr_request( - "mistral-ocr-latest", - document.clone(), - json!({param: value}).as_object().unwrap().clone(), - ) - .expect("request should transform"); - assert_eq!(result.data.get(param), Some(&value)); - assert_eq!(result.data.get("model"), Some(&json!("mistral-ocr-latest"))); - assert_eq!(result.data.get("document"), Some(&document)); - assert_eq!(result.files, None); - } - } - - #[test] - fn transform_ocr_request_includes_multiple_new_params() { - let document = json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }); - let optional_params = json!({ - "table_format": "html", - "confidence_scores_granularity": "page", - "extract_header": true - }) - .as_object() - .unwrap() - .clone(); - let result = transform_ocr_request("mistral-ocr-latest", document, optional_params) - .expect("request should transform"); - assert_eq!(result.data.get("table_format"), Some(&json!("html"))); - assert_eq!( - result.data.get("confidence_scores_granularity"), - Some(&json!("page")) - ); - assert_eq!(result.data.get("extract_header"), Some(&json!(true))); - } - - #[test] - fn transform_ocr_response_preserves_blocks_and_confidence_scores() { - let blocks = json!([{"type": "title", "content": "Invoice"}]); - let confidence_scores = json!({"page": 0.98}); - let response = json!({ - "pages": [{"index": 0, "markdown": "# Invoice", "blocks": blocks, "confidence_scores": confidence_scores}], - "model": "mistral-ocr-4-0", - "usage_info": {"pages_processed": 1} - }); - let result = - transform_ocr_response("mistral-ocr-4-0", response).expect("response should transform"); - assert_eq!(result.pages[0].get("blocks"), Some(&blocks)); - assert_eq!( - result.pages[0].get("confidence_scores"), - Some(&confidence_scores) - ); - } - - #[test] - fn transform_ocr_response_preserves_ocr4_page_fields() { - let response = json!({ - "pages": [{"index": 0, "markdown": "table page", "tables": [{"rows": 2, "cols": 3}], "hyperlinks": ["https://example.com"], "header": "Acme Corp", "footer": "Page 1"}], - "model": "mistral-ocr-4-0", - "usage_info": {"pages_processed": 1} - }); - let result = transform_ocr_response("mistral-ocr-4-0", response.clone()) - .expect("response should transform"); - assert_eq!(result.pages[0], response["pages"][0]); - } - - #[test] - fn transform_ocr_request_rejects_non_object_document() { - let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new()) - .expect_err("string document should be rejected"); - - assert_eq!( - err, - Error::InvalidType { - expected: "object", - actual: "string", - } - ); - } - - #[test] - fn transform_ocr_response_normalizes_mistral_json() { - let response = json!({ - "pages": [{"index": 0, "markdown": "hello"}], - "model": "mistral-ocr-2505-completion", - "document_annotation": null, - "usage_info": {"pages_processed": 1} - }); - - let result = transform_ocr_response("mistral-ocr-latest", response) - .expect("response should transform"); - - assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]); - assert_eq!(result.model, "mistral-ocr-2505-completion"); - assert_eq!(result.document_annotation, Some(Value::Null)); - assert_eq!(result.usage_info, Some(json!({"pages_processed": 1}))); - assert_eq!(result.object, "ocr"); - } - - #[test] - fn complete_url_defaults_and_dedupes_v1() { - assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr"); - assert_eq!(complete_url(Some(" ")), "https://api.mistral.ai/v1/ocr"); - assert_eq!( - complete_url(Some("https://proxy.internal")), - "https://proxy.internal/v1/ocr" - ); - assert_eq!( - complete_url(Some("https://proxy.internal/v1/")), - "https://proxy.internal/v1/ocr" - ); - } - - #[test] - fn resolve_api_key_prefers_param_then_env() { - let no_env = |_: &str| None; - assert_eq!( - resolve_api_key(Some("sk-param"), &no_env).unwrap(), - "sk-param" - ); - - let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); - assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env"); - // Blank param falls through to the environment. - assert_eq!(resolve_api_key(Some(" "), &with_env).unwrap(), "sk-env"); - } - - #[test] - fn resolve_api_key_errors_when_absent() { - let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); - } -} diff --git a/litellm-rust/crates/core/src/providers/mod.rs b/litellm-rust/crates/core/src/providers/mod.rs index c0c2c69831b..1aeb75063d6 100644 --- a/litellm-rust/crates/core/src/providers/mod.rs +++ b/litellm-rust/crates/core/src/providers/mod.rs @@ -2,7 +2,4 @@ pub mod anthropic; pub mod azure_ai; #[cfg(feature = "bedrock-auth")] pub mod bedrock; -pub mod mistral; pub mod openai; -pub mod reducto; -pub mod vertex_ai; diff --git a/litellm-rust/crates/core/src/providers/reducto/mod.rs b/litellm-rust/crates/core/src/providers/reducto/mod.rs deleted file mode 100644 index 3621ff6a2fd..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs deleted file mode 100644 index 8acee8f770c..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod transformation; - -#[cfg(test)] -mod tests; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs deleted file mode 100644 index 2b66d058b5d..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs +++ /dev/null @@ -1,202 +0,0 @@ -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; - -use super::transformation::*; -use crate::ocr::transformation::OcrProviderConfig; - -#[fixture] -fn parse_response() -> Value { - json!({ - "job_id": "job_123", - "usage": {"num_pages": 3, "credits": 3}, - "result": { - "chunks": [ - { - "content": "Page 1 block A", - "blocks": [{ - "content": "Page 1 block A", - "bbox": {"page": 1}, - "kind": "text", - }], - }, - { - "content": "Page 2 block A", - "blocks": [{ - "content": "Page 2 block A", - "bbox": {"page": 2}, - "kind": "table", - }], - }, - { - "content": "Page 1 block B", - "blocks": [{ - "content": "Page 1 block B", - "bbox": {"page": 1}, - "kind": "text", - }], - }, - { - "content": "Page 3 block A", - "blocks": [{ - "content": "Page 3 block A", - "bbox": {"page": 3}, - "kind": "figure", - }], - }, - ], - }, - }) -} - -#[rstest] -fn test_parse_v3_file_upload_and_response_mapping(parse_response: Value) { - let source = classify_document_source("data:application/pdf;base64,JVBERi0xLjQ=") - .expect("PDF data URI should be valid"); - let upload = build_upload_request( - source, - "Bearer test-key", - Some("https://platform.reducto.ai"), - ) - .expect("data URI should require upload"); - assert_eq!(upload.url, "https://platform.reducto.ai/upload"); - assert_eq!(upload.authorization, "Bearer test-key"); - assert_eq!(upload.file_name, "document"); - assert_eq!(upload.mime_type, "application/pdf"); - assert_eq!(upload.bytes, b"%PDF-1.4"); - - let optional_params = json!({ - "formatting": {"table_output_format": "html"}, - "retrieval": {"chunk_mode": "section"}, - "settings": {"ocr_system": "standard"}, - }) - .as_object() - .expect("params should be an object") - .clone(); - let request = build_parse_v3_request("reducto://uploaded.pdf", optional_params); - assert_eq!( - request.data, - json!({ - "input": "reducto://uploaded.pdf", - "formatting": {"table_output_format": "html"}, - "retrieval": {"chunk_mode": "section"}, - "settings": {"ocr_system": "standard"}, - }) - ); - - let transformed = transform_reducto_response("parse-v3", parse_response.clone()) - .expect("response should transform"); - assert_eq!( - transformed.usage_info, - Some(json!({"pages_processed": 3, "credits": 3})) - ); - assert_eq!(transformed.pages.len(), 3); - assert_eq!( - transformed.pages[0], - json!({ - "index": 0, - "markdown": "Page 1 block A\n\nPage 1 block B", - "blocks": [ - {"content": "Page 1 block A", "bbox": {"page": 1}, "kind": "text"}, - {"content": "Page 1 block B", "bbox": {"page": 1}, "kind": "text"}, - ], - }) - ); - assert_eq!(transformed.pages[1]["markdown"], "Page 2 block A"); - assert_eq!(transformed.pages[2]["markdown"], "Page 3 block A"); - assert_eq!(transformed.provider_native_response, Some(parse_response)); -} - -#[rstest] -fn test_parse_v3_reducto_id_passthrough_skips_upload(parse_response: Value) { - let document = json!({ - "type": "document_url", - "document_url": "reducto://already-uploaded.pdf", - }); - let source = extract_document_source(&document).expect("Reducto ID should be valid"); - assert!(build_upload_request(source.clone(), "Bearer test-key", None).is_none()); - assert_eq!( - source, - ReductoDocumentSource::FileId("reducto://already-uploaded.pdf".to_string()) - ); - - let request = REDUCTO_PARSE_V3_CONFIG - .transform_ocr_request( - "parse-v3", - document, - json!({"retrieval": {"chunk_mode": "section"}}) - .as_object() - .expect("params should be object") - .clone(), - ) - .expect("direct ID should transform"); - assert_eq!(request.data["input"], "reducto://already-uploaded.pdf"); - assert_eq!(request.data["retrieval"]["chunk_mode"], "section"); - - let response = REDUCTO_PARSE_V3_CONFIG - .transform_ocr_response("parse-v3", parse_response) - .expect("response should transform"); - assert!( - response.pages[0]["markdown"] - .as_str() - .expect("markdown should be string") - .starts_with("Page 1 block A") - ); -} - -#[rstest] -fn test_parse_legacy_wraps_enhance_under_options() { - let request = build_parse_legacy_request( - "reducto://legacy.pdf", - json!({"enhance": {"agentic": [{"type": "table"}]}}) - .as_object() - .expect("params should be object"), - ); - assert_eq!( - request.data, - json!({ - "document_url": "reducto://legacy.pdf", - "options": {"enhance": {"agentic": [{"type": "table"}]}}, - }) - ); -} - -#[rstest] -fn test_parse_v3_image_data_uri_upload_uses_image_mime() { - let source = classify_document_source("data:image/png;base64,iVBORw0KGgo=") - .expect("PNG data URI should be valid"); - let upload = build_upload_request( - source, - "Bearer programmatic-key", - Some("https://custom.reducto.test/"), - ) - .expect("data URI should require upload"); - assert_eq!(upload.url, "https://custom.reducto.test/upload"); - assert_eq!(upload.authorization, "Bearer programmatic-key"); - assert_eq!(upload.mime_type, "image/png"); - assert_eq!(upload.bytes, b"\x89PNG\r\n\x1a\n"); -} - -#[rstest] -#[case::http("http://example.com/document.pdf")] -#[case::https("https://example.com/document.pdf")] -fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) { - let error = classify_document_source(source).expect_err("plain URL should be rejected"); - assert!(error.to_string().contains("upload the file first")); -} - -#[rstest] -fn test_parse_v3_uses_programmatic_api_key_over_env() { - let key = resolve_api_key(Some("passed-key"), &|_| Some("env-reducto-key".to_string())) - .expect("explicit key should resolve"); - assert_eq!(key, "passed-key"); - - let headers = REDUCTO_PARSE_V3_CONFIG - .validate_environment(Vec::new(), Some("passed-key"), &|_| { - Some("env-reducto-key".to_string()) - }) - .expect("headers should validate"); - assert_eq!( - headers, - vec![("Authorization".to_string(), "Bearer passed-key".to_string())] - ); -} diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs deleted file mode 100644 index f025887846f..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs +++ /dev/null @@ -1,407 +0,0 @@ -use std::collections::BTreeMap; - -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use serde_json::{Map, Value, json}; - -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; - -pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; -pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; -pub const REDUCTO_ID_PREFIX: &str = "reducto://"; - -const PARSE_V3_SUPPORTED_OCR_PARAMS: &[&str] = &["formatting", "retrieval", "settings"]; -const PARSE_LEGACY_SUPPORTED_OCR_PARAMS: &[&str] = &["enhance"]; -const MISSING_KEY_MESSAGE: &str = "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"; -const DATA_URI_UPLOAD_REQUIRED: &str = - "Reducto data URI upload must complete before OCR request transformation"; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ReductoDocumentSource { - FileId(String), - Upload { bytes: Vec, mime_type: String }, -} - -#[derive(Clone, PartialEq, Eq)] -pub struct ReductoUploadRequest { - pub url: String, - pub authorization: String, - pub file_name: &'static str, - pub bytes: Vec, - pub mime_type: String, -} - -pub struct ReductoParseV3Config; -pub struct ReductoParseLegacyConfig; - -pub const REDUCTO_PARSE_V3_CONFIG: ReductoParseV3Config = ReductoParseV3Config; -pub const REDUCTO_PARSE_LEGACY_CONFIG: ReductoParseLegacyConfig = ReductoParseLegacyConfig; - -pub fn config_for_model(model: &str) -> Option<&'static dyn OcrProviderConfig> { - match model { - "parse-v3" => Some(&REDUCTO_PARSE_V3_CONFIG), - "parse-legacy" => Some(&REDUCTO_PARSE_LEGACY_CONFIG), - _ => None, - } -} - -pub fn normalize_api_base(api_base: Option<&str>) -> String { - api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(REDUCTO_API_BASE) - .trim_end_matches('/') - .to_string() -} - -pub fn parse_url(api_base: Option<&str>) -> String { - format!("{}/parse", normalize_api_base(api_base)) -} - -pub fn upload_url(api_base: Option<&str>) -> String { - format!("{}/upload", normalize_api_base(api_base)) -} - -pub fn resolve_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| { - env_lookup(REDUCTO_API_KEY_ENV) - .map(|key| key.trim().to_string()) - .filter(|key| !key.is_empty()) - }) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) -} - -pub fn extract_document_source(document: &Value) -> Result { - let document = document.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(document), - })?; - let source = document - .get("document_url") - .and_then(Value::as_str) - .filter(|source| !source.is_empty()) - .or_else(|| document.get("image_url").and_then(Value::as_str)) - .ok_or_else(|| { - Error::InvalidRequest( - "Reducto expected OCR preprocessing to produce document_url or image_url" - .to_string(), - ) - })?; - classify_document_source(source) -} - -pub fn classify_document_source(source: &str) -> Result { - if source.starts_with(REDUCTO_ID_PREFIX) { - return Ok(ReductoDocumentSource::FileId(source.to_string())); - } - if source.starts_with("http://") || source.starts_with("https://") { - return Err(Error::InvalidRequest( - "Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first." - .to_string(), - )); - } - if !source.starts_with("data:") { - return Err(Error::InvalidRequest( - "Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing." - .to_string(), - )); - } - - let (header, encoded) = source - .split_once(',') - .ok_or_else(|| Error::InvalidRequest("Invalid Reducto data URI provided.".to_string()))?; - if !header.split(';').any(|part| part == "base64") { - return Err(Error::InvalidRequest( - "Reducto only supports base64-encoded data URIs.".to_string(), - )); - } - - let mime_type = header - .strip_prefix("data:") - .and_then(|header| header.split(';').next()) - .filter(|mime| !mime.is_empty()) - .unwrap_or("application/octet-stream") - .to_string(); - let bytes = BASE64_STANDARD.decode(encoded).map_err(|_| { - Error::InvalidRequest("Invalid Reducto base64 payload provided.".to_string()) - })?; - - Ok(ReductoDocumentSource::Upload { bytes, mime_type }) -} - -pub fn build_upload_request( - source: ReductoDocumentSource, - authorization: &str, - api_base: Option<&str>, -) -> Option { - let ReductoDocumentSource::Upload { bytes, mime_type } = source else { - return None; - }; - - Some(ReductoUploadRequest { - url: upload_url(api_base), - authorization: authorization.to_string(), - file_name: "document", - bytes, - mime_type, - }) -} - -pub fn extract_upload_file_id(response_json: &Value) -> Result<&str, Error> { - response_json - .as_object() - .and_then(|response| response.get("file_id")) - .and_then(Value::as_str) - .filter(|file_id| !file_id.is_empty()) - .ok_or_else(|| { - Error::InvalidResponse(format!( - "Reducto /upload returned 200 without a file_id; got payload={response_json}" - )) - }) -} - -pub fn build_parse_v3_request( - file_id: &str, - optional_params: Map, -) -> OcrRequestData { - let data = std::iter::once(("input".to_string(), Value::String(file_id.to_string()))) - .chain(optional_params) - .collect(); - OcrRequestData { - data: Value::Object(data), - files: None, - } -} - -pub fn build_parse_legacy_request( - file_id: &str, - optional_params: &Map, -) -> OcrRequestData { - let options = optional_params - .get("enhance") - .filter(|enhance| !enhance.is_null()) - .map(|enhance| json!({"options": {"enhance": enhance}})); - let data = match options { - Some(Value::Object(options)) => std::iter::once(( - "document_url".to_string(), - Value::String(file_id.to_string()), - )) - .chain(options) - .collect(), - _ => Map::from_iter([( - "document_url".to_string(), - Value::String(file_id.to_string()), - )]), - }; - OcrRequestData { - data: Value::Object(data), - files: None, - } -} - -fn source_file_id(document: &Value) -> Result { - match extract_document_source(document)? { - ReductoDocumentSource::FileId(file_id) => Ok(file_id), - ReductoDocumentSource::Upload { .. } => Err(Error::Unsupported(DATA_URI_UPLOAD_REQUIRED)), - } -} - -fn page_number(block: &Map) -> Option { - let page = block.get("bbox")?.as_object()?.get("page")?; - page.as_i64() - .or_else(|| page.as_u64().and_then(|page| i64::try_from(page).ok())) - .or_else(|| page.as_str().and_then(|page| page.parse().ok())) -} - -fn chunks(result: &Map) -> &[Value] { - result - .get("chunks") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or_default() -} - -fn build_pages(result: &Map) -> Vec { - let blocks_by_page = chunks(result) - .iter() - .filter_map(Value::as_object) - .filter_map(|chunk| chunk.get("blocks").and_then(Value::as_array)) - .flatten() - .filter_map(|block| block.as_object().map(|object| (block, object))) - .filter_map(|(block, object)| page_number(object).map(|page| (page, block.clone()))) - .fold( - BTreeMap::>::new(), - |mut pages, (page, block)| { - pages.entry(page).or_default().push(block); - pages - }, - ); - - if blocks_by_page.is_empty() { - let markdown = chunks(result) - .iter() - .filter_map(Value::as_object) - .filter_map(|chunk| chunk.get("content").and_then(Value::as_str)) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - return if markdown.is_empty() { - Vec::new() - } else { - vec![json!({"index": 0, "markdown": markdown})] - }; - } - - blocks_by_page - .into_iter() - .map(|(page, blocks)| { - let markdown = blocks - .iter() - .filter_map(Value::as_object) - .filter_map(|block| block.get("content").and_then(Value::as_str)) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - json!({ - "index": page.saturating_sub(1).max(0), - "markdown": markdown, - "blocks": blocks, - }) - }) - .collect() -} - -pub fn transform_reducto_response( - model: &str, - response_json: Value, -) -> Result { - let response = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - let empty_result = Map::new(); - let result = match response.get("result") { - Some(Value::Object(result)) => result, - Some(Value::Null) => &empty_result, - Some(_) => { - return Err(Error::InvalidResponse( - "Reducto result must be an object".to_string(), - )); - } - None => response, - }; - let usage = response - .get("usage") - .and_then(Value::as_object) - .cloned() - .unwrap_or_default(); - let usage_info = Some(json!({ - "pages_processed": usage.get("num_pages").cloned().unwrap_or(Value::Null), - "credits": usage.get("credits").cloned().unwrap_or(Value::Null), - })); - - Ok(LiteLLMOcrResponse { - pages: build_pages(result), - model: model.to_string(), - document_annotation: None, - usage_info, - object: "ocr".to_string(), - extra_fields: Map::new(), - provider_native_response: Some(response_json), - }) -} - -impl OcrProviderConfig for ReductoParseV3Config { - fn supported_ocr_params(&self) -> &'static [&'static str] { - PARSE_V3_SUPPORTED_OCR_PARAMS - } - - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - optional_params: Map, - ) -> Result { - let file_id = source_file_id(&document)?; - Ok(build_parse_v3_request(&file_id, optional_params)) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_reducto_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(parse_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} - -impl OcrProviderConfig for ReductoParseLegacyConfig { - fn supported_ocr_params(&self) -> &'static [&'static str] { - PARSE_LEGACY_SUPPORTED_OCR_PARAMS - } - - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - optional_params: Map, - ) -> Result { - let file_id = source_file_id(&document)?; - Ok(build_parse_legacy_request(&file_id, &optional_params)) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_reducto_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(parse_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs b/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs deleted file mode 100644 index 3621ff6a2fd..00000000000 --- a/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs deleted file mode 100644 index b0e5a278d0b..00000000000 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ /dev/null @@ -1,467 +0,0 @@ -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; -use serde_json::{Map, Value, json}; - -use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; - -const VERTEX_DEFAULT_LOCATION: &str = "us-central1"; -const VERTEX_DEFAULT_DEEPSEEK_API_BASE: &str = "https://aiplatform.googleapis.com"; -const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY"; -const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY"; -const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT"; -const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION"; -const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION"; - -#[rustfmt::skip] -const DEEPSEEK_SUPPORTED_OCR_PARAMS: &[&str] = &[ - "stream", - "temperature", - "max_tokens", - "top_p", - "n", - "stop", -]; - -pub struct VertexAiOcrConfig; -pub struct VertexAiDeepSeekOcrConfig; - -pub const VERTEX_AI_OCR_CONFIG: VertexAiOcrConfig = VertexAiOcrConfig; -pub const VERTEX_AI_DEEPSEEK_OCR_CONFIG: VertexAiDeepSeekOcrConfig = VertexAiDeepSeekOcrConfig; - -fn string_param<'a>(params: &'a Map, keys: &[&str]) -> Option<&'a str> { - keys.iter() - .find_map(|key| params.get(*key).and_then(Value::as_str)) - .map(str::trim) - .filter(|value| !value.is_empty()) -} - -pub fn is_deepseek_model(model: &str) -> bool { - model.to_ascii_lowercase().contains("deepseek") -} - -pub fn resolve_vertex_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| env_lookup(VERTEX_AI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .or_else(|| env_lookup(VERTEXAI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| { - Error::Auth( - "Missing Vertex AI access token - pass api_key or provide Authorization via extra_headers" - .to_string(), - ) - }) -} - -fn vertex_project( - params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - string_param(params, &["vertex_project", "vertex_ai_project"]) - .map(str::to_string) - .or_else(|| env_lookup(VERTEXAI_PROJECT_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| { - Error::InvalidRequest( - "Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter" - .to_string(), - ) - }) -} - -fn vertex_location( - params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> String { - string_param(params, &["vertex_location", "vertex_ai_location"]) - .map(str::to_string) - .or_else(|| env_lookup(VERTEXAI_LOCATION_ENV).filter(|value| !value.trim().is_empty())) - .or_else(|| env_lookup(VERTEX_LOCATION_ENV).filter(|value| !value.trim().is_empty())) - .unwrap_or_else(|| VERTEX_DEFAULT_LOCATION.to_string()) -} - -fn vertex_mistral_api_base(api_base: Option<&str>, location: &str) -> String { - api_base - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .unwrap_or_else(|| format!("https://{location}-aiplatform.googleapis.com")) - .trim_end_matches('/') - .to_string() -} - -pub fn complete_vertex_mistral_url( - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let project = vertex_project(optional_params, env_lookup)?; - let location = vertex_location(optional_params, env_lookup); - let base = vertex_mistral_api_base(api_base, &location); - Ok(format!( - "{base}/v1/projects/{project}/locations/{location}/publishers/mistralai/models/{model}:rawPredict" - )) -} - -pub fn complete_vertex_deepseek_url( - api_base: Option<&str>, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let project = vertex_project(optional_params, env_lookup)?; - let location = vertex_location(optional_params, env_lookup); - let base = api_base - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(VERTEX_DEFAULT_DEEPSEEK_API_BASE) - .trim_end_matches('/'); - Ok(format!( - "{base}/v1/projects/{project}/locations/{location}/endpoints/openapi/chat/completions" - )) -} - -fn document_content_item(document: &Value) -> Result { - let object = document.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(document), - })?; - let doc_type = object - .get("type") - .and_then(Value::as_str) - .ok_or(Error::MissingField("document.type"))?; - let url_field = match doc_type { - "image_url" => "image_url", - "document_url" => "document_url", - other => { - return Err(Error::InvalidRequest(format!( - "Unsupported document type: {other}. Expected 'image_url' or 'document_url'" - ))); - } - }; - let url = object - .get(url_field) - .and_then(Value::as_str) - .filter(|value| !value.is_empty()) - .ok_or(Error::MissingField(url_field))?; - - Ok(json!({ - "type": "image_url", - "image_url": url, - })) -} - -fn deepseek_model_name(model: &str) -> String { - if model.starts_with("deepseek-ai/") { - model.to_string() - } else { - format!("deepseek-ai/{model}") - } -} - -fn first_choice_content(response: &Value) -> Result { - response - .get("choices") - .and_then(Value::as_array) - .and_then(|choices| choices.first()) - .and_then(|choice| choice.get("message")) - .and_then(|message| message.get("content")) - .cloned() - .filter(|content| match content { - Value::String(value) => !value.is_empty(), - Value::Object(_) => true, - _ => false, - }) - .ok_or_else(|| Error::InvalidResponse("No content in DeepSeek OCR response".to_string())) -} - -fn ocr_data_from_content(content: Value, usage: Option, model: &str) -> Value { - match content { - Value::String(content) => { - if content.trim_start().starts_with('{') { - serde_json::from_str(&content).unwrap_or_else(|_| { - json!({ - "pages": [{"index": 0, "markdown": content}], - "model": model, - "usage_info": usage.unwrap_or_else(|| json!({})), - }) - }) - } else { - json!({ - "pages": [{"index": 0, "markdown": content}], - "model": model, - "usage_info": usage.unwrap_or_else(|| json!({})), - }) - } - } - Value::Object(_) => content, - other => json!({ - "pages": [{"index": 0, "markdown": other.to_string()}], - "model": model, - "usage_info": usage.unwrap_or_else(|| json!({})), - }), - } -} - -impl OcrProviderConfig for VertexAiOcrConfig { - fn supported_ocr_params(&self) -> &'static [&'static str] { - MISTRAL_OCR_CONFIG.supported_ocr_params() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_vertex_mistral_url(api_base, model, optional_params, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_vertex_api_key(api_key, env_lookup) - } - - fn requires_data_uri_document(&self) -> bool { - true - } -} - -impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_ocr_params(&self) -> &'static [&'static str] { - DEEPSEEK_SUPPORTED_OCR_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params(&self, non_default_params: &Map) -> Map { - non_default_params - .iter() - .filter(|(name, _)| DEEPSEEK_SUPPORTED_OCR_PARAMS.contains(&name.as_str())) - .map(|(name, value)| (name.clone(), value.clone())) - .collect() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - let mut data = Map::new(); - data.insert( - "model".to_string(), - Value::String(deepseek_model_name(model)), - ); - data.insert( - "messages".to_string(), - json!([{"role": "user", "content": [document_content_item(&document)?]}]), - ); - for (key, value) in optional_params { - if DEEPSEEK_SUPPORTED_OCR_PARAMS.contains(&key.as_str()) { - data.insert(key, value); - } - } - Ok(OcrRequestData { - data: Value::Object(data), - files: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - let response = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - let usage = response.get("usage").cloned(); - let content = first_choice_content(&response_json)?; - let mut ocr_data = ocr_data_from_content(content.clone(), usage.clone(), model); - - if !ocr_data.get("pages").is_some_and(Value::is_array) { - ocr_data = json!({ - "pages": [{ - "index": 0, - "markdown": match content { - Value::String(value) => value, - other => other.to_string(), - } - }], - "model": ocr_data.get("model").and_then(Value::as_str).unwrap_or(model), - "usage_info": ocr_data.get("usage_info").cloned().or(usage).unwrap_or_else(|| json!({})), - }); - } - - let object = ocr_data.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&ocr_data), - })?; - let pages = object - .get("pages") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let usage_info = object - .get("usage_info") - .cloned() - .or_else(|| response.get("usage").cloned()); - Ok(LiteLLMOcrResponse { - pages, - model: object - .get("model") - .and_then(Value::as_str) - .unwrap_or(model) - .to_string(), - document_annotation: object.get("document_annotation").cloned(), - usage_info, - object: "ocr".to_string(), - extra_fields: Map::new(), - provider_native_response: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_vertex_deepseek_url(api_base, optional_params, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_vertex_api_key(api_key, env_lookup) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rstest::rstest; - - #[test] - fn vertex_mistral_url_uses_project_location_and_model() { - let params = Map::from_iter([ - ("vertex_project".to_string(), json!("proj-1")), - ("vertex_location".to_string(), json!("europe-west4")), - ]); - - let url = complete_vertex_mistral_url(None, "mistral-ocr-maas", ¶ms, &|_| None) - .expect("url builds"); - - assert_eq!( - url, - "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" - ); - } - - #[test] - fn vertex_mistral_reuses_mistral_body_transform() { - let body = VERTEX_AI_OCR_CONFIG - .transform_ocr_request( - "mistral-ocr-maas", - json!({"type": "image_url", "image_url": "data:image/png;base64,abc"}), - Map::new(), - ) - .expect("request transforms") - .data; - - assert_eq!(body["model"], "mistral-ocr-maas"); - assert_eq!(body["document"]["image_url"], "data:image/png;base64,abc"); - } - - #[test] - fn vertex_deepseek_request_uses_ocr_endpoint_shape() { - let body = VERTEX_AI_DEEPSEEK_OCR_CONFIG - .transform_ocr_request( - "deepseek-ocr-maas", - json!({"type": "document_url", "document_url": "gs://bucket/doc.pdf"}), - Map::from_iter([("temperature".to_string(), json!(0.1))]), - ) - .expect("request transforms") - .data; - - assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); - assert_eq!(body["temperature"], 0.1); - assert_eq!( - body["messages"][0]["content"][0], - json!({"type": "image_url", "image_url": "gs://bucket/doc.pdf"}) - ); - } - - #[rstest] - #[case::bare_model("deepseek-ocr-maas")] - #[case::namespaced_model("deepseek-ai/deepseek-ocr-maas")] - fn vertex_deepseek_request_uses_single_provider_namespace(#[case] model: &str) { - let body = VERTEX_AI_DEEPSEEK_OCR_CONFIG - .transform_ocr_request( - model, - json!({"type": "image_url", "image_url": "data:image/png;base64,AA=="}), - Map::new(), - ) - .expect("request transforms") - .data; - - assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); - } - - #[test] - fn vertex_deepseek_response_wraps_markdown_content() { - let response = VERTEX_AI_DEEPSEEK_OCR_CONFIG - .transform_ocr_response( - "deepseek-ocr-maas", - json!({ - "choices": [{"message": {"content": "# OCR text"}}], - "usage": {"prompt_tokens": 1} - }), - ) - .expect("response transforms"); - - assert_eq!( - response.pages, - vec![json!({"index": 0, "markdown": "# OCR text"})] - ); - assert_eq!(response.model, "deepseek-ocr-maas"); - assert_eq!(response.usage_info, Some(json!({"prompt_tokens": 1}))); - } -} diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core/src/url_utils.rs index 982dca0dbe3..1150f93a5c7 100644 --- a/litellm-rust/crates/core/src/url_utils.rs +++ b/litellm-rust/crates/core/src/url_utils.rs @@ -60,6 +60,14 @@ impl ApiUrl { } impl ApiUrl { + pub(crate) fn append_query_pairs<'a>( + mut self, + pairs: impl IntoIterator, + ) -> Self { + self.url.query_pairs_mut().extend_pairs(pairs); + self + } + pub(crate) fn into_string(self) -> String { self.url.into() } @@ -92,4 +100,19 @@ mod tests { .expect("url builds"); assert_eq!(actual, "https://example.test/v1/ocr?tenant=a"); } + + #[test] + fn appended_query_pairs_are_encoded() { + let actual = ApiUrl::parse("https://example.test") + .and_then(|url| url.complete_path(&["analyze"])) + .map(|url| { + url.append_query_pairs([("model", "name with spaces")]) + .into_string() + }) + .expect("url builds"); + assert_eq!( + actual, + "https://example.test/analyze?model=name+with+spaces" + ); + } } diff --git a/litellm-rust/crates/core/tests/azure_ai_ocr.rs b/litellm-rust/crates/core/tests/azure_ai_ocr.rs index 3d7fb2e54a3..d7d532cfef1 100644 --- a/litellm-rust/crates/core/tests/azure_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_ai_ocr.rs @@ -45,6 +45,28 @@ async fn facade_executes_azure_mistral_with_prepared_auth() { ); } +#[tokio::test] +async fn facade_acquires_supplied_entra_token_for_final_request() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let mut request = wire_request( + "azure_ai/model", + &base, + json!({"azure_ad_token":"rust-owned-token"}), + ); + request.connection.api_key = None; + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer rust-owned-token\r\n") + ); +} + struct ReplaceBodyDocument; impl OcrHooks for ReplaceBodyDocument { diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs new file mode 100644 index 00000000000..e4c81dea5a7 --- /dev/null +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -0,0 +1,395 @@ +use serde_json::{Value, json}; + +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use super::wire::{OcrWireRequest, decode_request}; + +fn query_value(url: &str, key: &str) -> Option { + url::Url::parse(url) + .unwrap() + .query_pairs() + .find_map(|(name, value)| (name == key).then(|| value.into_owned())) +} + +#[tokio::test] +async fn facade_maps_pages_features_and_url_document() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "status":"succeeded", + "analyzeResult":{"pages":[]} + }))]) + .await; + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"pages":[2,0,0,1],"features":["keyValuePairs","languages"]}), + ); + request.document = serde_json::from_value(json!({ + "type":"document_url", + "document_url":"https://example.com/document.pdf" + })) + .unwrap(); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let request = &seen.lock().unwrap()[0]; + let target = request.split_whitespace().nth(1).unwrap(); + let url = format!("{base}{target}"); + assert_eq!(query_value(&url, "pages").as_deref(), Some("1,2,3")); + assert_eq!( + query_value(&url, "features").as_deref(), + Some("keyValuePairs,languages") + ); + let body: Value = serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!( + body, + json!({"urlSource":"https://example.com/document.pdf"}) + ); +} + +#[tokio::test] +async fn rejects_invalid_pages_features_and_format() { + for options in [ + json!({"pages":[true]}), + json!({"pages":[1,"2"]}), + json!({"pages":[-1]}), + json!({"pages":"1&&features=bad"}), + json!({"features":"languages&pages=1"}), + json!({"req_format":"azure"}), + ] { + let result = decode_request(OcrWireRequest { + model: "azure_ai/doc-intelligence/prebuilt-read".into(), + document: json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + api_key: Some("key".into()), + api_base: Some("http://127.0.0.1:1".into()), + custom_llm_provider: None, + extra_headers: None, + optional_params: options.as_object().unwrap().clone(), + input_sources: Default::default(), + timeout_seconds: None, + }); + let rejected = match result { + Ok(request) => perform_ocr(request).await.is_err(), + Err(_) => true, + }; + assert!(rejected, "accepted {options}"); + } +} + +#[tokio::test] +async fn inline_document_decodes_to_base64_source() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "status":"succeeded" + }))]) + .await; + let request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let request = &seen.lock().unwrap()[0]; + let body: Value = serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body, json!({"base64Source":"YWJj"})); +} + +#[tokio::test] +async fn immediate_response_normalizes_pages_and_preserves_native() { + let operation = json!({ + "status":"succeeded", + "operationExtension":42, + "analyzeResult":{ + "content":"A\n\nB", + "tables":[{"cells":[]}], + "keyValuePairs":[{"key":{"content":"A"}}], + "pages":[{ + "pageNumber":"2", + "width":"8.5", + "height":11, + "unit":"inch", + "lines":[{"content":"A"},{"content":null},{"content":"B"}] + }] + } + }); + let (base, _, server) = mock_server(vec![MockResponse::json(operation.clone())]).await; + let result = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"req_format":"native"}), + )) + .await + .unwrap(); + server.await.unwrap(); + + assert_eq!(result.pages[0]["index"], 1); + assert_eq!(result.pages[0]["markdown"], "A\n\nB"); + assert_eq!( + result.pages[0]["dimensions"], + json!({"width":816,"height":1056,"dpi":96}) + ); + assert_eq!(result.usage_info, Some(json!({"pages_processed":1}))); + assert_eq!(result.provider_native_response, Some(operation)); +} + +#[tokio::test] +async fn accepted_response_polls_to_success_with_only_credentials() { + let operation = json!({"status":"succeeded","analyzeResult":{"pages":[]}}); + let (base, seen, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 200, + headers: vec![("Retry-After", "0".into())], + body: json!({"status":"running"}), + }, + MockResponse::json(operation.clone()), + ]) + .await; + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"req_format":"native"}), + ); + request + .connection + .extra_headers + .push(("X-Trace".into(), "initial-only".into())); + + let result = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(result.provider_native_response, Some(operation)); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests[0].to_ascii_lowercase().contains("x-trace:")); + for poll in &requests[1..] { + assert!(!poll.to_ascii_lowercase().contains("x-trace:")); + assert!( + poll.to_ascii_lowercase() + .contains("ocp-apim-subscription-key: test-key") + ); + } +} + +#[tokio::test] +async fn polling_forwards_bearer_credentials() { + let (base, seen, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse::json(json!({"status":"succeeded"})), + ]) + .await; + let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); + request.connection.api_key = None; + request.connection.extra_headers = vec![("Authorization".into(), "Bearer token".into())]; + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + assert!( + requests[1] + .to_ascii_lowercase() + .contains("authorization: bearer token") + ); +} + +#[tokio::test] +async fn polling_does_not_follow_redirects() { + let (base, seen, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 302, + headers: vec![("Location", "{base}/redirected".into())], + body: json!({}), + }, + MockResponse::json(json!({"status":"succeeded"})), + ]) + .await; + + let error = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .await + .unwrap_err(); + + assert!(error.to_string().contains("status 302"), "{error}"); + assert_eq!(seen.lock().unwrap().len(), 2); + server.abort(); +} + +#[tokio::test] +async fn polling_rejects_terminal_failure() { + let (base, _, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse::json(json!({"status":"failed"})), + ]) + .await; + + let error = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("status failed")); +} + +#[tokio::test] +async fn malformed_provider_pages_report_response_paths() { + for (analysis, path) in [ + (json!({"pages":null}), "pages"), + (json!({"pages":[null]}), "pages[0]"), + (json!({"pages":[{"lines":null}]}), "lines"), + (json!({"pages":[{"width":"bad"}]}), "width"), + ] { + let (base, _, server) = mock_server(vec![MockResponse::json(json!({ + "status":"succeeded", + "analyzeResult":analysis + }))]) + .await; + let error = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains(path), "{error}"); + } +} + +#[tokio::test] +async fn rejects_missing_invalid_and_cross_origin_operation_locations() { + for headers in [ + Vec::new(), + vec![("Operation-Location", "/relative".into())], + vec![("Operation-Location", "http://example.com/operation".into())], + vec![( + "Operation-Location", + "http://user:password@127.0.0.1/operation".into(), + )], + ] { + let (base, _, server) = mock_server(vec![MockResponse { + status: 202, + headers, + body: json!({}), + }]) + .await; + let error = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("operation-location")); + } +} + +#[tokio::test] +async fn polling_deadline_bounds_retry_delay() { + let (base, _, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 200, + headers: vec![("Retry-After", "9999".into())], + body: json!({"status":"notStarted"}), + }, + ]) + .await; + let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); + request.connection.poll_timeout = std::time::Duration::from_millis(100); + + let error = tokio::time::timeout(std::time::Duration::from_secs(1), perform_ocr(request)) + .await + .unwrap() + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("timed out")); +} + +#[tokio::test] +async fn model_id_is_encoded_and_dot_segments_are_rejected() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "status":"succeeded" + }))]) + .await; + perform_ocr(wire_request( + "azure_ai/doc-intelligence/a ?#é", + &base, + json!({}), + )) + .await + .unwrap(); + server.await.unwrap(); + assert!(seen.lock().unwrap()[0].contains("a%20%3F%23%C3%A9:analyze")); + + for model in [ + "azure_ai/doc-intelligence/.", + "azure_ai/doc-intelligence/..", + ] { + let error = perform_ocr(wire_request(model, "http://127.0.0.1:1", json!({}))) + .await + .unwrap_err(); + assert!(error.to_string().contains("dot segment")); + } +} + +#[tokio::test] +async fn pre_call_guardrail_receives_caller_pages_before_mapping() { + use crate::ocr::hooks::{OcrHookFuture, OcrHooks, OcrPreCallRequest}; + use std::sync::Arc; + + struct RewritePages; + impl OcrHooks for RewritePages { + fn has_guardrails(&self) -> bool { + true + } + + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { + assert_eq!(request.optional_params["pages"], json!([0, 2])); + Ok(OcrPreCallRequest { + optional_params: json!({"pages": [1]}), + ..request + }) + }) + } + } + let (base, seen, server) = + mock_server(vec![MockResponse::json(json!({"status": "succeeded"}))]).await; + let request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"pages": [0, 2]}), + ) + .with_host_hooks(Arc::new(RewritePages), None); + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + let target = requests[0].split_whitespace().nth(1).unwrap(); + assert_eq!( + query_value(&format!("{base}{target}"), "pages").as_deref(), + Some("2") + ); + assert_eq!(requests.len(), 1); +} diff --git a/litellm-rust/crates/core/tests/deepseek_ocr.rs b/litellm-rust/crates/core/tests/deepseek_ocr.rs new file mode 100644 index 00000000000..875fc9e3dc6 --- /dev/null +++ b/litellm-rust/crates/core/tests/deepseek_ocr.rs @@ -0,0 +1,95 @@ +use rstest::rstest; +use serde_json::{Value, json}; + +use crate::ocr::codecs::deepseek::{ + DeepSeekOcrParams, DeepSeekOcrResponse, transform_ocr_request, transform_ocr_response, +}; +use crate::ocr::types::OcrDocument; + +fn document() -> OcrDocument { + serde_json::from_value(json!({"type":"image_url","image_url":"gs://bucket/a.png"})).unwrap() +} + +#[rstest] +#[case("stream", json!(true))] +#[case("temperature", json!(0.1))] +#[case("max_tokens", json!(1024))] +#[case("top_p", json!(0.9))] +#[case("n", json!(2))] +#[case("stop", json!("done"))] +#[case("stop", json!(["done", "stop"]))] +fn request_mapping_matches_python(#[case] name: &str, #[case] value: Value) { + let params: DeepSeekOcrParams = + serde_json::from_value(json!({name: value.clone(), "ignored": true})).unwrap(); + let result = serde_json::to_value( + transform_ocr_request("deepseek-ai/deepseek-ocr-maas", document(), ¶ms).unwrap(), + ) + .unwrap(); + assert_eq!(result["model"], "deepseek-ai/deepseek-ocr-maas"); + assert_eq!( + result["messages"][0]["content"][0], + json!({"type":"image_url","image_url":"gs://bucket/a.png"}) + ); + assert_eq!(result[name], value); + assert!(result.get("ignored").is_none()); +} + +#[rstest] +#[case(json!("# hello"), "# hello")] +#[case(json!("{broken"), "{broken")] +#[case(json!(" {\"pages\":[]} "), " {\"pages\":[]} ")] +#[case(json!({"pages":[]}), "{\"pages\":[]}")] +#[case(json!({}), "{}")] +#[case(json!("[]"), "[]")] +#[case(json!("{\"pages\":[{\"markdown\":\"json text\"}]}"), "json text")] +#[case(json!({"pages":[{"markdown":"object"}]}), "object")] +fn response_codec_handles_text_json_and_objects(#[case] content: Value, #[case] expected: &str) { + let response: DeepSeekOcrResponse = serde_json::from_value( + json!({"choices":[{"message":{"content":content}}],"usage":{"prompt_tokens":1}}), + ) + .unwrap(); + let result = transform_ocr_response("model", response) + .unwrap() + .into_json(); + assert_eq!(result["pages"][0]["markdown"], expected); + assert_eq!(result["pages"][0]["index"], 0); + assert_eq!(result["usage_info"]["prompt_tokens"], 1); +} + +#[test] +fn structured_result_maps_pages_usage_model_and_annotation() { + let response: DeepSeekOcrResponse = serde_json::from_value(json!({ + "choices":[{"message":{"content":{ + "pages":[{"index":2,"markdown":"page","images":[{"id":"one"}],"dimensions":{"width":10}}], + "model":"provider-model", + "usage_info":{"pages_processed":1}, + "document_annotation":{"language":"en"}, + "future":"kept" + }}}] + })) + .unwrap(); + let result = transform_ocr_response("requested", response) + .unwrap() + .into_json(); + assert_eq!(result["pages"][0]["index"], 2); + assert_eq!(result["pages"][0]["images"][0]["id"], "one"); + assert_eq!(result["model"], "provider-model"); + assert_eq!(result["usage_info"]["pages_processed"], 1); + assert_eq!(result["document_annotation"]["language"], "en"); + assert_eq!(result["future"], "kept"); +} + +#[test] +fn response_codec_rejects_missing_empty_and_malformed_content() { + for value in [ + json!({"choices":[]}), + json!({"choices":[{"message":{"content":""}}]}), + json!({"choices":[{"message":{"content":"{\"pages\":[{\"markdown\":42}]}"}}]}), + json!({"choices":[{"message":{"content":{"pages":[{"markdown":42}]}}}]}), + ] { + let result = serde_json::from_value::(value) + .map_err(|_| ()) + .and_then(|response| transform_ocr_response("model", response).map_err(|_| ())); + assert!(result.is_err()); + } +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index d1828dfb816..cecd8869741 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -21,6 +21,7 @@ fn request_boundary_selects_mistral_and_rejects_unknown_providers() { .as_object() .unwrap() .clone(), + input_sources: Default::default(), timeout_seconds: None, }; assert!(decode_request(request).is_ok()); @@ -33,6 +34,7 @@ fn request_boundary_selects_mistral_and_rejects_unknown_providers() { custom_llm_provider: Some("unknown".into()), extra_headers: None, optional_params: serde_json::Map::new(), + input_sources: Default::default(), timeout_seconds: None, }) .is_err() diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs index 45047a0e62a..a2e67dffc7d 100644 --- a/litellm-rust/crates/core/tests/ocr/support.rs +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -30,6 +30,7 @@ pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> LiteLLMOc custom_llm_provider: None, extra_headers: None, optional_params: options.as_object().unwrap().clone(), + input_sources: Default::default(), timeout_seconds: Some(2.0), }) .unwrap() diff --git a/litellm-rust/crates/core/tests/reducto_ocr.rs b/litellm-rust/crates/core/tests/reducto_ocr.rs new file mode 100644 index 00000000000..8e86e4713ef --- /dev/null +++ b/litellm-rust/crates/core/tests/reducto_ocr.rs @@ -0,0 +1,232 @@ +use std::sync::Arc; + +use rstest::rstest; +use serde_json::{Value, json}; + +use super::hooks::{OcrDuringCallRequest, OcrHookFuture, OcrHooks}; +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; + +fn request_body(request: &str) -> Value { + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() +} + +#[rstest] +#[case( + "reducto/parse-v3", + json!({ + "formatting":{"table_output_format":"html"}, + "retrieval":{"chunk_mode":"section"}, + "settings":{"ocr_system":"standard"}, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + "reducto://already.pdf", + json!({ + "input":"reducto://already.pdf", + "formatting":{"table_output_format":"html"}, + "retrieval":{"chunk_mode":"section"}, + "settings":{"ocr_system":"standard"}, + "future_ocr_option":true, + "provider_option":"value" + }) +)] +#[case( + "reducto/parse-legacy", + json!({ + "enhance":{"agentic":[{"type":"table"}]}, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + "reducto://legacy.pdf", + json!({ + "document_url":"reducto://legacy.pdf", + "options":{"enhance":{"agentic":[{"type":"table"}]}}, + "future_ocr_option":true, + "provider_option":"value" + }) +)] +#[tokio::test] +async fn request_mapping_matches_python( + #[case] model: &str, + #[case] options: Value, + #[case] source: &str, + #[case] expected: Value, +) { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "result":{"chunks":[]} + }))]) + .await; + let mut request = wire_request(model, &base, options); + request.document = request.document.with_source(source.into()); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /parse ")); + assert_eq!(request_body(&requests[0]), expected); +} + +#[rstest] +#[case("parse-v3")] +#[case("parse-legacy")] +#[tokio::test] +async fn data_uri_upload_preserves_multipart_headers(#[case] model: &str) { + let (base, seen, server) = mock_server(vec![ + MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), + MockResponse::json(json!({"result":{"chunks":[{"content":"hello"}]}})), + ]) + .await; + let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); + request.connection.extra_headers = vec![ + ("Content-Type".into(), "application/json".into()), + ("X-Trace".into(), "upload-test".into()), + ]; + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0]["markdown"], "hello"); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with("POST /upload ")); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("content-type: multipart/form-data; boundary=") + ); + assert!(requests[0].contains("x-trace: upload-test")); + assert!(requests[0].contains("application/pdf")); + assert!(requests[0].contains("abc")); + assert!(requests[1].starts_with("POST /parse ")); +} + +#[rstest] +#[case(json!({"file_id":""}))] +#[case(json!({}))] +#[case(json!({"file_id":null}))] +#[tokio::test] +async fn invalid_upload_ids_stop_before_parse(#[case] response: Value) { + let (base, seen, server) = mock_server(vec![MockResponse::json(response)]).await; + let error = perform_ocr(wire_request("reducto/parse-v3", &base, json!({}))) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("file_id")); + assert_eq!(seen.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn upload_failure_stops_before_parse() { + let (base, seen, server) = mock_server(vec![MockResponse { + status: 503, + headers: vec![], + body: json!({"error":"unavailable"}), + }]) + .await; + assert!( + perform_ocr(wire_request("reducto/parse-v3", &base, json!({}))) + .await + .is_err() + ); + server.await.unwrap(); + assert_eq!(seen.lock().unwrap().len(), 1); +} + +#[rstest] +#[case("https://example.com/a.pdf")] +#[case("reducto://")] +#[case("data:application/pdf;base64")] +#[case("data:application/pdf;base64,INVALID!")] +#[tokio::test] +async fn rejects_invalid_document_sources_before_network(#[case] source: &str) { + let mut request = wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})); + request.document = request.document.with_source(source.into()); + assert!(perform_ocr(request).await.is_err()); +} + +#[test] +fn response_normalization_groups_blocks_and_distinguishes_null_result() { + use crate::ocr::codecs::reducto::{ReductoResponse, transform_ocr_response}; + + let raw = json!({"usage":{"num_pages":"2","credits":"3"},"result":{"chunks":[ + {"blocks":[{"content":"B","bbox":{"page":2},"kind":"table"}]}, + {"blocks":[{"content":"A","bbox":{"page":1},"kind":"text"},{"content":"C","bbox":{"page":1}}]} + ]}}); + let response: ReductoResponse = serde_json::from_value(raw).unwrap(); + let normalized = transform_ocr_response("parse-v3", response) + .unwrap() + .into_json(); + assert_eq!(normalized["pages"][0]["markdown"], "A\n\nC"); + assert_eq!(normalized["pages"][1]["markdown"], "B"); + assert_eq!(normalized["pages"][1]["blocks"][0]["kind"], "table"); + assert_eq!(normalized["usage_info"]["pages_processed"], 2); + assert_eq!(normalized["usage_info"]["credits"], 3.0); + + let missing: ReductoResponse = + serde_json::from_value(json!({"chunks":[{"content":"text"}]})).unwrap(); + let missing = transform_ocr_response("parse-v3", missing).unwrap(); + assert_eq!(missing.pages[0]["markdown"], "text"); + let null: ReductoResponse = serde_json::from_value( + json!({"result":null,"chunks":[{"content":"ignored"}],"usage":null}), + ) + .unwrap(); + let null = transform_ocr_response("parse-v3", null).unwrap(); + assert!(null.pages.is_empty()); +} + +#[tokio::test] +async fn facade_omits_native_response_by_default_and_preserves_auth_priority() { + let raw = json!({"job_id":"job-1","result":{"chunks":[]}}); + let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await; + let mut request = wire_request("reducto/parse-v3", &base, json!({})); + request.document = request.document.with_source("reducto://ready.pdf".into()); + request.connection.extra_headers = vec![("authorization".into(), "Bearer existing".into())]; + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.provider_native_response, None); + assert!( + seen.lock().unwrap()[0] + .to_ascii_lowercase() + .contains("authorization: bearer existing") + ); +} + +struct RewriteDocument; + +impl OcrHooks for RewriteDocument { + fn has_guardrails(&self) -> bool { + true + } + + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { + assert_eq!( + request.body["document_url"], + "data:application/pdf;base64,YWJj" + ); + Ok(OcrDuringCallRequest { + body: json!({"type":"document_url","document_url":"reducto://guarded.pdf"}), + ..request + }) + }) + } +} + +#[tokio::test] +async fn guardrail_rewrites_document_before_upload() { + let (base, seen, server) = + mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await; + let mut request = wire_request("reducto/parse-v3", &base, json!({})); + request.hooks = Arc::new(RewriteDocument); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /parse ")); + assert!(requests[0].contains("reducto://guarded.pdf")); +} diff --git a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs new file mode 100644 index 00000000000..6d3061d8f5d --- /dev/null +++ b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs @@ -0,0 +1,83 @@ +use serde_json::{Value, json}; + +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use crate::auth::InputSource; + +fn request_body(request: &str) -> Value { + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() +} + +#[tokio::test] +async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "choices":[{"message":{"content":"recognized"}}], + "usage":{"prompt_tokens":1} + }))]) + .await; + let mut request = wire_request( + "vertex_ai/deepseek-ocr-maas", + &base, + json!({ + "vertex_project":"project-1", + "vertex_location":"europe-west4", + "temperature":0.1, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + ); + request.document = request + .document + .with_source("gs://bucket/document.pdf".into()); + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0]["markdown"], "recognized"); + assert_eq!(response.usage_info.unwrap()["prompt_tokens"], 1); + let requests = seen.lock().unwrap(); + assert!(requests[0].starts_with( + "POST /v1/projects/project-1/locations/europe-west4/endpoints/openapi/chat/completions " + )); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key") + ); + let body = request_body(&requests[0]); + assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); + assert_eq!(body["temperature"], 0.1); + assert!(body.get("future_ocr_option").is_none()); + assert!(body.get("extra_body").is_none()); + assert_eq!( + body["messages"][0]["content"][0], + json!({"type":"document_url","document_url":"gs://bucket/document.pdf"}) + ); +} + +#[test] +fn host_registration_selects_deepseek_without_affecting_mistral() { + assert!(crate::ocr::wire::is_supported_request( + "deepseek-ocr-maas", + Some("vertex_ai") + )); + assert!(crate::ocr::wire::is_supported_request( + "mistral-ocr-maas", + Some("vertex_ai") + )); +} + +#[tokio::test] +async fn request_controlled_api_base_is_rejected_before_vertex_auth() { + let mut request = wire_request( + "vertex_ai/deepseek-ocr-maas", + "https://caller.example", + json!({"vertex_project":"project-1"}), + ); + request.connection.api_base_source = InputSource::Request; + + let error = perform_ocr(request).await.unwrap_err(); + assert!( + error + .to_string() + .contains("request-controlled Vertex AI endpoint") + ); +} diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs new file mode 100644 index 00000000000..96a19dd62b4 --- /dev/null +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -0,0 +1,161 @@ +use serde_json::{Value, json}; + +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use crate::auth::InputSource; + +fn request_body(request: &str) -> Value { + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() +} + +#[tokio::test] +async fn facade_executes_vertex_mistral_with_resolved_project_and_location() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "pages":[{"index":0,"markdown":"hello"}], + "usage_info":{"pages_processed":1} + }))]) + .await; + let request = wire_request( + "vertex_ai/mistral-ocr-maas", + &base, + json!({ + "vertex_project":"project-1", + "vertex_location":"europe-west4", + "extract_footer":true + }), + ); + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0]["markdown"], "hello"); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with( + "POST /v1/projects/project-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict " + )); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key") + ); + assert_eq!( + request_body(&requests[0]), + json!({ + "model":"mistral-ocr-maas", + "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, + "extract_footer":true + }) + ); +} + +#[tokio::test] +async fn supplied_authorization_is_forwarded_without_a_static_token() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let mut request = wire_request( + "vertex_ai/model", + &base, + json!({"vertex_project":"project-1"}), + ); + request.connection.api_key = None; + request.connection.extra_headers = vec![("authorization".into(), "Bearer supplied".into())]; + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert!( + seen.lock().unwrap()[0] + .to_ascii_lowercase() + .contains("authorization: bearer supplied") + ); +} + +#[tokio::test] +async fn invalid_credentials_fail_before_provider_http() { + let request = wire_request( + "vertex_ai/model", + "http://127.0.0.1:1", + json!({"vertex_credentials": true}), + ); + let error = perform_ocr(request).await.unwrap_err(); + assert!(error.to_string().contains("vertex_credentials")); +} + +#[tokio::test] +async fn request_controlled_api_base_is_rejected_before_vertex_auth() { + let mut request = wire_request( + "vertex_ai/mistral-ocr-maas", + "https://caller.example", + json!({"vertex_project":"project-1"}), + ); + request.connection.api_base_source = InputSource::Request; + + let error = perform_ocr(request).await.unwrap_err(); + assert!( + error + .to_string() + .contains("request-controlled Vertex AI endpoint") + ); +} + +#[tokio::test] +async fn adapters_build_complete_requests_and_share_mistral_normalization() { + use std::time::Duration; + + use crate::ocr::adapters::{MistralAdapter, OcrAdapter, VertexMistralAdapter}; + use crate::ocr::test_support::ocr_client; + + let client = ocr_client(); + let options = json!({ + "pages": [0, 2], + "include_image_base64": true, + "vertex_project": "project-1", + "vertex_location": "us-central1", + "unknown": "ignored" + }); + let direct = wire_request( + "mistral/mistral-ocr-maas", + "https://mistral.test", + options.clone(), + ); + let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options); + let direct_http = MistralAdapter + .prepare_request(&direct, &client) + .await + .unwrap(); + let vertex_http = VertexMistralAdapter + .prepare_request(&vertex, &client) + .await + .unwrap(); + assert_eq!(direct_http.url().as_str(), "https://mistral.test/v1/ocr"); + assert_eq!( + vertex_http.url().as_str(), + "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); + for http in [&direct_http, &vertex_http] { + assert_eq!(http.method(), reqwest::Method::POST); + assert_eq!(http.headers()["authorization"], "Bearer test-key"); + assert_eq!(http.headers()["content-type"], "application/json"); + assert_eq!(http.timeout(), Some(&Duration::from_secs(2))); + let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!( + body, + json!({ + "model": "mistral-ocr-maas", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "pages": [0, 2], + "include_image_base64": true + }) + ); + } + let payload = json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}); + let direct_response = MistralAdapter + .transform_ocr_response(&direct, serde_json::from_value(payload.clone()).unwrap()) + .unwrap() + .into_json(); + let vertex_response = VertexMistralAdapter + .transform_ocr_response(&vertex, serde_json::from_value(payload).unwrap()) + .unwrap() + .into_json(); + assert_eq!(direct_response, vertex_response); + assert_eq!(direct_response["model"], "mistral-ocr-maas"); + assert_eq!(direct_response["object"], "ocr"); + assert_eq!(direct_response["extra"], "preserved"); +} diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 0c30eab8112..e1f458ea0bc 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -43,7 +43,8 @@ pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { | Error::MissingField(_) | Error::MissingApiKey { .. } | Error::MissingAzureAiCredentials - | Error::MissingAzureAiCredentialsOrAdToken + | Error::MissingAzureDocumentIntelligenceCredentials + | Error::MissingReductoApiKey | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index bc51647cbad..97313651011 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -225,7 +225,7 @@ mod tests { ( "ocr", "aocr", - "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", + "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, input_sources=None, timeout_seconds=None)", ), ( "transcription", diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 2e6900f784f..c5def64c2f1 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -22,6 +22,12 @@ fn prepare_ocr( timeout_seconds: inputs.timeout_seconds, })?; let optional_params = object_or_empty("optional_params", inputs.optional_params)?; + let input_sources = inputs + .input_sources + .map(serde_json::from_value) + .transpose() + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))? + .unwrap_or_default(); Ok(async move { let RouteOptions { @@ -41,6 +47,7 @@ fn prepare_ocr( custom_llm_provider, extra_headers, optional_params, + input_sources, timeout_seconds: timeout.map(|value| value.as_secs_f64()), })?; return litellm_core::ocr::ocr(request) @@ -82,6 +89,8 @@ bridge_route! { extra_headers: Option, #[pyo3(from_py_with = litellm_python_interop::from_py)] optional_params: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + input_sources: Option, timeout_seconds: Option, }, prepare = prepare_ocr, @@ -93,13 +102,16 @@ mod tests { use litellm_core::ocr::wire::is_supported_request; #[test] - fn native_activation_excludes_unmigrated_azure_document_intelligence() { + fn native_activation_includes_migrated_providers() { assert!(is_supported_request("model", Some("mistral"))); assert!(is_supported_request("pixtral-12b", Some("azure_ai"))); - assert!(!is_supported_request( + assert!(is_supported_request( "documentintelligence/prebuilt-read", Some("azure_ai") )); - assert!(!is_supported_request("mistral-ocr", Some("vertex_ai"))); + assert!(is_supported_request("parse-v3", Some("reducto"))); + assert!(is_supported_request("parse-legacy", Some("reducto"))); + assert!(is_supported_request("mistral-ocr", Some("vertex_ai"))); + assert!(is_supported_request("deepseek-ocr", Some("vertex_ai"))); } } diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs index ee82c170d55..b4de50c5f1a 100644 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/token_counter.rs @@ -30,12 +30,17 @@ struct TokenCounter { impl TokenCounter { #[new] fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult { - let inner = release_gil(py, || CoreTokenCounter::from_json(tokenizer_json)) - .map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - encode_slots: Arc::new(Semaphore::new(encode_parallelism())), - }) + Self::load(py, || CoreTokenCounter::from_json(tokenizer_json)) + } + + #[staticmethod] + fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { + Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file)) + } + + #[staticmethod] + fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { + Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file)) } fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { @@ -58,6 +63,19 @@ impl TokenCounter { } } +impl TokenCounter { + fn load( + py: Python<'_>, + load: impl FnOnce() -> Result + Send, + ) -> PyResult { + let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; + Ok(Self { + inner: Arc::new(inner), + encode_slots: Arc::new(Semaphore::new(encode_parallelism())), + }) + } +} + fn encode_parallelism() -> usize { available_parallelism().map_or(TOKEN_COUNT_FALLBACK_PARALLELISM, NonZero::get) } @@ -70,7 +88,7 @@ fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result PyErr { let message = error.to_string(); match error { - Error::Load(_) => PyValueError::new_err(message), + Error::Load(_) | Error::Ranks(_) | Error::UnicodeClasses => PyValueError::new_err(message), Error::RequestParse(_) | Error::MissingInput | Error::FloatText diff --git a/litellm-rust/crates/token-counter/Cargo.toml b/litellm-rust/crates/token-counter/Cargo.toml index 61c9cf6e991..d0369631682 100644 --- a/litellm-rust/crates/token-counter/Cargo.toml +++ b/litellm-rust/crates/token-counter/Cargo.toml @@ -6,8 +6,10 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true indexmap = { version = "2.14.0", features = ["serde"] } itoa = "1.0" +rustc-hash = "2.1.3" serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/litellm-rust/crates/token-counter/src/byte_level.rs b/litellm-rust/crates/token-counter/src/byte_level.rs index 2b435eb39bb..ec6134a252e 100644 --- a/litellm-rust/crates/token-counter/src/byte_level.rs +++ b/litellm-rust/crates/token-counter/src/byte_level.rs @@ -12,7 +12,7 @@ use tokenizers::pre_tokenizers::PreTokenizerWrapper; use tokenizers::{Model, Tokenizer}; use unicode_normalization_alignments::{IsNormalized, UnicodeNormalization, is_nfkc_quick}; -use super::unicode_classes::UnicodeClasses; +use super::unicode_classes::{Class, UnicodeClasses, class, run_len}; const CONTRACTIONS: [&str; 7] = ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"]; @@ -110,27 +110,6 @@ fn mapped_len(piece: &str) -> usize { .count() } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Class { - Letter, - Number, - Space, - Other, -} - -fn class(character: char, unicode_classes: &UnicodeClasses) -> Class { - match character { - 'A'..='Z' | 'a'..='z' => Class::Letter, - '0'..='9' => Class::Number, - '\t'..='\r' | ' ' => Class::Space, - _ if character.is_ascii() => Class::Other, - _ if unicode_classes.is_letter(character) => Class::Letter, - _ if unicode_classes.is_number(character) => Class::Number, - _ if unicode_classes.is_space(character) => Class::Space, - _ => Class::Other, - } -} - /// The regex matches every character, so the pieces tile the text. fn pieces<'a>( text: &'a str, @@ -169,12 +148,6 @@ fn piece_len(text: &str, first: char, unicode_classes: &UnicodeClasses) -> usize } } -fn run_len(text: &str, run_class: Class, unicode_classes: &UnicodeClasses) -> usize { - text.char_indices() - .find(|(_, character)| class(*character, unicode_classes) != run_class) - .map_or(text.len(), |(index, _)| index) -} - /// `\s+(?!\S)|\s+`: whitespace followed by a non-space leaves its last /// character to start the next piece (` ?` on the following alternatives). fn space_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { diff --git a/litellm-rust/crates/token-counter/src/cl100k.rs b/litellm-rust/crates/token-counter/src/cl100k.rs new file mode 100644 index 00000000000..2b2811afc70 --- /dev/null +++ b/litellm-rust/crates/token-counter/src/cl100k.rs @@ -0,0 +1,125 @@ +//! Scanner for tiktoken's `cl100k_base` split regex, +//! `'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s`. + +use super::scanner::{contraction_len, digit_run_len, is_newline}; +use super::unicode_classes::{Class, UnicodeClasses, class, run_len}; + +/// The alternatives in regex order; the possessive quantifiers mean an +/// alternative that starts matching and runs out of input fails as a whole. +pub(super) fn piece_len(text: &str, first: char, unicode_classes: &UnicodeClasses) -> usize { + if let Some(len) = contraction_len(text) { + return len; + } + let first_class = class(first, unicode_classes); + match first_class { + Class::Letter => return run_len(text, Class::Letter, unicode_classes), + Class::Number => return digit_run_len(text, unicode_classes), + Class::Space | Class::Other => {} + } + let rest = &text[first.len_utf8()..]; + let second_class = rest + .chars() + .next() + .map(|character| class(character, unicode_classes)); + if !is_newline(first) && second_class == Some(Class::Letter) { + return first.len_utf8() + run_len(rest, Class::Letter, unicode_classes); + } + if first_class == Class::Other { + return symbol_run_len(text, unicode_classes); + } + if first == ' ' && second_class == Some(Class::Other) { + return 1 + symbol_run_len(rest, unicode_classes); + } + space_run_len(text, unicode_classes) +} + +/// `[^\s\p{L}\p{N}]++[\r\n]*+` +fn symbol_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { + let symbols = run_len(text, Class::Other, unicode_classes); + symbols + + text[symbols..] + .bytes() + .take_while(|byte| matches!(byte, b'\r' | b'\n')) + .count() +} + +/// `\s++$|\s*[\r\n]|\s+(?!\S)|\s`: whitespace to the end of the text is one +/// piece; otherwise the piece ends at the last newline of the run, or leaves +/// the run's last character for the next piece's optional leading space. +fn space_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { + let run = run_len(text, Class::Space, unicode_classes); + if run == text.len() { + return run; + } + if let Some(newline) = text[..run].rfind(['\r', '\n']) { + return newline + 1; + } + let last = text[..run].chars().next_back().map_or(0, char::len_utf8); + match run - last { + 0 => run, + shorter => shorter, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::scanner::pieces; + + fn split(text: &str) -> Vec<&str> { + pieces( + text, + piece_len, + UnicodeClasses::get().expect("Oniguruma exposes Unicode classes"), + ) + .collect() + } + + #[rstest] + #[case("", &[])] + #[case("Hello world", &["Hello", " world"])] + #[case("don't I'LL you'Ve we'RE he'd I'm", &["don", "'t", " I", "'LL", " you", "'Ve", " we", "'RE", " he", "'d", " I", "'m"])] + #[case("IT'SOK it'Dbe 'Sx 'Tx", &["IT", "'S", "OK", " it", "'D", "be", " '", "Sx", " '", "Tx"])] + #[case("'Sx'Tx'Mx'LLx'VEx'REx'Dx", &["'S", "x", "'T", "x", "'M", "x", "'LL", "x", "'VE", "x", "'RE", "x", "'D", "x"])] + #[case("'ſ 'lx", &["'ſ", " '", "lx"])] + #[case("12345 6", &["123", "45", " ", "6"])] + #[case("!abc !!abc", &["!abc", " !!", "abc"])] + #[case(" !!!\r\n\r\nx", &[" !!!\r\n\r\n", "x"])] + #[case("a b \n\n c", &["a", " ", " b", " \n\n", " ", " c"])] + #[case("a\nb\r\nc\n\nd \n e", &["a", "\n", "b", "\r\n", "c", "\n\n", "d", " \n", " e"])] + #[case("x \t\n \t y\n", &["x", " \t\n", " \t", " y", "\n"])] + #[case("end ", &["end", " "])] + #[case("\u{a0}abc\u{a0}!", &["\u{a0}abc", "\u{a0}", "!"])] + #[case("<|endoftext|>", &["<|", "endoftext", "|>"])] + #[case("e\u{301}a", &["e", "\u{301}a"])] + #[case("日本語 ١٢٣٤", &["日本語", " ", "١٢٣", "٤"])] + fn scanner_splits_like_the_regex(#[case] text: &str, #[case] expected: &[&str]) { + assert_eq!(split(text), expected); + } + + #[derive(serde::Deserialize)] + struct TextFixture { + text: String, + pieces: Vec, + } + + #[test] + fn scanner_splits_the_fixture_corpus_like_tiktoken_regex() { + let fixtures: Vec = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/cl100k/texts.jsonl" + )) + .lines() + .map(|line| serde_json::from_str(line).expect("fixture line is json")) + .collect(); + assert!(fixtures.len() > 3000); + let mismatches: Vec<_> = fixtures + .iter() + .filter(|fixture| split(&fixture.text) != fixture.pieces) + .map(|fixture| (&fixture.text, split(&fixture.text), &fixture.pieces)) + .collect(); + assert!(mismatches.is_empty(), "{mismatches:#?}"); + } +} diff --git a/litellm-rust/crates/token-counter/src/counter.rs b/litellm-rust/crates/token-counter/src/counter.rs index a3943515a64..7eedc449dd1 100644 --- a/litellm-rust/crates/token-counter/src/counter.rs +++ b/litellm-rust/crates/token-counter/src/counter.rs @@ -3,6 +3,7 @@ use serde::Serialize; use crate::Error; use crate::byte_level::ByteLevelCounter; use crate::python_json; +use crate::scanner::{SplitPattern, TiktokenCounter}; use crate::tools::format_function_definitions; use crate::types::{ ContentBlock, ContentItem, CountableRequest, Message, MessageContent, TextValue, ToolChoice, @@ -23,12 +24,19 @@ pub struct InputTokenCount { pub input_tokens: usize, } -/// A loaded HuggingFace tokenizer plus the message accounting Python applies on -/// top of it. Encoding is CPU-bound and synchronous; hosts run it off their -/// event loop. +enum Encoder { + HuggingFace { + tokenizer: Box, + byte_level: Option, + }, + Tiktoken(TiktokenCounter), +} + +/// A loaded tokenizer plus the message accounting Python applies on top of +/// it. Encoding is CPU-bound and synchronous; hosts run it off their event +/// loop. pub struct TokenCounter { - tokenizer: tokenizers::Tokenizer, - byte_level: Option, + encoder: Encoder, } impl TokenCounter { @@ -39,23 +47,50 @@ impl TokenCounter { .map_err(Error::Load)?; let byte_level = ByteLevelCounter::detect(&tokenizer); Ok(Self { - tokenizer, - byte_level, + encoder: Encoder::HuggingFace { + tokenizer: Box::new(tokenizer), + byte_level, + }, + }) + } + + /// Load tiktoken's `cl100k_base` rank file (`base64(token) rank` lines). + /// The host reads the file. + pub fn from_cl100k_ranks(rank_file: &str) -> Result { + Self::from_tiktoken_ranks(SplitPattern::Cl100k, rank_file) + } + + /// Load tiktoken's `o200k_base` rank file (`base64(token) rank` lines). + /// The host reads the file. + pub fn from_o200k_ranks(rank_file: &str) -> Result { + Self::from_tiktoken_ranks(SplitPattern::O200k, rank_file) + } + + fn from_tiktoken_ranks(split: SplitPattern, rank_file: &str) -> Result { + Ok(Self { + encoder: Encoder::Tiktoken(TiktokenCounter::from_ranks(split, rank_file)?), }) } pub fn count_text(&self, text: &str) -> Result { - if let Some(count) = self - .byte_level - .as_ref() - .and_then(|counter| counter.count(&self.tokenizer, text)) - { - return Ok(count); + match &self.encoder { + Encoder::Tiktoken(counter) => Ok(counter.count(text)), + Encoder::HuggingFace { + tokenizer, + byte_level, + } => { + if let Some(count) = byte_level + .as_ref() + .and_then(|counter| counter.count(tokenizer, text)) + { + return Ok(count); + } + tokenizer + .encode_fast(text, true) + .map(|encoding| encoding.len()) + .map_err(Error::Encode) + } } - self.tokenizer - .encode_fast(text, true) - .map(|encoding| encoding.len()) - .map_err(Error::Encode) } /// Mirrors the host's key precedence: `messages`, then `prompt`, then diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index dc590093f64..6b8668fe182 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -6,6 +6,10 @@ use thiserror::Error as ThisError; pub enum Error { #[error("failed to load tokenizer: {0}")] Load(#[source] tokenizers::Error), + #[error("failed to load tokenizer: tiktoken rank file: {0}")] + Ranks(String), + #[error("failed to load tokenizer: Unicode character classes are unavailable")] + UnicodeClasses, #[error("unsupported by the rust token counter: request body could not be parsed: {0}")] RequestParse(#[source] serde_json::Error), #[error("unsupported by the rust token counter: request has no countable input")] diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index eb602b3cade..fa0014e2bad 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -5,9 +5,13 @@ #![forbid(unsafe_code)] mod byte_level; +mod cl100k; mod counter; mod error; +mod o200k; mod python_json; +mod scanner; +mod tiktoken; mod tools; mod types; mod unicode_classes; diff --git a/litellm-rust/crates/token-counter/src/o200k.rs b/litellm-rust/crates/token-counter/src/o200k.rs new file mode 100644 index 00000000000..c0d85c45e78 --- /dev/null +++ b/litellm-rust/crates/token-counter/src/o200k.rs @@ -0,0 +1,217 @@ +//! Scanner for tiktoken's `o200k_base` split regex, +//! `[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`. +//! tiktoken runs it with a backtracking engine, so the letter alternatives +//! below reproduce where the greedy quantifiers settle, not only what the +//! classes say. + +use super::scanner::{contraction_len, digit_run_len, is_newline}; +use super::unicode_classes::{Case, Class, UnicodeClasses, case, case_run_len, class, run_len}; + +/// The alternatives in regex order: a number is never a letter piece, a +/// letter always is, and only whitespace and symbols reach the last three. +pub(super) fn piece_len(text: &str, first: char, unicode_classes: &UnicodeClasses) -> usize { + let first_class = class(first, unicode_classes); + if first_class == Class::Number { + return digit_run_len(text, unicode_classes); + } + if let Some(len) = letter_piece_len(text, first, first_class, unicode_classes) { + return len; + } + if first_class == Class::Other { + return symbol_run_len(text, unicode_classes); + } + let rest = &text[first.len_utf8()..]; + if first == ' ' + && rest + .chars() + .next() + .is_some_and(|character| class(character, unicode_classes) == Class::Other) + { + return 1 + symbol_run_len(rest, unicode_classes); + } + space_run_len(text, unicode_classes) +} + +/// The two letter alternatives, each first with then without the optional +/// `[^\r\n\p{L}\p{N}]` prefix: the order the engine tries them in. +fn letter_piece_len( + text: &str, + first: char, + first_class: Class, + unicode_classes: &UnicodeClasses, +) -> Option { + let prefix = (!is_newline(first) && matches!(first_class, Class::Space | Class::Other)) + .then(|| first.len_utf8()); + let after_prefix = |shape: fn(&str, &UnicodeClasses) -> Option| { + prefix.and_then(|prefix| shape(&text[prefix..], unicode_classes).map(|len| prefix + len)) + }; + after_prefix(upper_then_lower_len) + .or_else(|| upper_then_lower_len(text, unicode_classes)) + .or_else(|| after_prefix(upper_run_len)) + .or_else(|| upper_run_len(text, unicode_classes)) +} + +/// `[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?`. +/// The upper run is greedy; when no lower character follows it, the engine +/// gives characters back until the one it just gave back is lower too, and +/// that single character is the lower run. +fn upper_then_lower_len(text: &str, unicode_classes: &UnicodeClasses) -> Option { + let upper = case_run_len(text, Case::is_upper, unicode_classes); + let lower = case_run_len(&text[upper..], Case::is_lower, unicode_classes); + let letters = if lower > 0 { + upper + lower + } else { + let (index, last_both) = text[..upper] + .char_indices() + .rev() + .find(|(_, character)| case(*character, unicode_classes).is_lower())?; + index + last_both.len_utf8() + }; + Some(letters + contraction_len(&text[letters..]).unwrap_or(0)) +} + +/// `[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?` +fn upper_run_len(text: &str, unicode_classes: &UnicodeClasses) -> Option { + let upper = case_run_len(text, Case::is_upper, unicode_classes); + if upper == 0 { + return None; + } + let letters = upper + case_run_len(&text[upper..], Case::is_lower, unicode_classes); + Some(letters + contraction_len(&text[letters..]).unwrap_or(0)) +} + +/// `[^\s\p{L}\p{N}]+[\r\n/]*` +fn symbol_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { + let symbols = run_len(text, Class::Other, unicode_classes); + symbols + + text[symbols..] + .bytes() + .take_while(|byte| matches!(byte, b'\r' | b'\n' | b'/')) + .count() +} + +/// `\s*[\r\n]+|\s+(?!\S)|\s+`: a run with a newline ends at its last newline, +/// even at the end of the text; otherwise whitespace to the end of the text +/// is one piece, or the run leaves its last character for the next piece's +/// optional leading space. +fn space_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { + let run = run_len(text, Class::Space, unicode_classes); + if let Some(newline) = text[..run].rfind(['\r', '\n']) { + return newline + 1; + } + if run == text.len() { + return run; + } + let last = text[..run].chars().next_back().map_or(0, char::len_utf8); + match run - last { + 0 => run, + shorter => shorter, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use tokenizers::utils::SysRegex; + + use super::*; + use crate::scanner::pieces; + + fn split(text: &str) -> Vec<&str> { + pieces( + text, + piece_len, + UnicodeClasses::get().expect("Oniguruma exposes Unicode classes"), + ) + .collect() + } + + #[rstest] + #[case("", &[])] + #[case("Hello world", &["Hello", " world"])] + #[case("camelCase PascalCase ABCdef ABCdeF ABC", &["camel", "Case", " Pascal", "Case", " ABCdef", " ABCde", "F", " ABC"])] + #[case("日本ABC ABC日本 日本語abc abc日本語", &["日本", "ABC", " ABC日本", " 日本語abc", " abc日本語"])] + #[case("\u{301}ABC \u{301}abc \u{301}\u{301}A A\u{301}\u{301} E\u{301}A aE\u{301}", &["\u{301}", "ABC", " \u{301}abc", " \u{301}\u{301}", "A", " A\u{301}\u{301}", " E\u{301}", "A", " a", "E\u{301}"])] + #[case("ᵃbc ᵃBC Aᵃbc Aᵃ ᵃ' ᵃ's", &["ᵃbc", " ᵃ", "BC", " Aᵃbc", " Aᵃ", " ᵃ", "'", " ᵃ's"])] + #[case("Džungla aDžB ADžB ADžb", &["Džungla", " a", "DžB", " ADžB", " ADžb"])] + #[case("don'tx ABC's abc'S abc'ſ ABC'ſx IT'SOK it'Dbe", &["don't", "x", " ABC's", " abc'S", " abc'ſ", " ABC'ſ", "x", " IT'S", "OK", " it'D", "be"])] + #[case("'sabc x's 's 'Sx'Tx 9'9 a'9 ' s", &["'sabc", " x's", " '", "s", " '", "Sx'T", "x", " ", "9", "'", "9", " a", "'", "9", " '", " s"])] + #[case("!ABC !AbC !!abc !!\u{301}a \u{a0}\u{301}A", &["!ABC", " !", "Ab", "C", " !!", "abc", " !!\u{301}", "a", " ", "\u{a0}\u{301}", "A"])] + #[case("!!/\n/x a/b !!\n/x /x //", &["!!/\n/", "x", " a", "/b", " !!\n/", "x", " ", " /", "x", " ", " //"])] + #[case("12345 6 1abc abc1", &["123", "45", " ", "6", " ", "1", "abc", " abc", "1"])] + #[case("x \n x \r\n \r\n y", &["x", " \n", " x", " \r\n \r\n", " y"])] + #[case("x \n ", &["x", " \n", " "])] + #[case("a b \n\n c", &["a", " ", " b", " \n\n", " ", " c"])] + #[case("x\t\ty x\t\t", &["x", "\t", "\ty", " x", "\t\t"])] + #[case("end ", &["end", " "])] + #[case("\u{a0}abc\u{a0}!", &["\u{a0}abc", "\u{a0}", "!"])] + #[case("<|endoftext|>", &["<|", "endoftext", "|>"])] + #[case("İstanbul ΣΊΣΥΦΟΣ Ελληνικά Русский", &["İstanbul", " ΣΊΣΥΦΟΣ", " Ελληνικά", " Русский"])] + #[case("日本語 ١٢٣٤", &["日本語", " ", "١٢٣", "٤"])] + fn scanner_splits_like_the_regex(#[case] text: &str, #[case] expected: &[&str]) { + assert_eq!(split(text), expected); + } + + #[test] + fn every_scalar_alone_is_one_piece() { + for character in (0..=0x10FFFFu32).filter_map(char::from_u32) { + let text = character.to_string(); + assert_eq!( + split(&text), + [text.as_str()], + "U+{:04X}", + u32::from(character) + ); + } + } + + #[test] + fn cases_match_oniguruma() { + let unicode_classes = UnicodeClasses::get().expect("Oniguruma exposes Unicode classes"); + let upper = SysRegex::new(r"[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]").expect("regex"); + let lower = SysRegex::new(r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]").expect("regex"); + let whole = + |regex: &SysRegex, text: &str| regex.find_iter(text).next() == Some((0, text.len())); + let mut text = String::new(); + for character in (0..=0x10FFFFu32).filter_map(char::from_u32) { + text.clear(); + text.push(character); + let expected = match (whole(&upper, &text), whole(&lower, &text)) { + (true, true) => Case::Both, + (true, false) => Case::Upper, + (false, true) => Case::Lower, + (false, false) => Case::Neither, + }; + assert_eq!( + case(character, unicode_classes), + expected, + "U+{:04X}", + u32::from(character) + ); + } + } + + #[derive(serde::Deserialize)] + struct TextFixture { + text: String, + pieces: Vec, + } + + #[test] + fn scanner_splits_the_fixture_corpus_like_tiktoken_regex() { + let fixtures: Vec = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/o200k/texts.jsonl" + )) + .lines() + .map(|line| serde_json::from_str(line).expect("fixture line is json")) + .collect(); + assert!(fixtures.len() > 3000); + let mismatches: Vec<_> = fixtures + .iter() + .filter(|fixture| split(&fixture.text) != fixture.pieces) + .map(|fixture| (&fixture.text, split(&fixture.text), &fixture.pieces)) + .collect(); + assert!(mismatches.is_empty(), "{mismatches:#?}"); + } +} diff --git a/litellm-rust/crates/token-counter/src/scanner.rs b/litellm-rust/crates/token-counter/src/scanner.rs new file mode 100644 index 00000000000..c2c3057aeeb --- /dev/null +++ b/litellm-rust/crates/token-counter/src/scanner.rs @@ -0,0 +1,107 @@ +//! Exact token counting for tiktoken encodings. A hand-written scanner +//! reproduces the piece boundaries of the encoding's split regex, and each +//! piece is merged with the rank file. Special tokens are ordinary text, as +//! with `encode(text, disallowed_special=())`. + +use std::iter; + +use super::tiktoken::{MergeRanks, MergeScratch}; +use super::unicode_classes::{Class, UnicodeClasses, class}; +use super::{cl100k, o200k}; +use crate::Error; + +const MAX_DIGITS_PER_PIECE: usize = 3; + +/// Byte length of the piece the split regex matches at the start of the +/// text, given the text's first character. +pub(super) type PieceLen = fn(&str, char, &UnicodeClasses) -> usize; + +#[derive(Clone, Copy, Debug)] +pub(super) enum SplitPattern { + Cl100k, + O200k, +} + +impl SplitPattern { + fn piece_len(self) -> PieceLen { + match self { + Self::Cl100k => cl100k::piece_len, + Self::O200k => o200k::piece_len, + } + } +} + +pub(super) struct TiktokenCounter { + ranks: MergeRanks, + piece_len: PieceLen, + unicode_classes: &'static UnicodeClasses, +} + +impl TiktokenCounter { + pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result { + Ok(Self { + ranks: MergeRanks::parse(rank_file)?, + piece_len: split.piece_len(), + unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?, + }) + } + + pub(super) fn count(&self, text: &str) -> usize { + let mut scratch = MergeScratch::default(); + pieces(text, self.piece_len, self.unicode_classes) + .map(|piece| self.ranks.count_piece(piece.as_bytes(), &mut scratch)) + .sum() + } +} + +/// The regex matches every character, so the pieces tile the text. +pub(super) fn pieces<'a>( + text: &'a str, + piece_len: PieceLen, + unicode_classes: &'static UnicodeClasses, +) -> impl Iterator { + iter::successors( + split_piece(text, piece_len, unicode_classes), + move |(_, rest)| split_piece(rest, piece_len, unicode_classes), + ) + .map(|(piece, _)| piece) +} + +fn split_piece<'a>( + text: &'a str, + piece_len: PieceLen, + unicode_classes: &UnicodeClasses, +) -> Option<(&'a str, &'a str)> { + let first = text.chars().next()?; + Some(text.split_at(piece_len(text, first, unicode_classes))) +} + +/// `'(?i:s|t|re|ve|m|ll|d)`, the contraction both encodings spell out. Simple +/// case folding also maps U+017F (long s) onto `s`. +pub(super) fn contraction_len(text: &str) -> Option { + let mut characters = text.chars(); + if characters.next()? != '\'' { + return None; + } + let first = characters.next()?; + let len = match first { + 's' | 'S' | '\u{17F}' | 'd' | 'D' | 'm' | 'M' | 't' | 'T' => first.len_utf8(), + 'l' | 'L' => matches!(characters.next(), Some('l' | 'L')).then_some(2)?, + 'v' | 'V' | 'r' | 'R' => matches!(characters.next(), Some('e' | 'E')).then_some(2)?, + _ => return None, + }; + Some(1 + len) +} + +pub(super) fn is_newline(character: char) -> bool { + matches!(character, '\r' | '\n') +} + +/// `\p{N}{1,3}` +pub(super) fn digit_run_len(text: &str, unicode_classes: &UnicodeClasses) -> usize { + text.chars() + .take(MAX_DIGITS_PER_PIECE) + .take_while(|character| class(*character, unicode_classes) == Class::Number) + .map(char::len_utf8) + .sum() +} diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs new file mode 100644 index 00000000000..c479ae01be9 --- /dev/null +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -0,0 +1,215 @@ +//! tiktoken's byte-level BPE: a rank file of `base64(token) rank` lines and +//! the merge loop that turns one regex piece into tokens. The merge order is +//! tiktoken's (lowest rank first, leftmost pair on ties) so the token count is +//! identical, but pairs are tracked in a heap so a long piece costs +//! `O(n log n)` instead of tiktoken's `O(n^2)`. + +use std::cmp::Reverse; +use std::collections::BinaryHeap; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use rustc_hash::FxHashMap; + +use crate::Error; + +type Rank = u32; + +const NO_RANK: Rank = Rank::MAX; +const END: usize = usize::MAX; + +pub(super) struct MergeRanks(FxHashMap, Rank>); + +impl MergeRanks { + pub(super) fn parse(text: &str) -> Result { + let ranks = text + .lines() + .filter(|line| !line.is_empty()) + .map(parse_line) + .collect::, _>>()?; + if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) { + return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token"))); + } + Ok(Self(ranks)) + } + + fn rank(&self, bytes: &[u8]) -> Rank { + self.0.get(bytes).copied().unwrap_or(NO_RANK) + } + + /// Token count of one regex piece, as `encode_ordinary` would produce. + pub(super) fn count_piece(&self, piece: &[u8], scratch: &mut MergeScratch) -> usize { + if piece.len() < 2 || self.0.contains_key(piece) { + return 1; + } + scratch.reset(piece.len()); + for start in 0..piece.len() - 1 { + scratch.set_rank(start, self.rank(&piece[start..start + 2])); + } + let mut parts = piece.len(); + while let Some(Reverse((rank, start))) = scratch.heap.pop() { + if scratch.next[start] == END || scratch.rank[start] != rank { + continue; + } + let merged = scratch.next[start]; + let after = scratch.next[merged]; + scratch.next[merged] = END; + scratch.next[start] = after; + parts -= 1; + if after < piece.len() { + scratch.prev[after] = start; + scratch.set_rank(start, self.rank(&piece[start..scratch.end(after)])); + } else { + scratch.rank[start] = NO_RANK; + } + let before = scratch.prev[start]; + if before != END { + scratch.set_rank(before, self.rank(&piece[before..scratch.end(start)])); + } + } + parts + } +} + +fn parse_line(line: &str) -> Result<(Box<[u8]>, Rank), Error> { + let (token, rank) = line + .split_once(' ') + .ok_or_else(|| Error::Ranks(format!("line without a rank: {line:?}")))?; + let bytes = STANDARD + .decode(token) + .map_err(|error| Error::Ranks(format!("token is not base64: {error}")))?; + let rank = rank + .parse() + .map_err(|error| Error::Ranks(format!("rank is not an integer: {error}")))?; + Ok((bytes.into_boxed_slice(), rank)) +} + +/// Buffers reused across the pieces of one text. Parts are addressed by the +/// byte offset they start at, which also gives the leftmost-pair tie break. +#[derive(Default)] +pub(super) struct MergeScratch { + next: Vec, + prev: Vec, + rank: Vec, + heap: BinaryHeap>, +} + +impl MergeScratch { + fn reset(&mut self, len: usize) { + self.next.clear(); + self.next.extend(1..=len); + self.prev.clear(); + self.prev.push(END); + self.prev.extend(0..len - 1); + self.rank.clear(); + self.rank.resize(len, NO_RANK); + self.heap.clear(); + } + + fn end(&self, start: usize) -> usize { + self.next[start] + } + + fn set_rank(&mut self, start: usize, rank: Rank) { + self.rank[start] = rank; + if rank != NO_RANK { + self.heap.push(Reverse((rank, start))); + } + } +} + +#[cfg(test)] +mod tests { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + use super::*; + + fn ranks() -> MergeRanks { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4" + ); + MergeRanks::parse(&std::fs::read_to_string(path).expect("cl100k rank file is in the repo")) + .expect("rank file parses") + } + + /// tiktoken's `_byte_pair_merge`, transcribed, as the reference. + fn reference_count(ranks: &MergeRanks, piece: &[u8]) -> usize { + if piece.len() < 2 || ranks.0.contains_key(piece) { + return 1; + } + let mut parts: Vec<(usize, Rank)> = (0..piece.len() - 1) + .map(|index| (index, ranks.rank(&piece[index..index + 2]))) + .chain([(piece.len() - 1, NO_RANK), (piece.len(), NO_RANK)]) + .collect(); + let get_rank = |parts: &[(usize, Rank)], index: usize| { + if index + 3 < parts.len() { + ranks.rank(&piece[parts[index].0..parts[index + 3].0]) + } else { + NO_RANK + } + }; + loop { + let Some(index) = parts[..parts.len() - 1] + .iter() + .enumerate() + .filter(|(_, (_, rank))| *rank != NO_RANK) + .min_by_key(|(index, (_, rank))| (*rank, *index)) + .map(|(index, _)| index) + else { + return parts.len() - 1; + }; + if index > 0 { + parts[index - 1].1 = get_rank(&parts, index - 1); + } + parts[index].1 = get_rank(&parts, index); + parts.remove(index + 1); + } + } + + #[test] + fn every_byte_is_a_token() { + let ranks = ranks(); + assert_eq!(ranks.0.len(), 100_256); + assert!((0..=u8::MAX).all(|byte| ranks.rank(&[byte]) != NO_RANK)); + } + + #[test] + fn heap_merge_matches_tiktokens_merge_loop() { + let ranks = ranks(); + let mut scratch = MergeScratch::default(); + let mut rng = StdRng::seed_from_u64(99); + let alphabet = b" abcdeorstn.,'\n\xc3\xa9\xe2\x82\xac0123"; + for _ in 0..20_000 { + let piece: Vec = (0..rng.gen_range(1..24)) + .map(|_| alphabet[rng.gen_range(0..alphabet.len())]) + .collect(); + assert_eq!( + ranks.count_piece(&piece, &mut scratch), + reference_count(&ranks, &piece), + "piece {:?}", + String::from_utf8_lossy(&piece) + ); + } + } + + #[test] + fn long_repeated_runs_stay_cheap() { + let ranks = ranks(); + let mut scratch = MergeScratch::default(); + let piece = vec![b' '; 1 << 20]; + let started = std::time::Instant::now(); + let count = ranks.count_piece(&piece, &mut scratch); + assert!(count > 0); + assert!(started.elapsed().as_secs() < 5, "{:?}", started.elapsed()); + } + + #[test] + fn malformed_rank_files_are_rejected() { + assert!(MergeRanks::parse("IQ==").is_err()); + assert!(MergeRanks::parse("IQ== x").is_err()); + assert!(MergeRanks::parse("!!! 1").is_err()); + assert!(MergeRanks::parse("IQ== 1").is_err()); + } +} diff --git a/litellm-rust/crates/token-counter/src/unicode_classes.rs b/litellm-rust/crates/token-counter/src/unicode_classes.rs index 405cee34949..6aa34047adf 100644 --- a/litellm-rust/crates/token-counter/src/unicode_classes.rs +++ b/litellm-rust/crates/token-counter/src/unicode_classes.rs @@ -9,6 +9,8 @@ pub(super) struct UnicodeClasses { letters: Ranges, numbers: Ranges, spaces: Ranges, + uppers: Ranges, + lowers: Ranges, } static CLASSES: LazyLock> = LazyLock::new(|| { @@ -19,6 +21,8 @@ static CLASSES: LazyLock> = LazyLock::new(|| { letters: Ranges::load(r"\p{L}+", &scalars)?, numbers: Ranges::load(r"\p{N}+", &scalars)?, spaces: Ranges::load(r"\s+", &scalars)?, + uppers: Ranges::load(r"[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+", &scalars)?, + lowers: Ranges::load(r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]+", &scalars)?, }) }); @@ -59,15 +63,102 @@ impl UnicodeClasses { CLASSES.as_ref() } - pub(super) fn is_letter(&self, character: char) -> bool { + fn is_letter(&self, character: char) -> bool { self.letters.contains(character) } - pub(super) fn is_number(&self, character: char) -> bool { + fn is_number(&self, character: char) -> bool { self.numbers.contains(character) } - pub(super) fn is_space(&self, character: char) -> bool { + fn is_space(&self, character: char) -> bool { self.spaces.contains(character) } + + fn is_upper(&self, character: char) -> bool { + self.uppers.contains(character) + } + + fn is_lower(&self, character: char) -> bool { + self.lowers.contains(character) + } +} + +/// `\p{L}`, `\p{N}`, `\s` and everything else, the character classes the +/// split regexes are written in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Class { + Letter, + Number, + Space, + Other, +} + +pub(super) fn class(character: char, unicode_classes: &UnicodeClasses) -> Class { + match character { + 'A'..='Z' | 'a'..='z' => Class::Letter, + '0'..='9' => Class::Number, + '\t'..='\r' | ' ' => Class::Space, + _ if character.is_ascii() => Class::Other, + _ if unicode_classes.is_letter(character) => Class::Letter, + _ if unicode_classes.is_number(character) => Class::Number, + _ if unicode_classes.is_space(character) => Class::Space, + _ => Class::Other, + } +} + +/// Byte length of the leading run of `run_class` characters. +pub(super) fn run_len(text: &str, run_class: Class, unicode_classes: &UnicodeClasses) -> usize { + text.char_indices() + .find(|(_, character)| class(*character, unicode_classes) != run_class) + .map_or(text.len(), |(index, _)| index) +} + +/// Membership in the two letter classes of the o200k split regex, +/// `[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]` and `[\p{Ll}\p{Lm}\p{Lo}\p{M}]`; `Lm`, +/// `Lo` and `M` are in both. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Case { + Upper, + Lower, + Both, + Neither, +} + +impl Case { + pub(super) fn is_upper(self) -> bool { + matches!(self, Case::Upper | Case::Both) + } + + pub(super) fn is_lower(self) -> bool { + matches!(self, Case::Lower | Case::Both) + } +} + +pub(super) fn case(character: char, unicode_classes: &UnicodeClasses) -> Case { + match character { + 'A'..='Z' => Case::Upper, + 'a'..='z' => Case::Lower, + _ if character.is_ascii() => Case::Neither, + _ => match ( + unicode_classes.is_upper(character), + unicode_classes.is_lower(character), + ) { + (true, true) => Case::Both, + (true, false) => Case::Upper, + (false, true) => Case::Lower, + (false, false) => Case::Neither, + }, + } +} + +/// Byte length of the leading run of characters whose case passes `in_class`. +pub(super) fn case_run_len( + text: &str, + in_class: fn(Case) -> bool, + unicode_classes: &UnicodeClasses, +) -> usize { + text.char_indices() + .find(|(_, character)| !in_class(case(*character, unicode_classes))) + .map_or(text.len(), |(index, _)| index) } diff --git a/litellm-rust/crates/token-counter/tests/fixtures/cl100k/requests.jsonl b/litellm-rust/crates/token-counter/tests/fixtures/cl100k/requests.jsonl new file mode 100644 index 00000000000..c2a7b718cc3 --- /dev/null +++ b/litellm-rust/crates/token-counter/tests/fixtures/cl100k/requests.jsonl @@ -0,0 +1,10 @@ +{"body": "{\"model\": \"gpt-4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello, how are you today?\"}]}", "input_tokens": 14} +{"body": "{\"model\": \"gpt-4\", \"messages\": [{\"role\": \"system\", \"content\": \"You are a terse assistant.\"}, {\"role\": \"user\", \"name\": \"alice\", \"content\": [{\"type\": \"text\", \"text\": \"Summarise this paragraph about ships and harbours.\"}, \"plain string item\"]}, {\"role\": \"assistant\", \"content\": [{\"type\": \"text\", \"text\": \"Sure.\"}]}]}", "input_tokens": 39} +{"body": "{\"model\": \"gpt-4\", \"messages\": [{\"role\": \"user\", \"content\": \"weather?\"}], \"tools\": [{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get weather\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"City name\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}, \"days\": {\"type\": \"integer\"}, \"tags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, \"opts\": {\"type\": \"object\", \"properties\": {\"verbose\": {\"type\": \"boolean\"}, \"level\": {\"type\": \"integer\", \"enum\": [1, 2]}}, \"required\": [\"verbose\"]}, \"anything\": {}}, \"required\": [\"location\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"noop\"}}], \"tool_choice\": {\"type\": \"function\", \"function\": {\"name\": \"get_weather\"}}}", "input_tokens": 106} +{"body": "{\"model\": \"gpt-4\", \"messages\": [{\"role\": \"system\", \"content\": \"sys\"}, {\"role\": \"user\", \"content\": \"weather?\"}], \"tools\": [{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get weather\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"City name\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}, \"days\": {\"type\": \"integer\"}, \"tags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, \"opts\": {\"type\": \"object\", \"properties\": {\"verbose\": {\"type\": \"boolean\"}, \"level\": {\"type\": \"integer\", \"enum\": [1, 2]}}, \"required\": [\"verbose\"]}, \"anything\": {}}, \"required\": [\"location\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"noop\"}}], \"tool_choice\": \"none\"}", "input_tokens": 99} +{"body": "{\"model\": \"gpt-4\", \"prompt\": \"Write a haiku about ships.\"}", "input_tokens": 7} +{"body": "{\"model\": \"gpt-4\", \"prompt\": [\"first prompt\", \"second prompt\"]}", "input_tokens": 4} +{"body": "{\"model\": \"gpt-4\", \"input\": [{\"role\": \"user\", \"content\": [{\"type\": \"input_text\", \"text\": \"Summarise caf\\u00e9 menus, na\\u00efve \\u2014 ok? \\\"quoted\\\"\\n\"}]}, {\"role\": \"assistant\", \"content\": \"Sure.\"}], \"instructions\": \"be terse\"}", "input_tokens": 60} +{"body": "{\"model\": \"gpt-4\", \"input\": [[101, 2023, 5], [7]], \"encoding_format\": \"float\"}", "input_tokens": 5} +{"body": "{\"model\": \"gpt-4\", \"query\": \"best harbour\", \"documents\": [\"doc one\", {\"text\": \"doc two\", \"title\": \"T\", \"n\": 3, \"ok\": true, \"none\": null, \"tags\": [\"a\", \"b\"]}]}", "input_tokens": 42} +{"body": "{\"model\": \"gpt-4\", \"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant. Answer precisely and cite sources.\"}, {\"role\": \"user\", \"content\": \"\\ud83d\\ude42 a every WON'T They'RE involved counting caf\\u00e9 backtracking boundaries Z\\u00fcrich WON'T 100% \\\"quotes\\\" caf\\u00e9 tiktoken's budget They'RE request request regex v1.2.3 hand hand fox tiktoken's on 3.14159 mirrors that don't WON'T admission before budget the admission WON'T no 1999 100% no admission budget mirrors way caf\\u00e9 dog a quick mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 hand gateway regex on jumps because {braces} 'single' the the the {braces} piece hand scanned reservation [brackets] we'll 3.14159 request don't \\\"quotes\\\" na\\u00efve C++ caf\\u00e9 caf\\u00e9 for https://example.com/a/b?c=d we'll no over the node.js for while over way\"}, {\"role\": \"assistant\", \"content\": \"it tiktoken's node.js it scanned v1.2.3 boundaries (parens) and while reservation lazy the that https://example.com/a/b?c=d quick don't budget engine boundaries F# budget every we'll before jumps scanned jumps there's counting the don't jumps a once admission 100% 3.14159 brown brown that because jumps They'RE 100% caf\\u00e9 WON'T They'RE boundaries \\u6771\\u4eac exactly scanner caf\\u00e9 fox (parens) body dog a 1999 boundaries 'single' {braces} reservation mirrors while {braces} {braces} so admission the F# https://example.com/a/b?c=d brown a caf\\u00e9 on admission that on counting that we'll over lazy https://example.com/a/b?c=d way over engine reservation gateway body I'M for \\\"quotes\\\" 42 and F# there's 'single' quick \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T scanner written every (parens) so\"}, {\"role\": \"user\", \"content\": \"I'M over faster counting https://example.com/a/b?c=d way while it over way mirrors boundaries keep body hand na\\u00efve once because that node.js for every the 'single' every caf\\u00e9 caf\\u00e9 piece there's v1.2.3 v1.2.3 tiktoken's brown so counting there's for we'll boundaries 'single' no https://example.com/a/b?c=d node.js \\u6771\\u4eac written with budget (parens) because \\\"quotes\\\" \\u6771\\u4eac while \\u6771\\u4eac counting scanned 1999 \\u6771\\u4eac hand keep so that gateway caf\\u00e9 for scanned it request keep counting user@example.com admission request \\\"quotes\\\" v1.2.3 caf\\u00e9 over that F# we'll regex a faster with that They'RE piece because tiktoken's engine hand 100% WON'T reservation (parens) caf\\u00e9 on Z\\u00fcrich dog \\u6771\\u4eac that They'RE 'single' the 'single' na\\u00efve involved the {braces} there's scanned no engine dog backtracking there's\"}, {\"role\": \"assistant\", \"content\": \"every node.js C++ for 3.14159 \\u6771\\u4eac 'single' gateway once the user@example.com a 100% every every tokens written and every brown na\\u00efve the body the because quick brown engine fox 3.14159 (parens) F# while with a 100% C++ with the written WON'T on request Z\\u00fcrich hand on https://example.com/a/b?c=d don't way way 42 that we'll \\ud83d\\ude42 [brackets] admission tiktoken's request hand caf\\u00e9 every every (parens) They'RE that jumps the regex 100% \\\"quotes\\\" is budget\"}, {\"role\": \"user\", \"content\": \"engine with I'M backtracking while F# quick 'single' mirrors \\\"quotes\\\" 'single' regex caf\\u00e9 It's Z\\u00fcrich exactly a counting tiktoken's we'll once \\u0645\\u0631\\u062d\\u0628\\u0627 42 v1.2.3 scanner WON'T \\u6771\\u4eac there's node.js It's budget that budget {braces} because [brackets] It's a $1,234.56 that v1.2.3 42 gateway backtracking it C++ scanned keep \\ud83d\\ude42 I'M hand a counting scanner reservation \\u6771\\u4eac written 3.14159 there's once fox I'M C++ lazy the \\u0645\\u0631\\u062d\\u0628\\u0627 before don't we'll gateway exactly \\ud83d\\ude42 Z\\u00fcrich 3.14159 with WON'T there's node.js \\ud83d\\ude42 quick written faster counting 1999 backtracking 'single' counting we'll engine counting don't \\\"quotes\\\" engine \\\"quotes\\\" way I'M budget [brackets] backtracking \\\"quotes\\\" 3.14159 don't written written \\u0645\\u0631\\u062d\\u0628\\u0627 body I'M 'single' while on and reservation \\ud83d\\ude42 regex and no budget regex (parens) we'll They'RE na\\u00efve v1.2.3\"}, {\"role\": \"assistant\", \"content\": \"hand lazy dog budget lazy involved because scanned piece quick that involved 'single' dog quick na\\u00efve budget scanner exactly $1,234.56 fox quick I'M hand (parens) node.js while jumps boundaries \\ud83d\\ude42 They'RE way request engine 'single' fox backtracking regex \\u0645\\u0631\\u062d\\u0628\\u0627 because and over jumps 3.14159 WON'T counting for and mirrors admission 'single' caf\\u00e9 https://example.com/a/b?c=d that \\ud83d\\ude42 faster \\u0645\\u0631\\u062d\\u0628\\u0627 admission jumps quick for $1,234.56 exactly exactly $1,234.56 scanned keep 3.14159 backtracking piece tiktoken's is is hand reservation before regex budget 3.14159 tiktoken's I'M it no budget user@example.com budget written budget over that https://example.com/a/b?c=d no written so quick tokens C++\"}, {\"role\": \"user\", \"content\": \"once body involved every brown every it that hand engine with scanned reservation \\u6771\\u4eac backtracking and {braces} over [brackets] brown over for is \\ud83d\\ude42 100% before quick no counting no v1.2.3 counting \\\"quotes\\\" mirrors 3.14159 1999 there's way \\\"quotes\\\" piece on na\\u00efve They'RE no every exactly node.js 42 because over there's 1999 C++ we'll mirrors F# it scanner jumps It's scanner mirrors mirrors It's tokens backtracking body brown $1,234.56 keep admission 100% the exactly we'll caf\\u00e9 jumps over 3.14159 we'll so 42 \\u6771\\u4eac I'M WON'T lazy brown It's a na\\u00efve \\\"quotes\\\" https://example.com/a/b?c=d (parens) \\u6771\\u4eac tiktoken's so dog body 'single' scanned piece body 3.14159 scanned body the so \\\"quotes\\\" counting\"}, {\"role\": \"assistant\", \"content\": \"boundaries request the that 100% gateway the there's hand the They'RE caf\\u00e9 fox C++ scanner written \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE scanner WON'T keep before for it so counting that (parens) {braces} They'RE scanner every {braces} no node.js keep I'M jumps backtracking gateway https://example.com/a/b?c=d don't tiktoken's a is 1999 don't F# v1.2.3 involved hand scanner They'RE backtracking exactly and for exactly for $1,234.56 counting v1.2.3 request gateway mirrors that no \\ud83d\\ude42 every dog once They'RE 100% reservation It's a with and 100% because so It's admission there's gateway 42 on over gateway is every 3.14159 boundaries no for admission quick lazy lazy fox node.js because while $1,234.56 quick I'M lazy involved WON'T {braces} na\\u00efve {braces} there's with caf\\u00e9 https://example.com/a/b?c=d \\\"quotes\\\" [brackets] lazy lazy it 100% caf\\u00e9 way lazy tokens C++ that it \\u6771\\u4eac lazy\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors budget reservation 'single' it scanner F# is exactly They'RE \\ud83d\\ude42 I'M boundaries F# {braces} https://example.com/a/b?c=d over backtracking node.js is \\ud83d\\ude42 \\ud83d\\ude42 $1,234.56 so and counting It's Z\\u00fcrich once node.js once v1.2.3 on that we'll 'single' mirrors I'M \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking They'RE tokens hand counting 1999 user@example.com Z\\u00fcrich They'RE tokens reservation exactly for https://example.com/a/b?c=d mirrors and boundaries regex the brown while keep \\\"quotes\\\" we'll the involved 42 {braces} scanner reservation Z\\u00fcrich no we'll [brackets] caf\\u00e9 written that hand user@example.com $1,234.56 42 while budget every 3.14159 a exactly way body scanned admission C++ (parens) tiktoken's body for WON'T hand no dog 1999 (parens) on don't 1999 we'll I'M v1.2.3 that WON'T fox scanned 1999\"}, {\"role\": \"assistant\", \"content\": \"gateway $1,234.56 $1,234.56 \\\"quotes\\\" scanner there's scanner $1,234.56 100% it budget \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) admission admission with I'M admission 'single' hand jumps scanned It's \\\"quotes\\\" is F# before engine counting dog because admission tiktoken's backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 that 42 written it body quick Z\\u00fcrich I'M F# I'M that brown It's request caf\\u00e9 is boundaries jumps don't written written faster for admission Z\\u00fcrich engine reservation and Z\\u00fcrich scanned written keep scanned is keep jumps the so F# 'single' user@example.com keep because They'RE over because C++ written faster 42 mirrors na\\u00efve 42 tiktoken's [brackets] na\\u00efve hand on no written so $1,234.56 there's 100% 100% $1,234.56 is \\ud83d\\ude42 scanner dog lazy 100% https://example.com/a/b?c=d tiktoken's They'RE [brackets] user@example.com while hand \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d we'll 42 WON'T it a mirrors v1.2.3\"}, {\"role\": \"user\", \"content\": \"that involved dog C++ way that regex with https://example.com/a/b?c=d because 1999 jumps and and caf\\u00e9 F# backtracking that 3.14159 the quick v1.2.3 engine piece request brown (parens) because tokens na\\u00efve we'll and \\ud83d\\ude42 user@example.com that na\\u00efve \\u6771\\u4eac na\\u00efve tokens jumps 3.14159 tiktoken's na\\u00efve brown It's jumps before and a the user@example.com Z\\u00fcrich WON'T mirrors It's fox It's involved don't \\u6771\\u4eac 'single' written that written fox and the 3.14159 Z\\u00fcrich way on once na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 request fox so fox 'single' admission admission node.js mirrors backtracking it the\"}, {\"role\": \"assistant\", \"content\": \"engine so $1,234.56 don't caf\\u00e9 lazy I'M while 'single' budget scanned \\ud83d\\ude42 Z\\u00fcrich piece tokens exactly budget boundaries admission written tokens every \\u0645\\u0631\\u062d\\u0628\\u0627 before with backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 engine regex faster brown \\ud83d\\ude42 before we'll reservation na\\u00efve regex there's faster counting na\\u00efve reservation quick Z\\u00fcrich tiktoken's that gateway we'll C++ They'RE scanned for engine budget 100% na\\u00efve $1,234.56 written admission no we'll WON'T scanned body dog node.js while\"}, {\"role\": \"user\", \"content\": \"once exactly scanner boundaries scanned every there's mirrors $1,234.56 there's so dog dog They'RE lazy fox because 'single' so gateway Z\\u00fcrich faster v1.2.3 quick I'M \\u6771\\u4eac exactly written tokens node.js na\\u00efve brown [brackets] mirrors while on \\u6771\\u4eac $1,234.56 once hand over exactly quick backtracking over exactly {braces} it don't regex 3.14159 way the 100% $1,234.56 is I'M admission admission [brackets] scanned while boundaries piece counting that node.js reservation \\u6771\\u4eac body jumps while node.js I'M Z\\u00fcrich with fox C++ reservation F# \\\"quotes\\\" They'RE {braces} (parens) caf\\u00e9 1999 there's {braces} every WON'T no dog WON'T 1999 admission [brackets] that body 'single' gateway while mirrors with with scanner hand no over scanned hand na\\u00efve It's the while with once \\\"quotes\\\" boundaries \\\"quotes\\\" It's tokens and\"}, {\"role\": \"assistant\", \"content\": \"jumps 100% before body there's a C++ $1,234.56 on exactly exactly every hand lazy dog don't every that so F# Z\\u00fcrich jumps caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 written scanned tokens admission WON'T backtracking scanned 1999 engine brown exactly counting node.js 'single' 1999 counting that keep keep $1,234.56 Z\\u00fcrich They'RE before scanned They'RE \\u6771\\u4eac It's over tiktoken's once hand request tiktoken's while gateway node.js no \\ud83d\\ude42 it https://example.com/a/b?c=d It's 100% written there's hand {braces} there's admission $1,234.56 F# [brackets]\"}, {\"role\": \"user\", \"content\": \"hand mirrors exactly 42 They'RE [brackets] before \\u0645\\u0631\\u062d\\u0628\\u0627 that while written caf\\u00e9 body because fox tokens no WON'T dog faster and fox keep don't caf\\u00e9 'single' node.js (parens) reservation C++ I'M fox F# \\u6771\\u4eac mirrors over 1999 written keep na\\u00efve gateway a 3.14159 keep once body \\\"quotes\\\" F# we'll $1,234.56 https://example.com/a/b?c=d regex fox backtracking is admission request involved so over engine caf\\u00e9 way backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac They'RE It's a mirrors \\\"quotes\\\" jumps They'RE way way the engine C++ request because backtracking with with written the scanned boundaries https://example.com/a/b?c=d (parens) no for gateway dog \\ud83d\\ude42 boundaries node.js\"}, {\"role\": \"assistant\", \"content\": \"and tiktoken's They'RE caf\\u00e9 exactly the quick gateway caf\\u00e9 budget hand node.js the node.js we'll faster fox 'single' involved scanner na\\u00efve reservation https://example.com/a/b?c=d {braces} way we'll backtracking keep while and no with dog exactly the WON'T dog \\u0645\\u0631\\u062d\\u0628\\u0627 regex [brackets] written 1999 keep v1.2.3 I'M mirrors admission \\u6771\\u4eac before we'll \\\"quotes\\\" (parens) 100% over mirrors 42 a request F# WON'T a counting \\ud83d\\ude42 scanner no we'll fox gateway boundaries 1999 WON'T I'M Z\\u00fcrich faster there's caf\\u00e9 They'RE that the with for faster quick that body reservation 42 don't I'M I'M scanned faster hand for\"}, {\"role\": \"user\", \"content\": \"and the admission caf\\u00e9 admission once F# written reservation budget written https://example.com/a/b?c=d scanner a that Z\\u00fcrich once the Z\\u00fcrich faster C++ no I'M counting is hand caf\\u00e9 before engine on C++ 'single' \\\"quotes\\\" I'M Z\\u00fcrich quick that boundaries node.js lazy 3.14159 while na\\u00efve Z\\u00fcrich it reservation request that before F# boundaries once written the WON'T https://example.com/a/b?c=d the written piece mirrors 100% mirrors https://example.com/a/b?c=d over before regex scanner \\u0645\\u0631\\u062d\\u0628\\u0627 before node.js WON'T \\u0645\\u0631\\u062d\\u0628\\u0627 way jumps for scanned that involved for every (parens) lazy C++ boundaries backtracking it user@example.com no that C++ faster engine I'M piece with faster for that brown 3.14159 user@example.com it while so on involved that 42 once quick written we'll a budget\"}, {\"role\": \"assistant\", \"content\": \"admission \\ud83d\\ude42 100% tiktoken's on jumps we'll hand body is tiktoken's and 100% \\ud83d\\ude42 the it 3.14159 $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 scanner over 100% scanned \\ud83d\\ude42 so way engine that scanner 100% so so tokens boundaries node.js we'll jumps user@example.com that tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 once don't way is body so user@example.com on request is lazy and every engine tokens no lazy admission jumps reservation v1.2.3 scanned \\u0645\\u0631\\u062d\\u0628\\u0627 the \\ud83d\\ude42 node.js so \\\"quotes\\\" every it \\u6771\\u4eac F# regex don't faster every $1,234.56 on way engine regex every keep $1,234.56 no that engine written don't involved {braces} \\u6771\\u4eac v1.2.3 while request 42 (parens)\"}, {\"role\": \"user\", \"content\": \"keep fox we'll backtracking once it engine with lazy (parens) faster caf\\u00e9 F# with It's (parens) user@example.com for over counting we'll it https://example.com/a/b?c=d fox the (parens) way over because https://example.com/a/b?c=d before written fox involved written [brackets] before it tokens \\\"quotes\\\" there's once [brackets] 'single' tiktoken's C++ v1.2.3 quick 100% user@example.com dog \\u6771\\u4eac I'M 'single' keep a before node.js F# exactly request the Z\\u00fcrich exactly tokens so faster admission lazy and every counting $1,234.56 brown\"}, {\"role\": \"assistant\", \"content\": \"'single' every budget 42 It's quick way dog {braces} because don't \\u0645\\u0631\\u062d\\u0628\\u0627 way brown involved counting body don't (parens) body tiktoken's scanned \\u0645\\u0631\\u062d\\u0628\\u0627 dog it (parens) the for once once engine written there's that node.js every \\u0645\\u0631\\u062d\\u0628\\u0627 1999 $1,234.56 caf\\u00e9 written body dog gateway for It's WON'T 42 'single' 3.14159 that Z\\u00fcrich jumps C++ while WON'T admission so involved It's \\u0645\\u0631\\u062d\\u0628\\u0627 node.js on with It's it They'RE lazy is engine way scanner $1,234.56 and lazy boundaries 'single' before dog that faster 3.14159 tokens It's tokens we'll once and reservation over They'RE\"}, {\"role\": \"user\", \"content\": \"I'M node.js C++ na\\u00efve once engine with \\ud83d\\ude42 involved 100% brown scanned and is scanner scanned 'single' counting don't fox way way C++ before every faster piece hand They'RE so brown piece because while on a WON'T 1999 backtracking budget \\u6771\\u4eac piece and engine every we'll dog user@example.com na\\u00efve https://example.com/a/b?c=d brown Z\\u00fcrich it way and so hand on \\\"quotes\\\" (parens) before we'll\"}, {\"role\": \"assistant\", \"content\": \"They'RE there's node.js mirrors there's the there's reservation engine is boundaries fox a body fox 42 user@example.com (parens) 100% on (parens) written request https://example.com/a/b?c=d {braces} It's Z\\u00fcrich every counting $1,234.56 scanned once user@example.com C++ so 100% exactly exactly {braces} body jumps on no involved 100% and admission every scanned reservation tokens exactly keep there's Z\\u00fcrich v1.2.3 keep 42 the 42 the with boundaries request involved there's faster keep on https://example.com/a/b?c=d user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 on (parens) na\\u00efve (parens) WON'T \\u6771\\u4eac with engine na\\u00efve body 1999 the engine quick counting boundaries there's counting {braces} for 100% involved there's involved so WON'T lazy a over way keep They'RE tokens keep piece na\\u00efve node.js exactly reservation body every faster backtracking\"}, {\"role\": \"user\", \"content\": \"and scanned C++ jumps They'RE scanned boundaries C++ that hand budget tokens \\\"quotes\\\" scanned I'M 100% 'single' counting because (parens) regex (parens) na\\u00efve F# (parens) I'M and with so once once for regex piece dog quick They'RE while keep exactly before 1999 is 100% keep caf\\u00e9 before 42 hand that reservation 3.14159 because fox that regex body gateway don't once gateway and engine with once don't I'M piece way 3.14159 admission the don't it body piece \\\"quotes\\\" 42 mirrors on body don't 100% that quick backtracking {braces} is we'll and body we'll budget every every v1.2.3 exactly way I'M \\\"quotes\\\" involved gateway scanned once \\u0645\\u0631\\u062d\\u0628\\u0627 keep jumps \\u6771\\u4eac backtracking dog engine \\u6771\\u4eac tiktoken's that 42 user@example.com don't scanner (parens) I'M\"}, {\"role\": \"assistant\", \"content\": \"the piece 1999 over v1.2.3 C++ It's https://example.com/a/b?c=d because request that fox \\ud83d\\ude42 way \\u6771\\u4eac tokens tokens brown (parens) way v1.2.3 that na\\u00efve mirrors \\\"quotes\\\" admission dog 100% 100% regex backtracking reservation 1999 user@example.com \\\"quotes\\\" 3.14159 I'M budget mirrors 1999 lazy admission a on that user@example.com They'RE They'RE \\\"quotes\\\" user@example.com node.js 100% so na\\u00efve and It's \\\"quotes\\\" with the piece boundaries we'll 42 42 while https://example.com/a/b?c=d user@example.com budget faster na\\u00efve and 1999 a admission fox and 'single' for They'RE boundaries scanned\"}, {\"role\": \"user\", \"content\": \"quick gateway They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 fox the (parens) mirrors because while we'll we'll every mirrors counting Z\\u00fcrich 42 on gateway WON'T written backtracking no user@example.com caf\\u00e9 \\u6771\\u4eac that boundaries while so fox backtracking involved They'RE exactly 1999 {braces} [brackets] exactly regex mirrors \\u6771\\u4eac 1999 over reservation way involved \\u0645\\u0631\\u062d\\u0628\\u0627 42 3.14159 while quick so a (parens) hand there's F# a They'RE body engine F# v1.2.3 faster hand over dog backtracking while brown that before I'M 1999 way before piece counting request that budget boundaries v1.2.3 exactly that They'RE faster involved \\\"quotes\\\" scanner C++ It's 100% $1,234.56 written\"}, {\"role\": \"assistant\", \"content\": \"https://example.com/a/b?c=d way \\\"quotes\\\" fox fox C++ is admission It's \\ud83d\\ude42 tiktoken's WON'T tokens so caf\\u00e9 way admission a scanned $1,234.56 3.14159 that F# don't fox counting every engine scanner scanned don't 'single' tiktoken's https://example.com/a/b?c=d I'M keep there's piece \\u6771\\u4eac is while way hand over fox WON'T \\u6771\\u4eac scanner v1.2.3 \\u6771\\u4eac don't written \\\"quotes\\\" keep the body and brown that counting before budget request 3.14159 boundaries {braces}\"}, {\"role\": \"user\", \"content\": \"exactly It's 3.14159 hand body They'RE tokens don't v1.2.3 backtracking on quick (parens) 100% body quick so scanner so \\ud83d\\ude42 backtracking is once exactly regex https://example.com/a/b?c=d na\\u00efve before there's every C++ [brackets] tokens They'RE with the 1999 piece a reservation request caf\\u00e9 it 'single' while \\\"quotes\\\" boundaries because lazy \\u0645\\u0631\\u062d\\u0628\\u0627 for Z\\u00fcrich It's (parens) a 100% there's dog caf\\u00e9 \\ud83d\\ude42 (parens) because quick with node.js https://example.com/a/b?c=d engine piece \\ud83d\\ude42 counting brown way It's user@example.com user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 scanned reservation 'single' don't request admission\"}, {\"role\": \"assistant\", \"content\": \"reservation it so before no scanned backtracking a 1999 every exactly jumps 100% over It's 1999 we'll boundaries don't scanned once and before \\u6771\\u4eac faster backtracking regex backtracking every 'single' admission caf\\u00e9 brown 3.14159 \\ud83d\\ude42 there's \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T 42 $1,234.56 is before reservation it admission backtracking before over $1,234.56 {braces} mirrors It's {braces} before admission quick hand lazy scanned \\ud83d\\ude42 exactly faster while reservation C++ They'RE it exactly gateway it body for quick so quick 3.14159 1999 for user@example.com involved {braces} https://example.com/a/b?c=d quick while I'M lazy brown backtracking keep\"}, {\"role\": \"user\", \"content\": \"quick keep v1.2.3 request budget a and regex keep body gateway so It's before \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 I'M 3.14159 and written counting {braces} v1.2.3 for brown engine user@example.com mirrors mirrors over \\u6771\\u4eac (parens) regex jumps keep written and Z\\u00fcrich a no dog and jumps piece C++ the counting with quick on the user@example.com request is jumps boundaries quick 'single' 100% 'single' over 100% the over there's na\\u00efve 1999 engine quick https://example.com/a/b?c=d 3.14159 it jumps way no hand \\ud83d\\ude42 [brackets] {braces} admission F# scanner 3.14159 F# it 3.14159 boundaries budget keep don't that quick 'single' reservation C++ the gateway and dog exactly body so https://example.com/a/b?c=d \\\"quotes\\\" 100%\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 budget \\\"quotes\\\" quick once body I'M that way no once a user@example.com boundaries admission gateway engine caf\\u00e9 $1,234.56 They'RE They'RE reservation user@example.com They'RE They'RE budget for \\u0645\\u0631\\u062d\\u0628\\u0627 dog is WON'T faster involved lazy so while faster backtracking 1999 user@example.com we'll engine reservation v1.2.3 it on tokens counting \\ud83d\\ude42 every every over written it every tiktoken's no before (parens) body the request quick every admission the \\ud83d\\ude42 body don't admission regex there's \\u6771\\u4eac \\\"quotes\\\" dog don't no Z\\u00fcrich $1,234.56 and scanned scanned https://example.com/a/b?c=d request while way written engine reservation there's scanned the request that jumps it caf\\u00e9 and v1.2.3 scanned na\\u00efve involved brown no user@example.com tokens because counting there's \\\"quotes\\\" on node.js quick exactly every that written\"}, {\"role\": \"user\", \"content\": \"counting {braces} \\ud83d\\ude42 C++ admission boundaries C++ it 'single' that mirrors jumps the while that don't They'RE while 42 Z\\u00fcrich 1999 scanner so quick v1.2.3 there's don't keep Z\\u00fcrich no keep faster reservation request is (parens) body we'll 100% don't na\\u00efve (parens) for backtracking fox boundaries backtracking v1.2.3 $1,234.56 we'll 42 tokens faster counting once keep budget fox v1.2.3 'single' mirrors no engine exactly fox 42 way the C++ quick tiktoken's counting 1999 It's we'll counting because hand \\ud83d\\ude42 user@example.com is request quick 42 it keep don't They'RE WON'T once and fox v1.2.3 and with na\\u00efve way lazy user@example.com written is 'single' brown scanned request is caf\\u00e9 we'll node.js a so while we'll WON'T way admission They'RE fox C++ once node.js WON'T that v1.2.3 every\"}, {\"role\": \"assistant\", \"content\": \"C++ don't no boundaries scanner https://example.com/a/b?c=d node.js backtracking hand [brackets] keep F# counting caf\\u00e9 scanner v1.2.3 https://example.com/a/b?c=d counting no https://example.com/a/b?c=d the backtracking it I'M I'M F# involved WON'T with exactly (parens) brown body v1.2.3 body https://example.com/a/b?c=d keep it It's 1999 F# \\u6771\\u4eac so \\ud83d\\ude42 scanner for user@example.com v1.2.3 I'M don't fox written request engine once regex counting tokens the admission (parens) admission reservation faster C++ 3.14159 tokens hand \\\"quotes\\\" counting caf\\u00e9 [brackets] It's don't Z\\u00fcrich 3.14159 the tiktoken's I'M v1.2.3 admission C++ They'RE hand tokens dog no $1,234.56 engine while boundaries so caf\\u00e9 100% \\u0645\\u0631\\u062d\\u0628\\u0627 F# scanned jumps faster while na\\u00efve lazy\"}, {\"role\": \"user\", \"content\": \"WON'T hand keep brown \\u0645\\u0631\\u062d\\u0628\\u0627 because counting faster with that involved is because because 42 Z\\u00fcrich that engine with the the brown {braces} tiktoken's [brackets] I'M It's over (parens) C++ It's over faster \\u6771\\u4eac user@example.com user@example.com 100% on it on with mirrors 3.14159 42 budget It's WON'T node.js keep tiktoken's \\u6771\\u4eac https://example.com/a/b?c=d 1999 involved body scanned hand involved engine faster every is and that tokens every 42 on dog admission (parens) body for caf\\u00e9 once before a \\u6771\\u4eac before with don't tokens It's because \\\"quotes\\\" node.js na\\u00efve it engine is backtracking [brackets] regex brown They'RE so is is involved engine so scanner gateway scanned They'RE keep na\\u00efve body reservation 42 scanned WON'T faster there's backtracking it caf\\u00e9 v1.2.3 brown regex {braces} lazy node.js C++ don't written WON'T with user@example.com piece over we'll v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich jumps (parens)\"}, {\"role\": \"assistant\", \"content\": \"na\\u00efve scanned every reservation no we'll faster before it {braces} admission 1999 scanned budget written there's WON'T $1,234.56 faster brown reservation (parens) 'single' every (parens) $1,234.56 It's is counting way jumps no scanned I'M the node.js that na\\u00efve over faster [brackets] 1999 there's keep user@example.com 100% user@example.com before once reservation brown don't C++ dog it over backtracking every \\\"quotes\\\" quick a budget \\ud83d\\ude42 budget because v1.2.3 dog 100% once v1.2.3 and and every don't 3.14159 once once quick gateway before for 3.14159 v1.2.3 \\u6771\\u4eac tokens that on because it budget a involved scanned scanner lazy 'single' caf\\u00e9 They'RE engine request while Z\\u00fcrich engine WON'T\"}, {\"role\": \"user\", \"content\": \"100% over written reservation https://example.com/a/b?c=d WON'T scanner F# before https://example.com/a/b?c=d we'll that while counting is admission 42 caf\\u00e9 C++ body tiktoken's body 3.14159 the I'M with node.js It's \\u6771\\u4eac [brackets] don't https://example.com/a/b?c=d C++ before on [brackets] 'single' piece brown before 1999 {braces} written {braces} way reservation request F# so https://example.com/a/b?c=d 100% $1,234.56 piece piece keep https://example.com/a/b?c=d way budget 100% boundaries node.js lazy because F# tokens (parens) and C++ backtracking tiktoken's piece a 3.14159 WON'T scanned backtracking node.js because regex backtracking so keep 1999 the is because WON'T node.js we'll lazy quick with once and once \\\"quotes\\\" we'll keep It's the on It's so is faster\"}, {\"role\": \"assistant\", \"content\": \"hand every for written mirrors backtracking $1,234.56 It's WON'T because faster Z\\u00fcrich don't don't {braces} jumps 'single' regex gateway user@example.com once there's a on over \\\"quotes\\\" 100% node.js regex a body that F# 100% once on 3.14159 while backtracking v1.2.3 hand \\u6771\\u4eac tokens admission \\\"quotes\\\" 100% admission and It's Z\\u00fcrich that so while and $1,234.56 caf\\u00e9 every F# involved fox backtracking the \\u0645\\u0631\\u062d\\u0628\\u0627 and https://example.com/a/b?c=d exactly node.js before mirrors 'single' exactly tokens that scanned body https://example.com/a/b?c=d na\\u00efve once it the the faster v1.2.3 scanned fox that jumps v1.2.3 1999 gateway F# caf\\u00e9 'single' fox brown [brackets] 100% over user@example.com on na\\u00efve https://example.com/a/b?c=d node.js They'RE and scanner 'single' involved the because hand scanner fox user@example.com\"}, {\"role\": \"user\", \"content\": \"1999 $1,234.56 that 'single' no counting (parens) because Z\\u00fcrich on (parens) a fox user@example.com admission over is there's \\\"quotes\\\" {braces} exactly piece that with I'M F# over for counting and \\ud83d\\ude42 scanner over every \\ud83d\\ude42 100% admission before \\u0645\\u0631\\u062d\\u0628\\u0627 reservation WON'T brown scanner on faster 100% caf\\u00e9 piece [brackets] counting scanner It's written {braces} C++ that is boundaries exactly \\u0645\\u0631\\u062d\\u0628\\u0627 once 'single' quick F# hand 'single' user@example.com reservation jumps it F# reservation request that 'single' (parens) the way WON'T (parens) a backtracking backtracking that \\\"quotes\\\" C++ I'M C++ C++ gateway with body body\"}, {\"role\": \"assistant\", \"content\": \"budget that 42 It's body backtracking caf\\u00e9 1999 because hand hand F# piece is (parens) and Z\\u00fcrich jumps user@example.com don't I'M They'RE regex body reservation once (parens) with F# piece is every node.js no piece because {braces} with 42 Z\\u00fcrich tiktoken's keep I'M is scanner no body 'single' 1999 that request so \\u0645\\u0631\\u062d\\u0628\\u0627 3.14159 \\u6771\\u4eac and https://example.com/a/b?c=d engine tiktoken's on while $1,234.56 over budget 1999 1999 backtracking is \\ud83d\\ude42 the tiktoken's involved over don't Z\\u00fcrich I'M so gateway written dog before written v1.2.3 backtracking gateway keep dog [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9\"}, {\"role\": \"user\", \"content\": \"exactly involved user@example.com fox keep boundaries and 42 reservation {braces} mirrors the It's budget C++ tiktoken's budget mirrors faster [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 over scanner way quick while \\\"quotes\\\" exactly 'single' F# no faster [brackets] backtracking scanner WON'T WON'T tokens quick that the node.js it regex user@example.com involved while boundaries regex while hand mirrors way na\\u00efve a hand request that before v1.2.3 1999 3.14159 caf\\u00e9 because on over scanner so body tokens quick because counting exactly hand user@example.com that It's\"}, {\"role\": \"assistant\", \"content\": \"so quick dog (parens) Z\\u00fcrich backtracking for the \\ud83d\\ude42 because regex involved tokens dog we'll 'single' we'll Z\\u00fcrich once 100% regex we'll [brackets] 42 while with 1999 Z\\u00fcrich fox admission while written the C++ over every mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 on it https://example.com/a/b?c=d scanned gateway no 1999 dog https://example.com/a/b?c=d we'll 1999 before the every budget on 1999 with piece 1999 faster user@example.com I'M body keep while the once scanned I'M\"}, {\"role\": \"user\", \"content\": \"tiktoken's quick budget on budget the \\u0645\\u0631\\u062d\\u0628\\u0627 for Z\\u00fcrich I'M don't WON'T we'll user@example.com It's jumps \\u6771\\u4eac we'll involved gateway it that jumps for for counting that $1,234.56 so while 'single' WON'T quick the brown we'll once mirrors [brackets] \\u6771\\u4eac \\\"quotes\\\" boundaries way for because {braces} it user@example.com faster $1,234.56 no a 'single' body backtracking before the 3.14159 scanner {braces} backtracking hand because so body [brackets] because that involved scanned WON'T there's \\u0645\\u0631\\u062d\\u0628\\u0627 it hand we'll dog a before the and faster \\ud83d\\ude42 hand 100% on body [brackets] 42 the node.js it counting and jumps we'll a fox 3.14159 once (parens) \\ud83d\\ude42 hand every don't way jumps body over involved \\u6771\\u4eac is budget way scanned $1,234.56 because request gateway involved the that dog the reservation while \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"counting written brown on once it They'RE involved we'll every hand \\ud83d\\ude42 it They'RE dog for reservation on every no tokens regex piece on (parens) tiktoken's before gateway it every the while that we'll gateway a the a v1.2.3 the over lazy node.js gateway budget WON'T \\ud83d\\ude42 is exactly jumps over lazy piece backtracking written with 3.14159 jumps involved lazy scanner keep keep [brackets] boundaries that no that because quick F# v1.2.3 reservation involved\"}, {\"role\": \"user\", \"content\": \"caf\\u00e9 engine request na\\u00efve jumps involved lazy regex scanned {braces} 3.14159 engine mirrors \\\"quotes\\\" Z\\u00fcrich on there's (parens) They'RE regex for over regex 1999 tiktoken's scanner that once admission F# mirrors 42 the body WON'T the WON'T dog 100% 42 is lazy \\ud83d\\ude42 that request once 100% na\\u00efve dog tokens budget jumps \\ud83d\\ude42 $1,234.56 user@example.com $1,234.56 don't admission that brown (parens) it node.js tokens na\\u00efve we'll it v1.2.3 once so is written admission faster way we'll request jumps v1.2.3 fox the lazy gateway $1,234.56 42 counting Z\\u00fcrich the 'single' exactly once way I'M \\u6771\\u4eac They'RE before caf\\u00e9 boundaries over counting piece brown faster while so counting na\\u00efve hand \\ud83d\\ude42 quick the every Z\\u00fcrich {braces} scanned \\\"quotes\\\" with regex keep while once engine before fox\"}, {\"role\": \"assistant\", \"content\": \"mirrors {braces} because \\u6771\\u4eac mirrors scanned exactly budget way mirrors https://example.com/a/b?c=d boundaries involved 100% 3.14159 exactly engine brown They'RE is keep that hand lazy exactly tokens It's keep every \\ud83d\\ude42 the F# written {braces} user@example.com the counting $1,234.56 keep regex tiktoken's and mirrors fox the gateway \\ud83d\\ude42 keep They'RE written I'M there's we'll na\\u00efve WON'T is brown C++ involved brown and piece \\ud83d\\ude42 reservation [brackets] reservation I'M \\\"quotes\\\" while exactly way https://example.com/a/b?c=d don't \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac over keep scanner \\u6771\\u4eac scanned over tiktoken's boundaries mirrors once for quick faster a \\u6771\\u4eac lazy (parens) na\\u00efve gateway \\u6771\\u4eac \\u6771\\u4eac brown They'RE there's on keep once dog that 1999 [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 while it It's no\"}, {\"role\": \"user\", \"content\": \"that body over Z\\u00fcrich before that dog keep and \\\"quotes\\\" and regex tiktoken's way piece is scanner is quick C++ node.js written dog regex the It's dog https://example.com/a/b?c=d scanned engine we'll counting it 'single' counting a boundaries is \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 brown budget fox the Z\\u00fcrich jumps {braces} that boundaries piece because the 'single' v1.2.3 {braces} once that mirrors \\ud83d\\ude42 backtracking no no mirrors on on scanned F# They'RE regex scanned the every for faster v1.2.3 \\ud83d\\ude42 I'M we'll exactly that is once for because it caf\\u00e9 It's caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 keep budget admission\"}, {\"role\": \"assistant\", \"content\": \"and node.js is so admission body They'RE https://example.com/a/b?c=d the 1999 we'll It's keep mirrors request so piece gateway engine way hand exactly is C++ 100% exactly for body piece with jumps WON'T the no keep WON'T so F# don't $1,234.56 {braces} and \\ud83d\\ude42 request reservation written tiktoken's that written way written the tokens 3.14159 WON'T admission \\u0645\\u0631\\u062d\\u0628\\u0627 C++ budget gateway every before brown WON'T piece 1999 [brackets] a dog [brackets] while scanner \\ud83d\\ude42 we'll v1.2.3 scanner quick dog a while admission hand so there's every 42 node.js that engine faster na\\u00efve is \\\"quotes\\\"\"}, {\"role\": \"user\", \"content\": \"scanner hand boundaries admission it \\u6771\\u4eac WON'T na\\u00efve user@example.com Z\\u00fcrich F# F# They'RE a is caf\\u00e9 \\\"quotes\\\" before on request $1,234.56 boundaries (parens) don't brown the every 1999 (parens) [brackets] so 1999 boundaries 100% the hand 42 tokens every over that the the F# that quick They'RE before because tokens caf\\u00e9 we'll exactly boundaries because brown keep no 'single' there's brown v1.2.3 user@example.com 42 1999 \\u6771\\u4eac {braces} for mirrors on tiktoken's WON'T \\\"quotes\\\" na\\u00efve They'RE {braces} v1.2.3 brown written involved tokens jumps boundaries budget jumps boundaries way 42 body written na\\u00efve written is request every admission counting before backtracking with They'RE fox\"}, {\"role\": \"assistant\", \"content\": \"\\ud83d\\ude42 we'll backtracking involved counting request node.js caf\\u00e9 lazy $1,234.56 so I'M while is every it 42 exactly node.js that They'RE there's I'M I'M a and counting admission no 'single' admission v1.2.3 request boundaries 'single' tiktoken's budget gateway 100% caf\\u00e9 jumps hand no the \\u6771\\u4eac I'M we'll fox 1999 piece and body {braces} na\\u00efve hand Z\\u00fcrich v1.2.3 over quick for 100% while backtracking scanned scanner scanner request for hand fox the that hand scanned It's while (parens) written\"}, {\"role\": \"user\", \"content\": \"body there's scanned na\\u00efve quick It's a request lazy hand 1999 https://example.com/a/b?c=d jumps and caf\\u00e9 every It's before we'll on body no lazy there's 'single' mirrors we'll tokens over (parens) while \\u0645\\u0631\\u062d\\u0628\\u0627 C++ v1.2.3 keep gateway it because request mirrors for there's $1,234.56 keep over gateway https://example.com/a/b?c=d brown before They'RE 42 [brackets] tokens once once while scanner scanned and I'M Z\\u00fcrich it involved na\\u00efve I'M hand exactly before node.js 'single' because no lazy 3.14159 It's scanner F# 'single' while exactly It's (parens) \\ud83d\\ude42 counting while for tokens backtracking fox (parens) v1.2.3 once quick because is there's reservation because It's node.js that regex for don't\"}, {\"role\": \"assistant\", \"content\": \"with with so written budget is jumps admission before that I'M \\\"quotes\\\" because 3.14159 on WON'T \\\"quotes\\\" caf\\u00e9 is for scanned engine so engine on admission no body before once node.js node.js [brackets] They'RE Z\\u00fcrich written the \\ud83d\\ude42 reservation once so it a budget the request \\u6771\\u4eac https://example.com/a/b?c=d request \\u0645\\u0631\\u062d\\u0628\\u0627 a the 3.14159 with mirrors because body [brackets] exactly \\ud83d\\ude42 the a 1999 every lazy 'single' the request keep once dog user@example.com the scanned \\\"quotes\\\" is regex scanner every hand keep faster request quick WON'T while 3.14159 It's 3.14159 mirrors reservation written the a It's don't reservation C++ It's 42 v1.2.3 involved reservation lazy keep \\\"quotes\\\" the body \\ud83d\\ude42 while 3.14159 node.js dog no user@example.com budget [brackets] hand over\"}, {\"role\": \"user\", \"content\": \"before 1999 \\ud83d\\ude42 on that don't Z\\u00fcrich keep a way for request \\u6771\\u4eac every \\\"quotes\\\" https://example.com/a/b?c=d 3.14159 once for every hand that WON'T 'single' request written jumps regex F# once quick \\u6771\\u4eac dog na\\u00efve the node.js way [brackets] gateway (parens) piece exactly \\ud83d\\ude42 v1.2.3 jumps They'RE written fox the scanner exactly WON'T na\\u00efve so body faster brown the that is with exactly Z\\u00fcrich [brackets] Z\\u00fcrich budget it lazy and the engine budget 3.14159 counting \\ud83d\\ude42 https://example.com/a/b?c=d it WON'T $1,234.56 {braces} a because we'll exactly it with request the that don't scanner jumps involved and tiktoken's the quick so \\ud83d\\ude42 [brackets] regex engine involved is\"}, {\"role\": \"assistant\", \"content\": \"They'RE body because engine 100% WON'T {braces} the tiktoken's before node.js it every na\\u00efve lazy the don't caf\\u00e9 admission 1999 faster dog 3.14159 {braces} once tokens we'll before the admission WON'T the it and It's because engine with user@example.com it \\u6771\\u4eac C++ over is tokens piece exactly it piece backtracking gateway node.js no the 3.14159 exactly regex fox \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors a 1999 v1.2.3 request 100% scanned reservation node.js I'M the 100% so quick before piece brown caf\\u00e9 gateway 42 involved It's involved admission Z\\u00fcrich and {braces} It's quick \\ud83d\\ude42 body fox counting\"}, {\"role\": \"user\", \"content\": \"I'M admission regex that involved engine it \\ud83d\\ude42 counting every dog hand before scanned the 100% we'll exactly Z\\u00fcrich \\u6771\\u4eac v1.2.3 engine na\\u00efve node.js body don't lazy tiktoken's gateway don't node.js request F# scanner there's 100% scanner fox the a 42 quick na\\u00efve admission while 1999 way regex there's \\\"quotes\\\" scanned is so before tiktoken's involved [brackets] quick 1999 \\u6771\\u4eac 1999 for lazy v1.2.3 regex lazy body F# hand boundaries once mirrors so brown \\\"quotes\\\" dog $1,234.56 tokens that once request hand tokens lazy lazy [brackets] quick don't mirrors quick don't lazy It's body boundaries mirrors F# [brackets] there's while 100% body\"}, {\"role\": \"assistant\", \"content\": \"budget before over They'RE I'M request is that while written before $1,234.56 {braces} F# that admission and with for [brackets] fox \\ud83d\\ude42 42 body \\\"quotes\\\" the F# reservation dog 3.14159 WON'T It's tiktoken's exactly regex 3.14159 there's involved backtracking don't 1999 with faster dog body before tiktoken's with and over 42 written there's $1,234.56 hand 42 \\\"quotes\\\" lazy gateway backtracking 100% because \\\"quotes\\\" hand caf\\u00e9 faster request on over \\\"quotes\\\" once scanned over 100% engine mirrors \\u6771\\u4eac gateway user@example.com fox every before [brackets] body admission WON'T engine no dog for node.js gateway node.js na\\u00efve piece the way scanner a the 'single' gateway user@example.com They'RE is don't \\u6771\\u4eac before scanned\"}, {\"role\": \"user\", \"content\": \"It's way that \\u0645\\u0631\\u062d\\u0628\\u0627 written tiktoken's over faster It's with WON'T it because faster because with backtracking Z\\u00fcrich the while because scanned mirrors that scanned They'RE fox exactly 100% It's the tiktoken's hand so user@example.com request \\u6771\\u4eac faster v1.2.3 quick on once over dog piece counting node.js They'RE the because (parens) hand F# \\u6771\\u4eac with hand dog over budget written Z\\u00fcrich jumps there's budget C++ backtracking v1.2.3 'single' don't fox They'RE for \\ud83d\\ude42 admission involved v1.2.3 3.14159 the while 42 quick regex 100% on node.js tokens scanner written every with They'RE \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"100% because user@example.com caf\\u00e9 way 1999 Z\\u00fcrich over written 'single' 3.14159 $1,234.56 tokens I'M C++ [brackets] na\\u00efve $1,234.56 we'll $1,234.56 node.js we'll it (parens) {braces} v1.2.3 with written admission 100% exactly {braces} reservation there's {braces} fox with engine na\\u00efve for 1999 brown hand while dog backtracking backtracking na\\u00efve user@example.com tokens \\u0645\\u0631\\u062d\\u0628\\u0627 body brown don't C++ piece over scanner (parens) 100% no F# keep there's v1.2.3 [brackets] we'll don't hand dog every engine 'single' v1.2.3\"}, {\"role\": \"user\", \"content\": \"before and every $1,234.56 caf\\u00e9 the mirrors reservation https://example.com/a/b?c=d 3.14159 the quick gateway exactly it $1,234.56 1999 v1.2.3 $1,234.56 it on node.js backtracking (parens) there's \\ud83d\\ude42 no quick regex \\ud83d\\ude42 before once \\u0645\\u0631\\u062d\\u0628\\u0627 involved gateway caf\\u00e9 admission $1,234.56 caf\\u00e9 backtracking scanner na\\u00efve a engine [brackets] a way written is caf\\u00e9 1999 gateway 100% boundaries [brackets] backtracking there's tokens while 42 na\\u00efve counting it C++ v1.2.3 100% v1.2.3 scanned \\u0645\\u0631\\u062d\\u0628\\u0627 'single' \\u6771\\u4eac WON'T there's keep It's so WON'T\"}, {\"role\": \"assistant\", \"content\": \"keep 42 because quick and because faster is 'single' scanned fox regex {braces} C++ I'M it 100% written tiktoken's engine so scanned reservation \\ud83d\\ude42 3.14159 \\\"quotes\\\" before the that exactly for with there's 100% over with WON'T It's quick Z\\u00fcrich v1.2.3 'single' the tiktoken's (parens) while we'll written [brackets] reservation the on engine way na\\u00efve caf\\u00e9 exactly $1,234.56 https://example.com/a/b?c=d before written 'single' (parens) piece They'RE Z\\u00fcrich written counting the every that written na\\u00efve hand admission over piece Z\\u00fcrich we'll no quick https://example.com/a/b?c=d keep we'll while 'single' request quick on we'll tiktoken's on regex Z\\u00fcrich jumps written that a because mirrors\"}, {\"role\": \"user\", \"content\": \"\\\"quotes\\\" reservation na\\u00efve there's 100% dog reservation [brackets] 3.14159 F# node.js \\u0645\\u0631\\u062d\\u0628\\u0627 the fox boundaries C++ na\\u00efve before It's so written brown C++ lazy 3.14159 100% [brackets] tokens lazy tiktoken's {braces} with and and fox I'M for the exactly gateway keep backtracking that fox Z\\u00fcrich quick lazy every \\u0645\\u0631\\u062d\\u0628\\u0627 hand Z\\u00fcrich \\u6771\\u4eac caf\\u00e9 C++ there's keep the over v1.2.3 mirrors user@example.com It's They'RE tiktoken's 1999 because piece jumps it dog a budget boundaries brown exactly piece lazy 100% so scanned https://example.com/a/b?c=d there's that don't the engine fox tiktoken's 42 $1,234.56 \\u6771\\u4eac keep the dog admission boundaries piece It's mirrors fox user@example.com boundaries I'M faster reservation It's for no fox \\u0645\\u0631\\u062d\\u0628\\u0627 and because boundaries budget involved written They'RE v1.2.3 admission while brown F# so 'single' body that\"}, {\"role\": \"assistant\", \"content\": \"body tokens exactly over dog Z\\u00fcrich WON'T exactly F# no engine and \\ud83d\\ude42 scanner way {braces} scanner jumps tokens piece [brackets] hand that na\\u00efve \\\"quotes\\\" dog for node.js the v1.2.3 that They'RE once gateway so {braces} with keep that regex no [brackets] admission on caf\\u00e9 fox scanner tokens don't before quick They'RE budget dog C++ once jumps user@example.com a node.js (parens) because WON'T \\ud83d\\ude42 while request 3.14159 with because there's regex don't Z\\u00fcrich written user@example.com with while request keep They'RE mirrors 1999 over involved \\u6771\\u4eac $1,234.56 C++ counting involved that tokens way for \\ud83d\\ude42 regex jumps we'll because na\\u00efve lazy with I'M don't before regex reservation fox that so\"}, {\"role\": \"user\", \"content\": \"involved because dog 1999 exactly 1999 keep boundaries node.js tiktoken's once Z\\u00fcrich [brackets] way na\\u00efve v1.2.3 faster C++ scanner mirrors \\u6771\\u4eac \\u6771\\u4eac [brackets] hand \\ud83d\\ude42 node.js tiktoken's don't counting exactly tiktoken's keep WON'T while with 42 brown it way Z\\u00fcrich over because keep before way {braces} the before way boundaries that \\u6771\\u4eac 1999 request mirrors over It's gateway with scanner It's tiktoken's Z\\u00fcrich gateway faster we'll Z\\u00fcrich admission\"}, {\"role\": \"assistant\", \"content\": \"once They'RE node.js we'll \\u0645\\u0631\\u062d\\u0628\\u0627 don't Z\\u00fcrich admission {braces} It's \\ud83d\\ude42 \\u6771\\u4eac gateway brown is so scanner [brackets] {braces} there's tiktoken's a \\u0645\\u0631\\u062d\\u0628\\u0627 node.js and [brackets] don't counting \\\"quotes\\\" They'RE every mirrors reservation body on piece on regex node.js a 'single' \\u6771\\u4eac tiktoken's \\\"quotes\\\" for scanned there's faster gateway They'RE mirrors jumps https://example.com/a/b?c=d faster tokens every WON'T WON'T that request budget It's counting we'll scanner faster tokens that [brackets] tiktoken's there's F# lazy the reservation on the because with lazy user@example.com $1,234.56 jumps gateway C++ [brackets]\"}, {\"role\": \"user\", \"content\": \"over on regex lazy piece don't (parens) don't so https://example.com/a/b?c=d WON'T v1.2.3 brown node.js 100% \\u0645\\u0631\\u062d\\u0628\\u0627 over admission {braces} https://example.com/a/b?c=d scanned hand It's tiktoken's C++ quick don't They'RE {braces} WON'T scanner there's admission the body $1,234.56 dog \\u6771\\u4eac Z\\u00fcrich involved (parens) for so hand 3.14159 so scanned \\u0645\\u0631\\u062d\\u0628\\u0627 42 100% \\u6771\\u4eac counting tiktoken's 1999 fox keep reservation caf\\u00e9 there's tokens quick F# no is v1.2.3 body \\ud83d\\ude42 brown dog way boundaries scanned node.js quick over admission user@example.com boundaries faster regex on 42 admission na\\u00efve engine lazy WON'T that user@example.com 100% I'M with because faster exactly way for faster C++ because scanner written the \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"there's for I'M on reservation once once mirrors the body body 3.14159 fox dog \\\"quotes\\\" written we'll \\ud83d\\ude42 regex with user@example.com fox 1999 Z\\u00fcrich na\\u00efve is hand hand before on for over piece exactly that the exactly 100% caf\\u00e9 that brown counting way admission 100% 1999 v1.2.3 don't \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} mirrors it once piece Z\\u00fcrich gateway caf\\u00e9 and the budget that because hand scanner tokens request regex for exactly over piece F# we'll [brackets] I'M while and fox scanner \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} on fox quick mirrors It's scanner the scanner no it because node.js 'single' regex written caf\\u00e9 v1.2.3 https://example.com/a/b?c=d {braces} the don't jumps It's hand 'single' scanned 3.14159 involved every fox over we'll keep there's mirrors written on I'M WON'T F# that F# that 3.14159 1999 keep 'single'\"}, {\"role\": \"user\", \"content\": \"exactly involved caf\\u00e9 that 1999 over a there's it F# exactly \\ud83d\\ude42 tokens we'll don't regex fox because that while involved backtracking involved reservation there's over They'RE admission 1999 regex counting keep $1,234.56 for engine it 'single' regex the that body v1.2.3 WON'T once It's 42 tiktoken's written no 'single' piece fox and before it tiktoken's a caf\\u00e9 v1.2.3 user@example.com quick user@example.com scanned admission the scanner on is over reservation request \\u6771\\u4eac we'll 3.14159 with before \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking 3.14159 counting reservation don't way with WON'T involved before It's admission keep C++ written with counting 'single' brown the 100% They'RE 42 before while every and \\\"quotes\\\" for They'RE there's 42 tokens v1.2.3 https://example.com/a/b?c=d before involved there's body once\"}, {\"role\": \"assistant\", \"content\": \"Z\\u00fcrich backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 the 3.14159 involved \\\"quotes\\\" 42 node.js once scanner mirrors {braces} na\\u00efve I'M brown the admission over They'RE piece it faster so request don't over way na\\u00efve (parens) once mirrors 100% don't scanner Z\\u00fcrich hand budget fox there's there's boundaries Z\\u00fcrich don't \\u6771\\u4eac scanner 42 v1.2.3 \\\"quotes\\\" na\\u00efve 3.14159 100% every we'll 3.14159 quick 1999 backtracking {braces} don't lazy dog lazy a dog the tokens with body that quick brown tokens lazy \\u6771\\u4eac piece piece every dog every 1999 \\\"quotes\\\" mirrors faster fox the 3.14159 way piece boundaries user@example.com there's caf\\u00e9 faster on F# https://example.com/a/b?c=d we'll admission \\u0645\\u0631\\u062d\\u0628\\u0627 that written piece scanned counting and admission we'll Z\\u00fcrich dog 3.14159 node.js every engine Z\\u00fcrich body involved backtracking mirrors 100% 100%\"}, {\"role\": \"user\", \"content\": \"(parens) that on 3.14159 \\u6771\\u4eac over is 42 \\\"quotes\\\" 'single' body faster exactly mirrors F# way (parens) no exactly mirrors that \\ud83d\\ude42 $1,234.56 faster brown Z\\u00fcrich a brown tiktoken's \\ud83d\\ude42 tokens regex while They'RE I'M budget node.js https://example.com/a/b?c=d 42 tiktoken's https://example.com/a/b?c=d over over every 'single' 100% tiktoken's scanner gateway for don't tiktoken's a hand v1.2.3 {braces} fox that quick every that fox faster exactly 'single' (parens) v1.2.3 request dog over no is jumps for Z\\u00fcrich WON'T so body once scanner 3.14159 every that \\u6771\\u4eac so\"}, {\"role\": \"assistant\", \"content\": \"no while faster the because there's \\\"quotes\\\" piece tiktoken's a reservation Z\\u00fcrich because caf\\u00e9 gateway tokens gateway over Z\\u00fcrich They'RE that 3.14159 quick They'RE na\\u00efve mirrors faster a 100% reservation I'M on F# lazy 1999 and budget and $1,234.56 exactly budget written fox 100% body and there's don't once body the on while budget \\ud83d\\ude42 100% \\\"quotes\\\" request \\u0645\\u0631\\u062d\\u0628\\u0627 keep lazy 1999 before piece that before involved \\ud83d\\ude42 written don't gateway the once user@example.com (parens) that backtracking budget request involved the v1.2.3 42 hand tokens mirrors I'M written \\u6771\\u4eac \\\"quotes\\\" body [brackets] written 'single' brown so request 3.14159 node.js budget node.js dog (parens) scanner {braces} a WON'T v1.2.3 because there's that engine tiktoken's every no over before on mirrors mirrors don't \\u0645\\u0631\\u062d\\u0628\\u0627 budget for quick 42 quick over lazy over v1.2.3 gateway \\u0645\\u0631\\u062d\\u0628\\u0627 and\"}, {\"role\": \"user\", \"content\": \"that caf\\u00e9 user@example.com hand over we'll no that Z\\u00fcrich brown 42 so fox counting quick every regex https://example.com/a/b?c=d brown once body 'single' reservation v1.2.3 $1,234.56 I'M that on \\ud83d\\ude42 F# scanner faster over F# we'll dog because mirrors \\ud83d\\ude42 we'll scanned regex budget on and I'M admission is written It's fox na\\u00efve with WON'T involved 'single' user@example.com WON'T the [brackets] exactly 42 'single' {braces} involved user@example.com They'RE written before keep tokens dog It's over on \\ud83d\\ude42 a that 1999 reservation I'M for that fox it boundaries no hand for $1,234.56 They'RE jumps hand WON'T https://example.com/a/b?c=d faster over [brackets] 3.14159 Z\\u00fcrich\"}, {\"role\": \"assistant\", \"content\": \"100% the quick scanner is with node.js [brackets] 100% keep scanned because brown every that $1,234.56 \\\"quotes\\\" because 3.14159 it 42 tiktoken's because once 1999 user@example.com node.js the admission it quick scanner involved It's $1,234.56 user@example.com 1999 3.14159 exactly https://example.com/a/b?c=d na\\u00efve dog counting exactly quick v1.2.3 piece involved once v1.2.3 exactly 42 scanner gateway (parens) request \\u6771\\u4eac fox keep 42 node.js while keep with the gateway with F# over no involved caf\\u00e9 [brackets] request They'RE scanner on with v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 every the 42 It's every jumps a scanned involved the brown scanned quick the tokens I'M I'M over no over faster v1.2.3 caf\\u00e9 \\\"quotes\\\" body request that fox fox before node.js brown dog a dog body engine faster Z\\u00fcrich https://example.com/a/b?c=d before I'M we'll hand WON'T admission and counting while on quick and\"}, {\"role\": \"user\", \"content\": \"na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 mirrors mirrors while tiktoken's tiktoken's hand Z\\u00fcrich there's backtracking brown brown \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors is (parens) it that tokens v1.2.3 way admission that caf\\u00e9 so exactly engine with scanner budget WON'T \\u6771\\u4eac 'single' hand na\\u00efve there's 'single' \\u6771\\u4eac piece na\\u00efve $1,234.56 is before faster admission request \\ud83d\\ude42 \\u6771\\u4eac counting backtracking written \\\"quotes\\\" there's involved lazy quick faster that request scanned v1.2.3 $1,234.56 every dog with for regex once with [brackets] v1.2.3 so caf\\u00e9 It's written it Z\\u00fcrich lazy caf\\u00e9 scanner counting there's no caf\\u00e9 every F# 100% the C++ every faster we'll on tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 engine $1,234.56 faster faster 1999 that exactly mirrors for admission budget 'single' 3.14159 no engine\"}, {\"role\": \"assistant\", \"content\": \"https://example.com/a/b?c=d there's gateway mirrors body mirrors written for brown node.js for brown node.js 'single' scanner na\\u00efve request is every 100% user@example.com dog 3.14159 no we'll that Z\\u00fcrich don't involved with https://example.com/a/b?c=d fox dog F# reservation C++ a v1.2.3 v1.2.3 backtracking WON'T F# no piece C++ {braces} a mirrors way $1,234.56 that WON'T body They'RE no we'll jumps F# once caf\\u00e9 reservation WON'T that piece 1999 They'RE counting piece They'RE request tiktoken's a engine na\\u00efve scanned body lazy faster It's over It's that tokens Z\\u00fcrich C++ scanner with \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"user\", \"content\": \"scanner while tiktoken's fox on boundaries tiktoken's tokens before tiktoken's it body gateway and over (parens) the (parens) 'single' on C++ faster budget every request once on exactly every that it Z\\u00fcrich quick regex and user@example.com jumps it user@example.com quick 100% it exactly every 'single' mirrors body jumps counting caf\\u00e9 tokens admission that piece scanner {braces} tokens is \\ud83d\\ude42 request admission C++ is node.js body for body They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking WON'T\"}, {\"role\": \"assistant\", \"content\": \"dog C++ (parens) It's with 'single' we'll [brackets] request is there's 3.14159 scanned 100% for request on admission $1,234.56 over there's scanned engine scanner They'RE that lazy \\u6771\\u4eac before budget user@example.com once C++ $1,234.56 that scanner exactly tokens is engine \\ud83d\\ude42 dog user@example.com na\\u00efve They'RE we'll \\ud83d\\ude42 that 'single' jumps the Z\\u00fcrich on jumps involved hand involved tiktoken's It's C++ \\ud83d\\ude42 user@example.com that before user@example.com $1,234.56 don't once scanner and a user@example.com user@example.com exactly and user@example.com $1,234.56 admission backtracking budget written 100% we'll 1999 scanned mirrors every \\u0645\\u0631\\u062d\\u0628\\u0627 a brown dog mirrors reservation 1999 regex hand budget \\u6771\\u4eac user@example.com it before 'single' hand over with {braces} no the a keep quick tiktoken's 3.14159 caf\\u00e9 every Z\\u00fcrich F# \\u6771\\u4eac the I'M Z\\u00fcrich na\\u00efve no way gateway na\\u00efve https://example.com/a/b?c=d request {braces} backtracking\"}, {\"role\": \"user\", \"content\": \"\\\"quotes\\\" \\\"quotes\\\" scanned is lazy user@example.com v1.2.3 regex a It's tokens piece They'RE engine over Z\\u00fcrich 3.14159 {braces} 'single' (parens) a boundaries node.js request jumps body before for hand dog boundaries budget on the boundaries we'll every because {braces} Z\\u00fcrich way https://example.com/a/b?c=d 1999 body Z\\u00fcrich there's budget 1999 before so v1.2.3 WON'T user@example.com F# it written lazy with once dog faster faster once we'll \\ud83d\\ude42 exactly user@example.com faster 42 \\u0645\\u0631\\u062d\\u0628\\u0627 a admission over \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T lazy\"}, {\"role\": \"assistant\", \"content\": \"na\\u00efve the (parens) for the brown https://example.com/a/b?c=d and 42 Z\\u00fcrich on {braces} \\ud83d\\ude42 regex over regex na\\u00efve F# over C++ before with the for don't 'single' counting \\u6771\\u4eac tiktoken's before mirrors fox gateway faster so regex \\ud83d\\ude42 keep $1,234.56 reservation so \\ud83d\\ude42 42 https://example.com/a/b?c=d tokens user@example.com for the for exactly \\\"quotes\\\" it written for F# dog I'M engine faster \\u6771\\u4eac gateway dog na\\u00efve is {braces} tokens over request backtracking 1999 body request written 'single' They'RE tokens F# tokens brown C++ boundaries admission every budget we'll no budget it keep don't https://example.com/a/b?c=d every exactly caf\\u00e9 'single' there's C++ 'single' dog v1.2.3 backtracking They'RE engine (parens) 'single' gateway every it $1,234.56 1999 lazy once reservation over fox that {braces} 1999 budget node.js written C++ way [brackets] before \\ud83d\\ude42 gateway on tiktoken's and (parens) the written \\u6771\\u4eac\"}, {\"role\": \"user\", \"content\": \"user@example.com scanned 42 user@example.com piece backtracking every engine that \\u0645\\u0631\\u062d\\u0628\\u0627 request $1,234.56 lazy Z\\u00fcrich mirrors fox no that so It's {braces} It's piece mirrors v1.2.3 regex quick counting for They'RE F# 3.14159 once brown no that no \\u6771\\u4eac so 100% scanner 1999 counting fox there's once engine caf\\u00e9 brown [brackets] piece hand mirrors budget lazy over the counting fox counting is for I'M faster a with node.js we'll \\u6771\\u4eac it 1999 user@example.com that It's mirrors written lazy lazy the before faster piece na\\u00efve fox piece 3.14159 fox na\\u00efve dog regex [brackets] [brackets] \\u6771\\u4eac over body user@example.com 42 node.js while and https://example.com/a/b?c=d na\\u00efve piece faster brown (parens) \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 \\\"quotes\\\"\"}, {\"role\": \"assistant\", \"content\": \"faster written 100% $1,234.56 engine 'single' gateway faster the boundaries exactly on C++ so so mirrors once backtracking caf\\u00e9 quick 1999 while we'll WON'T a Z\\u00fcrich boundaries involved piece scanner {braces} written 1999 and 3.14159 there's \\ud83d\\ude42 that the because quick scanner 3.14159 that it way counting na\\u00efve reservation for before $1,234.56 over quick engine scanner no the we'll scanned backtracking that no don't before 1999 \\u6771\\u4eac we'll before 3.14159 $1,234.56 budget counting tokens jumps while before tokens node.js tokens gateway scanner that \\ud83d\\ude42 3.14159 'single' WON'T so way (parens) backtracking \\\"quotes\\\" the v1.2.3 {braces} body body C++ it 'single'\"}, {\"role\": \"user\", \"content\": \"because request that that dog counting 100% mirrors I'M fox there's the piece with budget regex while before tokens with over there's \\u0645\\u0631\\u062d\\u0628\\u0627 reservation body on reservation tiktoken's and \\u0645\\u0631\\u062d\\u0628\\u0627 while so quick that budget quick brown and user@example.com \\u6771\\u4eac 100% dog keep there's It's WON'T caf\\u00e9 quick there's $1,234.56 way user@example.com budget for v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 quick way boundaries is reservation 42 no that \\ud83d\\ude42 the hand \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) na\\u00efve $1,234.56 3.14159 quick 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors na\\u00efve hand once every F# the \\ud83d\\ude42 once piece I'M 42 with involved the {braces} [brackets] a 'single' caf\\u00e9 that once before once 'single' counting {braces} every reservation budget jumps WON'T\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 {braces} it They'RE scanned for \\u6771\\u4eac hand mirrors written 42 with node.js quick admission I'M counting \\u0645\\u0631\\u062d\\u0628\\u0627 brown \\u6771\\u4eac 100% over \\u6771\\u4eac that \\ud83d\\ude42 \\u6771\\u4eac It's \\ud83d\\ude42 scanner budget so dog that regex keep hand piece lazy no is user@example.com counting tiktoken's request admission we'll mirrors hand gateway boundaries no https://example.com/a/b?c=d on engine reservation no that {braces} mirrors brown Z\\u00fcrich admission piece user@example.com scanned and with 100% the tiktoken's fox WON'T written\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich is over 'single' C++ It's gateway [brackets] no over way quick admission no faster written node.js 42 v1.2.3 involved {braces} 3.14159 because [brackets] reservation body before the exactly every [brackets] tokens so fox piece boundaries boundaries v1.2.3 caf\\u00e9 I'M F# 1999 backtracking user@example.com with with don't body a C++ brown don't It's caf\\u00e9 regex They'RE way https://example.com/a/b?c=d scanned 'single' boundaries that reservation \\ud83d\\ude42 brown fox node.js backtracking gateway na\\u00efve (parens)\"}, {\"role\": \"assistant\", \"content\": \"piece WON'T body it $1,234.56 that 3.14159 1999 it reservation \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's boundaries 'single' It's reservation gateway C++ body budget mirrors is the backtracking and \\u0645\\u0631\\u062d\\u0628\\u0627 gateway a 'single' is and while 42 hand it They'RE caf\\u00e9 backtracking counting gateway exactly F# because na\\u00efve scanned that C++ is 100% involved admission piece 3.14159 quick it the lazy involved WON'T a before It's tokens They'RE it mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 reservation it \\ud83d\\ude42 na\\u00efve so is a and that request while that written body keep reservation the on every They'RE WON'T budget scanned reservation $1,234.56 fox is there's $1,234.56 brown involved admission\"}, {\"role\": \"user\", \"content\": \"way F# don't brown v1.2.3 the regex na\\u00efve involved brown budget because dog that WON'T and v1.2.3 for it counting because because because brown \\u6771\\u4eac backtracking (parens) I'M that dog reservation keep that scanner request WON'T They'RE backtracking admission every 42 because 100% [brackets] https://example.com/a/b?c=d admission way the so don't I'M \\ud83d\\ude42 3.14159 that C++ 3.14159 https://example.com/a/b?c=d na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com node.js way I'M over https://example.com/a/b?c=d engine once mirrors Z\\u00fcrich F# fox on \\u0645\\u0631\\u062d\\u0628\\u0627 na\\u00efve mirrors piece with the \\ud83d\\ude42 WON'T counting involved don't written boundaries gateway keep 1999 boundaries so Z\\u00fcrich way piece every user@example.com exactly They'RE piece boundaries written brown I'M C++ gateway keep the tiktoken's reservation counting Z\\u00fcrich a involved\"}, {\"role\": \"assistant\", \"content\": \"way is a for scanned dog exactly don't body na\\u00efve [brackets] fox na\\u00efve {braces} that body for scanner reservation user@example.com quick fox hand faster reservation mirrors (parens) counting no we'll a exactly reservation user@example.com tiktoken's hand tokens don't C++ body na\\u00efve hand with backtracking for engine for They'RE admission {braces} fox boundaries keep quick it with once body the body the budget \\\"quotes\\\" caf\\u00e9 a admission $1,234.56 we'll way v1.2.3 3.14159 reservation fox WON'T 3.14159 once that written Z\\u00fcrich for na\\u00efve once piece tiktoken's over engine \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 F# 3.14159 lazy \\ud83d\\ude42 user@example.com na\\u00efve \\u6771\\u4eac don't because 'single' They'RE \\ud83d\\ude42 \\u6771\\u4eac body because lazy a for exactly request tiktoken's gateway no C++ before with every {braces} keep v1.2.3 faster mirrors (parens) (parens) and quick scanned I'M budget engine with\"}, {\"role\": \"user\", \"content\": \"regex with exactly \\u0645\\u0631\\u062d\\u0628\\u0627 dog budget that 42 {braces} tokens https://example.com/a/b?c=d tiktoken's that I'M we'll 100% engine counting F# no regex node.js no boundaries way tiktoken's because \\\"quotes\\\" exactly 3.14159 and [brackets] mirrors written keep that F# backtracking scanned They'RE it gateway over {braces} node.js 100% budget scanner budget \\\"quotes\\\" we'll v1.2.3 we'll caf\\u00e9 \\ud83d\\ude42 tokens don't once because mirrors backtracking na\\u00efve every faster once budget caf\\u00e9 it I'M caf\\u00e9 a backtracking regex fox while engine faster scanned I'M tiktoken's don't \\\"quotes\\\" 3.14159 scanner It's na\\u00efve request boundaries hand lazy no $1,234.56 every for jumps written 1999 mirrors {braces} WON'T lazy engine and\"}, {\"role\": \"assistant\", \"content\": \"and reservation caf\\u00e9 it (parens) before piece backtracking and WON'T brown counting that 1999 $1,234.56 tokens user@example.com user@example.com it faster It's the body and request brown caf\\u00e9 keep scanner \\u0645\\u0631\\u062d\\u0628\\u0627 for tokens caf\\u00e9 'single' request boundaries 'single' request They'RE user@example.com tokens quick there's keep exactly a scanned is regex it hand quick hand {braces} so 'single' gateway and\"}, {\"role\": \"user\", \"content\": \"na\\u00efve is request that It's the way caf\\u00e9 the 42 there's quick user@example.com the over that $1,234.56 with user@example.com quick They'RE budget the it F# gateway It's fox we'll involved tiktoken's na\\u00efve over while https://example.com/a/b?c=d that na\\u00efve tiktoken's 42 scanned we'll na\\u00efve and tiktoken's (parens) piece 'single' exactly Z\\u00fcrich budget WON'T (parens) It's engine we'll body involved and keep\"}, {\"role\": \"assistant\", \"content\": \"way v1.2.3 it so once before \\\"quotes\\\" WON'T WON'T written and over F# the with no boundaries v1.2.3 body that dog exactly Z\\u00fcrich backtracking there's on \\u6771\\u4eac mirrors request on They'RE hand regex engine reservation it scanned \\\"quotes\\\" hand boundaries for reservation before brown WON'T once hand \\u0645\\u0631\\u062d\\u0628\\u0627 body 42 \\u0645\\u0631\\u062d\\u0628\\u0627 once every admission body $1,234.56 before $1,234.56 WON'T {braces} na\\u00efve mirrors before the that 3.14159 way brown It's 1999 gateway reservation F# for user@example.com exactly jumps https://example.com/a/b?c=d is and node.js 100% boundaries v1.2.3 'single' we'll no gateway They'RE with F# [brackets] C++ mirrors for tokens written every\"}, {\"role\": \"user\", \"content\": \"WON'T na\\u00efve 'single' It's 3.14159 \\ud83d\\ude42 don't 'single' \\\"quotes\\\" involved C++ caf\\u00e9 with mirrors 100% It's that no user@example.com WON'T before is gateway once node.js gateway while on quick \\ud83d\\ude42 boundaries \\u6771\\u4eac budget 1999 admission They'RE mirrors engine I'M is \\u0645\\u0631\\u062d\\u0628\\u0627 I'M budget for 'single' on \\u6771\\u4eac with WON'T and because brown regex hand reservation scanner 100% 100% user@example.com there's it\"}, {\"role\": \"assistant\", \"content\": \"there's exactly [brackets] faster budget while $1,234.56 (parens) engine caf\\u00e9 They'RE no 100% for because $1,234.56 regex Z\\u00fcrich https://example.com/a/b?c=d tokens jumps because counting brown user@example.com request lazy \\u0645\\u0631\\u062d\\u0628\\u0627 request scanned WON'T $1,234.56 exactly brown every mirrors with 'single' engine 42 \\ud83d\\ude42 engine reservation hand lazy 'single' counting hand (parens) gateway that counting user@example.com 42 fox request \\\"quotes\\\" hand lazy 100% 42 I'M It's WON'T https://example.com/a/b?c=d engine while\"}, {\"role\": \"user\", \"content\": \"C++ \\\"quotes\\\" we'll gateway \\ud83d\\ude42 counting written piece na\\u00efve \\u6771\\u4eac WON'T over caf\\u00e9 backtracking engine https://example.com/a/b?c=d the v1.2.3 that so quick counting na\\u00efve \\u6771\\u4eac don't WON'T It's we'll that the 100% request I'M brown na\\u00efve tokens engine budget before the scanned mirrors dog engine {braces} {braces} 42 don't WON'T It's piece \\\"quotes\\\" that no quick 3.14159 F# once fox don't for scanner \\u6771\\u4eac it brown backtracking tokens $1,234.56 reservation written They'RE the \\\"quotes\\\" caf\\u00e9 backtracking C++ reservation\"}, {\"role\": \"assistant\", \"content\": \"regex (parens) Z\\u00fcrich engine https://example.com/a/b?c=d written tiktoken's C++ dog It's lazy dog exactly with it that admission way no scanner a 3.14159 on They'RE It's involved caf\\u00e9 brown on dog that https://example.com/a/b?c=d don't \\u6771\\u4eac jumps \\ud83d\\ude42 and \\u6771\\u4eac so dog so don't (parens) scanned engine budget the exactly F# tokens over \\u0645\\u0631\\u062d\\u0628\\u0627 jumps backtracking 3.14159 budget 100% boundaries gateway backtracking don't scanner dog with [brackets] gateway 3.14159 caf\\u00e9 faster budget na\\u00efve caf\\u00e9 way for every reservation backtracking request They'RE It's \\u6771\\u4eac written\"}, {\"role\": \"user\", \"content\": \"42 1999 so boundaries $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} budget backtracking [brackets] we'll admission piece node.js admission Z\\u00fcrich fox a involved request is it node.js node.js so \\u6771\\u4eac [brackets] no \\u6771\\u4eac brown node.js quick na\\u00efve scanner C++ user@example.com admission 'single' involved 1999 user@example.com I'M v1.2.3 It's 100% there's gateway F# node.js gateway scanned that caf\\u00e9 3.14159 Z\\u00fcrich no admission {braces} tiktoken's over It's I'M 1999 [brackets] body written scanned that scanner on the on with that scanned hand\"}, {\"role\": \"assistant\", \"content\": \"{braces} regex [brackets] piece \\ud83d\\ude42 mirrors there's tiktoken's It's lazy mirrors every v1.2.3 I'M that boundaries 'single' node.js on C++ user@example.com budget user@example.com no user@example.com every engine while a I'M over [brackets] WON'T piece over there's lazy I'M that $1,234.56 budget https://example.com/a/b?c=d it a request exactly 42 keep They'RE caf\\u00e9 tokens \\u0645\\u0631\\u062d\\u0628\\u0627 keep scanner while dog F# mirrors backtracking we'll gateway scanner so we'll request there's budget it lazy scanner for exactly faster we'll reservation scanner no WON'T request (parens) tiktoken's exactly before so while hand involved with $1,234.56 mirrors 3.14159 on the [brackets] boundaries 100% with request way engine keep the https://example.com/a/b?c=d \\u6771\\u4eac the WON'T that so\"}, {\"role\": \"user\", \"content\": \"body hand backtracking (parens) counting na\\u00efve They'RE \\ud83d\\ude42 WON'T boundaries v1.2.3 budget na\\u00efve admission once the that because tokens dog tiktoken's scanner scanner (parens) I'M \\\"quotes\\\" budget 100% there's once WON'T keep {braces} the before fox (parens) the counting because lazy 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 admission 1999 and there's while admission 3.14159 a counting C++ with exactly don't don't lazy \\ud83d\\ude42 42 \\\"quotes\\\" backtracking C++ there's keep user@example.com request is node.js piece before brown budget 42 100% because admission and with tiktoken's no the 100%\"}, {\"role\": \"assistant\", \"content\": \"with 1999 Z\\u00fcrich that is with mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 the brown $1,234.56 way scanner https://example.com/a/b?c=d lazy written scanner quick v1.2.3 \\\"quotes\\\" regex \\\"quotes\\\" They'RE 3.14159 \\ud83d\\ude42 [brackets] Z\\u00fcrich the counting for so $1,234.56 It's WON'T the quick hand that mirrors the 42 caf\\u00e9 exactly node.js we'll backtracking while I'M and fox is before written it 1999 no for involved \\u0645\\u0631\\u062d\\u0628\\u0627 the \\\"quotes\\\" piece the way involved that no 100% before there's 100% we'll is [brackets] C++ the we'll the node.js reservation is while [brackets] engine scanner we'll no C++ don't dog v1.2.3 that so tiktoken's before \\ud83d\\ude42 mirrors 'single' it once caf\\u00e9 there's backtracking gateway body over quick way and that once \\u6771\\u4eac 'single' we'll (parens) and before for quick They'RE 1999 regex there's \\ud83d\\ude42\"}, {\"role\": \"user\", \"content\": \"engine They'RE https://example.com/a/b?c=d we'll quick tokens written WON'T don't regex backtracking It's it lazy 1999 brown budget on [brackets] node.js {braces} They'RE involved a mirrors exactly tokens with $1,234.56 F# 42 backtracking involved engine It's and {braces} \\u6771\\u4eac Z\\u00fcrich it it [brackets] gateway keep na\\u00efve request scanned don't regex we'll C++ counting tokens so request while scanned tokens scanner for v1.2.3 with tokens \\ud83d\\ude42 They'RE na\\u00efve 'single' keep node.js v1.2.3 'single' it way gateway It's keep reservation backtracking body fox with that [brackets] the and 1999 request no engine counting is [brackets] lazy on request quick involved \\ud83d\\ude42 brown v1.2.3 involved fox exactly hand {braces} we'll every regex\"}, {\"role\": \"assistant\", \"content\": \"(parens) faster tiktoken's with the every mirrors boundaries that caf\\u00e9 backtracking involved once 42 \\u6771\\u4eac Z\\u00fcrich {braces} boundaries that [brackets] 1999 it admission before a jumps and 3.14159 way scanner don't {braces} budget for a 3.14159 mirrors boundaries while caf\\u00e9 counting \\u6771\\u4eac a written tiktoken's and we'll https://example.com/a/b?c=d jumps while so [brackets] F# 3.14159 mirrors that {braces} request request I'M keep the over hand engine keep node.js dog na\\u00efve [brackets] every body $1,234.56 \\\"quotes\\\" $1,234.56 we'll \\u0645\\u0631\\u062d\\u0628\\u0627 that 42 budget scanned it They'RE no with the {braces} we'll tiktoken's a \\ud83d\\ude42 tiktoken's https://example.com/a/b?c=d It's exactly It's for over that 100% a \\\"quotes\\\" caf\\u00e9 no 3.14159 hand gateway quick budget over user@example.com I'M\"}, {\"role\": \"user\", \"content\": \"mirrors engine so Z\\u00fcrich and reservation caf\\u00e9 1999 na\\u00efve exactly involved caf\\u00e9 and 1999 over tiktoken's user@example.com over once that It's jumps involved C++ and is node.js brown gateway 100% 100% keep hand backtracking 3.14159 involved hand 'single' once because budget They'RE brown 3.14159 every faster request it regex while \\ud83d\\ude42 1999 exactly way we'll 'single' request scanner boundaries on (parens) we'll\"}, {\"role\": \"assistant\", \"content\": \"once 1999 fox that Z\\u00fcrich \\u6771\\u4eac that we'll 42 'single' dog \\u6771\\u4eac mirrors (parens) because na\\u00efve every and piece C++ 3.14159 https://example.com/a/b?c=d 42 once so gateway is scanned no user@example.com node.js caf\\u00e9 faster node.js tiktoken's It's F# brown it admission body don't don't before scanner before F# 3.14159 na\\u00efve backtracking user@example.com F# that (parens) (parens) there's exactly a reservation it before backtracking It's piece tiktoken's lazy written is {braces} 1999 scanned keep body because mirrors no written that piece boundaries request once scanner \\\"quotes\\\" with engine hand v1.2.3 we'll v1.2.3 the we'll $1,234.56 the quick and tokens backtracking is quick 'single' 42 once reservation admission fox scanner a scanner engine 1999 quick [brackets] \\u6771\\u4eac budget for quick engine keep user@example.com 100% C++ [brackets] hand on Z\\u00fcrich hand Z\\u00fcrich regex 1999\"}, {\"role\": \"user\", \"content\": \"\\u0645\\u0631\\u062d\\u0628\\u0627 scanner They'RE budget C++ exactly {braces} is \\ud83d\\ude42 because way body brown counting with regex it \\\"quotes\\\" we'll body node.js over a node.js that while it v1.2.3 3.14159 quick request that dog node.js They'RE 1999 that gateway 3.14159 gateway there's backtracking lazy C++ admission node.js 42 request They'RE we'll gateway so once 1999 brown na\\u00efve 3.14159 budget faster \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" backtracking that 100% because a fox once involved on involved v1.2.3 WON'T mirrors 'single' a \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" is They'RE that involved reservation They'RE (parens) v1.2.3 Z\\u00fcrich WON'T Z\\u00fcrich fox that don't boundaries engine regex don't exactly na\\u00efve don't caf\\u00e9 request budget [brackets] F# tokens exactly It's request it\"}, {\"role\": \"assistant\", \"content\": \"scanned (parens) reservation {braces} engine tokens v1.2.3 node.js scanner {braces} It's with admission the 100% F# \\u0645\\u0631\\u062d\\u0628\\u0627 body na\\u00efve with 'single' boundaries regex (parens) because mirrors there's keep once lazy it brown involved boundaries request engine before on It's \\ud83d\\ude42 na\\u00efve fox WON'T we'll backtracking jumps exactly scanner it admission mirrors don't that \\u6771\\u4eac faster body mirrors reservation (parens) \\\"quotes\\\" regex mirrors keep exactly They'RE 3.14159 there's a \\u0645\\u0631\\u062d\\u0628\\u0627 and with admission written C++ https://example.com/a/b?c=d the piece jumps that is lazy tokens before 100% boundaries tokens dog tiktoken's it $1,234.56 I'M 100% once (parens) I'M way brown keep\"}, {\"role\": \"user\", \"content\": \"no no hand F# jumps so faster every don't so tiktoken's 'single' I'M [brackets] \\\"quotes\\\" v1.2.3 over admission exactly scanner node.js 100% quick gateway \\ud83d\\ude42 faster budget over faster over and engine request faster fox 'single' backtracking body brown no na\\u00efve is it \\\"quotes\\\" piece with \\ud83d\\ude42 for tokens I'M scanned while admission way faster na\\u00efve mirrors tokens boundaries \\u6771\\u4eac jumps tiktoken's it there's faster\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 no It's piece request \\u6771\\u4eac Z\\u00fcrich don't \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE 3.14159 quick Z\\u00fcrich once tiktoken's with engine jumps over I'M because caf\\u00e9 faster tiktoken's that 'single' we'll dog 42 \\u6771\\u4eac way once counting it C++ on that the that backtracking we'll scanned before 100% na\\u00efve request dog every request 1999 request quick over 1999 gateway https://example.com/a/b?c=d because 3.14159 that reservation mirrors that a over I'M with regex Z\\u00fcrich mirrors piece regex caf\\u00e9 engine Z\\u00fcrich the written v1.2.3 \"}, {\"role\": \"user\", \"content\": \"hand is It's scanned regex 1999 way on It's we'll tokens with reservation while way mirrors regex no over 42 is user@example.com scanner scanned keep budget {braces} brown budget while is involved [brackets] body jumps F# dog no 3.14159 exactly over 100% (parens) It's jumps so we'll \\u6771\\u4eac faster admission exactly admission that for tiktoken's 100% once engine and every a before budget regex They'RE counting\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 involved backtracking regex na\\u00efve way tokens we'll 100% gateway the gateway the body lazy I'M budget is regex body (parens) C++ that body no the jumps [brackets] exactly 1999 budget request brown \\ud83d\\ude42 jumps keep scanned user@example.com for exactly regex dog node.js scanner that node.js 42 engine fox \\ud83d\\ude42 tiktoken's WON'T mirrors written faster while 42 100% \\u6771\\u4eac user@example.com dog engine lazy [brackets] exactly quick node.js written written 100% every \\ud83d\\ude42 body 3.14159 hand because once gateway faster before with It's a 1999 42 boundaries jumps that {braces} 100% caf\\u00e9 a They'RE 3.14159 [brackets] reservation 1999 faster \\ud83d\\ude42 hand 1999 [brackets] scanner\"}, {\"role\": \"user\", \"content\": \"https://example.com/a/b?c=d dog for C++ v1.2.3 because every 100% \\u6771\\u4eac and brown is request piece boundaries on before because that body WON'T [brackets] that keep tokens lazy WON'T keep on for WON'T body quick {braces} gateway body written tokens quick the mirrors is is with I'M 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 don't \\u0645\\u0631\\u062d\\u0628\\u0627 piece involved user@example.com written tiktoken's fox written and no 1999 don't on because counting \\\"quotes\\\" They'RE 42 a gateway \\u6771\\u4eac scanner admission user@example.com 3.14159 there's on over 'single' written that a {braces} that because gateway the\"}, {\"role\": \"assistant\", \"content\": \"WON'T backtracking \\ud83d\\ude42 node.js 3.14159 keep it \\u0645\\u0631\\u062d\\u0628\\u0627 there's is so user@example.com on there's 3.14159 mirrors involved keep \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's {braces} no engine brown brown there's dog way regex that 'single' for and 3.14159 node.js every there's a every They'RE WON'T exactly scanner tiktoken's They'RE regex the dog counting over the for request scanner with and mirrors dog the counting tokens gateway https://example.com/a/b?c=d for https://example.com/a/b?c=d backtracking\"}, {\"role\": \"user\", \"content\": \"don't because that fox Z\\u00fcrich with Z\\u00fcrich way piece (parens) every (parens) user@example.com {braces} keep so a that boundaries 100% keep C++ node.js 1999 piece $1,234.56 \\ud83d\\ude42 backtracking {braces} that involved gateway boundaries no 42 hand 3.14159 \\ud83d\\ude42 na\\u00efve na\\u00efve that for Z\\u00fcrich $1,234.56 engine lazy fox scanned before once hand no https://example.com/a/b?c=d admission WON'T admission It's no a because \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac exactly written \\u0645\\u0631\\u062d\\u0628\\u0627 faster that is user@example.com v1.2.3 'single' boundaries gateway faster backtracking no engine fox $1,234.56 lazy the $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 that WON'T before I'M scanned\"}, {\"role\": \"assistant\", \"content\": \"it is admission admission brown the They'RE we'll we'll user@example.com [brackets] way tokens $1,234.56 on caf\\u00e9 I'M gateway 3.14159 written 'single' on involved mirrors node.js that fox 'single' quick once \\u6771\\u4eac gateway $1,234.56 no exactly body Z\\u00fcrich that 'single' engine and Z\\u00fcrich don't no so tiktoken's I'M that over hand dog we'll (parens) 'single' \\ud83d\\ude42 while 100% is node.js no quick over while on budget that \\\"quotes\\\" budget It's don't regex on C++ 1999 [brackets] scanned 1999 that there's $1,234.56 way\"}, {\"role\": \"user\", \"content\": \"budget while tiktoken's backtracking gateway and hand no while \\ud83d\\ude42 request node.js we'll and don't while na\\u00efve quick WON'T don't [brackets] quick there's fox that that I'M keep \\ud83d\\ude42 tiktoken's v1.2.3 the body the \\u0645\\u0631\\u062d\\u0628\\u0627 reservation written F# jumps admission boundaries user@example.com while I'M brown lazy It's regex fox user@example.com {braces} is on 'single' written request there's engine exactly tiktoken's WON'T caf\\u00e9 the tokens on and brown exactly scanner with involved regex on v1.2.3 jumps that written F# on exactly before over once body \\u6771\\u4eac \\ud83d\\ude42 lazy that \\ud83d\\ude42\"}, {\"role\": \"assistant\", \"content\": \"don't every there's scanned while They'RE a tokens is scanned mirrors so is request counting once way (parens) {braces} counting 100% \\\"quotes\\\" before boundaries brown tiktoken's that way engine quick tiktoken's over involved 100% before for the C++ It's that node.js [brackets] because https://example.com/a/b?c=d engine 100% budget we'll with tokens 42 budget it dog 1999 user@example.com 42 C++ keep the for a 42 keep no caf\\u00e9 node.js 'single' the for don't It's C++ admission a involved 3.14159 'single' don't They'RE WON'T quick so backtracking because once body \\\"quotes\\\" WON'T user@example.com caf\\u00e9 v1.2.3 engine 'single' scanner hand I'M 'single' $1,234.56 [brackets] {braces} counting it don't there's no that\"}, {\"role\": \"user\", \"content\": \"so jumps dog hand no 100% while request tiktoken's there's that so regex $1,234.56 don't 'single' boundaries \\\"quotes\\\" that 100% the boundaries keep backtracking (parens) no \\u6771\\u4eac fox F# mirrors 3.14159 we'll there's \\\"quotes\\\" scanner backtracking $1,234.56 100% request the WON'T boundaries caf\\u00e9 tiktoken's node.js backtracking counting user@example.com C++ exactly is reservation 100% we'll quick regex \\ud83d\\ude42 'single' I'M request written https://example.com/a/b?c=d boundaries They'RE there's It's piece F# v1.2.3 boundaries [brackets] a 100% that scanner v1.2.3 42 because https://example.com/a/b?c=d [brackets] that I'M once body faster because with engine 'single' scanned on engine the 1999 \\u6771\\u4eac quick written no piece involved boundaries [brackets] counting while I'M Z\\u00fcrich scanner for the written $1,234.56 (parens) that boundaries the because\"}, {\"role\": \"assistant\", \"content\": \"(parens) https://example.com/a/b?c=d 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich Z\\u00fcrich \\u6771\\u4eac involved don't reservation there's we'll \\\"quotes\\\" reservation They'RE tokens because lazy there's a exactly It's jumps every $1,234.56 v1.2.3 exactly 42 a quick https://example.com/a/b?c=d caf\\u00e9 tiktoken's piece reservation every C++ caf\\u00e9 v1.2.3 scanned backtracking na\\u00efve 1999 piece engine while that I'M involved on fox 42 https://example.com/a/b?c=d (parens) tiktoken's for dog once the involved a dog once there's https://example.com/a/b?c=d tokens na\\u00efve (parens) once 1999 over na\\u00efve caf\\u00e9 hand They'RE every tiktoken's involved we'll node.js\"}, {\"role\": \"user\", \"content\": \"that request while jumps faster exactly written 3.14159 scanner that \\\"quotes\\\" with exactly dog that keep with piece piece boundaries no [brackets] backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 F# hand $1,234.56 engine written (parens) and It's involved node.js Z\\u00fcrich boundaries with backtracking na\\u00efve and body counting node.js keep Z\\u00fcrich that while \\u6771\\u4eac because there's \\u6771\\u4eac boundaries no They'RE counting jumps v1.2.3 with don't a \\u0645\\u0631\\u062d\\u0628\\u0627 fox user@example.com no that it F# once it so every 100% a caf\\u00e9 the \\\"quotes\\\" [brackets] that gateway while we'll scanned caf\\u00e9 keep involved \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"the while I'M gateway WON'T tiktoken's 'single' over 'single' request faster (parens) faster faster \\u0645\\u0631\\u062d\\u0628\\u0627 hand tiktoken's keep while v1.2.3 $1,234.56 that written body WON'T way \\u0645\\u0631\\u062d\\u0628\\u0627 brown a F# scanned \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE keep lazy caf\\u00e9 100% 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} we'll exactly the C++ body Z\\u00fcrich Z\\u00fcrich way with reservation mirrors C++ backtracking involved WON'T exactly with involved WON'T way a counting the admission 1999 hand because once They'RE the backtracking 1999 no a 42 brown\"}, {\"role\": \"user\", \"content\": \"node.js so jumps is piece a way on because the that backtracking request the reservation {braces} {braces} the https://example.com/a/b?c=d it tokens reservation the lazy They'RE reservation a 1999 I'M boundaries no with no C++ is way 3.14159 WON'T hand 'single' counting Z\\u00fcrich on F# with 1999 admission every (parens) we'll request mirrors that the 'single' it faster dog \\\"quotes\\\" jumps that faster WON'T faster every 100% involved a there's \\ud83d\\ude42 way way 'single' na\\u00efve no on it (parens) WON'T user@example.com na\\u00efve boundaries mirrors [brackets] it regex regex 42 engine engine and faster \\\"quotes\\\" gateway \\u0645\\u0631\\u062d\\u0628\\u0627 over request $1,234.56 They'RE user@example.com that node.js I'M https://example.com/a/b?c=d involved on (parens) it a that 3.14159 way over no\"}, {\"role\": \"assistant\", \"content\": \"boundaries so budget involved exactly written node.js [brackets] with jumps caf\\u00e9 node.js (parens) [brackets] 100% tiktoken's 42 it that piece (parens) engine user@example.com involved no gateway \\\"quotes\\\" exactly It's budget exactly once It's a involved (parens) budget we'll fox node.js tokens Z\\u00fcrich no body \\\"quotes\\\" na\\u00efve while it is I'M the hand mirrors WON'T the caf\\u00e9 \\u6771\\u4eac mirrors https://example.com/a/b?c=d counting before way because \\\"quotes\\\" a admission lazy dog keep the budget quick mirrors there's [brackets] hand Z\\u00fcrich https://example.com/a/b?c=d mirrors user@example.com 42 regex request scanned \\\"quotes\\\" engine because the It's Z\\u00fcrich F# 42 boundaries there's keep engine \\ud83d\\ude42 I'M They'RE because over $1,234.56 admission for fox because exactly because piece regex on \\u6771\\u4eac 'single' backtracking a\"}, {\"role\": \"user\", \"content\": \"They'RE backtracking \\u6771\\u4eac boundaries mirrors that 100% {braces} 42 gateway boundaries tokens so on counting gateway tiktoken's tiktoken's tokens tiktoken's there's while tokens It's C++ mirrors the tokens budget hand we'll over over [brackets] jumps the \\ud83d\\ude42 way and that boundaries 100% counting reservation there's [brackets] 3.14159 They'RE keep the regex It's while budget request Z\\u00fcrich it https://example.com/a/b?c=d C++ admission quick engine keep quick \\u6771\\u4eac $1,234.56 engine budget hand \\u6771\\u4eac body 100% don't scanned $1,234.56 dog the \\u6771\\u4eac piece over 1999 $1,234.56 $1,234.56 the dog we'll gateway is dog there's fox Z\\u00fcrich over They'RE so a engine 'single' regex and lazy na\\u00efve tokens tokens \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 \\ud83d\\ude42 (parens)\"}, {\"role\": \"assistant\", \"content\": \"\\\"quotes\\\" {braces} so with brown (parens) that the because keep boundaries before because fox exactly every because F# so mirrors way C++ request gateway there's that on request tokens it \\\"quotes\\\" regex written dog scanner budget while it that $1,234.56 scanned every no we'll is because while reservation They'RE don't dog scanner dog WON'T fox no scanned \\ud83d\\ude42 budget no dog request it scanned is hand over no 3.14159 so regex backtracking exactly backtracking the 42 100% faster quick scanner before 'single' reservation engine it we'll caf\\u00e9 every quick for no faster 100% $1,234.56 on Z\\u00fcrich written keep\"}, {\"role\": \"user\", \"content\": \"on tokens don't F# counting keep exactly every \\ud83d\\ude42 keep Z\\u00fcrich for fox counting no caf\\u00e9 reservation na\\u00efve mirrors They'RE It's F# the that piece \\\"quotes\\\" \\u6771\\u4eac \\u6771\\u4eac engine no $1,234.56 \\\"quotes\\\" because dog 100% na\\u00efve user@example.com tokens [brackets] no with that boundaries I'M Z\\u00fcrich before (parens) while and scanned gateway user@example.com faster request boundaries \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d written C++ I'M while quick [brackets] before 1999 and They'RE C++ v1.2.3 for no {braces} request C++ They'RE over WON'T on caf\\u00e9 {braces} 100% the body quick keep C++ \\ud83d\\ude42 tokens scanned admission keep hand a node.js WON'T keep 100% $1,234.56 hand $1,234.56 (parens) 1999 because the \\ud83d\\ude42 is 42 body Z\\u00fcrich way [brackets] it once that dog C++ jumps fox tiktoken's piece \\\"quotes\\\" v1.2.3 exactly for 42 \\ud83d\\ude42 'single' na\\u00efve admission \\u6771\\u4eac They'RE fox mirrors regex\"}, {\"role\": \"assistant\", \"content\": \"They'RE scanner tiktoken's quick (parens) https://example.com/a/b?c=d budget quick for engine (parens) hand tiktoken's piece written https://example.com/a/b?c=d counting counting over no once engine I'M because fox exactly regex no once keep and written regex faster 100% the admission counting a scanner C++ quick involved gateway brown piece Z\\u00fcrich regex written is faster \\u6771\\u4eac body brown no tokens scanned every request it I'M \\ud83d\\ude42 for (parens) Z\\u00fcrich written 3.14159 'single' I'M no na\\u00efve quick regex gateway it counting (parens) the over so hand gateway is body way dog v1.2.3 faster so request while before counting body F# faster [brackets] engine involved a $1,234.56 dog that that budget mirrors user@example.com the brown that no C++ it every na\\u00efve exactly that body Z\\u00fcrich \\u6771\\u4eac with don't so It's \\u0645\\u0631\\u062d\\u0628\\u0627 that gateway a the\"}, {\"role\": \"user\", \"content\": \"dog is once budget \\ud83d\\ude42 quick regex fox (parens) [brackets] gateway \\ud83d\\ude42 no regex because lazy request with that written we'll It's that request user@example.com is \\\"quotes\\\" \\\"quotes\\\" faster the don't quick node.js \\u0645\\u0631\\u062d\\u0628\\u0627 faster na\\u00efve They'RE the 100% written that the 100% backtracking \\ud83d\\ude42 written the fox scanner na\\u00efve it piece It's no so It's before na\\u00efve gateway there's because gateway reservation \\ud83d\\ude42 quick involved \\u0645\\u0631\\u062d\\u0628\\u0627 with WON'T request F# scanned exactly F# reservation scanned engine jumps v1.2.3 we'll 'single' it quick WON'T jumps na\\u00efve tokens brown node.js backtracking reservation 100% v1.2.3 once exactly\"}, {\"role\": \"assistant\", \"content\": \"fox user@example.com it (parens) we'll 100% involved dog and brown 'single' exactly mirrors 100% WON'T na\\u00efve 1999 F# faster 'single' tokens tiktoken's admission \\\"quotes\\\" the WON'T request that on we'll I'M piece there's body Z\\u00fcrich boundaries because jumps the They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 boundaries \\u6771\\u4eac the https://example.com/a/b?c=d caf\\u00e9 piece don't brown exactly node.js the budget every so admission that and 100% fox 'single' {braces} 100% $1,234.56 C++ counting WON'T brown don't https://example.com/a/b?c=d boundaries I'M 100% [brackets] lazy quick It's brown \\u6771\\u4eac for is \\u6771\\u4eac for is and request\"}, {\"role\": \"user\", \"content\": \"{braces} counting I'M request fox involved boundaries with WON'T WON'T $1,234.56 is faster don't request https://example.com/a/b?c=d while It's every body the a brown engine 42 for we'll request (parens) involved is scanned backtracking jumps C++ gateway keep scanner for counting 42 reservation hand \\\"quotes\\\" hand C++ every reservation caf\\u00e9 that don't with $1,234.56 there's WON'T {braces} that 3.14159 F# before over exactly caf\\u00e9 budget \\u6771\\u4eac the before and body gateway hand admission It's C++ that F# It's faster no that we'll written (parens) fox mirrors we'll user@example.com no https://example.com/a/b?c=d node.js that brown request\"}, {\"role\": \"assistant\", \"content\": \"I'M admission counting Z\\u00fcrich user@example.com faster a \\ud83d\\ude42 mirrors once lazy is the involved \\ud83d\\ude42 \\u6771\\u4eac every scanned every because budget hand and it jumps there's while there's [brackets] na\\u00efve on \\\"quotes\\\" \\\"quotes\\\" a \\ud83d\\ude42 hand \\u6771\\u4eac we'll and It's with the {braces} piece WON'T 'single' for \\u6771\\u4eac no backtracking lazy fox a [brackets] fox 42 \\ud83d\\ude42 (parens) 1999 https://example.com/a/b?c=d 1999 lazy and 1999 reservation way (parens) 100% it na\\u00efve 100%\"}, {\"role\": \"user\", \"content\": \"regex on scanner once $1,234.56 1999 (parens) gateway budget dog F# before user@example.com once brown jumps 100% $1,234.56 that budget Z\\u00fcrich the reservation because a on \\\"quotes\\\" backtracking hand dog because backtracking a is reservation F# F# brown gateway for the we'll the node.js piece I'M keep tokens once body WON'T there's tokens piece 'single' They'RE node.js so 100% body over a is user@example.com no \\\"quotes\\\" boundaries F# They'RE the C++ backtracking and \\ud83d\\ude42 over WON'T\"}, {\"role\": \"assistant\", \"content\": \"is 'single' scanned \\u6771\\u4eac 3.14159 mirrors {braces} there's scanner every reservation mirrors reservation backtracking involved that tiktoken's {braces} node.js we'll F# jumps v1.2.3 v1.2.3 exactly keep \\u0645\\u0631\\u062d\\u0628\\u0627 1999 scanner with tokens once counting so for https://example.com/a/b?c=d piece the while request Z\\u00fcrich it [brackets] dog involved the involved the admission and (parens) engine F# \\u0645\\u0631\\u062d\\u0628\\u0627 gateway quick \\ud83d\\ude42 v1.2.3 tiktoken's https://example.com/a/b?c=d faster \\ud83d\\ude42 that 42 (parens) jumps caf\\u00e9 so hand budget admission [brackets] gateway brown I'M quick admission C++ C++ caf\\u00e9 \\u6771\\u4eac 'single' user@example.com hand that don't C++ once\"}, {\"role\": \"user\", \"content\": \"there's na\\u00efve reservation na\\u00efve before gateway 100% 1999 request that jumps tiktoken's for admission on quick on scanned boundaries jumps \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors we'll dog because a budget \\u6771\\u4eac it na\\u00efve 1999 that the caf\\u00e9 involved counting jumps tokens with \\u0645\\u0631\\u062d\\u0628\\u0627 involved before the over admission I'M dog v1.2.3 engine tiktoken's we'll scanner every tiktoken's They'RE once It's exactly request jumps node.js WON'T 3.14159 over that exactly engine \\u0645\\u0631\\u062d\\u0628\\u0627 once budget reservation because it 42 keep 'single' boundaries so \\u0645\\u0631\\u062d\\u0628\\u0627 node.js scanner budget 3.14159 engine node.js C++ I'M budget [brackets] a caf\\u00e9 user@example.com jumps tokens every 1999 It's\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 scanner (parens) fox gateway mirrors 42 They'RE reservation we'll fox written brown \\\"quotes\\\" mirrors so 42 hand because budget 100% I'M counting request quick 42 caf\\u00e9 C++ C++ 3.14159 request https://example.com/a/b?c=d jumps jumps scanned brown keep brown and \\ud83d\\ude42 once tokens admission fox F# Z\\u00fcrich user@example.com the gateway tiktoken's https://example.com/a/b?c=d \\\"quotes\\\" tokens \\ud83d\\ude42 \\ud83d\\ude42 [brackets] while node.js [brackets] 1999 on \\\"quotes\\\" caf\\u00e9 1999 jumps 3.14159 request over body admission tiktoken's \\ud83d\\ude42 regex that the once scanner because They'RE regex 'single' admission 42\"}, {\"role\": \"user\", \"content\": \"gateway \\u0645\\u0631\\u062d\\u0628\\u0627 keep quick https://example.com/a/b?c=d F# involved hand It's written request {braces} over mirrors there's gateway every so it caf\\u00e9 involved {braces} quick that gateway over 42 fox lazy involved every keep don't (parens) It's request every for \\u6771\\u4eac admission piece on 3.14159 backtracking written quick gateway and jumps F# (parens) 100% the budget quick keep it 100% \\ud83d\\ude42 for (parens) is \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"way boundaries F# (parens) that It's \\\"quotes\\\" scanned \\ud83d\\ude42 \\u6771\\u4eac that brown regex is over na\\u00efve 3.14159 because Z\\u00fcrich 1999 (parens) \\u0645\\u0631\\u062d\\u0628\\u0627 C++ over way so that user@example.com and user@example.com tiktoken's scanned \\u6771\\u4eac [brackets] engine lazy 1999 involved it 100% a so \\u0645\\u0631\\u062d\\u0628\\u0627 on every body na\\u00efve faster regex engine fox that tokens caf\\u00e9 na\\u00efve 3.14159 and hand engine with\"}, {\"role\": \"user\", \"content\": \"keep there's that every [brackets] $1,234.56 \\\"quotes\\\" on body https://example.com/a/b?c=d involved https://example.com/a/b?c=d tokens scanner 1999 https://example.com/a/b?c=d https://example.com/a/b?c=d node.js 1999 backtracking user@example.com that [brackets] don't body lazy WON'T 42 written hand we'll WON'T no \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 that $1,234.56 way piece https://example.com/a/b?c=d admission admission C++ boundaries we'll reservation scanner the C++ \\u0645\\u0631\\u062d\\u0628\\u0627 hand engine na\\u00efve I'M backtracking while regex \\ud83d\\ude42 budget quick v1.2.3 dog v1.2.3 quick [brackets] fox for node.js [brackets] admission \\ud83d\\ude42 They'RE that on boundaries every that regex 42 a {braces} way every scanned backtracking boundaries budget we'll \\\"quotes\\\" it no caf\\u00e9 scanner jumps backtracking while involved and it 1999 with way {braces} \\\"quotes\\\" 'single'\"}, {\"role\": \"assistant\", \"content\": \"42 is \\\"quotes\\\" budget It's They'RE dog fox exactly fox written regex backtracking written with \\\"quotes\\\" written WON'T that backtracking for a every a engine lazy 42 the once budget on because dog we'll and node.js it $1,234.56 mirrors (parens) It's 42 na\\u00efve it scanned https://example.com/a/b?c=d v1.2.3 C++ 'single' boundaries caf\\u00e9 1999 before once 3.14159 is [brackets] don't tiktoken's scanner fox [brackets] \\u6771\\u4eac over once [brackets] They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 we'll with 42 counting \\u0645\\u0631\\u062d\\u0628\\u0627 C++ 3.14159 it the quick\"}, {\"role\": \"user\", \"content\": \" (parens) with before so is that node.js C++ we'll on admission the \\u6771\\u4eac faster the caf\\u00e9 jumps Z\\u00fcrich (parens) fox \\u6771\\u4eac {braces} every {braces} fox mirrors They'RE They'RE fox counting C++ keep \\\"quotes\\\" 1999 once 100% C++ don't once way dog They'RE once involved exactly \\ud83d\\ude42 we'll piece request reservation quick the don't (parens) scanner I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 Z\\u00fcrich once admission no regex tiktoken's exactly 1999 over engine regex there's involved 42 admission no 'single' over there's (parens) caf\\u00e9 because request keep \\u6771\\u4eac keep \\\"quotes\\\" budget piece piece faster $1,234.56 quick WON'T it piece hand\"}, {\"role\": \"assistant\", \"content\": \"involved scanner WON'T {braces} {braces} It's F# piece it \\\"quotes\\\" I'M because body before quick counting piece na\\u00efve reservation hand WON'T hand the request 1999 involved because {braces} so don't F# and [brackets] faster na\\u00efve way for brown brown hand it Z\\u00fcrich 3.14159 the user@example.com with 1999 while every scanner lazy (parens) tokens [brackets] for tokens body exactly I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 over caf\\u00e9 exactly counting a tiktoken's while no https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 lazy lazy (parens) backtracking that involved keep and a tiktoken's is body They'RE it it 100% scanned jumps jumps (parens) na\\u00efve so for v1.2.3 F# 'single' way no exactly 'single' tiktoken's there's dog caf\\u00e9 engine $1,234.56 is tokens\"}, {\"role\": \"user\", \"content\": \"a C++ 100% faster the node.js gateway every It's budget reservation there's scanner before Z\\u00fcrich \\ud83d\\ude42 written written tokens faster tokens $1,234.56 counting quick Z\\u00fcrich for lazy tokens node.js before before backtracking WON'T It's jumps before Z\\u00fcrich $1,234.56 with boundaries mirrors the request once node.js faster that jumps backtracking 1999 boundaries node.js fox admission tiktoken's mirrors way the we'll involved backtracking request 3.14159 the jumps na\\u00efve na\\u00efve \\\"quotes\\\" mirrors 3.14159 na\\u00efve scanner F# because node.js 3.14159 that caf\\u00e9 100% and and (parens) \\\"quotes\\\" WON'T $1,234.56 \\u6771\\u4eac quick $1,234.56 reservation while the na\\u00efve exactly once written backtracking written the na\\u00efve\"}, {\"role\": \"assistant\", \"content\": \"tokens \\u0645\\u0631\\u062d\\u0628\\u0627 scanned [brackets] every admission [brackets] scanned 'single' exactly scanner tokens that [brackets] tokens scanned the before before They'RE every because \\\"quotes\\\" C++ scanner with with brown exactly is for user@example.com while caf\\u00e9 boundaries it WON'T scanner that admission reservation 100% is admission before gateway reservation caf\\u00e9 100% request tiktoken's 3.14159 before backtracking while so https://example.com/a/b?c=d 42 faster and and faster It's we'll mirrors 42 tiktoken's regex node.js there's scanned\"}, {\"role\": \"user\", \"content\": \"written v1.2.3 exactly reservation piece every F# reservation node.js node.js \\u6771\\u4eac that scanner https://example.com/a/b?c=d 42 {braces} it and keep It's no regex jumps with user@example.com body over It's \\\"quotes\\\" and 'single' counting I'M that \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) engine the way $1,234.56 scanned They'RE for caf\\u00e9 \\\"quotes\\\" lazy don't backtracking tokens \\\"quotes\\\" quick {braces} \\\"quotes\\\" involved involved engine scanner because hand tiktoken's v1.2.3 F# regex $1,234.56 engine {braces} gateway counting that (parens) that the before admission that every backtracking \\\"quotes\\\" [brackets] way gateway engine lazy fox 42 don't [brackets] once na\\u00efve \\\"quotes\\\" They'RE They'RE is jumps every 100% written backtracking body caf\\u00e9 that backtracking v1.2.3 (parens) F# keep exactly piece once v1.2.3 F# They'RE backtracking 100% the fox the Z\\u00fcrich involved F#\"}, {\"role\": \"assistant\", \"content\": \"{braces} 42 engine over na\\u00efve hand involved They'RE request It's Z\\u00fcrich before {braces} 42 dog every node.js is over https://example.com/a/b?c=d na\\u00efve engine fox is [brackets] is on over \\ud83d\\ude42 (parens) \\ud83d\\ude42 regex They'RE They'RE \\ud83d\\ude42 (parens) C++ node.js 100% jumps once that scanner on mirrors regex that faster tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's while F# exactly 100% 3.14159 engine \\u6771\\u4eac scanner {braces} na\\u00efve I'M the over brown written don't tokens user@example.com is F# once faster on It's a there's C++ 1999 we'll [brackets] over the dog there's request dog C++ the brown regex way na\\u00efve written before Z\\u00fcrich engine caf\\u00e9 WON'T 1999 with engine and scanned $1,234.56 keep I'M every user@example.com 'single' a and 1999 don't I'M written while and that\"}, {\"role\": \"user\", \"content\": \"budget {braces} hand so and (parens) fox for that no way is counting tokens WON'T tokens tokens lazy tokens piece on that request dog mirrors I'M mirrors scanned regex on na\\u00efve C++ once user@example.com before lazy user@example.com \\\"quotes\\\" I'M once over involved that request written backtracking \\\"quotes\\\" is no brown there's {braces} because \\ud83d\\ude42 keep reservation written 100% caf\\u00e9 v1.2.3 tiktoken's body F# \\u6771\\u4eac counting quick every faster It's while \\u6771\\u4eac every backtracking lazy They'RE tokens fox {braces} tokens \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" 1999 piece involved [brackets] tokens They'RE the a don't hand caf\\u00e9 quick before 'single' so \\u6771\\u4eac that [brackets] quick \\\"quotes\\\" counting gateway it that is It's scanner the 'single' the \\u0645\\u0631\\u062d\\u0628\\u0627 keep 100% it fox way na\\u00efve exactly mirrors caf\\u00e9 mirrors there's because piece every a\"}, {\"role\": \"assistant\", \"content\": \"counting no tiktoken's hand dog {braces} They'RE $1,234.56 involved \\\"quotes\\\" on \\u0645\\u0631\\u062d\\u0628\\u0627 the because tiktoken's 3.14159 that because counting a 3.14159 so node.js counting backtracking https://example.com/a/b?c=d C++ quick involved (parens) request (parens) F# no gateway exactly every backtracking hand on user@example.com It's written scanned before reservation {braces} there's [brackets] $1,234.56 I'M engine involved brown reservation Z\\u00fcrich request F# brown a way I'M written exactly faster \\u6771\\u4eac Z\\u00fcrich mirrors regex 42 tiktoken's caf\\u00e9 42 100% F# way 1999 no node.js way involved while and {braces} that 1999 They'RE node.js admission I'M the is there's 'single' scanner na\\u00efve scanned tiktoken's \\u6771\\u4eac before backtracking 'single'\"}, {\"role\": \"user\", \"content\": \"na\\u00efve tiktoken's boundaries admission 42 100% {braces} 'single' (parens) and and way I'M 3.14159 budget jumps with gateway the fox engine because on scanner \\\"quotes\\\" 1999 user@example.com that written 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 quick [brackets] WON'T 42 \\u6771\\u4eac \\\"quotes\\\" jumps so budget scanned 'single' boundaries and F# quick every https://example.com/a/b?c=d regex tiktoken's lazy involved we'll \\ud83d\\ude42 it admission v1.2.3 reservation counting scanner that \\u0645\\u0631\\u062d\\u0628\\u0627 over there's counting because 100% keep F# lazy $1,234.56 so that is once request is \\ud83d\\ude42 before once na\\u00efve keep\"}, {\"role\": \"assistant\", \"content\": \"with Z\\u00fcrich a 3.14159 faster faster faster jumps 'single' before over dog It's $1,234.56 node.js 'single' na\\u00efve body way we'll It's once regex the counting 3.14159 [brackets] gateway C++ on piece jumps body is node.js lazy user@example.com don't with v1.2.3 exactly user@example.com [brackets] once It's it Z\\u00fcrich [brackets] {braces} faster \\u0645\\u0631\\u062d\\u0628\\u0627 1999 that C++ involved on quick C++ scanner reservation reservation regex node.js faster because jumps It's mirrors that F# brown the request with every reservation backtracking (parens) v1.2.3 I'M {braces} once tokens reservation 42 a 3.14159 F# we'll (parens) way caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 1999 F# piece 1999 quick for scanned\"}, {\"role\": \"user\", \"content\": \"scanned piece F# reservation because regex regex counting written na\\u00efve budget 42 the before keep keep way It's It's exactly node.js because 3.14159 tokens keep with \\u0645\\u0631\\u062d\\u0628\\u0627 body exactly that reservation v1.2.3 42 [brackets] keep scanner dog piece it F# [brackets] regex that tokens exactly body node.js that once faster hand with lazy exactly boundaries jumps that scanned that written\"}, {\"role\": \"assistant\", \"content\": \"brown They'RE gateway the Z\\u00fcrich gateway user@example.com over quick and \\\"quotes\\\" boundaries regex \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 the tokens 'single' is every tokens 42 reservation (parens) request I'M backtracking regex tokens boundaries on is hand scanner boundaries caf\\u00e9 engine and jumps It's the over user@example.com user@example.com I'M on lazy while 1999 WON'T brown that reservation It's v1.2.3 WON'T exactly gateway with quick F# F# the I'M gateway I'M don't way boundaries \\\"quotes\\\" reservation Z\\u00fcrich and 3.14159 on faster na\\u00efve 42 v1.2.3 v1.2.3 3.14159 jumps hand we'll once (parens) C++ the faster keep They'RE node.js fox scanned 42 jumps faster way gateway hand 1999 it that request keep \\u6771\\u4eac $1,234.56 tokens engine that we'll F# we'll written \\\"quotes\\\" a is no keep tiktoken's node.js node.js on counting user@example.com \\\"quotes\\\" {braces} 1999 user@example.com hand fox I'M node.js is\"}, {\"role\": \"user\", \"content\": \"'single' \\u6771\\u4eac and scanned v1.2.3 dog \\u6771\\u4eac with budget $1,234.56 a exactly and scanned body way na\\u00efve on brown mirrors a 3.14159 dog fox jumps {braces} (parens) jumps the way over and we'll involved faster user@example.com fox body there's gateway body brown jumps there's for 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking It's mirrors Z\\u00fcrich 'single' tokens 'single' for user@example.com 100% faster there's keep C++ and don't gateway budget {braces} faster hand don't is involved \\u6771\\u4eac regex regex reservation scanner\"}, {\"role\": \"assistant\", \"content\": \"F# body is regex jumps WON'T reservation Z\\u00fcrich tokens piece that and \\u0645\\u0631\\u062d\\u0628\\u0627 [brackets] for 'single' we'll engine F# there's \\\"quotes\\\" so counting \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d [brackets] piece [brackets] involved exactly fox {braces} \\ud83d\\ude42 3.14159 way F# 3.14159 scanned while keep every request \\u6771\\u4eac way C++ na\\u00efve engine engine user@example.com because admission I'M gateway v1.2.3 v1.2.3 It's it Z\\u00fcrich https://example.com/a/b?c=d 3.14159 Z\\u00fcrich hand [brackets] [brackets] dog that 'single' exactly \\u6771\\u4eac regex brown reservation involved and WON'T F# na\\u00efve before It's a tiktoken's I'M we'll with written budget engine caf\\u00e9 tokens 100% scanned It's brown every a for\"}, {\"role\": \"user\", \"content\": \"involved because caf\\u00e9 so there's I'M 42 no na\\u00efve na\\u00efve exactly tiktoken's once scanner tiktoken's counting exactly dog the a and faster WON'T so They'RE 'single' faster for written C++ 3.14159 tiktoken's $1,234.56 C++ way exactly https://example.com/a/b?c=d every way node.js budget 42 before before gateway it mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 way faster budget (parens) budget every that don't over so v1.2.3 request we'll because with \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 don't 'single' reservation fox tokens there's request it They'RE [brackets] it node.js gateway 'single' node.js body once way quick 3.14159\"}, {\"role\": \"assistant\", \"content\": \"3.14159 while reservation body before on once (parens) request because https://example.com/a/b?c=d involved 42 user@example.com and no request They'RE dog F# na\\u00efve They'RE exactly [brackets] regex engine mirrors scanner engine It's there's counting \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich way 3.14159 over counting dog gateway no we'll for \\\"quotes\\\" no lazy 42 involved counting involved written counting 1999 a $1,234.56 tokens regex quick and no once \\u6771\\u4eac every so brown is \\u0645\\u0631\\u062d\\u0628\\u0627 fox https://example.com/a/b?c=d caf\\u00e9 brown \\u6771\\u4eac 42 exactly a scanner over \\ud83d\\ude42 we'll budget dog {braces} for before faster\"}, {\"role\": \"user\", \"content\": \" faster the over keep while \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's once the because involved scanner regex that \\u6771\\u4eac there's brown WON'T dog \\u0645\\u0631\\u062d\\u0628\\u0627 budget lazy v1.2.3 v1.2.3 caf\\u00e9 counting so don't the scanner I'M caf\\u00e9 1999 don't once engine lazy (parens) request keep user@example.com the keep fox the Z\\u00fcrich https://example.com/a/b?c=d 3.14159 caf\\u00e9 (parens) It's piece I'M written \\\"quotes\\\" 'single' It's that {braces} reservation piece\"}, {\"role\": \"assistant\", \"content\": \"fox quick C++ brown before no lazy faster the that the every 100% na\\u00efve that 100% {braces} scanner we'll involved na\\u00efve before that {braces} faster mirrors because 'single' it faster v1.2.3 involved is so with request piece piece while that engine admission tokens gateway scanned before F# every $1,234.56 hand 1999 request is over v1.2.3 over quick regex v1.2.3 v1.2.3 boundaries Z\\u00fcrich and\"}, {\"role\": \"user\", \"content\": \"node.js WON'T faster \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 dog tokens and the no because They'RE and WON'T WON'T caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 42 na\\u00efve written keep don't the WON'T is admission $1,234.56 and every the I'M is C++ budget node.js tokens (parens) node.js caf\\u00e9 'single' with engine https://example.com/a/b?c=d v1.2.3 written node.js {braces} tiktoken's boundaries tiktoken's lazy scanned user@example.com quick body regex backtracking involved jumps faster quick They'RE request na\\u00efve body on written \\ud83d\\ude42 regex a It's a a no \\\"quotes\\\" \\ud83d\\ude42 while involved the jumps and \\ud83d\\ude42 node.js https://example.com/a/b?c=d boundaries na\\u00efve node.js brown hand 42 na\\u00efve for before C++ while https://example.com/a/b?c=d 42 because that for reservation \\\"quotes\\\" gateway budget is a so It's Z\\u00fcrich budget regex once scanned piece regex budget before over na\\u00efve regex\"}, {\"role\": \"assistant\", \"content\": \"WON'T on scanned user@example.com They'RE exactly faster the for a boundaries faster faster $1,234.56 I'M with don't {braces} jumps \\u0645\\u0631\\u062d\\u0628\\u0627 jumps so is keep while Z\\u00fcrich 42 so tiktoken's [brackets] written {braces} request engine no faster node.js counting \\u6771\\u4eac lazy counting faster is mirrors it there's jumps tiktoken's piece [brackets] \\u6771\\u4eac so brown admission brown Z\\u00fcrich way mirrors reservation user@example.com v1.2.3 $1,234.56 budget because the \"}, {\"role\": \"user\", \"content\": \"every jumps 'single' engine so dog scanned F# there's \\ud83d\\ude42 https://example.com/a/b?c=d user@example.com exactly backtracking node.js admission is na\\u00efve v1.2.3 tokens the fox involved gateway that admission don't reservation hand engine C++ so for we'll over written dog https://example.com/a/b?c=d while way dog and written with brown F# budget backtracking so tiktoken's \\u6771\\u4eac exactly written involved They'RE because reservation a caf\\u00e9 over boundaries regex is 'single' admission keep admission I'M WON'T admission counting scanned dog {braces} admission admission every F# 1999 scanner They'RE and the na\\u00efve 42 there's with involved 42 exactly 100% it [brackets] way They'RE node.js \\ud83d\\ude42 keep we'll I'M so I'M tiktoken's reservation WON'T caf\\u00e9 boundaries C++\"}, {\"role\": \"assistant\", \"content\": \"node.js the It's so C++ scanned so 1999 the budget request F# fox body body keep is [brackets] 42 over scanner there's \\u0645\\u0631\\u062d\\u0628\\u0627 node.js \\u0645\\u0631\\u062d\\u0628\\u0627 don't involved F# hand na\\u00efve scanned with involved engine jumps WON'T {braces} mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 is counting C++ fox it (parens) 3.14159 regex engine {braces} it keep 'single' the \\u0645\\u0631\\u062d\\u0628\\u0627 gateway engine WON'T user@example.com keep [brackets] once before budget the reservation regex hand keep user@example.com It's written reservation no fox \\\"quotes\\\" on \\u6771\\u4eac (parens) Z\\u00fcrich backtracking \\ud83d\\ude42 [brackets] mirrors gateway scanner boundaries v1.2.3 brown admission 1999 because way tokens regex admission so \\u0645\\u0631\\u062d\\u0628\\u0627 with counting fox tiktoken's na\\u00efve tokens counting gateway\"}, {\"role\": \"user\", \"content\": \"v1.2.3 dog counting 42 1999 with (parens) 42 scanner involved the it 'single' jumps over and user@example.com once I'M I'M engine before mirrors a faster brown quick Z\\u00fcrich v1.2.3 \\u6771\\u4eac body it mirrors na\\u00efve fox budget 'single' with mirrors WON'T user@example.com faster so Z\\u00fcrich scanned It's that written reservation is that 42 v1.2.3 over 1999 on 1999 that over na\\u00efve brown there's the 100% \\u6771\\u4eac request \\\"quotes\\\" $1,234.56 budget regex piece na\\u00efve 3.14159 the scanner \\u6771\\u4eac scanner scanner It's v1.2.3 on tokens engine 1999 42\"}, {\"role\": \"assistant\", \"content\": \"faster $1,234.56 piece user@example.com for \\u0645\\u0631\\u062d\\u0628\\u0627 while so on Z\\u00fcrich backtracking no It's \\\"quotes\\\" hand node.js before engine it tokens jumps reservation dog jumps [brackets] boundaries scanner for (parens) {braces} and request before scanned because before the for admission mirrors every piece admission keep and gateway we'll brown 1999 is I'M They'RE over https://example.com/a/b?c=d 3.14159 while hand once exactly way counting caf\\u00e9 3.14159 hand na\\u00efve every scanned \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors because They'RE backtracking \\ud83d\\ude42 \\\"quotes\\\" over admission piece on don't scanned exactly reservation and (parens)\"}, {\"role\": \"user\", \"content\": \"every quick that that while na\\u00efve fox {braces} while brown that v1.2.3 dog a C++ F# over for regex while every jumps it is 1999 caf\\u00e9 it and there's C++ we'll over counting boundaries brown 42 I'M involved and reservation boundaries so before I'M with scanned every is involved https://example.com/a/b?c=d lazy regex piece It's a while v1.2.3 user@example.com the 'single' I'M a is is \\\"quotes\\\" user@example.com because node.js faster F# Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 that na\\u00efve because brown involved the so involved WON'T body backtracking quick engine backtracking gateway that a user@example.com body [brackets] tokens every boundaries while \\ud83d\\ude42 is on boundaries scanner node.js Z\\u00fcrich is there's for regex it 100% piece tiktoken's\"}, {\"role\": \"assistant\", \"content\": \"reservation hand before F# counting $1,234.56 caf\\u00e9 faster \\u6771\\u4eac [brackets] tokens Z\\u00fcrich tiktoken's on written {braces} It's keep \\\"quotes\\\" while tokens na\\u00efve the no because keep https://example.com/a/b?c=d because counting request \\\"quotes\\\" it a because hand involved reservation node.js and backtracking 42 on piece I'M gateway 'single' mirrors the Z\\u00fcrich quick lazy brown {braces} admission Z\\u00fcrich C++ before https://example.com/a/b?c=d It's engine backtracking don't \\u6771\\u4eac piece dog no keep Z\\u00fcrich piece and 42 regex C++ caf\\u00e9 with we'll 3.14159 'single' mirrors counting Z\\u00fcrich it dog scanner a node.js piece {braces} we'll we'll because They'RE user@example.com fox caf\\u00e9 1999 42 every counting tiktoken's It's before backtracking scanner 42 the \\u6771\\u4eac admission lazy F# gateway mirrors It's C++ tiktoken's is dog no written https://example.com/a/b?c=d \\u6771\\u4eac exactly \\\"quotes\\\" body exactly\"}, {\"role\": \"user\", \"content\": \"that https://example.com/a/b?c=d the over boundaries scanner don't [brackets] quick every because because I'M because so once dog mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" quick on 'single' (parens) 42 and that before while while engine while over tiktoken's counting exactly {braces} tiktoken's piece gateway user@example.com It's that admission scanner It's request \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com that 42 for so \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 user@example.com for F# exactly that 100% backtracking boundaries it fox tiktoken's piece counting once engine admission piece Z\\u00fcrich a budget keep and {braces} is reservation scanner with https://example.com/a/b?c=d gateway no user@example.com na\\u00efve so user@example.com written brown\"}, {\"role\": \"assistant\", \"content\": \"piece They'RE body v1.2.3 \\ud83d\\ude42 C++ na\\u00efve It's we'll backtracking on because the counting a involved counting that there's request WON'T jumps that node.js Z\\u00fcrich quick https://example.com/a/b?c=d They'RE Z\\u00fcrich 3.14159 engine budget \\ud83d\\ude42 \\\"quotes\\\" scanner don't {braces} F# request don't $1,234.56 \\\"quotes\\\" tokens over while admission every with quick I'M \\\"quotes\\\" {braces} fox [brackets] gateway dog keep Z\\u00fcrich every [brackets] every mirrors C++ \\ud83d\\ude42 They'RE is [brackets] 3.14159 scanner hand and request 3.14159 with scanner no the regex 1999 They'RE every dog there's the admission that tiktoken's 'single' engine\"}, {\"role\": \"user\", \"content\": \"boundaries It's way dog I'M body lazy https://example.com/a/b?c=d budget faster for I'M regex every body piece with \\ud83d\\ude42 regex before for quick 'single' exactly with over brown there's jumps the don't {braces} and [brackets] it jumps on while don't no exactly every tokens and the 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 no [brackets] keep \\ud83d\\ude42 with Z\\u00fcrich gateway while backtracking the [brackets] every is \\u6771\\u4eac while faster for while budget [brackets] \\ud83d\\ude42 don't once quick so hand 'single' It's on dog exactly with once engine 'single' no written involved over tokens way over [brackets] with so regex caf\\u00e9 3.14159 \\ud83d\\ude42 a node.js every because faster reservation budget v1.2.3 with lazy I'M engine because and for before before engine mirrors \\\"quotes\\\" that node.js node.js counting a admission counting on WON'T no piece I'M once boundaries\"}, {\"role\": \"assistant\", \"content\": \"piece counting quick \\u0645\\u0631\\u062d\\u0628\\u0627 brown the there's mirrors regex faster involved tokens backtracking F# backtracking \\\"quotes\\\" \\ud83d\\ude42 with \\u6771\\u4eac body with gateway counting lazy exactly request [brackets] 'single' because while keep is I'M jumps body {braces} mirrors 1999 $1,234.56 faster it so {braces} v1.2.3 keep 42 admission admission It's that involved https://example.com/a/b?c=d na\\u00efve \\u6771\\u4eac \\u6771\\u4eac 42 fox node.js the (parens) (parens) [brackets] on faster hand node.js engine before \\u0645\\u0631\\u062d\\u0628\\u0627 jumps body we'll engine regex Z\\u00fcrich body admission 3.14159 They'RE Z\\u00fcrich scanner before scanner reservation \\u6771\\u4eac admission involved don't lazy $1,234.56 $1,234.56 that lazy scanned F# for request https://example.com/a/b?c=d there's before scanner reservation They'RE on \\u0645\\u0631\\u062d\\u0628\\u0627 quick we'll 100% lazy while budget 'single' once quick brown admission node.js gateway 1999 v1.2.3 piece because https://example.com/a/b?c=d reservation $1,234.56 for na\\u00efve \\ud83d\\ude42 exactly gateway that exactly on node.js and\"}, {\"role\": \"user\", \"content\": \"hand counting request while reservation there's body regex tokens don't regex every no a engine it a 1999 F# reservation with on mirrors for before Z\\u00fcrich v1.2.3 on the \\u0645\\u0631\\u062d\\u0628\\u0627 quick involved budget every node.js the hand it with node.js 1999 quick faster backtracking quick 42 while counting mirrors over so reservation involved [brackets] no written and tokens fox tiktoken's It's written \\ud83d\\ude42 over admission with over don't \\u6771\\u4eac way counting \\ud83d\\ude42 $1,234.56 https://example.com/a/b?c=d body hand over request tiktoken's that budget https://example.com/a/b?c=d exactly scanned gateway regex the while It's there's written for lazy F# counting body caf\\u00e9 jumps that counting jumps\"}, {\"role\": \"assistant\", \"content\": \"faster Z\\u00fcrich boundaries faster don't counting before dog engine I'M Z\\u00fcrich 3.14159 C++ \\u0645\\u0631\\u062d\\u0628\\u0627 for scanned involved engine backtracking brown \\u6771\\u4eac mirrors is (parens) backtracking tokens admission that Z\\u00fcrich don't C++ tiktoken's user@example.com that mirrors lazy scanner (parens) piece the body every \\\"quotes\\\" the request tokens body C++ (parens) a don't They'RE exactly the 3.14159 is budget user@example.com it \\\"quotes\\\" on \\ud83d\\ude42 backtracking admission written \\ud83d\\ude42 C++ quick fox 1999 I'M for it don't 1999 because way there's body reservation 100% WON'T dog 1999 tokens once a over once so $1,234.56 jumps They'RE scanner 3.14159 regex request 'single' exactly $1,234.56 exactly\"}, {\"role\": \"user\", \"content\": \"we'll 1999 lazy over 42 user@example.com over scanned gateway lazy piece so lazy exactly 'single' caf\\u00e9 node.js there's and once v1.2.3 while faster gateway every way 1999 gateway while 1999 1999 on caf\\u00e9 so written backtracking \\\"quotes\\\" backtracking scanner https://example.com/a/b?c=d body admission dog piece way brown fox 3.14159 with WON'T \\u6771\\u4eac \\\"quotes\\\" and exactly lazy na\\u00efve 1999 (parens) and involved keep that 3.14159 F# keep lazy no na\\u00efve request https://example.com/a/b?c=d jumps 'single' fox 'single' [brackets] tiktoken's counting once there's na\\u00efve admission faster F# we'll \\\"quotes\\\" https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 exactly \\u6771\\u4eac hand over https://example.com/a/b?c=d F# fox scanned 1999\"}, {\"role\": \"assistant\", \"content\": \"1999 no there's is 'single' C++ na\\u00efve that over admission \\\"quotes\\\" F# scanner gateway piece 100% while for no {braces} with fox scanned counting \\\"quotes\\\" there's gateway user@example.com keep body involved 3.14159 Z\\u00fcrich Z\\u00fcrich \\\"quotes\\\" [brackets] quick admission request a is 42 scanned caf\\u00e9 \\\"quotes\\\" I'M piece C++ mirrors 'single' every $1,234.56 3.14159 regex quick dog 42 WON'T brown no {braces} piece body quick before Z\\u00fcrich for for request {braces} engine while \\u6771\\u4eac Z\\u00fcrich brown gateway counting backtracking before don't $1,234.56 the request so {braces} \\u6771\\u4eac [brackets] because a node.js a 3.14159 boundaries every quick we'll jumps na\\u00efve that on we'll scanner tokens \\u6771\\u4eac https://example.com/a/b?c=d tokens 'single' keep\"}, {\"role\": \"user\", \"content\": \"fox admission exactly dog \\u6771\\u4eac backtracking {braces} node.js \\\"quotes\\\" way gateway on 1999 engine scanner keep quick way written boundaries because $1,234.56 don't there's way so over tokens F# that we'll v1.2.3 scanned F# for \\u0645\\u0631\\u062d\\u0628\\u0627 involved there's while {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 admission admission [brackets] They'RE \\ud83d\\ude42 regex body quick that v1.2.3 dog quick user@example.com over before admission hand is budget brown admission we'll body node.js na\\u00efve boundaries hand (parens) there's we'll v1.2.3 100% caf\\u00e9 for [brackets] 100% dog \\\"quotes\\\" piece boundaries I'M admission backtracking gateway engine no C++ there's we'll tiktoken's regex \\u0645\\u0631\\u062d\\u0628\\u0627 way WON'T {braces} once counting 1999 it before quick request WON'T we'll I'M admission lazy tokens involved 42 gateway lazy faster written that reservation exactly \\u6771\\u4eac 100% piece 100% scanner admission\"}, {\"role\": \"assistant\", \"content\": \"exactly because body and we'll so backtracking over fox F# I'M that jumps don't don't lazy scanned I'M lazy way piece It's They'RE scanner gateway 1999 so fox no because gateway 1999 that boundaries engine They'RE on admission scanner over Z\\u00fcrich that Z\\u00fcrich written v1.2.3 WON'T [brackets] tiktoken's scanner 3.14159 don't \\u6771\\u4eac tiktoken's user@example.com [brackets] before \\\"quotes\\\" no C++ node.js scanned 42\"}, {\"role\": \"user\", \"content\": \"once so 3.14159 because 100% with $1,234.56 faster no jumps once written gateway written tokens [brackets] before brown a counting \\\"quotes\\\" tiktoken's They'RE the quick (parens) exactly request jumps before involved mirrors dog budget \\ud83d\\ude42 every scanned \\ud83d\\ude42 fox https://example.com/a/b?c=d gateway body node.js lazy na\\u00efve v1.2.3 exactly way [brackets] gateway scanned WON'T [brackets] https://example.com/a/b?c=d piece \\u6771\\u4eac C++ 3.14159 fox v1.2.3 tokens mirrors caf\\u00e9 engine brown Z\\u00fcrich They'RE hand quick $1,234.56 WON'T tokens lazy written na\\u00efve 'single' dog lazy it that way \\ud83d\\ude42\"}, {\"role\": \"assistant\", \"content\": \"dog a They'RE It's so They'RE counting user@example.com jumps F# request no node.js there's na\\u00efve mirrors \\u6771\\u4eac tiktoken's scanner backtracking hand boundaries 1999 brown written mirrors written faster that budget because hand with 3.14159 and over WON'T every that way regex boundaries C++ \\u0645\\u0631\\u062d\\u0628\\u0627 piece Z\\u00fcrich because it They'RE involved https://example.com/a/b?c=d body 42 a fox that every and request {braces} with node.js F# [brackets] $1,234.56 on no budget way on backtracking node.js so 100%\"}, {\"role\": \"user\", \"content\": \"They'RE scanned dog body on that don't 1999 \\ud83d\\ude42 scanner 1999 that exactly C++ They'RE every caf\\u00e9 brown so \\ud83d\\ude42 caf\\u00e9 mirrors and while request WON'T budget scanner na\\u00efve Z\\u00fcrich the fox 3.14159 100% 'single' before \\ud83d\\ude42 Z\\u00fcrich scanner before that 1999 v1.2.3 body every boundaries that and v1.2.3 1999 100% because user@example.com dog engine keep the we'll I'M scanned na\\u00efve \\\"quotes\\\" there's that lazy https://example.com/a/b?c=d {braces} Z\\u00fcrich exactly \\\"quotes\\\" budget reservation involved that mirrors boundaries na\\u00efve that backtracking \\ud83d\\ude42 once \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 WON'T WON'T I'M node.js involved don't {braces} a and lazy piece backtracking for hand regex\"}, {\"role\": \"assistant\", \"content\": \"WON'T admission admission F# because don't [brackets] tiktoken's before the so na\\u00efve don't hand no na\\u00efve request on mirrors dog \\u0645\\u0631\\u062d\\u0628\\u0627 and before while body $1,234.56 v1.2.3 mirrors piece reservation so backtracking user@example.com every 100% F# regex 1999 brown [brackets] gateway 42 and on with 42 regex (parens) {braces} WON'T v1.2.3 {braces} a there's every I'M the 'single' 42 F# involved \\ud83d\\ude42 piece keep scanner the before I'M reservation written 3.14159 before there's piece once It's \\\"quotes\\\" dog written brown (parens) because while fox quick exactly user@example.com $1,234.56 boundaries Z\\u00fcrich is admission WON'T scanner we'll $1,234.56 hand [brackets] written {braces} {braces} {braces} the\"}, {\"role\": \"user\", \"content\": \"counting once C++ it the and involved keep it so there's I'M C++ 1999 no keep piece It's no user@example.com 'single' I'M WON'T node.js there's 100% quick 100% dog \\ud83d\\ude42 faster Z\\u00fcrich is before there's on don't F# {braces} keep that it na\\u00efve faster counting faster is over hand hand before involved tokens v1.2.3 {braces} boundaries backtracking so on before regex hand backtracking for while keep so (parens) tokens is with exactly backtracking (parens) admission $1,234.56 once no 42 C++ brown reservation Z\\u00fcrich \\\"quotes\\\"\"}, {\"role\": \"assistant\", \"content\": \"so {braces} It's involved body $1,234.56 lazy scanner caf\\u00e9 na\\u00efve v1.2.3 mirrors 3.14159 na\\u00efve the fox It's before we'll over no involved counting C++ brown quick hand \\u6771\\u4eac every exactly 42 boundaries $1,234.56 dog WON'T quick scanned na\\u00efve dog na\\u00efve boundaries quick backtracking body [brackets] tokens piece 100% exactly F# on admission the WON'T budget scanner \\\"quotes\\\" and WON'T we'll boundaries on budget scanned with \\ud83d\\ude42 $1,234.56 before we'll tiktoken's https://example.com/a/b?c=d https://example.com/a/b?c=d over no the there's It's don't over way keep [brackets] with body They'RE the once jumps keep admission They'RE tokens tokens jumps no 1999 fox 3.14159 100% reservation (parens) lazy [brackets] because https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 reservation 3.14159 keep while They'RE I'M brown quick They'RE admission exactly F# regex gateway and v1.2.3 while (parens) because with \\\"quotes\\\"\"}, {\"role\": \"user\", \"content\": \"scanner while node.js scanner 42 hand tokens tiktoken's They'RE engine is 'single' 'single' piece and v1.2.3 while it faster don't brown don't body na\\u00efve (parens) \\ud83d\\ude42 node.js the \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d caf\\u00e9 that $1,234.56 mirrors admission \\ud83d\\ude42 scanner that \\\"quotes\\\" C++ (parens) WON'T Z\\u00fcrich They'RE It's \\ud83d\\ude42 caf\\u00e9 that with WON'T it hand that over \\\"quotes\\\" 'single' na\\u00efve 1999 piece keep {braces} quick the no regex \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 piece na\\u00efve the \\u0645\\u0631\\u062d\\u0628\\u0627 'single' no that a body with\"}, {\"role\": \"assistant\", \"content\": \"that C++ over caf\\u00e9 tiktoken's scanned \\u0645\\u0631\\u062d\\u0628\\u0627 fox \\\"quotes\\\" for node.js once keep C++ counting once body so for jumps every before reservation user@example.com budget [brackets] boundaries 42 1999 written jumps $1,234.56 that the https://example.com/a/b?c=d faster Z\\u00fcrich request and once involved [brackets] so over over na\\u00efve caf\\u00e9 for for \\\"quotes\\\" while that scanned tokens scanner no no It's F# request request reservation Z\\u00fcrich jumps the na\\u00efve (parens) the backtracking jumps na\\u00efve scanner caf\\u00e9 keep I'M \\u6771\\u4eac quick scanner body hand scanner I'M way https://example.com/a/b?c=d it caf\\u00e9 mirrors (parens) F# keep {braces} brown WON'T regex engine user@example.com reservation user@example.com brown don't engine brown hand They'RE written https://example.com/a/b?c=d way (parens) dog\"}, {\"role\": \"user\", \"content\": \"\\u6771\\u4eac F# scanner that regex It's lazy na\\u00efve 100% involved keep the admission na\\u00efve user@example.com a for Z\\u00fcrich {braces} exactly lazy counting while tokens with hand written a is counting I'M [brackets] [brackets] 100% tokens body \\\"quotes\\\" \\\"quotes\\\" tiktoken's v1.2.3 https://example.com/a/b?c=d counting $1,234.56 $1,234.56 don't keep that before scanned lazy before gateway na\\u00efve scanner fox $1,234.56 tiktoken's faster so I'M involved over (parens) admission lazy dog caf\\u00e9 don't jumps 100% node.js request because caf\\u00e9 https://example.com/a/b?c=d 42 before 1999 [brackets] {braces} once [brackets] caf\\u00e9 scanner quick there's\"}, {\"role\": \"assistant\", \"content\": \"scanner caf\\u00e9 (parens) keep so {braces} on F# it because so over brown It's exactly I'M I'M F# faster $1,234.56 tokens exactly $1,234.56 caf\\u00e9 the na\\u00efve engine user@example.com \\\"quotes\\\" written C++ so regex don't backtracking engine the caf\\u00e9 C++ They'RE fox lazy written WON'T boundaries keep It's caf\\u00e9 while {braces} C++ keep jumps no node.js 3.14159 there's faster no 'single' so It's so a once scanner \\u0645\\u0631\\u062d\\u0628\\u0627 engine every once before backtracking request because F# scanned jumps I'M exactly we'll we'll and once [brackets] there's exactly 100%\"}, {\"role\": \"user\", \"content\": \"C++ tiktoken's $1,234.56 body scanner dog I'M regex that https://example.com/a/b?c=d mirrors every before user@example.com \\\"quotes\\\" with involved scanner budget I'M reservation hand that for 42 fox 42 tokens scanned They'RE They'RE \\\"quotes\\\" keep reservation piece Z\\u00fcrich the 3.14159 counting counting boundaries F# every while body boundaries 42 don't hand faster the reservation no \\u0645\\u0631\\u062d\\u0628\\u0627 that \\u6771\\u4eac while involved lazy involved caf\\u00e9 that on \\u6771\\u4eac dog 'single' on over once keep Z\\u00fcrich lazy https://example.com/a/b?c=d backtracking scanned 100% 1999 C++ body \\u0645\\u0631\\u062d\\u0628\\u0627 scanner https://example.com/a/b?c=d {braces} don't 3.14159 node.js regex fox faster I'M we'll lazy because backtracking \\ud83d\\ude42 fox F# \\\"quotes\\\" mirrors tokens It's and the for a WON'T regex reservation 42 dog They'RE request on node.js 'single' tokens node.js that admission 1999 on WON'T budget \\ud83d\\ude42 admission exactly 1999 that we'll \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich\"}, {\"role\": \"assistant\", \"content\": \"it 1999 engine \\\"quotes\\\" v1.2.3 every body boundaries over fox before I'M faster that fox \\u0645\\u0631\\u062d\\u0628\\u0627 involved node.js keep They'RE fox the with WON'T jumps hand caf\\u00e9 (parens) exactly user@example.com engine Z\\u00fcrich for request that involved na\\u00efve boundaries faster before no way 'single' piece don't tokens \\ud83d\\ude42 regex $1,234.56 \\ud83d\\ude42 keep [brackets] over 100% once that exactly node.js (parens) Z\\u00fcrich v1.2.3 scanned the admission there's F# and is tiktoken's on \\u0645\\u0631\\u062d\\u0628\\u0627 no involved I'M C++ 42 there's while before v1.2.3 piece over brown we'll 1999 lazy that user@example.com budget C++ on quick engine 1999 {braces} faster and that node.js budget body\"}, {\"role\": \"user\", \"content\": \"user@example.com no tiktoken's involved jumps exactly \\ud83d\\ude42 They'RE \\ud83d\\ude42 once because faster node.js I'M engine hand lazy we'll [brackets] fox budget quick every the tokens C++ a quick They'RE {braces} faster caf\\u00e9 for mirrors caf\\u00e9 it user@example.com node.js written on that engine regex https://example.com/a/b?c=d there's tokens don't na\\u00efve counting \\ud83d\\ude42 user@example.com mirrors engine $1,234.56 3.14159 backtracking hand request brown keep tiktoken's \\u6771\\u4eac a there's it scanner keep engine because fox scanned no I'M no scanned faster 42 [brackets] hand \\u0645\\u0631\\u062d\\u0628\\u0627 that budget scanned that tokens admission C++ and [brackets] that 'single'\"}, {\"role\": \"assistant\", \"content\": \"node.js regex while and WON'T body 100% gateway there's https://example.com/a/b?c=d WON'T keep brown 100% budget with budget \\u0645\\u0631\\u062d\\u0628\\u0627 and so don't while don't exactly backtracking faster tokens gateway node.js lazy over written counting 100% node.js tiktoken's brown dog v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 we'll C++ the with don't we'll tokens 100% no no written scanned counting involved hand user@example.com \\\"quotes\\\" lazy \\u0645\\u0631\\u062d\\u0628\\u0627 body counting for [brackets] 'single' backtracking piece It's regex don't admission once boundaries over lazy F# \\u6771\\u4eac tokens it \\u6771\\u4eac user@example.com fox that while tiktoken's Z\\u00fcrich and (parens) every [brackets] keep counting 100% 42 3.14159 counting it {braces} piece 'single' involved https://example.com/a/b?c=d involved quick \\ud83d\\ude42 tokens written that request tiktoken's lazy admission 42 I'M tokens \\\"quotes\\\" Z\\u00fcrich engine reservation tiktoken's once so a scanned {braces} scanner dog before piece written keep https://example.com/a/b?c=d while It's {braces} Z\\u00fcrich it\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich 3.14159 They'RE They'RE there's budget we'll gateway a brown [brackets] jumps once fox there's F# boundaries node.js for once 42 F# dog once so jumps 42 and 100% while it user@example.com exactly brown so [brackets] Z\\u00fcrich 'single' request with tiktoken's Z\\u00fcrich every reservation hand boundaries keep a \\ud83d\\ude42 it once admission piece and we'll budget dog na\\u00efve 3.14159 backtracking so gateway 42 Z\\u00fcrich faster (parens) node.js 100% lazy jumps fox scanner engine 'single' counting request \\u6771\\u4eac a regex brown that 1999 https://example.com/a/b?c=d user@example.com They'RE regex I'M \\ud83d\\ude42 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 scanner budget tokens [brackets] WON'T that quick budget v1.2.3 way way user@example.com request the [brackets] $1,234.56 gateway They'RE quick scanned $1,234.56 \\ud83d\\ude42 na\\u00efve don't admission \\u0645\\u0631\\u062d\\u0628\\u0627 budget \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"WON'T admission piece \\u6771\\u4eac gateway no \\ud83d\\ude42 WON'T tokens 'single' piece admission 3.14159 It's while I'M Z\\u00fcrich the keep boundaries Z\\u00fcrich WON'T It's [brackets] brown tokens They'RE exactly boundaries WON'T admission piece (parens) so tiktoken's backtracking is \\u0645\\u0631\\u062d\\u0628\\u0627 before mirrors 3.14159 dog backtracking written caf\\u00e9 hand scanned way that \\ud83d\\ude42 tiktoken's quick way They'RE {braces} counting na\\u00efve once reservation fox admission $1,234.56 hand and fox we'll hand because quick the is \\u6771\\u4eac {braces} \\\"quotes\\\" exactly and dog so scanner involved \\u6771\\u4eac [brackets]\"}, {\"role\": \"user\", \"content\": \"a \\u0645\\u0631\\u062d\\u0628\\u0627 I'M we'll before backtracking [brackets] scanner don't tokens admission tokens https://example.com/a/b?c=d C++ $1,234.56 is \\ud83d\\ude42 exactly dog piece no tiktoken's admission 'single' the quick that every before before dog It's engine so tiktoken's the boundaries https://example.com/a/b?c=d 'single' 1999 written body {braces} there's scanned budget admission don't for we'll \\\"quotes\\\" body quick regex quick there's we'll it https://example.com/a/b?c=d user@example.com involved na\\u00efve 3.14159 on we'll the over na\\u00efve na\\u00efve \\u6771\\u4eac gateway I'M v1.2.3 it way the scanner jumps budget C++ C++ \\u6771\\u4eac it with quick F# F# with piece budget involved request before backtracking the tiktoken's scanner $1,234.56 backtracking body admission over They'RE on engine request node.js while that They'RE backtracking request v1.2.3 42 3.14159 and 100% scanned a caf\\u00e9 every 100% exactly request tokens is gateway before a exactly caf\\u00e9 we'll node.js because \\ud83d\\ude42 admission\"}, {\"role\": \"assistant\", \"content\": \"F# \\u0645\\u0631\\u062d\\u0628\\u0627 the we'll user@example.com mirrors F# \\u6771\\u4eac faster over with C++ no because piece scanned because 42 piece (parens) scanned admission 100% Z\\u00fcrich It's 42 for 1999 the regex the dog faster budget (parens) brown engine It's on (parens) so https://example.com/a/b?c=d piece over v1.2.3 scanned don't It's piece regex admission https://example.com/a/b?c=d every don't dog F# no regex hand is dog piece no counting v1.2.3 don't https://example.com/a/b?c=d the 3.14159 because once https://example.com/a/b?c=d \\u6771\\u4eac\"}, {\"role\": \"user\", \"content\": \"we'll body tiktoken's F# lazy is is request we'll because dog \\u6771\\u4eac boundaries lazy caf\\u00e9 exactly quick jumps They'RE 3.14159 brown \\u6771\\u4eac 3.14159 'single' on keep faster 'single' because I'M no 'single' tiktoken's exactly F# v1.2.3 tokens hand because faster that over and gateway don't 42 \\u0645\\u0631\\u062d\\u0628\\u0627 lazy before that once for keep body lazy Z\\u00fcrich before faster piece gateway caf\\u00e9 exactly involved scanner the \\ud83d\\ude42 node.js WON'T way before C++ written 1999 It's with that is every na\\u00efve 1999 keep \\u6771\\u4eac quick request regex budget It's 1999 before a hand lazy engine WON'T don't while brown admission \\u0645\\u0631\\u062d\\u0628\\u0627 1999 caf\\u00e9 na\\u00efve WON'T boundaries counting reservation \\\"quotes\\\" there's the once over counting request\"}, {\"role\": \"assistant\", \"content\": \"gateway reservation once F# C++ written 42 on piece https://example.com/a/b?c=d [brackets] that \\u0645\\u0631\\u062d\\u0628\\u0627 we'll reservation They'RE node.js the fox piece https://example.com/a/b?c=d user@example.com every F# $1,234.56 \\\"quotes\\\" admission body [brackets] 'single' that and because regex we'll \\u6771\\u4eac 42 keep na\\u00efve F# user@example.com keep admission we'll user@example.com 1999 {braces} 100% because no $1,234.56 counting the 42 there's boundaries F# no admission 42 so so v1.2.3 for mirrors while the regex piece budget 42 node.js {braces} na\\u00efve regex with jumps that counting a involved $1,234.56 that admission tiktoken's na\\u00efve involved before $1,234.56 budget 1999 because v1.2.3 that backtracking \\u6771\\u4eac mirrors [brackets] node.js hand {braces} don't brown counting Z\\u00fcrich scanned over \\ud83d\\ude42 with I'M lazy tokens\"}, {\"role\": \"user\", \"content\": \"quick keep it body so every C++ gateway It's tokens 'single' 100% we'll quick regex so scanner v1.2.3 the involved It's while {braces} $1,234.56 that written that C++ scanned dog 100% 100% boundaries (parens) gateway exactly scanner regex gateway \\u6771\\u4eac mirrors brown 1999 regex v1.2.3 $1,234.56 caf\\u00e9 scanned brown there's caf\\u00e9 1999 \\\"quotes\\\" no F# for that 100% backtracking exactly admission the quick (parens) 'single' [brackets] {braces} that written boundaries reservation https://example.com/a/b?c=d request on quick 1999 hand so brown regex while exactly piece hand piece reservation tiktoken's scanner \\\"quotes\\\" caf\\u00e9 way because na\\u00efve (parens) tiktoken's 3.14159 admission so don't on involved na\\u00efve C++ dog reservation It's boundaries admission it (parens) gateway\"}, {\"role\": \"assistant\", \"content\": \"keep there's with scanner that budget no on node.js budget scanner dog before exactly quick 1999 WON'T reservation node.js it F# $1,234.56 caf\\u00e9 1999 scanned engine and \\ud83d\\ude42 [brackets] with scanned and admission \\u6771\\u4eac \\ud83d\\ude42 faster {braces} WON'T on I'M F# scanned while dog no exactly written C++ because tiktoken's request reservation written don't 100% gateway budget \\u6771\\u4eac that jumps counting \\u6771\\u4eac every engine scanned They'RE \\ud83d\\ude42 for engine F# and {braces} exactly exactly \\\"quotes\\\" the hand before\"}, {\"role\": \"user\", \"content\": \"jumps $1,234.56 we'll we'll {braces} user@example.com way hand fox way It's brown 3.14159 https://example.com/a/b?c=d regex jumps don't once dog written quick lazy tiktoken's WON'T way it budget the that quick It's WON'T They'RE brown keep because so keep \\u6771\\u4eac don't written because way with is I'M https://example.com/a/b?c=d 3.14159 scanner counting v1.2.3 gateway the it 42 while WON'T tokens caf\\u00e9 jumps once regex budget v1.2.3 C++ backtracking brown exactly involved every regex boundaries the mirrors F# F# $1,234.56 [brackets] WON'T \\u6771\\u4eac $1,234.56 jumps every that It's \\u0645\\u0631\\u062d\\u0628\\u0627 the keep while keep hand regex once jumps v1.2.3 caf\\u00e9 don't brown (parens) 'single'\"}, {\"role\": \"assistant\", \"content\": \"quick because faster $1,234.56 reservation F# don't the boundaries v1.2.3 on user@example.com reservation WON'T body 1999 brown \\u6771\\u4eac jumps \\\"quotes\\\" quick WON'T that (parens) the keep $1,234.56 C++ written {braces} so tokens on {braces} dog https://example.com/a/b?c=d \\ud83d\\ude42 3.14159 written backtracking gateway the way 'single' is gateway that jumps it fox piece backtracking tokens that faster backtracking there's a request request [brackets] and tokens $1,234.56 for user@example.com \\u6771\\u4eac so 'single' 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T don't 100% [brackets] for lazy the node.js hand because It's faster\"}, {\"role\": \"user\", \"content\": \"is faster admission budget that 1999 counting quick engine no while exactly while the is written v1.2.3 faster v1.2.3 mirrors once They'RE [brackets] tiktoken's 3.14159 quick with because fox while so [brackets] a dog while involved keep 1999 that involved \\ud83d\\ude42 keep is v1.2.3 100% once C++ node.js piece backtracking I'M 3.14159 that (parens) They'RE admission budget fox lazy scanned boundaries on on once that keep https://example.com/a/b?c=d lazy exactly because brown with exactly written \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"that 1999 node.js C++ \\u0645\\u0631\\u062d\\u0628\\u0627 no tiktoken's boundaries request so engine before 42 user@example.com a faster piece involved regex body for before lazy hand faster 'single' and hand tiktoken's lazy that node.js jumps budget fox way [brackets] v1.2.3 [brackets] 'single' brown piece It's counting mirrors every 3.14159 \\\"quotes\\\" https://example.com/a/b?c=d na\\u00efve we'll F# 100% once budget because scanned tokens Z\\u00fcrich engine scanned\"}, {\"role\": \"user\", \"content\": \"'single' It's over lazy piece 100% gateway caf\\u00e9 caf\\u00e9 \\\"quotes\\\" a way the request written the budget reservation counting $1,234.56 with Z\\u00fcrich counting I'M keep tokens once we'll the while https://example.com/a/b?c=d a on the tokens involved caf\\u00e9 gateway backtracking before fox na\\u00efve with on 3.14159 written lazy {braces} \\\"quotes\\\" there's don't quick 'single' that for They'RE faster because I'M is scanner \\u6771\\u4eac counting the every Z\\u00fcrich body $1,234.56 quick keep backtracking scanner 'single' lazy and 42 counting written so tokens engine 'single' because scanned They'RE v1.2.3 tokens \\u6771\\u4eac exactly 3.14159 on jumps jumps reservation tokens \\u0645\\u0631\\u062d\\u0628\\u0627 budget They'RE na\\u00efve na\\u00efve request\"}, {\"role\": \"assistant\", \"content\": \"lazy is reservation budget it jumps quick {braces} body 'single' is no jumps \\u6771\\u4eac Z\\u00fcrich no admission 100% and WON'T admission 'single' piece is we'll v1.2.3 is the we'll admission engine request body that body WON'T user@example.com 42 and piece I'M backtracking {braces} WON'T boundaries [brackets] engine request so backtracking budget {braces} caf\\u00e9 3.14159 {braces} It's hand once $1,234.56 They'RE budget It's quick brown regex 42 (parens) and 1999 regex Z\\u00fcrich Z\\u00fcrich Z\\u00fcrich exactly https://example.com/a/b?c=d way tokens no tiktoken's counting that hand backtracking jumps way over WON'T na\\u00efve {braces} piece it that 100% (parens) exactly and is over the WON'T user@example.com 1999 request written counting brown while on because 42 tokens no because (parens) Z\\u00fcrich lazy reservation admission once counting 42 exactly They'RE is dog\"}, {\"role\": \"user\", \"content\": \"that They'RE the admission F# (parens) https://example.com/a/b?c=d tiktoken's 3.14159 scanned reservation 'single' boundaries written exactly faster F# na\\u00efve while and [brackets] It's brown \\ud83d\\ude42 caf\\u00e9 jumps with \\\"quotes\\\" backtracking on reservation C++ written over regex [brackets] counting [brackets] lazy scanned $1,234.56 faster 'single' {braces} (parens) They'RE \\\"quotes\\\" on request F# while the dog faster https://example.com/a/b?c=d brown tokens 100% because WON'T way counting don't scanner a scanner 'single' dog because hand admission tiktoken's {braces} hand caf\\u00e9 that a \\ud83d\\ude42 fox \\ud83d\\ude42 \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors $1,234.56 once tiktoken's node.js \\u6771\\u4eac exactly \\\"quotes\\\" Z\\u00fcrich WON'T (parens) the hand It's once every no They'RE F# C++ lazy every don't 1999 user@example.com piece C++ admission 1999 because I'M budget\"}, {\"role\": \"assistant\", \"content\": \"dog over caf\\u00e9 the $1,234.56 before is that 'single' piece It's jumps exactly na\\u00efve every budget hand tokens $1,234.56 \\ud83d\\ude42 we'll (parens) hand for (parens) exactly way na\\u00efve body gateway brown keep It's before every lazy and while quick budget 100% (parens) no 1999 the 3.14159 regex that fox user@example.com 1999 it quick $1,234.56 there's no that exactly written there's \\u0645\\u0631\\u062d\\u0628\\u0627 request mirrors on involved involved C++ no [brackets] exactly na\\u00efve https://example.com/a/b?c=d 3.14159 gateway the jumps once scanned with that admission budget They'RE 'single' caf\\u00e9 mirrors WON'T \\u0645\\u0631\\u062d\\u0628\\u0627 reservation request It's the body admission reservation scanned we'll node.js regex counting before with \\ud83d\\ude42 backtracking [brackets] https://example.com/a/b?c=d 3.14159 the once It's request a written WON'T mirrors 1999\"}, {\"role\": \"user\", \"content\": \"1999 WON'T so user@example.com jumps fox \\ud83d\\ude42 {braces} there's budget before written quick mirrors {braces} for with 42 and gateway counting we'll request it so a for engine {braces} WON'T Z\\u00fcrich user@example.com because https://example.com/a/b?c=d the node.js is fox I'M 3.14159 tiktoken's faster is once 42 involved no 'single' I'M caf\\u00e9 WON'T I'M They'RE [brackets] $1,234.56 we'll I'M so 42 It's every (parens) we'll once \\ud83d\\ude42 once mirrors jumps budget body {braces} \\ud83d\\ude42 F# fox {braces} exactly \\ud83d\\ude42 WON'T that on 100% budget engine reservation C++ fox F# 42 no (parens) counting mirrors v1.2.3 for fox [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 while way 1999 boundaries piece 100% involved every \\ud83d\\ude42 'single' brown once v1.2.3 before 'single' (parens) v1.2.3 Z\\u00fcrich we'll involved 100% {braces}\"}, {\"role\": \"assistant\", \"content\": \"engine backtracking budget on na\\u00efve mirrors dog reservation don't jumps budget gateway counting dog tokens brown for way Z\\u00fcrich dog piece 'single' that quick it there's quick budget 42 It's that dog They'RE F# written dog piece \\u0645\\u0631\\u062d\\u0628\\u0627 F# regex every no hand body once (parens) every lazy 100% over quick once mirrors don't and v1.2.3 They'RE faster brown 1999 a \\ud83d\\ude42 42 gateway https://example.com/a/b?c=d Z\\u00fcrich for over na\\u00efve it lazy admission admission brown hand \\\"quotes\\\" gateway reservation 1999 keep and {braces} keep F# request written tiktoken's every budget every regex there's exactly I'M \\\"quotes\\\" \\u0645\\u0631\\u062d\\u0628\\u0627 there's backtracking and don't boundaries F# caf\\u00e9 3.14159 (parens) 3.14159 user@example.com https://example.com/a/b?c=d tiktoken's keep every\"}, {\"role\": \"user\", \"content\": \"gateway 'single' v1.2.3 request that hand tiktoken's over v1.2.3 'single' so \\ud83d\\ude42 1999 once once na\\u00efve admission na\\u00efve piece that \\\"quotes\\\" 3.14159 way exactly engine F# the \\u0645\\u0631\\u062d\\u0628\\u0627 3.14159 user@example.com on https://example.com/a/b?c=d It's https://example.com/a/b?c=d F# there's written with {braces} admission {braces} a on user@example.com dog dog [brackets] that caf\\u00e9 over for admission v1.2.3 100% node.js boundaries scanner 42 piece because is (parens) (parens) body no is {braces} that user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 hand \\\"quotes\\\" user@example.com on that \\\"quotes\\\" backtracking \\\"quotes\\\" on no hand 3.14159 brown faster 100% (parens)\"}, {\"role\": \"assistant\", \"content\": \"there's user@example.com They'RE It's lazy user@example.com before before https://example.com/a/b?c=d request WON'T WON'T fox WON'T and faster for body involved 3.14159 keep written admission on gateway written I'M with quick v1.2.3 C++ there's jumps quick admission scanned jumps node.js before while 1999 over node.js with reservation every once node.js brown don't there's mirrors a [brackets] so faster before \\ud83d\\ude42 tiktoken's admission F# don't written na\\u00efve admission is fox we'll way dog tokens scanned the \\\"quotes\\\" hand no 'single' \\ud83d\\ude42 over we'll that regex so every for (parens) is with the \\\"quotes\\\" exactly piece regex exactly brown for for hand tokens C++ user@example.com tiktoken's engine I'M the It's faster regex is dog 'single' C++ user@example.com \\ud83d\\ude42 Z\\u00fcrich {braces} node.js for \\u6771\\u4eac brown regex the with tiktoken's 42 $1,234.56 (parens)\"}, {\"role\": \"user\", \"content\": \"request it reservation that https://example.com/a/b?c=d so engine user@example.com jumps that and the 3.14159 because 3.14159 brown tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 scanner \\u0645\\u0631\\u062d\\u0628\\u0627 fox counting there's dog lazy node.js tiktoken's I'M we'll 42 it tokens faster faster before \\u0645\\u0631\\u062d\\u0628\\u0627 reservation so with the regex it once backtracking for It's hand regex way there's 100% exactly dog no we'll 42 brown lazy scanned budget na\\u00efve WON'T because and we'll 42 way tiktoken's WON'T gateway we'll I'M dog https://example.com/a/b?c=d scanned and that mirrors \\ud83d\\ude42 C++ They'RE with counting on keep na\\u00efve 3.14159 once request keep admission \\\"quotes\\\" C++ \\ud83d\\ude42 once quick boundaries user@example.com tokens budget budget scanner caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 exactly node.js hand\"}, {\"role\": \"assistant\", \"content\": \"fox for user@example.com there's jumps quick fox I'M boundaries mirrors before involved over node.js admission that is lazy It's 1999 1999 fox involved user@example.com \\\"quotes\\\" boundaries 100% \\u0645\\u0631\\u062d\\u0628\\u0627 that \\u6771\\u4eac backtracking 100% there's jumps because 42 tiktoken's caf\\u00e9 that v1.2.3 tokens brown \\ud83d\\ude42 backtracking user@example.com F# 100% 3.14159 no boundaries It's quick https://example.com/a/b?c=d Z\\u00fcrich tokens 42 100% over WON'T They'RE for lazy body involved every on on user@example.com \\u6771\\u4eac (parens) quick once faster quick over backtracking don't once They'RE engine caf\\u00e9 tokens fox because there's every dog we'll \\u6771\\u4eac exactly 3.14159 \\ud83d\\ude42 boundaries no every the F# 100% faster faster once boundaries a I'M {braces} 1999 over \\\"quotes\\\" mirrors that fox 3.14159 lazy na\\u00efve admission that reservation no for written because scanned caf\\u00e9 body\"}, {\"role\": \"user\", \"content\": \"and involved body jumps \\\"quotes\\\" {braces} it we'll regex it na\\u00efve counting the reservation keep admission once before while admission v1.2.3 I'M reservation 1999 scanned tokens counting don't boundaries lazy exactly we'll $1,234.56 and \\u0645\\u0631\\u062d\\u0628\\u0627 \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 engine I'M 42 F# keep every (parens) budget 100% written tiktoken's (parens) don't na\\u00efve I'M body because the is (parens) admission brown hand piece 42 every request don't $1,234.56 once there's 1999 with we'll that backtracking F# with fox mirrors [brackets] for WON'T with jumps request fox scanned a request while no that faster way I'M \\u6771\\u4eac boundaries that brown dog so (parens) written\"}, {\"role\": \"assistant\", \"content\": \" 42 exactly and \\\"quotes\\\" mirrors admission F# faster F# a on faster with quick brown reservation piece (parens) fox \\u0645\\u0631\\u062d\\u0628\\u0627 budget C++ v1.2.3 so over node.js Z\\u00fcrich WON'T tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 F# 1999 42 the involved once 100% is quick (parens) before jumps dog don't over because $1,234.56 reservation 3.14159 that WON'T caf\\u00e9 backtracking request the \\u0645\\u0631\\u062d\\u0628\\u0627 piece every involved 100% budget there's boundaries counting once written we'll na\\u00efve 42 \\ud83d\\ude42 I'M 3.14159 regex scanner reservation tiktoken's on I'M \\u6771\\u4eac boundaries on brown there's that 'single' the exactly scanned no admission request we'll tiktoken's 'single' because \\\"quotes\\\" \\u6771\\u4eac no tiktoken's caf\\u00e9 jumps it tiktoken's lazy faster exactly body tiktoken's fox budget\"}, {\"role\": \"user\", \"content\": \"so fox a backtracking Z\\u00fcrich admission budget fox scanned 42 request don't because reservation way admission gateway v1.2.3 the so \\u6771\\u4eac involved 'single' with a 100% 'single' v1.2.3 user@example.com dog backtracking body C++ piece the so mirrors written once $1,234.56 while fox 42 [brackets] F# we'll Z\\u00fcrich piece budget while for we'll the we'll written exactly $1,234.56 mirrors counting \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors on 'single' na\\u00efve is node.js while is a on\"}, {\"role\": \"assistant\", \"content\": \"on involved 42 'single' They'RE piece while [brackets] dog that and there's before once jumps counting that so is \\u6771\\u4eac [brackets] over and exactly with involved it once on 1999 42 3.14159 is keep \\u6771\\u4eac jumps that there's They'RE piece lazy over with lazy request \\ud83d\\ude42 written node.js admission backtracking way the over backtracking node.js and while $1,234.56 [brackets] while 42 \\u0645\\u0631\\u062d\\u0628\\u0627 I'M 'single' while user@example.com WON'T \\\"quotes\\\" before admission the faster budget mirrors counting we'll once jumps involved WON'T 3.14159 v1.2.3 [brackets] keep over for 'single' \\u6771\\u4eac regex written na\\u00efve way (parens) user@example.com request there's every involved I'M for scanned It's WON'T https://example.com/a/b?c=d C++ caf\\u00e9 once user@example.com [brackets] a 1999 it tiktoken's 'single' a over mirrors jumps request there's dog reservation mirrors tokens jumps tiktoken's is user@example.com tokens\"}, {\"role\": \"user\", \"content\": \"backtracking once over user@example.com body scanner piece faster \\ud83d\\ude42 over for C++ exactly https://example.com/a/b?c=d no engine 'single' piece 42 over hand for for backtracking quick way 1999 regex scanned every reservation no before over \\\"quotes\\\" so $1,234.56 [brackets] tokens na\\u00efve request a budget fox budget user@example.com tokens brown I'M \\\"quotes\\\" hand I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac engine once engine backtracking while there's no [brackets] jumps the for [brackets] there's \\u0645\\u0631\\u062d\\u0628\\u0627 dog 42 admission so na\\u00efve F# there's \\u6771\\u4eac reservation F# keep piece keep user@example.com caf\\u00e9 the that It's $1,234.56 v1.2.3 It's na\\u00efve engine lazy They'RE counting faster 'single' piece scanned $1,234.56 user@example.com boundaries [brackets] 'single' \\u6771\\u4eac brown so scanned 'single' scanner boundaries 'single' admission admission \\\"quotes\\\" caf\\u00e9 piece\"}, {\"role\": \"assistant\", \"content\": \"Z\\u00fcrich counting while the the no 100% \\u0645\\u0631\\u062d\\u0628\\u0627 body 3.14159 before the while 1999 once that on caf\\u00e9 1999 on it tiktoken's piece scanned 'single' Z\\u00fcrich and [brackets] regex with regex \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com counting admission way keep scanner scanner \\u0645\\u0631\\u062d\\u0628\\u0627 every so [brackets] tokens $1,234.56 every budget boundaries because reservation and keep 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d tiktoken's https://example.com/a/b?c=d scanner admission caf\\u00e9 $1,234.56 F# faster so engine so it regex 'single' before hand with that admission fox keep Z\\u00fcrich no scanned I'M tiktoken's the it admission a jumps brown that there's it that (parens) 100% (parens) gateway for\"}, {\"role\": \"user\", \"content\": \"so 100% once it there's (parens) that with 1999 lazy written written keep na\\u00efve brown caf\\u00e9 regex the tiktoken's that a while brown It's dog and It's 'single' $1,234.56 42 \\u6771\\u4eac I'M v1.2.3 once na\\u00efve I'M {braces} node.js scanned It's hand piece no caf\\u00e9 backtracking admission with we'll is on node.js fox on tokens brown brown regex scanner admission $1,234.56 lazy regex piece regex node.js (parens) once on while quick dog gateway body that admission for written it scanned I'M it mirrors because the v1.2.3 every 3.14159 boundaries request na\\u00efve It's because faster hand dog tokens v1.2.3 hand that tiktoken's keep caf\\u00e9 keep tokens we'll it jumps no \\ud83d\\ude42 node.js exactly na\\u00efve na\\u00efve every involved tiktoken's \\u6771\\u4eac counting dog over user@example.com https://example.com/a/b?c=d 1999 3.14159 node.js is hand F# written a\"}, {\"role\": \"assistant\", \"content\": \"we'll fox there's 3.14159 C++ so budget because we'll Z\\u00fcrich the on before we'll that Z\\u00fcrich fox keep scanned dog once that \\\"quotes\\\" while scanned brown engine for fox while there's (parens) 42 every we'll caf\\u00e9 https://example.com/a/b?c=d budget exactly gateway F# https://example.com/a/b?c=d I'M dog way before brown with it 3.14159 don't no engine the piece keep user@example.com Z\\u00fcrich It's once written and They'RE the admission scanner with written that fox engine body on scanner 1999 the admission WON'T quick user@example.com I'M admission tiktoken's https://example.com/a/b?c=d while https://example.com/a/b?c=d every with involved the admission gateway there's 'single' gateway a scanner and request that is engine counting dog user@example.com that hand counting lazy admission boundaries engine while way WON'T brown \\ud83d\\ude42 F# I'M F# scanner \\u6771\\u4eac 3.14159 a backtracking scanner I'M brown\"}, {\"role\": \"user\", \"content\": \"I'M lazy It's over on faster fox tiktoken's while that It's for that reservation https://example.com/a/b?c=d keep node.js the \\u6771\\u4eac the admission body 1999 exactly and way dog (parens) node.js \\u6771\\u4eac on They'RE quick that faster F# boundaries tokens boundaries reservation reservation backtracking tokens is brown 100% {braces} quick engine we'll node.js 100% scanned over F# hand na\\u00efve that request jumps quick because regex\"}, {\"role\": \"assistant\", \"content\": \"with 100% $1,234.56 boundaries we'll so no \\\"quotes\\\" $1,234.56 Z\\u00fcrich we'll brown on exactly 1999 WON'T caf\\u00e9 every dog we'll regex before tokens before [brackets] lazy over faster once Z\\u00fcrich request lazy They'RE tokens https://example.com/a/b?c=d [brackets] don't It's with keep hand hand (parens) I'M budget {braces} faster it way is there's every dog gateway \\\"quotes\\\" it Z\\u00fcrich way the over quick tokens C++ hand mirrors It's It's (parens) F# 100% boundaries caf\\u00e9 F# we'll scanner admission\"}, {\"role\": \"user\", \"content\": \"it node.js caf\\u00e9 'single' quick C++ $1,234.56 3.14159 hand 42 $1,234.56 I'M every exactly 100% lazy the \\u6771\\u4eac tiktoken's WON'T jumps with \\u0645\\u0631\\u062d\\u0628\\u0627 I'M counting written piece and keep it regex on F# keep 3.14159 'single' dog and boundaries tokens \\\"quotes\\\" engine with hand that 3.14159 brown WON'T don't scanner keep I'M WON'T fox F# exactly 'single' piece engine and so on request and It's reservation written that boundaries is caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 on every counting a written so jumps with scanner with quick don't jumps way \\u6771\\u4eac keep mirrors regex (parens) we'll it scanned node.js $1,234.56 reservation They'RE caf\\u00e9 node.js for while scanner it faster [brackets] gateway it scanner for mirrors na\\u00efve admission dog on\"}, {\"role\": \"assistant\", \"content\": \"that v1.2.3 written node.js I'M keep user@example.com tiktoken's tiktoken's way backtracking They'RE quick while scanned way we'll user@example.com way 1999 way caf\\u00e9 https://example.com/a/b?c=d hand faster is involved 1999 (parens) quick is (parens) on brown scanner with dog don't admission involved exactly F# They'RE is with exactly way before counting \\\"quotes\\\" backtracking once gateway faster 3.14159 [brackets] way [brackets] a keep \\ud83d\\ude42 hand (parens) counting boundaries budget I'M mirrors v1.2.3 F# once way 1999 WON'T reservation [brackets] for it scanned because 100% reservation 100% tiktoken's They'RE engine a it that so mirrors written over mirrors node.js F# \\ud83d\\ude42 brown dog it v1.2.3 user@example.com the 1999 reservation counting [brackets] budget way mirrors C++ F# and so there's so They'RE I'M scanner request request WON'T and written budget Z\\u00fcrich faster written lazy C++ brown user@example.com \\ud83d\\ude42 lazy reservation caf\\u00e9 node.js caf\\u00e9 brown\"}, {\"role\": \"user\", \"content\": \"budget budget and \\u6771\\u4eac tiktoken's tiktoken's don't that boundaries that 1999 backtracking with (parens) tiktoken's there's fox counting because exactly the every [brackets] engine Z\\u00fcrich admission way scanned boundaries \\ud83d\\ude42 user@example.com counting na\\u00efve F# fox 100% so faster 1999 because so quick engine way fox that engine lazy quick 1999 way $1,234.56 quick while body that 'single' Z\\u00fcrich They'RE na\\u00efve keep 3.14159 https://example.com/a/b?c=d the involved I'M jumps\"}, {\"role\": \"assistant\", \"content\": \"every jumps Z\\u00fcrich piece 'single' F# we'll 'single' over exactly written C++ 3.14159 with scanner exactly while user@example.com a body engine there's engine way 100% request [brackets] exactly way jumps Z\\u00fcrich budget once gateway before I'M \\\"quotes\\\" https://example.com/a/b?c=d user@example.com written admission exactly I'M {braces} \\ud83d\\ude42 They'RE with WON'T on backtracking node.js a with a lazy request na\\u00efve that scanner that once scanned quick 'single'\"}, {\"role\": \"user\", \"content\": \"v1.2.3 budget brown every [brackets] that na\\u00efve there's the and (parens) is brown They'RE F# there's every there's because the so because \\u0645\\u0631\\u062d\\u0628\\u0627 and over 3.14159 brown piece F# involved (parens) so regex while Z\\u00fcrich (parens) counting user@example.com while on $1,234.56 it v1.2.3 {braces} engine v1.2.3 while 1999 keep regex while https://example.com/a/b?c=d once engine WON'T involved \\u6771\\u4eac I'M it with for on hand \\ud83d\\ude42 v1.2.3 on a na\\u00efve regex there's na\\u00efve before v1.2.3 dog (parens) for brown 3.14159 v1.2.3 [brackets] exactly exactly no user@example.com on mirrors It's with that {braces} faster\"}, {\"role\": \"assistant\", \"content\": \"for once admission backtracking engine because once for I'M brown fox the quick quick backtracking there's lazy engine mirrors every don't caf\\u00e9 before mirrors engine 3.14159 on user@example.com with don't keep it node.js before hand regex reservation while scanner na\\u00efve body body no with F# admission Z\\u00fcrich F# WON'T no I'M \\ud83d\\ude42 way scanned $1,234.56 with a no it caf\\u00e9 lazy the dog https://example.com/a/b?c=d involved caf\\u00e9 jumps we'll scanner caf\\u00e9 42 na\\u00efve is we'll\"}, {\"role\": \"user\", \"content\": \"'single' scanner 100% we'll it a {braces} while it v1.2.3 while mirrors there's exactly caf\\u00e9 It's gateway (parens) fox every dog (parens) a written while while before hand involved admission written counting It's mirrors C++ 1999 scanned fox don't written don't reservation exactly mirrors it https://example.com/a/b?c=d there's no scanned that I'M involved $1,234.56 lazy fox scanned written I'M 100% is we'll every don't F# brown every It's keep while scanned jumps tokens 'single' hand Z\\u00fcrich request admission It's 'single' quick backtracking user@example.com request tokens budget exactly gateway jumps {braces} 'single' once tokens a Z\\u00fcrich before admission admission way jumps fox piece hand They'RE we'll exactly reservation on it written mirrors every once $1,234.56 scanner scanner Z\\u00fcrich jumps is over (parens) written piece Z\\u00fcrich $1,234.56 before request 'single' is 'single' keep\"}, {\"role\": \"assistant\", \"content\": \"scanned $1,234.56 gateway over because It's because tokens (parens) C++ admission is hand C++ brown v1.2.3 user@example.com brown \\\"quotes\\\" once backtracking lazy [brackets] with scanner dog lazy 'single' budget 1999 we'll reservation exactly on exactly the counting the [brackets] engine They'RE scanner caf\\u00e9 (parens) involved na\\u00efve on dog (parens) before don't while fox F# request once the lazy \\ud83d\\ude42 gateway user@example.com once that for 42 involved body user@example.com on regex the for lazy backtracking node.js body over involved v1.2.3 \\u6771\\u4eac regex mirrors there's involved regex faster that before request 3.14159 3.14159 that body 1999 faster for 42 'single' user@example.com request user@example.com piece scanned 1999 keep 3.14159 every fox F# scanned before jumps that admission counting They'RE that tokens I'M\"}, {\"role\": \"user\", \"content\": \"exactly quick and https://example.com/a/b?c=d a brown is dog hand regex It's tiktoken's the every with on scanner every keep scanned backtracking because budget keep fox there's so with so that on exactly counting once it exactly (parens) 3.14159 and that while fox admission 42 reservation It's the na\\u00efve 42 regex we'll piece hand so written regex faster fox backtracking that They'RE while 1999 \\u6771\\u4eac \\\"quotes\\\" request \\u6771\\u4eac They'RE and 100% the written for over way 1999 tokens https://example.com/a/b?c=d brown I'M brown brown keep while with $1,234.56 hand user@example.com {braces} [brackets] that faster na\\u00efve scanner on it caf\\u00e9 counting They'RE \\\"quotes\\\" {braces} counting there's body don't jumps budget fox reservation faster \\u0645\\u0631\\u062d\\u0628\\u0627 faster we'll with gateway [brackets] over counting It's there's \\u6771\\u4eac tokens that backtracking scanner that it so\"}, {\"role\": \"assistant\", \"content\": \"boundaries body fox counting that F# on there's caf\\u00e9 scanned jumps 42 dog hand jumps lazy \\\"quotes\\\" for WON'T is 42 on while 1999 scanner over there's engine 100% \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" jumps before don't [brackets] is counting involved It's I'M boundaries the (parens) with I'M engine 100% for exactly 'single' piece C++ na\\u00efve {braces} we'll every a no a \\\"quotes\\\" user@example.com body the C++ written https://example.com/a/b?c=d counting I'M a v1.2.3 dog 'single' Z\\u00fcrich keep scanner written before over lazy scanned the with keep over 'single' lazy 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) dog scanned the the dog no $1,234.56 with a so because quick Z\\u00fcrich fox v1.2.3 once faster \\ud83d\\ude42 faster boundaries it 1999 mirrors way a 3.14159 so it written \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d because the before involved Z\\u00fcrich budget while v1.2.3 scanned boundaries regex we'll backtracking\"}, {\"role\": \"user\", \"content\": \"involved $1,234.56 while jumps \\ud83d\\ude42 \\\"quotes\\\" piece hand written fox jumps v1.2.3 hand piece a jumps [brackets] brown involved Z\\u00fcrich engine that faster 'single' way faster budget F# 1999 so that tiktoken's involved regex WON'T budget mirrors mirrors F# before gateway I'M They'RE https://example.com/a/b?c=d lazy counting gateway regex boundaries \\\"quotes\\\" \\\"quotes\\\" 100% {braces} keep written 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 boundaries before written na\\u00efve 3.14159 that v1.2.3 no before request brown dog for $1,234.56 we'll node.js written mirrors scanned WON'T because we'll we'll way F# piece They'RE and while\"}, {\"role\": \"assistant\", \"content\": \"and that scanner brown They'RE we'll Z\\u00fcrich C++ the it 1999 hand F# admission \\u6771\\u4eac is mirrors so once the it admission counting is boundaries gateway that tiktoken's gateway lazy \\u6771\\u4eac na\\u00efve the we'll fox we'll is 42 that hand 3.14159 while reservation piece there's 1999 (parens) so involved we'll because involved written it caf\\u00e9 before once because https://example.com/a/b?c=d hand fox $1,234.56 admission is hand $1,234.56 boundaries for backtracking WON'T 1999 gateway the They'RE [brackets] budget na\\u00efve because that 42 They'RE C++ is don't \\ud83d\\ude42 C++ I'M $1,234.56 once 3.14159 1999 Z\\u00fcrich boundaries reservation quick exactly gateway $1,234.56 {braces}\"}, {\"role\": \"user\", \"content\": \"42 written (parens) backtracking jumps 1999 a \\\"quotes\\\" involved scanned admission They'RE gateway so backtracking 42 100% keep written It's WON'T user@example.com scanner that we'll WON'T Z\\u00fcrich don't na\\u00efve \\ud83d\\ude42 lazy mirrors (parens) before dog 'single' backtracking tokens faster 100% the body with 42 while {braces} while lazy brown I'M way exactly involved $1,234.56 'single' tokens engine counting tiktoken's $1,234.56 it user@example.com so reservation a 3.14159 dog that brown body fox mirrors quick before don't lazy gateway 100% \\u6771\\u4eac tokens the 42 dog 100% the gateway lazy because request there's $1,234.56 It's and regex 100% It's 'single' {braces} that keep involved caf\\u00e9 with {braces} They'RE faster the the $1,234.56 I'M backtracking I'M reservation 42 1999 before lazy request keep mirrors [brackets] 3.14159 42 {braces} $1,234.56 body 42 It's request 3.14159 {braces}\"}, {\"role\": \"assistant\", \"content\": \"and dog body user@example.com fox quick faster scanned the it faster $1,234.56 $1,234.56 with user@example.com over gateway fox tokens piece quick Z\\u00fcrich and counting engine no dog dog reservation on it it 100% every regex {braces} node.js mirrors before the Z\\u00fcrich v1.2.3 mirrors \\ud83d\\ude42 scanned WON'T brown there's \\u6771\\u4eac that mirrors for I'M it mirrors it node.js is \\u6771\\u4eac body \\\"quotes\\\" dog WON'T tiktoken's regex the It's the It's keep 3.14159 quick regex budget WON'T don't and counting over piece scanned C++ scanned is no \\u0645\\u0631\\u062d\\u0628\\u0627 1999 we'll it https://example.com/a/b?c=d keep https://example.com/a/b?c=d user@example.com admission backtracking once user@example.com piece request that (parens) budget written faster we'll budget scanned the user@example.com regex for WON'T piece counting\"}, {\"role\": \"user\", \"content\": \"{braces} 100% for It's the 1999 \\\"quotes\\\" scanned the 'single' quick caf\\u00e9 \\ud83d\\ude42 once once there's C++ It's we'll I'M engine no It's it na\\u00efve involved fox we'll written jumps 3.14159 dog exactly there's It's gateway tiktoken's every It's C++ WON'T we'll na\\u00efve reservation user@example.com 3.14159 every that with backtracking hand every v1.2.3 before so tiktoken's brown It's 3.14159 v1.2.3 caf\\u00e9 because \\\"quotes\\\" faster user@example.com It's before It's (parens) for I'M request written written dog \\u6771\\u4eac counting faster boundaries because tiktoken's on and gateway fox C++ admission It's {braces} v1.2.3 [brackets] 3.14159 exactly that budget that C++ (parens) so 1999 regex involved budget because scanner {braces} dog because I'M reservation exactly [brackets] reservation don't\"}, {\"role\": \"assistant\", \"content\": \"scanner They'RE \\\"quotes\\\" gateway 100% 3.14159 $1,234.56 so request 1999 C++ na\\u00efve quick faster we'll WON'T dog before engine 'single' tiktoken's node.js \\u0645\\u0631\\u062d\\u0628\\u0627 scanner it and na\\u00efve \\ud83d\\ude42 regex so once there's (parens) before 3.14159 dog [brackets] hand 100% backtracking over don't is once jumps caf\\u00e9 https://example.com/a/b?c=d so lazy that 3.14159 reservation on admission \\ud83d\\ude42 (parens) exactly budget v1.2.3 backtracking with brown request It's it with tokens (parens) scanner 1999 backtracking a fox faster v1.2.3 $1,234.56 I'M node.js the reservation piece [brackets] so \\u6771\\u4eac lazy counting the $1,234.56 I'M backtracking over 3.14159 fox fox keep the backtracking with It's [brackets] 3.14159 (parens) that They'RE so [brackets] piece [brackets] {braces} counting written boundaries mirrors budget (parens) gateway \\ud83d\\ude42 exactly hand engine WON'T 3.14159 faster WON'T fox request\"}, {\"role\": \"user\", \"content\": \"while quick \\u0645\\u0631\\u062d\\u0628\\u0627 so every 42 \\ud83d\\ude42 boundaries counting gateway that \\u0645\\u0631\\u062d\\u0628\\u0627 'single' no 100% and regex because before request and 'single' the once F# na\\u00efve $1,234.56 quick $1,234.56 reservation don't node.js scanner 42 node.js 1999 $1,234.56 https://example.com/a/b?c=d jumps lazy 'single' WON'T They'RE WON'T brown {braces} that scanner because Z\\u00fcrich written dog 3.14159 budget \\\"quotes\\\" engine 3.14159 3.14159 for before exactly tiktoken's jumps user@example.com is (parens) scanner lazy piece request brown Z\\u00fcrich node.js admission It's\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 C++ node.js because it 42 dog boundaries mirrors piece don't C++ 3.14159 https://example.com/a/b?c=d scanner there's is keep \\ud83d\\ude42 once before because backtracking over $1,234.56 [brackets] quick we'll gateway because we'll exactly once every gateway with over because 3.14159 keep every [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 the because 42 1999 no 100% backtracking 1999 \\u6771\\u4eac faster (parens) faster 42 the F# budget dog 1999 regex before\"}, {\"role\": \"user\", \"content\": \"no it \\ud83d\\ude42 mirrors https://example.com/a/b?c=d reservation C++ on budget \\u6771\\u4eac the for {braces} [brackets] and tiktoken's written 42 so body \\\"quotes\\\" involved C++ budget is written WON'T jumps It's {braces} counting reservation tokens brown piece hand before no It's gateway gateway [brackets] piece mirrors there's \\ud83d\\ude42 while a caf\\u00e9 \\u6771\\u4eac caf\\u00e9 counting counting it 3.14159 don't caf\\u00e9 so written for tiktoken's boundaries tokens 1999 fox keep quick that boundaries that no scanner Z\\u00fcrich budget it written that backtracking involved quick mirrors backtracking brown jumps the lazy They'RE and 42 backtracking 100% brown tokens gateway boundaries caf\\u00e9 reservation way https://example.com/a/b?c=d $1,234.56 100% regex \\u0645\\u0631\\u062d\\u0628\\u0627 lazy we'll \\u0645\\u0631\\u062d\\u0628\\u0627 involved jumps \\u0645\\u0631\\u062d\\u0628\\u0627 while node.js Z\\u00fcrich Z\\u00fcrich counting 100% (parens) that 100% on {braces} counting once user@example.com faster gateway \\u0645\\u0631\\u062d\\u0628\\u0627 that with\"}, {\"role\": \"assistant\", \"content\": \"faster on that boundaries on They'RE F# https://example.com/a/b?c=d that node.js quick for 100% the regex v1.2.3 don't with 42 C++ gateway is before keep is I'M the (parens) They'RE It's \\\"quotes\\\" so mirrors every backtracking tiktoken's v1.2.3 the 3.14159 They'RE scanner over no so that and WON'T that every the don't \\\"quotes\\\" 42 3.14159 node.js budget every 'single' caf\\u00e9 node.js before counting engine\"}, {\"role\": \"user\", \"content\": \"3.14159 piece tiktoken's with request regex on body [brackets] Z\\u00fcrich every gateway the that tiktoken's https://example.com/a/b?c=d no so budget node.js 1999 Z\\u00fcrich we'll They'RE keep and because piece every \\ud83d\\ude42 reservation faster written while request 3.14159 because {braces} fox WON'T written \\\"quotes\\\" (parens) over jumps mirrors body that jumps hand keep 3.14159 a the the it because so we'll gateway tiktoken's regex the scanned 1999 Z\\u00fcrich and don't and\"}, {\"role\": \"assistant\", \"content\": \"quick v1.2.3 100% node.js the na\\u00efve na\\u00efve involved once hand that \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors that request I'M tokens C++ don't na\\u00efve it mirrors there's with tokens https://example.com/a/b?c=d a there's so I'M mirrors $1,234.56 hand mirrors $1,234.56 admission C++ \\u0645\\u0631\\u062d\\u0628\\u0627 scanned user@example.com piece fox admission node.js https://example.com/a/b?c=d caf\\u00e9 It's \\ud83d\\ude42 scanned before https://example.com/a/b?c=d (parens) way https://example.com/a/b?c=d because F# I'M is the It's is reservation exactly user@example.com It's https://example.com/a/b?c=d lazy while the \\ud83d\\ude42 42 because for tiktoken's\"}, {\"role\": \"user\", \"content\": \"fox over {braces} is because caf\\u00e9 hand {braces} backtracking scanner F# engine engine user@example.com It's 42 quick jumps C++ \\\"quotes\\\" Z\\u00fcrich involved C++ na\\u00efve don't we'll admission exactly I'M a {braces} regex $1,234.56 exactly for [brackets] piece F# tokens \\\"quotes\\\" na\\u00efve dog lazy na\\u00efve there's on that with request \\u0645\\u0631\\u062d\\u0628\\u0627 quick a the gateway \\\"quotes\\\" and user@example.com 1999 (parens) over dog involved on engine over so hand over gateway Z\\u00fcrich counting keep that and $1,234.56 tokens scanned [brackets] reservation Z\\u00fcrich it node.js\"}, {\"role\": \"assistant\", \"content\": \"while on because fox \\u6771\\u4eac mirrors quick lazy I'M F# while a the because {braces} \\\"quotes\\\" once mirrors 100% a that on before body over request tokens we'll jumps that for 1999 mirrors hand lazy we'll over dog \\\"quotes\\\" because over way scanned that brown gateway over brown Z\\u00fcrich don't \\ud83d\\ude42 involved mirrors that once v1.2.3 \\ud83d\\ude42 v1.2.3 counting (parens) quick 'single' $1,234.56 scanner https://example.com/a/b?c=d no It's boundaries tokens hand caf\\u00e9 request body scanned 42 a counting with engine there's v1.2.3 fox gateway hand so counting scanned Z\\u00fcrich written https://example.com/a/b?c=d that {braces} v1.2.3 involved there's hand it there's is the exactly \\u0645\\u0631\\u062d\\u0628\\u0627 that scanner\"}, {\"role\": \"user\", \"content\": \"\\u0645\\u0631\\u062d\\u0628\\u0627 piece way scanned we'll hand written jumps fox https://example.com/a/b?c=d lazy $1,234.56 Z\\u00fcrich na\\u00efve WON'T scanner brown \\\"quotes\\\" mirrors It's caf\\u00e9 I'M \\\"quotes\\\" while admission fox engine brown tokens hand over while no scanner 3.14159 reservation budget \\\"quotes\\\" 100% hand while the It's faster tokens hand admission no and F# because \\u6771\\u4eac It's 'single' body that tiktoken's reservation tiktoken's is They'RE every (parens) Z\\u00fcrich scanner request the that F# caf\\u00e9 it They'RE way because\"}, {\"role\": \"assistant\", \"content\": \"brown \\\"quotes\\\" and don't regex [brackets] before on a that is every don't mirrors (parens) over lazy with F# on mirrors boundaries body jumps dog the It's \\u0645\\u0631\\u062d\\u0628\\u0627 so way before jumps WON'T request lazy v1.2.3 caf\\u00e9 it https://example.com/a/b?c=d $1,234.56 tiktoken's and {braces} admission \\u0645\\u0631\\u062d\\u0628\\u0627 fox on jumps jumps faster node.js user@example.com user@example.com so https://example.com/a/b?c=d counting It's \\\"quotes\\\" admission scanner on that while I'M piece written node.js way dog dog 100% before with \\u6771\\u4eac node.js hand mirrors involved 1999 a mirrors\"}, {\"role\": \"user\", \"content\": \"hand tiktoken's [brackets] 42 before 'single' F# F# for 'single' with body way every gateway {braces} brown keep scanner I'M the tiktoken's https://example.com/a/b?c=d for dog 100% tokens body counting over v1.2.3 https://example.com/a/b?c=d no 1999 They'RE {braces} tokens exactly scanner a for \\u0645\\u0631\\u062d\\u0628\\u0627 node.js way $1,234.56 gateway backtracking the brown a tiktoken's dog na\\u00efve I'M tokens for node.js boundaries a It's it C++ that tokens way C++\"}, {\"role\": \"assistant\", \"content\": \"a a counting don't WON'T no tokens quick we'll engine that written \\u0645\\u0631\\u062d\\u0628\\u0627 so 'single' every I'M Z\\u00fcrich there's v1.2.3 lazy there's caf\\u00e9 that scanned keep is fox 1999 100% tokens (parens) jumps 1999 v1.2.3 3.14159 WON'T {braces} for dog [brackets] the involved brown fox [brackets] (parens) They'RE (parens) gateway 1999 before a is scanned keep piece don't I'M exactly na\\u00efve we'll don't that 42 every gateway (parens) the na\\u00efve with Z\\u00fcrich backtracking lazy 100% is node.js faster while quick 42 engine tokens with \\u0645\\u0631\\u062d\\u0628\\u0627 C++ scanner\"}, {\"role\": \"user\", \"content\": \"that hand every exactly \\u6771\\u4eac $1,234.56 tokens {braces} body we'll engine quick once \\u6771\\u4eac on F# v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 I'M over over scanner tiktoken's tokens na\\u00efve that piece tiktoken's once and scanned scanned request with it jumps counting {braces} scanner 100% WON'T scanned exactly is piece 'single' request \\u6771\\u4eac regex for quick written 'single' every F# we'll way na\\u00efve and we'll tokens v1.2.3 regex piece v1.2.3 scanned is with counting with dog gateway admission 100% lazy because that \\ud83d\\ude42 3.14159 F# way hand exactly $1,234.56 while user@example.com It's jumps faster reservation It's don't it mirrors scanned $1,234.56 it while keep $1,234.56 fox $1,234.56 backtracking involved admission 3.14159 mirrors while with on over mirrors C++ for there's fox budget a keep\"}, {\"role\": \"assistant\", \"content\": \"100% scanner user@example.com backtracking every na\\u00efve quick hand jumps body tokens for v1.2.3 exactly \\u0645\\u0631\\u062d\\u0628\\u0627 that \\\"quotes\\\" \\ud83d\\ude42 gateway no caf\\u00e9 that written https://example.com/a/b?c=d the engine piece They'RE budget https://example.com/a/b?c=d involved the https://example.com/a/b?c=d the for involved once exactly every written exactly while 3.14159 user@example.com backtracking request fox is engine way budget counting 1999 piece so 3.14159 tokens F# involved brown a \\u6771\\u4eac scanned keep v1.2.3 for tiktoken's 1999 They'RE and on backtracking counting boundaries na\\u00efve regex\"}, {\"role\": \"user\", \"content\": \"fox is regex https://example.com/a/b?c=d is node.js mirrors user@example.com 100% involved request piece with \\u6771\\u4eac admission the on quick way \\u6771\\u4eac backtracking keep tiktoken's because F# lazy 100% WON'T tiktoken's Z\\u00fcrich involved quick reservation [brackets] written don't {braces} na\\u00efve caf\\u00e9 a $1,234.56 {braces} regex hand 100% 100% so reservation is there's engine \\u0645\\u0631\\u062d\\u0628\\u0627 is 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking v1.2.3 we'll so quick is before\"}, {\"role\": \"assistant\", \"content\": \"[brackets] request lazy C++ 100% scanned F# a because budget They'RE 1999 that is so \\u0645\\u0631\\u062d\\u0628\\u0627 I'M hand involved 3.14159 admission node.js faster once admission tiktoken's jumps is hand quick tokens mirrors lazy It's 42 fox gateway because request once $1,234.56 mirrors there's so backtracking a over engine we'll \\ud83d\\ude42 body engine fox {braces} {braces} counting hand we'll (parens) that 'single' \\\"quotes\\\" lazy budget na\\u00efve lazy WON'T body v1.2.3 the (parens) \"}, {\"role\": \"user\", \"content\": \"for before quick v1.2.3 scanner tokens faster is hand gateway lazy backtracking C++ budget \\u6771\\u4eac brown (parens) They'RE \\u6771\\u4eac It's \\ud83d\\ude42 3.14159 tokens na\\u00efve backtracking (parens) don't WON'T request WON'T that \\u6771\\u4eac piece v1.2.3 mirrors engine the over mirrors na\\u00efve gateway Z\\u00fcrich C++ written 3.14159 dog involved written \\\"quotes\\\" keep \\\"quotes\\\" They'RE fox written 'single' for backtracking gateway written 1999 the a for F# we'll F# brown faster once engine https://example.com/a/b?c=d it quick piece {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors [brackets] boundaries $1,234.56 {braces} request keep piece involved body quick once for for They'RE that involved scanned on for 100% mirrors involved budget 3.14159 They'RE faster with \\u6771\\u4eac backtracking reservation the 1999 for \\u0645\\u0631\\u062d\\u0628\\u0627 \\u0645\\u0631\\u062d\\u0628\\u0627 42 \\ud83d\\ude42 keep no caf\\u00e9 engine\"}, {\"role\": \"assistant\", \"content\": \"that tokens dog exactly piece Z\\u00fcrich quick the request 100% every it user@example.com that https://example.com/a/b?c=d way the reservation 1999 F# while a that They'RE backtracking It's Z\\u00fcrich admission budget C++ They'RE we'll keep counting \\ud83d\\ude42 \\\"quotes\\\" that \\ud83d\\ude42 lazy the engine They'RE \\u6771\\u4eac quick involved while no jumps the https://example.com/a/b?c=d is that 3.14159 100% 42 the no while tiktoken's over while written I'M backtracking node.js tokens F# $1,234.56 caf\\u00e9 it while keep (parens) boundaries admission so over no that fox involved F# brown na\\u00efve $1,234.56 reservation lazy body mirrors every $1,234.56 v1.2.3 that scanned user@example.com before quick body with [brackets] brown tiktoken's v1.2.3 fox budget \\\"quotes\\\" 42 every tokens budget don't counting counting no tiktoken's request C++ It's I'M and \\\"quotes\\\" tiktoken's every (parens) user@example.com the the we'll no I'M boundaries F# over on over tokens\"}, {\"role\": \"user\", \"content\": \"tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 It's counting piece $1,234.56 42 once counting there's faster [brackets] 100% piece counting quick \\ud83d\\ude42 caf\\u00e9 exactly written node.js is way on lazy counting it 42 scanned lazy It's v1.2.3 dog dog and because tiktoken's lazy \\ud83d\\ude42 don't involved and once 3.14159 1999 reservation boundaries caf\\u00e9 written written reservation we'll regex {braces} it the https://example.com/a/b?c=d lazy a once body na\\u00efve 1999 tiktoken's caf\\u00e9 caf\\u00e9 for body WON'T \\ud83d\\ude42 no because regex lazy 3.14159 with dog request scanned and involved involved scanned keep that fox https://example.com/a/b?c=d on we'll we'll budget boundaries exactly exactly admission lazy backtracking that F# while fox 1999 boundaries request \\u6771\\u4eac mirrors brown brown 100% Z\\u00fcrich {braces} They'RE 3.14159 v1.2.3 there's v1.2.3 once 1999 faster and WON'T there's we'll on written C++ v1.2.3\"}, {\"role\": \"assistant\", \"content\": \"user@example.com \\u6771\\u4eac because before so keep written for WON'T the They'RE is hand \\ud83d\\ude42 tiktoken's on 42 F# we'll written body user@example.com \\u6771\\u4eac 1999 admission with reservation 42 and that before 3.14159 piece request exactly 1999 for node.js gateway 100% no hand hand it and admission body scanner https://example.com/a/b?c=d involved user@example.com dog boundaries on while 42 body written don't once keep that node.js on reservation \\u0645\\u0631\\u062d\\u0628\\u0627 lazy \\\"quotes\\\" They'RE that with regex while Z\\u00fcrich na\\u00efve that lazy caf\\u00e9 don't jumps before {braces} don't for https://example.com/a/b?c=d keep user@example.com once lazy because counting brown written that 'single' F# \\u6771\\u4eac regex tiktoken's admission I'M a that It's exactly while with so hand don't we'll on\"}, {\"role\": \"user\", \"content\": \"{braces} (parens) fox on over tiktoken's budget lazy for \\\"quotes\\\" 100% so because 100% boundaries Z\\u00fcrich gateway boundaries don't jumps faster don't F# the \\\"quotes\\\" {braces} tokens backtracking They'RE regex every tokens the \\\"quotes\\\" once user@example.com {braces} every \\\"quotes\\\" counting budget It's gateway there's exactly regex is node.js request for F# piece so don't quick 'single' is user@example.com once I'M because user@example.com before user@example.com it v1.2.3 piece a lazy scanned 100% lazy counting dog and we'll we'll for tiktoken's exactly no fox once 1999 \\u6771\\u4eac we'll every the because regex $1,234.56 100% scanner so \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 on while that na\\u00efve body brown way 3.14159 scanner scanner on They'RE don't over because the 42 involved node.js that a They'RE caf\\u00e9 for \\ud83d\\ude42 user@example.com user@example.com I'M so faster\"}, {\"role\": \"assistant\", \"content\": \"a don't It's gateway fox for \\u6771\\u4eac exactly that exactly I'M piece https://example.com/a/b?c=d \\ud83d\\ude42 faster so once jumps reservation piece boundaries v1.2.3 scanned a while that user@example.com [brackets] keep no there's brown the 1999 backtracking They'RE brown node.js v1.2.3 tiktoken's with is v1.2.3 that hand so I'M and engine boundaries C++ user@example.com tokens keep na\\u00efve {braces} gateway because backtracking [brackets] gateway don't we'll for we'll 100% boundaries counting 3.14159 is the F# it fox node.js for is tiktoken's no backtracking engine and 'single' and involved before fox tokens It's https://example.com/a/b?c=d mirrors written boundaries (parens) reservation the tiktoken's (parens) fox hand body (parens) (parens) it is It's and way because {braces} dog tiktoken's the counting while \\ud83d\\ude42 42 scanned over the\"}, {\"role\": \"user\", \"content\": \"it brown brown tokens backtracking jumps over boundaries with quick counting faster gateway {braces} \\u6771\\u4eac 42 mirrors \\ud83d\\ude42 engine hand \\u6771\\u4eac reservation v1.2.3 quick no no that involved over the scanner faster every mirrors for admission I'M we'll and scanned node.js tokens brown budget 42 written gateway lazy 3.14159 engine it we'll it 3.14159 $1,234.56 the that it (parens) over \\u0645\\u0631\\u062d\\u0628\\u0627 node.js for that {braces} \\\"quotes\\\" way before that no quick tokens and scanner the WON'T boundaries dog C++ boundaries while F# once backtracking 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 a while counting node.js Z\\u00fcrich engine fox faster it F# 1999 gateway written It's 'single' na\\u00efve there's 3.14159 na\\u00efve Z\\u00fcrich brown 3.14159 \\u6771\\u4eac user@example.com WON'T exactly Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 node.js 100% gateway regex while user@example.com and no quick lazy with\"}, {\"role\": \"assistant\", \"content\": \"1999 100% dog so for mirrors tiktoken's backtracking on that fox I'M every for involved involved \\\"quotes\\\" 100% so while 42 and it for F# node.js that gateway backtracking $1,234.56 fox boundaries on budget backtracking mirrors body Z\\u00fcrich for on 42 hand 1999 counting way body brown with scanned 42 [brackets] a jumps I'M written there's no They'RE faster backtracking I'M the don't once admission scanner 'single' \\ud83d\\ude42 way on\"}, {\"role\": \"user\", \"content\": \"every regex \\ud83d\\ude42 3.14159 we'll fox it tiktoken's Z\\u00fcrich \\\"quotes\\\" user@example.com na\\u00efve quick \\u0645\\u0631\\u062d\\u0628\\u0627 fox fox it C++ fox \\u0645\\u0631\\u062d\\u0628\\u0627 body gateway 3.14159 100% fox there's keep They'RE so reservation it lazy with lazy gateway request keep scanned hand every fox budget for quick $1,234.56 piece piece faster hand we'll 42 user@example.com written 1999 \\u6771\\u4eac every na\\u00efve request It's involved caf\\u00e9 [brackets] faster scanned keep 42 It's \\u6771\\u4eac every way is 3.14159 written mirrors brown WON'T once WON'T exactly the They'RE there's scanner because [brackets] gateway v1.2.3 fox tiktoken's 'single' Z\\u00fcrich {braces} jumps caf\\u00e9 I'M https://example.com/a/b?c=d C++ boundaries \\\"quotes\\\" with written we'll quick a with the every way involved scanned so scanner request scanner $1,234.56 exactly the caf\\u00e9 so there's is lazy that jumps user@example.com so with don't faster 1999 request admission They'RE while boundaries reservation\"}, {\"role\": \"assistant\", \"content\": \"jumps no that dog \\u0645\\u0631\\u062d\\u0628\\u0627 the lazy request user@example.com dog with lazy Z\\u00fcrich and tokens \\u6771\\u4eac boundaries engine keep boundaries 'single' so C++ \\ud83d\\ude42 gateway \\ud83d\\ude42 mirrors F# on {braces} once the before body user@example.com boundaries na\\u00efve and keep counting \\\"quotes\\\" while hand request body tiktoken's request we'll so I'M \\\"quotes\\\" on {braces} https://example.com/a/b?c=d They'RE is regex don't dog They'RE (parens) once caf\\u00e9 involved written faster keep before scanner while They'RE boundaries admission once gateway reservation every caf\\u00e9 dog tokens regex 'single' 3.14159 there's counting that tokens dog https://example.com/a/b?c=d piece F# before engine scanner exactly \\\"quotes\\\" don't while\"}, {\"role\": \"user\", \"content\": \"WON'T node.js v1.2.3 involved mirrors on brown keep every tiktoken's F# 100% 'single' They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 na\\u00efve don't 3.14159 \\\"quotes\\\" there's once I'M $1,234.56 dog tokens that the Z\\u00fcrich \\\"quotes\\\" exactly C++ 3.14159 (parens) v1.2.3 no It's no keep user@example.com 42 Z\\u00fcrich a over written tiktoken's keep tiktoken's written with no WON'T $1,234.56 scanner keep keep that fox regex admission every [brackets] WON'T \\u6771\\u4eac 42 engine (parens) C++ we'll tokens a it while [brackets] counting involved the \\u0645\\u0631\\u062d\\u0628\\u0627 lazy 'single' WON'T 3.14159 1999 $1,234.56 regex that 3.14159 because caf\\u00e9 mirrors \\\"quotes\\\" counting 3.14159 with exactly admission every They'RE because 100% body piece na\\u00efve regex counting I'M that admission lazy \\ud83d\\ude42 scanner quick fox we'll brown admission C++\"}, {\"role\": \"assistant\", \"content\": \"\\ud83d\\ude42 the that \\u6771\\u4eac exactly while engine counting \\ud83d\\ude42 42 admission admission once caf\\u00e9 for tokens v1.2.3 jumps piece piece scanner caf\\u00e9 is \\\"quotes\\\" boundaries so jumps hand so while exactly involved written lazy (parens) quick F# with 1999 C++ jumps Z\\u00fcrich faster because quick the scanned C++ because because F# for no there's keep caf\\u00e9 jumps while is tokens quick lazy C++ boundaries 'single' exactly once tiktoken's node.js Z\\u00fcrich https://example.com/a/b?c=d before exactly {braces} boundaries written no caf\\u00e9 100% dog regex WON'T counting dog jumps F# for tokens no budget scanner that {braces} user@example.com C++ scanned \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 for every way and 'single' mirrors\"}, {\"role\": \"user\", \"content\": \"gateway because tokens jumps \\u6771\\u4eac exactly 3.14159 I'M 'single' It's WON'T body 42 and way is tiktoken's dog that admission on (parens) \\u6771\\u4eac C++ node.js over boundaries piece piece backtracking node.js dog hand on there's C++ 'single' so mirrors (parens) that (parens) the 1999 scanner hand node.js keep (parens) {braces} every [brackets] user@example.com that They'RE over tokens is with every so is quick request piece so reservation 42 brown on \\u6771\\u4eac fox They'RE $1,234.56 They'RE na\\u00efve boundaries jumps 42 quick exactly \\u0645\\u0631\\u062d\\u0628\\u0627 the 3.14159 over na\\u00efve is reservation mirrors it tokens no \\u6771\\u4eac on \\u6771\\u4eac mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 dog for engine before before node.js $1,234.56 100% every no 100% a budget backtracking jumps there's faster there's boundaries tokens every tiktoken's way F# $1,234.56 that every exactly piece\"}, {\"role\": \"assistant\", \"content\": \"over 3.14159 body 100% admission is F# tiktoken's user@example.com once with \\u0645\\u0631\\u062d\\u0628\\u0627 no with quick \\\"quotes\\\" the and $1,234.56 with faster exactly is involved keep exactly before 'single' there's body regex for involved counting https://example.com/a/b?c=d every over user@example.com C++ there's there's They'RE I'M scanned https://example.com/a/b?c=d scanned hand \\ud83d\\ude42 every counting don't fox 'single' WON'T na\\u00efve 100% gateway 'single' I'M no once involved node.js 1999 every admission caf\\u00e9 regex with regex written mirrors $1,234.56 tokens while backtracking we'll on it dog na\\u00efve that I'M every [brackets] a tokens I'M engine \\u0645\\u0631\\u062d\\u0628\\u0627 F# body don't request admission mirrors WON'T keep Z\\u00fcrich engine $1,234.56 request $1,234.56\"}, {\"role\": \"user\", \"content\": \"It's way body mirrors involved written 100% WON'T [brackets] no boundaries piece brown and while over before is there's we'll jumps scanned so \\ud83d\\ude42 backtracking keep don't dog and 42 that \\u0645\\u0631\\u062d\\u0628\\u0627 over and before and \\u6771\\u4eac on for (parens) https://example.com/a/b?c=d the request user@example.com I'M \\u0645\\u0631\\u062d\\u0628\\u0627 [brackets] 3.14159 so Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 involved the hand once there's caf\\u00e9 jumps is body gateway v1.2.3 budget They'RE that it I'M 'single' gateway scanner that brown brown They'RE {braces} caf\\u00e9 hand admission budget scanner $1,234.56 mirrors hand 1999 scanned 3.14159 a It's\"}, {\"role\": \"assistant\", \"content\": \"regex Z\\u00fcrich https://example.com/a/b?c=d lazy engine keep regex admission with before body \\\"quotes\\\" boundaries once tiktoken's gateway 'single' and the way I'M $1,234.56 backtracking the engine dog Z\\u00fcrich 3.14159 Z\\u00fcrich user@example.com fox that keep while over once {braces} \\ud83d\\ude42 admission tokens regex tokens budget no I'M that on caf\\u00e9 faster I'M dog 'single' dog we'll with quick no because tiktoken's that counting tiktoken's we'll involved node.js involved with C++ body (parens) dog over before v1.2.3 100% na\\u00efve while request F# every jumps gateway over a that for it body exactly Z\\u00fcrich \\u6771\\u4eac a mirrors caf\\u00e9 I'M that Z\\u00fcrich brown that https://example.com/a/b?c=d no 'single' boundaries \\u0645\\u0631\\u062d\\u0628\\u0627 we'll They'RE exactly request once lazy It's 3.14159 before \\\"quotes\\\" \\u6771\\u4eac scanned scanned It's jumps C++ They'RE C++ and\"}, {\"role\": \"user\", \"content\": \"the fox They'RE budget faster with brown 3.14159 because backtracking \\ud83d\\ude42 that because reservation the so {braces} is WON'T request 3.14159 100% over a \\u0645\\u0631\\u062d\\u0628\\u0627 brown while admission because there's and once tokens [brackets] is no with a over there's over written scanned don't exactly I'M gateway is boundaries it tokens faster exactly tokens 42 lazy node.js https://example.com/a/b?c=d is engine with\"}, {\"role\": \"assistant\", \"content\": \"3.14159 I'M gateway scanner tokens C++ node.js 1999 engine for jumps user@example.com It's request (parens) so [brackets] \\ud83d\\ude42 that it admission 3.14159 budget WON'T caf\\u00e9 that involved budget v1.2.3 [brackets] node.js They'RE mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 request with v1.2.3 WON'T we'll admission brown don't 1999 \\u6771\\u4eac \\u6771\\u4eac F# for the scanner is 'single' is it piece tokens body body Z\\u00fcrich tiktoken's we'll involved \\u6771\\u4eac on no tokens on {braces} It's mirrors I'M [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 lazy Z\\u00fcrich a while caf\\u00e9 involved request $1,234.56 tiktoken's once lazy\"}, {\"role\": \"user\", \"content\": \"3.14159 lazy piece fox on lazy v1.2.3 dog while piece 'single' involved written 100% jumps hand scanner v1.2.3 don't 'single' scanned with piece don't They'RE piece (parens) for with F# scanned don't no backtracking written 100% 100% lazy backtracking involved user@example.com scanner way we'll 42 'single' reservation 'single' lazy the written don't 'single' every and before reservation so it Z\\u00fcrich before a node.js before F# involved jumps counting involved that scanned regex so that user@example.com it with \\u0645\\u0631\\u062d\\u0628\\u0627 involved admission caf\\u00e9 the reservation [brackets] over mirrors that hand {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 don't that we'll [brackets] hand faster I'M regex boundaries piece keep reservation WON'T hand backtracking counting piece $1,234.56 \\\"quotes\\\" Z\\u00fcrich tokens jumps it body before don't scanner {braces} body It's 42 caf\\u00e9 $1,234.56 scanner tokens\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 engine involved faster involved keep with $1,234.56 while piece F# Z\\u00fcrich 100% tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 is budget that tiktoken's admission scanned node.js 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 that on on body dog with regex admission 3.14159 piece na\\u00efve so 'single' mirrors boundaries boundaries keep on no while \\u0645\\u0631\\u062d\\u0628\\u0627 reservation the node.js F# 42 piece reservation It's tiktoken's reservation $1,234.56 hand tiktoken's that I'M it \\ud83d\\ude42 (parens) counting Z\\u00fcrich is [brackets] https://example.com/a/b?c=d reservation boundaries lazy (parens) that with over Z\\u00fcrich don't counting\"}, {\"role\": \"user\", \"content\": \"body dog backtracking that we'll don't \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com keep quick before scanned caf\\u00e9 It's body quick na\\u00efve 100% before quick jumps \\\"quotes\\\" once regex 42 I'M \\ud83d\\ude42 and keep a faster brown caf\\u00e9 regex \\\"quotes\\\" we'll engine exactly so 42 written They'RE every caf\\u00e9 the the once na\\u00efve piece no hand that boundaries scanned before there's there's reservation https://example.com/a/b?c=d admission I'M v1.2.3 dog They'RE dog scanned engine It's user@example.com no reservation \\u0645\\u0631\\u062d\\u0628\\u0627 regex scanned Z\\u00fcrich is it a and \\\"quotes\\\" involved exactly exactly way hand jumps \\ud83d\\ude42 reservation quick tokens mirrors over user@example.com boundaries 3.14159 I'M node.js piece F# https://example.com/a/b?c=d before tiktoken's I'M piece brown is na\\u00efve and {braces} WON'T It's don't it is {braces} on so so \\u6771\\u4eac it it user@example.com brown \\\"quotes\\\" I'M 100% scanned C++ body tokens \\u6771\\u4eac [brackets]\"}, {\"role\": \"assistant\", \"content\": \"mirrors F# $1,234.56 for dog (parens) keep that [brackets] (parens) 3.14159 v1.2.3 WON'T is don't the a It's https://example.com/a/b?c=d faster 'single' it user@example.com is dog dog counting so $1,234.56 v1.2.3 boundaries every on regex backtracking before quick brown quick C++ quick v1.2.3 scanner written \\ud83d\\ude42 user@example.com before is 42 3.14159 on engine dog dog once request https://example.com/a/b?c=d lazy on caf\\u00e9 faster don't before is exactly written brown C++ piece a 3.14159 no WON'T faster it https://example.com/a/b?c=d and boundaries \\ud83d\\ude42 reservation that engine we'll fox while the dog v1.2.3 because boundaries scanned It's \\u6771\\u4eac \\\"quotes\\\" scanned that so is tiktoken's scanned {braces} gateway exactly regex quick over is fox It's we'll {braces} faster\"}, {\"role\": \"user\", \"content\": \"Summarise the conversation so far in three sentences.\"}]}", "input_tokens": 50354} diff --git a/litellm-rust/crates/token-counter/tests/fixtures/cl100k/texts.jsonl b/litellm-rust/crates/token-counter/tests/fixtures/cl100k/texts.jsonl new file mode 100644 index 00000000000..3dd681d43f9 --- /dev/null +++ b/litellm-rust/crates/token-counter/tests/fixtures/cl100k/texts.jsonl @@ -0,0 +1,4053 @@ +{"text": "", "tokens": 0, "pieces": []} +{"text": "Hello, how are you today?", "tokens": 7, "pieces": ["Hello", ",", " how", " are", " you", " today", "?"]} +{"text": "I'm sure they're right, we'll see. WE'LL SEE, I'M SURE THEY'RE RIGHT, IT'S HERS AND IT'D BE 'D", "tokens": 34, "pieces": ["I", "'m", " sure", " they", "'re", " right", ",", " we", "'ll", " see", ".", " WE", "'LL", " SEE", ",", " I", "'M", " SURE", " THEY", "'RE", " RIGHT", ",", " IT", "'S", " HERS", " AND", " IT", "'D", " BE", " '", "D"]} +{"text": "don't Don'T DON'T won'T i've I'VE i'Ve you'RE 'S 'T 'M 'D 'LL 'VE 'RE 'ſ 'x", "tokens": 37, "pieces": ["don", "'t", " Don", "'T", " DON", "'T", " won", "'T", " i", "'ve", " I", "'VE", " i", "'Ve", " you", "'RE", " '", "S", " '", "T", " '", "M", " '", "D", " '", "LL", " '", "VE", " '", "RE", " '", "ſ", " '", "x"]} +{"text": "1234567890 123 12 1 0000000 ٣٤٥٦٧٨ ३४५६ 1,234,567.89 2026-09-11T18:00:00Z", "tokens": 58, "pieces": ["123", "456", "789", "0", " ", "123", " ", "12", " ", "1", " ", "000", "000", "0", " ", "٣٤٥", "٦٧٨", " ", "३४५", "६", " ", "1", ",", "234", ",", "567", ".", "89", " ", "202", "6", "-", "09", "-", "11", "T", "18", ":", "00", ":", "00", "Z"]} +{"text": "$abc %def &ghi @jkl _mno #pqr ~stu ^vwx |yz \\a /b :c ;d ?e !f (g )h [i ]j {k }l n =o +p *q", "tokens": 56, "pieces": ["$abc", " %", "def", " &", "ghi", " @", "jkl", " _", "mno", " #", "pqr", " ~", "stu", " ^", "vwx", " |", "yz", " \\", "a", " /", "b", " :", "c", " ;", "d", " ?", "e", " !", "f", " (", "g", " )", "h", " [", "i", " ]", "j", " {", "k", " }", "l", " <", "m", " >", "n", " =", "o", " +", "p", " *", "q"]} +{"text": "foo bar baz \t qux\t\tquux \n\nline\r\nline\r\n\r\n \n\t\r\n x ", "tokens": 22, "pieces": ["foo", " ", " bar", " ", " baz", " \t", " qux", "\t", "\tquux", " \n\n", "line", "\r\n", "line", "\r\n\r\n \n\t\r\n", " ", " x", " "]} +{"text": "trailing spaces ", "tokens": 4, "pieces": ["trailing", " spaces", " "]} +{"text": "trailing tabs\t\t", "tokens": 4, "pieces": ["trailing", " tabs", "\t\t"]} +{"text": "trailing newline\n", "tokens": 4, "pieces": ["trailing", " newline", "\n"]} +{"text": "\n\n\n", "tokens": 1, "pieces": ["\n\n\n"]} +{"text": "\r\n\r\n\r\n", "tokens": 1, "pieces": ["\r\n\r\n\r\n"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "😀😃😄 👍🏽 🇺🇸 👨‍👩‍👧‍👦 ✈️ ❤️‍🔥 ٭ ※ ⌘ ⏎", "tokens": 54, "pieces": ["😀😃😄", " 👍🏽", " 🇺🇸", " 👨‍👩‍👧‍👦", " ✈️", " ❤️‍🔥", " ٭", " ※", " ⌘", " ⏎"]} +{"text": "漢字かな交じり文、東京都千代田区。日本語のテキストです。中文测试。한국어 텍스트", "tokens": 42, "pieces": ["漢字かな交じり文", "、東京都千代田区", "。日本語のテキストです", "。中文测试", "。한국어", " 텍스트"]} +{"text": "مرحبا بالعالم، هذا نص عربي مع أرقام ١٢٣٤٥٦٧ و علامات ترقيم!", "tokens": 52, "pieces": ["مرحبا", " بالعالم", "،", " هذا", " نص", " عربي", " مع", " أرقام", " ", "١٢٣", "٤٥٦", "٧", " و", " علامات", " ترقيم", "!"]} +{"text": "Zürich, façade, naïve, Ærøskøbing, Ελληνικά, Русский текст, עברית, हिन्दी, ไทย", "tokens": 52, "pieces": ["Zürich", ",", " façade", ",", " naïve", ",", " Ærøskøbing", ",", " Ελληνικά", ",", " Русский", " текст", ",", " עברית", ",", " ह", "िन", "्द", "ी,", " ไทย"]} +{"text": "é å ḍ̇ ́́ combining̈ markś!", "tokens": 19, "pieces": ["e", "́", " a", "̊", " ḋ", "̣", " ́́", " combining", "̈", " marks", "́!"]} +{"text": "ΣΊΣΥΦΟΣ Džungla İstanbul file flow Ａｂｃ ㍿ ㋿ ꟲ 𐞁", "tokens": 50, "pieces": ["ΣΊΣΥΦΟΣ", " Džungla", " İstanbul", " file", " flow", " Ａｂｃ", " ㍿", " ㋿", " ꟲ", " 𐞁"]} +{"text": "<|endoftext|> <|fim_prefix|>code<|fim_middle|>more<|fim_suffix|> <|endofprompt|> <|im_start|>", "tokens": 40, "pieces": ["<|", "endoftext", "|>", " <|", "fim", "_prefix", "|>", "code", "<|", "fim", "_middle", "|>", "more", "<|", "fim", "_suffix", "|>", " <|", "endofprompt", "|>", " <|", "im", "_start", "|>"]} +{"text": " [INST] [/INST] <>", "tokens": 22, "pieces": ["", " <", "META", "_START", ">", " <", "s", ">", " ", " [", "INST", "]", " [/", "INST", "]", " <<", "SYS", ">>"]} +{"text": "def f(x):\n return {'a': x ** 2, \"b\": [1, 2, 3]} # comment\n\nprint(f(10))\n", "tokens": 35, "pieces": ["def", " f", "(x", "):\n", " ", " return", " {'", "a", "':", " x", " **", " ", "2", ",", " \"", "b", "\":", " [", "1", ",", " ", "2", ",", " ", "3", "]}", " ", " #", " comment", "\n\n", "print", "(f", "(", "10", "))\n"]} +{"text": "{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\\n\"}],\"temperature\":0.7}", "tokens": 26, "pieces": ["{\"", "model", "\":\"", "gpt", "-", "4", "\",\"", "messages", "\":[{\"", "role", "\":\"", "user", "\",\"", "content", "\":\"", "hi", "\\n", "\"}],\"", "temperature", "\":", "0", ".", "7", "}"]} +{"text": "https://example.com/path?query=1&other=two#fragment user@example.com 192.168.0.1", "tokens": 26, "pieces": ["https", "://", "example", ".com", "/path", "?query", "=", "1", "&other", "=two", "#fragment", " user", "@example", ".com", " ", "192", ".", "168", ".", "0", ".", "1"]} +{"text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "tokens": 375, "pieces": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]} +{"text": " ", "tokens": 24, "pieces": [" "]} +{"text": "........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................", "tokens": 48, "pieces": ["........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................"]} +{"text": "abababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab", "tokens": 1500, "pieces": ["abababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab"]} +{"text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", "tokens": 95, "pieces": ["\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"]} +{"text": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "tokens": 1000, "pieces": ["000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000"]} +{"text": "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", "tokens": 375, "pieces": ["!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"]} +{"text": "😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀", "tokens": 2000, "pieces": ["😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀"]} +{"text": "漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢", "tokens": 2000, "pieces": ["漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢"]} +{"text": " abc ! 
x 　y ​z ‍‍ q", "tokens": 18, "pieces": [" abc", " ", "!", " ", "
x", " ", "　y", " ​", "z", " ‍‍", " q"]} +{"text": "x…y \u000b\f z", "tokens": 8, "pieces": ["x", "…y", " \u000b\f", " z"]} +{"text": "\u0000\u0001\u0002  �", "tokens": 6, "pieces": ["\u0000\u0001\u0002", " ", " �"]} +{"text": "tab\tseparated\tvalues\n1\t2\t3\n", "tokens": 11, "pieces": ["tab", "\tseparated", "\tvalues", "\n", "1", "\t", "2", "\t", "3", "\n"]} +{"text": "MiXeD cAsE wOrDs AND ACRONYMS like NASA, HTTP/2, gRPC, iOS, macOS", "tokens": 28, "pieces": ["MiXeD", " cAsE", " wOrDs", " AND", " ACRONYMS", " like", " NASA", ",", " HTTP", "/", "2", ",", " gRPC", ",", " iOS", ",", " macOS"]} +{"text": "snake_case_identifier camelCaseIdentifier PascalCaseIdentifier SCREAMING_SNAKE_CASE kebab-case", "tokens": 19, "pieces": ["snake", "_case", "_identifier", " camelCaseIdentifier", " PascalCaseIdentifier", " SCREAMING", "_SNAKE", "_CASE", " kebab", "-case"]} +{"text": "x'sy x'ty x'rey x'vey x'my x'lly x'dy x'S x'T x'RE x'VE x'M x'LL x'D x'sS x'llL", "tokens": 43, "pieces": ["x", "'s", "y", " x", "'t", "y", " x", "'re", "y", " x", "'ve", "y", " x", "'m", "y", " x", "'ll", "y", " x", "'d", "y", " x", "'S", " x", "'T", " x", "'RE", " x", "'VE", " x", "'M", " x", "'LL", " x", "'D", " x", "'s", "S", " x", "'ll", "L"]} +{"text": "IT'SOK it'Dbe x'Sy x'Ty x'My x'Dy x'LLy x'VEy x'REy x'Ly x'Vy x'Ry 'Sx'Tx'Mx'LLx'VEx'REx'Dx", "tokens": 55, "pieces": ["IT", "'S", "OK", " it", "'D", "be", " x", "'S", "y", " x", "'T", "y", " x", "'M", "y", " x", "'D", "y", " x", "'LL", "y", " x", "'VE", "y", " x", "'RE", "y", " x", "'Ly", " x", "'Vy", " x", "'Ry", " '", "Sx", "'T", "x", "'M", "x", "'LL", "x", "'VE", "x", "'RE", "x", "'D", "x"]} +{"text": "'s't're've'm'll'd 'S'T'RE'VE'M'LL'D ''s '''s", "tokens": 21, "pieces": ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d", " '", "S", "'T", "'RE", "'VE", "'M", "'LL", "'D", " ''", "s", " '''", "s"]} +{"text": "9'9 9's a'9 '9 ' 's' ' 's", "tokens": 18, "pieces": ["9", "'", "9", " ", "9", "'s", " a", "'", "9", " '", "9", " '", " '", "s", "'", " '", " '", "s"]} +{"text": "١٢٣٤ ½⅓¼ ⅣⅤ 𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡 ①②③", "tokens": 56, "pieces": ["١٢٣", "٤", " ", "½⅓¼", " ", "ⅣⅤ", " ", "𝟘𝟙𝟚", "𝟛𝟜𝟝", "𝟞𝟟𝟠", "𝟡", " ", "①②③"]} +{"text": "camelCase PascalCase ABCdef ABCdeF ABC aB Ab ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABC", "tokens": 21, "pieces": ["camelCase", " PascalCase", " ABCdef", " ABCdeF", " ABC", " aB", " Ab", " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABC"]} +{"text": "日本ABC ABC日本 日本語abc abc日本語 漢字Kanji kanji漢字 KANJI漢字kanji مرحباABC ABCمرحبا abcمرحبا", "tokens": 53, "pieces": ["日本ABC", " ABC日本", " 日本語abc", " abc日本語", " 漢字Kanji", " kanji漢字", " KANJI漢字kanji", " مرحباABC", " ABCمرحبا", " abcمرحبا"]} +{"text": "́ABC ́abc ́́A Á́ ÉA aÉ !!́a  ́A ẍY Ẍy", "tokens": 33, "pieces": ["́ABC", " ́", "abc", " ́́", "A", " A", "́́", " E", "́A", " aE", "́", " !!́", "a", " ", " ", "́A", " x", "̈Y", " X", "̈y"]} +{"text": "ᵃbc ᵃBC Aᵃbc Aᵃ ᵃ' ᵃ's Džungla aDžB ADžB ADžb DžDž Ljx İi ΣΊΣΥΦΟΣσ ΣσΣ", "tokens": 73, "pieces": ["ᵃbc", " ᵃBC", " Aᵃbc", " Aᵃ", " ᵃ", "'", " ᵃ", "'s", " Džungla", " aDžB", " ADžB", " ADžb", " DžDž", " Ljx", " İi", " ΣΊΣΥΦΟΣσ", " ΣσΣ"]} +{"text": "don'tx ABC's abc'S abc'ſ ABC'ſx IT'SOK it'Dbe 'sabc x's 's 'Sx'Tx x’s X'LLx X'Ll", "tokens": 43, "pieces": ["don", "'t", "x", " ABC", "'s", " abc", "'S", " abc", "'ſ", " ABC", "'ſ", "x", " IT", "'S", "OK", " it", "'D", "be", " '", "sabc", " x", "'s", " '", "s", " '", "Sx", "'T", "x", " x", "’s", " X", "'LL", "x", " X", "'Ll"]} +{"text": "!ABC !AbC !!abc #camelCase (ABCdef)  ABC abc Abc \tABC\tabc", "tokens": 27, "pieces": ["!ABC", " !", "AbC", " !!", "abc", " #", "camelCase", " (", "ABCdef", ")", " ", " ABC", " abc", " Abc", " ", "\tABC", "\tabc"]} +{"text": "!!/\n/x a/b !!\n/x /x // path/to/file.rs http://x.y/z?a=b/c \\/\\/ //\r\n//\n", "tokens": 29, "pieces": ["!!/\n", "/x", " a", "/b", " !!\n", "/x", " ", " /", "x", " ", " //", " path", "/to", "/file", ".rs", " http", "://", "x", ".y", "/z", "?a", "=b", "/c", " \\/\\/", " //\r\n", "//\n"]} +{"text": "x \n x \r\n \r\n y x \n a b \n\n c x\t\ty x\t\t end \n \n", "tokens": 22, "pieces": ["x", " \n", " x", " \r\n \r\n", " y", " x", " \n", " ", " a", " ", " b", " \n\n", " ", " c", " x", "\t", "\ty", " x", "\t\t", " end", " \n \n"]} +{"text": "12345 6 1abc abc1 ABC123abc 123ABC ١٢٣٤٥abc", "tokens": 27, "pieces": ["123", "45", " ", "6", " ", "1", "abc", " abc", "1", " ABC", "123", "abc", " ", "123", "ABC", " ", "١٢٣", "٤٥", "abc"]} +{"text": "Ⅳ٣٤٥٦<|endoftext|>9
Dž#$%", "tokens": 24, "pieces": ["Ⅳ٣٤", "٥٦", "<|", "endoftext", "|>", "9", "
Dž", "#$%"]} +{"text": "́!!ſİ'D​a'll 字0'MZſⅣ ḍ̇éfi㍿𐞁<|endoftext|>'reＡ'S#$%", "tokens": 46, "pieces": ["́!!", "ſİ", "'D", "​a", "'ll", " 字", "0", "'M", "Zſ", "Ⅳ", " ḋ", "̣éfi", "㍿𐞁", "<|", "endoftext", "|>'", "reＡ", "'S", "#$%"]} +{"text": "ع字'T\r\ń½sꟲ㋿'VE'S<😀🏽!!12345678 ٣٤٥٦Džſḍ̇\réEOT­'ſ<|endoftext|><|fim_prefix|>ś
\tm", "tokens": 77, "pieces": ["ع字", "'T", "\r\n", "́", "½", "sꟲ", "㋿'", "VE", "'", "S", "<😀🏽!!", "123", "456", "78", " <", "EOT", ">", "٣٤٥", "٦", "Džſḋ", "̣\r", "e", "́EOT", "­'", "ſ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "s", "́", "
", "\tm"]} +{"text": "9'Re\r\n'ſ　 'T'Re \né#$%<½!!٣٤٥٦'ſ<\"عßZt\u000bDž'T<|fim_prefix|>ſ㋿\"éſ", "tokens": 61, "pieces": ["9", "'", "Re", "\r\n", "'ſ", "　 ", " '", "T", "'Re", " \n", "é", "#$%<", "½", "!!", "٣٤٥", "٦", "'ſ", "<<", "META", "_START", ">\"", "عßZt", "\u000bDž", "'T", "<|", "fim", "_prefix", "|>", "ſ", "㋿\"", "e", "́ſ"]} +{"text": "<|endoftext|>12345678#$%tعⅣ'T0'D<|endoftext|>é'M-'ſ'sß12345678ꟲ0(>\r\n'MZ'Sa'M", "tokens": 57, "pieces": ["<|", "endoftext", "|>", "123", "456", "78", "#$%", "tع", "Ⅳ", "'T", "0", "'D", "<|", "endoftext", "|>", "e", "́'", "M", "-<", "EOT", ">'", "ſ", "'s", "ß", "123", "456", "78", "ꟲ", "0", "(>\r\n", "'M", "Z", "'S", "a", "'M"]} +{"text": " \n'll\"‍ d \nß㍿\u000baⅣ😀🏽́ſ\n \n
'Reİ\tDž٣٤٥٦ع'llEOT.\nݽ٣٤٥٦>å\u000b<|fim_prefix|>\"𐞁", "tokens": 70, "pieces": [" \n", "'ll", "\"‍", " ", " d", " \n", "ß", "㍿", "\u000ba", "Ⅳ", "😀🏽́", "ſ", "\n \n", "
", "'Re", "İ", "\tDž", "٣٤٥", "٦", "ع", "'ll", "EOT", ".\n", "İ", "½٣٤", "٥٦", ">a", "̊", "\u000b", "<|", "fim", "_prefix", "|>\"", "𐞁"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "å'remfi㍿s", "tokens": 11, "pieces": ["a", "̊'", "remfi", "㍿s"]} +{"text": "<|endoftext|>", "tokens": 7, "pieces": ["<|", "endoftext", "|>"]} +{"text": "🙂३'M🙂 12345678'VÉ,\u000bİ<|fim_prefix|> Ａ'TDž　's!!", " ", "Ａ", "'T", "Dž", "　", "'s", "!!<", "t", "\"!", "d", " \n", "m", "'M", "('", "ſ"]} +{"text": "\t", "tokens": 1, "pieces": ["\t"]} +{"text": "…<|fim_prefix|>\n.-åß\r\n\r\nEOT'llå½-fi!é'VE12345678EOT字'VE!🙂<|fim_prefix|>'ſ'D \r\r漢0…", "tokens": 61, "pieces": ["…", "<|", "fim", "_prefix", "|>\n", ".-", "a", "̊ß", "\r\n\r\n", "EOT", "'ll", "a", "̊", "½", "-fi", "!e", "́'", "VE", "123", "456", "78", "EOT字", "'VE", "!🙂<|", "fim", "_prefix", "|>'", "ſ", "'D", " \r\r", "漢", "0", "…"]} +{"text": "a'S'T👍🏽> \n-s㍿.㍿#$%́\r\n\r\n", "tokens": 23, "pieces": ["a", "'S", "'T", "👍🏽>", " \n", "-s", "㍿.㍿#$%́\r\n\r\n"]} +{"text": " 漢#$%­…­ع­ß#$%\t0é", "tokens": 18, "pieces": [" 漢", "#$%­", "…", "­ع", "­ß", "#$%", "\t", "0", "e", "́"]} +{"text": ",9'㍿-", "tokens": 10, "pieces": [",", "9", "'㍿-"]} +{"text": "٣٤٥٦Ⅳ漢İ'sع", "tokens": 15, "pieces": ["٣٤٥", "٦Ⅳ", "漢İ", "'s", "ع"]} +{"text": ".éé'sZ>éEOTZ'0​e½ 0#$%́fiⅣ", "tokens": 25, "pieces": [".e", "́é", "'s", "Z", ">éEOTZ", "'", "0", "​e", "½", " ", " ", "0", "#$%́", "fi", "Ⅳ"]} +{"text": "t<Ⅳ…Dž'VE ꟲ٣٤٥٦éåع㍿'s字👍🏽ع ३EOTⅣ😀🏽e \nＡ\"", "tokens": 63, "pieces": ["t", "<", "Ⅳ", "…Dž", "'VE", "", " ", " ꟲ", "٣٤٥", "٦", "éa", "̊ع", "㍿'<", "META", "_START", ">s字", "👍🏽", "ع", " ", " ", "३", "EOT", "Ⅳ", "😀🏽", "e", " \n", "Ａ", "\""]} +{"text": "' . t\t<|fim_prefix|>㍿Dž३s😀🏽\t㍿EOTå𐞁​EOT
\t- \ns \n#$%ḍ̇é\r\n　ee漢", "tokens": 64, "pieces": ["'", " .", " ", " t", "\t", "<|", "fim", "_prefix", "|>㍿", "Dž", "३", "s", "😀🏽", "\t", "㍿EOTa", "̊𐞁", "​EOT", "
", "\t", "-", " \n", "s", " \n", "#$%", "ḋ", "̣é", "\r\n", "　ee漢"]} +{"text": "'ſEOTéⅣ\nİ'S!!!t<|endoftext|>éＡ<|fim_prefix|>Ⅳ'Z‍'Re'
<|endoftext|>\u000b<㋿　#$%漢Ａ\"ꟲ㍿'T'T", "tokens": 72, "pieces": ["'ſ", "EOTe", "́", "Ⅳ", "\n", "İ", "'S", "!!!", "t", "<|", "endoftext", "|>", "e", "́Ａ", "<|", "fim", "_prefix", "|>", "Ⅳ", "'Z", "‍'", "Re", "'", "
", "<|", "endoftext", "|>", "\u000b", "<㋿", "　", "#$%", "漢Ａ", "\"ꟲ", "㍿'", "T", "'T"]} +{"text": "'Re😀🏽½(.>\u000bſa㍿<|fim_prefix|>>t!'ll३ꟲ \né\n0e\r\n\r\n…😀🏽½́dḍ̇𐞁\r\n\r\n<|fim_prefix|>.<|endoftext|>9", "tokens": 73, "pieces": ["'Re", "😀🏽", "½", "(.>", "\u000bſa", "㍿<|", "fim", "_prefix", "|>>", "t", "!'", "ll", "३", "ꟲ", " \n", "e", "́\n", "0", "e", "\r\n\r\n", "…", "😀🏽", "½", "́dḋ", "̣𐞁", "\r\n\r\n", "<|", "fim", "_prefix", "|>.<|", "endoftext", "|>", "9"]} +{"text": "\r\nå👍🏽!!é", "tokens": 13, "pieces": ["\r\n", "a", "̊👍🏽!!", "e", "́"]} +{"text": "dİ(.عع \n字㍿\nå(Z'ſ㍿\r\n\r\n,<|fim_prefix|>", "tokens": 33, "pieces": ["dİ", "(.", "عع", " \n", "字", "㍿\n", "a", "̊(", "Z", "'ſ", "㍿\r\n\r\n", ",<|", "fim", "_prefix", "|>"]} +{"text": "!", "tokens": 1, "pieces": ["!"]} +{"text": " \n\r\n.s <|endoftext|>aꟲ'sꟲ३…\r\n\r\n\u000bDž‍\t9🙂ſ", "tokens": 33, "pieces": [" \n\r\n", ".s", " <|", "endoftext", "|>", "aꟲ", "'s", "ꟲ", "३", "…\r\n\r\n", "\u000bDž", "‍", "\t", "9", "🙂ſ"]} +{"text": "'re𐞁é३ fi", "tokens": 12, "pieces": ["'re", "𐞁e", "́", "३", " ", " fi"]} +{"text": "12345678 #$%<|fim_prefix|>‍㍿'T😀🏽fi'll's'S12345678½é
,🙂٣٤٥٦#$%👍🏽🙂12345678<Ⅳ!\"'VE
", "tokens": 74, "pieces": ["123", "456", "78", " ", " #$%<|", "fim", "_prefix", "|>‍㍿'", "T", "😀🏽", "fi", "'ll", "'s", "'S", "123", "456", "78½", "e", "́", "
", ",🙂<", "META", "_START", ">", "٣٤٥", "٦", "#$%👍🏽🙂", "123", "456", "78", "<", "Ⅳ", "!\"'", "VE", "
"]} +{"text": "fi>İ𐞁…\u000b'D­\tß👍🏽 Ⅳß'Dé\r\n \nß　👍🏽", "tokens": 43, "pieces": ["fi", ">İ𐞁", "…", "\u000b", "'D", "­", "\t", "ß", "👍🏽", " ", " ", "Ⅳ", "ß", "'D", "e", "́\r\n", " \n", "ß", "　", "👍🏽"]} +{"text": "#$% ßع́'T<Ａ012345678 \n<|fim_prefix|> ㍿👍🏽'", "tokens": 79, "pieces": ["👍🏽>", "Ａ", "012", "345", "678", " \n", "<|", "fim", "_prefix", "|>", " ", "㍿👍🏽'"]} +{"text": "ꟲ'll#$%(\r\n\r\nZ0\u000b👍🏽
'VE ,½'ll­EOT", "tokens": 27, "pieces": ["ꟲ", "'ll", "#$%(\r\n\r\n", "Z", "0", "\u000b", "👍🏽", "
", "'VE", " ", ",", "½", "'ll", "­EOT"]} +{"text": "İ#$%\n9sßEOTd-!!<|endoftext|>　'ReＡＡfiⅣ'ſéſ🙂étſ\ne'ſ㋿é'VE\"ꟲ漢", "tokens": 56, "pieces": ["İ", "#$%\n", "9", "sßEOTd", "-!!<|", "endoftext", "|>", "　", "'Re", "ＡＡfi", "Ⅳ", "'ſ", "éſ", "🙂étſ", "\n", "e", "'ſ", "㋿é", "'VE", "\"ꟲ漢"]} +{"text": "<३…😀🏽m>'s<|endoftext|>\r\n\r\n३'ſ'S<|endoftext|><|fim_prefix|>Dž🙂ſⅣＡ㋿-'re#$%!é\r<|fim_prefix|>å9…s'VE", "tokens": 74, "pieces": ["<", "३", "…", "😀🏽", "m", ">'", "s", "<|", "endoftext", "|>\r\n\r\n", "३", "'ſ", "'S", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "Dž", "🙂ſ", "Ⅳ", "Ａ", "㋿-'", "re", "#$%!", "é", "\r", "<|", "fim", "_prefix", "|>", "a", "̊", "9", "…s", "'VE"]} +{"text": " <|endoftext|>\r\t<|endoftext|>…ßſ\n#$%🙂㋿ḍ̇\r\n\r\n", "tokens": 38, "pieces": [" ", "<|", "endoftext", "|>\r", "\t", "<|", "endoftext", "|>", "…ßſ", "\n", "#$%🙂㋿", "ḋ", "̣\r\n\r\n"]} +{"text": " \n(\u000b!!\u000b\r\n\t́'t漢३!\nß \n㍿\t'T,-m-\u000b>ſ \ns३
<|endoftext|>㋿ \n'S'VE>\u000b🙂\r\n", "tokens": 54, "pieces": [" \n", "(", "\u000b", "!!", "\u000b\r\n", "\t", "́'", "t漢", "३", "!\n", "ß", " \n", "㍿", "\t", "'T", ",-", "m", "-", "\u000b", ">ſ", " \n", "s", "३", "
", "<|", "endoftext", "|>㋿", " \n", "'S", "'VE", ">", "\u000b", "🙂\r\n"]} +{"text": "d'Rea'0㍿İé👍🏽s.٣٤٥٦('S\",​
12345678…ꟲ.\r\n\r\n'T㋿ ½'D", "tokens": 59, "pieces": ["d", "'Re", "a", "'", "0", "㍿İé", "👍🏽", "s", ".", "٣٤٥", "٦", "('", "S", "<", "META", "_START", ">\",​", "
", "123", "456", "78", "…ꟲ", ".\r\n\r\n", "<", "EOT", ">'", "T", "㋿", " ", "½", "'D"]} +{"text": "'ſ'S…\" 'll㍿. ! (s!!\r\nß字㋿ḍ̇Z'ſ<|fim_prefix|>'Tß㍿ſ\n9\r\n#$%
㍿'T", "tokens": 65, "pieces": ["'ſ", "'S", "…", "\"", " '", "ll", "㍿.", " !", " ", "(<", "EOT", ">s", "!!\r\n", "ß字", "㋿ḋ", "̣Z", "'ſ", "<|", "fim", "_prefix", "|>'", "Tß", "㍿ſ", "\n", "9", "\r\n", "#$%", "
", "㍿'", "T"]} +{"text": "İm#$%🙂é'll'VE'VEfi \n\r\r0漢عt'llİd's\r'M­9", "tokens": 30, "pieces": ["İm", "#$%🙂", "é", "'ll", "'VE", "'VE", "fi", " \n\r\r", "0", "漢عt", "'ll", "İd", "'s", "\r", "'M", "­", "9"]} +{"text": "́'T'ſ \ne\u000bⅣ\ns9ḍ̇'S,\r\n\r\néed're<|fim_prefix|> 👍🏽\u000b½'Re", "tokens": 41, "pieces": ["́'", "T", "'ſ", " \n", "e", "\u000b", "Ⅳ", "\n", "s", "9", "ḋ", "̣'", "S", ",\r\n\r\n", "éed", "'re", "<|", "fim", "_prefix", "|>", " ", " 👍🏽", "\u000b", "½", "'Re"]} +{"text": "🙂'VE‍<​<|endoftext|>ⅣEOTdſ‍Dž-'D(", "tokens": 29, "pieces": ["🙂'", "VE", "‍<​<|", "endoftext", "|>", "Ⅳ", "EOTdſ", "‍Dž", "-'", "D", "("]} +{"text": "İ'Reİ𐞁'ſ\re", "tokens": 12, "pieces": ["İ", "'Re", "İ𐞁", "'ſ", "\r", "e"]} +{"text": "…åt\r'VE\nع​<|fim_prefix|>Ⅳß😀🏽s㍿<|fim_prefix|>,\r ­ꟲ…İ're!!,'T< 9EOT", "tokens": 60, "pieces": ["…a", "̊t", "\r", "'VE", "\n", "ع", "​<|", "fim", "_prefix", "|>", "Ⅳ", "ß", "😀🏽", "s", "㍿<|", "fim", "_prefix", "|>,\r", " ", " ­", "ꟲ", "…İ", "'re", "!!,'", "T", "<", " ", "9", "EOT"]} +{"text": "a‍\"mé,\rßꟲ'llé,t…#$% 'M!!t㍿'VE<|endoftext|>t('ſ", "tokens": 39, "pieces": ["a", "‍\"", "me", "́,\r", "ßꟲ", "'ll", "é", ",t", "…", "#$%", " '", "M", "!!", "t", "㍿'", "VE", "<|", "endoftext", "|>", "t", "('", "ſ"]} +{"text": "ع'S,", "tokens": 3, "pieces": ["ع", "'S", ","]} +{"text": "漢 a字", "tokens": 5, "pieces": ["漢", " a字"]} +{"text": "d'Reé're \n,!!<|fim_prefix|>😀🏽 \n​12345678m \n㍿　\r\n\r\nḍ̇'VE'S‍tZ>å#$%'S'D!,​,#$%\"٣٤٥٦Ａ<漢,", "tokens": 71, "pieces": ["d", "'Re", "e", "́'", "re", " \n", ",!!<|", "fim", "_prefix", "|>😀🏽", " \n", "​", "123", "456", "78", "m", " \n", "㍿", "　\r\n\r\n", "ḋ", "̣'", "VE", "'S", "‍tZ", ">a", "̊#$%'", "S", "'D", "!,​,#$%\"", "٣٤٥", "٦", "Ａ", "<漢", ","]} +{"text": "'Sefi're\t­<|fim_prefix|>‍'Re\u000b🙂!12345678!! \na𐞁'S12345678EOT­Ａ<|endoftext|>㍿'llİeé", "tokens": 56, "pieces": ["'S", "efi", "'re", "\t", "­<|", "fim", "_prefix", "|>‍'", "Re", "\u000b", "🙂!", "123", "456", "78", "!!", " \n", "a𐞁", "'S", "123", "456", "78", "EOT", "­Ａ", "<|", "endoftext", "|>㍿'", "llİeé"]} +{"text": " 𐞁‍३Dž́­!½\r\n\r\nZsＡ!'T", "tokens": 22, "pieces": [" 𐞁", "‍", "३", "Dž", "́­!", "½", "\r\n\r\n", "ZsＡ", "!'", "T"]} +{"text": "0‍'Re.٣٤٥٦'ſ 's\ta\r½\r\n>ée'Dع\u000b𐞁a'Dİ 0　🙂'D'så漢'D'D३é'M>", "tokens": 56, "pieces": ["0", "‍'", "Re", ".", "٣٤٥", "٦", "'ſ", " '", "s", "\ta", "\r", "½", "\r\n", ">ée", "'D", "ع", "\u000b𐞁a", "'D", "İ", " ", " ", "0", "　", "🙂'", "D", "'s", "a", "̊漢", "'D", "'D", "३", "é", "'M", ">"]} +{"text": "عⅣ,9!!s …ع<
🙂,0　å\tDž👍🏽\r\n\r\nḍ̇ !!३ \n\r\n\r\n𐞁éfi'M", "tokens": 52, "pieces": ["ع", "Ⅳ", ",", "9", "!!", "s", " ", "…ع", "<", "
", "🙂,", "0", "　a", "̊", "\tDž", "👍🏽\r\n\r\n", "ḋ", "̣", " ", "!!", "३", " \n\r\n\r\n", "𐞁e", "́fi", "'M"]} +{"text": "!<|endoftext|>३t\"३,😀🏽\t'D𐞁12345678'½", "tokens": 33, "pieces": ["!<|", "endoftext", "|>", "३", "t", "\"", "३", ",<", "META", "_START", ">😀🏽", "\t", "'D", "𐞁", "123", "456", "78", "'", "½"]} +{"text": "Dž<|endoftext|>", "tokens": 9, "pieces": ["Dž", "<|", "endoftext", "|>"]} +{"text": "#$%३>
\r\n\r\n<|endoftext|>字٣٤٥٦fifiå\r
ZEOT\rå㋿‍#$%", "tokens": 49, "pieces": ["#$%", "३", ">", "
\r\n\r\n", "<|", "endoftext", "|>", "字", "٣٤٥", "٦", "fifia", "̊\r", "
ZEOT", "\r", "a", "̊㋿‍#$%"]} +{"text": "𐞁㍿s9​…!ſḍ̇'Re.<|endoftext|>(ꟲs \n'll0 …ḍ̇ 'TDžfi<|fim_prefix|>0EOT​🙂½a0'sＡ\u000b", "tokens": 74, "pieces": ["𐞁", "㍿s", "9", "​", "…", "!ſḋ", "̣'", "Re", ".<", "META", "_START", "><|", "endoftext", "|>(", "ꟲs", " \n", "'ll", "0", " ", "…ḋ", "̣", " ", " '", "TDžfi", "<|", "fim", "_prefix", "|>", "0", "EOT", "​🙂", "½", "a", "0", "'s", "Ａ", "\u000b"]} +{"text": ">09(!!ſＡDž- 'Sfi​\u000b'D'VE0!!\t'Se'VE's'D12345678''M", "tokens": 42, "pieces": [">", "09", "(!!", "ſＡDž", "-", " ", " '", "Sfi", "​", "\u000b", "'D", "'VE", "0", "!!<", "EOT", ">", "\t", "'S", "e", "'VE", "'", "s", "'D", "123", "456", "78", "''", "M"]} +{"text": "fi0m>-'sé \n\r‍9fi,Z\r\n½é9
㋿'re>'lĺéⅣ", "tokens": 49, "pieces": ["fi", "0", "m", ">-<", "EOT", ">'", "sé", " \n\r", "‍<", "EOT", ">", "9", "fi", ",Z", "\r\n", "½", "e", "́<", "META", "_START", ">", "9", "
", "㋿'", "re", ">'", "ll", "́e", "́", "Ⅳ", ""]} +{"text": "😀🏽½　\r-\rEOTét#$%é\r'Tع>é İ'D.㍿<|fim_prefix|>½é½ ‍a\"DžDžＡⅣ Ａ<'VE𐞁", "tokens": 63, "pieces": ["😀🏽", "½", "　\r", "-\r", "EOTét", "#$%", "é", "\r", "'T", "ع", ">e", "́", " İ", "'D", ".㍿<|", "fim", "_prefix", "|>", "½", "e", "́", "½", " ‍", "a", "\"DžDžＡ", "Ⅳ", " ", " Ａ", "<<", "EOT", ">'", "VE𐞁"]} +{"text": "ꟲ'M😀🏽🙂­", "tokens": 16, "pieces": ["ꟲ", "'", "M", "😀🏽🙂­"]} +{"text": "㍿Z㍿åfié'ſZ㋿>'VEdeع \n \nm 👍🏽éå३é", "tokens": 43, "pieces": ["㍿Z", "㍿a", "̊fié", "'ſ", "Z", "㋿>'", "VEdeع", " \n \n", "m", " 👍🏽<", "META", "_START", ">e", "́a", "̊", "३", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "…''re　!m㋿ \n'T9\r\n\r\n<|fim_prefix|>m", "tokens": 22, "pieces": ["…", "''", "re", "　", "!m", "㋿", " \n", "'T", "9", "\r\n\r\n", "<|", "fim", "_prefix", "|>", "m"]} +{"text": "\t9EOT　 's9'reİåt\n'D#$%s字>ꟲ", "tokens": 24, "pieces": ["\t", "9", "EOT", "　", " '", "s", "9", "'re", "İa", "̊t", "\n", "'D", "#$%", "s字", ">ꟲ"]} +{"text": "'D(mEOT½\u000bß\r\n\r\n,ḍ̇m's'T́>'ſ'D\r\na字0\t(ß'VE\r\u000b 🙂.عs9(", "tokens": 44, "pieces": ["'D", "(mEOT", "½", "\u000bß", "\r\n\r\n", ",ḋ", "̣m", "'s", "'T", "́>'", "ſ", "'D", "\r\n", "a字", "0", "\t", "(", "ß", "'VE", "\r", "\u000b", " ", "🙂.", "عs", "9", "("]} +{"text": "å're㋿ḍ̇'reZ㋿́'S漢!!
(Ａé'S३\r\n\r\n#$% \n're
'VE#$%fi\n,\u000b", "tokens": 50, "pieces": ["a", "̊'", "re", "㋿ḋ", "̣'", "reZ", "㋿́'", "S漢", "!!", "
", "(Ａé", "'S", "३", "\r\n\r\n", "#$%", " \n", "'re", "
", "'VE", "#$%", "fi", "\n", ",", "\u000b"]} +{"text": "!!é åéİ'Re㋿((!㋿", "tokens": 17, "pieces": ["!!", "é", " ", " a", "̊éİ", "'Re", "㋿((!㋿"]} +{"text": "'sDž\n字\r\n\r\nm#$%fi
漢'Ret½\u000bß'Tḍ̇9 ½ éEOT're'ſⅣ字३\tm", "tokens": 44, "pieces": ["'s", "Dž", "\n", "字", "\r\n\r\n", "m", "#$%", "fi", "
漢", "'Re", "t", "½", "\u000bß", "'T", "ḋ", "̣", "9", " ", "½", " ", " e", "́EOT", "'re", "'ſ", "Ⅳ", "字", "३", "\tm"]} +{"text": "ع'S\"", "tokens": 7, "pieces": ["ع", "'", "S", "\""]} +{"text": " \nZsⅣ\"sⅣ0é12345678<|fim_prefix|>>", "tokens": 20, "pieces": [" \n", "Zs", "Ⅳ", "\"s", "Ⅳ0", "é", "123", "456", "78", "<|", "fim", "_prefix", "|>>"]} +{"text": " \n
d㍿́12345678ſ'Ａ㋿\" \né#$%\rfi<\r\n\r\n'lle", "tokens": 33, "pieces": [" \n", "
d", "㍿́", "123", "456", "78", "ſ", "'", "Ａ", "㋿\"", " \n", "é", "#$%\r", "fi", "<\r\n\r\n", "'ll", "e"]} +{"text": "'VEmDžd'Re\r\n'Re< ㍿ é ‍
 漢…'TZ t\r'Refi!", "tokens": 41, "pieces": ["'VE", "mDžd", "'Re", "\r\n", "'Re", "<", " ", " ㍿", " ", " e", "́", " ", " ‍<", "EOT", ">", "
", " 漢", "…", "'T", "Z", " ", " t", "\r", "'Re", "fi", "!"]} +{"text": "\t\"३!½字#$%\"'Sḍ̇𐞁ꟲ… \nDž́ſéⅣ​👍🏽", "tokens": 56, "pieces": ["\t", "\"", "३", "!", "½", "字", "<", "META", "_START", ">#$%<", "META", "_START", ">\"'", "Sḋ", "̣𐞁ꟲ", "… \n", "Dž", "́ſe", "́", "Ⅳ", "​👍🏽"]} +{"text": "'S(ß-'ll'T!

½>'VE'Td漢ع'Sd漢'VEＡ'så\r<|endoftext|>ⅣEOT'S 漢 '<|fim_prefix|>\t", "tokens": 60, "pieces": ["'S", "(ß", "-'", "ll", "'T", "!", "
", "
", "½", ">'", "VE", "'T", "d漢ع", "'S", "d漢", "'VE", "Ａ", "'s", "a", "̊\r", "<|", "endoftext", "|><", "META", "_START", ">", "Ⅳ", "EOT", "'S", " ", " 漢", " ", " '<|", "fim", "_prefix", "|>", "\t"]} +{"text": "🙂́Ⅳ
éßZ字,­漢'ſ漢'T<|fim_prefix|>", "tokens": 29, "pieces": ["🙂́", "Ⅳ", "
e", "́ßZ字", ",­", "漢", "'ſ", "漢", "'T", "<|", "fim", "_prefix", "|>"]} +{"text": "!!\tßß", "tokens": 4, "pieces": ["!!", "\tßß"]} +{"text": "Z!ḍ̇'Sꟲ<#$%a#$%a's'edⅣ\teeſmeİ9'Dm", "tokens": 33, "pieces": ["Z", "!ḋ", "̣'", "Sꟲ", "<#$%", "a", "#$%", "a", "'s", "'ed", "Ⅳ", "\teeſmeİ", "9", "'D", "m"]} +{"text": "Dž\u000b12345678\"
tꟲ'Mt\"t½", "tokens": 18, "pieces": ["Dž", "\u000b", "123", "456", "78", "\"", "
tꟲ", "'M", "t", "\"t", "½"]} +{"text": "٣٤٥٦'ſ'M\r0EOT<'re9½<​㍿'ll'lla-s‍<|endoftext|>​tm½a aa½,å\"'D", "tokens": 51, "pieces": ["٣٤٥", "٦", "'ſ", "'M", "\r", "0", "EOT", "<'", "re", "9½", "<​㍿'", "ll", "'ll", "a", "-s", "‍<|", "endoftext", "|>​", "tm", "½", "a", " aa", "½", ",a", "̊\"'", "D"]} +{"text": "123456780Ａfi", "tokens": 11, "pieces": ["", "123", "456", "780", "Ａfi"]} +{"text": "İ‍d'll\nEOT'Dd<|endoftext|>'S\u000b", "tokens": 19, "pieces": ["İ", "‍d", "'ll", "\n", "EOT", "'D", "d", "<|", "endoftext", "|>'", "S", "\u000b"]} +{"text": "'M'ReeＡ12345678!😀🏽, Dž<|endoftext|>'s(\"ſ'\tſ0<|endoftext|>EOT ꟲ#$% \n😀🏽DžDž9عſe<'ReＡⅣé", "tokens": 72, "pieces": ["'M", "'Re", "eＡ", "123", "456", "78", "!😀🏽,", " Dž", "<|", "endoftext", "|>'", "s", "(\"", "ſ", "'", "\tſ", "0", "<|", "endoftext", "|>", "EOT", " ", " ꟲ", "#$%", " \n", "😀🏽", "DžDž", "9", "عſe", "<'", "ReＡ", "Ⅳ", "e", "́"]} +{"text": "Ⅳ'ḍ̇　👍🏽0ḍ̇12345678EOT<|fim_prefix|>\r\nⅣ😀🏽'VE\"é!!'S\nß 'T.𐞁é
>ḍ̇\u000b<|fim_prefix|>,́s", "tokens": 80, "pieces": ["Ⅳ", "'<", "EOT", ">ḋ", "̣", "　", "👍🏽", "0", "ḋ", "̣", "123", "456", "78", "EOT", "<|", "fim", "_prefix", "|>\r\n", "Ⅳ", "😀🏽'", "VE", "\"é", "!!'", "S", "\n", "ß", " ", " '", "T", ".𐞁e", "́", "
", ">ḋ", "̣", "\u000b", "<|", "fim", "_prefix", "|>,́", "s"]} +{"text": "!Z\r\n½\u000b\u000bsع\n<.<|fim_prefix|>
'VEsDž㋿d𐞁' \n<|fim_prefix|>İ#$%'MZ३ Ⅳ 'VE\r\n\r\nm9　　\r\n", "tokens": 59, "pieces": ["!Z", "\r\n", "½", "\u000b", "\u000bsع", "\n", "<.<|", "fim", "_prefix", "|>", "
", "'VE", "sDž", "㋿d𐞁", "'", " \n", "<|", "fim", "_prefix", "|>", "İ", "#$%'", "MZ", "३", " ", " ", "Ⅳ", " ", " '", "VE", "\r\n\r\n", "m", "9", "　　\r\n"]} +{"text": "!åİ㋿('Re\r\n'VEḍ̇t<|endoftext|>\n0İ<9!0(", "tokens": 36, "pieces": ["!a", "̊İ", "㋿<", "META", "_START", ">('", "Re", "\r\n", "'VE", "ḋ", "̣t", "<|", "endoftext", "|>\n", "0", "İ", "<", "9", "!", "0", "("]} +{"text": "fi<''Tde,\t… \n<😀🏽ḍ̇Dž👍🏽 é
\tſعḍ̇Ⅳ🙂t's\"㋿ mm(", "tokens": 60, "pieces": ["fi", "<''", "Tde", ",", "\t… \n", "<😀🏽", "ḋ", "̣Dž", "👍🏽", " e", "́", "
", "\tſعḋ", "̣", "Ⅳ", "🙂t", "'s", "\"㋿", " ", " mm", "("]} +{"text": "'re'Re're ,ß
​㋿'reEOTＡ!!㋿tDž#$%> \ne9🙂é…9å'red(s\r\n", "tokens": 44, "pieces": ["'re", "'Re", "'re", " ", " ,", "ß", "
", "​㋿'", "reEOTＡ", "!!㋿", "tDž", "#$%>", " \n", "e", "9", "🙂é", "…", "9", "a", "̊'", "red", "(s", "\r\n"]} +{"text": "fia.9㍿.aꟲ 'llß", "tokens": 17, "pieces": ["fia", ".", "9", "㍿.", "aꟲ", " ", " '", "llß"]} +{"text": "\nDžé <|endoftext|>…字 ­e\r\n…<|endoftext|><'s　ꟲ\t \t-!…<|fim_prefix|>𐞁 ſ\r\n\r\n'VEſḍ̇dع", "tokens": 63, "pieces": ["\n", "Džé", " <|", "endoftext", "|>", "…字", " ­", "e", "\r\n", "…", "<|", "endoftext", "|><'", "s", "　ꟲ", "\t ", "\t", "-!", "…", "<|", "fim", "_prefix", "|>", "𐞁", " ſ", "\r\n\r\n", "'VE", "ſḋ", "̣dع"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "½,字å<|fim_prefix|>🙂字\u000bé\n'Mß're.ſ½é'D<🙂", "字", "\u000bé", "\n", "'M", "ß", "'re", ".ſ", "½", "é", "'D", "<<", "d漢"]} +{"text": "'S
　ḍ̇m㋿Ａe字́ḍ̇㋿'TŹ>mＡé
　 <|endoftext|>'VE,å \n'ſ", "tokens": 56, "pieces": ["'S", "
", "　ḋ", "̣m", "㋿Ａe字", "́ḋ", "̣㋿'", "TZ", "́>", "mＡ", "e", "́", "
　", " ", "<|", "endoftext", "|>'", "VE", ",a", "̊", " \n", "'ſ"]} +{"text": "9ZDž<字é字<Dž½d‍'T!sⅣEOT \n‍\tEOTåa'ſＡå٣٤٥٦İ𐞁", "tokens": 53, "pieces": ["9", "ZDž", "<字é字", "<Dž", "½", "d", "‍'", "T", "!s", "Ⅳ", "EOT", " \n", "‍", "\tEOTa", "̊a", "'ſ", "Ａa", "̊", "٣٤٥", "٦", "İ𐞁"]} +{"text": "'ll>३ \n'12345678#$%\r!d‍́,-t😀🏽\r\n\r\n'D>́👍🏽", "tokens": 38, "pieces": ["'ll", ">", "३", " \n", "'", "123", "456", "78", "#$%\r", "!d", "‍́<", "META", "_START", ">,-", "t", "😀🏽\r\n\r\n", "'D", ">́👍🏽"]} +{"text": "12345678'S as­<|fim_prefix|>Dž12345678>es!<|endoftext|>\t", "tokens": 28, "pieces": ["123", "456", "78", "'S", " as", "­<|", "fim", "_prefix", "|>", "Dž", "123", "456", "78", ">es", "!<|", "endoftext", "|>", "\t"]} +{"text": "Z12345678e'S\r\n<|fim_prefix|>é½-", "tokens": 20, "pieces": ["Z", "123", "456", "78", "e", "'S", "\r\n", "<|", "fim", "_prefix", "|>", "e", "́", "½", "-"]} +{"text": "<|fim_prefix|>\r\nm𐞁\u000b漢߅'s'VE'Dßåß ,  'Re'D'MDž.'ll字'Sꟲ…ع", "tokens": 51, "pieces": ["<|", "fim", "_prefix", "|>\r\n", "m𐞁", "\u000b漢ß", "…", "'s", "'VE", "'D", "ß", "a", "̊ß", " ", ",", " ", " <", "META", "_START", ">'", "Re", "'D", "'M", "Dž", ".'", "ll字", "'S", "ꟲ", "…ع"]} +{"text": "'Dḍ̇'D\nfi!'s\r\n\r\nİ́t …,'re\tḍ̇<|endoftext|>", "tokens": 39, "pieces": ["'D", "ḋ", "̣'", "D", "\n", "fi", "!'", "s", "\r\n\r\n", "İ", "́t", " ", "…", ",'", "re", "\t", "ḋ", "̣<|", "endoftext", "|>"]} +{"text": "İ.,'VE'S
\r( !­ßſ'Sꟲs\tꟲ\u000b-'ſع,Ａå", "tokens": 35, "pieces": ["İ", ".,'", "VE", "'S", "
\r", "(", " ", " !­", "ßſ", "'S", "ꟲs", "\tꟲ", "\u000b", "-'", "ſع", ",Ａa", "̊"]} +{"text": "12345678­٣٤٥٦('Re0ſ<…ع🙂\u000btéⅣa're're", "tokens": 37, "pieces": ["123", "456", "78", "­", "٣٤٥", "٦", "('", "Re", "0", "ſ", "<", "…ع", "🙂", "\u000b", "t", "e", "́", "Ⅳ", "a", "'re", "'re"]} +{"text": "ꟲ字>t́㋿\r\n\r\n字👍🏽ds'Tß<|fim_prefix|>🙂 字", "tokens": 34, "pieces": ["ꟲ字", ">t", "́<", "META", "_START", ">㋿\r\n\r\n", "字", "👍🏽", "ds", "'T", "ß", "<|", "fim", "_prefix", "|>🙂", " 字"]} +{"text": " a  ​'sꟲdés!!\"'VE<|endoftext|>'re👍🏽EOT'sZ're' ́Ａ字é½𐞁's㋿9㍿ſ", "tokens": 60, "pieces": [" a", " ", " ", "​'", "sꟲ", "de", "́s", "!!\"'", "VE", "<|", "endoftext", "|>'", "re", "👍🏽", "EOT", "'s", "Z", "'re", "'", " ́", "Ａ字e", "́", "½", "𐞁", "'s", "㋿", "9", "㍿ſ"]} +{"text": "‍\n'sm,fiع́t,'s!!٣٤٥٦'字😀🏽👍🏽‍'re३!
", "tokens": 44, "pieces": ["‍\n", "'s", "m", ",fiع", "́t", ",'", "s", "!!", "٣٤٥", "٦", "'字", "😀🏽👍🏽‍'", "re", "३", "!", "
"]} +{"text": "　…𐞁­'D­Ⅳ0EOT\r🙂字\r\nß'VE३é<|endoftext|>Z👍🏽😀🏽ß٣٤٥٦d𐞁ꟲ㋿<|fim_prefix|>㍿\n", "tokens": 81, "pieces": ["　", "…𐞁", "­'", "D", "­", "Ⅳ0", "EOT", "\r", "🙂字", "\r\n", "ß", "'VE", "३", "e", "́<|", "endoftext", "|>", "Z", "👍🏽😀🏽", "ß", "٣٤٥", "٦", "d𐞁ꟲ", "㋿<|", "fim", "_prefix", "|>㍿<", "META", "_START", ">\n"]} +{"text": "-\r\n\r\nß🙂're<|fim_prefix|>EOT…ésfi<'ll…ꟲ12345678㋿e>", "tokens": 41, "pieces": ["-\r\n\r\n", "ß", "🙂<", "META", "_START", ">'", "re", "<|", "fim", "_prefix", "|>", "EOT", "…e", "́sfi", "<'", "ll", "…ꟲ", "123", "456", "78", "㋿e", ">"]} +{"text": "字.EOTå \n ½ ‍'re'VE'llß
sſ漢'T'M'Då(#$%<|fim_prefix|><­'re\r\n\r\nZ½ß're<|fim_prefix|>'s　é", "tokens": 63, "pieces": ["字", ".EOTa", "̊", " \n", " ", " ", "½", " ", "‍'", "re", "'VE", "'ll", "ß", "
sſ漢", "'T", "'M", "'D", "a", "̊(#$%<|", "fim", "_prefix", "|><­'", "re", "\r\n\r\n", "Z", "½", "ß", "'re", "<|", "fim", "_prefix", "|><", "META", "_START", ">'", "s", "　é"]} +{"text": "Ⅳ", "tokens": 5, "pieces": ["", "Ⅳ"]} +{"text": "　9ß(😀🏽㋿漢< <|endoftext|>''Mfi𐞁'D\r\n\r\n 12345678\r\n漢å<ع're,\r\nZ", "tokens": 46, "pieces": ["　", "9", "ß", "(😀🏽㋿", "漢", "<", " <|", "endoftext", "|>''", "Mfi𐞁", "'D", "\r\n\r\n", " ", "123", "456", "78", "\r\n", "漢a", "̊<", "ع", "'re", ",\r\n", "Z"]} +{"text": "٣٤٥٦ß!!Ａ\t👍🏽!!e.ꟲ'VE", "tokens": 27, "pieces": ["٣٤٥", "٦", "ß", "!!", "Ａ", "\t", "👍🏽!!", "e", ".ꟲ", "'VE"]} +{"text": "'D\r\n\r\n'sſ𐞁<|fim_prefix|><|endoftext|>12345678<|endoftext|>é́́İ< ſDž'T \nfiⅣſＡ>Zꟲ㋿ع\r(>ſ' \n \n12345678٣٤٥٦Ａ", "tokens": 84, "pieces": ["'D", "\r\n\r\n", "'s", "ſ𐞁", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "123", "456", "78", "<|", "endoftext", "|>", "é", "́́", "İ", "<", " ", " ſDž", "'T", " \n", "fi", "Ⅳ", "ſＡ", "><", "EOT", ">Zꟲ", "㋿ع", "\r", "(>", "ſ", "'", " \n \n", "123", "456", "78٣", "٤٥٦", "Ａ"]} +{"text": "'sḍ̇-é३<|fim_prefix|>'T'sſ'Mé<|fim_prefix|>", "tokens": 32, "pieces": ["'s", "ḋ", "̣-", "e", "́", "३", "<|", "fim", "_prefix", "|>'", "T", "'s", "ſ", "'M", "e", "́<|", "fim", "_prefix", "|>"]} +{"text": ">漢d\"\nZ're­'S'D,\n", "tokens": 12, "pieces": [">漢d", "\"\n", "Z", "'re", "­'", "S", "'D", ",\n"]} +{"text": "m 🙂'TＡtⅣ\rd'Reſ'VE٣٤٥٦Ａ३㍿'Reİ'Ss å😀🏽́!…\r(\n'M'S'ſ漢Ⅳ 漢…Ⅳ ", "tokens": 70, "pieces": ["m", " ", "🙂'", "TＡt", "Ⅳ", "\r", "d", "'Re", "ſ", "'VE", "٣٤٥", "٦", "Ａ", "३", "㍿'", "Reİ", "'S", "s", " a", "̊😀🏽́!", "…\r", "(\n", "'M", "'S", "'ſ", "漢", "Ⅳ", " ", " 漢", "…", "Ⅳ", " "]} +{"text": "🙂́ 12345678<|endoftext|>'M(", "tokens": 16, "pieces": ["🙂́", " ", "123", "456", "78", "<|", "endoftext", "|>'", "M", "("]} +{"text": "<'ſ½'ſ're½'VE\"\n
'S,ßꟲ\n!!ſ'ſ 
!'MßéZ", "tokens": 35, "pieces": ["<'", "ſ", "½", "'ſ", "'re", "½", "'VE", "\"\n", "
", "'S", ",ßꟲ", "\n", "!!", "ſ", "'ſ", " ", "
", "!'", "MßéZ"]} +{"text": "
d́EOT漢!<|fim_prefix|>.½  ३…\"Ａ́'S<|endoftext|>", "tokens": 36, "pieces": ["
d", "́EOT漢", "!<|", "fim", "_prefix", "|>.", "½", "  ", " ", "३", "…", "\"Ａ", "́'", "S", "<|", "endoftext", "|>"]} +{"text": "<|endoftext|>0ع👍🏽'Re字ع're🙂\r\n\r\n𐞁ḍ̇,", "tokens": 33, "pieces": ["<|", "endoftext", "|>", "0", "ع", "👍🏽'", "Re字ع", "'re", "🙂\r\n\r\n", "𐞁ḋ", "̣,"]} +{"text": "'re\n㋿\u000bs३\r\nḍ̇ꟲ's'llå-ḍ̇Ａ \n​㍿'!'Re'rem㋿\r\n \nḍ̇㍿😀🏽½", "tokens": 60, "pieces": ["'re", "\n", "㋿", "\u000bs", "३", "\r\n", "ḋ", "̣ꟲ", "'s", "'ll", "a", "̊-", "ḋ", "̣Ａ", " \n", "​㍿'!'", "Re", "'re", "m", "㋿\r\n", " \n", "ḋ", "̣㍿😀🏽", "½"]} +{"text": "e🙂fi'S<|endoftext|>
met'M's½å🙂㋿👍🏽'M!, ß<|fim_prefix|>\r\n\r\n<|fim_prefix|>　m,", "tokens": 55, "pieces": ["e", "🙂fi", "'S", "<|", "endoftext", "|>", "
met", "'M", "'s", "½", "a", "̊🙂㋿👍🏽'", "M", "!,", " ß", "<|", "fim", "_prefix", "|>\r\n\r\n", "<|", "fim", "_prefix", "|>", "　m", ","]} +{"text": "<Ａét<|fim_prefix|>İ<|fim_prefix|>,
", "tokens": 21, "pieces": ["<Ａét", "<|", "fim", "_prefix", "|>", "İ", "<|", "fim", "_prefix", "|>,", "
"]} +{"text": "٣٤٥٦ß aＡİa \t\r\n'Dꟲ🙂　", "tokens": 27, "pieces": ["٣٤٥", "٦", "ß", " aＡİa", "", " \t\r\n", "'D", "ꟲ", "🙂", "　"]} +{"text": "½#$%​é字12345678ß½fi'llDž­Ⅳ㋿(,Dž \n'S'VEd", "tokens": 32, "pieces": ["½", "#$%​", "e", "́字", "123", "456", "78", "ß", "½", "fi", "'ll", "Dž", "­", "Ⅳ", "㋿(,", "Dž", " \n", "'S", "'VE", "d"]} +{"text": ">'S(#$%'D…Z‍'M㋿\r\n>", "tokens": 18, "pieces": [">'", "S", "(#$%'", "D", "…Z", "‍'", "M", "㋿\r\n", ">"]} +{"text": "aⅣ12345678<|fim_prefix|>>m🙂 !!…'Re< \r\nＡ​😀🏽#$%-Ａ'Re🙂​'Tt'Re(s٣٤٥٦Z­m", "tokens": 54, "pieces": ["a", "Ⅳ12", "345", "678", "<|", "fim", "_prefix", "|>>", "m", "🙂", " !!", "…", "'Re", "<", " \r\n", "Ａ", "​😀🏽#$%-", "Ａ", "'Re", "🙂​'", "Tt", "'Re", "(s", "٣٤٥", "٦", "Z", "­m"]} +{"text": "å.'M'VE
\tEOT漢漢🙂.12345678ع
EOT!!Zß  dع9ſ.<'S漢\n,👍🏽9🙂\u000b", "tokens": 53, "pieces": ["a", "̊.'", "M", "'VE", "
", "\tEOT漢漢", "🙂.", "123", "456", "78", "ع", "
EOT", "!!", "Zß", "  ", " dع", "9", "ſ", ".<'", "S漢", "\n", ",👍🏽", "9", "🙂", "\u000b"]} +{"text": "ḍ̇Ａ'ſ<", "tokens": 11, "pieces": ["ḋ", "̣Ａ", "'ſ", "<"]} +{"text": "㍿\u000b>e㋿#$%'S\r<|endoftext|>fi \ń(<|fim_prefix|>
.İDžs'M'Ret0
'S'MＡ12345678", "tokens": 50, "pieces": ["㍿", "\u000b", ">e", "㋿#$%'", "S", "\r", "<|", "endoftext", "|>", "fi", " \n", "́(<|", "fim", "_prefix", "|>", "
", ".İDžs", "'M", "'Re", "t", "0", "
", "'S", "'M", "Ａ", "123", "456", "78"]} +{"text": "å", "tokens": 3, "pieces": ["a", "̊"]} +{"text": " ḍ̇Dž", "tokens": 8, "pieces": [" ", " ḋ", "̣Dž"]} +{"text": "'ſå३ß'MDž'M<|endoftext|>​12345678٣٤٥٦😀🏽 'VE'T'(ع\t(åⅣ", "tokens": 57, "pieces": ["'ſ", "a", "̊", "३", "ß", "'M", "Dž", "'M", "<|", "endoftext", "|>​", "123", "456", "78", "", "٣٤٥", "٦", "😀🏽", " ", " '", "VE", "'T", "'(", "ع", "\t", "(a", "̊", "Ⅳ"]} +{"text": "EOT𐞁d-!!‍'D👍🏽ß'T漢0👍🏽字12345678'Re(…('T字　å(Z½éḍ̇0#$%'S ٣٤٥٦​", "tokens": 75, "pieces": ["EOT𐞁d", "-<", "META", "_START", ">!!‍'", "D", "👍🏽", "ß", "'T", "漢", "0", "👍🏽", "字", "123", "456", "78", "'Re", "(", "…", "('", "T字", "　", "a", "̊(", "Z", "½", "éḋ", "̣", "0", "#$%'", "S", " ", " ", "٣٤٥", "٦", "​"]} +{"text": "0'Re\r\n\r\n-mⅣ½a½́ ꟲ…", "tokens": 15, "pieces": ["0", "'Re", "\r\n\r\n", "-m", "Ⅳ½", "a", "½", "́", " ꟲ", "…"]} +{"text": "9EOT٣٤٥٦'ſe'VE٣٤٥٦字é\"'ll", "tokens": 30, "pieces": ["9", "EOT", "٣٤٥", "٦", "'ſ", "e", "'VE", "٣٤٥", "٦", "字e", "́\"'", "ll"]} +{"text": "　EOT'ſfi'S-12345678.ꟲfi🙂🙂 'D½12345678\tEOTdm'll\n'!åå-'D", "tokens": 48, "pieces": ["　EOT", "'ſ", "fi", "'S", "-", "123", "456", "78", ".ꟲfi", "🙂<", "EOT", ">🙂", " '", "D", "½12", "345", "678", "\tEOTdm", "'ll", "\n", "'!", "a", "̊a", "̊-'", "D"]} +{"text": "'VE EOT'Mḍ̇'ſ're𐞁İ'ſ\"'reß \n­0­<|endoftext|>漢sDž 'Re­'VEع𐞁.\n३㍿Zdé'M", "tokens": 71, "pieces": ["'VE", " EOT", "'M", "ḋ", "̣<", "EOT", ">'", "ſ", "'re", "𐞁İ", "'ſ", "\"'", "re", "ß", " \n", "­", "0", "­<|", "endoftext", "|>", "漢sDž", " ", "'Re", "­'", "VEع𐞁", ".\n", "३", "㍿Zdé", "'M"]} +{"text": "!!½!!s12345678!!.", "tokens": 8, "pieces": ["!!", "½", "!!", "s", "123", "456", "78", "!!."]} +{"text": "fi㍿t३…Dž12345678é३Z\u000bfi,0­d'ſ‍\r\n\r\nⅣ…fi ", "tokens": 49, "pieces": ["fi", "㍿t", "३", "…Dž", "123", "456", "78", "é", "३", "Z", "\u000bfi", ",", "0", "­<", "EOT", ">d", "'ſ", "‍\r\n\r\n", "", "Ⅳ", "…fi", " "]} +{"text": "dع'Rem\r'M>'S​'ſＡdéDž́ſİ9ḍ̇,\nå-字­- 'Ms'M!!٣٤٥٦12345678३Ａé́\t", "tokens": 59, "pieces": ["dع", "'Re", "m", "\r", "'M", ">'", "S", "​'", "ſＡdéDž", "́ſİ", "9", "ḋ", "̣,\n", "a", "̊-", "字", "­-", " '", "Ms", "'M", "!!", "٣٤٥", "٦12", "345", "678", "३", "Ａe", "́́", "\t"]} +{"text": "é🙂\t\n,Ａsé　🙂'lĺ'Ddß\"字 !!'S'D\u000b'll< \n(DžＡå<|endoftext|>0!'re", "tokens": 55, "pieces": ["e", "́🙂", "\t\n", ",Ａsé", "　", "🙂'", "ll", "́'", "D", "dß", "\"字", " ", "!!'", "S", "'D", "\u000b", "'ll", "<", " \n", "(DžＡa", "̊<|", "endoftext", "|>", "0", "!'", "re"]} +{"text": "é", "tokens": 2, "pieces": ["e", "́"]} +{"text": " '​🙂-字'ſ<|endoftext|>'M㋿\"tt㍿d-ſ'Tꟲ'M½٣٤٥٦\r\n\r\n'ſ \n'llt'reZ٣٤٥٦0d", "tokens": 62, "pieces": [" '​🙂-", "字", "'ſ", "<|", "endoftext", "|>'", "M", "㋿\"", "tt", "㍿d", "-ſ", "'T", "ꟲ", "'M", "½٣٤", "٥٦", "\r\n\r\n", "'ſ", " \n", "'ll", "t", "'re", "Z", "٣٤٥", "٦0", "d"]} +{"text": "\"\nt 'ꟲß", "tokens": 11, "pieces": ["\"\n", "t", " ", "'<", "META", "_START", ">ꟲß"]} +{"text": "12345678'll\t​12345678.\r\n\r\nſ!tßfi‍\"'s< \r!!0é½\"('Dḍ̇et'S.", "tokens": 43, "pieces": ["123", "456", "78", "'ll", "\t", "​", "123", "456", "78", ".\r\n\r\n", "ſ", "!tßfi", "‍\"'", "s", "<", " \r", "!!", "0", "e", "́", "½", "\"('", "Dḋ", "̣et", "'S", "."]} +{"text": "-m𐞁å", "tokens": 12, "pieces": ["-m𐞁a", "̊<", "EOT", ">"]} +{"text": "\r\nZ🙂👍🏽­　\u000b", "tokens": 13, "pieces": ["\r\n", "Z", "🙂👍🏽­", "　\u000b"]} +{"text": "\t👍🏽e12345678m'Sé>㋿'MEOT's\t㍿\né́'S\"!!0 ßå!d ,Ａ٣٤٥٦ \n‍\n'Re", "tokens": 62, "pieces": ["\t", "👍🏽", "e", "123", "456", "78", "m", "'S", "é", ">㋿'", "MEOT", "'s", "\t", "㍿\n", "é", "́'", "S", "\"!!", "0", " ", " ßa", "̊!", "d", " ", ",Ａ", "٣٤٥", "٦", " \n", "‍\n", "'Re"]} +{"text": "𐞁( 😀🏽\n\r\n\r\n", "tokens": 13, "pieces": ["𐞁", "(", " ", "😀🏽\n\r\n\r\n"]} +{"text": "9tm,­ع-字३. 漢‍ 　Ⅳ'Re'Re\u000b9Z", "tokens": 25, "pieces": ["9", "tm", ",­", "ع", "-字", "३", ".", " ", " 漢", "‍", " ", "　", "Ⅳ", "'Re", "'Re", "\u000b", "9", "Z"]} +{"text": "‍👍🏽'M­́İḍ̇ 🙂(mDžＡ㋿\r\n 9'll ><|endoftext|>ſ\n!!🙂ét9🙂>'VE-9 ", "tokens": 61, "pieces": ["‍👍🏽'", "M", "­́", "İḋ", "̣", " ", " 🙂(", "mDžＡ", "㋿\r\n", " ", " ", "9", "'ll", " ><|", "endoftext", "|>", "ſ", "\n", "!!🙂", "e", "́t", "9", "🙂>'", "VE", "-", "9", "", " "]} +{"text": "-0 ḍ̇", "tokens": 7, "pieces": ["-", "0", " ḋ", "̣"]} +{"text": "é字fifi!!é's👍🏽ſm<|fim_prefix|>(漢s\r\n\r\n
İ३
ݽ's9İaſs'sé'ſ \néİ😀🏽\t", "tokens": 62, "pieces": ["e", "́字fifi", "!!", "é", "'s", "👍🏽", "ſm", "<|", "fim", "_prefix", "|>(", "漢s", "\r\n\r\n", "
İ", "३", "
İ", "½", "'s", "9", "İaſs", "'s", "e", "́'", "ſ", " \n", "e", "́İ", "😀🏽", "\t"]} +{"text": "عs", "tokens": 2, "pieces": ["عs"]} +{"text": "İ's\refi　Ａt's­9😀🏽åⅣ\r\n'd‍", "tokens": 27, "pieces": ["İ", "'s", "\r", "efi", "　Ａt", "'s", "­", "9", "😀🏽", "a", "̊", "Ⅳ", "\r\n", "'d", "‍"]} +{"text": "<\r漢\u000ba\"½t'T𐞁字0㋿\t३😀🏽'Re'T", "tokens": 30, "pieces": ["<\r", "漢", "\u000ba", "\"", "½", "t", "'T", "𐞁字", "0", "㋿", "\t", "३", "😀🏽'", "Re", "'T"]} +{"text": "👍🏽", "tokens": 6, "pieces": ["👍🏽"]} +{"text": "　å👍🏽𐞁\r\n'' 0e!!'D🙂
", "tokens": 26, "pieces": ["　a", "̊👍🏽", "𐞁", "\r\n", "''", " ", "0", "e", "!!'", "D", "🙂", "
"]} +{"text": "12345678Z", "tokens": 4, "pieces": ["123", "456", "78", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi…123456780Z0'M", "tokens": 10, "pieces": ["fi", "…", "123", "456", "780", "Z", "0", "'M"]} +{"text": "'ſ 'VE,\t३Ⅳ😀🏽", "tokens": 17, "pieces": ["'ſ", " ", " '", "VE", ",", "\t", "३Ⅳ", "😀🏽"]} +{"text": "字­'漢ꟲ\r\n­‍'S('S\"́sa\u000bßßt#$% <|fim_prefix|>👍🏽 ḍ̇t𐞁ḍ̇ …!ß𐞁㍿", "tokens": 71, "pieces": ["字", "­'", "漢ꟲ", "\r\n", "­‍'", "S", "('", "S", "\"́", "sa", "\u000b", "ßßt", "#$%", " <|", "fim", "_prefix", "|><", "EOT", ">👍🏽", " ḋ", "̣t𐞁ḋ", "̣", " ", "…", "!ß𐞁", "㍿"]} +{"text": "ſ́😀🏽㋿0'T­漢ع‍\t d12345678'VE\tⅣſ🙂㋿dDž't!!", "tokens": 44, "pieces": ["ſ", "́😀🏽㋿", "0", "'T", "­漢", "ع", "‍", "\t", " d", "123", "456", "78", "'VE", "\t", "Ⅳ", "ſ", "🙂㋿", "dDž", "'t", "!!"]} +{"text": "t…عꟲß́
👍🏽\u000b'MＡdſ'VE'D", "tokens": 27, "pieces": ["t", "…عꟲß", "́", "
", "👍🏽", "\u000b", "'M", "Ａdſ", "'VE", "'D"]} +{"text": "İß漢e㋿ ३…<|endoftext|>9Ａ́", "tokens": 25, "pieces": ["İß漢e", "㋿", " ", " ", "३", "…", "<|", "endoftext", "|>", "9", "Ａ", "́"]} +{"text": "ß'ret(ßt'Re👍🏽0,\u000b!!s\rＡ'​𐞁​字ꟲع\u000b", "tokens": 41, "pieces": ["ß", "'re", "t", "(ßt", "'Re", "👍🏽", "0", ",", "\u000b", "!!", "s", "\r", "Ａ", "'​", "𐞁", "​字ꟲع", "", "\u000b"]} +{"text": "٣٤٥٦Ａ'T\t  mꟲ9 >'T𐞁ＡaＡ\r\n>👍🏽'S'Dm
​", "tokens": 52, "pieces": ["٣٤٥", "٦", "Ａ", "'T", "\t ", " mꟲ", "9", "", " >'", "T𐞁ＡaＡ", "\r\n", ">👍🏽'", "S", "'D", "m", "
", "​<", "EOT", ">"]} +{"text": "Džḍ̇㍿(<|fim_prefix|>ḍ̇½!!\n('Re<|fim_prefix|>👍🏽 \n…字\n12345678Ａ㋿ḍ̇d éİEOTt.…ſ\u000bå're​9", "tokens": 79, "pieces": ["Džḋ", "̣㍿(<|", "fim", "_prefix", "|>", "ḋ", "̣", "½", "!!\n", "('", "Re", "<|", "fim", "_prefix", "|>👍🏽", " \n", "…字", "\n", "123", "456", "78", "Ａ", "㋿ḋ", "̣d", " éİEOTt", ".", "…ſ", "\u000ba", "̊'", "re", "​", "9"]} +{"text": "\r\n\r\nⅣſ'redé,é \n<|endoftext|>ßſ\rꟲＡ,\n\r👍🏽(👍🏽😀🏽<|endoftext|>\"-\n𐞁…9", "tokens": 62, "pieces": ["\r\n\r\n", "Ⅳ", "ſ", "'re", "de", "́,", "é", " \n", "<|", "endoftext", "|>", "ßſ", "\r", "ꟲＡ", ",\n\r", "👍🏽(👍🏽😀🏽<|", "endoftext", "|>\"-\n", "𐞁", "…", "9"]} +{"text": "Ａ0's", "tokens": 4, "pieces": ["Ａ", "0", "'s"]} +{"text": "😀🏽Z\n'Tſfi\r\n'll>.'SßéEOT0　漢…😀🏽́Ⅳ́𐞁ＡéEOT\u000bt", "tokens": 47, "pieces": ["😀🏽", "Z", "\n", "'T", "ſfi", "\r\n", "'ll", ">.'", "Sße", "́EOT", "0", "　漢", "…", "😀🏽́", "Ⅳ", "́𐞁ＡéEOT", "\u000bt"]} +{"text": "🙂漢!!İ🙂٣٤٥٦EOTß\r\n\t
#$%å0>-👍🏽\ŕ𐞁𐞁३İ", "tokens": 49, "pieces": ["🙂漢", "!!", "İ", "🙂", "٣٤٥", "٦", "EOTß", "\r\n", "\t", "
", "#$%", "a", "̊", "0", ">-👍🏽\r", "́𐞁𐞁", "३", "İ"]} +{"text": "Z'T're ㋿​½­‍å \n<　<'T \nmm½", "tokens": 26, "pieces": ["Z", "'T", "'re", " ", "㋿​", "½", "­‍", "a", "̊", " \n", "<", "　", "<'", "T", " \n", "mm", "", "½"]} +{"text": "'VE​㍿­½<|endoftext|>", "tokens": 19, "pieces": ["'VE", "​㍿­<", "META", "_START", ">", "½", "<|", "endoftext", "|>"]} +{"text": "\r\nå\r ٣٤٥٦!!'ll\nsß're'ſ'S9sZ🙂\r\nß­s", "tokens": 37, "pieces": ["\r\n", "a", "̊\r", " ", " ", "٣٤٥", "٦", "!!'", "ll", "\n", "s", "ß", "'re", "'ſ", "'S", "9", "sZ", "🙂\r\n", "ß", "­s"]} +{"text": "'VE🙂", "tokens": 4, "pieces": ["'VE", "🙂"]} +{"text": "é\u000bDž‍s­!", "tokens": 10, "pieces": ["e", "́", "\u000bDž", "‍s", "­!"]} +{"text": "'SZ#$%Z½m🙂ſ字ع字\råeſ'll😀🏽å", "tokens": 30, "pieces": ["'S", "Z", "#$%", "Z", "½", "m", "🙂ſ字ع字", "\r", "a", "̊eſ", "'ll", "😀🏽", "a", "̊"]} +{"text": "ḍ̇'T<|endoftext|>
9<|fim_prefix|>漢㋿‍0ß🙂　a\u000bＡ३", "tokens": 42, "pieces": ["ḋ", "̣'", "T", "<|", "endoftext", "|>", "
", "9", "<|", "fim", "_prefix", "|>", "漢", "㋿‍", "0", "ß", "🙂", "　a", "\u000bＡ", "३"]} +{"text": "'ll-'ll0'Re字ZꟲⅣ漢👍🏽\r\n\r\n😀🏽\u000bZ字,dß", "tokens": 31, "pieces": ["'ll", "-'", "ll", "0", "'Re", "字Zꟲ", "Ⅳ", "漢", "👍🏽\r\n\r\n", "😀🏽", "\u000bZ字", ",dß"]} +{"text": "\r\n\r\nſDžZ're \nⅣ9Ａ'ſ\r\n\r\n'VÉé٣٤٥٦𐞁ém!<'Re #$%EOT!!", "tokens": 45, "pieces": ["\r\n\r\n", "ſDžZ", "'re", " \n", "Ⅳ9", "Ａ", "'ſ", "\r\n\r\n", "'VE", "́e", "́", "٣٤٥", "٦", "𐞁e", "́m", "!<'", "Re", " #$%", "EOT", "!!"]} +{"text": "'S<|fim_prefix|>ꟲ😀🏽s　d'VE漢<|endoftext|>'llDžſ😀🏽", "tokens": 45, "pieces": ["'S", "<|", "fim", "_prefix", "|>", "ꟲ", "😀🏽", "s", "　d", "'VE", "漢", "<|", "endoftext", "|>'", "llDžſ", "😀🏽"]} +{"text": "'Sde㍿漢>🙂Dž!!<👍🏽 \n½ḍ̇'ll­fi12345678\"é<|fim_prefix|>", "tokens": 44, "pieces": ["'S", "de", "㍿漢", ">🙂", "Dž", "!!<👍🏽", " \n", "½", "ḋ", "̣'", "ll", "­fi", "123", "456", "78", "\"é", "<|", "fim", "_prefix", "|>"]} +{"text": "\"\r\n'VE<|fim_prefix|>🙂\"", "tokens": 13, "pieces": ["\"\r\n", "'VE", "<|", "fim", "_prefix", "|>🙂\""]} +{"text": " ,<​ß'T<\r\n\r\nåeß' é<|endoftext|>>'ll-ß's㋿'VEß‍<|fim_prefix|>İt…-12345678'>٣٤٥٦　ß", "tokens": 61, "pieces": [" ,<​", "ß", "'T", "<\r\n\r\n", "a", "̊eß", "'", " e", "́<|", "endoftext", "|>>'", "ll", "-ß", "'s", "㋿'", "VEß", "‍<|", "fim", "_prefix", "|>", "İt", "…", "-", "123", "456", "78", "'>", "٣٤٥", "٦", "　ß"]} +{"text": "t​!12345678'll😀🏽\r\n", "tokens": 13, "pieces": ["t", "​!", "123", "456", "78", "'ll", "😀🏽\r\n"]} +{"text": ".é　,<|fim_prefix|>ḍ̇\rEOT'Ret​åé\t'ſt㋿ſⅣ🙂漢e\r\n\r\n'SＡa12345678　!­", "tokens": 54, "pieces": [".e", "́", "　", ",<|", "fim", "_prefix", "|>", "ḋ", "̣\r", "EOT", "'Re", "t", "​a", "̊e", "́", "\t", "'ſ", "t", "㋿ſ", "Ⅳ", "🙂漢e", "\r\n\r\n", "'S", "Ａa", "123", "456", "78", "　", "!­"]} +{"text": "　́  're12345678dm", "tokens": 9, "pieces": ["　", "́", " ", " ", "'re", "123", "456", "78", "dm"]} +{"text": "ſéſ9́EOTEOT!!\" ßİ!!fia é!…s字e", "tokens": 29, "pieces": ["ſe", "́ſ", "9", "́EOTEOT", "!!\"", " ", " ßİ", "!!", "fia", " é", "!", "…s字e"]} +{"text": "३'s\r'M
㍿,\r\n\r\nعEOT 'll,'re\r\n'M'ſİ!", "tokens": 28, "pieces": ["३", "'s", "\r", "'M", "
", "㍿,\r\n\r\n", "عEOT", " ", "'ll", ",'", "re", "\r\n", "'M", "'ſ", "İ", "!<", "EOT", ">"]} +{"text": "'ſm'S'llſ'Re漢éꟲ9", "tokens": 19, "pieces": ["'ſ", "m", "'S", "'ll", "ſ", "'Re", "漢é", "ꟲ", "9"]} +{"text": "-.ꟲſ'\r'll'VE
å", "tokens": 16, "pieces": ["-.", "ꟲſ", "'\r", "'ll", "'VE", "
a", "̊"]} +{"text": "'S٣٤٥٦é0Ａ🙂é<|endoftext|>s\r\n\r\n", "tokens": 29, "pieces": ["'S", "٣٤٥", "٦", "e", "́", "0", "Ａ", "🙂e", "́<|", "endoftext", "|><", "EOT", ">s", "\r\n\r\n"]} +{"text": "\u000b", "tokens": 1, "pieces": ["\u000b"]} +{"text": ".\"-ßⅣEOT🙂aⅣé
", "tokens": 16, "pieces": [".\"-", "ß", "Ⅳ", "EOT", "🙂a", "Ⅳ", "e", "́", "
"]} +{"text": "½\u000b🙂‍0!a ३9\r\n😀🏽߅.'S", "tokens": 24, "pieces": ["½", "\u000b", "🙂‍", "0", "!a", " ", "३9", "\r\n", "😀🏽", "ß", "…", ".'", "S"]} +{"text": "漢Ⅳḍ̇Ａ((\r\r\n .<|endoftext|>
ſ'Re३٣٤٥٦ꟲm", "tokens": 40, "pieces": ["漢", "Ⅳ", "ḋ", "̣Ａ", "((\r\r\n", " ", ".<|", "endoftext", "|>", "
ſ", "'Re", "३٣٤", "٥٦", "ꟲm"]} +{"text": "!!٣٤٥٦\r\n\n", "tokens": 10, "pieces": ["!!", "٣٤٥", "٦", "\r\n\n"]} +{"text": "٣٤٥٦字İ!!m'T…eé", "tokens": 21, "pieces": ["٣٤٥", "٦", "字İ", "!!", "m", "'T", "…eé", ""]} +{"text": ".　😀🏽d(Ａ́Z \nＡ🙂ß'Dİd \n½㍿é‍", "tokens": 30, "pieces": [".", "　", "😀🏽", "d", "(Ａ", "́Z", " \n", "Ａ", "🙂ß", "'D", "İd", " \n", "½", "㍿é", "‍"]} +{"text": "\r\n\r\n​Dž😀🏽s<|endoftext|>½0½½'re…", "tokens": 28, "pieces": ["\r\n\r\n", "​Dž", "😀🏽", "s", "<|", "endoftext", "|>", "½0", "", "½½", "'re", "…"]} +{"text": "Z'Dꟲ‍åéḍ̇mع\r", "tokens": 22, "pieces": ["Z", "'D", "ꟲ", "‍a", "̊éḋ", "̣mع", "\r"]} +{"text": "­>'S'ſ!!étß'll字𐞁m>ꟲ漢\nEOT٣٤٥٦'ll12345678(fi​9å0'D🙂́ Ａm", "tokens": 56, "pieces": ["­>'", "S", "'ſ", "!!", "e", "́tß", "'ll", "字𐞁m", ">ꟲ漢", "\n", "EOT", "٣٤٥", "٦", "'ll", "123", "456", "78", "(fi", "​", "9", "a", "̊", "0", "'D", "🙂́", " ", " Ａm"]} +{"text": "s,'ll
…0's9é㍿é३🙂́Džſ0\r\n\r\n'sع'M 
Z!!ḍ̇", "tokens": 45, "pieces": ["s", ",'", "ll", "
", "…", "0", "'s", "9", "e", "́㍿", "e", "́", "३", "🙂́", "Džſ", "0", "\r\n\r\n", "'s", "ع", "'M", " ", "
Z", "!!", "ḋ", "̣"]} +{"text": "é ,İ<|endoftext|>عéé𐞁👍🏽​ſa👍🏽ZZ𐞁\u000ba0 ­9½0\t​\u000bfi''re'Re<|fim_prefix|>​\u000bå‍(<", "tokens": 71, "pieces": ["é", " ", ",İ", "<|", "endoftext", "|>", "عe", "́é𐞁", "👍🏽​", "ſa", "👍🏽", "ZZ𐞁", "\u000ba", "0", " ", "­", "9½0", "\t", "​", "\u000bfi", "''", "re", "'Re", "<|", "fim", "_prefix", "|>​", "\u000ba", "̊‍(<"]} +{"text": "'Tḍ̇½ \n''Re<'re‍½.e12345678'D́é 𐞁fi🙂12345678\t٣٤٥٦fi漢'ſḍ̇12345678'D𐞁　漢é٣٤٥٦", "tokens": 82, "pieces": ["'T", "ḋ", "̣", "½", " \n", "''", "Re", "<'", "re", "‍", "½", ".e", "123", "456", "78", "'D", "́e", "́", " ", " 𐞁fi", "🙂", "123", "456", "78", "\t", "٣٤٥", "٦", "fi漢", "'ſ", "ḋ", "̣<", "META", "_START", ">", "123", "456", "78", "'D", "𐞁", "　漢e", "́", "٣٤٥", "٦"]} +{"text": " s, ­
fi're \nEOT　ꟲ३字é漢\"\"!😀🏽ع
s‍méݽ 12345678‍㍿ſ.", "tokens": 56, "pieces": [" ", " s", ",", " ", "­", "
fi", "'re", " \n", "EOT", "　ꟲ", "३", "字", "é漢", "\"\"!😀🏽", "ع", "
s", "‍me", "́İ", "½", " ", " ", "123", "456", "78", "‍㍿", "ſ", "."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " s's…å,ſa\"t½𐞁're\rEOT½'llⅣfi\">\"Ａ½İ-३t", "tokens": 45, "pieces": [" ", " s", "'s", "…a", "̊,", "ſa", "\"t", "½", "𐞁", "'re", "\r", "EOT", "½", "'ll", "Ⅳ", "fi", "\">\"", "Ａ", "½", "İ", "-<", "META", "_START", ">", "३", "t"]} +{"text": "\r\n\r\n<,9Ⅳ३'Re'Re0\n12345678३ḍ̇\u000b\r\n t.\t<|endoftext|>Ａ\r\n\r\n \n\ŕfi'Ss<|fim_prefix|>\r\n\r\n\n!!‍", "tokens": 55, "pieces": ["\r\n\r\n", "<,", "9Ⅳ३", "'Re", "'Re", "0", "\n", "123", "456", "78३", "ḋ", "̣", "\u000b\r\n", " t", ".", "\t", "<|", "endoftext", "|>", "Ａ", "\r\n\r\n \n\r", "́fi", "'S", "s", "<|", "fim", "_prefix", "|>\r\n\r\n\n", "!!‍"]} +{"text": "Ⅳ́é \n​Ⅳع́ ", "tokens": 11, "pieces": ["Ⅳ", "́é", " \n", "​", "Ⅳ", "ع", "́", " "]} +{"text": ">ꟲ\r\n👍🏽٣٤٥٦\"!é​
's#$%½'Re𐞁\r\n\r\né,", "tokens": 41, "pieces": [">ꟲ", "\r\n", "👍🏽", "٣٤٥", "٦", "\"!", "é", "​<", "META", "_START", ">", "
", "'s", "#$%", "½", "'Re", "𐞁", "\r\n\r\n", "é", ","]} +{"text": "İ's👍🏽<|fim_prefix|><|fim_prefix|>'ll#$%'saḍ̇
ſ12345678 'S'VE\n", "tokens": 42, "pieces": ["İ", "'s", "👍🏽<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "ll", "#$%'", "saḋ", "̣", "
ſ", "123", "456", "78", " '", "S", "'VE", "\n"]} +{"text": "३\r\nꟲåéDž
(㋿éfiꟲEOT-a'VE \"'DéⅣ\n'\n
.Ａ!!\" \n👍🏽 🙂…é\r\n\r\n\n<|endoftext|>", "tokens": 66, "pieces": ["३", "\r\n", "ꟲa", "̊éDž", "
", "(㋿", "e", "́fiꟲEOT", "-a", "'VE", " ", "\"'", "Dé", "Ⅳ", "\n", "'\n", "
", ".Ａ", "!!\"", " \n", "👍🏽", " 🙂", "…é", "\r\n\r\n\n", "<|", "endoftext", "|>"]} +{"text": "ß𐞁½,!!­½-😀🏽\r\n\r\n'ſ　‍#$%(,Z,\r\n,tt\u000b'ſ#$%", "tokens": 42, "pieces": ["ß𐞁", "½", ",!!­", "½", "-😀🏽\r\n\r\n", "'ſ", "　", "‍#$%(<", "META", "_START", "><", "EOT", ">,", "Z", ",\r\n", ",tt", "\u000b", "'ſ", "#$%"]} +{"text": "🙂३'ſ\rꟲ", "tokens": 11, "pieces": ["🙂", "३", "'ſ", "\r", "ꟲ"]} +{"text": "ع,'S", "tokens": 3, "pieces": ["ع", ",'", "S"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'VE'VE٣٤٥٦㋿İ'Re‍İ\r\n'VE'ss", "tokens": 25, "pieces": ["'VE", "'VE", "٣٤٥", "٦", "㋿İ", "'Re", "‍İ", "\r\n", "'VE", "'s", "s"]} +{"text": " 'M", "tokens": 2, "pieces": [" '", "M"]} +{"text": "👍🏽 \nع><|fim_prefix|>(㍿", "tokens": 18, "pieces": ["👍🏽", " \n", "ع", "><|", "fim", "_prefix", "|>(㍿"]} +{"text": "½Z", "tokens": 2, "pieces": ["½", "Z"]} +{"text": "'s\r'T\r\nfi\tésEOT \nEOT​Z　12345678ع \né👍🏽…!!'llZ㋿́٣٤٥٦", "tokens": 48, "pieces": ["'s", "\r", "'T", "\r\n", "fi", "\te", "́sEOT", " \n", "EOT", "​Z", "　", "123", "456", "78", "ع", " \n", "e", "́👍🏽", "…", "!!'", "llZ", "㋿́", "٣٤٥", "٦"]} +{"text": "(字'reſd", "tokens": 6, "pieces": ["(字", "'re", "ſd"]} +{"text": "'s…EOT\r\nع　\nſås(m#$%㍿'D", "tokens": 23, "pieces": ["'s", "…EOT", "\r\n", "ع", "　\n", "ſa", "̊s", "(m", "#$%㍿'", "D"]} +{"text": "
('VEꟲ㋿12345678 \n😀🏽'Rea\u000b \nⅣ\u000b漢'S \u000bß٣٤٥٦…३m'VE<|fim_prefix|>", "tokens": 55, "pieces": ["
", "('", "VEꟲ", "㋿", "123", "456", "78", " \n", "😀🏽'", "Rea", "\u000b \n", "Ⅳ", "\u000b漢", "'S", " ", "\u000bß", "٣٤٥", "٦", "…", "३", "m", "'VE", "<|", "fim", "_prefix", "|>"]} +{"text": "ع\t½0fiß<|endoftext|>-'D٣٤٥٦!", "tokens": 28, "pieces": ["ع", "\t", "½0", "fiß", "<|", "endoftext", "|>-'", "D", "٣٤٥", "٦", "!"]} +{"text": "㍿字tꟲ　'D 'ſéea\n३tDžع字'12345678漢0'S\"", "tokens": 34, "pieces": ["㍿字tꟲ", "　", "'D", " ", "'ſ", "e", "́ea", "\n", "३", "tDžع字", "'", "123", "456", "78", "漢", "0", "'S", "\""]} +{"text": "ſ字'sⅣsⅣZ 12345678're½0'T>👍🏽­t'Séé́३,Ａ㍿\r\n>é EOT \n…'VE'\r\n\r\n‍😀🏽", "tokens": 61, "pieces": ["ſ字", "'s", "Ⅳ", "s", "Ⅳ", "Z", " ", " ", "123", "456", "78", "'re", "½0", "'T", ">👍🏽­", "t", "'S", "e", "́e", "́́", "३", ",Ａ", "㍿\r\n", ">e", "́", " ", " EOT", " \n", "…", "'VE", "'\r\n\r\n", "‍😀🏽"]} +{"text": "३!! \n\r\n \n🙂'D-😀🏽fi\rع12345678'.👍🏽३‍\té👍🏽0ꟲ㋿ſå", "tokens": 58, "pieces": ["३", "!!", " \n\r\n \n", "🙂'", "D", "-😀🏽", "fi", "\r", "ع", "123", "456", "78", "'.<", "META", "_START", ">👍🏽", "३", "‍", "\te", "́👍🏽", "0", "ꟲ", "㋿ſa", "̊"]} +{"text": "漢𐞁é", "tokens": 7, "pieces": ["漢𐞁é"]} +{"text": "ſ'Mfi fi'ré'Reß'Re!", "tokens": 15, "pieces": ["ſ", "'M", "fi", " ", " fi", "'re", "́'", "Reß", "'Re", "!"]} +{"text": "'T\rfi­
\r\n\u000bEOTDžfi \n　fis\r½'Re,
٣٤٥٦'T​(ꟲfi　½… ", "tokens": 47, "pieces": ["'T", "\r", "fi", "­", "
\r\n", "\u000bEOTDžfi", " \n", "　fis", "\r", "½", "'Re", ",", "
", "٣٤٥", "٦", "'T", "​(", "ꟲfi", "　", "½", "… "]} +{"text": "㍿👍🏽d!\t\t́12345678", "tokens": 17, "pieces": ["㍿👍🏽", "d", "!", "\t", "\t", "́", "123", "456", "78"]} +{"text": "'D😀🏽'M𐞁Ａ<ꟲꟲع'T,.㍿'T\n'll<|fim_prefix|>'D<|fim_prefix|>12345678e's\tꟲ'\u000b😀🏽İ(\"", "tokens": 67, "pieces": ["'D", "😀🏽'", "M𐞁Ａ", "<ꟲꟲع", "'T", ",.㍿<", "META", "_START", ">'", "T", "\n", "'ll", "<|", "fim", "_prefix", "|>'", "D", "<|", "fim", "_prefix", "|>", "123", "456", "78", "e", "'s", "\tꟲ", "'", "\u000b", "😀🏽", "İ", "(\""]} +{"text": "9'D'ſ𐞁\r\n\r\n'M字㋿å👍🏽sm'll\rfi(😀🏽'ſ𐞁#$%", "tokens": 44, "pieces": ["9", "'D", "'ſ", "𐞁", "\r\n\r\n", "'M", "字", "㋿a", "̊👍🏽", "sm", "'ll", "\r", "fi", "(😀🏽'", "ſ𐞁", "#$%"]} +{"text": " ­'ll's!", "tokens": 5, "pieces": [" ­'", "ll", "'s", "!"]} +{"text": "Džİ 'SＡ​0 …dꟲ'VE9'Re\r\n\r\nEOTꟲḍ̇ mmå-\"‍字ⅣＡ", "tokens": 47, "pieces": ["Džİ", " ", "'S", "Ａ", "​", "0", " ", "…dꟲ", "'VE", "9", "'Re", "\r\n\r\n", "EOTꟲḋ", "̣", " mma", "̊<", "EOT", ">-\"‍", "字", "Ⅳ", "Ａ"]} +{"text": "-'M.d(", "tokens": 4, "pieces": ["-'", "M", ".d", "("]} +{"text": "\t", "tokens": 1, "pieces": ["\t"]} +{"text": "\n👍🏽'Rem<|fim_prefix|>'M字e'Reée'D!!'Re-", "tokens": 26, "pieces": ["\n", "👍🏽'", "Rem", "<|", "fim", "_prefix", "|>'", "M字e", "'Re", "ée", "'D", "!!'", "Re", "-"]} +{"text": "\t#$%Ⅳ'M\r\n İſ ​'ll\u000b'reDžḍ̇e're(ßZ字㍿ſ<㋿<|fim_prefix|><|endoftext|>㋿ß12345678'", "tokens": 63, "pieces": ["\t", "#$%", "Ⅳ", "'M", "\r\n", " İſ", " ", "​'", "ll", "\u000b", "'re", "Džḋ", "̣e", "'re", "(ßZ字", "㍿ſ", "<㋿<", "META", "_START", "><|", "fim", "_prefix", "|><|", "endoftext", "|>㋿", "ß", "123", "456", "78", "'"]} +{"text": "'ſ'D<|endoftext|>sfi'T<é \r\n\r\ndm\"½Dž́'T\t㋿EOTDž fi", "tokens": 40, "pieces": ["'ſ", "'D", "<|", "endoftext", "|>", "sfi", "'T", "<é", " \r\n\r\n", "dm", "\"", "½", "Dž", "́'", "T", "", "\t", "㋿EOTDž", " fi"]} +{"text": "'ReⅣs'VE🙂'D", "tokens": 10, "pieces": ["'Re", "Ⅳ", "s", "'VE", "🙂'", "D"]} +{"text": " å\u000bع\"><'VEé👍🏽 ", "tokens": 17, "pieces": [" ", " a", "̊", "\u000bع", "\"><'", "VEé", "👍🏽", " "]} +{"text": "t!!'ſ!!!\td½ſ,<|fim_prefix|>'s'D12345678>Z…́Džſ\u000bé𐞁\"'T'Re​'D", "tokens": 49, "pieces": ["t", "!!<", "EOT", ">'", "ſ", "!!!", "\td", "½", "ſ", ",<|", "fim", "_prefix", "|>'", "s", "'D", "123", "456", "78", ">Z", "…", "́Džſ", "\u000bé𐞁", "\"'", "T", "'", "Re", "​'", "D"]} +{"text": "Ⅳ…eé,'ſ'Re\r\nd😀🏽-\nß0Z(👍🏽'S12345678m½\"㍿㍿å٣٤٥٦​<|endoftext|>\te \nZ", "tokens": 67, "pieces": ["Ⅳ", "…ee", "́,'", "ſ", "'Re", "\r\n", "d", "😀🏽-\n", "ß", "0", "Z", "(👍🏽'", "S", "123", "456", "78", "m", "½", "\"㍿㍿", "a", "̊", "٣٤٥", "٦", "​<|", "endoftext", "|>", "\te", " \n", "Z"]} +{"text": "'ll\r\nꟲd", "tokens": 6, "pieces": ["'ll", "\r\n", "ꟲd"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿‍s#$%\t\r\n\r\na<|endoftext|>٣٤٥٦åع\n 0㋿\r\n<​Ⅳ,​9!!'<😀🏽 \n", "tokens": 52, "pieces": ["㍿‍", "s", "#$%", "\t\r\n\r\n", "a", "<|", "endoftext", "|>", "٣٤٥", "٦", "a", "̊ع", "\n", " ", " ", "0", "㋿\r\n", "<​", "Ⅳ", ",​", "9", "!!'<😀🏽", " \n"]} +{"text": "''Re　12345678漢\ntḍ̇0'DEOTEOT'VEꟲ-ꟲ­", "tokens": 30, "pieces": ["''", "Re", "　", "123", "456", "78", "漢", "\n", "tḋ", "̣", "0", "'D", "EOTEOT", "'VE", "ꟲ", "-ꟲ", "­"]} +{"text": "0", "tokens": 4, "pieces": ["", "0"]} +{"text": "<ß're'S🙂'Reſ𐞁　0 ㋿㍿0\r\n𐞁\u000bé>👍🏽　
 \nſfi\t\t́Z'Re'D", "tokens": 53, "pieces": ["<ß", "'re", "'S", "🙂'", "Reſ𐞁", "　", "0", " ㋿㍿", "0", "\r\n", "𐞁", "\u000be", "́>👍🏽", "　
 \n", "ſfi", "\t", "\t", "́Z", "'Re", "'D"]} +{"text": "'Re\t#$%<|endoftext|>é\u000b'ḍ̇,tＡ‍éＡß'T9're 're,㍿'D​𐞁a'D\"fi३", "tokens": 53, "pieces": ["'Re", "\t", "#$%<|", "endoftext", "|>", "é", "\u000b", "'ḋ", "̣,", "tＡ", "‍éＡß", "'T", "9", "'re", " ", " '", "re", ",㍿'", "D", "​𐞁a", "'D", "\"fi", "३"]} +{"text": "'S\r\nZ\rd!!  ​\t　", "tokens": 11, "pieces": ["'S", "\r\n", "Z", "\r", "d", "!!", " ", " ", "​", "\t　"]} +{"text": "𐞁'ſt'Re", "tokens": 9, "pieces": ["𐞁", "'ſ", "t", "'Re"]} +{"text": "éd0 \n'ſ \nⅣİ12345678e,Dž漢me字😀🏽", "tokens": 28, "pieces": ["e", "́d", "0", " \n", "'ſ", " \n", "Ⅳ", "İ", "123", "456", "78", "e", ",Dž漢me字", "😀🏽"]} +{"text": "　!!<|endoftext|>'VE'T　字'", "tokens": 25, "pieces": ["　", "!!<|", "endoftext", "|>'", "VE", "'", "T", "", "　字", "'<", "META", "_START", ">"]} +{"text": "'s", "tokens": 1, "pieces": ["'s"]} +{"text": "漢!!'VE\t<|fim_prefix|>'Reem㍿\r\n🙂Ａ'Me٣٤٥٦ꟲ\u000bİ👍🏽३ 👍🏽 'T", "tokens": 58, "pieces": ["漢", "!!'", "VE", "\t", "<|", "fim", "_prefix", "|>'", "Reem", "㍿\r\n", "🙂<", "META", "_START", ">Ａ", "'M", "e", "٣٤٥", "٦", "ꟲ", "\u000bİ", "👍🏽", "३", " ", " 👍🏽", " ", "'T"]} +{"text": "'s!!‍'Re'llt-‍​-s12345678 9ḍ̇", "tokens": 29, "pieces": ["'s", "!!‍'", "Re", "'ll", "t", "-‍​-", "s", "123", "456", "78", " ", " ", "9", "ḋ", "̣"]} +{"text": "é'VE!s\"Dž字s𐞁e\r\n\r\nZé's", "tokens": 25, "pieces": ["e", "́'", "VE", "!<", "EOT", "><", "META", "_START", ">s", "\"Dž字s𐞁e", "\r\n\r\n", "Zé", "'s"]} +{"text": "　\tⅣ<|fim_prefix|>\n12345678🙂<|fim_prefix|>'Sé'Re<|fim_prefix|>　ع\r\nfi𐞁'T'D𐞁́ꟲ…Ａd fi's<9s<|endoftext|>😀🏽Ⅳ٣٤٥٦12345678", "tokens": 90, "pieces": ["　", "\t", "Ⅳ", "<|", "fim", "_prefix", "|>\n", "123", "456", "78", "🙂<|", "fim", "_prefix", "|>'", "Sé", "'Re", "<|", "fim", "_prefix", "|>", "　ع", "\r\n", "fi𐞁", "'T", "'D", "𐞁", "́ꟲ", "…Ａd", " fi", "'s", "<", "9", "s", "<|", "endoftext", "|>😀🏽", "Ⅳ٣٤", "٥٦1", "234", "567", "8"]} +{"text": " 9👍🏽\r㋿>d㍿!\u000b㋿\n'VEdét\u000b>éé👍🏽३ ३😀🏽㍿aＡéİ…m‍", "tokens": 61, "pieces": [" ", "9", "👍🏽\r", "㋿>", "d", "㍿!", "\u000b", "㋿\n", "'VE", "de", "́t", "\u000b", ">e", "́é", "👍🏽", "३", " ", "३", "😀🏽㍿", "aＡéİ", "…m", "‍"]} +{"text": "s(.<|endoftext|> !ḍ̇'Mꟲ'VEDž", "tokens": 28, "pieces": ["s", "(.<|", "endoftext", "|>", " ", "!<", "META", "_START", ">ḋ", "̣'", "Mꟲ", "'VE", "Dž"]} +{"text": "​'TDž \n…㍿\u000b ſt㍿åé½😀🏽é­9👍🏽<|endoftext|>ḍ̇'reⅣ#$%…ſＡ", "tokens": 70, "pieces": ["​'", "TDž", " \n", "…", "㍿", "\u000b ", " ſt", "㍿", "a", "̊é", "½", "😀🏽", "é", "­", "9", "👍🏽<|", "endoftext", "|>", "ḋ", "̣'", "re", "Ⅳ", "#$%", "…ſＡ", ""]} +{"text": "!!!\tß'ſ漢s'Re9<|endoftext|>d'S -'!!​ 字t½\n‍ ḍ̇½३", "tokens": 44, "pieces": ["!!!", "\tß", "'ſ", "漢s", "'Re", "9", "<|", "endoftext", "|>", "d", "'S", " -'!!​", " ", " 字t", "½", "\n", "‍<", "EOT", ">", " ", " ḋ", "̣", "½३"]} +{"text": "'llＡ -​\u000b'VÉ字'ſß३Ⅳſ٣٤٥٦'😀🏽é<.Dž'VE.0<|fim_prefix|>ع…<|endoftext|>", "tokens": 64, "pieces": ["'ll", "Ａ", " ", "-​", "\u000b", "'VE", "́字", "'ſ", "ß", "३Ⅳ", "ſ", "٣٤٥", "٦", "'😀🏽", "é", "<.", "Dž", "'VE", ".", "0", "<|", "fim", "_prefix", "|>", "ع", "…", "<|", "endoftext", "|>"]} +{"text": "'reꟲd٣٤٥٦EOT9'D\n 字😀🏽㋿字m'VE😀🏽-0 -㍿😀🏽's　!!", "tokens": 56, "pieces": ["'re", "ꟲd", "٣٤٥", "٦", "EOT", "9", "'D", "\n", " ", " 字", "😀🏽<", "EOT", ">㋿", "字m", "'VE", "😀🏽-", "0", " -㍿😀🏽'", "s", "　", "!!"]} +{"text": "mt'S.Ａ \n‍३!!<|fim_prefix|>🙂'sZ<|endoftext|>Ａḍ̇.Dž㍿12345678s- 'ś12345678ß　Ⅳ㍿ꟲåＡ#$%字å½m", "tokens": 80, "pieces": ["mt", "'S", ".Ａ", " \n", "‍", "३", "!!<|", "fim", "_prefix", "|>🙂'", "sZ", "<|", "endoftext", "|>", "Ａḋ", "̣.", "Dž", "㍿", "123", "456", "78", "s", "-", " '", "s", "́", "123", "456", "78", "ß", "", "　", "Ⅳ", "㍿ꟲa", "̊Ａ", "#$%", "字a", "̊", "½", "m"]} +{"text": "👍🏽Ⅳ漢<|endoftext|>EOT́\t㋿\r\nع㍿", "tokens": 29, "pieces": ["👍🏽", "Ⅳ", "漢", "<|", "endoftext", "|>", "EOT", "́", "\t", "㋿\r\n", "ع", "㍿"]} +{"text": "　, é12345678Ａfi​0
t\r\n\r\nİDž😀🏽<|endoftext|>'ll\u000b'S'll  ㋿­EOT­­\r", "tokens": 54, "pieces": ["　", ",", " é", "123", "456", "78", "Ａfi", "​", "0", "
t", "\r\n\r\n", "İDž", "😀🏽<|", "endoftext", "|><", "META", "_START", ">'", "ll", "\u000b", "'S", "'ll", " ", " <", "EOT", ">", " ", " ㋿­", "EOT", "­­\r"]} +{"text": "\"\r\n\r\n>́ß ", "tokens": 5, "pieces": ["\"\r\n\r\n", ">́", "ß", " "]} +{"text": "字t'SEOTعZ㋿''ll'sEOTfi", "tokens": 17, "pieces": ["字t", "'S", "EOTعZ", "㋿''", "ll", "'s", "EOTfi"]} +{"text": "!!​👍🏽漢aꟲa\r\n🙂́\tⅣḍ̇Ⅳßd🙂Ⅳé'VEEOT…​㍿㍿\r\n\r\n!!…🙂 tfiⅣ🙂'Reé", "tokens": 67, "pieces": ["!!​👍🏽", "漢aꟲa", "\r\n", "🙂́", "\t", "Ⅳ", "ḋ", "̣", "Ⅳ", "ßd", "🙂", "Ⅳ", "e", "́'", "VEEOT", "…", "​㍿㍿\r\n\r\n", "!!", "…", "🙂", " ", " tfi", "Ⅳ", "🙂'", "Reé"]} +{"text": "
…\tfiḍ̇'Re0'é\u000b", "tokens": 18, "pieces": ["
…", "\tfiḋ", "̣'", "Re", "0", "'e", "́", "\u000b"]} +{"text": "Džs㍿-ſ", "tokens": 9, "pieces": ["Džs", "㍿-", "ſ"]} +{"text": "'VEé <|fim_prefix|>é\r'M", "tokens": 16, "pieces": ["'VE", "e", "́", " ", "<|", "fim", "_prefix", "|>", "e", "́\r", "'M"]} +{"text": "ßfi'ſ'ſ𐞁d", "tokens": 14, "pieces": ["ßfi", "'ſ", "'ſ", "𐞁d"]} +{"text": "'ſ　t\t fi\tEOT<|fim_prefix|>åafi\n!!m'\r\nع…'ſ ३ع'll…é👍🏽漢t'VEé \u000b­…漢<|endoftext|>", "tokens": 73, "pieces": ["'ſ", "　t", "\t ", " <", "EOT", ">fi", "\tEOT", "<|", "fim", "_prefix", "|>", "a", "̊afi", "\n", "!!", "m", "'\r\n", "ع", "…", "'ſ", " ", " ", "३", "ع", "'ll", "…e", "́👍🏽", "漢t", "'VE", "é", " ", "\u000b", "­", "…漢", "<|", "endoftext", "|>"]} +{"text": "😀🏽\r\n12345678 \nⅣ's,\t.9'é­'M😀🏽\rs㍿'M‍å㍿'Msa…ß'ſ9t'Re\tꟲZ 'S're", "tokens": 62, "pieces": ["😀🏽\r\n", "123", "456", "78", " \n", "Ⅳ", "'s", ",", "\t", ".", "9", "'e", "́­'", "M", "😀🏽\r", "s", "㍿'", "M", "‍a", "̊㍿'", "Msa", "…ß", "'ſ", "9", "t", "'Re", "\tꟲZ", " '", "S", "'re"]} +{"text": "EOT", "tokens": 2, "pieces": ["EOT"]} +{"text": "'S'D　!9 ­३é,　'ReＡßs", "tokens": 18, "pieces": ["'S", "'D", "　", "!", "9", " ", "­", "३", "e", "́,", "　", "'Re", "Ａßs"]} +{"text": " 'ſ're#$%0'De'عDž'Dعfi😀🏽Ⅳmå'sd\r\n\r\n'll\r'S\r\n\r\n", "tokens": 39, "pieces": [" ", " '", "ſ", "'re", "#$%<", "EOT", ">", "0", "'D", "e", "'عDž", "'D", "عfi", "😀🏽", "Ⅳ", "ma", "̊'", "sd", "\r\n\r\n", "'ll", "\r", "'S", "\r\n\r\n"]} +{"text": "\n\r\n\r\n<㍿½\"é's​­…㍿ꟲ३(𐞁㋿\r\n\r\n é
字å9\"aå\"", "tokens": 55, "pieces": ["\n\r\n\r\n", "<㍿", "½", "\"e", "́'", "s", "​­", "…", "㍿", "ꟲ", "", "३", "(𐞁", "㋿\r\n\r\n", " ", " e", "́", "
字a", "̊", "9", "\"aa", "̊\""]} +{"text": ".'ſ'ſZt­ 'M-İḍ̇'S12345678#$%0'D㋿İfi\rEOTm'DZßع", "tokens": 42, "pieces": [".'", "ſ", "'ſ", "Zt", "­", " ", " '", "M", "-İḋ", "̣'", "S", "123", "456", "78", "#$%", "0", "'D", "㋿İfi", "\r", "EOTm", "'D", "Zßع"]} +{"text": "9½Dž\r\n\r\nZ!!‍Ａ!\t㋿-åꟲ…fiém 'M'sa­㍿,😀🏽🙂 é\r\n👍🏽's'", "tokens": 60, "pieces": ["9½", "Dž", "\r\n\r\n", "Z", "!!‍", "Ａ", "!", "\t", "㋿-", "a", "̊ꟲ", "…", "fiém", " ", " '", "M", "'s", "a", "­㍿,😀🏽🙂", " e", "́\r\n", "👍🏽'", "s", "'"]} +{"text": "­'Tİ \n'ß\"…'VE'Mé​-<|endoftext|>\r\n\r\n​३'ſ.-a\"", "tokens": 33, "pieces": ["­'", "Tİ", " \n", "'ß", "\"", "…", "'VE", "'M", "e", "́​-<|", "endoftext", "|>\r\n\r\n", "​", "३", "'ſ", ".-", "a", "\""]} +{"text": "\r\n12345678ßm'D
٣٤٥٦ḍ̇​ß,عs9<|fim_prefix|><|fim_prefix|><12345678'S­Ze9", "tokens": 53, "pieces": ["\r\n", "123", "456", "78", "ßm", "'D", "
", "٣٤٥", "٦", "ḋ", "̣​", "ß", ",", "عs", "9", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><", "123", "456", "78", "'S", "­Ze", "9"]} +{"text": "'T\r\n'S\r\n\r\nꟲ,- 　å12345678EOT㋿ \r\n\r\nⅣ<|endoftext|>t#$%👍🏽ßaé \n㋿", "tokens": 48, "pieces": ["'T", "\r\n", "'S", "\r\n\r\n", "ꟲ", ",-", " ", "　a", "̊", "123", "456", "78", "EOT", "㋿", " \r\n\r\n", "Ⅳ", "<|", "endoftext", "|>", "t", "#$%👍🏽", "ßaé", " \n", "㋿"]} +{"text": "'s", "tokens": 1, "pieces": ["'s"]} +{"text": "ß字ع#$%e.ee\r\u000bm\n​字\u000bß\r\n", "tokens": 20, "pieces": ["ß字ع", "#$%", "e", ".ee", "\r", "", "\u000bm", "\n", "​字", "\u000bß", "\r\n"]} +{"text": "­
m'lle-Ａ'MＡ😀🏽!!٣٤٥٦🙂-<|endoftext|>'Re'ſ'S12345678'D\t'll'safi ", "tokens": 52, "pieces": ["­", "
m", "'ll", "e", "-Ａ", "'M", "Ａ", "😀🏽!!", "٣٤٥", "٦", "🙂-<|", "endoftext", "|>'", "Re", "'ſ", "'S", "123", "456", "78", "'D", "\t", "'ll", "'s", "afi", " "]} +{"text": "aDž　e'Sİ
d½s'VE　…m'S…\nꟲ
ꟲ'Re'T…‍#$%å𐞁 'Re<|endoftext|>'VEEOT'Tꟲeé", "tokens": 66, "pieces": ["aDž", "　e", "'S", "İ", "
d", "½", "s", "'VE", "　", "…", "m", "'S", "…\n", "ꟲ", "
ꟲ", "'Re", "'T", "…", "‍#$%", "a", "̊𐞁", " '", "Re", "<|", "endoftext", "|>'", "VEEOT", "'T", "ꟲeé"]} +{"text": "å🙂\t'S\t𐞁ⅣdaEOT\t<|fim_prefix|>Ⅳ½(12345678fi're漢­㍿#$%\n ,>\n<|endoftext|>å(ꟲ're \nḍ̇", "tokens": 67, "pieces": ["a", "̊🙂", "\t", "'S", "\t𐞁", "Ⅳ", "daEOT", "\t", "<|", "fim", "_prefix", "|>", "Ⅳ½", "(", "123", "456", "78", "fi", "'re", "漢", "­㍿#$%\n", " ", " ,>\n", "<|", "endoftext", "|>", "a", "̊(", "ꟲ", "'re", " \n", "ḋ", "̣"]} +{"text": "\r\n", "tokens": 1, "pieces": ["\r\n"]} +{"text": "😀🏽
　.\n👍🏽'‍\n12345678t \n'VE\"", "tokens": 30, "pieces": ["😀🏽", "
", "　", ".\n", "👍🏽'‍\n", "123", "456", "78", "t", " \n", "'VE", "\"<", "META", "_START", ">"]} +{"text": "​👍🏽😀🏽.<|endoftext|>", "tokens": 19, "pieces": ["​👍🏽😀🏽.<|", "endoftext", "|>"]} +{"text": "9'D…é.㍿!dm 'Rem's漢Džd​9s\r　\r\n字Ａ'DⅣ㋿𐞁㋿<|endoftext|>'M漢a12345678<", "tokens": 62, "pieces": ["9", "'D", "…é", ".㍿!", "dm", " ", "'Re", "m", "'s", "漢Džd", "​", "9", "s", "\r　\r\n", "字Ａ", "'D", "Ⅳ", "㋿𐞁", "㋿<|", "endoftext", "|>'", "M漢a", "123", "456", "78", "<"]} +{"text": "½'m", "tokens": 2, "pieces": ["½", "'m"]} +{"text": "'Re　
<'D
 t<ß \n'VEé',e<|fim_prefix|>३‍<|fim_prefix|>fi\t\r\n🙂9\"<|fim_prefix|>\r\n s…ع", "tokens": 57, "pieces": ["'Re", "　", "
", "<'", "D", "
 ", " t", "<ß", " \n", "'VE", "é", "',", "e", "<|", "fim", "_prefix", "|>", "३", "‍<|", "fim", "_prefix", "|>", "fi", "\t\r\n", "🙂", "9", "\"<|", "fim", "_prefix", "|>\r\n", "", " ", " s", "…ع"]} +{"text": "'s\r\n\t!!s'VEs'Dßå\nd😀🏽EOT \n　 \n漢eé​🙂\">Z\t >", "tokens": 44, "pieces": ["'s", "\r\n", "\t", "!!", "s", "'VE", "s", "'D", "ßa", "̊\n", "d", "😀🏽", "EOT", " \n　 \n", "漢ee", "́<", "EOT", ">​🙂\">", "Z", "\t ", " >"]} +{"text": "‍sİé́<|fim_prefix|>🙂'ſfi'Mعt٣٤٥٦ ſDž🙂👍🏽‍-mⅣ
́ 'S'漢", "tokens": 61, "pieces": ["‍sİe", "́́<|", "fim", "_prefix", "|>🙂'", "ſfi", "'M", "عt", "", "٣٤٥", "٦", " ſDž", "🙂👍🏽‍-", "m", "Ⅳ", "
", "́", " '", "S", "'漢"]} +{"text": "'MⅣfiḍ̇eſ㍿漢0٣٤٥٦-m's\r\n'Dع　ꟲ字字", "tokens": 38, "pieces": ["'M", "Ⅳ", "fiḋ", "̣eſ", "㍿漢", "0٣٤", "٥٦", "-m", "'s", "\r\n", "'D", "ع", "　ꟲ字字"]} +{"text": "ḍ̇ḍ̇‍٣٤٥٦e's!", "tokens": 23, "pieces": ["ḋ", "̣ḋ", "̣‍", "٣٤٥", "٦", "e", "'s", "!"]} +{"text": " 're'ſ.‍ꟲ'llå'ſ'ſİDž>­　(Ａt<", "tokens": 35, "pieces": [" '", "re", "'ſ", ".‍", "ꟲ", "'ll", "a", "̊'", "ſ", "'ſ", "İDž", ">­", "　", "(<", "META", "_START", ">Ａt", "<"]} +{"text": "　-''D字9", "tokens": 10, "pieces": ["　", "-''", "D", "字", "9"]} +{"text": "(\r 's👍🏽é>\t…'Tعḍ̇'T<ꟲ'reDž½- 'ſ(t㍿\r \n-́", "tokens": 46, "pieces": ["(\r", " ", "'s", "👍🏽", "e", "́>", "\t", "…", "'T", "عḋ", "̣'", "T", "<ꟲ", "'re", "Dž", "½", "-", " ", "'ſ", "(t", "㍿\r", " \n", "-́"]} +{"text": "­åEOT<　ßå😀🏽t'ſt३\r\n\r\n'Reع \r
ꟲs<|fim_prefix|> ­", "tokens": 47, "pieces": ["­a", "̊EOT", "<", "　", "ßa", "̊😀🏽", "t", "'ſ", "t", "३", "\r\n\r\n", "'Re", "ع", " \r", "
ꟲs", "<|", "fim", "_prefix", "|>", " ­"]} +{"text": "m\nEOT\u000b'VE,'D३㋿t३<|endoftext|>fi㍿ꟲmEOT漢㍿Z\"\rfi\r\nＡ\r漢İ👍🏽३,", "tokens": 61, "pieces": ["m", "\n", "EOT", "\u000b", "'VE", ",'", "D", "३", "㋿t", "३", "<|", "endoftext", "|>", "fi", "㍿ꟲmEOT漢", "㍿Z", "\"\r", "fi", "\r\n", "Ａ", "\r", "漢İ", "👍🏽", "३", ","]} +{"text": " 👍🏽\u000b-\r\"\r\nsＡ Dž'VE", "tokens": 23, "pieces": [" ", " 👍🏽", "\u000b", "-<", "EOT", ">\r", "\"\r\n", "sＡ", " ", " Dž", "'VE"]} +{"text": "ꟲ--,eع㍿…ḍ̇\nZ \n\"sع \r\nſ'D
Z'M(", "tokens": 30, "pieces": ["ꟲ", "--,", "eع", "㍿", "…ḋ", "̣\n", "Z", " \n", "\"sع", " \r\n", "ſ", "'D", "
Z", "'M", "("]} +{"text": "'M'Re Ａ𐞁ḍ̇,é\r𐞁…　Z.\na٣٤٥٦12345678é'T", "tokens": 42, "pieces": ["'M", "'Re", " ", " Ａ𐞁ḋ", "̣,", "é", "\r", "𐞁", "…", "　Z", ".\n", "a", "٣٤٥", "٦12", "345", "678", "e", "́'", "T"]} +{"text": "'S'T́", "tokens": 3, "pieces": ["'S", "'T", "́"]} +{"text": "0😀🏽­𐞁sſé 'MⅣEOT\u000b", "tokens": 23, "pieces": ["0", "😀🏽­", "𐞁sſé", " ", " '", "M", "Ⅳ", "EOT", "\u000b"]} +{"text": "Z<fi're9…fi\n!!\r!å9\r\n\r\nZEOTß\"12345678…㍿", "tokens": 31, "pieces": ["Z", "<fi", "'re", "9", "…fi", "\n", "!!\r", "!a", "̊", "9", "\r\n\r\n", "ZEOTß", "\"", "123", "456", "78", "…", "㍿"]} +{"text": "\t\r\n\r\n　Ⅳ \u000bꟲt'Sſꟲ'ſ\r\n
 ㋿'M\u000bfi㍿\né0!!㍿", "tokens": 46, "pieces": ["\t\r\n\r\n", "　", "Ⅳ", " ", "\u000bꟲt", "'S", "ſ", "ꟲ", "'ſ", "\r\n", "
", " ㋿'", "M", "\u000bfi", "㍿\n", "e", "́", "0", "!!㍿"]} +{"text": "́Ａ👍🏽 \n'Tع!　're'reİ'Dß,ß\u000b🙂㍿!!㍿
'T½t'M'll<|fim_prefix|>s'!t㋿🙂‍é\r\n\r\n㍿", "tokens": 64, "pieces": ["́Ａ", "👍🏽", " \n", "'T", "ع", "!", "　", "'re", "'re", "İ", "'D", "ß", ",ß", "\u000b", "🙂㍿!!㍿", "
", "'T", "½", "t", "'M", "'ll", "<|", "fim", "_prefix", "|>", "s", "'<", "META", "_START", ">!", "t", "㋿🙂‍", "é", "\r\n\r\n", "㍿"]} +{"text": "#$%ßEOT…½Dž　٣٤٥٦ \n'M,𐞁㋿s　", "tokens": 31, "pieces": ["#$%", "ßEOT", "…", "½", "Dž", "　", "٣٤٥", "٦", " \n", "'M", ",𐞁", "㋿s", "　"]} +{"text": "0' \nḍ̇ßꟲ<\"
<|endoftext|>漢́amEOT‍Dž 🙂👍🏽 a", "tokens": 44, "pieces": ["0", "'", " \n", "ḋ", "̣ßꟲ", "<\"", "
", "<|", "endoftext", "|><", "META", "_START", ">漢", "́amEOT", "‍Dž", " ", " 🙂👍🏽", " a"]} +{"text": "½'ſå\t­'Daꟲ👍🏽fi㋿!!EOTé𐞁m👍🏽
 's'Mß're's \n 's\"\u000b'T\r\n", "tokens": 57, "pieces": ["½", "'ſ", "a", "̊", "\t", "­'", "Daꟲ", "👍🏽", "fi", "㋿!!", "EOTe", "́𐞁m", "👍🏽", "
", " ", "'s", "'M", "ß", "'re", "'s", " \n", " ", " '", "s", "\"", "\u000b", "'T", "\r\n"]} +{"text": "­ع́ ٣٤٥٦'Sſ 9٣٤٥٦'T 漢٣٤٥٦-'T!!EOT​<|endoftext|>Ⅳ0\" ꟲ㋿
'ſ<'Mع", "tokens": 72, "pieces": ["­ع", "́", " ", "٣٤٥", "٦", "'S", "ſ", " ", "9٣٤", "٥٦", "'T", " 漢", "٣٤٥", "٦", "-'", "T", "!!", "EOT", "​<|", "endoftext", "|>", "Ⅳ0", "\"", " ꟲ", "㋿", "
", "'ſ", "<'", "Mع"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": "-!,'ll
́　　.,.İ'\t#$%s're'VE(­\u000b!​<|endoftext|>'VE'D\r\n\u000bＡ'SDž", "tokens": 48, "pieces": ["-!,'", "ll", "
", "́", "　", "　", ".,.", "İ", "'", "\t", "#$%", "s", "'", "re", "'VE", "(­", "\u000b", "!<", "EOT", ">​<|", "endoftext", "|>'", "VE", "'D", "\r\n", "\u000bＡ", "'S", "Dž"]} +{"text": "
\u000b\tß(<|fim_prefix|>-　 ㍿😀🏽'll'ſ", "tokens": 27, "pieces": ["
\u000b", "\tß", "(<|", "fim", "_prefix", "|>-", "　", " ㍿😀🏽'", "ll", "'ſ"]} +{"text": "½'३😀🏽((…'Ts>å", "tokens": 17, "pieces": ["½", "'", "३", "😀🏽((", "…", "'T", "s", ">a", "̊"]} +{"text": "e½Z👍🏽ḍ̇12345678<|fim_prefix|>३\"s>\t!!'re'llDž<|endoftext|>ſ́'ll㋿0d😀🏽Dž", "tokens": 63, "pieces": ["e", "½", "Z", "👍🏽", "ḋ", "̣<", "EOT", ">", "123", "456", "78", "<|", "fim", "_prefix", "|>", "३", "\"s", ">", "\t", "!!'", "re", "'ll", "Dž", "<|", "endoftext", "|>", "ſ", "́'", "ll", "㋿", "0", "d", "😀🏽", "Dž"]} +{"text": "ḍ̇字'ſ<|fim_prefix|>字t'S#$%ßa'M　'T\u000b'M­ＡⅣ'< eé!!fi'reéé !ꟲ!!", "tokens": 53, "pieces": ["ḋ", "̣字", "'ſ", "<|", "fim", "_prefix", "|>", "字t", "'S", "#$%", "ßa", "'M", "　", "'T", "\u000b", "'M", "­Ａ", "Ⅳ", "'<", " ", " ee", "́!!", "fi", "'re", "e", "́é", " ", "!<", "EOT", ">ꟲ", "!!"]} +{"text": "İ㋿🙂\r٣٤٥٦عfi३Ⅳ'Re\u000b'VE‍🙂'D ३\t'ſ½🙂9'll٣٤٥٦ſ­å'S字.eéEOT\n'T !é字", "tokens": 72, "pieces": ["İ", "㋿🙂\r", "٣٤٥", "٦", "عfi", "३Ⅳ", "'Re", "\u000b", "'VE", "‍🙂'", "D", " ", "३", "\t", "'ſ", "½", "🙂", "9", "'ll", "٣٤٥", "٦", "ſ", "­a", "̊'", "S字", ".eéEOT", "\n", "'T", " ", " !", "e", "́字"]} +{"text": "́fi.'S\t'VEDž㍿ \"<ꟲt-𐞁", "tokens": 24, "pieces": ["́fi", ".'", "S", "\t", "'VE", "Dž", "㍿", " ", " \"<", "ꟲt", "-𐞁"]} +{"text": "9Ａ\u000bZＡéſſ\r\n\r\nꟲe !!\r\n\r\nİ", "tokens": 28, "pieces": ["9", "Ａ", "\u000bZＡe", "́<", "EOT", ">ſſ", "\r\n\r\n", "ꟲe", " ", "!!\r\n\r\n", "İ"]} +{"text": "fi½", "tokens": 3, "pieces": ["fi", "½"]} +{"text": "'s 　…㋿'D'Re　'S\r\n's 9's­!İsEOTs12345678İ's'S😀🏽　fié", "tokens": 51, "pieces": ["'s", " 　", "…", "㋿<", "e", "́​,>'", "D", "'Re", "", "　", "'S", "\r\n", "'s", " ", " ", "9", "'s", "­!", "İsEOTs", "123", "456", "78", "İ", "'s", "'S", "😀🏽", "　fie", "́"]} +{"text": "!!\r\n\r\n'　ß
>…<|endoftext|>'D<|endoftext|>字!!.😀🏽'sß­'Re'VE's\t 字'ſ.(", "tokens": 47, "pieces": ["!!\r\n\r\n", "'", "　ß", "
", ">", "…", "<|", "endoftext", "|>'", "D", "<|", "endoftext", "|>", "字", "!!.😀🏽'", "sß", "­'", "Re", "'VE", "'s", "\t", " 字", "'ſ", ".("]} +{"text": " ع(", "tokens": 3, "pieces": [" ", " ع", "("]} +{"text": "字EOT.\r\nfi!ꟲ>.\r", "tokens": 12, "pieces": ["字EOT", ".\r\n", "fi", "!ꟲ", ">.\r"]} +{"text": " <'M0'VE ㍿…👍🏽é>9 漢12345678900EOT\r", "tokens": 35, "pieces": [" ", "<'", "M", "0", "'VE", " ", "㍿", "…", "👍🏽", "é", ">", "9", " ", " 漢", "123", "456", "789", "00", "EOT", "\r"]} +{"text": "'S,'ſ.<|fim_prefix|><|endoftext|>'ſ'sß0Ａ\r9", "tokens": 26, "pieces": ["'S", ",'", "ſ", ".<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "ſ", "'s", "ß", "0", "Ａ", "\r", "9"]} +{"text": "عDžEOT!(👍🏽", "tokens": 12, "pieces": ["عDžEOT", "!(👍🏽"]} +{"text": "<😀🏽'Ａfi \r\n0Dž-'MDž‍0ſ'll''re\n­ \nع٣٤٥٦👍🏽s
ßfi0　३12345678​𐞁å\r\n\r\n", "tokens": 79, "pieces": ["<😀🏽'", "Ａfi", " \r\n", "0", "Dž", "-'", "MDž", "‍<", "META", "_START", ">", "0", "ſ", "'", "ll", "''", "re", "\n", "­", " \n", "ع", "٣٤٥", "٦", "👍🏽", "s", "
ßfi", "0", "　", "३12", "345", "678", "​", "𐞁a", "̊\r\n\r\n"]} +{"text": "'S-a​'reéß𐞁½ 'SZ漢fi　", "tokens": 19, "pieces": ["'S", "-a", "​'", "ree", "́ß𐞁", "½", " '", "SZ漢fi", "　"]} +{"text": "\t㋿'VE<|endoftext|>", "tokens": 13, "pieces": ["\t", "㋿'", "VE", "<|", "endoftext", "|>"]} +{"text": "<#$%#$% a9EOT𐞁a \r\n\r\nt<'s　 \n-ḍ̇fi٣٤٥٦åaådfi 0…-", "tokens": 55, "pieces": ["<#$%#$%", " a", "9", "EOT𐞁a", " \r\n\r\n", "t", "<'", "s", "　 \n", "-ḋ", "̣fi", "٣٤٥", "٦", "a", "̊<", "EOT", ">aa", "̊dfi", " ", " ", "0", "…", "-"]} +{"text": "­😀🏽\"'ع👍🏽12345678's 'll🙂12345678<|fim_prefix|>😀🏽<.'M", "tokens": 45, "pieces": ["­😀🏽\"'<", "EOT", ">ع", "👍🏽", "123", "456", "78", "'s", " ", " '", "ll", "🙂", "123", "456", "78", "<|", "fim", "_prefix", "|>😀🏽<.'", "M"]} +{"text": ". ½.\u000b㍿d𐞁　12345678‍\n'DＡ", "tokens": 23, "pieces": [".", " ", "½", ".", "\u000b", "㍿d𐞁", "　", "123", "456", "78", "‍\n", "'D", "Ａ"]} +{"text": "½a…😀🏽EOT'll-", "tokens": 13, "pieces": ["½", "a", "…", "😀🏽", "EOT", "'ll", "-"]} +{"text": "\tＡ‍(😀🏽 \nعꟲ㍿(a!!㍿", "tokens": 29, "pieces": ["\t", "Ａ", "‍(😀🏽", " \n", "عꟲ", "㍿(", "a", "!!㍿"]} +{"text": "́🙂'T\u000b!!a𐞁\te　\",!!", "tokens": 16, "pieces": ["́🙂'", "T", "\u000b", "!!", "a𐞁", "\te", "　", "\",!!"]} +{"text": "\r\n\r\n'll👍🏽ts㋿m#$%.tＡ😀🏽㍿ 'ſعſ12345678‍ſfi>३é's'Ms\t\r\n\r\nDžع", "tokens": 62, "pieces": ["\r\n\r\n", "'ll", "👍🏽", "ts", "㋿m", "#$%.", "tＡ", "😀🏽㍿", " ", " '", "ſعſ", "123", "456", "78", "‍<", "EOT", ">ſfi", ">", "३", "e", "́'", "s", "'M", "s", "\t\r\n\r\n", "Džع"]} +{"text": "!<|fim_prefix|>㍿", "tokens": 10, "pieces": ["!<|", "fim", "_prefix", "|>㍿"]} +{"text": "-漢é tm
,åß,'S", "tokens": 14, "pieces": ["-漢é", " tm", "
", ",a", "̊ß", ",'", "S"]} +{"text": "\r\n\r\n३㍿!!", "tokens": 7, "pieces": ["\r\n\r\n", "३", "㍿!!"]} +{"text": "'D'smⅣé𐞁ꟲ'12345678EOTå'ſ12345678 \n'S㋿ \néⅣع👍🏽́fi
å é'Dſ-Z'", "sm", "Ⅳ", "é𐞁ꟲ", "'", "123", "456", "78", "EOTa", "̊'", "ſ", "123", "456", "78", " \n", "'S", "㋿", " \n", "é", "Ⅳ", "ع", "👍🏽́", "fi", "
a", "̊", " e", "́'", "Dſ", "-Z", "<|endoftext|>s½,ꟲ\u000b9", "tokens": 30, "pieces": ["etſ", "9", "'re", "㋿\r\n", "½0३", "<|", "endoftext", "|>", "s", "½", ",ꟲ", "\u000b", "9"]} +{"text": "å\r\nsſ字Ⅳ​Ａ­٣٤٥٦𐞁!EOTⅣfi\nßt's­'ll…d", "tokens": 43, "pieces": ["a", "̊\r\n", "sſ字", "Ⅳ", "​Ａ", "­", "٣٤٥", "٦", "𐞁", "!EOT", "Ⅳ", "fi", "\n", "ßt", "'s", "­'", "ll", "…d"]} +{"text": " ḍ̇\r\n>\r\n'll\r\n½Ⅳ㋿,Dž'MDž㋿Ａ'VE\"\t'så…㋿é", "tokens": 41, "pieces": [" ", " ḋ", "̣\r\n", ">\r\n", "'ll", "\r\n", "½Ⅳ", "㋿,", "Dž", "'M", "Dž", "㋿Ａ", "'VE", "\"", "\t", "'s", "a", "̊", "…", "㋿é"]} +{"text": "
‍ésⅣ👍🏽s\r\n\r\n'VE\nḍ̇EOTd.  !!'T", "tokens": 34, "pieces": ["
", "‍e", "́s", "Ⅳ", "👍🏽", "s", "\r\n\r\n", "'VE", "\n", "ḋ", "̣EOTd", ".", "  ", " !!'", "T"]} +{"text": "'Sḍ̇.\r\n\r\n­३　­!㍿", "tokens": 16, "pieces": ["'S", "ḋ", "̣.\r\n\r\n", "­", "३", "　", "­!㍿"]} +{"text": "'VE‍㍿\naå'S.", "tokens": 14, "pieces": ["'VE", "‍㍿\n", "aa", "̊'", "S", "."]} +{"text": "åDžſ'T😀🏽\r\n\r\n\n'SEOT\r\n\r\nḍ̇İ9 ㋿'Re㋿Dž''re'Ma㋿'T!a,\"\r\n\r\n\na'VE0'Reé", "tokens": 57, "pieces": ["a", "̊Džſ", "'T", "😀🏽\r\n\r\n\n", "'S", "EOT", "\r\n\r\n", "ḋ", "̣İ", "9", " ", "㋿'", "Re", "㋿Dž", "''", "re", "'M", "a", "㋿'", "T", "!a", ",\"\r\n\r\n\n", "a", "'VE", "0", "'Re", "é"]} +{"text": "tt\r\nDžm'a,<🙂\u000b­.‍9EOT<\rع\r
9! \tİع٣٤٥٦👍🏽Z'ſꟲåİ", "tokens": 60, "pieces": ["tt", "\r\n", "Džm", "'<", "EOT", ">a", ",<🙂", "\u000b", "­.‍", "9", "EOT", "<\r", "ع", "\r", "
", "9", "!", " ", "\tİع", "", "٣٤٥", "٦", "👍🏽", "Z", "'ſ", "ꟲa", "̊İ"]} +{"text": "\n\u000bfi<|fim_prefix|>, t'M-a\r\n\r\néⅣ'reＡfi　\r\n", "tokens": 29, "pieces": ["\n", "\u000bfi", "<|", "fim", "_prefix", "|>,", " ", " t", "'M", "-a", "\r\n\r\n", "é", "Ⅳ", "'re", "Ａfi", "　\r\n"]} +{"text": "'sع!! \nعm👍🏽fiß ́Ａ(<㋿'ſ'D!\t漢's­🙂,", "tokens": 39, "pieces": ["'s", "ع", "!!", " \n", "عm", "👍🏽", "fiß", " ́", "Ａ", "(<㋿'", "ſ", "'D", "!", "\t漢", "'s", "­🙂,"]} +{"text": "́<|fim_prefix|>å12345678>½'S‍ع😀🏽'T‍12345678\nfi", "tokens": 39, "pieces": ["́<|", "fim", "_prefix", "|>", "a", "̊", "123", "456", "78", ">", "½", "'S", "‍ع", "😀🏽'", "T", "‍<", "EOT", ">", "123", "456", "78", "\n", "fi"]} +{"text": "\r\n\r\n DžZfi'ſ<|endoftext|> \n漢09\r'VE😀🏽Z.ß'ſ, \n𐞁\u000b'Re<|endoftext|>éß", "tokens": 56, "pieces": ["\r\n\r\n", "", " DžZfi", "'ſ", "<|", "endoftext", "|>", " \n", "漢", "09", "\r", "'VE", "😀🏽", "Z", ".ß", "'ſ", ",", " \n", "𐞁", "\u000b", "'Re", "<|", "endoftext", "|>", "éß"]} +{"text": "'D㍿'re'ſ's\naé​d>\ré३Ａt​ \n-½ \nfiİfi字㍿'VE'D!Z\r\n\u000b'ſ", "tokens": 52, "pieces": ["'D", "㍿'", "re", "'ſ", "'s", "\n", "ae", "́<", "META", "_START", ">​", "d", ">\r", "e", "́", "३", "Ａt", "​", " \n", "-", "½", " \n", "fiİfi字", "㍿'", "VE", "'D", "!Z", "\r\n", "\u000b", "'ſ"]} +{"text": "🙂'VEſ👍🏽İ're­字( \n", "tokens": 18, "pieces": ["🙂'", "VEſ", "👍🏽", "İ", "'re", "­字", "(", " \n"]} +{"text": "a!عDža​ 😀🏽…åع'M­d‍\"", "tokens": 23, "pieces": ["a", "!عDža", "​", " 😀🏽", "…a", "̊ع", "'M", "­d", "‍\""]} +{"text": "9\r\n\"<> \n…(Ⅳ'M'S
'VE㋿>­٣٤٥٦'M0…字…", "tokens": 38, "pieces": ["9", "\r\n", "\"<<", "EOT", ">>", " \n", "…", "(", "Ⅳ", "'M", "'S", "
", "'VE", "㋿>­", "٣٤٥", "٦", "'M", "0", "…字", "…"]} +{"text": "'Re!­'३　d​-\u000b", "tokens": 11, "pieces": ["'Re", "!­'", "३", "　d", "​-", "\u000b"]} +{"text": "½'re'VE…Z9ꟲ", "tokens": 11, "pieces": ["½", "'re", "'VE", "…Z", "9", "ꟲ"]} +{"text": "åⅣd👍🏽Ⅳ㍿dd,३́'Re<|fim_prefix|>\u000b \n>.", "tokens": 37, "pieces": ["a", "̊", "Ⅳ", "d", "👍🏽", "Ⅳ", "㍿dd", ",", "३", "́'", "Re", "<|", "fim", "_prefix", "|>", "\u000b", "", " \n", ">."]} +{"text": "𐞁İ㋿\u000b'Ḿ \nZ‍é't0éZＡ9३字fi", "tokens": 31, "pieces": ["𐞁İ", "㋿", "\u000b", "'M", "́", " \n", "Z", "‍e", "́'", "t", "0", "e", "́ZＡ", "9३", "字fi"]} +{"text": "'ll'Sİ(́s­字\r\n\r\nEOT'👍🏽'İ…>é'll 𐞁ݽ字'T'D🙂…sⅣé٣٤٥٦\" e\n", "tokens": 60, "pieces": ["'ll", "'S", "İ", "(́", "s", "­字", "\r\n\r\n", "EOT", "'👍🏽'", "İ", "…", ">é", "'ll", " ", " 𐞁İ", "½", "字", "'T", "'D", "🙂", "…s", "Ⅳ", "e", "́", "٣٤٥", "٦", "\"", " e", "\n"]} +{"text": " 字a(\r\nعⅣfi'M!d😀🏽12345678", "tokens": 20, "pieces": [" ", " 字a", "(\r\n", "ع", "Ⅳ", "fi", "'M", "!d", "😀🏽", "123", "456", "78"]} +{"text": "㍿<ß'VE>d's३'llm…​(ꟲ!'reé漢12345678're\u000b're'M'D", "tokens": 33, "pieces": ["㍿<", "ß", "'VE", ">d", "'s", "३", "'ll", "m", "…", "​(", "ꟲ", "!'", "ree", "́漢", "123", "456", "78", "'re", "\u000b", "'re", "'M", "'D"]} +{"text": "e​ae!<ع", "tokens": 12, "pieces": ["e", "​", "a", "e", "!<", "ع"]} +{"text": "​字e𐞁EOTaſḍ̇#$%　\"0\u000bſßm字… \n<|fim_prefix|>åEOT", "tokens": 47, "pieces": ["​字e𐞁EOTaſḋ", "̣#$%", "　", "\"", "0", "\u000bſßm", "字", "… \n", "<|", "fim", "_prefix", "|>", "a", "̊EOT"]} +{"text": "#$%!!Dž½३'VE.!e's Dž#$%Z'D​-🙂३'VEs'S,", "tokens": 35, "pieces": ["#$%!!", "Dž", "½३", "'VE", ".!", "e", "'s", " ", "Dž", "#$%", "Z", "'D", "​-🙂", "३", "'VE", "s", "'S", ","]} +{"text": "-'VE<|fim_prefix|>'<ꟲfi'T٣٤٥٦عſ字字fi\t́字", "tokens": 34, "pieces": ["-'", "VE", "<|", "fim", "_prefix", "|>'<", "ꟲfi", "'T", "٣٤٥", "٦", "عſ字字fi", "\t", "́字"]} +{"text": "<'M's<|endoftext|>d(👍🏽٣٤٥٦字𐞁.t<漢'ſ\n​é<|endoftext|>'re9𐞁", "tokens": 55, "pieces": ["<'", "M", "'s", "<|", "endoftext", "|>", "d", "(👍🏽", "٣٤٥", "٦", "字𐞁", ".t", "<漢", "'ſ", "\n", "​e", "́<|", "endoftext", "|>'", "re", "9", "𐞁"]} +{"text": "\"ع३EOT", "tokens": 6, "pieces": ["\"ع", "३", "EOT"]} +{"text": "'ś\r!.,'M", "tokens": 7, "pieces": ["'s", "́\r", "!.,'", "M"]} +{"text": ">,<|endoftext|>9!!㋿½Ⅳ\r\n\r\nd'T9𐞁İİ\n­㍿​EOT‍\"'M><|endoftext|> \n\r", "tokens": 47, "pieces": [">,<|", "endoftext", "|>", "9", "!!㋿", "½Ⅳ", "\r\n\r\n", "d", "'T", "9", "𐞁İİ", "\n", "­㍿​", "EOT", "‍\"'", "M", "><|", "endoftext", "|>", " \n\r"]} +{"text": "s漢漢　٣٤٥٦", "tokens": 14, "pieces": ["s漢漢", "　", "٣٤٥", "٦"]} +{"text": "'ſ \n字‍\rḍ̇Dž\"EOT'ſⅣ\"'M'S\u000b!!‍ßm's.fi­'ReⅣ", "tokens": 43, "pieces": ["'ſ", " \n", "字", "‍\r", "ḋ", "̣Dž", "\"EOT", "'ſ", "Ⅳ", "\"<", "EOT", ">'", "M", "'S", "\u000b", "!!‍", "ßm", "'s", ".fi", "­'", "Re", "Ⅳ"]} +{"text": "t#$%#$%''s\r\n\r\nſZ\"<|endoftext|><٣٤٥٦<|fim_prefix|>!!-'Sé😀🏽٣٤٥٦d字#$%0s'Re漢é'Reeḍ̇s㋿ع", "tokens": 77, "pieces": ["t", "#$%#$%''", "s", "\r\n\r\n", "ſZ", "\"<|", "endoftext", "|><", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>!!-'", "Sé", "😀🏽", "٣٤٥", "٦", "d字", "#$%", "0", "s", "'Re", "漢e", "́<", "META", "_START", ">'", "Reeḋ", "̣s", "㋿ع"]} +{"text": "\ta #$%,a\u000bt,Dža'll'Re-efi𐞁ſ٣٤٥٦
,éa㍿12345678👍🏽\r\n'D12345678d㋿12345678åEOT", "tokens": 66, "pieces": ["\ta", " #$%,", "a", "\u000bt", ",Dža", "'ll", "'Re", "-efi𐞁ſ", "٣٤٥", "٦", "
", ",éa", "㍿", "123", "456", "78", "👍🏽\r\n", "'D", "123", "456", "78", "d", "㋿", "123", "456", "78", "a", "̊EOT"]} +{"text": "éḍ̇'T👍🏽　‍ Ⅳ'ReZ…\"Ⅳİ!12345678ſd, \n 漢s\r\n\r\nDžå<|endoftext|>t(😀🏽 \n'Re!eé'red", "tokens": 69, "pieces": ["éḋ", "̣'", "T", "👍🏽", "　", "‍", " ", " ", "Ⅳ", "'Re", "Z", "…", "\"", "Ⅳ", "İ", "!", "123", "456", "78", "ſd", ",", " \n", " 漢s", "\r\n\r\n", "Dža", "̊<|", "endoftext", "|>", "t", "(😀🏽", " \n", "'Re", "!eé", "'re", "d"]} +{"text": "İ
‍ḍ̇ꟲfi…", "tokens": 17, "pieces": ["İ", "
", "‍ḋ", "̣ꟲfi", "…"]} +{"text": "!!㍿́'ll𐞁㋿\r", "tokens": 15, "pieces": ["!!㍿́'", "ll𐞁", "㋿\r"]} +{"text": " 0<|fim_prefix|>'ll㋿'så­t😀🏽३'Taå\r\nZ㍿12345678𐞁​
ꟲİ ꟲ\r\n", "tokens": 54, "pieces": [" ", " ", "0", "<|", "fim", "_prefix", "|>'", "ll", "㋿'", "sa", "̊­", "t", "😀🏽", "३", "'T", "aa", "̊\r\n", "Z", "㍿", "123", "456", "78", "𐞁", "​", "
ꟲİ", " ꟲ", "\r\n"]} +{"text": "e𐞁㍿é 12345678'sß　12345678meZ㋿字0'Re!'MZ-ſ'T𐞁", "tokens": 41, "pieces": ["e𐞁", "㍿<", "META", "_START", ">é", " ", "123", "456", "78", "'s", "ß", "　", "123", "456", "78", "meZ", "㋿字", "0", "'Re", "!'", "MZ", "-ſ", "'T", "𐞁"]} +{"text": " 0𐞁Z…#$%<|endoftext|>'D …👍🏽12345678á<|fim_prefix|>'re!!'Sfi \n😀🏽's٣٤٥٦'s'T<(", "tokens": 65, "pieces": [" ", "0", "𐞁Z", "…", "#$%<|", "endoftext", "|>'", "D", " ", "…", "👍🏽", "123", "456", "78", "a", "́<|", "fim", "_prefix", "|>'", "re", "!!'", "Sfi", " \n", "😀🏽'", "s", "٣٤٥", "٦", "'s", "'T", "<("]} +{"text": "👍🏽Ａ'reé'reİ\n३字字,åZt३ꟲ<|endoftext|> 'Mꟲ", "tokens": 44, "pieces": ["👍🏽", "Ａ", "'re", "e", "́'", "reİ", "\n", "३", "字", "字", ",a", "̊Zt", "३", "ꟲ", "<|", "endoftext", "|>", " ", "'M", "ꟲ"]} +{"text": " 　‍…\r漢ع're­ع \né'S👍🏽", "tokens": 22, "pieces": [" ", "　", "‍", "…\r", "漢ع", "'re", "­ع", " \n", "é", "'S", "👍🏽"]} +{"text": "\r#$%<|endoftext|>\u000b𐞁'T𐞁#$%e's \u000b…!'T́漢㍿>\r>👍🏽\r
å9,fi'VE‍\r㍿ < 𐞁<|fim_prefix|>漢ſ", "tokens": 82, "pieces": ["\r", "#$%<|", "endoftext", "|>", "\u000b𐞁", "'T", "𐞁", "#$%", "e", "'s", " \u000b", "…", "!'", "T", "́漢", "㍿>\r", ">👍🏽\r", "
a", "̊", "9", ",fi", "'VE", "‍\r", "㍿", " ", "<", " ", " 𐞁", "<|", "fim", "_prefix", "|>", "漢ſ"]} +{"text": "ſ9ع'Reé३'reſ'll\t'ſ㋿Ⅳ'Retfi­'re'VE​", "tokens": 31, "pieces": ["ſ", "9", "ع", "'Re", "é", "३", "'re", "ſ", "'ll", "\t", "'ſ", "㋿", "Ⅳ", "'Re", "tfi", "­'", "re", "'VE", "​"]} +{"text": "EOTd90'llＡ.'ll\" ", "tokens": 11, "pieces": ["EOTd", "90", "'ll", "Ａ", ".'", "ll", "\"", " "]} +{"text": "dＡ٣٤٥٦½td<­३å'Té12345678㍿字>\t'D <|fim_prefix|>ꟲ\n'MＡ'reå　'Dß🙂å 9", "tokens": 62, "pieces": ["dＡ", "٣٤٥", "٦½", "td", "<­", "३", "a", "̊'", "Té", "123", "456", "78", "㍿字", ">", "\t", "'D", " ", "<|", "fim", "_prefix", "|>", "ꟲ", "\n", "'M", "Ａ", "'re", "a", "̊", "　", "'D", "ß", "🙂a", "̊", " ", "9"]} +{"text": "'İDž\t½(漢", "tokens": 9, "pieces": ["'İDž", "\t", "½", "(漢"]} +{"text": ".ß𐞁‍İ \r\n", "tokens": 17, "pieces": [".", "ß𐞁", "‍", "İ", " \r\n"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'ſ ㋿ée👍🏽9\rEOT'\n<|endoftext|> ꟲ'S", " ꟲ", "'S", " \n​😀🏽 \nſDžſ'VE٣٤٥٦t\t<>", "tokens": 62, "pieces": ["'re", "🙂ḋ", "̣", "9", "​\n", "㋿'", "S", "­", "9", "éd漢", "
eİ", "", " \n", "​😀🏽", " \n", "ſDžſ", "'VE", "٣٤٥", "٦", "<", "META", "_START", ">t", "\t", "<>"]} +{"text": " #$%'D🙂9,…'re", "tokens": 11, "pieces": [" ", "#$%'", "D", "🙂", "9", ",", "…", "'re"]} +{"text": "ſß!<", "tokens": 4, "pieces": ["ſß", "!<"]} +{"text": "ßd'Re", "tokens": 3, "pieces": ["ßd", "'Re"]} +{"text": "fi( 'M\t''ReEOTéa\r\n㍿𐞁½ ३ é😀🏽\u000b>😀🏽'Dع'Dé\"İ ", "tokens": 48, "pieces": ["fi", "(", " ", "'M", "\t", "''", "ReEOTe", "́a", "\r\n", "㍿𐞁", "½", " ", "३", " e", "́😀🏽", "\u000b", ">😀🏽'", "Dع", "'D", "é", "\"İ", " "]} +{"text": "㋿́!!-😀🏽'0ع\t'ſ'D!!#$%<é漢fi<­", "tokens": 47, "pieces": ["㋿́!!-😀🏽'", "0", "ع", "", "\t", "'ſ", "'D", "!!<", "a", "̊a", "̊", " ", " '", "D", "#$%<", "é漢fi", "<­"]} +{"text": ".'Re­>", "tokens": 4, "pieces": [".'", "Re", "­>"]} +{"text": " <|endoftext|>​ \n-", "tokens": 9, "pieces": [" <|", "endoftext", "|>​", " \n", "-"]} +{"text": "'ſ!m漢😀🏽m㋿ſ9( \n𐞁Zꟲ'llé'M‍'s#$%(åḍ̇㍿٣٤٥٦'Re \n\r\nZ", "tokens": 63, "pieces": ["'ſ", "!m漢", "😀🏽", "m", "㋿ſ", "9", "(", " \n", "𐞁Zꟲ", "'ll", "e", "́'", "M", "‍'", "s", "#$%(", "a", "̊ḋ", "̣㍿", "٣٤٥", "٦", "'Re", " \n\r\n", "Z"]} +{"text": "t'ReEOTdＡ'M\r\n'D'VE\r\n'ſſ𐞁㋿'M'ſt>字", "tokens": 36, "pieces": ["t", "'Re", "EOTdＡ", "'", "M", "\r\n", "'D", "'VE", "\r\n", "'ſ", "ſ𐞁", "㋿'", "M", "'ſ", "t", ">字"]} +{"text": "'Mſ'T३d​\r\n<|fim_prefix|>'ſḍ̇'VE-Z-'Sİ㋿\r­ⅣEOT㋿\r\n\né😀🏽㍿'D>'M", "tokens": 55, "pieces": ["'M", "ſ", "'T", "३", "d", "​\r\n", "<|", "fim", "_prefix", "|>'", "ſḋ", "̣'", "VE", "-Z", "-'", "Sİ", "㋿\r", "­", "Ⅳ", "EOT", "㋿\r\n\n", "é", "😀🏽㍿'", "D", ">'", "M"]} +{"text": "漢　\r
  
'll½字Ⅳm\r\n\r\n,eémDž<|fim_prefix|>! 0​'S're-!!'D'D
ꟲ\"𐞁ḍ̇\u000b‍", "tokens": 66, "pieces": ["漢", "　\r", "
  ", "
", "'ll", "½", "字", "Ⅳ", "m", "\r\n\r\n", ",<", "EOT", ">eémDž", "<|", "fim", "_prefix", "|>!", " ", " ", "0", "​'", "S", "'re", "-!!'", "D", "'", "D", "
ꟲ", "\"𐞁ḋ", "̣", "\u000b", "‍"]} +{"text": "\t12345678ßd\r\nm'Re​", "tokens": 10, "pieces": ["\t", "123", "456", "78", "ßd", "\r\n", "m", "'Re", "​"]} +{"text": ".!!㋿ḍ̇éⅣ'red३'Dåt", "tokens": 26, "pieces": [".!!㋿", "ḋ", "̣e", "́", "Ⅳ", "'re", "d", "", "३", "'D", "a", "̊t"]} +{"text": "-‍\",
", "tokens": 6, "pieces": ["-‍\",", "
"]} +{"text": "'s'T DžEOT12345678'S", "tokens": 11, "pieces": ["'s", "'T", " DžEOT", "123", "456", "78", "'S"]} +{"text": "­㋿\r字\r\n\r\n㍿ (​é\r\n", "tokens": 20, "pieces": ["­㋿<", "META", "_START", ">\r", "字", "\r\n\r\n", "㍿", " ", "(​", "e", "́\r\n"]} +{"text": "字!9", "tokens": 3, "pieces": ["字", "!", "9"]} +{"text": "e\nḍ̇字Ａ-٣٤٥٦\"​
\"'DZⅣ𐞁'S<|endoftext|>> (İ \nm,><漢", "tokens": 58, "pieces": ["e", "\n", "ḋ", "̣字Ａ", "-", "٣٤٥", "٦", "\"<", "EOT", ">​", "
", "\"'", "DZ", "", "Ⅳ", "𐞁", "'S", "<|", "endoftext", "|>>", " ", " (", "İ", " \n", "m", ",<", "META", "_START", ">><", "漢"]} +{"text": "é<‍㍿漢…\u000b​9'M字漢 \r\n㍿ꟲ<'så\rm漢fi‍!!㋿\"٣٤٥٦ſ", "tokens": 38, "pieces": ["é", "Ⅳ", "…", "!!", "m", "(d", "'VE", "漢", ">a", "̊\r", "m漢fi", "‍!!㋿\"", "٣٤٥", "٦", "ſ"]} +{"text": "… \nßⅣ12345678
ḍ̇mعßＡ​İ-é12345678🙂'M‍ 'M9EOT\r\n'll\r\n\r\n٣٤٥٦ſ　", "tokens": 54, "pieces": ["… \n", "ß", "Ⅳ12", "345", "678", "
ḋ", "̣mعßＡ", "​İ", "-e", "́", "123", "456", "78", "🙂'", "M", "‍", " ", " '", "M", "9", "EOT", "\r\n", "'ll", "\r\n\r\n", "٣٤٥", "٦", "ſ", "　"]} +{"text": "'ll🙂\rꟲ #$%, 'Re½​㋿…'D漢ꟲ'SＡＡ e\t\tḍ̇'reİ👍🏽e😀🏽fi🙂s, 'VEt!", "tokens": 69, "pieces": ["'ll", "🙂\r", "ꟲ", " ", "#$%,", " '", "Re", "½", "​㋿", "…", "'D", "漢ꟲ", "'S", "ＡＡ", " ", " e", "\t", "\tḋ", "̣'", "reİ", "👍🏽", "e", "😀🏽<", "EOT", ">fi", "🙂s", ",", " ", " '", "VEt", "!"]} +{"text": "漢ß\t​", "tokens": 5, "pieces": ["漢ß", "\t", "​"]} +{"text": "　​må12345678#$%<|fim_prefix|>½Ⅳ㋿字'D字­!é\rꟲ'T", "tokens": 45, "pieces": ["", "　", "​ma", "̊", "123", "456", "78", "#$%<|", "fim", "_prefix", "|>", "½", "<", "META", "_START", ">", "Ⅳ", "㋿字", "'D", "字", "­!", "e", "́\r", "ꟲ", "'T"]} +{"text": "㍿
'D9𐞁½dß\r\n\r\n\u000bEOTd'sDžså'M<|endoftext|>ZDž 
.é'Re'TZ \n𐞁Ⅳ\t", "tokens": 62, "pieces": ["㍿", "
", "'D", "9", "𐞁", "½", "dß", "\r\n\r\n", "\u000bEOTd", "'", "sDžsa", "̊'", "M", "<|", "endoftext", "|>", "ZDž", " ", "
", ".e", "́'", "Re", "'T", "Z", " \n", "𐞁", "", "Ⅳ", "\t"]} +{"text": "EOTm'll're'reé ­　m字,<|fim_prefix|>'Re'tfi‍'VE🙂0é'M'VEt' ㋿-\n
'ſ'ſ‍9EOT𐞁'S", "tokens": 58, "pieces": ["EOTm", "'ll", "'re", "'re", "é", " ­", "　m字", ",<|", "fim", "_prefix", "|>'", "Re", "'t", "fi", "‍'", "VE", "🙂", "0", "é", "'M", "'VE", "t", "'", " ㋿-\n", "
", "'ſ", "'ſ", "‍", "9", "EOT𐞁", "'S"]} +{"text": "Ⅳ#$%ååß 'VE0fi㋿½'T'Re0-.12345678'>'s.🙂>ꟲ…", "tokens": 40, "pieces": ["Ⅳ", "#$%", "a", "̊a", "̊ß", " ", "'VE", "0", "fi", "㋿", "½", "'T", "'Re", "0", "-.", "123", "456", "78", "'>'", "s", ".🙂>", "ꟲ", "…"]} +{"text": "<|endoftext|>fi-㍿'T​", "tokens": 16, "pieces": ["<|", "endoftext", "|>", "fi", "-㍿'", "T", "​"]} +{"text": "İſ٣٤٥٦09字'­#$%\t'VE's'­e\"३\r \n<|fim_prefix|>s㋿Dž'T ", "tokens": 44, "pieces": ["İſ", "٣٤٥", "٦09", "字", "'­#$%", "\t", "'VE", "'s", "'­", "e", "\"", "३", "\r \n", "<|", "fim", "_prefix", "|>", "s", "㋿Dž", "'T", " "]} +{"text": "Ａ​Ａſſé-12345678.\u000b-　३½s'Tḍ̇#$%(ع 
ß\nDž'D'MDž 'seß", "tokens": 49, "pieces": ["Ａ", "​Ａ", "ſſé", "-", "123", "456", "78", ".", "\u000b", "-", "　", "३½", "s", "'T", "ḋ", "̣#$%(", "ع", " ", "
ß", "\n", "Dž", "'D", "'M", "Dž", " ", "'s", "eß"]} +{"text": "- \né'Sa'VEſ漢عſ𐞁..EOT<字,'s'ſ<\r\n\r\nt٣٤٥٦ꟲ\r\n\r\n👍🏽é­字9", "tokens": 58, "pieces": ["-", " \n", "e", "́'", "Sa", "'VE", "ſ漢عſ𐞁", "..", "EOT", "<字", ",'", "s", "'ſ", "<\r\n\r\n", "t", "٣٤٥", "٦", "ꟲ", "\r\n\r\n", "👍🏽", "e", "́­", "字", "9", ""]} +{"text": "9\n \n　'D,Dž'ع<<|fim_prefix|><|fim_prefix|>​EOT>İ'!ع字éée字('ll‍👍🏽\t'VEḍ̇字🙂a字 ,", "tokens": 63, "pieces": ["9", "\n \n", "　", "'D", ",Dž", "'ع", "<<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><", "EOT", ">​", "EOT", ">İ", "'!", "ع字e", "́ée字", "('", "ll", "‍👍🏽", "\t", "'VE", "ḋ", "̣字", "🙂a字", " ", ","]} +{"text": "å<'re<|endoftext|>½ \n<|endoftext|>\r\n\r\n٣٤٥٦fi ​\r0'reZ \nEOTDž. ㍿­\r\n#$%İ, 9 \n\"'M'TＡ", "tokens": 62, "pieces": ["a", "̊<'", "re", "<|", "endoftext", "|>", "½", " \n", "<|", "endoftext", "|>\r\n\r\n", "٣٤٥", "٦", "fi", " ", " ​\r", "0", "'re", "Z", " \n", "EOTDž", ".", " ", "㍿­\r\n", "#$%", "İ", ",", " ", " ", "9", " \n", "\"'", "M", "'T", "Ａ"]} +{"text": "\u000b<|endoftext|>\u000b㋿'s३ſ‍.'VE
\u000ba㍿Ⅳt​'Re\r ꟲ", "tokens": 47, "pieces": ["\u000b", "<|", "endoftext", "|>", "\u000b", "㋿'", "s", "३", "ſ", "‍.'", "VE", "
", "\u000ba", "㍿", "Ⅳ", "t", "​'", "Re", "\r", " ꟲ", ""]} +{"text": "EOT#$%😀🏽9㋿'T'D
<|endoftext|>'M,", "tokens": 27, "pieces": ["EOT", "#$%😀🏽", "9", "㋿'", "T", "'D", "
", "<|", "endoftext", "|>'", "M", ","]} +{"text": "'ſ ", "tokens": 7, "pieces": ["'ſ", "", " "]} +{"text": "éå'M字
>­​dé\r\n\r\n½\r\n, \u000bmfi.'llſ👍🏽 \n ع​t …é'T12345678fi('t\n½", "tokens": 52, "pieces": ["éa", "̊'", "M字", "
", ">­​", "dé", "\r\n\r\n", "½", "\r\n", ",", " ", "\u000bmfi", ".'", "llſ", "👍🏽", " \n", " ع", "​t", " ", "…é", "'T", "123", "456", "78", "fi", "('", "t", "\n", "½"]} +{"text": "٣٤٥٦å👍🏽t'M'!'ſm'Ś12345678…(ſ\u000b👍🏽漢's'Re'T!", "tokens": 50, "pieces": ["٣٤٥", "٦", "a", "̊👍🏽", "t", "'M", "'!'", "ſm", "'S", "́", "123", "456", "78", "…", "(ſ", "", "\u000b", "👍🏽", "漢", "'s", "'Re", "'T", "!"]} +{"text": "'s­'S\r", "tokens": 5, "pieces": ["'s", "­'", "S", "\r"]} +{"text": "́㋿ 
Ⅳ字! ‍Dž'T\rß(Ａß漢字🙂.'M'VEDže'VE'ét٣٤٥٦…(9\r\n\r\n'VE\r\n\r\n
Ａ", "tokens": 60, "pieces": ["́㋿", " ", "
", "Ⅳ", "字", "!", " ", " ‍", "Dž", "'T", "\r", "ß", "(Ａß漢字", "🙂.'", "M", "'VE", "Dže", "'VE", "'e", "́t", "٣٤٥", "٦", "…", "(", "9", "\r\n\r\n", "'VE", "\r\n\r\n", "
Ａ"]} +{"text": "DžéaＡ㋿ .,é
٣٤٥٦ßⅣ12345678", "tokens": 29, "pieces": ["Dže", "́aＡ", "㋿", " .,", "é", "
", "٣٤٥", "٦", "ß", "Ⅳ12", "345", "678"]} +{"text": " 👍🏽!!'M\t!!>½s३<|fim_prefix|>åＡ३'D\tZ\r\n\r\n'Re>#$%ḍ̇Ⅳd\r\n\r\n\tdt!!< t \"ع", "tokens": 66, "pieces": [" ", " 👍🏽!!'", "M", "\t", "!!>", "½", "s", "३", "<|", "fim", "_prefix", "|>", "a", "̊Ａ", "३", "'D", "\tZ", "\r\n\r\n", "'Re", ">#$%", "ḋ", "̣", "Ⅳ", "d", "\r\n\r\n", "\t", "dt", "!!<", " t", " ", " \"", "ع"]} +{"text": "㍿३عＡſ,>#$%!!'ll12345678'", "tokens": 24, "pieces": ["㍿", "३", "عＡ", "ſ", ",>#$%!!'", "ll", "123", "456", "78", "'"]} +{"text": "'S-　!!\"😀🏽 aåé　'S
", "tokens": 25, "pieces": ["<", "EOT", ">'", "S", "-", "　", "!!\"😀🏽", " aa", "̊é", "　", "'S", "
"]} +{"text": "Dž( \n12345678sé#$%(12345678e😀🏽", "tokens": 20, "pieces": ["Dž", "(", " \n", "123", "456", "78", "sé", "#$%(", "123", "456", "78", "e", "😀🏽"]} +{"text": "'ReéDžé 😀🏽­㍿　'lle", "tokens": 23, "pieces": ["'Re", "e", "́Dž", "e", "́", " 😀🏽­㍿", "　", "'ll", "e", ""]} +{"text": "३!㋿m\t'D㍿\r\n\r\n.'llDž　'lls'é<", "tokens": 26, "pieces": ["३", "!㋿", "m", "", "\t", "'D", "㍿\r\n\r\n", ".'", "llDž", "　", "'ll", "s", "'é", "<"]} +{"text": " \n'VE'Sd­　ع'Må\r\n\r\n'ree 'ReeZ's‍　('re🙂 (\r\u000bꟲ", "tokens": 39, "pieces": [" \n", "'VE", "'S", "d", "­", "　ع", "'M", "a", "̊\r\n\r\n", "'re", "e", " ", " '", "ReeZ", "'s", "‍", "　", "('", "re", "🙂<", "META", "_START", ">", " ", "(\r", "\u000bꟲ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'sZ", "tokens": 2, "pieces": ["'s", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n'S!!‍字\r\n\r\ns漢Ａ‍漢0!!‍dém0漢-́Dž \r\n\r\n<漢're‍'M12345678ḍ̇", "tokens": 51, "pieces": ["\r\n", "'S", "!!‍", "字", "\r\n\r\n", "s漢Ａ", "‍漢", "0", "!!‍", "de", "́m", "0", "漢", "-́", "Dž", " \r\n\r\n", "<漢", "'re", "‍'", "M", "123", "456", "78", "ḋ", "̣"]} +{"text": "<|endoftext|>𐞁३'S,ſ́!!e\t­\r\n'Reḍ̇'s'Ss३ ㍿#$%Z'ſa½<(!9", "tokens": 53, "pieces": ["<|", "endoftext", "|>", "𐞁", "३", "'S", ",ſ", "́!!", "e", "\t", "­<", "META", "_START", ">\r\n", "'Re", "ḋ", "̣'", "s", "'S", "s", "३", " ", "㍿#$%", "Z", "'ſ", "a", "½", "<(!", "9"]} +{"text": "méꟲé\n漢's<|fim_prefix|>
9३ éa Ａ'VE३!!漢 \na'VE>éEOT<|endoftext|>ḍ̇ Ａ,字'll#$%ſ", "tokens": 67, "pieces": ["me", "́ꟲe", "́\n", "漢", "'s", "<|", "fim", "_prefix", "|>", "
", "9", "", "३", " éa", " ", " Ａ", "'VE", "३", "!!", "漢", " \n", "a", "'VE", ">éEOT", "<|", "endoftext", "|>", "ḋ", "̣", " Ａ", ",字", "'ll", "#$%", "ſ"]} +{"text": "EOT३('ll㍿e😀🏽Ａ'ſ­é<12345678
漢\u000b ́12345678", "tokens": 36, "pieces": ["EOT", "३", "('", "ll", "㍿e", "😀🏽", "Ａ", "'ſ", "­é", "<", "123", "456", "78", "
漢", "\u000b", " ", "́", "123", "456", "78"]} +{"text": "!👍🏽'S'ſ \r\n'VE‍'!!'ḍ̇>Dž\r\n'ſ٣٤٥٦md é字ع!!<|endoftext|>0!!!'S
𐞁d", "tokens": 64, "pieces": ["!👍🏽'", "S", "'ſ", " \r\n", "'VE", "‍'!!'", "ḋ", "̣>", "Dž", "\r\n", "'ſ", "٣٤٥", "٦", "md", " e", "́字ع", "!!<|", "endoftext", "|>", "0", "!!!'", "S", "
𐞁d"]} +{"text": "é'ſtＡ'ſ\r\n<|fim_prefix|>\t'ſ३漢\r\n\u000b\nét!'s\r\n\r\n-Dž<|fim_prefix|>ع>9'VEİé㍿", "tokens": 55, "pieces": ["é", "'ſ", "tＡ", "'ſ", "\r\n", "<|", "fim", "_prefix", "|>", "\t", "'ſ", "३", "漢", "\r\n\u000b\n", "e", "́t", "!'", "s", "\r\n\r\n", "-Dž", "<|", "fim", "_prefix", "|>", "ع", ">", "9", "'VE", "İé", "㍿"]} +{"text": "<12345678👍🏽\r­‍", "tokens": 14, "pieces": ["<", "123", "456", "78", "👍🏽\r", "­‍"]} +{"text": "'s <'s!!ع\n-​\u000b Dž👍🏽Ⅳ's're<‍<|endoftext|>😀🏽\"'M 'll>٣٤٥٦ \tEOTm ", "tokens": 56, "pieces": ["'s", " ", "<'", "s", "!!", "ع", "\n", "-​", "\u000b", " Dž", "👍🏽", "Ⅳ", "'s", "'re", "<‍<|", "endoftext", "|>😀🏽\"'", "M", " ", "'ll", ">", "٣٤٥", "٦", " ", "\tEOTm", " "]} +{"text": "\r\n\"🙂fi é‍٣٤٥٦ḍ̇å<|endoftext|>
!å​　'Mé'T9!!!३㍿.d12345678'VE'D<(", "tokens": 61, "pieces": ["\r\n", "\"🙂", "fi", " ", " e", "́‍", "٣٤٥", "٦", "ḋ", "̣a", "̊<|", "endoftext", "|>", "
", "!a", "̊​", "　", "'M", "é", "'T", "9", "!!!", "३", "㍿.", "d", "123", "456", "78", "'VE", "'D", "<("]} +{"text": "́ ḍ̇\"㍿'VEsⅣſ(#$%ß 　
\rＡعt\t𐞁!!👍🏽­sfi!'Reå-fiEOT", "tokens": 57, "pieces": ["́", " ḋ", "̣\"㍿'", "VEs", "Ⅳ", "ſ", "(#$%", "ß", " 　
\r", "Ａعt", "\t𐞁", "!!👍🏽­", "sfi", "!'", "Rea", "̊-", "fiEOT"]} +{"text": "'T‍३Ⅳ", "tokens": 7, "pieces": ["'T", "‍", "३Ⅳ"]} +{"text": "!!tＡ'reꟲ½ع­३a!s'll\r\n<<|fim_prefix|>'S're<|endoftext|>👍🏽 Ａ \n,fi\r\n,İ'ſ👍🏽d#$%t-", "tokens": 64, "pieces": ["!!", "tＡ", "'re", "ꟲ", "½", "ع", "­", "३", "a", "!s", "'ll", "\r\n", "<<|", "fim", "_prefix", "|>'", "S", "'re", "<|", "endoftext", "|>👍🏽", " Ａ", " \n", ",fi", "\r\n", ",İ", "'ſ", "👍🏽", "d", "#$%", "t", "-"]} +{"text": ">'s'll‍ꟲ\u000b​'s#$%'Td👍🏽", "tokens": 21, "pieces": [">'", "s", "'ll", "‍ꟲ", "\u000b", "​'", "s", "#$%'", "Td", "👍🏽"]} +{"text": "\r\n\"​'ſ漢\u000bⅣ-٣٤٥٦t.", "tokens": 24, "pieces": ["\r\n", "\"​'", "ſ漢", "\u000b", "Ⅳ", "-", "٣٤٥", "٦", "t", "."]} +{"text": "9​å\r\n\r\n''VE३d,​́és're'T'ſmḍ̇, \n é
\"mſİ", "tokens": 63, "pieces": ["9", "​a", "̊\r\n\r\n", "''", "VE", "३", "d", ",​́", "e", "́s", "'re", "'T", "'ſ", "mḋ", "̣<", "t", "­👍🏽", " \r\n", "👍🏽!", "ꟲ", "<-<", "META", "_START", ">,", " \n", " é", "
", "\"mſİ"]} +{"text": "‍\r\n㍿Zİ9é.>fi'VEEOT ٣٤٥٦å😀🏽'D  \n#$%‍'re߅\r\n\r\n\u000b\"a　12345678", "tokens": 58, "pieces": ["‍<", "EOT", ">\r\n", "㍿Zİ", "9", "é", ".>", "fi", "'VE", "EOT", " ", "٣٤٥", "٦", "a", "̊😀🏽'", "D", "  \n", "#$%‍'", "reß", "…\r\n\r\n", "\u000b", "\"a", "　", "123", "456", "78"]} +{"text": "Ａ.३-\u000b9字ꟲZ'll㋿(a!𐞁sEOT'll.'å!! ḍ̇", "tokens": 43, "pieces": ["Ａ", ".", "३", "-", "\u000b", "9", "字ꟲZ", "'ll", "㋿(", "a", "!𐞁s", "EOT", "'ll", ".'", "a", "̊!!", " ḋ", "̣"]} +{"text": "!!s'fifi👍🏽m…\u000ba\"Ⅳ're½.9İEOTꟲſe字ꟲ'ſ", "tokens": 41, "pieces": ["!!", "s", "'fifi", "👍🏽", "m", "…", "\u000ba", "\"", "Ⅳ", "'re", "½", ".", "9", "İEOTꟲſe字ꟲ", "'ſ"]} +{"text": "𐞁'Sß​<|fim_prefix|>ſ \n>㋿9ꟲＡ9 a", "tokens": 30, "pieces": ["𐞁", "'S", "ß", "​<|", "fim", "_prefix", "|>", "ſ", " \n", ">㋿", "9", "ꟲＡ", "9", " ", " a"]} +{"text": "fi३\u000b're'VEع", "tokens": 9, "pieces": ["fi", "३", "\u000b", "'re", "'VE", "ع"]} +{"text": "👍🏽'séd<­Ａ㋿
'Mm'M½
", "tokens": 25, "pieces": ["👍🏽'", "se", "́d", "<­", "Ａ", "㋿", "
", "'M", "m", "'M", "½", "
"]} +{"text": "'VEs-漢>Z'", "tokens": 8, "pieces": ["'VE", "s", "-漢", ">Z", "'"]} +{"text": "İḍ̇t're½e0😀🏽'\"d", "tokens": 18, "pieces": ["İḋ", "̣t", "'re", "½", "e", "0", "😀🏽'\"", "d"]} +{"text": "efi're🙂\r㋿(字
…", "tokens": 16, "pieces": ["efi", "'re", "🙂\r", "㋿(", "字", "
…"]} +{"text": "9​é", "tokens": 9, "pieces": ["", "9", "​", "é"]} +{"text": "ß😀🏽å's", "tokens": 11, "pieces": ["ß", "😀🏽", "a", "̊'", "s"]} +{"text": "‍Z<|fim_prefix|>३!!\n­", "tokens": 14, "pieces": ["‍Z", "<|", "fim", "_prefix", "|>", "३", "!!\n", "­"]} +{"text": "漢é.…", "tokens": 10, "pieces": ["漢e", "́.", "…", ""]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "9.9 ㋿é(\r\nİ,s
", "tokens": 18, "pieces": ["9", ".", "9", " ", " ㋿<", "EOT", ">é", "(\r\n", "İ", ",s", "
"]} +{"text": "\u000b　#$%Ａ \n'VE'ſ🙂́'Tsé'ſt ", "tokens": 28, "pieces": ["\u000b", "　", "#$%", "Ａ", " \n", "'VE", "'ſ", "🙂<", "META", "_START", ">́'", "Tse", "́'", "ſt", " "]} +{"text": "𐞁३ \n٣٤٥٦'ll­‍'ReꟲEOT'D👍🏽'M'll<'ll.d\"'Md'T漢(", "tokens": 48, "pieces": ["𐞁", "३", " \n", "٣٤٥", "٦", "'ll", "­‍'", "ReꟲEOT", "'D", "👍🏽'", "M", "'ll", "<'", "ll", ".d", "\"'", "Md", "'T", "漢", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "0fiß.'re\"\u000b 'ſfiß!!'ll\r\n'S​'D㋿\r\n\r\n…", "tokens": 33, "pieces": ["0", "fiß", ".'", "re", "\"", "\u000b", "", " ", " '", "ſfiß", "!!'", "ll", "\r\n", "'S", "​'", "D", "㋿\r\n\r\n", "…"]} +{"text": "ß३'VEع<|endoftext|>Ⅳ👍🏽ß,", "tokens": 23, "pieces": ["ß", "३", "'VE", "ع", "<|", "endoftext", "|>", "Ⅳ", "👍🏽", "ß", ","]} +{"text": "t\r'T9­Ａ'S🙂　#$%!'T\tt½'VE㋿
-m👍🏽\n
 \n", "tokens": 38, "pieces": ["t", "\r", "'T", "9", "­Ａ", "'S", "🙂", "　", "#$%!'", "T", "", "\tt", "½", "'VE", "㋿", "
", "-m", "👍🏽\n", "
 \n"]} +{"text": "0ḍ̇Zdé.\r\n\r\nZ­<|endoftext|>as…'ll'M12345678<|fim_prefix|><\n‍ 'S👍🏽9!e'Re'Re", "tokens": 55, "pieces": ["0", "ḋ", "̣Zdé", ".\r\n\r\n", "Z", "­<|", "endoftext", "|>", "as", "…", "'ll", "'M", "123", "456", "78", "<|", "fim", "_prefix", "|><\n", "‍", " ", " '", "S", "👍🏽", "9", "!e", "'Re", "'Re"]} +{"text": "é漢ḍ̇", "tokens": 8, "pieces": ["é漢ḋ", "̣"]} +{"text": "ß٣٤٥٦'TZt३<|endoftext|>å.漢a\tm'TＡ Ａ½e‍​ ㋿㋿'M
 \n<'Re \n٣٤٥٦́", "tokens": 67, "pieces": ["ß", "٣٤٥", "٦", "'T", "Zt", "३", "<|", "endoftext", "|>", "a", "̊.", "漢a", "\tm", "'T", "Ａ", " Ａ", "½", "e", "‍​", " ", "㋿㋿'", "M", "
 \n", "<'", "Re", " \n", "٣٤٥", "٦", "́"]} +{"text": " ㍿ 'll Ａ字<|fim_prefix|>'Re　<|fim_prefix|>٣٤٥٦'M'Mꟲ\r\nå>Ⅳ.𐞁EOT‍'ll\n\rt!\nm<|fim_prefix|>", "tokens": 72, "pieces": [" ", " ㍿", " ", "'ll", " Ａ字", "<|", "fim", "_prefix", "|>'", "Re", "　", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "'M", "'M", "ꟲ", "\r\n", "a", "̊>", "Ⅳ", ".𐞁", "EOT", "‍'", "ll", "\n\r", "t", "!\n", "m", "<|", "fim", "_prefix", "|>"]} +{"text": "d\nḍ̇fiß\r'M😀🏽'S'T㋿ \r\n\r\n㍿<­漢٣٤٥٦'ReDž\n<|endoftext|>a--!!漢fi…\u000bé\r३­", "tokens": 65, "pieces": ["d", "\n", "ḋ", "̣fiß", "\r", "'M", "😀🏽'", "S", "'T", "㋿", " \r\n\r\n", "㍿<­", "漢", "٣٤٥", "٦", "'Re", "Dž", "\n", "<|", "endoftext", "|>", "a", "--!!", "漢fi", "…", "\u000bé", "\r", "३", "­"]} +{"text": "́a!!", "tokens": 3, "pieces": ["́a", "!!"]} +{"text": "a<|endoftext|>𐞁\nꟲ'S\r\nfiEOT12345678🙂", "tokens": 27, "pieces": ["a", "<|", "endoftext", "|>", "𐞁", "\n", "ꟲ", "'S", "\r\n", "fiEOT", "123", "456", "78", "🙂"]} +{"text": "👍🏽'reé㍿ 🙂0\t㋿\r#$%　
m!!é#$%🙂's㍿s'VEeſ'M#$%'ll漢Z,a( ", "tokens": 52, "pieces": ["👍🏽'", "reé", "㍿", " 🙂", "0", "\t", "㋿\r", "#$%", "　", "
m", "!!", "é", "#$%🙂'", "s", "㍿s", "'VE", "eſ", "'M", "#$%'", "ll漢Z", ",a", "(", " "]} +{"text": "👍🏽…㋿İd ", "tokens": 14, "pieces": ["👍🏽", "…", "㋿İd", " "]} +{"text": " 字　\u000bmEOT​ḍ̇ İ٣٤٥٦👍🏽'ſDža'T\r>'TDžDž!!٣٤٥٦\"'VE", "tokens": 57, "pieces": [" 字", "　", "\u000bmEOT", "​ḋ", "̣", " İ", "٣٤٥", "٦", "👍🏽'", "ſDža", "'T", "\r", ">'", "TDžDž", "!!", "٣٤٥", "٦", "\"<", "EOT", ">'", "VE"]} +{"text": "ḍ̇  \n é<|endoftext|>'ll<|endoftext|>é…\t \né\r\nm.'DéＡ#$%\u000bⅣ­", "tokens": 45, "pieces": ["ḋ", "̣", "  \n", " e", "́<|", "endoftext", "|>'", "ll", "<|", "endoftext", "|>", "é", "…\t \n", "é", "\r\n", "m", ".'", "Dé", "Ａ", "#$%", "\u000b", "Ⅳ", "­"]} +{"text": "'­ßع<|endoftext|>'llss\"٣٤٥٦é", "tokens": 23, "pieces": ["'­", "ßع", "<|", "endoftext", "|>'", "llss", "\"", "٣٤٥", "٦", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Ⅳ!!éé<|endoftext|>m
Dž'D ḍ̇\"३<|endoftext|>'👍🏽́'́'VEa!\n'll㋿́EOT<|endoftext|>!!‍9ḍ̇", "tokens": 69, "pieces": ["Ⅳ", "!!", "ée", "́<|", "endoftext", "|>", "m", "
Dž", "'D", " ḋ", "̣\"", "३", "<|", "endoftext", "|>'👍🏽́'́'", "VEa", "!\n", "'ll", "㋿́", "EOT", "<|", "endoftext", "|>!!‍", "9", "ḋ", "̣"]} +{"text": " ٣٤٥٦ſ!!ع𐞁-漢å漢३İ‍'M\"ع!Ａ#$%EOTe9\u000b<|endoftext|>‍'VE ٣٤٥٦12345678 ée9>\r\n\r\n\r\n\r\n'S", "tokens": 81, "pieces": [" ", " ", "٣٤٥", "٦", "ſ", "!!", "ع𐞁", "-漢a", "̊漢", "३", "İ", "‍'", "M", "\"ع", "!Ａ", "#$%", "EOTe", "9", "\u000b", "<|", "endoftext", "|>‍'", "VE", " ", "٣٤٥", "٦12", "345", "678", " e", "́e", "9", ">\r\n\r\n\r\n\r\n", "<", "EOT", ">'", "S"]} +{"text": "'D½!é0- \r\n\r\nd😀🏽 9­#$%Z…éꟲ　e'Reé>漢'S9", "tokens": 37, "pieces": ["'D", "½", "!e", "́", "0", "-", " \r\n\r\n", "d", "😀🏽", " ", " ", "9", "­#$%", "Z", "…éꟲ", "　e", "'Re", "é", ">漢", "'S", "9"]} +{"text": "　\r\né t! ' #$%12345678!e12345678! \r", "tokens": 23, "pieces": ["　\r\n", "e", "́", " t", "!", " ", "'", " ", " #$%", "123", "456", "78", "!e", "123", "456", "78", "!", " \r"]} +{"text": "EOT!'ſ­\r\nع", "tokens": 8, "pieces": ["EOT", "!'", "ſ", "­\r\n", "ع"]} +{"text": "㍿9#$%(d!!Ａ́
ع٣٤٥٦\"\"eß\u000b \n!!𐞁𐞁٣٤٥٦​👍🏽'll　
", "tokens": 64, "pieces": ["㍿", "9", "#$%(", "d", "!!", "Ａ", "́", "
ع", "", "٣٤٥", "٦", "\"\"", "eß", "\u000b \n", "!!", "𐞁", "𐞁", "٣٤٥", "٦", "​👍🏽'", "ll", "　
"]} +{"text": ", EOT\n<|endoftext|>'ReEOT
ſ'ſ㍿İſ\r\n\r\n㍿½🙂<|endoftext|>'\r\n\r\nⅣꟲm", "tokens": 51, "pieces": [",", " EOT", "\n", "<|", "endoftext", "|>'", "ReEOT", "
ſ", "'ſ", "㍿İſ", "\r\n\r\n", "㍿", "½", "🙂<|", "endoftext", "|>'\r\n\r\n", "Ⅳ", "ꟲ", "m"]} +{"text": "ß \n<|endoftext|>Ａ'llEOT \r åé9é👍🏽㍿㍿'Ḿm\"ꟲḍ̇漢'ſ12345678'T३ \n're'('D\u000b", "tokens": 66, "pieces": ["ß", " \n", "<|", "endoftext", "|>", "Ａ", "'ll", "EOT", " \r", " a", "̊e", "́", "9", "e", "́👍🏽㍿㍿'", "M", "́m", "\"ꟲḋ", "̣漢", "'ſ", "123", "456", "78", "'T", "३", " \n", "'re", "'('", "D", "\u000b"]} +{"text": "\n're😀🏽‍éḍ̇9㋿", "tokens": 20, "pieces": ["\n", "'re", "😀🏽‍", "e", "́ḋ", "̣", "9", "㋿"]} +{"text": "漢\u000b\r\n\r\n
 漢'redİ\r\n", "tokens": 13, "pieces": ["漢", "\u000b\r\n\r\n", "
", " 漢", "'re", "dİ", "\r\n"]} +{"text": "½!e", "tokens": 3, "pieces": ["½", "!e"]} +{"text": "　'VE​t३", "tokens": 7, "pieces": ["　", "'VE", "​t", "३"]} +{"text": "'M,d㋿\r>as>!'re\tſ́<|fim_prefix|>#$%-fi𐞁½㋿́\ré", "tokens": 37, "pieces": ["'M", ",d", "㋿\r", ">as", ">!'", "re", "\tſ", "́<|", "fim", "_prefix", "|>#$%-", "fi𐞁", "½", "㋿́\r", "é"]} +{"text": "e,😀🏽9\r\n#$%'Dt👍🏽s'ReⅣ", "tokens": 26, "pieces": ["e", ",😀🏽", "9", "\r\n", "#$%'", "Dt", "👍🏽", "s", "'Re", "Ⅳ", ""]} +{"text": "…\"\"½#$%😀🏽's!!\"ea< \n", "tokens": 18, "pieces": ["…", "\"\"", "½", "#$%😀🏽'", "s", "!!\"", "ea", "<", " \n"]} +{"text": "'TⅣ're㋿\r\n\r\n🙂'Re<|endoftext|>　'reꟲ ḍ̇m😀🏽字'S'ReåＡ", "tokens": 47, "pieces": ["'T", "Ⅳ", "'re", "㋿\r\n\r\n", "🙂<", "META", "_START", ">'", "Re", "<|", "endoftext", "|>", "　", "'re", "ꟲ", " ", " ḋ", "̣m", "😀🏽", "字", "'S", "'Re", "a", "̊Ａ"]} +{"text": "'<漢…🙂<|endoftext|> ßé\r\n\r\nḍ̇Dž", "tokens": 28, "pieces": ["'<", "漢", "…", "🙂<", "META", "_START", "><|", "endoftext", "|>", " ße", "́\r\n\r\n", "ḋ", "̣Dž"]} +{"text": "a'fi.EOT( 'T-\r\n\r\n12345678….", "tokens": 17, "pieces": ["a", "'fi", ".EOT", "(", " '", "T", "-\r\n\r\n", "123", "456", "78", "…", "."]} +{"text": "­'Re<㋿", "tokens": 7, "pieces": ["­'", "Re", "<㋿"]} +{"text": " \n>😀🏽 -ع'S३
🙂ß,!'s<-'re'Mmm'TdEOT-­\r\n\r\n", "tokens": 38, "pieces": [" \n", "><", "META", "_START", ">😀🏽", " -", "ع", "'S", "३", "
", "🙂ß", ",!'", "s", "<-'", "re", "'M", "mm", "'T", "dEOT", "-­\r\n\r\n"]} +{"text": "å'Refie　<|fim_prefix|>'TEOT a३d'sſ' \n!'reDž(߅(Ⅳeع", "tokens": 41, "pieces": ["a", "̊'", "Refie", "　", "<|", "fim", "_prefix", "|>'", "TEOT", " a", "३", "d", "'s", "ſ", "'", " \n", "!'", "reDž", "(ß", "…", "(", "Ⅳ", "eع"]} +{"text": "ع‍'ſDž12345678😀🏽 \n #$%(s<|endoftext|>'VE", "tokens": 29, "pieces": ["ع", "‍'", "ſDž", "123", "456", "78", "😀🏽", " \n", " ", " #$%(", "s", "<|", "endoftext", "|>'", "VE"]} +{"text": "‍👍🏽", "tokens": 8, "pieces": ["‍👍🏽"]} +{"text": "å　字­👍🏽ésm٣٤٥٦Dže12345678\r\n,9İ'Re
Ａ", "tokens": 37, "pieces": ["a", "̊", "　字", "­👍🏽", "ésm", "٣٤٥", "٦", "Dže", "123", "456", "78", "\r\n", ",", "9", "İ", "'Re", "
Ａ"]} +{"text": "#$%Zd (\r\n,'ſ漢\r\nDž#$%‍Ａ𐞁", "tokens": 24, "pieces": ["#$%", "Zd", " ", "(\r\n", ",'", "ſ漢", "\r\n", "Dž", "#$%‍", "Ａ𐞁"]} +{"text": "(a'D's<|fim_prefix|>(Z åt😀🏽EOTe!\r\na'TEOT…!", "tokens": 32, "pieces": ["(a", "'D", "'s", "<|", "fim", "_prefix", "|>(", "Z", " ", " a", "̊t", "😀🏽", "EOTe", "!\r\n", "a", "'T", "EOT", "…", "!"]} +{"text": "\u000b'De\ns", "tokens": 5, "pieces": ["\u000b", "'D", "e", "\n", "s"]} +{"text": "s", "tokens": 1, "pieces": ["s"]} +{"text": "'Refi EOTſ 𐞁\u000b字!'ReⅣ'ſ", "tokens": 23, "pieces": ["'Re", "fi", " ", " EOTſ", " ", " 𐞁", "\u000b字", "!'", "Re", "Ⅳ", "'ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "- 'llＡſ İ.\r\n\r\n𐞁#$%½9\u000b㋿\u000b　​m'St‍å0ſ… ", "tokens": 46, "pieces": ["-", " '", "llＡſ", " <", "EOT", ">İ", ".\r\n\r\n", "𐞁", "#$%", "½9", "\u000b", "㋿", "\u000b", "　", "​m", "'S", "t", "‍a", "̊", "0", "ſ", "… "]} +{"text": "\r\n'Rete\u000bfi'M\rꟲ9'll㍿ḍ̇ és'9Dž‍'T!​", "tokens": 33, "pieces": ["\r\n", "'Re", "te", "\u000bfi", "'M", "\r", "ꟲ", "9", "'ll", "㍿ḋ", "̣", " ", " és", "'", "9", "Dž", "‍'", "T", "!​"]} +{"text": "‍Ａ'\r\n…\u000b#$%.9'T12345678
\u000b mſ\r\n\r\n", "tokens": 27, "pieces": ["‍Ａ", "'\r\n", "…", "\u000b", "#$%.", "9", "'T", "123", "456", "78", "
\u000b", " m", "ſ", "\r\n\r\n"]} +{"text": "🙂Ａ, 12345678…㋿­३٣٤٥٦12345678Ａ३\r#$%<​­å", "tokens": 46, "pieces": ["🙂Ａ", ",", " ", "123", "456", "78", "…", "㋿­", "३٣٤", "٥٦1", "234", "567", "8", "Ａ", "३", "\r", "#$%<​­", "a", "̊"]} +{"text": "㍿é'M\t'S!!s\n,aİZé12345678‍­('M\"å😀🏽'M<|endoftext|>fiḍ̇d \n𐞁", "tokens": 54, "pieces": ["㍿e", "́'", "M", "\t", "'S", "!!", "s", "\n", ",aİZé", "123", "456", "78", "‍­('", "M", "\"a", "̊😀🏽'", "M", "<|", "endoftext", "|>", "fiḋ", "̣d", " \n", "𐞁"]} +{"text": " '😀🏽ḍ̇<|endoftext|><|endoftext|>Ⅳ\" 'ḍ̇🙂'S'S'll m'VE Ⅳ \n'🙂字'ſ'VE\rEOT\r‍‍!!\r9", "tokens": 69, "pieces": [" ", " '😀🏽", "ḋ", "̣<|", "endoftext", "|><|", "endoftext", "|>", "Ⅳ", "\"", " ", " '", "ḋ", "̣🙂'", "S", "'S", "'ll", " m", "'VE", " ", "Ⅳ", " \n", "'🙂", "字", "'ſ", "'VE", "\r", "EOT", "\r", "‍‍!!\r", "9"]} +{"text": "0<|fim_prefix|>(Dž \n㍿\u000bDž<…​漢Ａ(,.12345678٣٤٥٦\"字​'Re㍿\r\n\r\n‍㍿", "tokens": 52, "pieces": ["0", "<|", "fim", "_prefix", "|>(", "Dž", " \n", "㍿", "\u000bDž", "<", "…", "​漢Ａ", "(,.", "123", "456", "78٣", "٤٥٦", "\"字", "​'", "Re", "㍿\r\n\r\n", "‍㍿"]} +{"text": "漢३DžⅣ\n'ſ३fi \n 'Re'MＡ<|fim_prefix|>\r\n\r\n\r\n-ꟲ½!!<|endoftext|>½ 's½\r\n३ḍ̇\r\n\r\nſ
𐞁Ⅳa　‍٣٤٥٦", "tokens": 81, "pieces": ["漢", "३", "Dž", "Ⅳ", "\n", "'", "ſ", "३", "fi", " \n", " ", " '", "Re", "'M", "Ａ", "<|", "fim", "_prefix", "|>\r\n\r\n\r\n", "-ꟲ", "½", "!!<|", "endoftext", "|>", "½", " ", " '", "s", "½", "\r\n", "३", "ḋ", "̣\r\n\r\n", "ſ", "
𐞁", "Ⅳ", "a", "　", "‍", "٣٤٥", "٦"]} +{"text": "\t\r\n<am
>('S're\n㋿'Re\u000b‍\"́ \n<ß0
t 're''½>­\"'S'reDž-", "tokens": 44, "pieces": ["\t", "\r\n", "<<", "META", "_START", ">am", "
", ">('", "S", "'re", "\n", "㋿'", "Re", "\u000b", "‍\"́", " \n", "<ß", "0", "
t", " ", "'re", "''", "½", ">­\"'", "S", "'re", "Dž", "-"]} +{"text": "\r\n<𐞁9'M​ß🙂'D", "tokens": 14, "pieces": ["\r\n", "<𐞁", "9", "'M", "​ß", "🙂'", "D"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "…ꟲ 𐞁'Té​Z(é\u000bＡ\n\u000b!\"!!#$%  ſ\r0're 'T(٣٤٥٦<|fim_prefix|>\r're<|endoftext|>12345678Ⅳ㍿\r 😀🏽", "tokens": 77, "pieces": ["…ꟲ", " 𐞁", "'T", "e", "́​", "Z", "(e", "́", "\u000bＡ", "\n", "\u000b", "!\"!!#$%", " ", " ſ", "\r", "0", "'re", " ", "'T", "(", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\r", "'", "re", "<|", "endoftext", "|>", "123", "456", "78Ⅳ", "㍿\r", " 😀🏽"]} +{"text": "#$%eİ\nḍ̇", "tokens": 10, "pieces": ["#$%", "eİ", "\n", "ḋ", "̣"]} +{"text": "
-­😀🏽", "tokens": 9, "pieces": ["
", "-­😀🏽"]} +{"text": "(Ⅳß　'ſ-<|endoftext|>", "tokens": 16, "pieces": ["(", "Ⅳ", "ß", "　", "'ſ", "-<|", "endoftext", "|>"]} +{"text": "mt0#$%sＡ\nع 'T\r\n\r\nfiå(Džḍ̇İ \n\r", "tokens": 36, "pieces": ["mt", "0", "#$%", "sＡ", "\n", "ع", " ", "'T", "\r\n\r\n", "fia", "̊(<", "EOT", ">Džḋ", "̣İ", " \n\r"]} +{"text": "İ㋿#$%EOT𐞁३­#$%­-ꟲåḍ̇ⅣⅣḍ̇t's漢漢ꟲs'M'VEs\r\n½ İEOT٣٤٥٦Ａ'll'M12345678", "tokens": 74, "pieces": ["İ", "㋿#$%", "EOT𐞁", "३", "­#$%­-", "ꟲa", "̊ḋ", "̣", "ⅣⅣ", "ḋ", "̣t", "'s", "漢漢ꟲs", "'M", "'VE", "s", "\r\n", "½", " İEOT", "٣٤٥", "٦", "Ａ", "'ll", "'M", "123", "456", "78"]} +{"text": "ſd\r\n\r\ne!!12345678'a'Dſ漢  EOT<|endoftext|>t!! -',\n🙂<|endoftext|> <|endoftext|>", "tokens": 48, "pieces": ["ſd", "\r\n\r\n", "e", "!!", "123", "456", "78", "'a", "'D", "ſ漢", " ", " EOT", "<|", "endoftext", "|>", "t", "!!", " ", " -',\n", "🙂<|", "endoftext", "|>", " ", "<|", "endoftext", "|>"]} +{"text": "!.Z'Dfié", "tokens": 7, "pieces": ["!.", "Z", "'D", "fie", "́"]} +{"text": "​Zé<#$%12345678d
EOT३́\r\n\r\né­!EOTå'T'T३
\r\n12345678
", "tokens": 40, "pieces": ["​Ze", "́<#$%", "123", "456", "78", "d", "
EOT", "३", "́\r\n\r\n", "e", "́­!", "EOTa", "̊'", "T", "'T", "३", "
\r\n", "123", "456", "78", "
"]} +{"text": "ſ🙂'lléع́>😀🏽'VEe#$%😀🏽\tfiDžé'Re‍🙂ß'<Ⅳ9​'M㋿\r\nd'S　\t>\n", "tokens": 64, "pieces": ["ſ", "🙂'", "lle", "́ع", "́>😀🏽'", "VEe", "#$%😀🏽", "\tfiDžé", "'Re", "‍🙂", "ß", "'<", "Ⅳ9", "​'", "M", "㋿\r\n", "d", "'S", "　", "", "\t", ">\n"]} +{"text": "İas👍🏽'ſé'llع\tDžé12345678!é!<İ\r\nfi　'VE", "tokens": 42, "pieces": ["İ", "as", "👍🏽'", "ſe", "́'", "ll", "ع", "\tDže", "́", "123", "456", "78", "!e", "́!<", "İ", "\r\n", "fi", "　", "'VE"]} +{"text": "Zdꟲ(EOT'T\n", "tokens": 9, "pieces": ["Zdꟲ", "(EOT", "'T", "\n"]} +{"text": "字'D­'M

'll>İ(½'s३!㍿‍İ's#$%ééaß're.'T \n\"\r\nß­٣٤٥٦ꟲfi😀🏽", "tokens": 57, "pieces": ["字", "'D", "­'", "M", "
", "
", "'ll", ">İ", "(", "½", "'s", "३", "!㍿‍", "İ", "'s", "#$%", "e", "́éaß", "'re", ".'", "T", " \n", "\"\r\n", "ß", "­", "٣٤٥", "٦", "ꟲfi", "😀🏽"]} +{"text": "9('VEß 'T<|fim_prefix|><|endoftext|>🙂\t३>!!å'D㋿(", "tokens": 49, "pieces": ["9", "('", "VEß", "", " ", "'", "T", "<|", "fim", "_prefix", "|><|", "endoftext", "|>🙂", "\t", "३", ">!!", "a", "̊'", "D", "㋿("]} +{"text": "½'D́e'sßEOT >'ReDžt", "tokens": 17, "pieces": ["½", "'", "D", "́e", "'s", "ßEOT", " ", ">'", "ReDžt"]} +{"text": "‍­", "tokens": 3, "pieces": ["‍­"]} +{"text": "<|endoftext|> 'M㋿ß\r\nⅣ㋿​'Re\ne\n 12345678　'Reḍ̇'VE\"ḍ̇", "tokens": 49, "pieces": ["<|", "endoftext", "|>", " '", "M", "㋿ß", "\r\n", "", "Ⅳ", "㋿​'", "Re", "\n", "e", "\n", " ", " ", "123", "456", "78", "　", "'Re", "ḋ", "̣'", "VE", "\"ḋ", "̣"]} +{"text": "é'SZß\r\n\r\n\u000b<|endoftext|>!! 　\u000b >(ḍ̇Džds😀🏽'll'", "SZß", "\r\n\r\n", "\u000b", "<|", "endoftext", "|>!!", " 　\u000b ", " >(", "ḋ", "̣Džds", "😀🏽'", "ll", "३#$%a'ſ'S­
's字m", "tokens": 24, "pieces": ["३", "\"<|", "endoftext", "|>", "३", "#$%", "a", "'ſ", "'S", "­", "
", "'s", "字m"]} +{"text": "'re٣٤٥٦‍ḍ̇'S'ſ漢m…३ſꟲ‍½  'ſ<|fim_prefix|>́'Re'Reع\té,ḍ̇.ß \n字'ſ", "tokens": 71, "pieces": ["'re", "٣٤٥", "٦", "‍ḋ", "̣'", "S", "'ſ", "漢m", "…", "३", "ſꟲ", "‍", "½", " ", " ", "'ſ", "<|", "fim", "_prefix", "|>́'", "Re", "'Re", "ع", "\te", "́,", "ḋ", "̣.", "ß", " \n", "字", "'ſ", ""]} +{"text": "-d(\u000b\"३fi३ \n🙂\r'T'M", "tokens": 16, "pieces": ["-d", "(", "\u000b", "\"", "३", "fi", "३", " \n", "🙂\r", "'T", "'M"]} +{"text": "\r\n字!\n0٣٤٥٦👍🏽㍿ⅣZ𐞁ß㍿\" é,Z 
३'M", "tokens": 44, "pieces": ["\r\n", "字", "!\n", "0٣٤", "٥٦", "👍🏽㍿", "Ⅳ", "Z𐞁ß", "㍿\"", " é", ",Z", " ", "
", "३", "'M"]} +{"text": ">'d12345678\"…ßⅣ\n12345678Džع𐞁<|endoftext|><|endoftext|>\r'llEOTd½ 'T😀🏽0'​EOT\"٣٤٥٦㋿'ll12345678'll'll,'D", "tokens": 78, "pieces": ["><", "EOT", ">'", "d", "123", "456", "78", "\"", "…ß", "Ⅳ", "\n", "123", "456", "78", "Džع𐞁", "<|", "endoftext", "|><|", "endoftext", "|>\r", "'ll", "EOTd", "½", " ", " '", "T", "😀🏽", "0", "'​", "EOT", "\"", "٣٤٥", "٦", "㋿'", "ll", "123", "456", "78", "'ll", "'ll", ",'", "D"]} +{"text": "­", "tokens": 1, "pieces": ["­"]} +{"text": "Džå\u000b'Re🙂é>'re", "tokens": 13, "pieces": ["Dža", "̊", "\u000b", "'Re", "🙂e", "́>'", "re"]} +{"text": "٣٤٥٦d,\n'll'sعfi \r\n!!é!\t'Sfi😀🏽(­", "tokens": 31, "pieces": ["٣٤٥", "٦", "d", ",\n", "'ll", "'s", "عfi", " \r\n", "!!", "é", "!", "\t", "'S", "fi", "😀🏽(­"]} +{"text": "🙂'sＡå \ts漢Z'll-éع", "tokens": 18, "pieces": ["🙂'", "sＡa", "̊", " ", "\ts漢Z", "'ll", "-e", "́ع"]} +{"text": "'Re½\r\nm", "tokens": 4, "pieces": ["'Re", "½", "\r\n", "m"]} +{"text": ">'s!!　漢३<|endoftext|>Ａعa>!!<|endoftext|>\r!s\r\n‍ Dž \nⅣ's#$% \r\né٣٤٥٦<|endoftext|>'D0Z\t", "tokens": 71, "pieces": [">'", "s", "!!<", "META", "_START", ">", "　漢", "३", "<|", "endoftext", "|>", "Ａعa", ">!!<|", "endoftext", "|>\r", "!s", "\r\n", "‍", " ", " Dž", " \n", "Ⅳ", "'s", "#$%", " \r\n", "e", "́", "٣٤٥", "٦", "<|", "endoftext", "|>'", "D", "0", "Z", "\t"]} +{"text": "'s,ßDž'Re'!!!e9'Då\r\nİ \n-㍿d\u000b'T", "tokens": 24, "pieces": ["'s", ",ßDž", "'Re", "'!!!", "e", "9", "'D", "a", "̊\r\n", "İ", " \n", "-㍿", "d", "\u000b", "'T"]} +{"text": "\u000b'T9　😀🏽'Té\n're\t́ß!'M　<…
­😀🏽Ⅳa", "tokens": 34, "pieces": ["\u000b", "'T", "9", "　", "😀🏽'", "Té", "\n", "'re", "\t", "́ß", "!'", "M", "　", "<", "…", "
", "­😀🏽", "Ⅳ", "a"]} +{"text": "ḍ̇ꟲ‍m ­　'Re㋿,'M ㍿-EOT‍́ \nع\n'ḾDž \n\"é'Red½éEOT'S", "tokens": 48, "pieces": ["ḋ", "̣ꟲ", "‍m", " ", " ­", "　", "'Re", "㋿,'", "M", " ", " ㍿-", "EOT", "‍́", " \n", "ع", "\n", "'M", "́Dž", " \n", "\"é", "'Re", "d", "½", "éEOT", "'S"]} +{"text": "'Re \n's­ع㋿字é \nEOT'Re\" 'M😀🏽EOTḍ̇0<|fim_prefix|>漢-!३½Ⅳ𐞁ꟲ's", "tokens": 59, "pieces": ["'Re", " \n", "'s", "­ع", "㋿字e", "́", " \n", "EOT", "'Re", "\"", " ", "'M", "😀🏽", "EOTḋ", "̣", "0", "<|", "fim", "_prefix", "|>", "漢", "-!", "३½Ⅳ", "𐞁ꟲ", "'s"]} +{"text": "fi!fi<‍Ａ‍́😀🏽\r\n\r\n'll\"Ａ漢'Re<|fim_prefix|><|fim_prefix|><‍㍿", "tokens": 44, "pieces": ["fi", "!fi", "<‍", "Ａ", "‍́😀🏽\r\n\r\n", "'ll", "\"Ａ漢", "'Re", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><‍㍿"]} +{"text": "👍🏽'VE'Reé👍🏽d\" \n-\"𐞁d<|endoftext|>\r\n­EOT(12345678३'reＡ㋿", "tokens": 57, "pieces": ["👍🏽'", "VE", "'Re", "é", "👍🏽", "d", "\"", " \n", "-\"", "𐞁", "d", "<|", "endoftext", "|>\r\n", "­", "EOT", "(", "123", "456", "78३", "'re", "Ａ", "㋿"]} +{"text": "Ａå🙂.Ａ㋿½EOTꟲ12345678İ'M's", "tokens": 29, "pieces": ["Ａa", "̊🙂.", "Ａ", "㋿<", "META", "_START", ">", "½", "EOTꟲ", "123", "456", "78", "İ", "'M", "'s"]} +{"text": "-Džt,#$%EOT's0ꟲ
👍🏽mé३ꟲ'S. ع字ß\"''re
å'S　\u000bé ", "tokens": 53, "pieces": ["-Džt", ",#$%", "EOT", "'s", "0", "ꟲ", "
", "👍🏽", "mé", "३", "ꟲ", "'S", ".<", "META", "_START", ">", " ", " ع字ß", "\"''", "re", "
a", "̊'", "S", "　", "\u000be", "́", " "]} +{"text": "३ dDž", "tokens": 12, "pieces": ["", "३", " ", " <", "META", "_START", ">dDž"]} +{"text": "‍'ReDžfiİé'ſ\r'ſꟲＡ٣٤٥٦👍🏽e½\r\né.'M'", "tokens": 44, "pieces": ["‍'", "ReDžfiİé", "'ſ", "\r", "'ſ", "ꟲＡ", "٣٤٥", "٦", "👍🏽", "e", "½", "\r\n", "e", "́.'", "M", "'"]} +{"text": "Ⅳ　́0 \n>a

a½'<|fim_prefix|>𐞁ع(t\"<|fim_prefix|>'ſDžt", "tokens": 42, "pieces": ["", "Ⅳ", "　", "́", "0", " \n", ">a", "
", "
a", "½", "'<|", "fim", "_prefix", "|>", "𐞁ع", "(t", "\"<|", "fim", "_prefix", "|>'", "ſDžt"]} +{"text": "\r\n\r\n​३<|endoftext|>­'ll\u000b　s㋿a
", "tokens": 23, "pieces": ["\r\n\r\n", "​", "३", "<|", "endoftext", "|>­'", "ll", "\u000b", "　s", "㋿a", "
"]} +{"text": "\u000b<|endoftext|>0\n9𐞁fi'Så\u000b‍>㍿é𐞁m　İtéé (12345678", "tokens": 47, "pieces": ["\u000b", "<|", "endoftext", "|>", "0", "\n", "9", "𐞁fi", "'S", "a", "̊", "\u000b", "‍>㍿", "e", "́𐞁m", "　İ", "téé", " ", "(", "123", "456", "78"]} +{"text": "½'ſfi👍🏽‍\u000b\r\nEOT \r\n\r\nfi'M12345678\u000bdＡ9 \"٣٤٥٦<<|fim_prefix|> ", "tokens": 52, "pieces": ["", "½", "'ſ", "fi", "👍🏽‍", "\u000b\r\n", "EOT", " \r\n\r\n", "fi", "'M", "123", "456", "78", "\u000bdＡ", "9", " \"", "٣٤٥", "٦", "<<|", "fim", "_prefix", "|>", " "]} +{"text": "(\n漢EOT😀🏽", "tokens": 10, "pieces": ["(\n", "漢EOT", "😀🏽"]} +{"text": "ſ㍿aZ ß'Re'é…​ＡعⅣéfi12345678½", "tokens": 27, "pieces": ["ſ", "㍿aZ", " ß", "'Re", "'e", "́", "…", "​Ａع", "Ⅳ", "éfi", "123", "456", "78½"]} +{"text": "İ'D㍿ \n字é‍ßd😀🏽ḍ̇a>'ſ漢 >12345678ß<>'ll fi'😀🏽<|fim_prefix|>", "tokens": 53, "pieces": ["İ", "'D", "㍿", " \n", "字e", "́‍", "ßd", "😀🏽", "ḋ", "̣a", ">'", "ſ漢", " >", "123", "456", "78", "ß", "<>'", "ll", " fi", "'😀🏽<|", "fim", "_prefix", "|>"]} +{"text": "\"'M\t\r\ns12345678.éé\"\tß‍\"!!é-　
a#$%ß　 \n 'VE", "tokens": 36, "pieces": ["'Re", "½", "👍🏽", "½", "'ſ", "ſ", "!<|", "endoftext", "|>\"!!", "é", "-", "　", "
a", "#$%", "ß", "　 \n", " ", " '", "VE"]} +{"text": "㍿!Ａ字Dž", "tokens": 9, "pieces": ["㍿!", "Ａ字Dž"]} +{"text": "٣٤٥٦'Sꟲ\r\n\r\n😀🏽é(!>'re字é㋿e
-́​😀🏽́\r\néDž<|fim_prefix|>𐞁​​'llå\"😀🏽", "tokens": 71, "pieces": ["٣٤٥", "٦", "'S", "ꟲ", "\r\n\r\n", "😀🏽", "e", "́(!>'", "re字e", "́㋿", "e", "
", "-́<", "EOT", ">​😀🏽́\r\n", "éDž", "<|", "fim", "_prefix", "|>", "𐞁", "​​'", "lla", "̊\"😀🏽"]} +{"text": "…

…‍Ⅳ'D'ſ'll.½fi\r\n\r\n-9<|endoftext|> 漢!!", "tokens": 36, "pieces": ["…

", "…", "‍", "Ⅳ", "'D", "'ſ", "'ll", ".", "½", "fi", "\r\n\r\n", "-", "9", "<|", "endoftext", "|>", " ", " 漢", "!!"]} +{"text": "'D́
's​EOT٣٤٥٦İꟲ\r\n\r\nＡ 9dß'Sm<|endoftext|>😀🏽
's", "tokens": 60, "pieces": ["'D", "́", "
", "'s", "​EOT", "٣٤٥", "٦", "İꟲ", "\r\n\r\n", "Ａ", " ", " ", "9", "d", "", "ß", "'S", "m", "<|", "endoftext", "|>😀🏽", "
", "'s"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ع\n'D(ß㍿‍-'D😀🏽ß½Z'll\u000b'reİ", "tokens": 27, "pieces": ["ع", "\n", "'D", "(ß", "㍿‍-'", "D", "😀🏽", "ß", "½", "Z", "'", "ll", "\u000b", "'re", "İ"]} +{"text": "Ⅳ!!👍🏽Ⅳ🙂ſ ㋿!>½\rß½ ''D<fi'ſ", "tokens": 40, "pieces": ["Ⅳ", "!!👍🏽", "Ⅳ", "🙂ſ", " ", "㋿!>", "½", "\r", "ß", "", "½", " ", " ''", "D", "<fi", "'ſ"]} +{"text": "́EOT'D<‍!'S9字🙂\nå<|fim_prefix|>.a\t\r\né!!0<|endoftext|>Ⅳİ'll!\" 0", "tokens": 44, "pieces": ["́EOT", "'D", "<‍!'", "S", "9", "字", "🙂\n", "a", "̊<|", "fim", "_prefix", "|>.", "a", "\t\r\n", "e", "́!!", "0", "<|", "endoftext", "|>", "Ⅳ", "İ", "'ll", "!\"", " ", "0"]} +{"text": "<9\u000b'ſmⅣ'MßtＡ😀🏽ꟲ'reع ع'M<,ſİé'ſ-\r\n9Ａ­漢　9👍🏽.३mt", "tokens": 59, "pieces": ["<", "9", "\u000b", "'ſ", "m", "Ⅳ", "'M", "ßtＡ", "😀🏽", "ꟲ", "'re", "ع", " ع", "'M", "<,", "ſİe", "́'", "ſ", "-\r\n", "9", "Ａ", "­漢", "　", "9", "👍🏽<", "EOT", ">.", "३", "mt"]} +{"text": "0'M", "tokens": 2, "pieces": ["0", "'M"]} +{"text": "åe𐞁
\r,ß'VEa12345678ḍ̇a'Ree.> 
́😀🏽'T㋿Z,'ſ\r\n 𐞁åé\t", "tokens": 62, "pieces": ["a", "̊e𐞁", "
\r", ",ß", "'VE", "a", "123", "456", "78", "ḋ", "̣a", "'Re", "e", ".>", " ", "
", "́😀🏽'", "T", "㋿Z", ",'", "ſ", "\r\n", " ", " 𐞁a", "̊é", "\t"]} +{"text": "'ſ́é \n½\n's𐞁>a​ꟲfi-Z'ſ\r\n㋿ é \n
Dž…́'M…­éⅣ😀🏽字ſ're'Re­d'VE", "tokens": 61, "pieces": ["'ſ", "́é", " \n", "½", "\n", "'s", "𐞁", ">a", "​ꟲfi", "-Z", "'ſ", "\r\n", "㋿", " e", "́", " \n", "
Dž", "…", "́'", "M", "…", "­e", "́", "Ⅳ", "😀🏽", "字ſ", "'re", "'Re", "­d", "'VE"]} +{"text": "\r\n\r\nt 😀🏽ßfi'reé'ſe\u000b漢!ß　'Reſ'Mع<|endoftext|>ſ Ａfi-s字㋿", "tokens": 48, "pieces": ["\r\n\r\n", "t", " ", "😀🏽", "ßfi", "'re", "e", "́'", "ſe", "\u000b漢", "!ß", "　", "'Re", "ſ", "'M", "ع", "<|", "endoftext", "|>", "ſ", " ", " Ａfi", "-s字", "㋿"]} +{"text": "\n👍🏽!!9'Re😀🏽", "tokens": 15, "pieces": ["\n", "👍🏽!!", "9", "'Re", "😀🏽"]} +{"text": "<|fim_prefix|>½'½　'D", "tokens": 15, "pieces": ["<|", "fim", "_prefix", "|>", "½", "'", "½", "　", "'", "D"]} +{"text": "٣٤٥٦Ⅳ", "tokens": 10, "pieces": ["٣٤٥", "٦Ⅳ"]} +{"text": "#$% \nꟲ\"Ａ.😀🏽 9're( ٣٤٥٦ ㍿😀🏽", "tokens": 38, "pieces": ["#$%", " \n", "ꟲ", "\"Ａ", ".😀🏽", " ", " ", "9", "'re", "(", " ", "٣٤٥", "٦", " ", "㍿😀🏽"]} +{"text": "'re'T 😀🏽½EOTſ\r\n\r\n㋿912345678<|endoftext|>'ReeＡ \n12345678Z,\n'reḍ̇ḍ̇\n", "tokens": 49, "pieces": ["'re", "'T", " ", "😀🏽", "½", "EOTſ", "\r\n\r\n", "㋿", "912", "345", "678", "<|", "endoftext", "|>'", "ReeＡ", " \n", "123", "456", "78", "Z", ",\n", "'re", "ḋ", "̣ḋ", "̣\n"]} +{"text": "​.ſ", "tokens": 4, "pieces": ["​.", "ſ"]} +{"text": "㍿Ⅳ\téfia\t́,''M> İ३", "tokens": 19, "pieces": ["㍿", "Ⅳ", "\te", "́fia", "\t", "́,''", "M", ">", " ", " İ", "३"]} +{"text": ">👍🏽e㋿\né​ ३('D", "tokens": 23, "pieces": [">👍🏽", "e", "㋿\n", "e", "́​", " ", "३", "('", "D"]} +{"text": "\n'>'Ddé🙂12345678å'Mع0 9 ३㍿!!", "tokens": 29, "pieces": ["\n", "'>'", "Dde", "́🙂", "123", "456", "78", "a", "̊'", "Mع", "0", " ", " ", "9", " ", " ", "३", "㍿!!"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "İåfiå \n­\r\n\r\nt​.ꟲ­-'llꟲ\r\né!!DžＡ!d🙂\"　a'D \né‍ßde(fi　\n🙂", "tokens": 54, "pieces": ["İa", "̊fia", "̊", " \n", "­\r\n\r\n", "t", "​.", "ꟲ", "­-'", "llꟲ", "\r\n", "e", "́!!", "DžＡ", "!d", "🙂\"", "　a", "'D", " \n", "e", "́‍", "ßde", "(fi", "　\n", "🙂"]} +{"text": "\n­\nꟲ½e\n😀🏽…عİ \n㍿!\"'Re", "tokens": 25, "pieces": ["\n", "­\n", "ꟲ", "½", "e", "\n", "😀🏽", "…عİ", " \n", "㍿!\"'", "Re"]} +{"text": "e><<|fim_prefix|>å'Dع9<'VEé'D…½३!　'S😀🏽's👍🏽", "tokens": 43, "pieces": ["e", "><<|", "fim", "_prefix", "|>", "a", "̊'", "Dع", "9", "<'", "VEe", "́'", "D", "…", "½३", "!", "　", "'S", "😀🏽'", "s", "👍🏽"]} +{"text": "　", "tokens": 1, "pieces": ["　"]} +{"text": " \u000b'M\nEOT　 \n<|fim_prefix|>'SEOTſ'll'VE‍Dž'll字'\r9.'reع　", "tokens": 37, "pieces": [" ", "\u000b", "'M", "\n", "EOT", "　 \n", "<|", "fim", "_prefix", "|>'", "SEOTſ", "'ll", "'VE", "‍Dž", "'ll", "字", "'\r", "9", ".'", "reع", "　"]} +{"text": "t (12345678😀🏽字½ع'DⅣ👍🏽㍿\"👍🏽<|endoftext|>İfiꟲⅣ(", "tokens": 52, "pieces": ["t", " ", "(", "123", "456", "78", "😀🏽", "字", "½", "ع", "'D", "Ⅳ", "👍🏽㍿\"👍🏽<", "EOT", "><|", "endoftext", "|>", "İfiꟲ", "Ⅳ", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\n \n12345678👍🏽İ'llZ!!ſ\u000b\"<㋿Džtİ'ReİＡa👍🏽'Så\t<½,. d'T", "tokens": 57, "pieces": ["\n \n", "123", "456", "78", "👍🏽", "İ", "'ll", "Z", "!!", "ſ", "\u000b", "\"<㋿", "Dž", "tİ", "'Re", "İＡa", "👍🏽'", "Sa", "̊", "\t", "<", "½", ",.", " d", "'", "T"]} +{"text": "ḍ̇㍿ḍ̇", "tokens": 13, "pieces": ["ḋ", "̣㍿", "ḋ", "̣"]} +{"text": "'VE㍿EOT\nİ", "tokens": 9, "pieces": ["'VE", "㍿EOT", "\n", "İ"]} +{"text": "Ⅳ́ 's \n", "tokens": 6, "pieces": ["Ⅳ", "́", " '", "s", " \n"]} +{"text": "(", "tokens": 10, "pieces": ["a", "̊<|", "endoftext", "|>("]} +{"text": "!‍ \n'Tſ9<𐞁Ⅳ\rt'sfi!!", "tokens": 21, "pieces": ["!‍", " \n", "'T", "ſ", "9", "<𐞁", "Ⅳ", "\r", "t", "'s", "fi", "!!"]} +{"text": "ḍ̇‍Ａ'Mſİع́İ0fi", "tokens": 23, "pieces": ["ḋ", "̣‍", "Ａ", "'M", "ſİع", "́<", "EOT", ">İ", "0", "fi"]} +{"text": "ß👍🏽 ꟲ.'D­字\"😀🏽'll>ع< ſ \r\n\r\n
𐞁😀🏽Z‍٣٤٥٦e", "tokens": 55, "pieces": ["ß", "👍🏽", " ꟲ", ".'", "D", "­字", "\"😀🏽'", "ll", ">", "ع", "<", " ſ", " \r\n\r\n", "
𐞁", "😀🏽", "Z", "‍", "٣٤٥", "٦", "e"]} +{"text": "㋿\"'sꟲ. åéé(😀🏽", "tokens": 20, "pieces": ["㋿\"'", "sꟲ", ".", " a", "̊éé", "(😀🏽"]} +{"text": "!", "tokens": 1, "pieces": ["!"]} +{"text": "<ḍ̇\"　'Re!!#$%​!
😀🏽'T", "tokens": 27, "pieces": ["<ḋ", "̣\"", "　", "'Re", "!!#$%​!", "
", "😀🏽'", "T"]} +{"text": "'VE'T­​Ａ #$%t​　\r\n\r\nꟲm0å'३!\tfit<|fim_prefix|>\r12345678'D.'VE<|fim_prefix|>'re", "tokens": 52, "pieces": ["'VE", "'T", "­​", "Ａ", " ", "#$%", "t", "​", "　\r\n\r\n", "ꟲm", "0", "a", "̊'", "३", "!", "\tfit", "<|", "fim", "_prefix", "|>\r", "123", "456", "78", "'D", ".'", "VE", "<|", "fim", "_prefix", "|>'", "re"]} +{"text": "ZZß👍🏽٣٤٥٦d'D漢𐞁\r\n'll'ſ…😀🏽!!\"㍿", "tokens": 44, "pieces": ["ZZ", "ß", "👍🏽", "٣٤٥", "٦", "d", "'D", "漢𐞁", "\r\n", "'ll", "'ſ", "…", "😀🏽!!\"㍿"]} +{"text": "½'Mdꟲ　\"9Ａfi's'D'ſåé
\r\n\n!ḍ̇\"\r\n'T0ḍ̇👍🏽漢 漢12345678👍🏽ع!\"", "tokens": 64, "pieces": ["½", "'M", "dꟲ", "　", "\"", "9", "Ａfi", "'", "s", "'D", "'ſ", "a", "̊é", "
\r\n\n", "!ḋ", "̣\"\r\n", "'T", "0", "ḋ", "̣👍🏽", "漢", " 漢", "123", "456", "78", "👍🏽", "ع", "!\""]} +{"text": "EOT\r\n\r\n'MZ'D<|endoftext|>ḍ̇'T\"'ſ<|endoftext|>", "tokens": 32, "pieces": ["EOT", "\r\n\r\n", "'", "MZ", "'D", "<|", "endoftext", "|>", "ḋ", "̣'", "T", "\"'", "ſ", "<|", "endoftext", "|>"]} +{"text": "İ\rß>t٣٤٥٦éfi", "tokens": 16, "pieces": ["İ", "\r", "ß", ">t", "٣٤٥", "٦", "e", "́fi"]} +{"text": " ſ!!!\r\n\r\n㍿'TDž३é!'Ś'll9३ḍ̇", "tokens": 33, "pieces": [" ſ", "!!!\r\n\r\n", "㍿'", "TDž", "३", "e", "́!'", "S", "́'", "ll", "9३", "ḋ", "̣"]} +{"text": "\"#$%é🙂fi", "tokens": 8, "pieces": ["\"#$%", "é", "🙂fi"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "(٣٤٥٦३𐞁\r-'D́'漢'ſعEOT.ſt\u000b'…-<|fim_prefix|>😀🏽٣٤٥٦\"12345678ꟲs𐞁\u000b<|fim_prefix|>ḍ̇EOT-,", "tokens": 88, "pieces": ["(", "٣٤٥", "٦३", "𐞁", "\r", "-'", "D", "́'", "漢", "'ſ", "عEOT", ".ſt", "\u000b", "'", "…", "-<|", "fim", "_prefix", "|>😀🏽", "٣٤٥", "٦", "\"", "123", "456", "78", "ꟲs𐞁", "", "\u000b", "<|", "fim", "_prefix", "|>", "ḋ", "̣EOT", "-,"]} +{"text": "-­(Ⅳe>Zع,", "tokens": 13, "pieces": ["-­(", "Ⅳ", "e", ">Z", "ع", ","]} +{"text": "㋿\u000b㍿\r\nİ'\r😀🏽'ſ😀🏽a!!'ſ\t>.漢٣٤٥٦ 's㋿9.ås \n\r\n\r\n0 \n'll\"㋿<|endoftext|>m", "tokens": 72, "pieces": ["㋿", "\u000b", "㍿\r\n", "İ", "'\r", "😀🏽'", "ſ", "😀🏽", "a", "!!'", "ſ", "\t", ">.", "漢", "٣٤٥", "٦", " ", " <", "META", "_START", ">'", "s", "㋿", "9", ".a", "̊s", " \n\r\n\r\n", "0", " \n", "'ll", "\"㋿<|", "endoftext", "|>", "m"]} +{"text": "ſ　eⅣt're漢<ḍ̇'Dd'DEOT're​", "tokens": 30, "pieces": ["ſ", "　e", "Ⅳ", "t", "'re", "漢", "<ḋ", "̣'", "Dd", "'", "D", "EOT", "'re", "​"]} +{"text": "\u000bEOT00<|endoftext|>‍Z,ḍ̇12345678<|fim_prefix|>sEOTs­am <|fim_prefix|>å👍🏽sＡ漢", "tokens": 60, "pieces": ["\u000bEOT", "00", "<|", "endoftext", "|>‍", "Z", ",<", "EOT", ">ḋ", "̣", "123", "456", "78", "<|", "fim", "_prefix", "|>", "sEOTs", "­am", " ", " <|", "fim", "_prefix", "|>", "a", "̊👍🏽", "sＡ漢"]} +{"text": "<|endoftext|>9<ꟲEOTⅣfi!tⅣ ß\r\n\r\n​-३'Re", "tokens": 31, "pieces": ["<|", "endoftext", "|>", "9", "<ꟲEOT", "Ⅳ", "fi", "!t", "Ⅳ", " ", " ß", "\r\n\r\n", "​-", "३", "'Re"]} +{"text": "'D😀🏽 'D0٣٤٥٦\r9m🙂\r \n‍", "tokens": 26, "pieces": ["'D", "😀🏽", " '", "D", "0٣٤", "٥٦", "\r", "9", "m", "🙂\r", " \n", "‍"]} +{"text": "é'M🙂Dž
,<㍿'ll\"\t字aḍ̇字<|fim_prefix|>'ſ!!d🙂<|fim_prefix|>İ'VE .‍'s㍿'VE\t12345678İ\u000bⅣé", "tokens": 76, "pieces": ["é", "'M", "🙂<", "META", "_START", ">Dž", "
", ",<㍿'", "ll", "\"", "\t字aḋ", "̣字", "<|", "fim", "_prefix", "|>'", "ſ", "!!", "d", "🙂<|", "fim", "_prefix", "|>", "İ", "'VE", " ", " .‍'", "s", "㍿'", "VE", "\t", "123", "456", "78", "İ", "\u000b", "Ⅳ", "e", "́"]} +{"text": "0,　å漢 ſ", "tokens": 11, "pieces": ["0", ",", "　a", "̊漢", " ", " ſ"]} +{"text": "'s<‍\n're‍🙂 \n#$%　'Re'Ms😀🏽s\r'llt…fiḍ̇'ll​ßİ0…\"<ß‍", "tokens": 52, "pieces": ["'s", "<‍\n", "'re", "‍🙂", " \n", "#$%", "　", "'Re", "'M", "s", "😀🏽", "s", "\r", "'ll", "t", "…fiḋ", "̣'", "ll", "​ßİ", "0", "…", "\"<", "ß", "‍"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "9'SDž<|fim_prefix|> 𐞁<|endoftext|>'ll !>㍿\u000b", "tokens": 31, "pieces": ["9", "'S", "Dž", "<|", "fim", "_prefix", "|>", " 𐞁", "<|", "endoftext", "|>'", "ll", " ", " !>㍿", "\u000b"]} +{"text": "㍿EOT'ſ㋿\u000b\r\n\r\n　's \nd \nع🙂0!!ſDž'T>éd३é,Ⅳt<​‍İ12345678\r\n\r\n'S\r", "tokens": 56, "pieces": ["㍿EOT", "'ſ", "㋿", "\u000b\r\n\r\n", "　", "'s", " \n", "d", " \n", "ع", "🙂", "0", "!!", "ſDž", "'T", ">éd", "३", "e", "́,", "Ⅳ", "t", "<​<", "EOT", ">‍", "İ", "123", "456", "78", "\r\n\r\n", "'S", "\r", ""]} +{"text": "㍿-㋿٣٤٥٦d🙂>t 's\r\n\r\nZ.'VEé\r\nDž å👍🏽‍ ", "tokens": 43, "pieces": ["㍿-㋿", "٣٤٥", "٦", "d", "🙂>", "t", " '", "s", "\r\n\r\n", "Z", ".'", "VEé", "\r\n", "Dž", " ", " a", "̊👍🏽‍", " "]} +{"text": "́('D…😀🏽ß\r\n 'll'Re'sⅣ\r\n", "tokens": 19, "pieces": ["́('", "D", "…", "😀🏽", "ß", "\r\n", " '", "ll", "'Re", "'s", "Ⅳ", "\r\n"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "½‍😀🏽'll'Mfiå<　12345678­👍🏽å
ḍ̇\r\ń㍿'Reå​ḍ̇", "tokens": 57, "pieces": ["", "½", "‍😀🏽'", "ll", "'M", "fia", "̊<", "　", "123", "456", "78", "­👍🏽", "a", "̊", "
ḋ", "̣\r\n", "́㍿'", "Rea", "̊​", "ḋ", "̣"]} +{"text": ",ß\r\n'S<ſée‍tt", "tokens": 11, "pieces": [",ß", "\r\n", "'S", "<ſée", "‍tt"]} +{"text": "ſEOT<|fim_prefix|>\u000b0 ꟲ<|fim_prefix|>İ\t <|endoftext|>​<|endoftext|>t,9", "tokens": 43, "pieces": ["ſEOT", "<|", "fim", "_prefix", "|>", "\u000b", "0", " ", " ꟲ", "<|", "fim", "_prefix", "|>", "İ", "\t ", " <|", "endoftext", "|>​<|", "endoftext", "|>", "t", ",", "9"]} +{"text": "\r\n\r\nع३", "tokens": 4, "pieces": ["\r\n\r\n", "ع", "३"]} +{"text": "#$%\r‍­Džḍ̇'T\nḍ̇🙂>!!é'VEé.-𐞁­­Ａ🙂ſ'Sſ\t'ſع字>", "tokens": 60, "pieces": ["#$%\r", "‍­", "Džḋ", "̣'", "T", "\n", "ḋ", "̣🙂>!!", "e", "́'", "VEe", "́.-", "𐞁", "­­", "Ａ", "🙂ſ", "'S", "ſ", "", "\t", "'", "ſع字", ">"]} +{"text": "𐞁\r\n½d \n'Reséé'ſſ\"­'llⅣİ😀🏽<|endoftext|>fi", "tokens": 38, "pieces": ["𐞁", "\r\n", "½", "d", " \n", "'Re", "se", "́é", "'ſ", "ſ", "\"­'", "ll", "Ⅳ", "İ", "😀🏽<|", "endoftext", "|>", "fi"]} +{"text": "å<|fim_prefix|>'T<\n\na👍🏽\rſ#$%'ll\r<|endoftext|>-'T(\u000b'm0‍ß'ReDž", "tokens": 50, "pieces": ["a", "̊<|", "fim", "_prefix", "|>'", "T", "<\n\n", "a", "👍🏽\r", "ſ", "#$%'", "ll", "\r", "<|", "endoftext", "|>-'", "T", "(", "\u000b", "'m", "0", "‍ß", "'", "ReDž"]} +{"text": "ßع👍🏽m-'ſ'('M", "tokens": 15, "pieces": ["ßع", "👍🏽", "m", "-'", "ſ", "'('", "M"]} +{"text": "‍EOTꟲ٣٤٥٦", "tokens": 15, "pieces": ["‍EOTꟲ", "٣٤٥", "٦"]} +{"text": "'M½!!\n​ \nⅣꟲ́ ㋿३Ⅳꟲ<|fim_prefix|>'　 stfi<|fim_prefix|>'VEm då\n​", "tokens": 49, "pieces": ["'M", "½", "!!\n", "​", " \n", "Ⅳ", "ꟲ", "́", " ", " ㋿", "३Ⅳ", "ꟲ", "<|", "fim", "_prefix", "|>'", "　 ", " stfi", "<|", "fim", "_prefix", "|>'", "VEm", " da", "̊\n", "​"]} +{"text": "👍🏽‍½t", "tokens": 14, "pieces": ["👍🏽‍<", "EOT", ">", "½", "t"]} +{"text": "\r\n,s!!(　'M ꟲ\"‍😀🏽　​😀🏽EOT'reé", "tokens": 29, "pieces": ["\r\n", ",s", "!!(", "　", "'M", " ꟲ", "\"‍😀🏽", "　", "​😀🏽", "EOT", "'re", "e", "́"]} +{"text": "'M- é \n字Dž'll9…EOTḍ̇eİaⅣ\nfi'llＡ'S漢.٣٤٥٦½​9.…\n", "tokens": 53, "pieces": ["'M", "-", " é", " \n", "字Dž", "'ll", "9", "…EOTḋ", "̣eİa", "Ⅳ", "\n", "fi", "'ll", "Ａ", "'S", "漢", ".", "٣٤٥", "٦½", "​<", "EOT", ">", "9", ".", "…\n"]} +{"text": "e!🙂'Reſ<ſ́👍🏽㍿'sm👍🏽　0eＡm", "tokens": 35, "pieces": ["e", "!🙂'", "Reſ", "<ſ", "́👍🏽㍿'", "sm", "👍🏽", "　", "0", "eＡm"]} +{"text": "\n0'<|endoftext|>'ll\u000b­(Ａd́", "tokens": 17, "pieces": ["\n", "0", "'<|", "endoftext", "|>'", "ll", "\u000b", "­(", "Ａd", "́"]} +{"text": "ſ\t", "tokens": 3, "pieces": ["ſ", "\t"]} +{"text": "‍½m ́​Dž", "tokens": 9, "pieces": ["‍", "½", "m", " ́​", "Dž"]} +{"text": "ḍ̇　fiEOT", "tokens": 13, "pieces": ["ḋ", "̣", "　fi", "EOT"]} +{"text": ".a(İ
d'D…ع're字㍿😀🏽…'ll字😀🏽>#$%\r\n\r\nm<|fim_prefix|><|endoftext|>!'Mfi,-.é>­'S9dİ", "tokens": 62, "pieces": [".a", "(İ", "
d", "'D", "…ع", "'re", "字", "㍿😀🏽", "…", "'ll", "字", "😀🏽>#$%\r\n\r\n", "m", "<|", "fim", "_prefix", "|><|", "endoftext", "|>!'", "Mfi", ",-.", "e", "́>­'", "S", "9", "dİ"]} +{"text": "ß🙂ع('s'Da👍🏽å'T'M ́عfiZİ'M", "tokens": 28, "pieces": ["ß", "🙂ع", "('", "s", "'D", "a", "👍🏽", "a", "̊'", "T", "'M", " ", "́عfiZİ", "'M"]} +{"text": "ع'VE'sZ'\tİ​　😀🏽", "tokens": 19, "pieces": ["ع", "'VE", "'s", "Z", "'", "\t", "İ", "​", "　", "😀🏽"]} +{"text": "\r३'s'VEfiꟲ>", "tokens": 12, "pieces": ["\r", "३", "'s", "'VE", "fiꟲ", ">"]} +{"text": "👍🏽\tEOT漢'Re .　tEOT\u000bß​\n\t!Ⅳ👍🏽\r\nع.a𐞁'VEZ
漢字é'ſ", "tokens": 59, "pieces": ["👍🏽", "\tEOT漢", "'Re", " ", " <", "META", "_START", ">", " .", "　tEOT", "\u000bß", "​\n", "\t", "!", "Ⅳ", "👍🏽\r\n", "ع", ".a", "𐞁", "'VE", "Z", "
漢字e", "́'", "ſ"]} +{"text": "d<|fim_prefix|>\r\n\r\n漢'Re\"t　عéå-0　字''ſ'D́9
　 EOT", "tokens": 34, "pieces": ["d", "<|", "fim", "_prefix", "|>\r\n\r\n", "漢", "'Re", "\"t", "　عéa", "̊-", "0", "　字", "''", "ſ", "'D", "́", "9", "
　", " EOT"]} +{"text": " t !", "tokens": 4, "pieces": [" ", " t", " ", " !"]} +{"text": " 👍🏽 Ⅳ\n", "tokens": 10, "pieces": [" ", " 👍🏽", " ", "Ⅳ", "\n"]} +{"text": "<|fim_prefix|>漢s'Me'T'll<|endoftext|><|endoftext|>e!!9㋿,'reⅣ'VEee‍'T>dfiḍ̇İ'lld३", "tokens": 64, "pieces": ["<|", "fim", "_prefix", "|>", "漢s", "'M", "e", "'T", "'ll", "<|", "endoftext", "|><|", "endoftext", "|>", "e", "!!", "9", "㋿,'", "re", "Ⅳ", "'VE", "ee", "‍'", "T", ">dfi", "ḋ", "̣İ", "'", "lld", "३"]} +{"text": "½a\"\t😀🏽ſé٣٤٥٦漢\u000b,\t9Ⅳ'T\r\n", "tokens": 34, "pieces": ["½", "a", "\"", "\t", "😀🏽", "ſe", "́", "٣٤٥", "٦", "漢", "\u000b", ",", "\t", "9", "", "Ⅳ", "'T", "\r\n"]} +{"text": "t! 😀🏽ſⅣ \n ꟲ　 m'll漢㍿'漢!!12345678ſd́\t🙂ꟲ\"'Re<|endoftext|>", "tokens": 54, "pieces": ["t", "!", " 😀🏽", "ſ", "Ⅳ", " \n", " ", " ꟲ", "　 ", " m", "'ll", "漢", "㍿'", "漢", "!!", "123", "456", "78", "ſd", "́", "\t", "🙂ꟲ", "\"'", "Re", "<|", "endoftext", "|>"]} +{"text": "e\né'T<😀🏽ꟲ.\u000ba \nＡ'llfi'll\r\n\r\n'll,\rDžeع<|fim_prefix|>👍🏽ét'(t'Re'ſ#$%Z👍🏽​'reZ", "tokens": 66, "pieces": ["e", "\n", "é", "'T", "<😀🏽", "ꟲ", ".", "\u000ba", " \n", "Ａ", "'ll", "fi", "'ll", "\r\n\r\n", "'ll", ",\r", "Džeع", "<|", "fim", "_prefix", "|>👍🏽", "ét", "'(", "t", "'Re", "'", "ſ", "#$%", "Z", "👍🏽​'", "reZ"]} +{"text": "́!!ⅣmEOT", "tokens": 7, "pieces": ["́!!", "Ⅳ", "mEOT"]} +{"text": "!👍🏽 İ\nß>", "tokens": 12, "pieces": ["!👍🏽", " İ", "\n", "ß", ">"]} +{"text": "ſ\r\n\r\n'll(𐞁're㋿'M", "tokens": 15, "pieces": ["ſ", "\r\n\r\n", "'ll", "(𐞁", "'re", "㋿'", "M"]} +{"text": "-(½'Rea!\r\n('Re", "tokens": 7, "pieces": ["-(", "½", "'Re", "a", "!\r\n", "('", "Re"]} +{"text": "!Zع\r\n­'Da漢ZİDž'　#$%\r\n", "tokens": 17, "pieces": ["!Zع", "\r\n", "­'", "Da漢ZİDž", "'", "　", "#$%\r\n"]} +{"text": "ſع \n'ſ#$%!é­s\u000b​ß<字m字𐞁éⅣ'T.<<|endoftext|>'ſ́½9­-(🙂'reſ\n12345678🙂", "tokens": 55, "pieces": ["ſع", " \n", "'ſ", "#$%!", "e", "́­", "s", "\u000b", "​ß", "<字m字𐞁é", "Ⅳ", "'T", ".<<|", "endoftext", "|>'", "ſ", "́", "½9", "­-(🙂'", "reſ", "\n", "123", "456", "78", "🙂"]} +{"text": "åEOT \nＡ", "tokens": 8, "pieces": ["a", "̊EOT", " \n", "Ａ"]} +{"text": "'sd'T　 \n漢<>'llas(ḍ̇漢​㍿३字­<<|endoftext|>,…ß㍿㍿", "tokens": 47, "pieces": ["'s", "d", "'T", "　 \n", "漢", "<>'", "llas", "(ḋ", "̣漢", "​㍿", "३", "字", "­<<|", "endoftext", "|>,", "…ß", "㍿㍿<", "META", "_START", ">"]} +{"text": "#$%\t½㍿tßEOT\u000b\r\nZ.'llé३>", "tokens": 20, "pieces": ["#$%", "\t", "½", "㍿tßEOT", "\u000b\r\n", "Z", ".'", "llé", "३", ">"]} +{"text": "!!👍🏽👍🏽eİ
­é'll𐞁'll!!<|fim_prefix|>0'Mé \r\n\r\ń㍿<|fim_prefix|>́'ſ'SⅣ-ZＡt", "tokens": 61, "pieces": ["!!👍🏽👍🏽", "eİ", "
", "­é", "'ll", "𐞁", "'ll", "!!<|", "fim", "_prefix", "|>", "0", "'M", "e", "́", " \r\n\r\n", "́㍿<|", "fim", "_prefix", "|>́'", "ſ", "'S", "Ⅳ", "-ZＡt"]} +{"text": " Dž𐞁 \n字EOT​ḍ̇é𐞁t'Re\u000bm \r\n\r\n'M12345678(<|endoftext|>fiZ.'VE,ꟲ㍿\r\n\r\n🙂'll", "tokens": 61, "pieces": [" ", " Dž𐞁", " \n", "字EOT", "​ḋ", "̣e", "́𐞁t", "'Re", "\u000bm", "", " \r\n\r\n", "'M", "123", "456", "78", "(<|", "endoftext", "|>", "fiZ", ".'", "VE", ",ꟲ", "㍿\r\n\r\n", "🙂'", "ll"]} +{"text": "12345678
عEOTİ,'VE👍🏽Džfi,'VE'👍🏽Ⅳ", "tokens": 35, "pieces": ["", "123", "456", "78", "
عEOTİ", ",'", "VE", "👍🏽", "Džfi", ",'", "VE", "'👍🏽", "Ⅳ"]} +{"text": "< 0s", "tokens": 4, "pieces": ["<", " ", "0", "s"]} +{"text": "ꟲ", "tokens": 3, "pieces": ["ꟲ"]} +{"text": "é'S漢.d'Z!!\ta\u000bＡ<Ⅳ­😀🏽 é'T\t㋿\"0'ſDž< ", "tokens": 46, "pieces": ["é", "'S", "漢", ".d", "'Z", "!!", "\ta", "\u000bＡ", "<", "Ⅳ", "­<", "META", "_START", ">😀🏽", " e", "́'", "T", "", "\t", "㋿\"", "0", "'ſ", "Dž", "<", " "]} +{"text": "'re𐞁åm'D \n12345678漢ع<😀🏽\r\n\r\n٣٤٥٦'Seꟲ\r\n\r\n!!<|endoftext|>\t‍'VE<漢٣٤٥٦­ꟲ e'M", "tokens": 69, "pieces": ["'re", "𐞁a", "̊m", "'D", " \n", "123", "456", "78", "漢ع", "<😀🏽\r\n\r\n", "٣٤٥", "٦", "'S", "eꟲ", "\r\n\r\n", "!!<|", "endoftext", "|>", "\t", "‍'", "VE", "<漢", "٣٤٥", "٦", "­ꟲ", " e", "'M"]} +{"text": ",🙂\r\n\r\nعİ ßḍ̇😀🏽12345678 'ſ'", "tokens": 29, "pieces": [",🙂\r\n\r\n", "عİ", " ßḋ", "̣<", "EOT", ">😀🏽", "123", "456", "78", " '", "ſ", "'"]} +{"text": "s\n ­👍🏽90'Re𐞁Ａ>", "tokens": 18, "pieces": ["s", "\n", " ­👍🏽", "90", "'Re", "𐞁Ａ", ">"]} +{"text": "ꟲ(<|endoftext|>", "tokens": 10, "pieces": ["ꟲ", "(<|", "endoftext", "|>"]} +{"text": "‍eEOT12345678'Dꟲ👍🏽ꟲ漢e　İ!'Ⅳ­.e٣٤٥٦字\u000b🙂0'Sع'ſ e'Dꟲte", "tokens": 57, "pieces": ["‍eEOT", "123", "456", "78", "'D", "ꟲ", "👍🏽", "ꟲ漢e", "　İ", "!'", "Ⅳ", "­.", "e", "٣٤٥", "٦", "字", "\u000b", "🙂", "0", "'S", "ع", "'ſ", " ", " e", "'D", "ꟲte"]} +{"text": "Ａ<|endoftext|>\rEOTé!!é>ḍ̇åé
\nmfi \r\n", "tokens": 33, "pieces": ["Ａ", "<|", "endoftext", "|>\r", "EOTé", "!!", "e", "́>", "ḋ", "̣a", "̊é", "
\n", "mfi", " \r\n"]} +{"text": "'sḍ̇<|fim_prefix|>'٣٤٥٦'re
…ꟲe　٣٤٥٦'VE's\u000b '", "٣٤٥", "٦", "'re", "
", "…ꟲe", "　", "٣٤٥", "٦", "'VE", "'s", "\u000b", " <", "Zİ", "½", "\t", "(㋿", "Ａ", "!ß", "\"", "३", "EOT", "!!🙂\r\n", "'re", "İ", " \n", " ", " !'", "T"]} +{"text": "!! 😀🏽- ​s​Ａ\r\nع‍ß\t字'Re'　ſ<'Re\"👍🏽\t>\n<|endoftext|>😀🏽ß\u000b", "tokens": 50, "pieces": ["!!", " ", "😀🏽-", " ​", "s", "​Ａ", "\r\n", "ع", "‍ß", "\t字", "'Re", "'", "　ſ", "<'", "Re", "\"👍🏽", "\t", ">\n", "<|", "endoftext", "|>😀🏽", "ß", "\u000b"]} +{"text": "🙂-<|endoftext|>‍𐞁(", "tokens": 17, "pieces": ["🙂-<|", "endoftext", "|>‍", "𐞁", "("]} +{"text": "0\n字\ré!👍🏽'ſ'llEOT0<|endoftext|>.😀🏽\rꟲⅣⅣⅣ㍿'T9🙂٣٤٥٦'VEꟲé­​Ⅳ
", "tokens": 70, "pieces": ["0", "\n", "字", "\r", "e", "́!👍🏽'", "ſ", "'ll", "EOT", "0", "<|", "endoftext", "|>.😀🏽\r", "ꟲ", "ⅣⅣⅣ", "㍿'", "T", "9", "🙂", "٣٤٥", "٦", "'VE", "ꟲé", "­​", "Ⅳ", "
"]} +{"text": "Ａ…fi٣٤٥٦ \r\n\r\n🙂!'s…'Red\r\nZ字#$%🙂12345678ꟲm!!漢'T½𐞁#$%", "tokens": 53, "pieces": ["Ａ", "…fi", "٣٤٥", "٦", " \r\n\r\n", "🙂!'", "s", "…", "'Re", "d", "\r\n", "Z字", "#$%🙂", "123", "456", "78", "ꟲm", "!!", "漢", "'T", "½", "𐞁", "#$%<", "EOT", ">"]} +{"text": "0!a.字9'M'llḍ̇𐞁9漢'ſ>fi\t'T", "tokens": 32, "pieces": ["0", "!a", ".字", "9", "'M", "'", "llḋ", "̣𐞁", "9", "漢", "'ſ", ">fi", "\t", "'T"]} +{"text": "👍🏽Ⅳ\"!İ0,aİ'D字Ⅳ<ꟲ\t½\u000b 😀🏽", "tokens": 38, "pieces": ["👍🏽", "Ⅳ", "\"!", "İ", "0", ",aİ", "'D", "字", "Ⅳ", "<ꟲ", "\t", "½", "<", "EOT", ">", "\u000b", " ", "😀🏽"]} +{"text": "s\"mＡ12345678å ́🙂'Tع👍🏽'M…(漢ꟲ- 'M\n9", "tokens": 43, "pieces": ["s", "\"mＡ", "123", "456", "78", "a", "̊", " ́🙂'", "T", "ع", "👍🏽'", "M", "…", "(漢ꟲ", "-", " ", "'M", "\n", "9"]} +{"text": ">ꟲ٣٤٥٦\n漢.ß漢Ⅳ0عå 'ſ'ReEOT'T
​-(12345678, 's🙂stꟲ", "tokens": 51, "pieces": [">ꟲ", "٣٤٥", "٦", "\n", "漢", ".ß漢", "Ⅳ0", "عa", "̊", " ", " '", "ſ", "'Re", "EOT", "'T", "
", "​-(", "123", "456", "78", ",", " ", " '", "s", "🙂stꟲ"]} +{"text": "ß>12345678🙂<|fim_prefix|>\"<|endoftext|>ع <|fim_prefix|>>0'SEOT  <|endoftext|><|fim_prefix|>漢३,é字é‍\r\n're漢e(d(#$%ßå\u000b", "tokens": 76, "pieces": ["ß", ">", "123", "456", "78", "🙂<|", "fim", "_prefix", "|>\"<|", "endoftext", "|>", "ع", " ", " <|", "fim", "_prefix", "|>>", "0", "'S", "EOT", " ", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "漢", "३", ",e", "́字e", "́‍\r\n", "'re", "漢e", "(d", "(#$%", "ßa", "̊", "\u000b", ""]} +{"text": "fi İDž\r\n
0.'T'VE'Dḍ̇\r\n\r\nt a­𐞁#$%🙂.'", "T", "'VE", "'D", "ḋ", "̣\r\n\r\n", "t", " a", "­𐞁", "#$%🙂<", "dé", "(𐞁", "½", "s", "\r\n\r\n", "s"]} +{"text": "!!<'ſå \r\n𐞁fiZ 12345678'reİ( \nḍ̇३#$%'S'ſ\r,m­㍿.\n\r\n​\n'VE \n'ſ", "tokens": 52, "pieces": ["!!<'", "ſa", "̊", " \r\n", "𐞁fiZ", " ", "123", "456", "78", "'re", "İ", "(", " \n", "ḋ", "̣", "३", "#$%'", "S", "'ſ", "\r", ",m", "­㍿.\n\r\n", "​\n", "'VE", " \n", "'ſ"]} +{"text": "å\r'VE\u000b\u000b0>m'sa字a0٣٤٥٦.'S0\t", "tokens": 27, "pieces": ["a", "̊\r", "'VE", "\u000b", "\u000b", "0", ">m", "'s", "a字a", "0٣٤", "٥٦", ".'", "S", "0", "\t"]} +{"text": " Z!!'D<|fim_prefix|>­ å('VÉ", "tokens": 23, "pieces": [" Z", "!!'", "D", "<|", "fim", "_prefix", "|>­", " a", "̊('", "VE", "́"]} +{"text": "… 'T'VE👍🏽!‍", "tokens": 16, "pieces": ["… ", " '", "T", "'VE", "👍🏽!‍"]} +{"text": "\r\n\r\n(𐞁ꟲ \u000b'VE㋿👍🏽", "tokens": 22, "pieces": ["\r\n\r\n", "(𐞁ꟲ", " ", "\u000b", "'VE", "㋿👍🏽"]} +{"text": "​s'llé", "tokens": 9, "pieces": ["​s", "'ll", "e", "́<", "META", "_START", ">"]} +{"text": " \u000b 'VE‍m!!!éå. 're'M", "tokens": 20, "pieces": [" \u000b", " '", "VE", "‍<", "EOT", ">m", "!!!", "e", "́a", "̊.", " '", "re", "'M"]} +{"text": "३Ⅳ0\n\r\n\r\n,å", "tokens": 14, "pieces": ["३Ⅳ0", "\n\r\n\r\n", ",a", "̊<", "EOT", ">"]} +{"text": "ms's\r\nd\r\n\r\n12345678'S''Té(Ａ字'VE\tet(
 a
é12345678'D𐞁\r\r\né", "tokens": 40, "pieces": ["ms", "'s", "\r\n", "d", "\r\n\r\n", "123", "456", "78", "'S", "''", "Té", "(Ａ字", "'VE", "\tet", "(", "
", " a", "
e", "́", "123", "456", "78", "'D", "𐞁", "\r\r\n", "e", "́"]} +{"text": " fi👍🏽 'll
<　'
'ſ'S \ne👍🏽𐞁‍'sꟲḍ̇", "tokens": 49, "pieces": ["", " ", " fi", "👍🏽", " ", "'ll", "
", "<", "　", "'", "
", "'ſ", "'S", " \n", "e", "👍🏽", "𐞁", "‍'", "sꟲḋ", "̣"]} +{"text": "'re-㍿\nå-㍿\r\"𐞁!!½🙂
Dž𐞁#$%ſꟲ,\n!!😀🏽'll'Re𐞁\r\n…e👍🏽", "tokens": 62, "pieces": ["'re", "-㍿\n", "a", "̊-㍿\r", "\"𐞁", "!!", "½", "🙂", "
Dž𐞁", "#$%", "ſꟲ", ",\n", "!!😀🏽'", "ll", "'Re", "𐞁", "\r\n", "…e", "👍🏽"]} +{"text": "½Ａ><|endoftext|>½'M>Ⅳع\t>字é🙂 \n!　d\u000b ,​t㋿漢>\r\n", "tokens": 41, "pieces": ["½", "Ａ", "><|", "endoftext", "|>", "½", "'M", ">", "Ⅳ", "ع", "\t", ">字e", "́🙂", " \n", "!", "　d", "\u000b ", " ,​", "t", "㋿漢", ">\r\n"]} +{"text": "'٣٤٥٦ßꟲ漢 'Re​ \n … ​'VE\"0İ12345678　…0'T𐞁", "tokens": 40, "pieces": ["'", "٣٤٥", "٦", "ßꟲ漢", " '", "Re", "​", " \n", " …", " ​'", "VE", "\"", "0", "İ", "123", "456", "78", "　", "…", "0", "'T", "𐞁"]} +{"text": "e漢'S ", "tokens": 5, "pieces": ["e漢", "'S", " "]} +{"text": "ḍ̇ß 
\n", "tokens": 14, "pieces": ["ḋ", "̣ß", " 
\n"]} +{"text": "ſＡ", "tokens": 4, "pieces": ["ſＡ"]} +{"text": "\n­Dž٣٤٥٦,.
'D'ſ s'EOT'字e😀🏽0aEOT'M㍿'S\"㋿'VEع", "tokens": 50, "pieces": ["\n", "­Dž", "٣٤٥", "٦", ",.", "
", "'D", "'ſ", " s", "'EOT", "'字e", "😀🏽", "0", "aEOT", "'M", "㍿'", "S", "\"㋿'", "VE", "ع"]} +{"text": "字😀🏽9ḍ̇EOTZ'll\t३३'T
s­İ­12345678'VE漢(\r\n\r\n漢İ're㋿ع'VEİ漢👍🏽Ａa", "tokens": 67, "pieces": ["字", "😀🏽", "9", "ḋ", "̣EOTZ", "'ll", "\t", "३३", "'T", "
s", "­İ", "­", "123", "456", "78", "'VE", "漢", "(\r\n\r\n", "漢İ", "'", "re", "㋿ع", "'VE", "İ漢", "👍🏽<", "META", "_START", ">Ａa"]} +{"text": "­\"…‍.
d#$%'ſ'S'll'VE.ſ12345678𐞁's<|fim_prefix|> 'ſ12345678d<|endoftext|>\"!!​ å𐞁DžⅣ\r", "tokens": 73, "pieces": ["­\"", "…", "‍.", "
d", "#$%'", "ſ", "'S", "'ll", "'VE", ".ſ", "123", "456", "78", "𐞁", "'s", "<|", "fim", "_prefix", "|>", " ", " '", "ſ", "123", "456", "78", "d", "<|", "endoftext", "|>\"<", "EOT", "><", "META", "_START", ">!!​", " ", " a", "̊𐞁Dž", "Ⅳ", "\r"]} +{"text": "å'SéꟲDžⅣ३㍿m\r\n😀🏽Ⅳ'Reḍ̇㍿ ٣٤٥٦", "tokens": 48, "pieces": ["a", "̊'", "S", "éꟲDž", "Ⅳ३", "㍿m", "\r\n", "😀🏽", "Ⅳ", "'Re", "ḋ", "̣㍿", " ", "٣٤٥", "٦"]} +{"text": "\r\n>ée'S<|endoftext|>ḍ̇Ａ字<|fim_prefix|>fi", "tokens": 29, "pieces": ["\r\n", ">e", "́e", "'S", "<|", "endoftext", "|>", "ḋ", "̣Ａ字", "<|", "fim", "_prefix", "|>", "fi"]} +{"text": "\n'VE…\"ſß's'ſ<|fim_prefix|>\r", "tokens": 21, "pieces": ["\n", "'VE", "…", "\"ſß", "'s", "'ſ", "<|", "fim", "_prefix", "|>\r"]} +{"text": "'Dm​'VEåع12345678!!　👍🏽\n \né", "tokens": 23, "pieces": ["'D", "m", "​'", "VEa", "̊ع", "123", "456", "78", "!!", "　", "👍🏽\n", " \n", "é"]} +{"text": "字-0'VE,­ع#$%'D'T\r\n-fi", "tokens": 20, "pieces": ["字", "-", "0", "'VE", ",­", "ع", "#$%'", "D", "'T", "\r\n", "-fi"]} +{"text": "'ll>🙂漢eḍ̇'M\n\r\n‍ 👍🏽", "tokens": 25, "pieces": ["'ll", ">🙂", "漢eḋ", "̣'", "M", "\n\r\n", "‍", " ", "👍🏽"]} +{"text": "12345678éſ漢!\n9­'ſ>ḍ̇'́ ", "tokens": 24, "pieces": ["123", "456", "78", "e", "́ſ漢", "!\n", "9", "­'", "ſ", ">ḋ", "̣'́", " "]} +{"text": "\t,漢>'　dDž>\rDžZꟲ­'ll \n Ⅳ d's0'Re'D​", "tokens": 31, "pieces": ["\t", ",漢", ">'", "　dDž", ">\r", "DžZꟲ", "­'", "ll", " \n", " ", "Ⅳ", " ", " d", "'s", "0", "'Re", "'D", "​"]} +{"text": "\r\n\r\n-'S !!-\u000b‍", "tokens": 9, "pieces": ["\r\n\r\n", "-'", "S", " ", " !!-", "\u000b", "‍"]} +{"text": "é​ꟲ'll‍'re…\rع😀🏽tꟲ\t\"a🙂m,\r\n\r\n
३½ \nt,'ſDžع👍🏽𐞁EOTfiß#$%Dž'll'M ", "tokens": 64, "pieces": ["é", "​ꟲ", "'ll", "‍'", "re", "…\r", "ع", "😀🏽", "tꟲ", "\t", "\"a", "🙂m", ",\r\n\r\n", "
", "३½", " \n", "t", ",'", "ſDžع", "👍🏽", "𐞁EOTfiß", "#$%", "Dž", "'ll", "'M", " "]} +{"text": "́㍿'llt'll\"'S　字", "tokens": 12, "pieces": ["́㍿'", "llt", "'ll", "\"'", "S", "　字"]} +{"text": "'M\t　é>'D㋿İ", "tokens": 10, "pieces": ["'M", "\t", "　é", ">'", "D", "㋿İ"]} +{"text": "\"é>t'D<|endoftext|>", "tokens": 11, "pieces": ["\"é", ">t", "'D", "<|", "endoftext", "|>"]} +{"text": "Ａ😀🏽'ſ#$%'re'll#$%​ ㍿　 ", "tokens": 26, "pieces": ["Ａ", "😀🏽'", "ſ", "#$%'", "re", "'", "ll", "#$%​", " ㍿", "　 "]} +{"text": "ꟲꟲfi< m漢9EOTs𐞁
'll㋿𐞁.a.Ⅳ… \n<|fim_prefix|>٣٤٥٦🙂½é'Reſ", "tokens": 67, "pieces": ["ꟲꟲfi", "<<", "EOT", ">", " m漢", "9", "EOTs𐞁", "
", "'ll", "㋿𐞁", ".a", ".", "Ⅳ", "… \n", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "🙂", "½", "e", "́<", "META", "_START", ">'", "Reſ"]} +{"text": "'S'D'se'ſ9Ⅳſ🙂㍿Z\r\n­\u000b", "tokens": 21, "pieces": ["'S", "'D", "'s", "e", "'ſ", "9Ⅳ", "ſ", "🙂㍿", "Z", "\r\n", "­", "\u000b"]} +{"text": "!\n,(a…aEOT𐞁'VE0'", "VE", "0", "#$%\"", "tokens": 17, "pieces": ["'ll", "ſ", "\r", "123", "456", "78", "d", "<|", "endoftext", "|>#$%\""]} +{"text": "'", "tokens": 1, "pieces": ["'"]} +{"text": "㋿Ａ漢s'Tع
'Mꟲḍ̇é(-　åaſ​Džt\tＡ'S!漢 -漢 ‍<|endoftext|>😀🏽'D", "tokens": 63, "pieces": ["㋿Ａ漢s", "'T", "ع", "
", "'M", "ꟲḋ", "̣e", "́(-", "　a", "̊aſ", "​Džt", "\tＡ", "'S", "!漢", " ", " -", "漢", " ", " ‍<|", "endoftext", "|>😀🏽'", "D"]} +{"text": "'ll'VE३ß 0字­ 'ReİeEOTⅣ\nZ​", "tokens": 22, "pieces": ["'ll", "'VE", "३", "ß", " ", "0", "字", "­", " ", " '", "ReİeEOT", "Ⅳ", "\n", "Z", "​"]} +{"text": "İＡſ　#$%㋿'M", "tokens": 13, "pieces": ["İＡſ", "　", "#$%㋿'", "M"]} +{"text": "\"", "tokens": 1, "pieces": ["\""]} +{"text": "m#$%>'Mm.ZZ👍🏽㋿'T
㍿e\"㋿٣٤٥٦9\tḍ̇0½", "tokens": 50, "pieces": ["m", "#$%>'", "Mm", ".ZZ", "👍🏽<", "META", "_START", ">㋿'", "T", "
", "㍿e", "\"㋿", "٣٤٥", "٦9", "\tḋ", "̣", "0½"]} +{"text": " \n\t(½EOTßt漢\t㋿EOTß\r\n\r\n㍿'𐞁\u000ba½🙂", "tokens": 31, "pieces": [" \n", "\t", "(", "½", "EOTßt漢", "\t", "㋿EOTß", "\r\n\r\n", "㍿'", "𐞁", "\u000ba", "½", "🙂"]} +{"text": "9漢'SDž𐞁㍿ع'ſ३Ⅳ(é½👍🏽\t\r\n,", "tokens": 32, "pieces": ["9", "漢", "'S", "Dž𐞁", "㍿ع", "'ſ", "३Ⅳ", "(e", "́", "½", "👍🏽", "\t\r\n", ","]} +{"text": "'ſ😀🏽#$%\r\n­<|fim_prefix|><> EOT>​éⅣ'D\r'D\tt­'s 'Re>\"‍EOT#$%\r\nḍ̇fi d\r\n\r\n😀🏽", "tokens": 64, "pieces": ["'ſ", "😀🏽#$%\r\n", "­<|", "fim", "_prefix", "|><>", " ", " EOT", ">​", "e", "́", "Ⅳ", "'", "D", "\r", "'D", "\tt", "­'", "s", " ", " '", "Re", ">\"‍", "EOT", "#$%\r\n", "ḋ", "̣fi", " ", " d", "\r\n\r\n", "😀🏽"]} +{"text": "'Ree'sé(#$%Dž🙂́'ſ‍", "tokens": 17, "pieces": ["'Re", "e", "'s", "é", "(#$%", "Dž", "🙂́'", "ſ", "‍"]} +{"text": "ع𐞁eＡ\r\n\r\nfi‍\"　'ſ'VE'sſ'Re.Dž𐞁å'll<|endoftext|>,😀🏽㋿👍🏽ḍ̇<|endoftext|>t .㍿<<🙂
", "tokens": 85, "pieces": ["ع𐞁eＡ", "\r\n\r\n", "fi", "‍\"", "　", "'ſ", "'", "VE", "'s", "ſ", "'Re", ".Dž𐞁a", "̊'", "ll", "<|", "endoftext", "|>,😀🏽㋿👍🏽", "ḋ", "̣<|", "endoftext", "|>", "t", " ", " <", "META", "_START", ">.㍿<<🙂", "
"]} +{"text": "'ll>'s
\r,𐞁<|fim_prefix|>Z afi9字're字 ḍ̇<|endoftext|>字\r\n\r\n'sＡ0 d\n#$%12345678‍s'VEm字…", "tokens": 66, "pieces": ["'ll", ">'", "s", "", "
\r", ",𐞁", "<|", "fim", "_prefix", "|>", "Z", " ", " afi", "9", "字", "'re", "字", " ḋ", "̣<|", "endoftext", "|>", "字", "\r\n\r\n", "'s", "Ａ", "0", " ", " d", "\n", "#$%", "123", "456", "78", "‍s", "'VE", "m字", "…"]} +{"text": "Ａ\n-漢Dž👍🏽٣٤٥٦'ſ'D३ 漢　é", "tokens": 34, "pieces": ["Ａ", "\n", "-漢Dž", "👍🏽", "٣٤٥", "٦", "'ſ", "'D", "३", " 漢", "　e", "́"]} +{"text": "'S#$%'ß(😀🏽,é½EOTꟲ'VE\r'SⅣ12345678<(ßt'D‍　'M'Ret'VE", "tokens": 40, "pieces": ["'S", "#$%'", "ß", "(😀🏽,", "e", "́", "½", "EOTꟲ", "'VE", "\r", "'S", "Ⅳ12", "345", "678", "<(", "ßt", "'D", "‍", "　", "'M", "'Re", "t", "'VE"]} +{"text": "'re字'M(\u000b're𐞁'Ś9Dž\r\n\r\nDž \n('reꟲḍ̇
\rs \n", "tokens": 37, "pieces": ["'re", "字", "'M", "(", "\u000b", "'re", "𐞁", "'S", "́", "9", "Dž", "\r\n\r\n", "Dž", " \n", "('", "reꟲḋ", "̣", "
\r", "s", " \n"]} +{"text": "​!!İＡ'M!!'sfimⅣ　ꟲ👍🏽t👍🏽…é0é's>å　'ſ<|endoftext|>٣٤٥٦éＡ　Dž(  Dž㍿́ḍ̇😀🏽", "tokens": 85, "pieces": ["​!!", "İＡ", "'M", "!!'", "sfim", "Ⅳ", "　ꟲ", "👍🏽", "t", "👍🏽", "…é", "0", "é", "'s", ">a", "̊", "　", "'ſ", "<|", "endoftext", "|>", "٣٤٥", "٦", "e", "́Ａ", "　Dž", "(", " ", " Dž", "㍿́", "ḋ", "̣😀🏽"]} +{"text": "🙂\t😀🏽dＡ-😀🏽éⅣ́\r\n\r\n👍🏽 ſ́ \n- m\" 'Sd", "tokens": 41, "pieces": ["🙂", "\t", "😀🏽", "dＡ", "-😀🏽", "e", "́", "Ⅳ", "́\r\n\r\n", "👍🏽", " ſ", "́", " \n", "-", " ", " m", "\"", " ", "'S", "d"]} +{"text": "😀🏽!!éꟲEOT́ꟲ😀🏽'T𐞁½…👍🏽'EOT.ßa𐞁a", "tokens": 47, "pieces": ["😀🏽!!", "éꟲEOT", "́ꟲ", "😀🏽'", "T𐞁", "½", "…", "👍🏽'", "EOT", ".ßa𐞁a"]} +{"text": "!!t.99٣٤٥٦Ⅳع,漢字\u000bİ9\u000b", "tokens": 23, "pieces": ["!!", "t", ".", "99٣", "٤٥٦", "Ⅳ", "ع", ",漢字", "\u000bİ", "9", "\u000b"]} +{"text": ".\"Ａ\r>\u000bte<|endoftext|>#$%<|fim_prefix|>,eeå'Re👍🏽 s'Re 'Reſ\n>12345678#$%漢… \n<|fim_prefix|>\t", "tokens": 61, "pieces": [".\"", "Ａ", "\r", ">", "\u000bte", "<|", "endoftext", "|>#$%<|", "fim", "_prefix", "|>,", "eea", "̊'", "Re", "👍🏽", " s", "'Re", " ", " '", "Reſ", "\n", ">", "123", "456", "78", "#$%", "漢", "… \n", "<|", "fim", "_prefix", "|>", "\t"]} +{"text": "'VE're​​\u000b\n\n'> 'll\u000b𐞁 ß' t㍿m٣٤٥٦‍t0­ İ👍🏽Dž‍t\r", "tokens": 53, "pieces": ["'VE", "'re", "​​", "\u000b\n\n", "'>", " '", "ll", "\u000b𐞁", " ß", "'", " ", " t", "㍿", "m", "٣٤٥", "٦", "‍t", "0", "­", " ", " İ", "👍🏽", "Dž", "‍t", "\r"]} +{"text": " ㍿㍿\nſ‍!'0👍🏽", "tokens": 20, "pieces": [" ", "㍿㍿\n", "ſ", "‍!'", "0", "👍🏽"]} +{"text": "ḍ̇ḍ̇'M字sḍ̇\t#$%t,'VEéß३0'Sé'll‍👍🏽,s", "tokens": 46, "pieces": ["ḋ", "̣ḋ", "̣'", "M字sḋ", "̣", "\t", "#$%", "t", ",'", "VEe", "́ß", "३0", "'S", "e", "́'", "ll", "‍👍🏽,", "s"]} +{"text": "㍿sfi३Z
…😀🏽!!\"漢12345678'sfiZd字Dž𐞁Zå0.d9aß\"<|fim_prefix|>'12345678㋿字Z's're", "tokens": 66, "pieces": ["㍿sfi", "३", "Z", "", "
", "…", "😀🏽!!\"", "漢", "123", "456", "78", "'s", "fiZd字Dž𐞁Za", "̊", "0", ".d", "9", "aß", "\"<|", "fim", "_prefix", "|>'", "123", "456", "78", "㋿字Z", "'s", "'re"]} +{"text": " \"'VE're 9ḍ̇éeⅣ'VE!!㍿\t.", "tokens": 24, "pieces": [" ", " \"'", "VE", "'re", " ", "9", "ḋ", "̣e", "́e", "Ⅳ", "'VE", "!!㍿", "\t", "."]} +{"text": "½å㋿🙂𐞁\r\n12345678's…", "tokens": 20, "pieces": ["½", "a", "̊㋿🙂", "𐞁", "\r\n", "123", "456", "78", "'s", "…"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "
ع…
séİ\r\n\r\n,dſéZ \n 'DEOT'Re", "tokens": 21, "pieces": ["
ع", "…", "
se", "́İ", "\r\n\r\n", ",dſéZ", " \n", " '", "DEOT", "'Re"]} +{"text": "9dZ\rZＡ'll \t!!(\t​\u000bfi0　sİe­#$%'ſm\u000bm", "tokens": 40, "pieces": ["9", "dZ", "\r", "Z", "Ａ", "'", "ll", " ", "\t", "!!(", "\t", "​", "\u000bfi", "0", "　sİe", "­#$%'", "ſm", "\u000bm"]} +{"text": "<0‍ZEOTm'D
'll12345678.Z'llZé<ſfiſ0Ⅳ>", "tokens": 29, "pieces": ["<", "0", "‍ZEOTm", "'D", "
", "'ll", "123", "456", "78", ".Z", "'ll", "Zé", "<ſfiſ", "0Ⅳ", ">"]} +{"text": " å👍🏽12345678٣٤٥٦㍿ ३𐞁s \t\"३'lle'T'‍''VE 漢Ⅳ<|endoftext|> \n.👍🏽漢fí٣٤٥٦ſ12345678'VE", "tokens": 88, "pieces": [" ", " a", "̊👍🏽", "123", "456", "78٣", "٤٥٦", "㍿", " ", "३", "𐞁s", " ", "\t", "\"", "३", "'ll", "e", "'T", "'‍''", "VE", " 漢", "Ⅳ", "<|", "endoftext", "|>", " \n", ".👍🏽", "漢fi", "́", "٣٤٥", "٦", "ſ", "123", "456", "78", "'", "VE"]} +{"text": "Ⅳ \nḍ̇t<|endoftext|>Ａt　!,㍿!!٣٤٥٦🙂ⅣEOT#$%Ａ\r\n\r\n", "Ａt", "　", "!,㍿!!", "٣٤٥", "٦", "🙂", "Ⅳ", "EOT", "#$%", "Ａ", "\r\n\r\n", ",½\"\té'reİ\t(漢#$%\r\n\r\nå'!!٣٤٥٦\r'M
Zé٣٤٥٦́İ ́­'३😀🏽å'<|fim_prefix|>", "tokens": 75, "pieces": ["<|", "fim", "_prefix", "|>,", "½", "\"", "\te", "́'", "reİ", "\t", "(漢", "#$%\r\n\r\n", "a", "̊'!!", "٣٤٥", "٦", "\r", "'M", "
Ze", "́", "٣٤٥", "٦", "́", "İ", " ", " ́­'", "३", "😀🏽", "a", "̊'<|", "fim", "_prefix", "|>"]} +{"text": "\r\n…\u000b𐞁é-mſ…\r\n\r\nß<|endoftext|>d0 (字३½s字́'T\r\t'ſ'sⅣ́\tDž\r\n", "tokens": 48, "pieces": ["\r\n", "…", "\u000b𐞁é", "-mſ", "…\r\n\r\n", "ß", "<|", "endoftext", "|>", "d", "0", " (", "字", "३½", "s字", "́'", "T", "\r", "\t", "'ſ", "'s", "Ⅳ", "́", "\tDž", "\r\n"]} +{"text": "#$%!EOTdé'S\u000bß \n३<|endoftext|>ß­s‍'ll0‍", "tokens": 37, "pieces": ["#$%!", "EOTde", "́'", "S", "\u000b", "ß", " \n", "३", "<|", "endoftext", "|>", "ß", "­", "s", "‍'", "ll", "0", "‍"]} +{"text": "İꟲ12345678", "tokens": 7, "pieces": ["İꟲ", "123", "456", "78"]} +{"text": "Dž(!!Zİ<­ 'Ree,
", "tokens": 15, "pieces": ["Dž", "(!!", "Zİ", "<­", " ", " '", "Ree", ",", "
"]} +{"text": "字 🙂s…Dž\r\n\r\n'ſ\tsİ𐞁\r'T\rm'VE-a9,e0!漢eİ'VE's s'Re字#$%‍<|endoftext|>", "tokens": 50, "pieces": ["(s", "!!", "ع", "​d", "'", "ſ", "\tsİ𐞁", "\r", "'T", "\r", "m", "'VE", "-a", "9", ",e", "0", "!漢eİ", "'VE", "'s", " s", "'Re", "字", "#$%‍<|", "endoftext", "|>"]} +{"text": "٣٤٥٦٣٤٥٦EOTé\u000b<|fim_prefix|>𐞁!9漢(字é…😀🏽eİ\"e \t!!Ａ'reḍ̇'saZ\r\ntḍ̇­㍿\r\n\r\n𐞁", "tokens": 80, "pieces": ["٣٤٥", "٦٣٤", "٥٦", "EOTé", "\u000b", "<|", "fim", "_prefix", "|>", "𐞁", "!", "9", "漢", "(字e", "́", "…", "😀🏽", "eİ", "\"e", " ", "\t", "!!", "Ａ", "'re", "ḋ", "̣'", "saZ", "\r\n", "tḋ", "̣­㍿\r\n\r\n", "𐞁"]} +{"text": "éع(Ａ>३#$%!
\n\t'llع\r\n\r\nⅣd'VEs(\n'Re", "tokens": 29, "pieces": ["éع", "(Ａ", ">", "३", "#$%!", "
\n", "\t", "'ll", "ع", "\r\n\r\n", "Ⅳ", "d", "'VE", "s", "(\n", "'Re"]} +{"text": "عＡZ٣٤٥٦#$%", "tokens": 18, "pieces": ["عＡZ", "٣٤٥", "٦", "#$%"]} +{"text": "\u000bd😀🏽DžＡ३٣٤٥٦\u000bs#$%\t㍿'S,عé‍m\u000b-s!!­Z'#$%", "tokens": 46, "pieces": ["\u000bd", "😀🏽", "DžＡ", "३٣٤", "٥٦", "\u000bs", "#$%", "\t", "㍿'", "S", ",عe", "́‍", "m", "\u000b", "-s", "!!­", "Z", "'#$%"]} +{"text": "eé
<|fim_prefix|>'re'D'", "re", "'D", "'T㋿! \n…!\"'s٣٤٥٦٣٤٥٦'re　", "tokens": 40, "pieces": ["Ⅳ", "'VE", "字", "'VE", "'", "T", "㋿!", " \n", "…", "!\"'", "s", "٣٤٥", "٦٣٤", "٥٦", "'re", "　"]} +{"text": "-🙂d𐞁🙂 <\r\n㍿t­#$%'T å<|endoftext|>🙂'lls", "tokens": 39, "pieces": ["𐞁", "🙂", " ", "<\r\n", "㍿t", "­#$%'", "T", " a", "̊<|", "endoftext", "|>🙂'", "lls"]} +{"text": "'ſ !㍿,\r\n\r\n<|fim_prefix|>ßefi,㍿'Dİ🙂\t\r\n<|endoftext|>.ß9🙂😀🏽", "tokens": 44, "pieces": ["'ſ", " !㍿,\r\n\r\n", "<|", "fim", "_prefix", "|>", "ßefi", ",㍿'", "Dİ", "🙂", "\t\r\n", "<|", "endoftext", "|>.", "ß", "9", "🙂😀🏽"]} +{"text": "漢漢eعés\rDž'VE　Z", "tokens": 14, "pieces": ["漢漢eعés", "\r", "Dž", "'VE", "　Z"]} +{"text": "!fit#$%‍Džfi İ🙂🙂 ㍿‍́'ſ<|fim_prefix|>", "tokens": 35, "pieces": ["!fit", "#$%‍", "Džfi", " İ", "🙂🙂", " ", "㍿‍́'", "ſ", "<|", "fim", "_prefix", "|>"]} +{"text": "Z👍🏽s#$%<|fim_prefix|>ḍ̇", "tokens": 22, "pieces": ["Z", "👍🏽", "s", "#$%<|", "fim", "_prefix", "|>", "ḋ", "̣"]} +{"text": "\n'VEZé'll,…😀🏽d<|endoftext|>㋿ß!<|endoftext|>½é\r'lléＡt👍🏽.'VE'!!Ⅳ", "tokens": 55, "pieces": ["\n", "'VE", "Ze", "́'", "ll", ",", "…", "😀🏽", "d", "<|", "endoftext", "|>㋿", "ß", "!<|", "endoftext", "|>", "½", "e", "́\r", "'ll", "éＡt", "👍🏽.'", "VE", "'!!", "Ⅳ"]} +{"text": "ſ½​­😀🏽½\n'VE('عⅣ…'reé'reee12345678😀🏽- <'ꟲ'ReDž12345678< 　'VEt!! é\r漢's", "tokens": 60, "pieces": ["ſ", "½", "​­😀🏽", "½", "\n", "'VE", "('", "ع", "Ⅳ", "…", "'re", "e", "́'", "reee", "123", "456", "78", "😀🏽-", " <'", "ꟲ", "'Re", "Dž", "123", "456", "78", "<", " ", "　", "'VE", "t", "!!", " e", "́\r", "漢", "'s"]} +{"text": "Ａ­ꟲ<'ll
", "tokens": 10, "pieces": ["Ａ", "­ꟲ", "<'", "ll", "
"]} +{"text": "Dž　#$%㋿字㍿٣٤٥٦d\t… \n🙂's,-", "tokens": 30, "pieces": ["Dž", "　", "#$%㋿", "字", "㍿", "٣٤٥", "٦", "d", "\t… \n", "🙂'", "s", ",-"]} +{"text": "🙂'VE\r­'ſⅣꟲfié<|fim_prefix|>,(", "tokens": 28, "pieces": ["🙂'", "VE", "\r", "­<", "EOT", ">'", "ſ", "Ⅳ", "ꟲfié", "<|", "fim", "_prefix", "|>,("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "12345678Z<|endoftext|>\".\t> ½\u000b'M 12345678#$%㋿\u000bå漢at\u000bd<|endoftext|>!t👍🏽 > …s\tſ", "tokens": 62, "pieces": ["123", "456", "78", "Z", "<|", "endoftext", "|><", "EOT", ">\".", "\t", ">", " ", "½", "\u000b", "'M", " ", "123", "456", "78", "#$%㋿", "\u000ba", "̊漢at", "\u000bd", "<|", "endoftext", "|>!", "t", "👍🏽", " ", " >", " ", "…s", "\tſ"]} +{"text": "٣٤٥٦ Z漢́ 'VE‍'VEعع🙂३té\t\r\n\r\n>🙂'Sé 'S\r½　", "tokens": 46, "pieces": ["٣٤٥", "٦", "", " Z漢", "́", " ", "'VE", "‍'", "VEعع", "🙂", "३", "te", "́", "\t\r\n\r\n", "><", "EOT", ">🙂'", "Se", "́", " '", "S", "\r", "½", "　"]} +{"text": " <|endoftext|><|endoftext|>..t<|fim_prefix|>'T#$%Dž<'D👍🏽ås'll'fi​'llꟲ'Sd", "tokens": 52, "pieces": [" ", "<|", "endoftext", "|><|", "endoftext", "|>..", "t", "<|", "fim", "_prefix", "|>'", "T", "#$%", "Dž", "<'", "D", "👍🏽", "a", "̊s", "'ll", "'fi", "​'", "llꟲ", "'S", "d"]} +{"text": "(.Ⅳ
Dž9", "tokens": 8, "pieces": ["(.", "Ⅳ", "
Dž", "9"]} +{"text": "… -,-Ａtfi.عſ'VE'VE㋿㋿'\r> \n'字 ㋿<|fim_prefix|>'ll
'S㋿ 漢9e漢Z", "tokens": 57, "pieces": ["… ", " -,-", "Ａtfi", ".عſ", "'VE", "'VE", "㋿㋿'\r", ">", " \n", "'字", " ", " ㋿<|", "fim", "_prefix", "|>'", "ll", "
", "'S", "㋿", " 漢", "9", "e漢Z"]} +{"text": "́㋿<|fim_prefix|>🙂\r\n\r\nＡ
e
\"d12345678fiDž　's'Re <|endoftext|><|fim_prefix|>fi𐞁å㋿ع…🙂­३'(ß🙂\r\n\r\n", "Ａ", "
e", "
", "\"d", "123", "456", "78", "fiDž", "　", "'s", "'Re", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "fi𐞁a", "̊㋿", "ع", "…", "🙂­", "३", "'(", "ß", "é>#$% '", "tokens": 47, "pieces": ["!'", "T", "٣٤٥", "٦", "m", "123", "456", "78", "e", "́", "　", "👍🏽", "é", " \n", "…é", ",'", "S", "(­", "İ", "'S", "\n", "<|", "endoftext", "|>", "é", ">#$%", " '"]} +{"text": "!👍🏽
é12345678'll'Re🙂…Ａ#$%!🙂're\rİ>", "tokens": 38, "pieces": ["!👍🏽", "
e", "́", "123", "456", "78", "'ll", "'Re", "🙂", "…Ａ", "#$%!🙂'", "re", "\r", "İ", "><", "EOT", ">"]} +{"text": "DžⅣ \nſ 're…'-tfi'reⅣ \ń.dma", "tokens": 30, "pieces": ["Dž", "Ⅳ", " \n", "ſ", " ", " '", "re", "…", "'-", "t", "fi", "'re", "Ⅳ", " \n", "́.", "dma", ""]} +{"text": "'(👍🏽e#$%'S😀🏽\tß-​Ａa㍿\r\n\r\n eå<|fim_prefix|>\u000b!\n\t😀🏽🙂\tt'llZ㍿　", "tokens": 59, "pieces": ["'(👍🏽", "e", "#$%'", "S", "😀🏽", "\tß", "-​", "Ａa", "㍿\r\n\r\n", " ", " <", "META", "_START", ">ea", "̊<|", "fim", "_prefix", "|>", "\u000b", "!\n", "\t", "😀🏽🙂", "\tt", "'ll", "Z", "㍿", "　"]} +{"text": "字ꟲ fi​t12345678<'llEOT ­👍🏽\r\n
<|fim_prefix|>a㍿́-<|fim_prefix|>\t <́#$%Z,\nİ \n\r\n", "tokens": 55, "pieces": ["字ꟲ", " fi", "​t", "123", "456", "78", "<'", "llEOT", " ­👍🏽\r\n", "
", "<|", "fim", "_prefix", "|>", "a", "㍿́-<|", "fim", "_prefix", "|>", "\t", " <́#$%", "Z", ",\n", "İ", " \n\r\n"]} +{"text": "㋿", "tokens": 3, "pieces": ["㋿"]} +{"text": "<|fim_prefix|>d字\r\n\r\n-('Reé'VEع　👍🏽٣٤٥٦\r\n\r\nſ𐞁\n٣٤٥٦İ!!>\r​😀🏽9", "tokens": 60, "pieces": ["<|", "fim", "_prefix", "|>", "d字", "\r\n\r\n", "-('", "Ree", "́'", "VEع", "　", "👍🏽", "٣٤٥", "٦", "\r\n\r\n", "ſ𐞁", "\n", "٣٤٥", "٦", "İ", "!!>\r", "​😀🏽", "9"]} +{"text": "
a'Dß,!'S<|endoftext|>\tⅣ'D'S½å😀🏽ع　­ßtع!\u000b!! \n­漢9㋿‍!", "tokens": 52, "pieces": ["
a", "'D", "ß", ",!'", "S", "<|", "endoftext", "|>", "\t", "Ⅳ", "'D", "'S", "½", "a", "̊😀🏽", "ع", "　", "­ß", "tع", "!", "\u000b", "!!", " \n", "­漢", "9", "㋿‍!"]} +{"text": "३ ß😀🏽<|endoftext|>½​́'S\"!!Ａ𐞁漢😀🏽\r٣٤٥٦'VE　😀🏽\u000b​\n
Z\r\n\r\n!!㍿EOT<|fim_prefix|>Ａ‍Z‍ (", "tokens": 82, "pieces": ["३", " ", " ß", "😀🏽<|", "endoftext", "|>", "½", "​́'", "S", "\"!!", "Ａ𐞁漢", "😀🏽\r", "٣٤٥", "٦", "'VE", "　", "😀🏽", "\u000b", "​\n", "
Z", "\r\n\r\n", "!!㍿", "EOT", "<|", "fim", "_prefix", "|>", "Ａ", "‍Z", "‍", " ("]} +{"text": ".a12345678å 'll 's!!𐞁 \n.t'Re३٣٤٥٦ⅣEOTEOT𐞁", "tokens": 40, "pieces": [".a", "123", "456", "78", "a", "̊", " ", " '", "ll", " ", " '", "s", "!!", "𐞁", " \n", ".t", "'Re", "३٣٤", "٥٦Ⅳ", "EOTEOT𐞁"]} +{"text": " \n٣٤٥٦", "tokens": 9, "pieces": [" \n", "٣٤٥", "٦"]} +{"text": ">ſ9!!İ,عß½te\"​é'Té漢m\u000b \t­ #$%9<İ\r's09a\r👍🏽", "tokens": 42, "pieces": [">ſ", "9", "!!", "İ", ",عß", "½", "te", "\"​", "e", "́'", "Te", "́漢m", "\u000b ", "\t", "­", " ", " #$%", "9", "<İ", "\r", "'s", "09", "a", "\r", "👍🏽"]} +{"text": "½​- ३𐞁𐞁éع字's9t'D🙂😀🏽#$%ß字٣٤٥٦s'M", "tokens": 48, "pieces": ["½", "​-<", "EOT", ">", " ", " ", "३", "𐞁𐞁e", "́ع字", "'s", "9", "t", "'D", "🙂😀🏽#$%", "ß字", "٣٤٥", "٦", "s", "'M"]} +{"text": "३é.'Re-٣٤٥٦Dž#$%𐞁​́é<|fim_prefix|>ꟲ\".😀🏽<𐞁t12345678dd\t\t(ḍ̇-　", "tokens": 62, "pieces": ["३", "e", "́.'", "Re", "-", "٣٤٥", "٦", "Dž", "#$%", "𐞁", "​́", "é", "<|", "fim", "_prefix", "|>", "ꟲ", "\".😀🏽<", "𐞁t", "123", "456", "78", "dd", "\t", "\t", "(ḋ", "̣-", "　"]} +{"text": "9…३Ⅳ漢EOTe👍🏽𐞁s漢EOTå9fi'll", "tokens": 34, "pieces": ["9", "…", "३Ⅳ", "漢EOTe", "👍🏽", "𐞁s漢EOTa", "̊", "9", "fi", "'ll"]} +{"text": "㋿\r…'VEEOTß12345678å'lléfi​
å'T㍿'s'M　\r'TåDžd>漢 d　㍿", "tokens": 58, "pieces": ["㋿\r", "…", "'VE", "EOTß", "123", "456", "78", "a", "̊'", "lléfi", "​<", "EOT", ">", "
a", "̊'", "T", "㍿'", "s", "'M", "　\r", "'T", "a", "̊Džd", ">漢", " ", " d", "　", "㍿"]} +{"text": "ꟲ's'ſ…Zt\r\n\r\n<🙂ꟲ\teß'T'Dꟲ'll.𐞁dZdtDžEOT", "tokens": 45, "pieces": ["ꟲ", "'", "s", "'ſ", "…Zt", "\r\n\r\n", "<🙂", "ꟲ", "\teß", "'T", "'D", "ꟲ", "'ll", ".𐞁dZdtDžEOT", ""]} +{"text": "s<|fim_prefix|>​ſ\u000b-d<漢're\"👍🏽(…\r'ſ#$%\"'S<|fim_prefix|>a
'D\n\té-åİ'Ⅳd<|endoftext|>字\r<|fim_prefix|>", "tokens": 76, "pieces": ["s", "<|", "fim", "_prefix", "|>​", "ſ", "\u000b", "-d", "<漢", "'re", "\"👍🏽(", "…\r", "'ſ", "#$%\"'", "S", "<|", "fim", "_prefix", "|>", "a", "
", "'D", "\n", "\té", "-a", "̊İ", "'<", "EOT", ">", "Ⅳ", "d", "<|", "endoftext", "|>", "字", "\r", "<|", "fim", "_prefix", "|>"]} +{"text": "́'re9\r\n\r\n>٣٤٥٦asß>​ع'ſ'VEe\t'S9, Z'Re­𐞁'Re", "tokens": 42, "pieces": ["́'", "re", "9", "\r\n\r\n", ">", "٣٤٥", "٦", "asß", ">​", "ع", "'ſ", "'VE", "e", "\t", "'S", "9", ",", " Z", "'Re", "­<", "META", "_START", ">𐞁", "'Re"]} +{"text": "😀🏽👍🏽\t", "tokens": 12, "pieces": ["😀🏽👍🏽", "\t"]} +{"text": "a㋿'ll!!\r\n\r\né\"! ' -9'reＡ३ß <#$%'VE\"0!#$%Dž漢'T\r\n", "tokens": 37, "pieces": ["a", "㋿'", "ll", "!!\r\n\r\n", "é", "\"!", " '", " ", "-", "9", "'re", "Ａ", "३", "ß", " ", "<#$%'", "VE", "\"", "0", "!#$%", "Dž漢", "'T", "\r\n"]} +{"text": " \nßDž😀🏽's,m-'ſEOT𐞁𐞁", "tokens": 28, "pieces": [" \n", "ßDž", "😀🏽<", "EOT", ">'", "s", ",m", "-'", "ſEOT𐞁𐞁"]} +{"text": "
\r㍿<|endoftext|>'ll(<|fim_prefix|>　fi​eⅣ٣٤٥٦", "tokens": 36, "pieces": ["
\r", "㍿<|", "endoftext", "|>'", "ll", "(<|", "fim", "_prefix", "|>", "　fi", "​e", "Ⅳ٣٤", "٥٦"]} +{"text": "İZ \n12345678fi'Re'sſ
字,　<|fim_prefix|>'D,12345678\"a ,㋿", "tokens": 34, "pieces": ["İZ", " \n", "123", "456", "78", "fi", "'Re", "'s", "ſ", "
字", ",", "　", "<|", "fim", "_prefix", "|>'", "D", ",", "123", "456", "78", "\"a", " ,㋿"]} +{"text": "EOT漢!fi,İ'se'Re㍿३'VE#$%t0Z\u000bⅣZ", "tokens": 33, "pieces": ["EOT漢", "!fi", ",İ", "'s", "e", "'Re", "㍿", "३", "'VE", "#$%<", "EOT", ">t", "0", "Z", "\u000b", "Ⅳ", "Z"]} +{"text": " \u000b🙂<|fim_prefix|>　
12345678'TtⅣꟲ>fi३…㍿sⅣEOT'VE½étİé", "tokens": 47, "pieces": [" ", "\u000b", "🙂<|", "fim", "_prefix", "|>", "　", "
", "123", "456", "78", "'T", "t", "Ⅳ", "ꟲ", ">fi", "३", "…", "㍿s", "Ⅳ", "EOT", "'VE", "½", "e", "́tİé"]} +{"text": "٣٤٥٦0\r\n12345678<|endoftext|>m'D \n٣٤٥٦'S३ſ\t漢'Re३'M12345678Ⅳ'VE're.字　.<|fim_prefix|>\u000b<|fim_prefix|>m", "tokens": 73, "pieces": ["٣٤٥", "٦0", "\r\n", "123", "456", "78", "<|", "endoftext", "|>", "m", "'D", " \n", "٣٤٥", "٦", "'S", "", "३", "ſ", "\t漢", "'Re", "३", "'M", "123", "456", "78Ⅳ", "'VE", "'re", ".字", "　", ".<|", "fim", "_prefix", "|>", "\u000b", "<|", "fim", "_prefix", "|>", "m"]} +{"text": "㋿ß(12345678\tmع'S", "tokens": 11, "pieces": ["㋿ß", "(", "123", "456", "78", "\tmع", "'S"]} +{"text": "ꟲ​½s\u000b㋿#$%Ａ\"​ Dž!!ḍ̇Z字>fi'S\r\n\rå!e漢.ßa.👍🏽>'", "tokens": 51, "pieces": ["ꟲ", "​", "½", "s", "\u000b", "㋿#$%", "Ａ", "\"​", " Dž", "!!", "ḋ", "̣Z字", ">fi", "'S", "\r\n\r", "a", "̊!", "e漢", ".ßa", ".👍🏽>'"]} +{"text": "‍\t\t'DéＡ \n <|fim_prefix|>", "tokens": 17, "pieces": ["‍", "\t", "\t", "'D", "e", "́Ａ", " \n", " ", " <|", "fim", "_prefix", "|>"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " Dž#$%'T'T'M漢🙂<|fim_prefix|><'Té'D㋿👍🏽…fi🙂0Dž9👍🏽're😀🏽
éع\téa-.", "tokens": 73, "pieces": [" ", " Dž", "#$%'", "T", "'T", "'M", "漢", "🙂<|", "fim", "_prefix", "|><'", "Té", "'D", "㋿<", "EOT", ">👍🏽", "…fi", "🙂", "0", "Dž", "9", "👍🏽'", "re", "😀🏽", "
e", "́ع", "\téa", "-."]} +{"text": "​\r!,ad字t…𐞁>🙂'VE\r\n\r\nſ Dž, 'ſ'Re \n're're㍿'S0<漢'D,", "tokens": 43, "pieces": ["​\r", "!,", "ad字t", "…𐞁", ">🙂'", "VE", "\r\n\r\n", "ſ", " Dž", ",", " ", " '", "ſ", "'Re", " \n", "'re", "'re", "㍿'", "S", "0", "<漢", "'D", ","]} +{"text": "Z12345678.\"३\u000bİ字fi!!​\r\n\r\n🙂!!'ſ​㍿
'D!!t😀🏽½ſ٣٤٥٦
fi", "tokens": 57, "pieces": ["Z", "123", "456", "78", ".\"", "३", "\u000bİ字fi", "!!​\r\n\r\n", "🙂!!<", "EOT", ">'", "ſ", "​<", "EOT", ">㍿", "
", "'D", "!!", "t", "😀🏽", "½", "ſ", "٣٤٥", "٦", "
fi"]} +{"text": "\r\n\r\ns\t>\"étfiém'VEa're字\"Z\u000b m#$%İ\r<|endoftext|>s🙂ꟲ\te>", "tokens": 39, "pieces": ["\r\n\r\n", "s", "\t", ">\"", "étfie", "́m", "'VE", "a", "'re", "字", "\"Z", "\u000b", " m", "#$%", "İ", "\r", "<|", "endoftext", "|>", "s", "🙂ꟲ", "\te", ">"]} +{"text": "'VE#$%ꟲ'll٣٤٥٦ſꟲ åⅣ.…<|fim_prefix|>٣٤٥٦ea'M!!३Ⅳå'D\r\n​\r\n'S㋿½'ll½ع'MZEOT'", "tokens": 75, "pieces": ["'VE", "#$%", "ꟲ", "'ll", "٣٤٥", "٦", "ſꟲ", " a", "̊", "Ⅳ", ".", "…", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ea", "'M", "!!", "३Ⅳ", "a", "̊'", "D", "\r\n", "​\r\n", "'S", "㋿", "½", "'ll", "½", "ع", "'M", "ZEOT", "'"]} +{"text": "​Ⅳ EOT<|endoftext|>as!㋿éſd'DDž é,㍿(d​
\r\né a🙂ſ", "tokens": 50, "pieces": ["​", "Ⅳ", " EOT", "<|", "endoftext", "|>", "as", "!㋿", "éſd", "'D", "Dž", " ", " e", "́,㍿(", "d", "​", "
\r\n", "e", "́", " ", " a", "🙂ſ"]} +{"text": "㍿<|endoftext|>fitZ 𐞁'VE'll'D…\t🙂<Dž>Ａ!!'VEEOT'sع字's \"(…😀🏽'T!!'ſ​Z", "tokens": 65, "pieces": ["㍿<|", "endoftext", "|>", "fitZ", " ", " 𐞁", "'VE", "'ll", "'D", "…", "\t", "🙂<", "Dž", ">Ａ", "!!'", "VEEOT", "'s", "ع字", "'s", " ", "\"(", "…", "😀🏽'", "T", "!!'", "ſ", "​Z"]} +{"text": "ḍ̇½'Sa'́9Ⅳfi
 \n\u000bfi e,d…", "tokens": 30, "pieces": ["ḋ", "̣", "½", "'S", "a", "'́", "9", "", "Ⅳ", "fi", "
 \n", "\u000bfi", " ", " e", ",d", "…"]} +{"text": "漢 éå🙂é \n(ſs#$%'s e( ,㋿🙂ꟲ\n㍿
", "tokens": 41, "pieces": ["漢", " éa", "̊🙂", "e", "́", " \n", "(ſs", "#$%'", "s", " ", " e", "(", " ", ",㋿🙂", "ꟲ", "\n", "㍿<", "EOT", ">", "
"]} +{"text": "s'VEß𐞁ſ9 mfi!!㍿<|fim_prefix|>s9 　", "tokens": 43, "pieces": ["s", "'VE", "ß𐞁ſ", "9", " ", " mfi", "!!㍿<|", "fim", "_prefix", "|>", "s", "", "9", " 　"]} +{"text": ">-ſEOT'T<<|endoftext|>\r　fi漢👍🏽 a\"å㍿ ​EOT\n're-Z \n<|endoftext|>sm٣٤٥٦字", "tokens": 62, "pieces": [">-", "ſEOT", "'T", "<<|", "endoftext", "|>\r", "", "　fi漢", "👍🏽", " ", " a", "\"a", "̊㍿", " ", " ​", "EOT", "\n", "'re", "-Z", " \n", "<|", "endoftext", "|>", "sm", "٣٤٥", "٦", "字"]} +{"text": "\"𐞁ſ-٣٤٥٦ßémİ!३٣٤٥٦á>‍\r\n\r\n<|fim_prefix|>#$%ß're
🙂a<|fim_prefix|><|endoftext|>12345678d\",", "tokens": 70, "pieces": ["\"𐞁ſ", "-", "٣٤٥", "٦", "ßémİ", "!", "३٣٤", "٥٦", "a", "́>‍\r\n\r\n", "<|", "fim", "_prefix", "|>#$%", "ß", "'re", "
", "🙂a", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "123", "456", "78", "d", "\","]} +{"text": "😀🏽…,‍('T𐞁#$%
Ⅳع\t'字's're\r\n\r\n'll👍🏽ع .½< 😀🏽é३'VE漢漢'S\nⅣع…Ａ#$%", "tokens": 66, "pieces": ["😀🏽", "…", ",‍('", "T𐞁", "#$%", "
", "Ⅳ", "ع", "\t", "'字", "'s", "'re", "\r\n\r\n", "'ll", "👍🏽", "ع", " ", " .", "½", "<", " ", " 😀🏽", "é", "३", "'VE", "漢漢", "'S", "\n", "Ⅳ", "ع", "…Ａ", "#$%"]} +{"text": "Z \n>३d'S<|endoftext|>12345678👍🏽😀🏽'lĺ(­…㍿\t㋿㋿字fie0'Ａ'll漢s's🙂👍🏽😀🏽'VE<🙂fi", "tokens": 78, "pieces": ["Z", " \n", ">", "३", "d", "'S", "<|", "endoftext", "|>", "123", "456", "78", "👍🏽😀🏽'", "ll", "́(­", "…", "㍿", "\t", "㋿㋿", "字fie", "0", "'Ａ", "'ll", "漢s", "'s", "🙂👍🏽😀🏽'", "VE", "<🙂", "fi"]} +{"text": "é>EOT'VE'VE　'VEⅣ
 'll३0d12345678👍🏽Ⅳ字å字Ａḍ̇½<|fim_prefix|>Ⅳ<　<|endoftext|>å", "EOT", "'VE", "'VE", "　", "'VE", "Ⅳ", "
", " '", "ll", "३0", "d", "123", "456", "78", "👍🏽", "Ⅳ", "字a", "̊字Ａḋ", "̣", "½", "<|", "fim", "_prefix", "|>", "Ⅳ", "<", "　", "<|", "endoftext", "|>", "a", "̊!字<|fim_prefix|>#$%عté12345678😀🏽İ३d­>ꟲ-'sa३½­…é", "tokens": 62, "pieces": ["٣٤٥", "٦", " fi", "!<", "META", "_START", ">字", "<|", "fim", "_prefix", "|>#$%", "عte", "́", "123", "456", "78", "😀🏽", "İ", "३", "d", "­>", "ꟲ", "-'", "sa", "३", "", "½", "­", "…e", "́"]} +{"text": "éEOTe.(　٣٤٥٦<|fim_prefix|>e😀🏽🙂 😀🏽><|endoftext|>é٣٤٥٦Z \n>　'llfié<|fim_prefix|>👍🏽漢ꟲ .'M", "tokens": 84, "pieces": ["éEOT", "e", ".(", "　", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "e", "😀🏽🙂", " 😀🏽><|", "endoftext", "|>", "e", "́", "٣٤٥", "٦", "Z", " \n", "><", "META", "_START", ">", "　", "'ll", "fié", "<|", "fim", "_prefix", "|>👍🏽", "漢ꟲ", " ", " .'", "M"]} +{"text": "ßꟲs𐞁🙂<|fim_prefix|>e\"\r\n\r\n \n", "tokens": 24, "pieces": ["ßꟲs𐞁", "🙂<", "META", "_START", "><|", "fim", "_prefix", "|>", "e", "\"\r\n\r\n", " \n"]} +{"text": "<|fim_prefix|>'T　#$%‍'ll<|fim_prefix|>'T \t0字ꟲé😀🏽é\n'🙂( m½é'M\u000b<|fim_prefix|>㋿>", "tokens": 59, "pieces": ["<|", "fim", "_prefix", "|>'", "T", "　", "#$%‍'", "ll", "<|", "fim", "_prefix", "|>'", "T", " ", "\t", "0", "字ꟲé", "😀🏽", "e", "́\n", "'🙂(", " m", "½", "é", "'M", "\u000b", "<|", "fim", "_prefix", "|>㋿>"]} +{"text": "́​ꟲ", "tokens": 5, "pieces": ["́​", "ꟲ"]} +{"text": "'lla३́ꟲ　 🙂
", "tokens": 15, "pieces": ["'ll", "a", "३", "́ꟲ", "", "　", " 🙂", "
"]} +{"text": "0٣٤٥٦İ's(12345678\u000b0'M ", "tokens": 68, "pieces": ["0", "", "٣٤٥", "٦", "İ", "'s", "(", "123", "456", "78", "\u000b", "0", "'M", " "]} +{"text": "\u000bſ9́ \n'Re𐞁\r漢<|fim_prefix|>", "tokens": 21, "pieces": ["\u000bſ", "9", "́", " \n", "'Re", "𐞁", "\r", "漢", "<|", "fim", "_prefix", "|>"]} +{"text": "\r\n'D\r\nİ ('M٣٤٥٦ ع", "tokens": 16, "pieces": ["\r\n", "'D", "\r\n", "İ", " ('", "M", "٣٤٥", "٦", " ع"]} +{"text": "Ⅳ\"ꟲ\nß😀🏽9👍🏽é㍿ع㍿Džع!!𐞁　'S9😀🏽9'Ssꟲ'S‍<|endoftext|>\r\n३-
<|fim_prefix|>'EOTḍ̇३a", "tokens": 87, "pieces": ["Ⅳ", "\"ꟲ", "\n", "ß", "😀🏽", "9", "👍🏽", "e", "́㍿", "ع", "㍿Džع", "!!", "𐞁", "　", "'S", "9", "😀🏽", "9", "'S", "sꟲ", "'S", "‍<|", "endoftext", "|>\r\n", "३", "-", "
", "<|", "fim", "_prefix", "|>'", "EOTḋ", "̣", "३", "a"]} +{"text": "🙂́\u000bß!!Ⅳ,\r(漢'sꟲ½字\r\nＡ,\t'Tå𐞁३!!👍🏽Ⅳ\nt'Se", "tokens": 54, "pieces": ["🙂́<", "META", "_START", ">", "\u000bß", "!!", "Ⅳ", ",\r", "(漢", "'s", "ꟲ", "½", "字", "\r\n", "Ａ", ",", "\t", "'T", "a", "̊𐞁", "३", "!!👍🏽", "Ⅳ", "\n", "t", "'S", "e"]} +{"text": "'D\"se🙂ع12345678e", "tokens": 10, "pieces": ["'D", "\"se", "🙂ع", "123", "456", "78", "e"]} +{"text": "<( \n (", "tokens": 12, "pieces": ["<(<", "META", "_START", ">", " \n", "", " ", "("]} +{"text": "İ'MEOT'ſ\u000b३\r\n\r\nع'M\r\n>\t ", "tokens": 20, "pieces": ["İ", "'M", "EOT", "'ſ", "", "\u000b", "३", "\r\n\r\n", "ع", "'M", "\r\n", ">", "\t "]} +{"text": " -'S\r\né漢0ſ​'s'T'ſ '\u000b'D,㋿'M​td㍿<|endoftext|>", "tokens": 45, "pieces": [" -<", "META", "_START", ">'", "S", "\r\n", "é漢", "0", "ſ", "​<", "EOT", ">'", "s", "'T", "'ſ", " ", " '", "\u000b", "'D", ",㋿'", "M", "​td", "㍿<|", "endoftext", "|>"]} +{"text": "\nꟲ㍿<|endoftext|> 'M'sm½EOT 　sꟲ ع🙂!\n!!Ⅳ-<|fim_prefix|>.m'M\r㍿'Rem", "tokens": 54, "pieces": ["\n", "ꟲ", "㍿<|", "endoftext", "|>", " ", " '", "M", "'s", "m", "½", "EOT", " ", "　sꟲ", " ع", "🙂!\n", "!!", "Ⅳ", "-<|", "fim", "_prefix", "|>.", "m", "'M", "\r", "㍿'", "Rem"]} +{"text": "ع 'Res", "tokens": 8, "pieces": ["ع", "", " ", "'Re", "s"]} +{"text": "9! \ń'SDž३'Ss'D0(Z🙂ſDž!!ꟲß0…fi", "tokens": 31, "pieces": ["9", "!", " \n", "́'", "SDž", "३", "'S", "s", "'D", "0", "(Z", "🙂ſDž", "!!", "ꟲß", "0", "…fi"]} +{"text": "\"́½ \nⅣＡéꟲ👍🏽<Ａ( ́d漢👍🏽'ſ'S12345678\t'D,Dž12345678३", "tokens": 51, "pieces": ["\"́", "½", " \n", "Ⅳ", "Ａéꟲ", "👍🏽<", "Ａ", "(", " ", " ́", "d漢", "👍🏽'", "ſ", "'S", "123", "456", "78", "\t", "'D", ",Dž", "123", "456", "78३"]} +{"text": "\r\nå, !!fi ́字½12345678\n're𐞁'Re　<|endoftext|>👍🏽tEOT👍🏽 emⅣe", "tokens": 51, "pieces": ["\r\n", "a", "̊,", " ", " !!", "fi", " ", "́字", "½12", "345", "678", "\n", "'re", "𐞁", "'Re", "　", "<|", "endoftext", "|>👍🏽", "tEOT", "👍🏽", " ", " em", "Ⅳ", "e"]} +{"text": "\réße字0's
३ḍ̇0EOT", "tokens": 23, "pieces": ["\r", "e", "́ß", "e字", "0", "'s", "
", "३", "ḋ", "̣", "0", "EOT"]} +{"text": "Ａ'ſ🙂EOT\"ḍ̇Ⅳ㍿́٣٤٥٦", "tokens": 32, "pieces": ["Ａ", "'ſ", "🙂EOT", "\"ḋ", "̣", "Ⅳ", "㍿́", "٣٤٥", "٦"]} +{"text": "३.'M\t", "tokens": 5, "pieces": ["३", ".'", "M", "\t"]} +{"text": "'Re'llİ< \r\n\r\n\nEOT.d'll!'S,", "tokens": 19, "pieces": ["'Re", "'ll", "İ", "<", " \r\n\r\n\n", "EOT", ".d", "'", "ll", "!'", "S", ","]} +{"text": "1234567812345678'D'M\"㍿🙂'S'S<|endoftext|> >\n.'S.é𐞁​🙂 ſ \n<\r\nḍ̇0👍🏽!३,", "tokens": 62, "pieces": ["123", "456", "781", "234", "567", "8", "'D", "'M", "\"㍿🙂'", "S", "'S", "<|", "endoftext", "|>", " >\n", ".'", "S", ".e", "́<", "EOT", ">𐞁", "​🙂", " ſ", " \n", "<\r\n", "ḋ", "̣", "0", "👍🏽!", "३", ","]} +{"text": "'Sß'll
‍ß-é'D.Ａ३!!字're'll(0a'\"ⅣEOT‍🙂\r\n㍿😀🏽Ａ#$%EOT \n!!fíḍ̇", "tokens": 61, "pieces": ["'S", "ß", "'ll", "
", "‍ß", "-e", "́'", "D", ".Ａ", "३", "!!", "字", "'re", "'ll", "(", "0", "a", "'\"", "Ⅳ", "EOT", "‍🙂\r\n", "㍿😀🏽", "Ａ", "#$%", "EOT", " \n", "!!", "fi", "́ḋ", "̣"]} +{"text": "👍🏽Z'ſ \r\n-\r\n!字fi \n,s \n㍿,<|fim_prefix|>12345678🙂漢عemsḍ̇'e\"<|endoftext|>\tⅣ", "tokens": 56, "pieces": ["👍🏽", "Z", "'ſ", " \r\n", "-\r\n", "!字fi", " \n", ",s", " \n", "㍿,<|", "fim", "_prefix", "|>", "123", "456", "78", "🙂漢عemsḋ", "̣'", "e", "\"<|", "endoftext", "|>", "\t", "Ⅳ"]} +{"text": "!!\t\r\n\r\n
😀🏽\r0\u000b😀🏽ع <|endoftext|>0 ٣٤٥٦\u000b​s३<|endoftext|> ́'VEEOT　mmå-३字३  漢", "tokens": 73, "pieces": ["!!", "\t\r\n\r\n", "
", "😀🏽\r", "0", "\u000b", "😀🏽", "ع", " <|", "endoftext", "|>", "0", " ", "٣٤٥", "٦", "\u000b", "​s", "३", "<|", "endoftext", "|>", " ", "́'", "VEEOT", "　mma", "̊<", "META", "_START", ">-", "३", "字", "", "३", " ", " 漢"]} +{"text": "ع½śdDž!!,Z'llt३́Ａ", "tokens": 16, "pieces": ["ع", "½", "s", "́dDž", "!!,", "Z", "'ll", "t", "३", "́Ａ"]} +{"text": "<|endoftext|>'ll12345678 \n漢're३㍿'S-\"'re…<|endoftext|>'Śḍ̇!!", "tokens": 42, "pieces": ["<|", "endoftext", "|>'", "ll", "123", "456", "78", " \n", "漢", "'re", "३", "㍿'", "S", "-\"'", "re", "…", "<|", "endoftext", "|>'", "S", "́ḋ", "̣!!"]} +{"text": "ḍ̇0!\t​👍🏽9ſ(dⅣt'VE­0'D<漢\r\n\r\nＡ<|fim_prefix|>EOTfi", "tokens": 44, "pieces": ["ḋ", "̣", "0", "!", "\t", "​👍🏽", "9", "ſ", "(d", "Ⅳ", "t", "'VE", "­", "0", "'D", "<漢", "\r\n\r\n", "Ａ", "<|", "fim", "_prefix", "|>", "EOTfi"]} +{"text": "‍ \"\rsß½ꟲß😀🏽…漢'M㍿<́-t‍漢Zعd<|endoftext|>", "tokens": 43, "pieces": ["‍", " ", " \"\r", "sß", "½", "ꟲß", "😀🏽", "…漢", "'M", "㍿<́-", "t", "‍漢Zعd", "<|", "endoftext", "|>"]} +{"text": " 're!ém<|fim_prefix|>​\r\n<|endoftext|>\rع​#$%'llḍ̇㋿𐞁Ａ!>\rå­12345678ſéꟲ٣٤٥٦Ａm\r\n\r\nع'ſ'", "tokens": 76, "pieces": [" '", "re", "!e", "́m", "<|", "fim", "_prefix", "|>​\r\n", "<|", "endoftext", "|>\r", "ع", "​#$%'", "llḋ", "̣㋿", "𐞁Ａ", "!>\r", "a", "̊­", "123", "456", "78", "ſe", "́ꟲ", "٣٤٥", "٦", "Ａm", "\r\n\r\n", "ع", "'ſ", "'"]} +{"text": "'sEOTefi'\r\n\r\ń­ß<|endoftext|>t'reⅣ🙂", "tokens": 23, "pieces": ["'s", "EOTefi", "'\r\n\r\n", "́­", "ß", "<|", "endoftext", "|>", "t", "'re", "Ⅳ", "🙂"]} +{"text": ">(😀🏽'M\téⅣ\u000b'VE!! \n(\tm\t12345678<.t<|fim_prefix|>'re​, ́ꟲ!!​\r\nDž'sḍ̇", "tokens": 56, "pieces": [">(😀🏽'", "M", "\té", "Ⅳ", "\u000b", "'VE", "!!", " \n", "(", "\tm", "\t", "123", "456", "78", "<.", "t", "<|", "fim", "_prefix", "|>'", "re", "​,", " ", " ́", "ꟲ", "!!​<", "EOT", ">\r\n", "Dž", "'s", "ḋ", "̣"]} +{"text": "🙂'ſⅣ'Re漢́㍿'Mmd-t!!🙂-!!'Sİ'ſ३㋿\r𐞁'S'T<|endoftext|>m'VEſ…dع'rem٣٤٥٦", "tokens": 70, "pieces": ["🙂'", "ſ", "Ⅳ", "'Re", "漢", "́㍿<", "META", "_START", ">'", "Mmd", "-t", "!!🙂-!!'", "Sİ", "'ſ", "३", "㋿\r", "𐞁", "'S", "'T", "<|", "endoftext", "|>", "m", "'VE", "ſ", "…dع", "'re", "m", "٣٤٥", "٦"]} +{"text": "-👍🏽­'ll12345678!३㋿ß‍ 'så\r\nⅣ\r\n\r\n
\r\n ㍿३ m \n\t", "tokens": 42, "pieces": ["-👍🏽­'", "ll", "123", "456", "78", "!", "३", "㋿ß", "‍", " '", "sa", "̊\r\n", "Ⅳ", "\r\n\r\n
\r\n", " ", "㍿", "३", " m", " \n\t"]} +{"text": "ḍ̇!m㍿s\r　,𐞁ع fí", "tokens": 23, "pieces": ["ḋ", "̣!", "m", "㍿s", "\r", "　", ",𐞁ع", " ", " fi", "́"]} +{"text": ",İ\r\n\u000b", "tokens": 4, "pieces": [",İ", "\r\n\u000b"]} +{"text": "‍#$%<'S'ReⅣee𐞁'M \r\n9-٣٤٥٦…,'re٣٤٥٦'M'VE 'T٣٤٥٦Ａ-\"-'VEſſ-", "tokens": 70, "pieces": ["‍#$%<'", "S", "'Re", "Ⅳ", "ee𐞁", "'M", " \r\n", "9", "-", "٣٤٥", "٦", "…", ",'", "re", "٣٤٥", "٦", "'M", "'VE", "", " ", " '", "T", "٣٤٥", "٦", "Ａ", "-\"-'", "VEſſ", "-"]} +{"text": "<|endoftext|>३ <‍İ'så३'Mta's'M'ſ're \r\n\r\nꟲ\n😀🏽'Reꟲå\ndꟲ9Z\r\u000b \n\r\nté­İ", "tokens": 64, "pieces": ["<|", "endoftext", "|>", "३", " <‍<", "EOT", ">İ", "'s", "a", "̊", "३", "'M", "ta", "'s", "'M", "'ſ", "'re", " \r\n\r\n", "ꟲ", "\n", "😀🏽'", "Reꟲa", "̊\n", "dꟲ", "9", "Z", "\r\u000b \n\r\n", "té", "­İ"]} +{"text": "㋿<|endoftext|>é‍å \n३!😀🏽 \n'VE\t\r\n\r\n\néßå🙂\r\n\r\nfia", "tokens": 51, "pieces": ["㋿<", "META", "_START", "><|", "endoftext", "|>", "é", "‍", "a", "̊", " \n", "३", "!😀🏽", " \n", "'VE", "\t\r\n\r\n\n", "éßa", "̊🙂\r\n\r\n", "fia"]} +{"text": "字s12345678", "tokens": 5, "pieces": ["字s", "123", "456", "78"]} +{"text": "ḍ̇m㋿\n👍🏽,😀🏽 漢 \n­'T漢\t́'MåEOT're9t­\rß!　Ⅳ‍'Re\r", "tokens": 49, "pieces": ["\u000b", "́'", "MİＡ", "123", "456", "78", "EOT", "<|", "fim", "_prefix", "|>'", "T漢", "\t", "́'", "Ma", "̊EOT", "'re", "", "9", "t", "­\r", "ß", "!", "　", "Ⅳ", "‍'", "Re", "\r"]} +{"text": "漢٣٤٥٦d<|fim_prefix|>'sEOT #$%​<|endoftext|> 'Sfiع!\"ſ漢", "tokens": 42, "pieces": ["漢", "٣٤٥", "٦", "d", "<|", "fim", "_prefix", "|>'", "sEOT", " ", "#$%​<|", "endoftext", "|>", " ", "'S", "fiع", "!\"", "ſ漢"]} +{"text": "'🙂ſ३'VE99
t…🙂字EOT0. \n​عEOTtꟲDžfí'llİ‍\r\né", "tokens": 44, "pieces": ["'🙂", "ſ", "३", "'VE", "99", "
t", "…", "🙂字EOT", "0", ".", " \n", "​عEOTtꟲDžfi", "́'", "llİ", "‍\r\n", "e", "́"]} +{"text": "'S​'ll‍
İ", "tokens": 9, "pieces": ["'S", "​'", "ll", "‍", "
İ"]} +{"text": "å'Så ३é👍🏽'VE><\r\n\r\n😀🏽𐞁'aEOT\r\n\r\nß'll0Dže​Ａꟲ", "tokens": 53, "pieces": ["a", "̊'", "Sa", "̊", " ", " ", "३", "é", "👍🏽'", "VE", "><\r\n\r\n", "😀🏽", "𐞁", "'<", "META", "_START", ">aEOT", "\r\n\r\n", "ß", "'ll", "0", "Dže", "​Ａꟲ"]} +{"text": "''Mع>🙂ꟲ\r\n 9e\"字
", "tokens": 20, "pieces": ["''", "Mع", ">🙂", "ꟲ", "\r\n", " ", "9", "e", "\"字", "
"]} +{"text": ",
!<|endoftext|>‍㍿…'M‍㍿ \r\n'Mtİḍ̇fi👍🏽.\r\n\r\né>'re\"\r\n", "tokens": 45, "pieces": [",", "
", "!<|", "endoftext", "|>‍㍿", "…", "'M", "‍㍿", " \r\n", "'M", "tİḋ", "̣fi", "👍🏽.\r\n\r\n", "é", ">'", "re", "\"\r\n"]} +{"text": "'M'D‍012345678'll12345678<|fim_prefix|>\u000bＡ　ḍ̇!Ⅳ,'ll‍Ａ>İé \n👍🏽're9''ll漢-.a\n㋿é", "tokens": 65, "pieces": ["'M", "'D", "‍", "012", "345", "678", "'ll", "123", "456", "78", "<|", "fim", "_prefix", "|>", "\u000bＡ", "　ḋ", "̣!", "Ⅳ", ",'", "ll", "‍Ａ", ">İe", "́", " \n", "👍🏽'", "re", "9", "''", "ll", "漢", "-.", "a", "\n", "㋿é"]} +{"text": "'VE-'llḍ̇fi'DⅣ.​,́", "tokens": 18, "pieces": ["'VE", "-'", "llḋ", "̣fi", "'D", "Ⅳ", ".​,́"]} +{"text": "<|endoftext|>tm😀🏽!!'T,'sfi,漢,'ſfiZé<|fim_prefix|>,,!!
\t", "tokens": 42, "pieces": ["<|", "endoftext", "|>", "tm", "😀🏽!!'", "T", ",'", "sfi", ",漢", ",'", "ſfiZe", "́<|", "fim", "_prefix", "|>,,!!", "
\t"]} +{"text": "́'Reꟲ \n'M>.　9 EOT#$%👍🏽t字 \nt​  ٣٤٥٦(<|fim_prefix|>'re\t", "tokens": 45, "pieces": ["́'", "Reꟲ", " \n", "'M", ">.", "　", "9", " EOT", "#$%👍🏽", "t字", " \n", "t", "​", " ", " ", "٣٤٥", "٦", "(<|", "fim", "_prefix", "|>'", "re", "\t"]} +{"text": "\t're…'VEſ字ea🙂\re٣٤٥٦'T字'reé'llꟲ'ſ½ \nⅣa", "tokens": 40, "pieces": ["\t", "'re", "…", "'VE", "ſ字ea", "🙂\r", "e", "٣٤٥", "٦", "'T", "字", "'re", "e", "́'", "llꟲ", "'ſ", "½", " \n", "Ⅳ", "a"]} +{"text": "\tⅣe　0漢\u000b'D,　,漢Ⅳع😀🏽é́
'D \n\t.‍dꟲİ‍#$%Ａ'T", "tokens": 52, "pieces": ["\t", "Ⅳ", "e", "　", "0", "漢", "\u000b", "'", "D", ",", "　", ",漢", "Ⅳ", "ع", "😀🏽", "é", "́", "
", "'D", "", " \n", "\t", ".‍", "dꟲİ", "‍#$%", "Ａ", "'T"]} +{"text": "字!!'s 're'D\r\n\r\n(​👍🏽\r\n\r\n𐞁'll fiDž123456780‍'Tå漢٣٤٥٦", "tokens": 47, "pieces": ["字", "!!'", "s", " ", " '", "re", "'D", "\r\n\r\n", "(​👍🏽\r\n\r\n", "𐞁", "'ll", " fiDž", "123", "456", "780", "‍'", "Ta", "̊漢", "٣٤٥", "٦"]} +{"text": "Ⅳ. ⅣEOT \raſ漢<👍🏽-!e.٣٤٥٦.m", "tokens": 35, "pieces": ["Ⅳ", ".", " ", "Ⅳ", "EOT", " \r", "aſ漢", "<👍🏽-!", "e", ".", "٣٤٥", "٦", ".m"]} +{"text": "-#$%'T…Z<|fim_prefix|>\"9 \n½<|fim_prefix|>'M", "tokens": 25, "pieces": ["-#$%'", "T", "…Z", "<|", "fim", "_prefix", "|>\"", "9", " \n", "½", "<|", "fim", "_prefix", "|>'", "M"]} +{"text": "ꟲ३'T .
<''T'ReＡe'S<|endoftext|>​a<|endoftext|>𐞁\r\n　>EOT're😀🏽Z'T\"#$%ßZ'll\r\n\r\n㍿", "tokens": 63, "pieces": ["ꟲ", "३", "'T", " ", " .", "
", "<''", "T", "'Re", "Ａe", "'S", "<|", "endoftext", "|>​", "a", "<|", "endoftext", "|>", "𐞁", "\r\n", "　", ">EOT", "'re", "😀🏽", "Z", "'T", "\"#$%", "ßZ", "'ll", "\r\n\r\n", "㍿"]} +{"text": ">m­​ \n'Ds\u000bé㋿́", "tokens": 12, "pieces": [">m", "­​", " \n", "'D", "s", "\u000bé", "㋿́"]} +{"text": "<‍\n'MZfiß\tfi!'S\r…'s!!tǻ漢عꟲعå'ſEOT", "tokens": 38, "pieces": ["<‍\n", "'M", "Zfiß", "\tfi", "!'", "S", "\r", "…", "'s", "!!", "ta", "̊́", "漢عꟲعa", "̊'", "ſEOT"]} +{"text": "字😀🏽👍🏽‍ع\u000b'S㍿<|endoftext|>a!'ll", "tokens": 30, "pieces": ["字", "😀🏽👍🏽‍", "ع", "\u000b", "'S", "㍿<|", "endoftext", "|>", "a", "!'", "ll"]} +{"text": "ꟲ<|endoftext|>‍😀🏽​('ſ<|endoftext|>\t ​\u000b'D!!㍿'s\r\n\r\n>\"字 \n‍", "tokens": 48, "pieces": ["ꟲ", "<|", "endoftext", "|>‍😀🏽<", "META", "_START", ">​('", "ſ", "<|", "endoftext", "|>", "\t ", " ​", "\u000b", "'D", "!!㍿'", "s", "\r\n\r\n", ">\"", "字", " \n", "‍"]} +{"text": "m \n漢'S \"İ漢𐞁", "tokens": 17, "pieces": ["m", " \n", "漢", "'", "S", " ", " \"", "İ漢𐞁"]} +{"text": " EOTé٣٤٥٦a#$%عEOT#$%ع\n🙂 aZ​EOT\r\nع字m'Re𐞁Ⅳ'SＡ'S  \u000b
!EOT'll字", "tokens": 58, "pieces": [" EOTe", "́", "٣٤٥", "٦", "a", "#$%", "عEOT", "#$%", "ع", "\n", "🙂", " ", " aZ", "​EOT", "\r\n", "ع字m", "'Re", "𐞁", "Ⅳ", "'S", "Ａ", "'S", "  \u000b", "
", "!EOT", "'ll", "字"]} +{"text": "́عé३'re!!.'­a !0s…\"åé's😀🏽9", "tokens": 29, "pieces": ["́عe", "́", "३", "'re", "!!.'­", "a", " !", "0", "s", "…", "\"a", "̊e", "́'", "s", "😀🏽", "9"]} +{"text": "ḍ̇é\t>\r.d‍'Re字 \tß!\t \n漢'fi t", "tokens": 29, "pieces": ["ḋ", "̣é", "\t", ">\r", ".d", "‍'", "Re字", " ", "\tß", "!<", "EOT", ">", "\t \n", "漢", "'fi", " t"]} +{"text": "Z'S0Ａ\r\n\r\n\r\n\r\nß,㋿", "tokens": 11, "pieces": ["Z", "'S", "0", "Ａ", "\r\n\r\n\r\n\r\n", "ß", ",㋿"]} +{"text": "½é½😀🏽#$%­😀🏽0 9ſ(\r\r>𐞁'Sß-<|fim_prefix|>👍🏽\n👍🏽İ\"عZ'Z,
", "tokens": 62, "pieces": ["½", "e", "́", "½", "😀🏽#$%­😀🏽", "0", " ", "9", "ſ", "(\r\r", ">𐞁", "'S", "ß", "-<|", "fim", "_prefix", "|>👍🏽\n", "👍🏽", "İ", "\"عZ", "'Z", ",", "
"]} +{"text": "!DžEOT's𐞁t ́\t🙂a٣٤٥٦ḍ̇ꟲＡZ .m0 !!İ'Dt\r'll<|endoftext|>Z\nZ#$%'s­㋿‍#$%", "tokens": 71, "pieces": ["!DžEOT", "'s", "𐞁t", " ́", "\t", "🙂a", "٣٤٥", "٦", "ḋ", "̣ꟲＡZ", " ", " <", "EOT", ">.", "m", "0", " ", " !!", "İ", "'D", "t", "\r", "'ll", "<|", "endoftext", "|>", "Z", "\n", "Z", "#$%'", "s", "­㋿‍#$%"]} +{"text": "<(d İs 9!!\r\n\r\n<|endoftext|>s…\r\n'ſ'D'ſ", "tokens": 27, "pieces": ["<(", "d", " ", " İs", " ", "9", "!!\r\n\r\n", "<|", "endoftext", "|>", "s", "…\r\n", "'ſ", "'D", "'ſ"]} +{"text": "́½0", "tokens": 3, "pieces": ["́", "½0"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㋿é­", "tokens": 5, "pieces": ["㋿é", "­"]} +{"text": "😀🏽aع'VE\"9㍿#$%😀🏽
eEOT!!'sſ<|fim_prefix|>'Re\u000bZ漢'T>­𐞁३　e'ſ‍", "tokens": 62, "pieces": ["😀🏽<", "EOT", ">aع", "'VE", "\"", "9", "㍿#$%😀🏽", "
eEOT", "!!'", "sſ", "<|", "fim", "_prefix", "|>'", "Re", "\u000bZ漢", "'T", ">­", "𐞁", "३", "　e", "'ſ", "‍"]} +{"text": "#$% #$%Zꟲ漢　EOT'ſ'reＡ­'VE漢😀🏽३😀🏽s9( >\r \n\u000b
m<|fim_prefix|>< \n٣٤٥٦​e\r\n٣٤٥٦
", "tokens": 76, "pieces": ["#$%", " ", "#$%", "Zꟲ漢", "　EOT", "'ſ", "'re", "Ａ", "­'", "VE漢", "😀🏽", "३", "😀🏽", "s", "9", "(", " >\r", " \n", "\u000b", "
m", "<|", "fim", "_prefix", "|><", " \n", "٣٤٥", "٦", "​e", "\r\n", "٣٤٥", "٦", "
"]} +{"text": "'s 
\nİ,'VE　-a \n\r12345678#$%'S", "tokens": 22, "pieces": ["'s", "", " 
\n", "İ", ",'", "VE", "　", "-a", " \n\r", "123", "456", "78", "#$%'", "S"]} +{"text": "'Mdꟲ字", "tokens": 6, "pieces": ["'M", "dꟲ字"]} +{"text": "…ſ　12345678…㍿  😀🏽m!!ḍ̇", "tokens": 27, "pieces": ["…ſ", "　", "123", "456", "78", "…", "㍿", " ", " ", "😀🏽", "m", "!!", "ḋ", "̣"]} +{"text": "12345678e.fi're#$%­ \n,漢𐞁👍🏽", "tokens": 25, "pieces": ["123", "456", "78", "e", ".fi", "'re", "#$%­", " \n", ",漢𐞁", "👍🏽"]} +{"text": ". 漢sss\u000b\r\n\r\nſs\n😀🏽\t'ret EOT!!‍㍿,\r're\u000b'Re\"Ⅳ'M ", "tokens": 38, "pieces": [".", " 漢sss", "\u000b\r\n\r\n", "ſs", "\n", "😀🏽", "\t", "'re", "t", " EOT", "!!‍㍿,\r", "'re", "\u000b", "'Re", "\"", "Ⅳ", "'M", " "]} +{"text": "!!(!!🙂Ａ㋿㍿0㍿", "tokens": 17, "pieces": ["!!(!!🙂", "Ａ", "㋿㍿", "0", "㍿"]} +{"text": "𐞁Ź\r\n\r\n're'Tſ'D\td'T're9­​e🙂Zع\n'D­\r\n\u000b😀🏽½.ع字😀🏽İEOT½", "tokens": 50, "pieces": ["𐞁Z", "́\r\n\r\n", "'re", "'T", "ſ", "'D", "\td", "'T", "'re", "9", "­<", "EOT", ">​", "e", "🙂Zع", "\n", "'D", "­\r\n", "\u000b", "😀🏽", "½", ".ع字", "😀🏽", "İEOT", "½"]} +{"text": "İ>", "tokens": 2, "pieces": ["İ", ">"]} +{"text": "é𐞁漢", "tokens": 7, "pieces": ["é𐞁漢"]} +{"text": "!! \n½👍🏽😀🏽#$%<|endoftext|><|fim_prefix|>.㋿<|endoftext|>\u000b", "tokens": 40, "pieces": ["!!", " \n", "½", "👍🏽😀🏽#$%<|", "endoftext", "|><|", "fim", "_prefix", "|>.㋿<|", "endoftext", "|>", "\u000b"]} +{"text": "'ll\r\n<|endoftext|>İ­\r\n\r\n's t's'reét\r٣٤٥٦dd(>m<|endoftext|>'D㍿é", "tokens": 46, "pieces": ["'ll", "\r\n", "<|", "endoftext", "|>", "İ", "­\r\n\r\n", "'s", " t", "'s", "'re", "e", "́t", "\r", "٣٤٥", "٦", "dd", "(><", "EOT", ">m", "<|", "endoftext", "|>'", "D", "㍿é"]} +{"text": "𐞁𐞁'T", "tokens": 9, "pieces": ["𐞁𐞁", "'T"]} +{"text": "12345678'T'ſ#$%d​EOTⅣ-…<|fim_prefix|>ꟲ's12345678Džé'llm'VE(Ａåİ́", "tokens": 47, "pieces": ["123", "456", "78", "'T", "'ſ", "#$%", "d", "​EOT", "Ⅳ", "-", "…", "<|", "fim", "_prefix", "|>", "ꟲ", "'s", "123", "456", "78", "Džé", "'ll", "m", "'VE", "(Ａa", "̊İ", "́"]} +{"text": "👍🏽12345678­  \n0å'S​́ſ0Dž9s12345678é\"ſEOT'llḍ̇-éDžfi're'S", "tokens": 55, "pieces": ["👍🏽", "123", "456", "78", "­", " ", "", " \n", "0", "a", "̊'", "S", "​́", "ſ", "0", "Dž", "9", "s", "123", "456", "78", "e", "́\"", "ſEOT", "'ll", "ḋ", "̣-", "e", "́Džfi", "'re", "'S"]} +{"text": "'llåfi😀🏽e\r\n\t　d<|fim_prefix|>", "tokens": 23, "pieces": ["'ll", "a", "̊fi", "😀🏽", "e", "\r\n", "\t", "　d", "<|", "fim", "_prefix", "|>"]} +{"text": "'Re\r\n\r\n<|endoftext|>#$%( \n", "tokens": 15, "pieces": ["'Re", "\r\n\r\n", "<|", "endoftext", "|>#$%(", " \n"]} +{"text": ",­s.\t\u000b㋿
!!😀🏽'M<|endoftext|>#$%㋿漢e's🙂\t३e'T'm!!.,EOT
­mꟲZ", "tokens": 62, "pieces": [",<", "META", "_START", ">­", "s", ".", "\t", "\u000b", "㋿", "
", "!!😀🏽'", "M", "<|", "endoftext", "|>#$%㋿", "漢e", "'s", "🙂", "\t", "३", "e", "'T", "'m", "!!.,", "EOT", "
", "­", "mꟲZ"]} +{"text": "​>ꟲꟲ㋿\r\nſ👍🏽'Re'T ½'M(", "tokens": 27, "pieces": ["​>", "ꟲꟲ", "㋿\r\n", "ſ", "👍🏽'", "Re", "'T", " ", "½", "'M", "("]} +{"text": "Ａꟲſ
!dfiEOT12345678", "tokens": 18, "pieces": ["Ａꟲſ", "
", "!dfiEOT", "123", "456", "78"]} +{"text": "字'S<|endoftext|>9('llꟲ,d-\r\n\r\n12345678té\r\n\r\n٣٤٥٦'S9🙂\"-ꟲＡ9 \n>'llſ­EOT", "tokens": 51, "pieces": ["字", "'S", "<|", "endoftext", "|>", "9", "('", "llꟲ", ",d", "-\r\n\r\n", "123", "456", "78", "té", "\r\n\r\n", "٣٤٥", "٦", "'S", "9", "🙂\"-", "ꟲＡ", "9", " \n", ">'", "llſ", "­EOT"]} +{"text": " ३fi㍿(\re'reſ\"'VE0😀🏽!!𐞁12345678<|endoftext|>ع ZZ'Reé­a9🙂́'ſ­s​", "tokens": 53, "pieces": [" ", " ", "३", "fi", "㍿(\r", "e", "'re", "ſ", "\"'", "VE", "0", "😀🏽!!", "𐞁", "123", "456", "78", "<|", "endoftext", "|>", "ع", " ZZ", "'Re", "é", "­a", "9", "🙂́'", "ſ", "­s", "​"]} +{"text": "‍'re!!'Re\rfiDž½🙂'Dḍ̇d'ſ…'s\r\n\r\n\r\n('", "tokens": 31, "pieces": ["‍'", "re", "!!'", "Re", "\r", "fiDž", "½", "🙂'", "Dḋ", "̣d", "'ſ", "…", "'s", "\r\n\r\n\r\n", "('"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "!
\t​İ'M ㋿'Dm>ꟲ
'llEOTꟲ ꟲꟲ\"fi\r\nm!ß(…\u000b…𐞁", "tokens": 49, "pieces": ["!", "
", "\t", "​İ", "'M", " ㋿'", "Dm", ">ꟲ", "
", "'ll", "EOTꟲ", " ꟲꟲ", "\"fi", "\r\n", "m", "!ß", "(", "…\u000b", "…𐞁"]} +{"text": "٣٤٥٦'D \n#$%\t0-aå😀🏽0‍!e㍿🙂#$%fi-…٣٤٥٦é", "tokens": 49, "pieces": ["٣٤٥", "٦", "'D", " \n", "#$%", "\t", "0", "-aa", "̊😀🏽", "0", "‍!", "e", "㍿🙂#$%", "fi", "-", "…", "٣٤٥", "٦", "é"]} +{"text": "'re\r\n​!! \n​漢Ａ\r\n㍿!!\r\nع.😀🏽'ſ<|fim_prefix|>\rZ<\r\n", "tokens": 39, "pieces": ["'re", "\r\n", "​!!", " \n", "​漢Ａ", "\r\n", "㍿!!\r\n", "ع", ".😀🏽'", "ſ", "<|", "fim", "_prefix", "|>\r", "Z", "<\r\n"]} +{"text": "­ꟲ9'M.Z　😀🏽'VE's😀🏽>\r\n\r\nß .
 \ń'sḍ̇'VEm", "tokens": 44, "pieces": ["­ꟲ", "9", "'M", ".Z", "　", "😀🏽'", "VE", "'s", "😀🏽>\r\n\r\n", "ß", " .", "
 \n", "́<", "META", "_START", ">'", "sḋ", "̣<", "EOT", ">'", "VEm"]} +{"text": "Dž \r\n\r\nß㍿'re<|fim_prefix|>👍🏽'Re ​0", "tokens": 27, "pieces": ["Dž", " \r\n\r\n", "ß", "㍿'", "re", "<|", "fim", "_prefix", "|>👍🏽'", "Re", " ​", "0"]} +{"text": "İe'll", "tokens": 3, "pieces": ["İe", "'ll"]} +{"text": "ſ<|endoftext|>
é0'Re#$%daꟲ 'S٣٤٥٦'Re½漢\r\n ٣٤٥٦́　㍿Ａ'll㋿(", "tokens": 63, "pieces": ["ſ", "<|", "endoftext", "|>", "
e", "́", "0", "'Re", "#$%<", "EOT", ">daꟲ", " ", " '", "S", "٣٤٥", "٦", "'Re", "½", "漢", "\r\n", " ", " ", "٣٤٥", "٦", "́", "　", "㍿Ａ", "'ll", "㋿("]} +{"text": "٣٤٥٦-tꟲ9EOTſ\u000b́Zm0-.<|endoftext|>\r\n ́ <'s<|endoftext|>
", "tokens": 52, "pieces": ["٣٤٥", "٦", "-tꟲ", "9", "EOT", "ſ", "\u000b", "́Zm", "0", "-.<|", "endoftext", "|>\r\n", " ", " ́", " ", " <'", "s", "<|", "endoftext", "|>", "
"]} +{"text": "a'd12345678𐞁", "tokens": 9, "pieces": ["a", "'d", "123", "456", "78", "𐞁"]} +{"text": "ß\r'ſ\u000b٣٤٥٦>Ⅳ㍿Ⅳséåſ𐞁\u000bİ'VE字EOT'Re😀🏽 0😀🏽eaⅣ", "tokens": 73, "pieces": ["ß", "\r", "'ſ", "\u000b", "٣٤٥", "٦", ">", "Ⅳ", "㍿", "Ⅳ", "séa", "̊<", "m", "\"", " \n \n\r\n\r\n", "'Re", "'D", "\u000b𐞁", ".<", "EOT", ">ſ𐞁", "\u000bİ", "'VE", "字EOT", "'Re", "😀🏽", " ", "0", "😀🏽", "ea", "Ⅳ"]} +{"text": "\r\n\r\n<|endoftext|>!!\u000b'Dḍ̇å\r …a<|fim_prefix|>!!#$%👍🏽ꟲⅣ-\tt.\r\n
\"​𐞁\"\r\n\r\n'Z'ſ'S#$%", "tokens": 72, "pieces": ["\r\n\r\n", "<|", "endoftext", "|>!!", "\u000b", "'D", "ḋ", "̣a", "̊\r", " ", "…a", "<|", "fim", "_prefix", "|>!!#$%👍🏽", "ꟲ", "Ⅳ", "-", "\tt", ".<", "EOT", ">\r\n", "
", "\"​", "𐞁", "\"\r\n\r\n", "'Z", "'ſ", "'S", "#$%<", "META", "_START", ">"]} +{"text": "\r'T½\n٣٤٥٦'\"m\"𐞁", "tokens": 26, "pieces": ["\r", "'T", "½", "\n", "٣٤٥", "٦", "'\"", "m", "\"𐞁", ""]} +{"text": "\r\nⅣ\r'Reع㋿​d🙂m<|endoftext|>12345678dḍ̇\"'sſ'Re½t#$%🙂e#$%　'll", "tokens": 50, "pieces": ["\r\n", "Ⅳ", "\r", "'Re", "ع", "㋿​", "d", "🙂m", "<|", "endoftext", "|>", "123", "456", "78", "dḋ", "̣\"'", "sſ", "'", "Re", "½", "t", "#$%🙂", "e", "#$%", "　", "'ll"]} +{"text": "🙂!!\n<|fim_prefix|>​'M0<|endoftext|>…‍fieſaet😀🏽​aZ 　'll👍🏽Ａ", "tokens": 51, "pieces": ["🙂!!\n", "<|", "fim", "_prefix", "|>​'", "M", "0", "<|", "endoftext", "|>", "…", "‍fieſaet", "😀🏽​", "aZ", " ", "　", "'ll", "👍🏽", "Ａ"]} +{"text": "#$%>. 😀🏽e\u000b'll👍🏽EOT\"٣٤٥٦eꟲ!!>٣٤٥٦12345678<\"ſa'Re'S\tDž12345678'D", "tokens": 57, "pieces": ["#$%>.", " 😀🏽", "e", "\u000b", "'ll", "👍🏽", "EOT", "\"", "٣٤٥", "٦", "eꟲ", "!!>", "٣٤٥", "٦12", "345", "678", "<\"", "ſa", "'Re", "'S", "\tDž", "123", "456", "78", "'D"]} +{"text": "㍿\nع\r'ſ912345678Ⅳå,'M'Ré", "tokens": 21, "pieces": ["㍿\n", "ع", "\r", "'ſ", "912", "345", "678", "Ⅳ", "a", "̊,'", "M", "'Re", "́"]} +{"text": "t'Dm🙂<\n0\r\n\r\n(d-🙂😀🏽-'SＡ😀🏽", "tokens": 26, "pieces": ["t", "'D", "m", "🙂<\n", "0", "\r\n\r\n", "(d", "-🙂😀🏽-'", "SＡ", "😀🏽"]} +{"text": "Ⅳ9små're३ >​\"fi's<|fim_prefix|>!٣٤٥٦d‍İ#$%<|fim_prefix|>", "tokens": 46, "pieces": ["Ⅳ9", "sma", "̊'", "re", "३", " >​\"", "fi", "'s", "<|", "fim", "_prefix", "|>!", "٣٤٥", "٦", "d", "‍İ", "#$%<|", "fim", "_prefix", "|>"]} +{"text": "㋿ Z", "tokens": 5, "pieces": ["㋿", " Z"]} +{"text": "\"'\r\nİ<|fim_prefix|>. \nå.(\u000bfi🙂ḍ̇　 ٣٤٥٦İ𐞁åſ👍🏽", "tokens": 53, "pieces": ["\"'\r\n", "İ", "<|", "fim", "_prefix", "|>.", " \n", "a", "̊.(", "\u000bfi", "🙂ḋ", "̣", "　", " ", "٣٤٥", "٦", "İ𐞁a", "̊ſ", "👍🏽"]} +{"text": "é \r𐞁½🙂३", "tokens": 16, "pieces": ["é", " \r", "𐞁", "½", "🙂<", "META", "_START", ">", "३"]} +{"text": "å👍🏽'Dع　　́𐞁㍿'re👍🏽㍿½漢<|fim_prefix|>　😀🏽'ſå e-㍿३", "tokens": 66, "pieces": ["a", "̊👍🏽'", "D", "ع", "　", "　", "́𐞁", "㍿'", "re", "👍🏽㍿", "½", "漢", "<|", "fim", "_prefix", "|>", "　", "😀🏽'", "ſa", "̊", " ", " e", "-㍿", "३"]} +{"text": "漢٣٤٥٦\r<|endoftext|>👍🏽٣٤٥٦½t…'VE'VE", "tokens": 40, "pieces": ["漢", "٣٤٥", "٦", "\r", "<|", "endoftext", "|>👍🏽", "٣٤٥", "٦½", "t", "…", "'VE", "'VE"]} +{"text": " \nİ-<漢\r\"\u000b
'S\r<|fim_prefix|>👍🏽sععt́㍿sꟲ𐞁\t'9\t😀🏽", "tokens": 60, "pieces": [" \n", "İ", "-<", "漢", "\r", "\"", "\u000b", "
", "'S", "\r", "<|", "fim", "_prefix", "|><", "META", "_START", ">👍🏽", "sععt", "́㍿", "s", "ꟲ𐞁", "\t", "'", "9", "\t", "😀🏽"]} +{"text": " 👍🏽 #$%-​<ꟲİ'ſ㍿EOT><,!me👍🏽Ａ\n.d😀🏽Z'VE'VE>m'sZ", "tokens": 51, "pieces": [" ", " 👍🏽", " #$%-​<", "META", "_START", "><", "ꟲİ", "'ſ", "㍿EOT", "><,!", "me", "👍🏽", "Ａ", "\n", ".d", "😀🏽", "Z", "'VE", "'VE", ">m", "'s", "Z"]} +{"text": "ſ 'Re-e\nDž\n", "tokens": 12, "pieces": ["ſ", " ", "'Re", "-e", "\n", "Dž", "\n", ""]} +{"text": "ḍ̇ſésＡ'é'lls'ſ­'M'll'sa😀🏽s", "tokens": 35, "pieces": ["ḋ", "̣ſe", "́sＡ", "'e", "́'", "lls", "'ſ", "­'", "M", "'ll", "'s", "a", "😀🏽", "s"]} +{"text": "'​\u000b'Mꟲs字👍🏽t\n𐞁\"İZ'ſ½\r ſ'T🙂㋿😀🏽٣٤٥٦<|fim_prefix|>é½ Ａ​漢'VE9🙂é", "tokens": 78, "pieces": ["'​", "\u000b", "'M", "ꟲs字", "👍🏽", "t", "\n", "𐞁", "\"İZ", "'ſ", "½", "\r", " ſ", "'T", "🙂㋿😀🏽", "٣٤٥", "٦", "<|", "fim", "_prefix", "|><", "META", "_START", ">é", "½", " Ａ", "​<", "META", "_START", ">漢", "'VE", "9", "🙂e", "́"]} +{"text": "
sé'M'VEsDžé.(‍\"'ſ", "tokens": 17, "pieces": ["
sé", "'M", "'VE", "sDžé", ".(‍\"'", "ſ"]} +{"text": "'T12345678-\nd!!mé,9'll 'D\" ḍ̇m'ſ'T…<|endoftext|>12345678Ａ字#$%(fi12345678漢", "tokens": 49, "pieces": ["'T", "123", "456", "78", "-\n", "d", "!!", "mé", ",", "9", "'ll", " '", "D", "\"", " ḋ", "̣m", "'ſ", "'T", "…", "<|", "endoftext", "|>", "123", "456", "78", "Ａ字", "#$%(", "fi", "123", "456", "78", "漢"]} +{"text": "<|endoftext|>Dž", "tokens": 9, "pieces": ["<|", "endoftext", "|>", "Dž"]} +{"text": "‍<|endoftext|>><|fim_prefix|>'ſ \nd's'llmⅣ'ſ'VE٣٤٥٦'VE\u000b'VE#$%dⅣꟲ字‍\r", "tokens": 58, "pieces": ["‍<|", "endoftext", "|>><", "EOT", "><|", "fim", "_prefix", "|>'", "ſ", " \n", "d", "'s", "'ll", "m", "Ⅳ", "'ſ", "'VE", "٣٤٥", "٦", "'VE", "\u000b", "'VE", "#$%", "d", "Ⅳ", "ꟲ字", "‍\r"]} +{"text": "'VEå9fi 'll-🙂!!
‍ééＡ\t\rfié'D", "tokens": 36, "pieces": ["'VE", "a", "̊", "9", "fi", " ", " '", "ll", "-🙂!!", "
", "‍e", "́e", "́Ａ", "\t\r", "fie", "́<", "EOT", ">'", "D"]} +{"text": "\rDž#$%0\r\n㍿́12345678å😀🏽'ſ👍🏽", "tokens": 31, "pieces": ["\r", "Dž", "#$%", "0", "\r\n", "㍿́", "123", "456", "78", "a", "̊😀🏽'", "ſ", "👍🏽"]} +{"text": " ́<'Re912345678!'re<|fim_prefix|>0\"EOT​t#$%9Z12345678­'ll\"'re🙂😀🏽　　0<'ſ", "tokens": 50, "pieces": [" ", " ́<'", "Re", "912", "345", "678", "!'", "re", "<|", "fim", "_prefix", "|><", "META", "_START", ">", "0", "\"EOT", "​t", "#$%", "9", "Z", "123", "456", "78", "­'", "ll", "\"'", "re", "🙂😀🏽", "　", "　", "0", "<'", "ſ"]} +{"text": "\r\n12345678#$%​<|endoftext|>>…EOTⅣ'Séå,́\tＡ\nßZ 😀🏽٣٤٥٦'M!#$%", "tokens": 51, "pieces": ["\r\n", "123", "456", "78", "#$%​<|", "endoftext", "|>>", "…EOT", "Ⅳ", "'S", "éa", "̊,́", "\tＡ", "\n", "ßZ", " ", "😀🏽", "٣٤٥", "٦", "'M", "!#$%"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'!e'ſ!漢'M're0\"İ\r\n\r\n\"😀🏽ꟲeåé>s", "tokens": 31, "pieces": ["'!", "e", "'ſ", "!漢", "'M", "'re", "0", "\"İ", "\r\n\r\n", "\"😀🏽", "ꟲea", "̊e", "́>", "s"]} +{"text": "Dž<|endoftext|>!!ſ …t0　å'ſ", "tokens": 24, "pieces": ["Dž", "<|", "endoftext", "|>!!", "ſ", " ", "…t", "0", "　a", "̊'", "ſ"]} +{"text": "😀🏽\r's12345678Dž'漢​…(s'0\r\n\r\n…\r٣٤٥٦İ漢<|endoftext|>'Re\u000b ", "tokens": 46, "pieces": ["😀🏽\r", "'s", "123", "456", "78", "Dž", "'漢", "​", "…", "(s", "'", "0", "\r\n\r\n…\r", "٣٤٥", "٦", "İ漢", "<|", "endoftext", "|>'", "Re", "\u000b "]} +{"text": "'D!!m'Sé​ß \"‍Z'T'ssİ‍Dž<\t>Dž", "tokens": 25, "pieces": ["'D", "!!", "m", "'S", "é", "​ß", " ", " \"‍", "Z", "'T", "'s", "sİ", "‍Dž", "<", "\t", ">Dž"]} +{"text": "ꟲ(9 \n
m👍🏽0Ⅳ\r'T 𐞁!\r\n\r\n>­ſ　\u000b>\t'd'S🙂Ⅳé<\"", "tokens": 44, "pieces": ["ꟲ", "(", "9", " \n", "
m", "👍🏽", "0Ⅳ", "\r", "'T", " ", " 𐞁", "!\r\n\r\n", ">­", "ſ", "　", "\u000b", ">", "\t", "'d", "'S", "🙂", "Ⅳ", "é", "<\""]} +{"text": "𐞁'S#$%字ſ\u000b", "tokens": 15, "pieces": ["𐞁", "'", "S", "#$%", "字ſ", "\u000b"]} +{"text": ".🙂ꟲ<|endoftext|>s𐞁३12345678!ß(½ét!é  t\r\n३.", "tokens": 39, "pieces": [".🙂", "ꟲ", "<|", "endoftext", "|>", "s𐞁", "३12", "345", "678", "!ß", "(", "½", "e", "́t", "!é", " ", " t", "\r\n", "३", "."]} +{"text": " ́s'VE.'S٣٤٥٦३٣٤٥٦-sſ'Ms…9're😀🏽t́Ａfi㋿‍'Re<½'M字0-'re!!EOT'Ⅳå", "tokens": 71, "pieces": [" ́", "s", "'VE", ".'", "S", "٣٤٥", "٦३٣", "٤٥٦", "-sſ", "'M", "s", "…", "9", "'re", "😀🏽", "t", "́Ａfi", "㋿‍'", "Re", "<", "½", "'M", "字", "0", "-'", "re", "!!", "EOT", "'", "Ⅳ", "a", "̊"]} +{"text": "é('D 9ḿ'ſ\r'S", "tokens": 12, "pieces": ["é", "('", "D", " ", "9", "m", "́'", "ſ", "\r", "'S"]} +{"text": " Zſ'\"", "tokens": 4, "pieces": [" Zſ", "'\""]} +{"text": "漢​'Re<|endoftext|>Ⅳfi!!", "tokens": 20, "pieces": ["漢", "​'", "Re", "<|", "endoftext", "|>", "Ⅳ", "fi", "!!"]} +{"text": "'M字㋿éİß!!㋿ḍ̇Ａsꟲfi'T's漢!(ḍ̇Ⅳ😀🏽0åå9é", "tokens": 55, "pieces": ["'M", "字", "㋿", "éİß", "!!㋿", "ḋ", "̣Ａsꟲfi", "'T", "'s", "漢", "!(", "ḋ", "̣", "Ⅳ", "😀🏽", "0", "a", "̊a", "̊", "9", "é"]} +{"text": "\te字ꟲ….ſ", "tokens": 10, "pieces": ["\te字ꟲ", "…", ".ſ"]} +{"text": "éd…­ ㋿٣٤٥٦​'t㋿İ.'Ts0𐞁
🙂'lltع !!fi👍🏽!!́ ", "tokens": 53, "pieces": ["e", "́d", "…", "­", " ㋿", "٣٤٥", "٦", "​'", "t", "㋿İ", ".'", "Ts", "0", "𐞁", "
", "🙂'", "lltع", " ", " !!", "fi", "👍🏽!!́", " "]} +{"text": "🙂'Re🙂'll​漢'D(\r\n\r\nİ!👍🏽's'M\n-😀🏽Z\r\r\n<<12345678İa\r\n\r\n", "tokens": 41, "pieces": ["🙂'", "Re", "🙂'", "ll", "​漢", "'D", "(\r\n\r\n", "İ", "!👍🏽'", "s", "'M", "\n", "-😀🏽", "Z", "\r\r\n", "<<", "123", "456", "78", "İa", "\r\n\r\n"]} +{"text": "'ll字é'M\r\n\r\n\n<|endoftext|>'D'🙂Džéİ<'reå'Reé<|endoftext|>　'T\r'D…
s12345678>< sḍ̇𐞁٣٤٥٦'ll<fi\r\n\r\n", "tokens": 77, "pieces": ["'ll", "字é", "'M", "\r\n\r\n\n", "<|", "endoftext", "|>'", "D", "'🙂", "Dže", "́İ", "<'", "rea", "̊'", "Reé", "<|", "endoftext", "|>", "　", "'T", "\r", "'", "D", "…", "
s", "123", "456", "78", "><", " ", " sḋ", "̣𐞁", "٣٤٥", "٦", "'ll", "<fi", "\r\n\r\n"]} +{"text": "
㍿'DＡ\r\n\r\n'!!>.'ſꟲ\neDž'Re\u000bع-🙂éé,m-\raİ ", "tokens": 40, "pieces": ["
", "㍿'", "DＡ", "\r\n\r\n", "'!!>.'", "ſꟲ", "\n", "eDž", "'Re", "\u000bع", "-🙂", "e", "́e", "́,", "m", "-\r", "aİ", " "]} +{"text": "é0'Re٣٤٥٦", "tokens": 12, "pieces": ["e", "́", "0", "'Re", "٣٤٥", "٦"]} +{"text": "d👍🏽 '\u000b…字'll٣٤٥٦ ſ'D", "tokens": 29, "pieces": ["d", "👍🏽", " ", " '", "\u000b", "…字", "'ll", "٣٤٥", "٦", "", " ſ", "'D"]} +{"text": "🙂­㋿́s​å​'redfiİ'D
\r\n\r\ne0'T,e…'re漢é!!", "tokens": 34, "pieces": ["🙂­㋿́", "s", "​a", "̊​'", "redfiİ", "'D", "
\r\n\r\n", "e", "0", "'T", ",e", "…", "'re", "漢e", "́!!"]} +{"text": "'0! \nⅣ9å㍿𐞁Z12345678é's…㍿!!('T
", "tokens": 35, "pieces": ["'", "0", "!", " \n", "Ⅳ9", "a", "̊㍿", "𐞁Z", "123", "456", "78", "e", "́'", "s", "…", "㍿!!('", "T", "
"]} +{"text": "m🙂'Ré", "tokens": 6, "pieces": ["m", "🙂'", "Re", "́"]} +{"text": "٣٤٥٦ع́ḍ̇,👍🏽(́\r\nm's­'VE
👍🏽\"\r\nſ\u000b sZ٣٤٥٦.'Te𐞁\r'Da,ḍ̇\u000bꟲ!!", "tokens": 72, "pieces": ["٣٤٥", "٦", "ع", "́ḋ", "̣,👍🏽(́\r\n", "m", "'s", "­'", "VE", "
", "👍🏽\"\r\n", "ſ", "\u000b", " sZ", "٣٤٥", "٦", ".'", "Te𐞁", "\r", "'D", "a", ",ḋ", "̣", "\u000bꟲ", "!!"]} +{"text": "12345678'MsＡ𐞁e٣٤٥٦🙂　", "tokens": 27, "pieces": ["123", "456", "78", "'M", "sＡ𐞁e", "٣٤٥", "٦", "🙂<", "EOT", ">", "　"]} +{"text": "fi­Ａ字mⅣ'D𐞁'T👍🏽😀🏽're!#$%٣٤٥٦\nd", "tokens": 41, "pieces": ["fi", "­Ａ字m", "Ⅳ", "'D", "𐞁", "'T", "👍🏽😀🏽'", "re", "!#$%", "٣٤٥", "٦", "\n", "d"]} +{"text": "<'ReＡ\r\ns's'ret㍿́𐞁.12345678-'S🙂>字\r\n\r\ns㍿\r\ne\r\na12345678👍🏽\r\n", "tokens": 63, "pieces": ["<'", "ReＡ", "\r\n", "s", "'", "s", "'re", "t", "㍿́", "𐞁", ".", "123", "456", "78", "-'", "S", "🙂>", "字", "\r\n\r\n", "s", "㍿\r\n", "e", "\r\n", "a", "123", "456", "78", "👍🏽\r\n"]} +{"text": "m'ReⅣ​㋿…'VEعßd'S'Re　漢12345678Dž \té \n's.\rfi \n.", "tokens": 39, "pieces": ["m", "'Re", "Ⅳ", "​㋿", "…", "'VE", "عßd", "'S", "'Re", "　漢", "123", "456", "78", "Dž", " ", "\té", " \n", "'s", ".\r", "fi", " \n", ".<", "EOT", ">"]} +{"text": "​>", "tokens": 2, "pieces": ["​>"]} +{"text": "!!t३#$% \n\r\n\r㍿#$%9ß'ḍ̇­mdé
́.", "tokens": 33, "pieces": ["!!", "t", "३", "#$%", " \n\r\n\r", "㍿#$%", "9", "ß", "'ḋ", "̣­", "mde", "́", "
", "́."]} +{"text": "m t　'reḍ̇​<|endoftext|>0<|endoftext|>👍🏽", "tokens": 36, "pieces": ["m", " ", " t", "　", "'re", "ḋ", "̣​<|", "endoftext", "|>", "0", "<|", "endoftext", "|>👍🏽<", "META", "_START", ">"]} +{"text": " s ,İꟲع''s'S字t漢-😀🏽12345678\r<|fim_prefix|>ꟲ's\ré's'Dfi<|endoftext|>ḍ̇'ſDž'll,字", "tokens": 71, "pieces": [" s", " ,", "İꟲع", "'<", "EOT", ">'", "s", "'S", "字t漢", "-<", "EOT", ">😀🏽", "123", "456", "78", "\r", "<|", "fim", "_prefix", "|>", "ꟲ", "'s", "\r", "e", "́'", "s", "'D", "fi", "<|", "endoftext", "|>", "ḋ", "̣'", "ſDž", "'ll", ",字"]} +{"text": "
m<|endoftext|><|endoftext|>㋿<|endoftext|>­\"(½\u000b\r'S\"(a'Sḍ̇é½'ſå😀🏽a­🙂­ \nſ'Re👍🏽,\u000bd 👍🏽", "tokens": 82, "pieces": ["
m", "<|", "endoftext", "|><|", "endoftext", "|>㋿<|", "endoftext", "|>­<", "EOT", ">\"(", "½", "\u000b\r", "'S", "\"(", "a", "'S", "ḋ", "̣é", "½", "'ſ", "a", "̊😀🏽", "a", "­🙂­", " \n", "ſ", "'Re", "👍🏽,", "\u000bd", " ", "👍🏽"]} +{"text": "d‍३", "tokens": 5, "pieces": ["d", "‍", "३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b,\r٣٤٥٦'ll ‍<-!­.\r", "tokens": 20, "pieces": ["\u000b", ",\r", "٣٤٥", "٦", "'ll", " ", "‍<-!­.\r"]} +{"text": "😀🏽'D​​字", "tokens": 9, "pieces": ["😀🏽'", "D", "​​", "字"]} +{"text": "'S­ \nⅣ>", "tokens": 6, "pieces": ["'S", "­", " \n", "Ⅳ", ">"]} +{"text": "́İ㋿ 字é!'Re😀🏽#$%字 ‍é''VE\tⅣd\rＡ ", "tokens": 32, "pieces": ["­m", "\r", "\"<('", "Re", " s", ">😀🏽#$%", "字", " ", " ‍", "e", "́''", "VE", "\t", "Ⅳ", "d", "\r", "Ａ", " "]} +{"text": "'VE<|endoftext|>\r\n\r\ntaEOT<\u000b🙂​Ａ-́𐞁-'ſ-👍🏽'́d​ſ​e'M<9\r\n-'s'Déet́9", "tokens": 58, "pieces": ["'VE", "<|", "endoftext", "|>\r\n\r\n", "taEOT", "<", "\u000b", "🙂​", "Ａ", "-́<", "META", "_START", ">𐞁", "-'", "ſ", "-👍🏽'́", "d", "​ſ", "​e", "'M", "<", "9", "\r\n", "-'", "s", "'D", "éet", "́", "9"]} +{"text": "३\t 'VE12345678'S \nſDž\r\néd'M'S…s\n　>t<|fim_prefix|>👍🏽<|fim_prefix|>́'VE1234567812345678<|endoftext|>ꟲ३", "tokens": 72, "pieces": ["३", "\t", " ", "'VE", "123", "456", "78", "'S", " \n", "ſDž", "\r\n", "éd", "'M", "'S", "…s", "\n", "　", ">", "t", "<|", "fim", "_prefix", "|>👍🏽<|", "fim", "_prefix", "|>́'", "VE", "123", "456", "781", "234", "567", "8", "<|", "endoftext", "|>", "ꟲ", "३"]} +{"text": "0'M　ḍ̇🙂　", "tokens": 11, "pieces": ["0", "'M", "　ḋ", "̣🙂", "　"]} +{"text": "-<|fim_prefix|>Dž́㋿-'TⅣ🙂३ \n漢're-e'VE'M(\"𐞁字", "tokens": 39, "pieces": ["-<|", "fim", "_prefix", "|>", "Dž", "́㋿-'", "T", "Ⅳ", "🙂", "३", " \n", "漢", "'re", "-e", "'VE", "'M", "(\"", "𐞁字"]} +{"text": ".㋿㍿", "tokens": 7, "pieces": [".㋿㍿"]} +{"text": ",0'll#$%ꟲ\t-…　३ Ａ　ſİ'S'Smfi\n
", "tokens": 30, "pieces": [",", "0", "'ll", "#$%", "ꟲ", "\t", "-", "…", "　", "३", " ", " Ａ", "　ſİ", "'S", "'S", "mfi", "\n
"]} +{"text": "ع́​ 　<字 \n'VÉ#$%\u000b('D å३½'TDžＡ-EOT ꟲßEOTＡ'S", "tokens": 40, "pieces": ["ع", "́​", " ", "　", "<字", " \n", "'VE", "́#$%", "\u000b", "('", "D", " a", "̊", "३½", "'T", "DžＡ", "-EOT", " ꟲßEOTＡ", "'S"]} +{"text": "ع‍'s'S٣٤٥٦Z字'VEİ<'reéé!!EOT12345678…½<'VE\tåaⅣ…éZ'S\n
Ａ-​\r\n\r\n", "tokens": 57, "pieces": ["ع", "‍'", "s", "'S", "٣٤٥", "٦", "Z字", "'VE", "İ", "<'", "re", "ée", "́!!", "EOT", "123", "456", "78", "…", "½", "<'", "VE", "\ta", "̊a", "Ⅳ", "…e", "́Z", "'S", "\n", "
Ａ", "-​\r\n\r\n"]} +{"text": "­　½d\u000bedZ're-m字#$%ḍ̇'re
ḍ̇́漢(>(İ👍🏽<|endoftext|>𐞁<|fim_prefix|>㋿'T字'Re\r\nİ0字", "tokens": 67, "pieces": ["­", "　", "½", "d", "\u000bedZ", "'re", "-m字", "#$%", "ḋ", "̣'", "re", "
ḋ", "̣́", "漢", "(>(", "İ", "👍🏽<|", "endoftext", "|>", "𐞁", "<|", "fim", "_prefix", "|>㋿'", "T字", "'Re", "\r\n", "İ", "0", "字"]} +{"text": "0‍é٣٤٥٦<|endoftext|><'Re३ -\r\n٣٤٥٦́\r\te ,''T9㍿👍🏽字​<|fim_prefix|>", "tokens": 63, "pieces": ["0", "‍é", "٣٤٥", "٦", "<|", "endoftext", "|><", "META", "_START", "><'", "Re", "३", " ", " -\r\n", "٣٤٥", "٦", "́\r", "\te", " ", " ,''", "T", "9", "㍿👍🏽", "字", "​<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|><|fim_prefix|>'S🙂a!३\"'T字\ré0,123456780　\t­''VE㋿<|endoftext|> é", "tokens": 48, "pieces": ["<|", "endoftext", "|><|", "fim", "_prefix", "|>'", "S", "🙂a", "!", "३", "\"'", "T字", "\r", "e", "́", "0", ",", "123", "456", "780", "　", "\t", "­''", "VE", "㋿<|", "endoftext", "|>", " e", "́"]} +{"text": "!'M\u000b<|endoftext|>å\u000b", "tokens": 14, "pieces": ["!'", "M", "\u000b", "<|", "endoftext", "|>", "a", "̊", "\u000b"]} +{"text": ",'ll\nd漢ſ", "tokens": 8, "pieces": [",'", "ll", "\n", "d漢ſ"]} +{"text": "eEOT'Tå\r'sḍ̇'ß\"EOT\n㋿", "tokens": 25, "pieces": ["eEOT", "'T", "a", "̊\r", "'s", "ḋ", "̣'", "ß", "\"<", "META", "_START", ">EOT", "\n", "㋿"]} +{"text": "'sfißḍ̇Ａé\n\u000bＡé'Mm<|fim_prefix|>å<|fim_prefix|>٣٤٥٦", "tokens": 46, "pieces": ["'s", "fißḋ", "̣Ａé", "\n", "\u000bＡe", "́'", "Mm", "<|", "fim", "_prefix", "|>", "a", "̊<|", "fim", "_prefix", "|>", "٣٤٥", "٦"]} +{"text": "'M漢#$%ß..'\r\n0\u000b(́\"\r\nééfiⅣDž　\n\u000bＡḍ̇Ａs", "tokens": 37, "pieces": ["'M", "漢", "#$%", "ß", "..'\r\n", "0", "\u000b", "(́\"\r\n", "ée", "́fi", "Ⅳ", "Dž", "", "　\n", "\u000bＡḋ", "̣Ａs"]} +{"text": "é \nſ", "tokens": 5, "pieces": ["e", "́", " \n", "ſ"]} +{"text": "t　Z㋿,12345678.　EOT'VE
३­ Ⅳ㋿<|endoftext|>ßEOT!EOT'S", "tokens": 42, "pieces": ["t", "　Z", "㋿,", "123", "456", "78", ".", "　EOT", "'VE", "
", "३", "­", " ", " ", "Ⅳ", "㋿<|", "endoftext", "|>", "ßEOT", "!EOT", "'S"]} +{"text": "٣٤٥٦<|endoftext|>😀🏽३😀🏽'DＡ'reß 'Re. \n　9éå're'ſ<'reع!! <‍me👍🏽'llm", "tokens": 69, "pieces": ["٣٤٥", "٦", "<|", "endoftext", "|>😀🏽", "३", "😀🏽'", "DＡ", "'re", "ß", " ", "'Re", ".<", "EOT", ">", " \n", "　", "9", "éa", "̊'", "re", "'ſ", "<'", "reع", "!!", " ", "<‍", "me", "👍🏽'", "llm"]} +{"text": "'M Zſtaé9́ Ⅳ'S٣٤٥٦\r\n\r\n'ſ're'Re…", "tokens": 31, "pieces": ["'M", " Zſtaé", "9", "́", " ", "Ⅳ", "'S", "٣٤٥", "٦", "\r\n\r\n", "'ſ", "'re", "'Re", "…"]} +{"text": "!!漢EOT字<漢", "tokens": 11, "pieces": ["!!", "漢", "EOT字", "<漢"]} +{"text": "\n \n🙂​>‍'s字㍿s
're½EOT'Re'reé. \n", "tokens": 26, "pieces": ["\n \n", "🙂​>‍'", "s字", "㍿s", "
", "'re", "½", "EOT", "'Re", "'re", "e", "́.", " \n"]} +{"text": "\r\n\r\n'D́é-'lld!𐞁\u000bm㋿ع'D", "tokens": 21, "pieces": ["\r\n\r\n", "'D", "́é", "-'", "lld", "!<", "META", "_START", ">𐞁", "\u000bm", "㋿ع", "'D"]} +{"text": "​…३(​9t ", "tokens": 10, "pieces": ["​", "…", "३", "(​", "9", "t", " "]} +{"text": "🙂\r\nd0'VE🙂'M字
㋿!!'!!'D🙂", "tokens": 27, "pieces": ["🙂\r\n", "d", "0", "'VE", "🙂'", "M字", "
", "㋿!!'<", "EOT", ">!!'", "D", "🙂"]} +{"text": "-.m'　\r\nt<|fim_prefix|>👍🏽👍🏽a😀🏽½!!'VEDž\r ", "tokens": 39, "pieces": ["-.", "m", "'", "　\r\n", "t", "<|", "fim", "_prefix", "|>👍🏽👍🏽", "a", "😀🏽", "½", "!!'", "VEDž", "\r "]} +{"text": " \n<'M#$%\n\r\nİéＡ🙂 漢å\"　ع", "tokens": 25, "pieces": [" \n", "<'", "M", "#$%\n\r\n", "İe", "́Ａ", "🙂", " 漢a", "̊\"", "　ع"]} +{"text": "9'Dİ👍🏽㍿🙂m 
e fi<|fim_prefix|>㋿t'M'll \n!Ⅳ'VEİ-'reé½ !fi9\t\n \n<|endoftext|>", "tokens": 60, "pieces": ["9", "'D", "İ", "👍🏽㍿🙂", "m", " ", "
e", " ", " fi", "<|", "fim", "_prefix", "|>㋿", "t", "'M", "'ll", " \n", "!", "Ⅳ", "'VE", "İ", "-'", "reé", "½", " ", " !", "fi", "9", "\t\n \n", "<|", "endoftext", "|>"]} +{"text": "́…́\"Dž ", "tokens": 11, "pieces": ["́", "…", "́\"", "Dž", " "]} +{"text": "\r\n\r\nds,㋿'VE", "tokens": 8, "pieces": ["\r\n\r\n", "ds", ",㋿'", "VE"]} +{"text": "-a<|endoftext|>'s12345678!!s　.　", "tokens": 21, "pieces": ["-a", "<|", "endoftext", "|>'", "s", "123", "456", "78", "!!", "s", "", "　", ".", "　"]} +{"text": "12345678‍'s(t's 𐞁éعꟲ٣٤٥٦>0\r\nİ12345678!!३\",,fie'rea12345678Džꟲé' .EOT", "tokens": 67, "pieces": ["123", "456", "78", "‍'", "s", "(t", "'s", " 𐞁e", "́ع", "ꟲ", "٣٤٥", "٦", ">", "0", "\r\n", "İ", "123", "456", "78", "!!", "३", "\",<", "META", "_START", ">,", "fie", "'", "rea", "123", "456", "78", "Džꟲé", "'", " .", "EOT"]} +{"text": "a'llEOT-ع𐞁're٣٤٥٦㋿dé<|fim_prefix|>at,é字#$%-0👍🏽😀🏽.½ḍ̇<|fim_prefix|>ß<𐞁
㍿ (", "tokens": 76, "pieces": ["a", "'ll", "EOT", "-ع𐞁", "'re", "٣٤٥", "٦", "㋿dé", "<|", "fim", "_prefix", "|>", "at", ",e", "́字", "#$%-", "0", "👍🏽😀🏽.", "½", "ḋ", "̣<|", "fim", "_prefix", "|>", "ß", "<𐞁", "
", "㍿", " ", "("]} +{"text": "a-ßt𐞁e<|endoftext|>éfi字're
\tß　'VE\r\n\r\n \n㋿'TfiEOTdm𐞁\rß́'M", "tokens": 55, "pieces": ["a", "-ßt𐞁e", "<|", "endoftext", "|>", "e", "́fi字", "'", "re", "
", "\tß", "　", "'VE", "\r\n\r\n \n", "㋿'", "TfiEOTdm𐞁", "\r", "ß", "́'", "M"]} +{"text": "ꟲ!!٣٤٥٦\r\ne's't\n字　EOTmⅣ ß漢0dZ.ßsꟲ \" 'refié", "tokens": 48, "pieces": ["ꟲ", "!!", "٣٤٥", "٦", "\r\n", "e", "'s", "'t", "\n", "字", "　EOTm", "Ⅳ", " ß漢", "0", "dZ", ".ßsꟲ", " ", "\"", " ", " '", "re", "fié"]} +{"text": "-😀🏽12345678å.0fiEOT'Tع!㋿　Z>字 'Tꟲ#$%\r\nt'VEع
\u000bßḍ̇'S12345678'S \n\t\u000b٣٤٥٦", "tokens": 69, "pieces": ["-😀🏽", "123", "456", "78", "a", "̊.", "0", "fiEOT", "'T", "ع", "!㋿", "　Z", ">字", " ", " '", "Tꟲ", "#$%\r\n", "t", "'VE", "ع", "
", "\u000bßḋ", "̣<", "META", "_START", ">'", "S", "123", "456", "78", "'S", " \n", "\t", "\u000b", "٣٤٥", "٦"]} +{"text": "\nİ\"\n'T.e9'reé'ſZé👍🏽,12345678  👍🏽'll漢　Ⅳ'VEta's'VE \n's'Tعꟲ", "tokens": 52, "pieces": ["\n", "İ", "\"\n", "'T", ".e", "9", "'re", "e", "́'", "ſZe", "́👍🏽,", "123", "456", "78", " ", " ", "👍🏽'", "ll漢", "　", "Ⅳ", "'VE", "ta", "'s", "'VE", " \n", "'s", "'T", "عꟲ"]} +{"text": "'VE'sDž½fi㍿漢ſ<\rDž\r\n<\u000b\"\u000b漢Ａḍ̇­ad🙂", "tokens": 37, "pieces": ["'VE", "'s", "Dž", "½", "fi", "㍿漢ſ", "<\r", "Dž", "\r\n", "<", "\u000b", "\"", "\u000b漢Ａḋ", "̣­", "ad", "🙂"]} +{"text": "\u000b!\ra漢 EOT.‍!\u000b​​a👍🏽 \r\n'́", "tokens": 28, "pieces": ["\u000b", "!\r", "a漢", " EOT", ".‍!", "\u000b", "​​", "a", "👍🏽", " \r\n", "'́"]} +{"text": " Z!!½-,#$%fi'MDž 漢ꟲ>'reꟲ0t'reéḍ̇12345678(12345678ſ३#$%'ſdＡet12345678t३", "tokens": 64, "pieces": [" Z", "!!", "½", "-,#$%", "fi", "'M", "Dž", " ", " 漢", "ꟲ", ">'", "reꟲ", "0", "t", "'re", "éḋ", "̣", "123", "456", "78", "(", "123", "456", "78", "ſ", "३", "#$%'", "ſdＡ", "et", "123", "456", "78", "t", "३"]} +{"text": "㋿\t\r\n12345678é\u000b!!'ſ'MＡ,\r\n\r\n\"'ll'EOT٣٤٥٦'MZ", "tokens": 34, "pieces": ["㋿", "\t\r\n", "123", "456", "78", "é", "\u000b", "!!'", "ſ", "'M", "Ａ", ",\r\n\r\n", "\"<", "META", "_START", ">'", "ll", "'EOT", "٣٤٥", "٦", "'M", "Z"]} +{"text": "'Re
‍\t\r\n\r\nİ ㍿d \nmİ#$%ḍ̇字#$%eZ(12345678\r\n\r\n t \n'ReＡ‍\r\n's#$%ⅣＡ<|endoftext|>'VE'VE", "tokens": 62, "pieces": ["'Re", "
", "‍", "\t\r\n\r\n", "İ", " ", " ㍿", "d", " \n", "mİ", "#$%", "ḋ", "̣字", "#$%", "eZ", "(", "123", "456", "78", "\r\n\r\n", " ", " t", " \n", "'Re", "Ａ", "‍\r\n", "'s", "#$%", "Ⅳ", "Ａ", "<|", "endoftext", "|>'", "VE", "'VE"]} +{"text": "'lld㍿
'red٣٤٥٦👍🏽aDž'reꟲ<<|fim_prefix|>\r\n'Re'VEa\"<|fim_prefix|>é''🙂EOT'0're'ſ'T", "tokens": 64, "pieces": ["'ll", "d", "㍿", "
", "'re", "d", "٣٤٥", "٦", "👍🏽", "aDž", "'re", "ꟲ", "<<", "EOT", "><|", "fim", "_prefix", "|>\r\n", "'Re", "'VE", "a", "\"<|", "fim", "_prefix", "|>", "é", "''🙂", "EOT", "'", "0", "'re", "'ſ", "'T"]} +{"text": " EOT　漢\"<\"٣٤٥٦ꟲ12345678!\u000b👍🏽'M!Ａ'VE", "tokens": 37, "pieces": [" ", " EOT", "　漢", "\"<\"", "٣٤٥", "٦", "ꟲ", "123", "456", "78", "!", "\u000b", "👍🏽'", "M", "!Ａ", "'VE"]} +{"text": "<|fim_prefix|>　'reſZ‍🙂\r\n\r\nİfi\r\n'S👍🏽ſİ…'S ", "tokens": 38, "pieces": ["<|", "fim", "_prefix", "|>", "　", "'re", "ſZ", "‍🙂\r\n\r\n", "İfi", "\r\n", "'S", "👍🏽", "ſİ", "…", "'", "S", " "]} +{"text": "de३　", "tokens": 8, "pieces": ["de", "", "३", "　"]} +{"text": "\n𐞁㍿Dž ㋿a'll!!ſ字…Ⅳ'sé12345678", "tokens": 35, "pieces": ["\n", "𐞁", "㍿Dž", " ", " ㋿", "a", "'ll", "!!", "ſ字", "…", "Ⅳ", "'s", "e", "́<", "EOT", ">", "123", "456", "78"]} +{"text": "<|fim_prefix|>e😀🏽tZ​\t#$%عå३　㍿ß😀🏽İ9", "tokens": 41, "pieces": ["<|", "fim", "_prefix", "|>", "e", "😀🏽", "tZ", "​", "\t", "#$%", "عa", "̊", "३", "　", "㍿<", "META", "_START", ">ß", "😀🏽", "İ", "9"]} +{"text": "#$%!!½ ٣٤٥٦'D'VEa
\t ٣٤٥٦('VE!　\"​.", "tokens": 37, "pieces": ["#$%!!", "½", " ", " ", "٣٤٥", "٦", "'D", "'VE", "a", "
\t", " ", "٣٤٥", "٦", "('", "VE", "!", "　", "\"​."]} +{"text": "9<'re'M'T😀🏽('ſ'M.'VE\r\n\r\n👍🏽éa\u000bⅣعé<|fim_prefix|>>'D\tZDž　 ", "tokens": 49, "pieces": ["9", "<'", "re", "'M", "'T", "😀🏽('", "ſ", "'M", ".'", "VE", "\r\n\r\n", "👍🏽", "e", "́a", "", "\u000b", "Ⅳ", "عe", "́<|", "fim", "_prefix", "|>>'", "D", "\tZDž", "　 "]} +{"text": "'‍ſ#$%…३\r\n\r\ne𐞁'T!٣٤٥٦-sé é​'s𐞁d\r\n\r\nſå<|endoftext|>字 \n👍🏽DžⅣ½\",Z👍🏽", "tokens": 78, "pieces": ["'‍", "ſ", "#$%", "…", "३", "\r\n\r\n", "e𐞁", "'T", "!", "٣٤٥", "٦", "-se", "́", " e", "́<", "META", "_START", ">​'", "s𐞁d", "\r\n\r\n", "ſa", "̊<|", "endoftext", "|>", "字", " \n", "👍🏽", "Dž", "Ⅳ½", "\",", "Z", "👍🏽"]} +{"text": "9Z…🙂३ß \n(<|endoftext|>\t9'D٣٤٥٦EOT ꟲ.é
å'll漢İＡ", "tokens": 51, "pieces": ["9", "Z", "…", "🙂", "३", "ß", " \n", "(<|", "endoftext", "|>", "\t", "", "9", "'D", "٣٤٥", "٦", "EOT", " ", " ꟲ", ".e", "́", "
a", "̊'", "ll漢İＡ"]} +{"text": "
-\u000b'Tß'M　𐞁😀🏽\r'll३'Ⅳ…9Zm9Ａe'Tİ漢<éßå", "tokens": 50, "pieces": ["
", "-<", "EOT", ">", "\u000b", "'T", "ß", "'M", "　𐞁", "😀🏽\r", "'ll", "३", "'", "Ⅳ", "…", "9", "Zm", "9", "Ａe", "'T", "İ漢", "<éßa", "̊"]} +{"text": "\r\n'é\r\nع㋿'reEOT a'll \r0'D\u000bİ#$%🙂\r\n>㍿>å\r's😀🏽́ 'S Ａ ", "tokens": 51, "pieces": ["\r\n", "'e", "́\r\n", "ع", "㋿'", "reEOT", " a", "'ll", " \r", "0", "'D", "\u000bİ", "#$%🙂\r\n", ">㍿>", "a", "̊\r", "'s", "😀🏽<", "META", "_START", ">́", " ", "'S", " Ａ", " "]} +{"text": ".Ａ<|fim_prefix|>'ſfiDž­'eée٣٤٥٦'VE<|endoftext|>👍🏽mİ!!
", "tokens": 51, "pieces": [".Ａ", "<|", "fim", "_prefix", "|>'", "ſfiDž", "­'", "e", "ée", "٣٤٥", "٦", "'VE", "<|", "endoftext", "|>👍🏽", "mİ", "!!", "
"]} +{"text": "!漢‍<|fim_prefix|>Ａ'D'ſ 𐞁- .  a", "tokens": 29, "pieces": ["!漢", "‍<|", "fim", "_prefix", "|>", "Ａ", "'D", "'ſ", " ", " 𐞁", "-", " .", " ", " a"]} +{"text": "İé<|fim_prefix|>'re'T½ꟲ!!t Z　​ꟲEOT\r\n(fi's#$%sDž́ ㋿…'T!m\u000b", "tokens": 50, "pieces": ["İe", "́<|", "fim", "_prefix", "|>'", "re", "'T", "½", "ꟲ", "!!", "t", " Z", "　", "​ꟲEOT", "\r\n", "(fi", "'s", "#$%", "sDž", "́", " ", "㋿", "…", "'T", "!m", "\u000b"]} +{"text": "'re漢<|endoftext|>½ (😀🏽a#$%0<|fim_prefix|>å𐞁<|endoftext|>'Tſ\r\n\r\n‍Ⅳ\né<٣٤٥٦('ſ'T 9​漢३é,\u000b12345678İ\u000b", "tokens": 85, "pieces": ["'re", "漢", "<|", "endoftext", "|>", "½", " ", "(😀🏽", "a", "#$%", "0", "<|", "fim", "_prefix", "|>", "a", "̊𐞁", "<|", "endoftext", "|>'", "Tſ", "\r\n\r\n", "‍", "Ⅳ", "\n", "é", "<", "٣٤٥", "٦", "('", "ſ", "'T", " ", " ", "9", "​漢", "३", "é", ",", "\u000b", "123", "456", "78", "İ", "", "\u000b"]} +{"text": "#$%
(㍿Z!!s👍🏽\"EOT<\t𐞁,'T!'DⅣ'Re👍🏽३ḍ̇", "tokens": 46, "pieces": ["#$%", "
", "(㍿", "Z", "!!", "s", "👍🏽\"", "EOT", "<", "\t𐞁", ",'", "T", "!'", "D", "Ⅳ", "'Re", "👍🏽", "३", "ḋ", "̣"]} +{"text": "mm 're𐞁då'ReⅣ🙂 　>\r\n'ſ", "tokens": 23, "pieces": ["mm", " ", " '", "re𐞁da", "̊'", "Re", "Ⅳ", "🙂", " ", "　", ">\r\n", "'ſ"]} +{"text": "٣٤٥٦'ll\r\nZ字9½́#$%!t", "tokens": 18, "pieces": ["٣٤٥", "٦", "'ll", "\r\n", "Z字", "9½", "́#$%!", "t"]} +{"text": "fi'Dt.'llḍ̇'så.​\u000b'D㍿ḍ̇'ll'T　>Ⅳ \n#$%'D!'ll\r", "tokens": 44, "pieces": ["fi", "'D", "t", ".<", "META", "_START", ">'", "llḋ", "̣'", "sa", "̊.​", "\u000b", "'D", "㍿ḋ", "̣'", "ll", "'T", "　", ">", "Ⅳ", " \n", "#$%'", "D", "!'", "ll", "\r"]} +{"text": "'Reİfi\t'VE12345678字's'Mꟲſé(>'S ٣٤٥٦,😀🏽ſa\r\nmDža…ⅣＡ'\r\n", "tokens": 53, "pieces": ["'Re", "İfi", "\t", "'VE", "123", "456", "78", "字", "'s", "'M", "ꟲſe", "́(>'", "S", " ", "٣٤٥", "٦", ",😀🏽", "ſa", "\r\n", "mDža", "…", "Ⅳ", "Ａ", "'\r\n"]} +{"text": "!!EOT½​\r\n\r\n\t½'ſ'll's½
\"d½'\r\n\r\n٣٤٥٦t½ ḍ̇(<|fim_prefix|>.ꟲm'D​sm!😀🏽9 ", "tokens": 57, "pieces": ["!!", "EOT", "½", "​\r\n\r\n", "\t", "½", "'ſ", "'ll", "'s", "½", "
", "\"d", "½", "'\r\n\r\n", "٣٤٥", "٦", "t", "½", " ḋ", "̣(<|", "fim", "_prefix", "|>.", "ꟲm", "'D", "​sm", "!😀🏽", "9", " "]} +{"text": "🙂fiع#$%\ŕ", "tokens": 9, "pieces": ["🙂fiع", "#$%\r", "́"]} +{"text": "EOTİ\n#$%٣٤٥٦fi9́'S'VEZa12345678字s0>'re'Re<<|fim_prefix|>́", "tokens": 47, "pieces": ["EOTİ", "\n", "#$%<", "EOT", ">", "٣٤٥", "٦", "fi", "9", "́'", "S", "'VE", "Za", "123", "456", "78", "字s", "0", ">'", "re", "'Re", "<<|", "fim", "_prefix", "|>́"]} +{"text": "…漢'ſeZꟲ'VE­'Sa　0\u000b\r", "tokens": 21, "pieces": ["…漢", "'ſ", "eZꟲ", "'VE", "­'", "Sa", "　", "0", "\u000b\r"]} +{"text": "­ \nm!", "tokens": 4, "pieces": ["­", " \n", "m", "!"]} +{"text": "'DＡm½é,'ſ,-e'D😀🏽'S,'T0é<
　㍿e㍿'D'Re٣٤٥٦#$%\"𐞁'DＡa", "tokens": 60, "pieces": ["'D", "Ａm", "½", "é", ",'", "ſ", ",-", "e", "'D", "😀🏽'", "S", ",'", "T", "0", "e", "́<", "
", "　", "㍿e", "㍿'", "D", "'Re", "٣٤٥", "٦", "#$%\"", "𐞁", "'D", "Ａa"]} +{"text": "\r𐞁<|endoftext|>9dd!'T\r\n\r\n'sDž#$%9ß́\u000b<|fim_prefix|>\u000bḍ̇‍ ", "tokens": 42, "pieces": ["\r", "𐞁", "<|", "endoftext", "|>", "9", "dd", "!'", "T", "\r\n\r\n", "'s", "Dž", "#$%", "9", "ß", "́", "\u000b", "<|", "fim", "_prefix", "|>", "\u000bḋ", "̣‍", " "]} +{"text": "-𐞁!!ḍ̇‍.　0ꟲ '字a‍fiEOT\"ſ \ns'DZ're ٣٤٥٦'s", "tokens": 47, "pieces": ["-𐞁", "!!", "ḋ", "̣‍.", "　", "0", "ꟲ", " ", "'字a", "‍fiEOT", "\"ſ", " \n", "s", "'D", "Z", "'re", " ", "٣٤٥", "٦", "'s"]} +{"text": "'VE!!!😀🏽\tEOT'M-t>
\re0\"t\r\n\r\n'SEOT́
Dž字‍(", "tokens": 36, "pieces": ["'VE", "!!!😀🏽", "\tEOT", "'M", "-t", ">", "
\r", "e", "0", "\"t", "\r\n\r\n", "'S", "EOT", "́", "
Dž字", "‍<", "EOT", ">("]} +{"text": "EOT'ſ​'M", "tokens": 8, "pieces": ["EOT", "'ſ", "​'", "M"]} +{"text": "𐞁👍🏽👍🏽Ａß0½'MDže!!
EOT'llع<|fim_prefix|>é", "tokens": 41, "pieces": ["𐞁", "👍🏽👍🏽", "Ａß", "0½", "'M", "Dže", "!!", "
EOT", "'ll", "ع", "<|", "fim", "_prefix", "|>", "e", "́"]} +{"text": "#$%ع'TEOTs'S#$%ßd­é​ 'Re-#$%́", "tokens": 30, "pieces": ["#$%", "ع", "'T", "EOTs", "'S", "#$%", "ßd", "­e", "́​", " ", " <", "META", "_START", ">'", "Re", "-#$%́<", "EOT", ">"]} +{"text": "ꟲ字​\r'll!!", "tokens": 8, "pieces": ["ꟲ字", "​\r", "'ll", "!!"]} +{"text": "……a㍿m
𐞁𐞁,0fiſ‍d .>👍🏽 -é,\u000b́Dž字👍🏽㍿\n\u000b", "tokens": 60, "pieces": ["…", "…a", "㍿m", "
𐞁𐞁", ",", "0", "fiſ", "‍d", " ", " .>👍🏽", " ", "-", "é", ",", "\u000b", "́Dž字", "👍🏽㍿\n", "\u000b"]} +{"text": "d½Džtaع​s#$%t'reع'VE👍🏽're", "tokens": 26, "pieces": ["d", "½", "Džt", "aع", "​s", "#$%", "t", "'re", "ع", "'VE", "👍🏽'", "re"]} +{"text": "ꟲaſ \n'D-", "tokens": 9, "pieces": ["ꟲaſ", " \n", "'D", "-"]} +{"text": "😀🏽(३'ſZ\r\nſ  t \n<|endoftext|> Ⅳꟲḍ̇İ12345678e<|fim_prefix|>٣٤٥٦\n'S t's　İd👍🏽d", "tokens": 75, "pieces": ["😀🏽(", "३", "'ſ", "Z", "\r\n", "ſ", " ", " t", " \n", "<|", "endoftext", "|>", " ", " ", "Ⅳ", "ꟲḋ", "̣İ", "123", "456", "78", "e", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "\n", "'S", " t", "'s", "　İd", "👍🏽", "d"]} +{"text": ",…'llḍ̇s\"<\"३a(#$%\"s👍🏽字é'D!!
", "tokens": 33, "pieces": [",", "…", "'ll", "ḋ", "̣s", "\"<<", "META", "_START", ">\"", "३", "a", "(#$%\"", "s", "👍🏽", "字é", "'D", "!!", "
"]} +{"text": "👍🏽 ㍿½'VE#$%­'Re३　ſ'D­t'M३👍🏽Ａ<,'lla<|fim_prefix|>ＡDž🙂sd", "tokens": 53, "pieces": ["👍🏽", " ", "㍿", "½", "'VE", "#$%­'", "Re", "३", "　ſ", "'D", "­t", "'M", "३", "👍🏽", "Ａ", "<,'", "lla", "<|", "fim", "_prefix", "|>", "ＡDž", "🙂sd"]} +{"text": "Ａ𐞁\r\n\r\n#$%0㋿EOT-عſ'MmⅣd#$%'D12345678d… ", "tokens": 34, "pieces": ["Ａ𐞁", "\r\n\r\n", "#$%", "0", "㋿EOT", "-عſ", "'M", "m", "Ⅳ", "d", "#$%'", "D", "123", "456", "78", "d", "… "]} +{"text": "<|endoftext|>…'㋿<|endoftext|>t> fim's㋿(…e👍🏽<|fim_prefix|>t ,ꟲ", "tokens": 54, "pieces": ["<|", "endoftext", "|>", "…", "'㋿<|", "endoftext", "|>", "t", ">", " ", " fim", "'s", "㋿(", "…e", "👍🏽<|", "fim", "_prefix", "|>", "t", " ,", "ꟲ"]} +{"text": "
字́ſ'M'D<|endoftext|> (́
­,9#$%\"'s'Tſḍ̇>9‍d \n\"३fi!!å'S!td<|fim_prefix|>", "tokens": 60, "pieces": ["
字", "́ſ", "'M", "'D", "<|", "endoftext", "|>", " (́", "
", "­,", "9", "#$%\"'", "s", "'T", "ſḋ", "̣>", "9", "‍d", " \n", "\"", "३", "fi", "!!", "a", "̊'", "S", "!td", "<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|>12345678sfi(a　d", "tokens": 16, "pieces": ["<|", "endoftext", "|>", "123", "456", "78", "sfi", "(a", "　d"]} +{"text": "Džß'll😀🏽EOT٣٤٥٦ſZ0<|fim_prefix|>-", "tokens": 34, "pieces": ["Džß", "'ll", "😀🏽", "EOT", "٣٤٥", "٦", "ſZ", "0", "<|", "fim", "_prefix", "|>-"]} +{"text": "字'VE'T'M'll(\t<'ſſ ‍!!!!d#$%!!", "tokens": 24, "pieces": ["字", "'VE", "'T", "'M", "'ll", "(", "\t", "<'", "ſſ", " ", " ‍!!!!", "d", "#$%!!"]} +{"text": "<|fim_prefix|>!!…́'㍿a­\"㋿\r\n -<|endoftext|>>eḍ̇🙂字'Re->
.s", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>!!", "…", "́'㍿", "a", "­\"㋿\r\n", " -<|", "endoftext", "|>>", "eḋ", "̣🙂", "字", "'Re", "->", "
", ".s"]} +{"text": "(9漢'S're\r\ń's'ſ\r\n\r\n
('Dd<​12345678'D٣٤٥٦½#$%t­
ſ'S\r\nꟲ", "tokens": 51, "pieces": ["(", "9", "漢", "'S", "'re", "\r\n", "́'", "s", "'ſ", "\r\n\r\n", "
", "('", "Dd", "<​", "123", "456", "78", "'D", "٣٤٥", "٦½", "#$%", "t", "­", "
ſ", "'S", "\r\n", "ꟲ"]} +{"text": "🙂m(('s.½  ſ.'Tfiåa‍-9\t\"#$%#$%​ꟲ-'Re \n'Tꟲ\n㍿字-‍#$%", "tokens": 50, "pieces": ["🙂m", "(('", "s", ".", "½", " ", " ſ", ".'", "Tfia", "̊a", "‍-", "9", "\t", "\"#$%#$%​", "ꟲ", "-'", "Re", " \n", "'T", "ꟲ", "\n", "㍿字", "-‍#$%"]} +{"text": "(\r<|fim_prefix|>½'re'll​aß", "tokens": 15, "pieces": ["(\r", "<|", "fim", "_prefix", "|>", "½", "'re", "'ll", "​aß"]} +{"text": "\r\n٣٤٥٦d,\r\n\r\n", "tokens": 11, "pieces": ["\r\n", "٣٤٥", "٦", "d", ",\r\n\r\n"]} +{"text": "é\t­ \n0(12345678👍🏽٣٤٥٦　ḍ̇", "tokens": 29, "pieces": ["é", "\t", "­", " \n", "0", "(", "123", "456", "78", "👍🏽", "٣٤٥", "٦", "　ḋ", "̣"]} +{"text": "㍿ßém'Tå't🙂<|endoftext|>!　'M३.09Dž'ReZ!!👍🏽\u000bfid(e​漢 
'ſ12345678éＡ'T", "tokens": 59, "pieces": ["㍿ßém", "'T", "a", "̊'", "t", "🙂<|", "endoftext", "|>!", "　", "'M", "३", ".", "09", "Dž", "'Re", "Z", "!!👍🏽", "\u000bfid", "(e", "​漢", " ", "
", "'ſ", "123", "456", "78", "éＡ", "'T"]} +{"text": "(½İ<|fim_prefix|>!!a'　", "tokens": 14, "pieces": ["(", "½", "İ", "<|", "fim", "_prefix", "|>!!", "a", "'", "　"]} +{"text": "\r\n½ſ㋿ ㋿👍🏽dſ½㋿", "tokens": 24, "pieces": ["\r\n", "½", "ſ", "㋿", " ", "㋿👍🏽", "dſ", "½", "㋿"]} +{"text": "å½EOT's漢EOT‍ Dž' \n9é \n😀🏽é\rع>ḍ̇'VEß'Dmsİḍ̇", "tokens": 51, "pieces": ["a", "̊", "½", "EOT", "'s", "漢EOT", "‍", " Dž", "'", " \n", "9", "e", "́", " \n", "😀🏽", "é", "\r", "ع", ">ḋ", "̣'", "VEß", "'", "Dmsİḋ", "̣"]} +{"text": "Ⅳ12345678ḍ̇字,㍿0عs\u000b0fi'Ré.eßＡDž​漢ſ", "tokens": 36, "pieces": ["Ⅳ12", "345", "678", "ḋ", "̣字", ",㍿", "0", "عs", "\u000b", "0", "fi", "'Re", "́.", "eßＡDž", "​漢ſ"]} +{"text": "é'Maa'lls😀🏽
,Dž \nع,", "tokens": 20, "pieces": ["e", "́'", "Maa", "'ll", "s", "😀🏽", "
", ",Dž", " \n", "ع", ","]} +{"text": "'sＡé… \ne'T½12345678s", "tokens": 15, "pieces": ["'s", "Ａe", "́", "… \n", "e", "'T", "½12", "345", "678", "s"]} +{"text": "12345678<|fim_prefix|>😀🏽t字é'ع㋿👍🏽é\t fid'VE\r\" s\r\nß", "tokens": 45, "pieces": ["123", "456", "78", "<|", "fim", "_prefix", "|>😀🏽", "t字e", "́'", "ع", "㋿👍🏽", "e", "́", "\t", " fid", "'VE", "\r", "\"", " ", " s", "\r\n", "ß"]} +{"text": "ꟲDž,'VE", "tokens": 7, "pieces": ["ꟲDž", ",'", "VE"]} +{"text": "Dž<|endoftext|>-'𐞁'll
𐞁å\u000b<😀🏽#$%\t Ａ\u000bé'S'Ree12345678𐞁‍éＡ #$%ß'VE", "tokens": 65, "pieces": ["Dž", "<|", "endoftext", "|>-'", "𐞁", "'ll", "
𐞁a", "̊", "\u000b", "<😀🏽#$%", "\t", " Ａ", "\u000be", "́'", "S", "'Re", "e", "123", "456", "78", "𐞁", "‍e", "́Ａ", " <", "EOT", ">#$%", "ß", "'VE"]} +{"text": "\té", "tokens": 2, "pieces": ["\té"]} +{"text": "　ḍ̇…㋿ꟲ३,𐞁İ\n-'M字mfi𐞁😀🏽ع👍🏽́!!!!d\"\u000b'VE ३ (　", "tokens": 58, "pieces": ["　ḋ", "̣", "…", "㋿ꟲ", "३", ",𐞁İ", "\n", "-'", "M字mfi𐞁", "😀🏽", "ع", "👍🏽́!!!!", "d", "\"", "\u000b", "'VE", " ", "३", " ", " (", "　"]} +{"text": "\r\n0'M\r😀🏽e½
\r\n\r\n\n(.#$% \nß'llعⅣéſⅣs　㍿\r\n\r\n#$%\"EOT'S　👍🏽🙂'VE\n'ſ", "tokens": 63, "pieces": ["\r\n", "0", "'", "M", "\r", "😀🏽", "e", "½", "
\r\n\r\n\n", "(.#$%", " \n", "ß", "'ll", "ع", "Ⅳ", "éſ", "", "Ⅳ", "s", "　", "㍿\r\n\r\n", "#$%\"", "EOT", "'S", "　", "👍🏽🙂'", "VE", "\n", "'ſ"]} +{"text": "𐞁㋿!\n'DＡ\n\tßé‍Ⅳ>İ.<|fim_prefix|> ", "tokens": 33, "pieces": ["𐞁", "㋿!\n", "'D", "Ａ", "\n", "\tßé", "‍", "Ⅳ", ">İ", ".<|", "fim", "_prefix", "|>", " "]} +{"text": "́㋿ḍ̇!字'ſ EOT​fiⅣ\t漢𐞁'sfi'T'M ‍ \n㍿é'Re-0", "tokens": 49, "pieces": ["́㋿", "ḋ", "̣!", "字", "'ſ", " EOT", "​fi", "Ⅳ", "\t漢𐞁", "'s", "fi", "'T", "'M", " ", " ‍", " \n", "㍿é", "'Re", "-<", "EOT", ">", "0"]} +{"text": "'Re‍sEOT#$%字'ſ,😀🏽d漢 \nＡ \n!!'M're…!!'T ㍿ İ(a 𐞁s\rß٣٤٥٦\t'VE'Mİa𐞁𐞁", "tokens": 71, "pieces": ["'Re", "‍sEOT", "#$%", "字", "'ſ", ",😀🏽", "d漢", " \n", "Ａ", " \n", "!!'", "M", "'re", "…", "!!'", "T", " ㍿", " İ", "(a", " 𐞁s", "\r", "ß", "٣٤٥", "٦", "\t", "'VE", "'M", "İa𐞁𐞁"]} +{"text": "𐞁#$%dDž漢", "tokens": 14, "pieces": ["𐞁", "#$%<", "EOT", ">dDž漢"]} +{"text": "<|fim_prefix|>ḍ̇'Séſ\n're\r\n\r\n", "tokens": 20, "pieces": ["<|", "fim", "_prefix", "|>", "ḋ", "̣'", "Se", "́ſ", "\n", "'re", "\r\n\r\n"]} +{"text": "ſ٣٤٥٦Ａ'T'll'ſ٣٤٥٦​ İ漢t", "tokens": 35, "pieces": ["ſ", "", "٣٤٥", "٦", "Ａ", "'T", "'ll", "'ſ", "٣٤٥", "٦", "​", " İ漢t"]} +{"text": "­\r'M'VE\u000b0𐞁'D\r\n\r\né😀🏽'…\u000b<Dž!!…
\"　d👍🏽ßm'ſ9½-", "tokens": 58, "pieces": ["­\r", "'M", "'VE", "\u000b", "0", "𐞁", "'D", "\r\n\r\n", "e", "́😀🏽'", "…", "\u000b", "<Dž", "!!", "…", "
", "\"<", "e", "́<|", "fim", "_prefix", "|>", "　d", "👍🏽", "ßm", "'ſ", "9½", "-"]} +{"text": "9", "tokens": 1, "pieces": ["9"]} +{"text": "(\"字Džé(ꟲ-", "tokens": 14, "pieces": ["(\"", "字Dže", "́(<", "EOT", ">ꟲ", "-"]} +{"text": "\r\n­m½ \r\n \n#$%ꟲ><|endoftext|>　0d.'s漢😀🏽é", "tokens": 32, "pieces": ["\r\n", "­m", "½", " \r\n \n", "#$%", "ꟲ", "><|", "endoftext", "|>", "　", "0", "d", ".'", "s漢", "😀🏽", "e", "́"]} +{"text": "ſEOT\r\n9\r \na", "tokens": 9, "pieces": ["ſEOT", "\r\n", "9", "\r \n", "a"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "👍🏽\n'D", "tokens": 8, "pieces": ["👍🏽\n", "'D"]} +{"text": "'S'S\n
-Z
३.½'ſ'VE½ İⅣ\n\t𐞁\">\n Ａ'VEs㋿\r!!'S🙂!'DⅣ", "tokens": 50, "pieces": ["'S", "'S", "\n", "
", "-Z", "
", "", "३", ".", "½", "'ſ", "'VE", "½", " İ", "Ⅳ", "\n", "\t𐞁", "\">\n", " ", " Ａ", "'VE", "s", "㋿\r", "!!'", "S", "🙂!'", "D", "Ⅳ"]} +{"text": "
\r\n\r\n㍿\r\n\r\n\r\n#$%ꟲ", "tokens": 12, "pieces": ["
\r\n\r\n", "㍿\r\n\r\n\r\n", "#$%", "ꟲ"]} +{"text": "\t'Re#$%ꟲ👍🏽　'ꟲ'llé\té,'ſß", "tokens": 31, "pieces": ["\t", "'Re", "#$%", "ꟲ", "👍🏽<", "EOT", ">", "　", "'ꟲ", "'ll", "e", "́", "\té", ",'", "ſß"]} +{"text": "<
\r-('VEDž\r'S\ré'Ree\r\n😀🏽's\né\r\nm\t", "tokens": 31, "pieces": ["<", "
\r", "-('", "VEDž", "\r", "'S", "\r", "é", "'Re", "e", "\r\n", "😀🏽'", "s", "\n", "e", "́\r\n", "m", "\t"]} +{"text": "<|fim_prefix|>𐞁\r\n'll\t's9fi12345678'12345678…-0\n > \n<|fim_prefix|>-é
😀🏽's'S!!🙂\n㋿‍\n0ḍ̇", "tokens": 67, "pieces": ["<|", "fim", "_prefix", "|>", "𐞁", "\r\n", "'ll", "\t", "'s", "9", "fi", "123", "456", "78", "'", "123", "456", "78", "…", "-", "0", "\n", " ", ">", " \n", "<|", "fim", "_prefix", "|>-", "é", "
", "😀🏽'", "s", "'S", "!!🙂\n", "㋿‍\n", "0", "ḋ", "̣"]} +{"text": "𐞁🙂𐞁ꟲⅣDž½a0,㍿éḍ̇½'s٣٤٥٦٣٤٥٦", "tokens": 54, "pieces": ["𐞁", "🙂𐞁ꟲ", "Ⅳ", "Dž", "½", "a", "0", ",<", "EOT", "><", "EOT", ">㍿", "éḋ", "̣", "½", "'s", "٣٤٥", "٦٣٤", "٥٦"]} +{"text": "''re​> \n\r\n\r\n>字s'M­fi㋿'s\r\n\r\n\r\nſ0.t", "tokens": 23, "pieces": ["''", "re", "​>", " \n\r\n\r\n", ">字s", "'M", "­fi", "㋿'", "s", "\r\n\r\n\r\n", "ſ", "0", ".t"]} +{"text": "9 fiZꟲＡ👍🏽'M­ع<|endoftext|>🙂#$%­ḍ̇t🙂　 字Z ½ꟲ\r'<|endoftext|>́tDž­", "tokens": 61, "pieces": ["9", " fiZꟲＡ", "👍🏽'", "M", "­ع", "<|", "endoftext", "|>🙂#$%­", "ḋ", "̣t", "🙂", "　", " 字Z", " ", " ", "½", "ꟲ", "\r", "'<|", "endoftext", "|>́", "tDž", "­"]} +{"text": "𐞁'S(㍿!!e'll'Tsfi㍿𐞁EOT're", "tokens": 30, "pieces": ["𐞁", "'S", "(㍿!!", "e", "'", "ll", "'T", "sfi", "㍿𐞁EOT", "'re"]} +{"text": "aⅣEOT字㍿́\"​ع㍿EOT'D\r<|fim_prefix|>'Sé\r𐞁'Dعİ㍿''VEå‍'Ree0 \u000bDž", "tokens": 61, "pieces": ["a", "Ⅳ", "EOT字", "㍿́\"​", "ع", "㍿EOT", "'D", "\r", "<|", "fim", "_prefix", "|><", "META", "_START", ">'", "Sé", "\r", "𐞁", "'D", "عİ", "㍿''", "VE", "a", "̊‍'", "Ree", "0", " ", "\u000bDž"]} +{"text": "å,🙂<|endoftext|>. \n​.\r\n字\r\nİ\r\n\r\n'MＡ㋿½9", "tokens": 65, "pieces": ["a", "̊<", "e", "'VE", "\u000bꟲ", " ", " '", "llḋ", "̣ḋ", "̣", "　", "३", "'S", " ", "'ſ", "Džꟲ", ",🙂<|", "endoftext", "|>.", " \n", "​.\r\n", "字", "\r\n", "İ", "\r\n\r\n", "'M", "Ａ", "㋿", "½9"]} +{"text": "
<|fim_prefix|>Dž", "tokens": 11, "pieces": ["
", "<|", "fim", "_prefix", "|>", "Dž"]} +{"text": "-\n
(\r\n\r\n\r\n　fi'seß\r漢㋿s ­", "tokens": 19, "pieces": ["-\n", "
", "(\r\n\r\n\r\n", "　fi", "'s", "eß", "\r", "漢", "㋿s", " ­"]} +{"text": "'VE٣٤٥٦👍🏽's
e😀🏽ß🙂>ß <ＡＡ'S!!㍿㋿ '‍㋿é>ḍ̇fi\r\n\r\n'ſ", "tokens": 64, "pieces": ["'VE", "٣٤٥", "٦", "👍🏽'", "s", "
e", "😀🏽", "ß", "🙂>", "ß", " <", "ＡＡ", "'S", "!!㍿㋿", " ", " '‍㋿", "é", ">ḋ", "̣fi", "\r\n\r\n", "'ſ"]} +{"text": "'D'ſ…'S9", "tokens": 8, "pieces": ["'D", "'ſ", "…", "'S", "9"]} +{"text": "\rm३　'S🙂漢㍿.㍿,'EOT!漢 EOTå", "tokens": 28, "pieces": ["\r", "m", "३", "　", "'S", "🙂漢", "㍿.㍿,'", "EOT", "!漢", " EOTa", "̊"]} +{"text": "­İ㍿!!🙂 ​EOT'S0㋿éßå㋿ \n", "tokens": 27, "pieces": ["­İ", "㍿!!🙂", " ", " ​", "EOT", "'S", "0", "㋿e", "́ßa", "̊㋿", " \n"]} +{"text": "s", "tokens": 1, "pieces": ["s"]} +{"text": "'llⅣ''<|endoftext|>a½\r\n\r\n m　!d!!aＡ　'Dß㋿Z<|fim_prefix|>>'red", "tokens": 43, "pieces": ["'ll", "Ⅳ", "''<|", "endoftext", "|>", "a", "½", "\r\n\r\n", " ", " m", "　", "!d", "!!", "aＡ", "　", "'D", "ß", "㋿Z", "<|", "fim", "_prefix", "|>>'", "red"]} +{"text": "𐞁𐞁t\r\n-́'Te'Dſ…<|fim_prefix|>　9ḍ̇", "tokens": 33, "pieces": ["𐞁𐞁t", "\r\n", "-́'", "Te", "'D", "ſ", "…", "<|", "fim", "_prefix", "|>", "　", "9", "ḋ", "̣"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": " #$%'ſé('ll ‍­'Re'ReDža", "tokens": 18, "pieces": [" ", "#$%'", "ſé", "('", "ll", " ", " ‍­'", "Re", "'Re", "Dža"]} +{"text": "٣٤٥٦\u000b<|fim_prefix|>　fi㋿", "tokens": 22, "pieces": ["٣٤٥", "٦", "\u000b", "<|", "fim", "_prefix", "|>", "　fi", "㋿"]} +{"text": "ꟲعae\r\n\r\n'ſ
字(-\t!!!!½åßéＡ!​'Reſ", "tokens": 29, "pieces": ["ꟲعae", "\r\n\r\n", "'ſ", "
字", "(-", "\t", "!!!!", "½", "a", "̊ßéＡ", "!​'", "Reſ"]} +{"text": "-३ #$%!!👍🏽", "tokens": 13, "pieces": ["-", "३", " ", " #$%!!👍🏽"]} +{"text": "𐞁'\t- 'M.𐞁​", "tokens": 20, "pieces": ["𐞁", "'", "\t", "-", " ", " '", "M", ".𐞁", "​"]} +{"text": " å　<|fim_prefix|>EOT字d‍㍿‍-12345678're\"t0👍🏽<|endoftext|>s>\r\n\r\n9Ⅳad'VE're\"­", "tokens": 55, "pieces": [" ", " a", "̊", "　", "<|", "fim", "_prefix", "|>", "EOT字d", "‍㍿‍-", "123", "456", "78", "'re", "\"t", "0", "👍🏽<|", "endoftext", "|>", "s", ">\r\n\r\n", "9Ⅳ", "ad", "'VE", "'re", "\"­"]} +{"text": "ſ", "tokens": 2, "pieces": ["ſ"]} +{"text": "å٣٤٥٦Z😀🏽>'T\r\n\r\n३ß\"½éZ㋿‍ḍ̇9s'D字\n漢ſdt३e", "tokens": 54, "pieces": ["a", "̊", "٣٤٥", "٦", "Z", "😀🏽<", "EOT", ">>'", "T", "\r\n\r\n", "३", "ß", "\"", "½", "éZ", "㋿‍", "ḋ", "̣", "9", "s", "'D", "字", "\n", "漢ſdt", "३", "e"]} +{"text": "㍿-.'s.e　­'VE ­\u000b\r\nå", "tokens": 17, "pieces": ["㍿-.'", "s", ".e", "　", "­'", "VE", " ­", "\u000b\r\n", "a", "̊"]} +{"text": "'T'T", "tokens": 2, "pieces": ["'T", "'T"]} +{"text": "-.('", "tokens": 2, "pieces": ["-.('"]} +{"text": "Z'ſ'VEé'T…ß'ſ\n're　­.>𐞁́å'VE😀🏽 -d\u000bＡ'T", "tokens": 43, "pieces": ["Z", "'ſ", "'VE", "e", "́'", "T", "…ß", "'ſ", "\n", "'re", "　", "­.>", "𐞁", "́a", "̊'", "VE", "😀🏽", " ", "-d", "\u000bＡ", "'T"]} +{"text": "ḍ̇३३㋿e'M'Re\r\n
\"d \u000b'D-漢 漢e\r\n\r\n\u000b㍿a㍿", "tokens": 42, "pieces": ["ḋ", "̣<", "META", "_START", ">", "३३", "㋿e", "'M", "'Re", "\r\n", "
", "\"d", " ", "\u000b", "'D", "-漢", " 漢e", "\r\n\r\n", "\u000b", "㍿a", "㍿"]} +{"text": "'ll'VE'VEå👍🏽é9\r\n\r\n<漢", "tokens": 28, "pieces": ["'ll", "'VE", "'", "VEa", "̊👍🏽", "e", "́<", "EOT", ">", "9", "\r\n\r\n", "<漢"]} +{"text": "عⅣ'M'Md٣٤٥٦0!
're ‍<>måm३\"ع-㍿㋿'re🙂0's'Ś​m<|endoftext|>½ \n­m-🙂", "tokens": 61, "pieces": ["ع", "Ⅳ", "'M", "'M", "d", "٣٤٥", "٦0", "!", "
", "'re", " ‍<>", "ma", "̊m", "३", "\"ع", "-㍿㋿'", "re", "🙂", "0", "'s", "'S", "́​", "m", "<|", "endoftext", "|>", "½", " \n", "­m", "-🙂"]} +{"text": "\u000b字'éfi'VEDžⅣ\tDžda\r\n\r\n 'VE½<|fim_prefix|><|endoftext|>🙂-'ll漢Ⅳ", "tokens": 45, "pieces": ["\u000b字", "'e", "́fi", "'VE", "Dž", "Ⅳ", "\tDžda", "\r\n\r\n", " ", " <", "EOT", ">'", "VE", "½", "<|", "fim", "_prefix", "|><|", "endoftext", "|>🙂-'", "ll漢", "Ⅳ"]} +{"text": "İeꟲＡ", "tokens": 7, "pieces": ["İeꟲＡ"]} +{"text": "9'Reعd(𐞁\n㋿३a\r\n\r\nⅣ(Ａ \n'S字\"a
fi\u000b're-!", "tokens": 41, "pieces": ["9", "'Re", "عd", "(𐞁", "\n", "㋿", "३", "a", "\r\n\r\n", "Ⅳ", "(Ａ", " \n", "'S", "字", "\"a", "
fi", "\u000b", "'re", "-!<", "META", "_START", ">"]} +{"text": "Dž٣٤٥٦#$%", "tokens": 12, "pieces": ["Dž", "٣٤٥", "٦", "#$%"]} +{"text": "㍿\r\n\r\nd　e>…Ａſé<|endoftext|>\tḍ̇\r\n\r\n\r", "tokens": 31, "pieces": ["㍿\r\n\r\n", "d", "　e", ">", "…Ａſe", "́<|", "endoftext", "|>", "\tḋ", "̣\r\n\r\n\r"]} +{"text": "!fi>'re'VEſ­å'Re.EOTſa 
㋿'ſ\r\n'Mß'M'Sem\r\n\r\n12345678ḍ̇ 𐞁!!­é12345678́👍🏽12345678", "tokens": 71, "pieces": ["!fi", ">'", "re", "'VE", "ſ", "­a", "̊'", "Re", ".EOTſa", " ", "
", "㋿'", "ſ", "\r\n", "'M", "ß", "'", "M", "'S", "em", "\r\n\r\n", "123", "456", "78", "ḋ", "̣", " ", " 𐞁", "!!­", "e", "́", "123", "456", "78", "́👍🏽", "123", "456", "78"]} +{"text": "e.́عſe​\r\nt -㋿½\n'VE('re0 0 ㍿'T ßＡ𐞁!!ḍ̇<|fim_prefix|>字‍½t㋿", "tokens": 65, "pieces": ["e", ".́", "عſe", "​\r\n", "t", " ", " -㋿", "½", "\n", "'VE", "('", "re", "", "0", " ", " ", "0", " ", " ㍿'", "T", " ", " ßＡ𐞁", "!!", "ḋ", "̣<|", "fim", "_prefix", "|>", "字", "‍", "½", "t", "㋿"]} +{"text": "#$% 's‍'ſém ſ'T0½'M㋿éZ", "tokens": 22, "pieces": ["#$%", " ", " '", "s", "‍'", "ſém", " ſ", "'T", "0½", "'M", "㋿éZ"]} +{"text": "　m٣٤٥٦ \n­'D㋿", "tokens": 20, "pieces": ["　m", "٣٤٥", "٦", " \n", "­'", "D", "㋿"]} +{"text": "d٣٤٥٦́­́३<\r\n..‍'S-.㍿́
İ<'T", "tokens": 37, "pieces": ["d", "٣٤٥", "٦", "́­́", "३", "<\r\n", "..<", "EOT", ">‍'", "S", "-.㍿́", "
İ", "<'", "T"]} +{"text": "'S\u000bs're>Ⅳt'VE<'Re><|endoftext|>​'Té
<|fim_prefix|>'s\r\n0½.'VE\u000b'M#$%'M​ſZeḍ̇.< !!d're", "tokens": 57, "pieces": ["'S", "\u000bs", "'re", ">", "Ⅳ", "t", "'VE", "<'", "Re", "><|", "endoftext", "|>​'", "Te", "́", "
", "<|", "fim", "_prefix", "|>'", "s", "\r\n", "0½", ".'", "VE", "\u000b", "'M", "#$%'", "M", "​ſZeḋ", "̣.<", " ", "!!", "d", "'re"]} +{"text": "
́عé'll‍ḍ̇!!'ſ're\r'VE…‍'VEſe'VE😀🏽🙂sع", "tokens": 44, "pieces": ["
", "́عé", "'ll", "‍ḋ", "̣!!'", "ſ", "'re", "\r", "'VE", "…", "‍'", "VEſe", "'VE", "😀🏽🙂", "sع"]} +{"text": "!!́ fie 
12345678-\u000bé0're(", "tokens": 21, "pieces": ["!!́", " fie", " ", "
", "123", "456", "78", "-", "\u000bé", "0", "'re", "("]} +{"text": "\r\n.́'Dt<\"　٣٤٥٦'Sd'Md'll😀🏽 ḍ̇İ9\t<|fim_prefix|>ém​Ａ　漢å", "tokens": 53, "pieces": ["\r\n", ".́'", "Dt", "<\"", "　", "٣٤٥", "٦", "'S", "d", "'M", "d", "'ll", "😀🏽", " ", " ḋ", "̣İ", "9", "\t", "<|", "fim", "_prefix", "|>", "e", "́m", "​Ａ", "　漢a", "̊"]} +{"text": "Zfi
åZ👍🏽.'VE're…é123456780ع㋿<|endoftext|>İ'!(9'(㋿
👍🏽'Re　㋿>", "tokens": 63, "pieces": ["Zfi", "
a", "̊Z", "👍🏽.'", "VE", "'re", "", "…e", "́", "123", "456", "780", "ع", "㋿<|", "endoftext", "|>", "İ", "'!(", "9", "'(㋿", "
", "👍🏽'", "Re", "　", "㋿>"]} +{"text": "'M'llEOT'll", "tokens": 9, "pieces": ["'M", "'", "llEOT", "'ll"]} +{"text": "…éd EOTع're'DDž>㍿>İ\r\n\r\n(　<­½é-'ſع're​ ", "tokens": 31, "pieces": ["…éd", " ", " EOTع", "'re", "'D", "Dž", ">㍿>", "İ", "\r\n\r\n", "(", "　", "<­", "½", "é", "-'", "ſع", "'re", "​", " "]} +{"text": "ßſßfi'St", "tokens": 8, "pieces": ["ßſßfi", "'S", "t"]} +{"text": "Dž0t'VEm(fi\r", "tokens": 11, "pieces": ["Dž", "0", "t", "'VE", "m", "(fi", "\r"]} +{"text": "<#$%'Re \n \n字\r\n\r\n…́<|fim_prefix|>12345678EOT­9Dž-'' ​m𐞁३'T'T\n", "tokens": 43, "pieces": ["<#$%'", "Re", " \n \n", "字", "\r\n\r\n", "…", "́<|", "fim", "_prefix", "|>", "123", "456", "78", "EOT", "­", "9", "Dž", "-''", " ​", "m𐞁", "३", "'T", "'T", "\n"]} +{"text": "<|fim_prefix|>!(EOTm👍🏽Z\"", "tokens": 22, "pieces": ["<|", "fim", "_prefix", "|>!(", "EOTm", "👍🏽", "Z", "\"<", "EOT", ">"]} +{"text": "\r\n\r\nḍ̇'M३'Dع😀🏽><|endoftext|>!>\r'Re漢'VE", "tokens": 32, "pieces": ["\r\n\r\n", "ḋ", "̣'", "M", "३", "'D", "ع", "😀🏽><|", "endoftext", "|>!>\r", "'Re", "漢", "'VE"]} +{"text": "३'S🙂é,​'reİ㍿ꟲa-<|endoftext|>éZ­-😀🏽\u000b", "tokens": 37, "pieces": ["३", "'S", "🙂e", "́,​'", "reİ", "㍿ꟲa", "-<|", "endoftext", "|>", "éZ", "­-😀🏽", "\u000b"]} +{"text": "　'VEعＡßꟲ…ſ'M​­d\ré're!\t!#$%\n😀🏽", "tokens": 37, "pieces": ["　", "'VE", "عＡßꟲ", "…ſ", "'M", "​­", "d", "\r", "e", "́'", "re", "!", "\t", "!#$%<", "EOT", ">\n", "😀🏽"]} +{"text": "'ſ…'llåé'Re.…m'reſ(\"e'Ⅳéİ­'D#$%a
'S½½३́\rعＡ३ḍ̇㍿", "tokens": 63, "pieces": ["'ſ", "…", "'ll", "a", "̊é", "'Re", ".", "…m", "'re", "ſ", "(\"", "e", "'", "Ⅳ", "é", "İ", "­'", "D", "#$%", "a", "
", "'S", "½½३", "́\r", "عＡ", "३", "ḋ", "̣㍿"]} +{"text": ",'ſ(,'é'Re😀🏽字Ａ㍿㍿(🙂m>'ſ٣٤٥٦å12345678३‍('T㍿‍", "tokens": 63, "pieces": [",'", "ſ", "(,'", "é", "'", "Re", "😀🏽", "字Ａ", "㍿㍿(<", "EOT", ">🙂", "m", ">'", "ſ", "٣٤٥", "٦", "a", "̊", "123", "456", "78३", "‍('", "T", "㍿‍"]} +{"text": "<🙂٣٤٥٦ⅣEOTEOTt'ſ12345678 \n , ́\"\u000b,\r\n\r\n字'VEfi.'s#$%", "tokens": 39, "pieces": ["<🙂", "٣٤٥", "٦Ⅳ", "EOTEOTt", "'ſ", "123", "456", "78", " \n", " ,", " ́\"", "\u000b", ",\r\n\r\n", "字", "'VE", "fi", ".'", "s", "#$%"]} +{"text": ".ß -<|fim_prefix|>99\r\n\n👍🏽ḍ̇㋿t㋿<|fim_prefix|>'refi", "tokens": 40, "pieces": [".ß", " -<|", "fim", "_prefix", "|>", "99", "\r\n\n", "👍🏽", "ḋ", "̣㋿", "t", "㋿<|", "fim", "_prefix", "|>'", "refi"]} +{"text": "­'s12345678.İ‍fit'D(Ａé\t'T9…a३İ", "tokens": 28, "pieces": ["­'", "s", "123", "456", "78", ".İ", "‍fit", "'D", "(Ａe", "́", "\t", "'T", "9", "…a", "३", "İ"]} +{"text": "​'Sſ漢t e,.'S ½ \nsſé'M㋿'llſ'Dž\t　\"🙂tZḍ̇…", "tokens": 50, "pieces": ["​'", "Sſ", "漢t", " e", ",.'", "S", " ", " ", "½", " \n", "sſe", "́'", "M", "㋿'", "llſ", "'Dž", "\t", "　", "\"🙂", "tZḋ", "̣", "…"]} +{"text": ",9\tḍ̇", "tokens": 8, "pieces": [",", "9", "\tḋ", "̣"]} +{"text": "٣٤٥٦عßé\r\n…,ḍ̇\n㍿'T>　<|fim_prefix|>ع<|endoftext|>s<", "tokens": 52, "pieces": ["'D", "ḋ", "̣", "\t字", "\n", "s", "'M", "a漢", "　", "㋿'", "re", "\t", ",🙂", "0", ".", "
e", "́<", "EOT", ">>", "　", "<|", "fim", "_prefix", "|>", "ع", "<|", "endoftext", "|>", "s", "<"]} +{"text": "!‍'D9Dž'llḍ̇​Ａ㋿३\r३té'llå'D'Re…­३0漢<|fim_prefix|>9(­½<|fim_prefix|>́ ́t", "tokens": 64, "pieces": ["!‍'", "D", "9", "Dž", "'ll", "ḋ", "̣​", "Ａ", "㋿", "३", "\r", "३", "te", "́'", "lla", "̊'", "D", "'Re", "…", "­", "३0", "漢", "<|", "fim", "_prefix", "|>", "9", "(­", "½", "<|", "fim", "_prefix", "|>́", " ́", "t"]} +{"text": "12345678<|fim_prefix|>\r\nå👍🏽#$%é漢12345678Ⅳ٣٤٥٦t🙂ع𐞁'sİ٣٤٥٦…漢👍🏽ḍ̇EOT're", "tokens": 73, "pieces": ["123", "456", "78", "<|", "fim", "_prefix", "|>\r\n", "a", "̊👍🏽#$%", "é漢", "123", "456", "78Ⅳ", "٣٤٥", "٦", "t", "🙂ع𐞁", "'s", "İ", "٣٤٥", "٦", "…漢", "👍🏽", "ḋ", "̣EOT", "'re"]} +{"text": "ßعs'㍿\"\r\n\r\n字👍🏽0afi'ſ12345678tİ<9　😀🏽\r\n\r\n-", "tokens": 39, "pieces": ["ßعs", "'㍿\"\r\n\r\n", "字", "👍🏽", "0", "afi", "'ſ", "123", "456", "78", "tİ", "<", "9", "　", "😀🏽\r\n\r\n", "-"]} +{"text": "'M字t12345678…#$%­,\n \nDž9字m\r\n", "tokens": 19, "pieces": ["'M", "字t", "123", "456", "78", "…", "#$%­,\n", " \n", "Dž", "9", "字m", "\r\n"]} +{"text": "字's👍🏽\u000b\r\n\r\nع<\u000b३ \nDž>㋿åßfi \n", "tokens": 55, "pieces": ["字", "'s", "👍🏽", "\u000b\r\n\r\n", "ع", "<", "\u000b", "", "३", " \n", "Dž", ">㋿", "a", "̊ßfi", " \n"]} +{"text": "'D'll fi👍🏽EOT‍s👍🏽'VEEOT a😀🏽\r!!
mZ\r\n(­😀🏽ḍ̇EOT\r\n \n", "tokens": 65, "pieces": ["'D", "'ll", "", " fi", "👍🏽", "EOT", "‍s", "👍🏽'", "VEEOT", " ", " <", "EOT", ">a", "😀🏽\r", "!!", "
mZ", "\r\n", "(­<", "META", "_START", ">😀🏽", "ḋ", "̣EOT", "\r\n \n"]} +{"text": "ßꟲ‍­<|fim_prefix|>12345678\r\n\r\n", "tokens": 18, "pieces": ["ßꟲ", "‍­<|", "fim", "_prefix", "|>", "123", "456", "78", "\r\n\r\n"]} +{"text": "́", "tokens": 1, "pieces": ["́"]} +{"text": "…é­<|endoftext|>d字,\nع'M9ḍ̇'Re<'sd", "tokens": 29, "pieces": ["…é", "­<|", "endoftext", "|>", "d字", ",\n", "ع", "'M", "9", "ḋ", "̣'", "Re", "<'", "sd"]} +{"text": "\r\n\r\n", "tokens": 1, "pieces": ["\r\n\r\n"]} +{"text": "🙂 12345678Z𐞁‍'ſZ٣٤٥٦'re😀🏽👍🏽#$%>'ſ90‍12345678m's", "tokens": 53, "pieces": ["🙂", " <", "EOT", ">", "123", "456", "78", "Z𐞁", "‍'", "ſZ", "٣٤٥", "٦", "'re", "😀🏽👍🏽#$%>'", "ſ", "90", "‍", "123", "456", "78", "m", "'s"]} +{"text": "!fi.''D​'S​aé🙂'ſ🙂‍eé'Re½!!\u000b!!'T㍿t", "tokens": 42, "pieces": ["!fi", ".<", "EOT", ">''", "D", "​'", "S", "​ae", "́🙂'", "ſ", "🙂‍", "ee", "́'", "Re", "½", "!!<", "META", "_START", ">", "\u000b", "!!'", "T", "㍿t"]} +{"text": "'S'", "tokens": 6, "pieces": ["'", "S", "'"]} +{"text": "\"ſİ#$%½-½ 𐞁'Re'ſ　e-\r\n'M'll
e👍🏽a'ſ", "tokens": 36, "pieces": ["\"ſİ", "#$%", "½", "-", "½", " 𐞁", "'Re", "'ſ", "　e", "-\r\n", "'M", "'ll", "
e", "👍🏽", "a", "'ſ"]} +{"text": "👍🏽'VE\"fi-t's'll漢ße'Tåefis\r\r\n're
ع!å㍿å𐞁 \né😀🏽9Ａ(é'T㋿\"-", "tokens": 62, "pieces": ["👍🏽'", "VE", "\"fi", "-t", "'s", "'ll", "漢ße", "'T", "a", "̊efis", "\r\r\n", "'re", "
ع", "!a", "̊㍿", "a", "̊𐞁", " \n", "é", "😀🏽", "9", "Ａ", "(e", "́'", "T", "㋿\"-"]} +{"text": "𐞁​㍿", "tokens": 8, "pieces": ["𐞁", "​㍿"]} +{"text": "ꟲEOT­<|endoftext|>m​ſ \n👍🏽<|fim_prefix|>EOT>#$%t \n'llfi😀🏽Ze,", "tokens": 54, "pieces": ["ꟲEOT", "­<", "META", "_START", "><|", "endoftext", "|>", "m", "​ſ", " \n", "👍🏽<|", "fim", "_prefix", "|>", "EOT", "><", "EOT", ">#$%", "t", " \n", "'ll", "fi", "😀🏽", "Ze", ","]} +{"text": "a\r\n\r\n‍ s'ſ­é \n!!t字0\t,'S", "tokens": 19, "pieces": ["a", "\r\n\r\n", "‍", " s", "'ſ", "­é", " \n", "!!", "t字", "0", "\t", ",'", "S"]} +{"text": "'M!<٣٤٥٦\nDž'ſ-12345678👍🏽'Sß字Z३\r\né.'T#$%-'t𐞁​㍿<é\u000bⅣ​", "tokens": 56, "pieces": ["'M", "!<", "٣٤٥", "٦", "\n", "Dž", "'ſ", "-", "123", "456", "78", "👍🏽'", "Sß字Z", "३", "\r\n", "e", "́.'", "T", "#$%-'", "t𐞁", "​㍿<", "é", "\u000b", "Ⅳ", "​"]} +{"text": "​ 'll\r\n're", "tokens": 9, "pieces": ["​", " ", " '", "ll", "\r\n", "'re"]} +{"text": " \"e'T漢<|endoftext|>Dž(…'VE㋿ſİ\r\n\r\nＡ'reḍ̇'Re.", "tokens": 58, "pieces": [" ", "\"e", "'T", "漢", "<|", "endoftext", "|>", "Dž", "(", "…", "'VE", "㋿ſİ", "\r\n\r\n", "Ａ", "'", "reḋ", "̣'", "Re", "."]} +{"text": "३٣٤٥٦½", "tokens": 11, "pieces": ["३٣٤", "٥٦½"]} +{"text": "ß'Re\"fi ३İ \n<<|endoftext|>12345678🙂'ſ12345678
٣٤٥٦", "tokens": 42, "pieces": ["ß", "'Re", "\"fi", " ", " <", "META", "_START", ">", "३", "İ", " \n", "<<|", "endoftext", "|>", "123", "456", "78", "🙂'", "ſ", "123", "456", "78", "
", "٣٤٥", "٦"]} +{"text": "<|fim_prefix|>'s\t ḍ̇'re🙂　ß漢's\r\n\r\nZ9'Resḍ̇\n'T\"漢\"Ⅳ", "tokens": 46, "pieces": ["<|", "fim", "_prefix", "|>'", "s", "\t ", " ḋ", "̣'", "re", "🙂", "　ß漢", "'s", "\r\n\r\n", "Z", "9", "'Re", "sḋ", "̣\n", "'T", "\"漢", "\"", "Ⅳ"]} +{"text": "‍‍㍿\u000b<|fim_prefix|> 𐞁t-​'aＡ>ḍ̇　'llß٣٤٥٦da'e㍿ꟲſ", " 𐞁t", "-​'", "aＡ", ">ḋ", "̣", "　", "'ll", "ß", "٣٤٥", "٦", "da", "'e", "㍿ꟲſ", "<|fim_prefix|>'Refié𐞁½\u000b३ 'Dſḍ̇12345678 
", "tokens": 41, "pieces": ["‍😀🏽><|", "fim", "_prefix", "|>'", "Refié𐞁", "½", "\u000b", "३", " '", "Dſḋ", "̣", "123", "456", "78", " 
"]} +{"text": "\t'T　İ​Z😀🏽", "tokens": 11, "pieces": ["\t", "'T", "　İ", "​Z", "😀🏽"]} +{"text": "'D!!Ⅳ​s're ­😀🏽'Re٣٤٥٦́ ㋿٣٤٥٦٣٤٥٦ Dž'M<|endoftext|>
", "tokens": 65, "pieces": ["'D", "!!", "Ⅳ", "​<", "EOT", ">s", "'re", " ", " ­😀🏽'", "Re", "٣٤٥", "٦", "́", " ㋿", "٣٤٥", "٦٣٤", "٥٦", " Dž", "'M", "<|", "endoftext", "|>", "
"]} +{"text": "ßåéḍ̇ꟲfi👍🏽'll\"", "tokens": 31, "pieces": ["ßa", "̊é", "<", "EOT", ">ḋ", "̣ꟲfi", "👍🏽'", "ll", "\""]} +{"text": "\"t'reع漢́٣٤٥٦Ⅳ\t٣٤٥٦<́s'Sع​m\tⅣ.\r'Déé", "tokens": 41, "pieces": ["\"t", "'re", "ع漢", "́", "٣٤٥", "٦Ⅳ", "\t", "٣٤٥", "٦", "<́", "s", "'S", "ع", "​m", "\t", "Ⅳ", ".\r", "'D", "éé"]} +{"text": "('M\r字…-३ \u000b…
é<|fim_prefix|>'M.'Re", "tokens": 26, "pieces": ["('", "M", "\r", "字", "…", "-", "३", " \u000b…", "
é", "<|", "fim", "_prefix", "|>'", "M", ".'", "Re"]} +{"text": "'VE.<|endoftext|>fi字\t​\r\n\r\n字​ꟲ\r Dž́𐞁'D12345678٣٤٥٦'T<\r\n'ſ12345678👍🏽漢
𐞁'll­12345678 ع½😀🏽#$%　\r\n\r\n", "tokens": 82, "pieces": ["'VE", ".<|", "endoftext", "|>", "fi字", "\t", "​\r\n\r\n", "字", "​ꟲ", "\r", " ", " Dž", "́𐞁", "'D", "123", "456", "78٣", "٤٥٦", "'T", "<\r\n", "'ſ", "123", "456", "78", "👍🏽", "漢", "
𐞁", "'ll", "­", "123", "456", "78", " ", " ع", "½", "😀🏽#$%", "　\r\n\r\n"]} +{"text": "漢!!'Re !!", "tokens": 7, "pieces": ["漢", "!!'", "Re", " ", " !!"]} +{"text": "-EOT", "tokens": 5, "pieces": ["-", "EOT"]} +{"text": "­
ꟲfi㋿٣٤٥٦👍🏽(३'M\"'T('re😀🏽>Ａfi12345678", "tokens": 50, "pieces": ["­", "
ꟲfi", "㋿", "٣٤٥", "٦", "👍🏽(", "३", "'M", "\"'", "T", "('", "re", "😀🏽>", "Ａfi", "", "123", "456", "78"]} +{"text": "㍿'s'ſ字\u000b'Re😀🏽,'T́­ 𐞁
!!fi　,漢漢३\t३ß́\rⅣ\r'ſ'ſå\r\r\n\r\n👍🏽", "tokens": 64, "pieces": ["㍿'", "s", "'ſ", "字", "\u000b", "'Re", "😀🏽,'", "T", "́­", " 𐞁", "
", "!!", "fi", "　", ",漢漢", "३", "\t", "३", "ß", "́\r", "Ⅳ", "\r", "'ſ", "'ſ", "a", "̊\r\r\n\r\n", "👍🏽"]} +{"text": "\t12345678🙂İ­Dža­< Ａ", "tokens": 15, "pieces": ["\t", "123", "456", "78", "🙂İ", "­Dža", "­<", " Ａ"]} +{"text": "a<|fim_prefix|>\r\n\r\n\r\n\r\n
­👍🏽 m'så'VE", "tokens": 25, "pieces": ["a", "<|", "fim", "_prefix", "|>\r\n\r\n\r\n\r\n", "
", "­👍🏽", " m", "'s", "a", "̊'", "VE"]} +{"text": "　\r\n\r\n😀🏽‍'VE-İ\naＡ<字<|endoftext|>#$%a🙂é>ع!! ꟲ​dſ½'T", "tokens": 53, "pieces": ["　\r\n\r\n", "😀🏽‍'", "VE", "-İ", "\n", "aＡ", "<字", "<|", "endoftext", "|>#$%", "a", "🙂e", "́>", "ع", "!!<", "EOT", ">", " ꟲ", "​dſ", "½", "'T"]} +{"text": "'D३a漢𐞁🙂12345678<|fim_prefix|>>㍿'VE٣٤٥٦ EOT'T\t🙂e字(", "tokens": 48, "pieces": ["'D", "३", "a漢𐞁", "🙂", "123", "456", "78", "<|", "fim", "_prefix", "|>>㍿<", "META", "_START", ">'", "VE", "٣٤٥", "٦", " EOT", "'T", "\t", "🙂e字", "("]} +{"text": "́　 \r\nfi", "tokens": 6, "pieces": ["́", "　 \r\n", "fi"]} +{"text": "'Sİt­.'D's𐞁\r\n-'ll'll.İ́\r\n\r\n'\n字éåt \nß", "tokens": 31, "pieces": ["'S", "İt", "­.'", "D", "'s", "𐞁", "\r\n", "-'", "ll", "'ll", ".<", "EOT", ">İ", "́\r\n\r\n", "'\n", "字éa", "̊t", " \n", "ß"]} +{"text": "🙂t\"ꟲ\r\nꟲİ<|endoftext|> ḍ̇é 'T​𐞁漢t　ſع12345678fiع", "tokens": 45, "pieces": ["🙂t", "\"ꟲ", "\r\n", "ꟲİ", "<|", "endoftext", "|>", " ḋ", "̣é", " ", "'T", "​𐞁漢t", "　ſع", "123", "456", "78", "fiع"]} +{"text": "'Re'D'VEDž.!!İꟲ", "tokens": 12, "pieces": ["'Re", "'D", "'VE", "Dž", ".!!", "İꟲ"]} +{"text": "fiDž­s㍿'
㋿'reſé👍🏽漢𐞁<|fim_prefix|>ḍ̇tDž'sa", "tokens": 52, "pieces": ["fiDž", "­s", "㍿'", "
", "㋿'", "reſé", "👍🏽<", "META", "_START", ">漢𐞁", "<|", "fim", "_prefix", "|>", "ḋ", "̣tDž", "'s", "a"]} +{"text": "éꟲ漢٣٤٥٦ 'M\tfi\u000b're字", "tokens": 23, "pieces": ["éꟲ漢", "٣٤٥", "٦", " ", " '", "M", "\tfi", "\u000b", "'re", "字"]} +{"text": " …ſe'9漢­字Ⅳ😀🏽㋿\u000b\r\n\r\nd'D🙂…　d#$%'Reḍ̇e㍿Zt…\u000bİ>e\r­a\r", "tokens": 59, "pieces": [" ", "…ſe", "'", "9", "漢", "­字", "", "Ⅳ", "😀🏽㋿", "\u000b\r\n\r\n", "d", "'D", "🙂", "…", "　d", "#$%'", "Reḋ", "̣e", "㍿Zt", "…", "\u000bİ", ">e", "\r", "­a", "\r"]} +{"text": ">,<|endoftext|>İ漢 \r\n\r\ns字!ſ0漢Z\u000bEOT'VE
Ａ👍🏽,", "tokens": 37, "pieces": [">,<|", "endoftext", "|>", "İ漢", " \r\n\r\n", "s字", "!ſ", "0", "漢Z", "\u000bEOT", "'VE", "
Ａ", "👍🏽,"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'TdⅣfi­t\t-­'s<\"'VE'M　'Re'VE漢ſ
'D
‍㍿9#$%ſ'd
\r", "tokens": 52, "pieces": ["'T", "d", "Ⅳ", "fi", "­t", "\t", "-­'", "s", "<<", "META", "_START", ">\"'", "VE", "'M", "　", "'Re", "'VE", "漢ſ", "
", "'D", "
", "‍㍿", "9", "#$%", "ſ", "'d", "
\r"]} +{"text": "eḍ̇ ,'M0tEOT \n'T\"👍🏽t漢
!!㍿\r\nſİ😀🏽<|endoftext|>'reZ's", "tokens": 49, "pieces": ["eḋ", "̣", " ,'", "M", "0", "tEOT", " \n", "'T", "\"👍🏽", "t漢", "
", "!!㍿\r\n", "ſİ", "😀🏽<|", "endoftext", "|>'", "reZ", "'s"]} +{"text": "𐞁'M'ſZ'T'Då\u000bßfi'VE'S\n\n漢'S\r\nſ'SEOT\" Ⅳ", "tokens": 38, "pieces": ["𐞁", "'M", "'ſ", "Z", "'T", "'D", "a", "̊", "\u000bßfi", "'VE", "'", "S", "\n\n", "漢", "'S", "\r\n", "ſ", "'S", "EOT", "\"", " ", "Ⅳ"]} +{"text": "😀🏽  -0'M \n㋿9", "tokens": 15, "pieces": ["😀🏽", " ", " ", "-", "0", "'M", " \n", "㋿", "9"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "½'Da'D-<|fim_prefix|>'re'D​m'å३\r\n\r\n३Ａfi(, e.'re'Reḍ̇'T", "tokens": 45, "pieces": ["½", "'D", "a", "'D", "-<|", "fim", "_prefix", "|>'", "re", "'D", "​m", "'a", "̊", "३", "\r\n\r\n", "३", "Ａfi", "(,", " ", " e", ".'", "re", "'Re", "ḋ", "̣<", "META", "_START", ">'", "T"]} +{"text": "\rm‍㍿0('Séع<|endoftext|>­ ", "tokens": 21, "pieces": ["\r", "m", "‍㍿", "0", "('", "Se", "́ع", "<|", "endoftext", "|>­", " "]} +{"text": "'ſaEOT \n\t🙂é漢're​Dž🙂㍿(dm㍿éfiEOT..́…,(0.👍🏽…İ­!㋿", "tokens": 54, "pieces": ["'ſ", "aEOT", " \n", "\t", "🙂e", "́漢", "'re", "​Dž", "🙂㍿(", "dm", "㍿éfiEOT", "..́", "…", ",(", "0", ".👍🏽", "…İ", "­!㋿"]} +{"text": ",'VEéDž", "tokens": 6, "pieces": [",'", "VEe", "́Dž"]} +{"text": "'llＡ́㋿12345678 é𐞁Ａa9𐞁fi㋿ḍ̇'ſſ'D\nḍ̇ß́>('VE'VE,㋿", "tokens": 57, "pieces": ["'ll", "Ａ", "́㋿", "123", "456", "78", " ", " e", "́𐞁Ａa", "9", "𐞁fi", "㋿ḋ", "̣'", "ſſ", "'D", "\n", "ḋ", "̣ß", "́>('", "VE", "'VE", ",㋿"]} +{"text": "'ll-ḍ̇ \r'ſd!! ३,𐞁½ſ　éḍ̇sḍ̇é'ſ!\t<|fim_prefix|>\t­
字😀🏽ß<,'re…\u000b.Ａ𐞁'M", "tokens": 77, "pieces": ["'ll", "-ḋ", "̣", " \r", "'ſ", "d", "!!", " ", "३", ",𐞁", "½", "ſ", "　e", "́ḋ", "̣sḋ", "̣é", "'ſ", "!", "\t", "<|", "fim", "_prefix", "|>", "\t", "­", "
字", "😀🏽", "ß", "<,'", "re", "…", "\u000b", ".Ａ𐞁", "'M"]} +{"text": "🙂s\n\r­'re EOT!漢🙂漢'reß'D<.'ſ　Ⅳ'T9\n's(m're12345678>Ａ<|endoftext|>#$%", "tokens": 52, "pieces": ["🙂s", "\n\r", "­'", "re", " EOT", "!漢", "🙂<", "EOT", ">漢", "'re", "ß", "'D", "<.'", "ſ", "　", "Ⅳ", "'T", "9", "\n", "'s", "(m", "'re", "123", "456", "78", ">Ａ", "<|", "endoftext", "|>#$%"]} +{"text": "fiİ​ſ'VEſ0-0 \n, 9<|fim_prefix|>漢½", "tokens": 30, "pieces": ["fiİ", "​ſ", "'VE", "ſ", "0", "-", "0", " \n", ",", " ", "9", "<|", "fim", "_prefix", "|>", "漢", "½"]} +{"text": "\r\n-\rs३\t́'T#$%\r\ne​½'ll<|fim_prefix|>", "tokens": 27, "pieces": ["\r\n", "-\r", "s", "३", "\t", "́'", "T", "#$%<", "META", "_START", ">\r\n", "e", "​", "½", "'ll", "<|", "fim", "_prefix", "|>"]} +{"text": "漢\"9'ſ're", "tokens": 8, "pieces": ["漢", "\"", "9", "'ſ", "'re"]} +{"text": "(Zs\u000b 
.<'VE\r\n\r\n㍿🙂㍿‍\t(🙂½\r're
 😀🏽🙂!Z \n😀🏽", "tokens": 48, "pieces": ["(Zs", "\u000b ", "
", ".<'", "VE", "\r\n\r\n", "㍿🙂㍿‍", "\t", "(🙂", "½", "\r", "'re", "
 ", " 😀🏽🙂!", "Z", "", " \n", "😀🏽"]} +{"text": "é<|fim_prefix|>d", "tokens": 10, "pieces": ["e", "́<|", "fim", "_prefix", "|>", "d"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "12345678Ａ🙂𐞁👍🏽.<|fim_prefix|>\u000b's\u000b!!!'\n\rEOTdꟲsta'S9fi<Ⅳ\u000b ع\r\n\r\n½ 'S", "tokens": 51, "pieces": ["123", "456", "78", "Ａ", "🙂𐞁", "👍🏽.<|", "fim", "_prefix", "|>", "\u000b", "'s", "\u000b", "!!!'\n\r", "EOTdꟲsta", "'S", "9", "fi", "<", "Ⅳ", "\u000b ", " ع", "\r\n\r\n", "½", " ", "'S"]} +{"text": "(­ꟲ'llZ😀🏽́½㋿< \nå\rßå \n", "tokens": 28, "pieces": ["(­", "ꟲ", "'ll", "Z", "😀🏽́", "½", "㋿<", " \n", "a", "̊\r", "ßa", "̊", " \n"]} +{"text": "!!#$%Ⅳḍ̇漢'llfißeDž'D　‍㍿.(字<|fim_prefix|>ee👍🏽12345678eDžé'T!'VE㋿ m", "tokens": 61, "pieces": ["!!#$%", "Ⅳ", "ḋ", "̣漢", "'ll", "fißeDž", "'D", "　", "‍㍿.(", "字", "<|", "fim", "_prefix", "|>", "ee", "👍🏽", "123", "456", "78", "eDže", "́'", "T", "!'", "VE", "㋿", " ", " m"]} +{"text": "\r'😀🏽'Re　 İ'ſ\"🙂　𐞁\r\n\t#$%　­.a!\t　 \n\u000bſ", "tokens": 38, "pieces": ["\r", "'😀🏽'", "Re", "　", " İ", "'ſ", "\"🙂", "　𐞁", "\r\n", "\t", "#$%", "　", "­.", "a", "!", "\t　 \n", "\u000bſ"]} +{"text": "ßعe're9'sßſ漢'VE'VE½\"👍🏽'ReZ😀🏽'VE12345678ḍ̇👍🏽
㋿Z…\t (\r\n'reⅣ'T'12345678'S
", "tokens": 69, "pieces": ["ßعe", "'re", "9", "'s", "ßſ漢", "'VE", "'VE", "½", "\"👍🏽'", "ReZ", "😀🏽'", "VE", "123", "456", "78", "ḋ", "̣👍🏽", "
", "㋿Z", "…\t", " ", "(\r\n", "'re", "Ⅳ", "'T", "'", "123", "456", "78", "'S", "
"]} +{"text": "ḍ̇
ꟲ'T'VE
'ſ.Dž\r\n'…𐞁 !'D\r\n\r\n'Mt字 ſ​🙂,­s\r\n\r\nꟲ'Mt(㋿½३", "tokens": 60, "pieces": ["ḋ", "̣", "
ꟲ", "'T", "'VE", "
", "'ſ", ".", "Dž", "\r\n", "'", "…𐞁", " ", "!'", "D", "\r\n\r\n", "'M", "t字", " ſ", "​🙂,­", "s", "\r\n\r\n", "ꟲ", "'M", "t", "(㋿", "½३"]} +{"text": "åＡⅣ🙂ßfi'", "tokens": 13, "pieces": ["a", "̊Ａ", "Ⅳ", "🙂ßfi", "'"]} +{"text": "'s(\nd👍🏽'll Ａ(\n.", "tokens": 15, "pieces": ["'s", "(\n", "d", "👍🏽'", "ll", " Ａ", "(\n", "."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "#$%😀🏽𐞁Ａ\r\n\"'re0‍s㍿­㋿'s
0½­fi\r\n>́", "tokens": 42, "pieces": ["#$%😀🏽", "𐞁Ａ", "\r\n", "\"'", "re", "0", "‍s", "㍿­㋿'", "s", "
", "0½", "­fi", "\r\n", ">́"]} +{"text": "'se'M!,‍eع\r\n,9", "tokens": 11, "pieces": ["'s", "e", "'M", "!,‍", "eع", "\r\n", ",", "9"]} +{"text": "<|fim_prefix|>,EOT<…-dfie'reZå 912345678>　'Tfi'D​٣٤٥٦!!🙂EOT\r\n\r\n'ſfi­\r٣٤٥٦‍'re㍿", "tokens": 69, "pieces": ["<|", "fim", "_prefix", "|>,", "EOT", "<", "…", "-dfie", "'re", "Za", "̊", " ", " ", "912", "345", "678", ">", "　", "'T", "fi", "'D", "​", "٣٤٥", "٦", "!!🙂", "EOT", "\r\n\r\n", "'ſ", "fi", "­\r", "٣٤٥", "٦", "‍'", "re", "㍿"]} +{"text": "ع!'Dß", "tokens": 8, "pieces": ["ع", "!'", "Dß", ""]} +{"text": "漢tZ<'ReDž-㍿('Re0'M\u000b-fi½\r\n!!Z", "tokens": 24, "pieces": ["漢tZ", "<'", "ReDž", "-㍿('", "Re", "0", "'M", "\u000b", "-fi", "½", "\r\n", "!!", "Z"]} +{"text": "'s'ſs,　-\r'séZDž<|fim_prefix|> \ns३a
👍🏽½!👍🏽's
!!\r㋿tİꟲ \nsåꟲ\u000b", "tokens": 64, "pieces": ["'s", "'ſ", "s", ",", "　", "-\r", "'s", "éZDž", "<|", "fim", "_prefix", "|>", " \n", "s", "३", "a", "
", "👍🏽", "½", "!👍🏽'", "s", "
", "!!\r", "㋿tİꟲ", " \n", "sa", "̊ꟲ", "\u000b"]} +{"text": "t‍!!fi 'VE", "tokens": 9, "pieces": ["t", "‍!!", "fi", " ", "'VE"]} +{"text": "㋿ \"'s​EOT#$%\ne", "tokens": 31, "pieces": ["'re", "عfi", ".­", "eعe", "́", "\tDžＡ", "'re", "Dž", "'S", "e", "!<", "EOT", ">'", "s", "​EOT", "#$%\n", "e"]} +{"text": "½Dž \n½,e'D٣٤٥٦𐞁\r\n\r\ne
<|endoftext|>\u000bſ 0<('Tfi", "tokens": 66, "pieces": ["½", "Dž", " \n", "½", ",e", "'D", "٣٤٥", "٦", "𐞁", "\r\n\r\n", "e", "
", "<|", "endoftext", "|>", "\u000bſ", " ", "0", "<('", "Tfi"]} +{"text": "​Ａ　​٣٤٥٦́a're'VE‍ ㍿å0'VÉ9're!!'D'S0\"!½'Re", "tokens": 43, "pieces": ["​Ａ", "　", "​", "٣٤٥", "٦", "́a", "'re", "'VE", "‍", " ", " ㍿", "a", "̊", "0", "'VE", "́", "9", "'re", "!!'", "D", "'S", "0", "\"!", "½", "'Re"]} +{"text": "'S½漢́ \r\n'S.,ḍ̇\"'SZ t<|fim_prefix|>😀🏽e'T㋿dé㋿Ⅳ9'ſ字fi!", "tokens": 54, "pieces": ["'S", "½", "漢", "́", " \r\n", "'S", ".,", "ḋ", "̣\"'", "SZ", " ", " t", "<|", "fim", "_prefix", "|>😀🏽", "e", "'T", "㋿dé", "㋿", "Ⅳ9", "'", "ſ字fi", "!"]} +{"text": "🙂fi‍',
é​ \n12345678𐞁…'VEＡ'Re", "tokens": 27, "pieces": ["🙂fi", "‍',", "
e", "́​", " \n", "123", "456", "78", "𐞁", "…", "'VE", "Ａ", "'Re"]} +{"text": "'Re漢fi\"12345678'ſ'ſ<|endoftext|>㍿𐞁३字9\u000b\" ſ漢m'VEİ<|endoftext|>,'llé 9DžDž٣٤٥٦\n", "tokens": 71, "pieces": ["'Re", "漢fi", "\"", "123", "456", "78", "'ſ", "'ſ", "<|", "endoftext", "|>㍿", "𐞁", "३", "字", "9", "\u000b", "\"", " ſ漢m", "'VE", "İ", "<|", "endoftext", "|>,'", "llé", " ", "9", "DžDž", "٣٤٥", "٦", "\n"]} +{"text": "عꟲ<|endoftext|>é\r\nİDž'Re​ꟲ>-漢字ع́!!ß'ree \n'👍🏽­İß𐞁٣٤٥٦", "tokens": 54, "pieces": ["عꟲ", "<|", "endoftext", "|>", "é", "\r\n", "İDž", "'Re", "​ꟲ", ">-", "漢字ع", "́!!", "ß", "'re", "e", " \n", "'👍🏽­", "İß𐞁", "٣٤٥", "٦"]} +{"text": "…́㍿'Re 'llt ३ \nmZ''re'ſ", "tokens": 25, "pieces": ["…", "́㍿'", "Re", " '", "llt", " ", "३", " \n", "mZ", "'<", "EOT", ">'", "re", "'ſ"]} +{"text": " ㍿Dž <|fim_prefix|>m's٣٤٥٦-éDž ع‍fi\"٣٤٥٦'Re'𐞁𐞁,.\r\n'Re漢", "tokens": 55, "pieces": [" ", "㍿Dž", " <|", "fim", "_prefix", "|>", "m", "'s", "٣٤٥", "٦", "-éDž", " ع", "‍fi", "\"", "٣٤٥", "٦", "'Re", "'𐞁𐞁", ",.\r\n", "'Re", "漢"]} +{"text": "'ReEOT'llꟲDžé0\tfi\"🙂'S'ſ'DZés'll' tḍ̇.", "tokens": 39, "pieces": ["'Re", "EOT", "'ll", "ꟲDžé", "0", "\tfi", "\"🙂<", "EOT", ">'", "S", "'ſ", "'D", "Ze", "́s", "'ll", "'", " tḋ", "̣."]} +{"text": "\u000b<-<|fim_prefix|>#$%ſ'D'ſEOTＡḍ̇
12345678!'Re'ſİ'VE're's'll'T0'S'Re\r\n<'re,>٣٤٥٦😀🏽½½\"\t…ع", "tokens": 71, "pieces": ["\u000b", "<-<|", "fim", "_prefix", "|>#$%", "ſ", "'D", "'ſ", "EOTＡḋ", "̣", "
", "123", "456", "78", "!'", "Re", "'ſ", "İ", "'VE", "'re", "'s", "'ll", "'T", "0", "'S", "'Re", "\r\n", "<'", "re", ",>", "٣٤٥", "٦", "😀🏽", "½½", "\"", "\t", "…ع"]} +{"text": "9 9İ३ſ\n'ree<|fim_prefix|>'llḍ̇…𐞁9m>ع'D!!\nse‍​å12345678漢'ſ
.Ⅳ", "tokens": 56, "pieces": ["9", " ", "9", "İ", "३", "ſ", "\n", "'re", "e", "<|", "fim", "_prefix", "|>'", "llḋ", "̣", "…𐞁", "9", "m", ">ع", "'D", "!!\n", "se", "‍​", "a", "̊", "123", "456", "78", "漢", "'ſ", "
", ".", "Ⅳ"]} +{"text": "\n99İ​a'Re漢　éſ\r\n\r\nDž", "tokens": 15, "pieces": ["\n", "99", "İ", "​a", "'Re", "漢", "　éſ", "\r\n\r\n", "Dž"]} +{"text": "́\"\r\n\r\n'VE.́Z٣٤٥٦😀🏽a<|endoftext|>\"'ſfié 　'S\r\n\r\n's12345678'T‍eDžéع", "tokens": 56, "pieces": ["́\"\r\n\r\n", "'VE", ".́", "Z", "٣٤٥", "٦", "😀🏽", "a", "<|", "endoftext", "|>\"'", "ſfié", " ", " <", "META", "_START", ">", " ", "　", "'S", "\r\n\r\n", "'s", "123", "456", "78", "'T", "‍eDže", "́ع"]} +{"text": "٣٤٥٦<|fim_prefix|>''s㍿Ⅳ́  \n३ſ🙂\r\n\r\n\n㍿'s\r\nå", "tokens": 42, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>''", "s", "㍿", "Ⅳ", "́", "  \n", "३", "ſ", "🙂\r\n\r\n\n", "㍿'", "s", "\r\n", "a", "̊"]} +{"text": "!३ \n", "tokens": 4, "pieces": ["!", "३", " \n"]} +{"text": "𐞁'Z㋿㋿Ⅳ‍!'Re 　", "tokens": 20, "pieces": ["𐞁", "'Z", "㋿㋿", "Ⅳ", "‍!'", "Re", " 　"]} +{"text": "-
", "tokens": 3, "pieces": ["-", "
"]} +{"text": "‍­ꟲ,<㋿'s🙂", "tokens": 14, "pieces": ["‍­", "ꟲ", ",<㋿'", "s", "🙂"]} +{"text": "\rİⅣ漢0\"​éétḍ̇\"İ\u000b!!'reⅣ　12345678\t㋿0<|endoftext|>12345678…́\r\n\u000bſ\"!éⅣ-", "tokens": 58, "pieces": ["\r", "İ", "Ⅳ", "漢", "0", "\"​", "ée", "́tḋ", "̣\"", "İ", "\u000b", "!!'", "re", "Ⅳ", "　", "123", "456", "78", "\t", "㋿", "0", "<|", "endoftext", "|>", "123", "456", "78", "…", "́\r\n", "\u000bſ", "\"!", "e", "́", "Ⅳ", "-"]} +{"text": "a<|fim_prefix|>", "tokens": 8, "pieces": ["a", "<|", "fim", "_prefix", "|>"]} +{"text": "é>Ź👍🏽­'T‍ …ß<İ'st\r\né­㋿\r\n😀🏽Ⅳfie\u000b", "tokens": 45, "pieces": ["é", ">Z", "́👍🏽­'", "T", "‍", " ", "…ß", "<İ", "'s", "t", "\r\n", "e", "́­㋿\r\n", "😀🏽", "Ⅳ", "fie", "\u000b"]} +{"text": "s\"å!Ⅳİe\r\n\r\n𐞁9\r㋿½\n'M-é ­\u000b#$%", "tokens": 46, "pieces": ["s", "\"a", "̊!", "Ⅳ", "İe", "\r\n\r\n", "𐞁", "9", "\r", "㋿", "½", "\n", "'M", "-é", " ", " <", "s漢", "'S", " ", "'s", "­", " \n", "'M", "Džꟲ", "­>­", "\u000b", "#$%"]} +{"text": "…'re\n\r9Ⅳ \n㋿åå\u000b's'Tꟲ­ꟲ\n'T\r\n\r\n12345678 😀🏽's'll12345678e½", "tokens": 50, "pieces": ["…", "'re", "\n\r", "9Ⅳ", " \n", "㋿a", "̊a", "̊", "\u000b", "'s", "'T", "ꟲ", "­ꟲ", "\n", "'T", "\r\n\r\n", "123", "456", "78", " ", " 😀🏽<", "META", "_START", ">'", "s", "'ll", "123", "456", "78", "e", "½"]} +{"text": "字漢\u000b‍fit'VE\r'S'😀🏽ع…\t", "tokens": 23, "pieces": ["字漢", "\u000b", "‍fit", "'VE", "\r", "'S", "'😀🏽", "ع", "…\t"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": ".0३'MＡ३ḍ̇\r<|endoftext|>12345678
㍿t", "tokens": 31, "pieces": [".", "0३", "'M", "Ａ", "३", "ḋ", "̣\r", "<|", "endoftext", "|>", "123", "456", "78", "
", "㍿t"]} +{"text": "\r\n\r\n'll㍿>\n字é<|endoftext|>\t\t12345678>-Dž", "tokens": 24, "pieces": ["\r\n\r\n", "'ll", "㍿>\n", "字e", "́<|", "endoftext", "|>", "\t", "\t", "123", "456", "78", ">-", "Dž"]} +{"text": "­ …!ḍ̇m𐞁'Tعſ\r\n\r\nſ́ 0m", "tokens": 33, "pieces": ["­", " ", "…", "!ḋ", "̣m𐞁", "'T", "عſ", "\r\n\r\n", "ſ", "́<", "EOT", ">", " ", "0", "m"]} +{"text": "字\"e9ع'M­tſ👍🏽>ßDž­", "tokens": 20, "pieces": ["字", "\"e", "9", "ع", "'M", "­tſ", "👍🏽>", "ßDž", "­"]} +{"text": "ع…Ａ'D🙂'reEOT#$%! 'M'VE ٣٤٥٦'T‍-👍🏽#$%'Re\n\n 字", "tokens": 48, "pieces": ["ع", "…Ａ", "'D", "🙂'", "reEOT", "#$%!<", "META", "_START", ">", " ", " '", "M", "'VE", " ", "٣٤٥", "٦", "'T", "‍-👍🏽#$%'", "Re", "\n\n", " ", " 字"]} +{"text": "'Tt𐞁ḍ̇ḍ̇३عEOT'Sſ'd.s​t🙂ݽ‍漢t<|fim_prefix|>", "tokens": 44, "pieces": ["'T", "t𐞁ḋ", "̣ḋ", "̣", "३", "عEOT", "'S", "ſ", "'d", ".s", "​t", "🙂İ", "½", "‍漢t", "<|", "fim", "_prefix", "|>"]} +{"text": " 😀🏽é'll!३'VEZs\r\n\r\n\t\n<|endoftext|>\n\r're", "tokens": 24, "pieces": [" 😀🏽", "é", "'ll", "!", "३", "'VE", "Zs", "\r\n\r\n\t\n", "<|", "endoftext", "|>\n\r", "'re"]} +{"text": "mdİa½a'VE-d​#$% ३", "tokens": 15, "pieces": ["mdİa", "½", "a", "'VE", "-d", "​#$%", " ", " ", "३"]} +{"text": "\u000b", "tokens": 1, "pieces": ["\u000b"]} +{"text": "😀🏽'VE㍿.--\u000b're㍿eé'S<|fim_prefix|>🙂𐞁'S-Ａ́İ<|endoftext|>\u000b12345678　Z'Ms\r\n0ſ字", "tokens": 58, "pieces": ["😀🏽'", "VE", "㍿.--", "\u000b", "'re", "㍿eé", "'S", "<|", "fim", "_prefix", "|>🙂", "𐞁", "'S", "-Ａ", "́İ", "<|", "endoftext", "|>", "\u000b", "123", "456", "78", "　Z", "'M", "s", "\r\n", "0", "ſ字"]} +{"text": "'D́عe
ꟲt'Re\"­.!!‍‍é
#$%ḍ̇#$%ꟲ>s漢e…mEOTꟲ…", "tokens": 48, "pieces": ["'D", "́عe", "
ꟲt", "'Re", "\"­.!!‍‍", "é", "
", "#$%", "ḋ", "̣#$%", "ꟲ", ">s漢e", "…mEOTꟲ", "…"]} +{"text": "a0­\r\n\r\n (s(字mع t'ſ'T!!", "tokens": 18, "pieces": ["a", "0", "­\r\n\r\n", " ", " (", "s", "(字mع", " t", "'ſ", "'T", "!!"]} +{"text": "‍'sꟲßs!!٣٤٥٦0ꟲ!!́㍿ſ'D\r\n½!!\r\n é\t😀🏽́́'S
…fiⅣ", "tokens": 58, "pieces": ["‍'", "sꟲßs", "!!", "٣٤٥", "٦0", "ꟲ", "!!́㍿", "ſ", "'", "D", "\r\n", "½", "!!\r\n", " ", " e", "́", "\t", "😀🏽́́'", "S", "
", "…fi", "Ⅳ"]} +{"text": "\r\n\r\n<|fim_prefix|>\"‍-''", "tokens": 12, "pieces": ["\r\n\r\n", "<|", "fim", "_prefix", "|>\"‍-''"]} +{"text": " <|fim_prefix|>!!0ſ'Re'ſ<|endoftext|> \n\n ㋿'ſ'Reå'll\r\n9३\"-字.d<Ⅳå", "tokens": 51, "pieces": [" ", "<|", "fim", "_prefix", "|>!!", "0", "ſ", "'Re", "'ſ", "<|", "endoftext", "|>", " \n\n", " ", " ㋿'", "ſ", "'Re", "a", "̊'", "ll", "\r\n", "9३", "\"-", "字", ".d", "<", "Ⅳ", "a", "̊"]} +{"text": "(Ａ <|fim_prefix|>🙂'll,漢 å㋿'VE9'VEDž.𐞁​Ａ\"\r'se", "tokens": 58, "pieces": ["(Ａ", "", " ", "<|", "fim", "_prefix", "|><", "META", "_START", ">🙂<", "EOT", ">'", "ll", ",漢", " ", " a", "̊㋿'", "VE", "9", "'VE", "Dž", ".𐞁", "​<", "META", "_START", ">Ａ", "\"\r", "'s", "e"]} +{"text": "e字'ſ㋿'ſſḍ̇Ⅳt字ſ'Ś…😀🏽\u000b\nİa\n​🙂'M,٣٤٥٦fi \u000b漢", "tokens": 64, "pieces": ["e字", "'ſ", "㋿'", "ſſḋ", "̣", "Ⅳ", "t字", "ſ", "'S", "́", "…", "😀🏽", "\u000b\n", "İa", "\n", "​🙂'", "M", ",", "٣٤٥", "٦", "fi", " ", "\u000b漢"]} +{"text": " '\u000bḍ̇'llß👍🏽­㋿Ⅳſ𐞁漢½漢­३#$%(😀🏽\"½ſⅣDž'VE'VEé a", "tokens": 63, "pieces": [" ", "'", "\u000bḋ", "̣'", "llß", "👍🏽­㋿", "Ⅳ", "ſ", "𐞁漢", "½", "漢", "­", "३", "#$%(😀🏽\"", "½", "ſ", "Ⅳ", "Dž", "'VE", "'VE", "é", " ", " a"]} +{"text": "<|fim_prefix|>!!ß're'reDž#$%12345678\rmå㋿'D\nḍ̇-12345678fi㍿'Re​12345678é\t", "tokens": 50, "pieces": ["<|", "fim", "_prefix", "|>!!", "ß", "'re", "'re", "Dž", "#$%", "123", "456", "78", "\r", "ma", "̊㋿'", "D", "\n", "ḋ", "̣-", "123", "456", "78", "fi", "㍿'", "Re", "​", "123", "456", "78", "é", "\t"]} +{"text": "fi㋿s'ſ́'T'<|fim_prefix|>-İe'll Dž߅'Re ㍿EOT #$%㍿…\u000bZ'VE'S
ſ\r\n\r\n (", "tokens": 60, "pieces": ["fi", "㋿s", "'ſ", "́'", "T", "'<|", "fim", "_prefix", "|>-<", "META", "_START", ">İe", "'ll", " Džß", "…", "'Re", " ", " ㍿", "EOT", " ", " #$%㍿", "…", "\u000bZ", "'VE", "'S", "
ſ", "\r\n\r\n", " ", "("]} +{"text": "　🙂#$%#$%9'VE!!'VE…eDž㋿d\"'SEOT 漢're
'#$%!'Ret0\u000b'S　EOT字åعa912345678\r\n\r\n\u000béDž<|fim_prefix|>", "tokens": 27, "pieces": ["e", "́<", "META", "_START", "><", "META", "_START", ">a", "912", "345", "678", "\r\n\r\n", "\u000bé", "Dž", "<|", "fim", "_prefix", "|>"]} +{"text": "👍🏽\r0s३🙂漢ée \n𐞁DžsⅣ(\n'ſ'll½,ſ<|fim_prefix|>İ́>ḍ̇<👍🏽!!'ll'S٣٤٥٦字'VE>", "tokens": 73, "pieces": ["👍🏽\r", "0", "s", "३", "🙂漢ée", " \n", "𐞁Džs", "Ⅳ", "(\n", "'ſ", "'ll", "½", ",ſ", "<|", "fim", "_prefix", "|>", "İ", "́>", "ḋ", "̣<👍🏽!!'", "ll", "'S", "٣٤٥", "٦", "字", "'VE", ">"]} +{"text": "\n!m- \n'll'Tt'VE…Dž👍🏽ع३'ſß\r\nſ å漢漢\u000bＡ! ́㋿0ع", "tokens": 55, "pieces": ["\n", "!m", "-", " \n", "'", "ll", "'T", "t", "'VE", "…Dž", "👍🏽", "ع", "३", "'ſ", "ß", "\r\n", "ſ", " a", "̊漢漢", "\u000bＡ", "!", " ", " ́㋿", "0", "ع"]} +{"text": " 'T!!'sDž", "tokens": 7, "pieces": [" '", "T", "!!'", "sDž"]} +{"text": "…\u000b's Ⅳß", "tokens": 11, "pieces": ["…", "\u000b", "'s", "", " ", "Ⅳ", "ß"]} +{"text": "#$%Ⅳ\r\n9 Ⅳ0'll𐞁
,'M'll‍'Dḍ̇Dž㍿٣٤٥٦ \n३,Z🙂é\r\n\r\ns", "tokens": 55, "pieces": ["#$%", "Ⅳ", "\r\n", "9", " ", " <", "META", "_START", ">", "Ⅳ0", "'ll", "𐞁", "
", ",'", "M", "'ll", "‍'", "Dḋ", "̣Dž", "㍿", "٣٤٥", "٦", " \n", "३", ",Z", "🙂é", "\r\n\r\n", "s"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿'ll<|fim_prefix|>s're're(<'S'Re㍿ Ａ​'D", "tokens": 30, "pieces": ["㍿'", "ll", "<|", "fim", "_prefix", "|>", "s", "'re", "'re", "(<'", "S", "'Re", "㍿", " Ａ", "​'", "D"]} +{"text": " 'S('ſ……'D'll🙂'٣٤٥٦­🙂<|endoftext|>​t　<|fim_prefix|>> #$%‍ßⅣ're\r
9३'D'Mİ́漢s", "tokens": 64, "pieces": [" '", "S", "('", "ſ", "…", "…", "'D", "'ll", "🙂'", "٣٤٥", "٦", "­🙂<|", "endoftext", "|>​", "t", "　", "<|", "fim", "_prefix", "|>>", " ", "#$%‍", "ß", "Ⅳ", "'re", "\r", "
", "9३", "'D", "'M", "İ", "́漢s"]} +{"text": "👍🏽s​<|endoftext|>'S \nm!!å٣٤٥٦\"fi", "tokens": 36, "pieces": ["👍🏽", "s", "​<|", "endoftext", "|>'", "S", " \n", "m", "!!", "a", "̊", "٣٤٥", "٦", "\"<", "META", "_START", ">fi"]} +{"text": "́'Réd-𐞁\t㍿mdé'́
é३", "tokens": 24, "pieces": ["́'", "Re", "́d", "-𐞁", "\t", "㍿mdé", "'́", "
e", "́", "३"]} +{"text": "ß​'D<'D\"🙂fi 'M…é‍ \n12345678𐞁字㍿ \n9ß😀🏽té0fi३ḍ̇'VE", "tokens": 56, "pieces": ["ß", "​'", "D", "<'", "D", "\"🙂", "fi", " ", " '", "M", "…é", "‍", " \n", "123", "456", "78", "𐞁", "字", "㍿", " \n", "9", "ß", "😀🏽", "te", "́", "0", "fi", "३", "ḋ", "̣'", "VE"]} +{"text": "½-ع12345678\n漢é'T'S\u000b𐞁dåå🙂عİ\r 
'ſḍ̇.'VE
　", "tokens": 46, "pieces": ["½", "-ع", "123", "456", "78", "\n", "漢e", "́'", "T", "'S", "\u000b𐞁da", "̊a", "̊🙂", "عİ", "\r", " ", "
", "'ſ", "ḋ", "̣.'", "VE", "
　"]} +{"text": ".😀🏽'Sm're'ſ<|fim_prefix|>Ａḍ̇.­.0'llⅣ<ß.<|endoftext|>漢𐞁😀🏽'res\u000b३‍", "tokens": 60, "pieces": [".😀🏽'", "Sm", "'re", "'ſ", "<|", "fim", "_prefix", "|>", "Ａḋ", "̣.­.", "0", "'ll", "Ⅳ", "<ß", ".<|", "endoftext", "|>", "漢𐞁", "😀🏽'", "res", "\u000b", "३", "‍"]} +{"text": "Z\u000b", "tokens": 2, "pieces": ["Z", "\u000b"]} +{"text": "s­\r\n'Mm…EOTa'VE😀🏽- \n…𐞁<|endoftext|>é'Tm㍿ 'Ret<|fim_prefix|>12345678'T३🙂!é'llEOT㋿s'VE\r\n\r\n​", "tokens": 72, "pieces": ["s", "­\r\n", "'M", "m", "…EOT", "a", "'VE", "😀🏽-", " \n", "…𐞁", "<|", "endoftext", "|>", "e", "́'", "Tm", "㍿", " '", "Ret", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "३", "🙂!", "é", "'ll", "EOT", "㋿s", "'VE", "\r\n\r\n", "​"]} +{"text": "字　0,\u000bZé('VE's\"\r\n\r\nꟲ漢'ReDž\nfiع㍿12345678(㍿", "tokens": 36, "pieces": ["字", "　", "0", ",", "\u000bZe", "́('", "VE", "'s", "\"\r\n\r\n", "ꟲ漢", "'", "ReDž", "\n", "fiع", "㍿", "123", "456", "78", "(㍿"]} +{"text": "'Ms-09ḍ̇DžⅣ d0'<ſß \n", "tokens": 25, "pieces": ["'M", "s", "-", "09", "ḋ", "̣Dž", "Ⅳ", " ", " d", "0", "'<<", "META", "_START", ">ſß", " \n"]} +{"text": "'ſ𐞁\r're\r\n\r\n\n", "tokens": 16, "pieces": ["'ſ", "𐞁", "\r", "'", "re", "\r\n\r\n\n"]} +{"text": "'VE eéＡ'Sa😀🏽
Ⅳ-'M𐞁,　 >'VEtd½\t'Mḍ̇‍\r\n\r\n !ꟲ \t३", "tokens": 53, "pieces": ["'VE", " e", "e", "́Ａ", "'S", "a", "😀🏽", "
", "Ⅳ", "-'", "M𐞁", ",", "　", " ", ">'", "VEtd", "½", "\t", "'M", "ḋ", "̣‍\r\n\r\n", " ", " !", "ꟲ", " ", "\t", "३"]} +{"text": "İ'S0
", "tokens": 5, "pieces": ["İ", "'S", "0", "
"]} +{"text": "<|fim_prefix|><|fim_prefix|>ß 's'Sm😀🏽fi'Tḍ̇\u000bs\"‍'ſ \nmEOT‍", "tokens": 49, "pieces": ["<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "ß", " ", " '", "s", "'S", "m", "😀🏽", "fi", "'T", "ḋ", "̣", "\u000bs", "\"‍'", "ſ", " \n", "mEOT", "‍"]} +{"text": "'Reḍ̇'s!𐞁d'VEſ👍🏽㍿a('St'ſ㋿𐞁  <|fim_prefix|>\r\n½Dž!!字<|endoftext|>'re字12345678'Reaa aEOT", "tokens": 72, "pieces": ["'Re", "ḋ", "̣'", "s", "!𐞁d", "'VE", "ſ", "👍🏽㍿", "a", "('", "St", "'ſ", "㋿𐞁", " ", " ", "<|", "fim", "_prefix", "|>\r\n", "½", "Dž", "!!", "字", "<|", "endoftext", "|>'", "re字", "123", "456", "78", "'Re", "aa", " ", " aEOT"]} +{"text": "'VE­s\"'M\r\n\r\nZ\r\n\r\n'VE'D9ꟲ", "tokens": 23, "pieces": ["'VE", "­s", "\"'", "M", "\r\n\r\n", "Z", "\r\n\r\n", "'VE", "'D", "9", "ꟲ", ""]} +{"text": ".d🙂", "tokens": 3, "pieces": [".d", "🙂"]} +{"text": "EOTEOTEOTß<|fim_prefix|>㋿ ́fi‍\nİ!! 
½ß", "tokens": 33, "pieces": ["EOTEOTEOTß", "<|", "fim", "_prefix", "|>㋿", " ", " ́", "fi", "‍\n", "İ", "!!", " ", "
", "½", "ß"]} +{"text": "'sdDžsſ ㍿ \n🙂\t!'T\u000bse.‍\n<|fim_prefix|>🙂ſ㍿ ­'D'S'Re٣٤٥٦Ⅳ'S", "tokens": 55, "pieces": ["'s", "dDžsſ", " ", " ㍿", " \n", "🙂", "\t", "!'", "T", "\u000bse", ".‍\n", "<|", "fim", "_prefix", "|>🙂", "ſ", "㍿", " ", " ­'", "D", "'S", "'Re", "٣٤٥", "٦Ⅳ", "'S"]} +{"text": "'ll😀🏽eEOT. \n \r'sعع́\u000b\rⅣé", "tokens": 23, "pieces": ["'ll", "😀🏽", "eEOT", ".", " \n \r", "'s", "عع", "́", "\u000b\r", "Ⅳ", "e", "́"]} +{"text": "<|endoftext|>😀🏽\r\n!İ\t \n'T'#$%m ­😀🏽m9<mꟲå\t \ne\n㍿", "tokens": 50, "pieces": ["<|", "endoftext", "|>😀🏽\r\n", "!İ", "\t \n", "'T", "'#$%", "m", " ", " ­😀🏽", "m", "9", "<<", "EOT", ">mꟲa", "̊", "\t \n", "e", "\n", "㍿"]} +{"text": "'reع\tfi''MEOT㍿9\r\né'T‍eع\r\n\r\n😀🏽<|fim_prefix|>\r\n>​\n\"", "tokens": 43, "pieces": ["'re", "ع", "\tfi", "''", "MEOT", "㍿", "9", "\r\n", "é", "'T", "‍eع", "\r\n\r\n", "😀🏽<|", "fim", "_prefix", "|><", "EOT", ">\r\n", ">​\n", "\""]} +{"text": "'ſ>\t", "tokens": 5, "pieces": ["'ſ", ">", "\t"]} +{"text": "'ſ\n𐞁'Re,-<|endoftext|>'VEſ…<|fim_prefix|>EOT​'Mſ'S0٣٤٥٦12345678👍🏽'llİ12345678!!\tſé'ſ‍漢'ſå\u000b're'T'D\t㍿½", "tokens": 89, "pieces": ["'ſ", "\n", "𐞁", "'Re", ",-<|", "endoftext", "|>'", "VEſ", "…", "<|", "fim", "_prefix", "|>", "EOT", "​'", "Mſ", "'S", "0٣٤", "٥٦1", "234", "567", "8", "👍🏽'", "llİ", "123", "456", "78", "!!", "\tſé", "'ſ", "‍漢", "'ſ", "a", "̊", "\u000b", "'re", "'T", "'D", "\t", "㍿", "½"]} +{"text": "㋿𐞁é'VE­­-å‍'漢,EOT字Ａ३<|fim_prefix|>​🙂🙂'VE''s…t​'re'St㍿　!!", "tokens": 61, "pieces": ["㋿𐞁é", "'VE", "­­-", "a", "̊<", "EOT", ">‍'", "漢", ",EOT字Ａ", "३", "<|", "fim", "_prefix", "|>​🙂🙂'", "VE", "''", "s", "…t", "​'", "re", "'S", "t", "㍿", "　", "!!"]} +{"text": "'ll字", "tokens": 2, "pieces": ["'ll", "字"]} +{"text": "\t㍿s㋿#$%12345678,e#$%'T'Re٣٤٥٦😀🏽👍🏽s
!", "tokens": 41, "pieces": ["\t", "㍿s", "㋿#$%", "123", "456", "78", ",e", "#$%'", "T", "'Re", "٣٤٥", "٦", "😀🏽👍🏽", "s", "
", "!"]} +{"text": "t'ſ‍\t🙂éd\r㋿😀🏽a\r\n\r\n9　'llＡ'll-(0‍٣٤٥٦­　字#$%(å'ſ
Ⅳ'SⅣa", "tokens": 68, "pieces": ["t", "'ſ", "‍", "\t", "🙂éd", "\r", "㋿😀🏽", "a", "\r\n\r\n", "9", "　", "'ll", "Ａ", "'ll", "-(", "0", "‍", "٣٤٥", "٦", "­", "　字", "#$%<", "META", "_START", ">(", "a", "̊'", "ſ", "
", "Ⅳ", "'S", "Ⅳ", "a"]} +{"text": "'\r\n\r​ḍ̇­a. EOT'VE­\r\n\r\n'Re​㋿ 👍🏽<|fim_prefix|>>㋿>\r\n\r\n\r\n!Dž'VE　'llſ'ré\t'ſ'sḍ̇,", "tokens": 65, "pieces": ["'\r\n\r", "​ḋ", "̣­", "a", ".", " ", " EOT", "'VE", "­\r\n\r\n", "'Re", "​㋿", " ", " 👍🏽<|", "fim", "_prefix", "|>>㋿>\r\n\r\n\r\n", "!Dž", "'", "VE", "　", "'ll", "ſ", "'re", "́", "\t", "'ſ", "'s", "ḋ", "̣,"]} +{"text": "👍🏽ＡéEOT\"Z\ré́\u000bＡعaZ𐞁<|fim_prefix|>9d३", "tokens": 38, "pieces": ["👍🏽", "Ａe", "́EOT", "\"Z", "\r", "é", "́", "\u000bＡعaZ𐞁", "<|", "fim", "_prefix", "|>", "9", "d", "३"]} +{"text": "9‍'T字'reⅣt'Re'Dꟲå0'reꟲé's.ع\"'ll字㍿>", "tokens": 36, "pieces": ["9", "‍'", "T字", "'re", "Ⅳ", "t", "'Re", "'D", "ꟲa", "̊", "0", "'re", "ꟲe", "́'", "s", ".ع", "\"'", "ll字", "㍿>"]} +{"text": "#$%㋿é", "tokens": 6, "pieces": ["#$%㋿", "é"]} +{"text": "é🙂‍'re
å'VE३s.…'T漢.!#$%12345678㋿٣٤٥٦m㋿#$%\r", "tokens": 56, "pieces": ["e", "́🙂‍'", "re", "", "
a", "̊'", "VE", "३", "s", ".", "…", "'T", "漢", ".!#$%", "123", "456", "78", "㋿", "٣٤٥", "٦", "m", "㋿<", "EOT", ">#$%\r"]} +{"text": "👍🏽!!", "tokens": 7, "pieces": ["👍🏽!!"]} +{"text": "'sḍ̇Z\r\n\r\n½İ🙂\r\n\r\n'Tß", "tokens": 33, "pieces": ["ße", "0", "Ａꟲ", "'D", "𐞁", "\u000b", "
", "Ⅳ", "'S", "<'", "Re", "-s", "<|", "fim", "_prefix", "|>🙂\r\n\r\n", "'T", "ß"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'S>éZſ<|endoftext|>\">३-ع#$%Z!'VE", "tokens": 23, "pieces": ["'S", ">éZſ", "<|", "endoftext", "|>\">", "३", "-ع", "#$%", "Z", "!'", "VE"]} +{"text": "d'ſ<\r🙂<|endoftext|>\nǻ#$%ع́ \n\n½\tå'Re‍🙂ḍ̇'S,m👍🏽EOTfi​㍿\u000b😀🏽<|fim_prefix|>", "tokens": 69, "pieces": ["d", "'ſ", "<\r", "🙂<|", "endoftext", "|>\n", "a", "̊́#$%", "ع", "́", " \n\n", "½", "\ta", "̊'", "Re", "‍🙂", "ḋ", "̣'", "S", ",m", "👍🏽", "EOTfi", "​㍿", "\u000b", "😀🏽<|", "fim", "_prefix", "|>"]} +{"text": "e…é'VE9漢é!!<|fim_prefix|>", "tokens": 19, "pieces": ["e", "…é", "'VE", "9", "漢e", "́!!<|", "fim", "_prefix", "|>"]} +{"text": "ḍ̇
\t३", "tokens": 10, "pieces": ["ḋ", "̣", "
", "\t", "३"]} +{"text": "\t 字é㍿İ<|endoftext|>'VEḍ̇Ⅳ", "tokens": 28, "pieces": ["\t", " 字e", "́㍿<", "EOT", ">İ", "<|", "endoftext", "|>'", "VEḋ", "̣", "Ⅳ"]} +{"text": "Ⅳ​😀🏽İ\"'D🙂'T​9d\råe12345678\r\n\r\n \nع's", "tokens": 38, "pieces": ["Ⅳ", "​😀🏽", "İ", "\"'", "D", "🙂'", "T", "​<", "EOT", ">", "9", "d", "\r", "a", "̊e", "123", "456", "78", "\r\n\r\n \n", "ع", "'s"]} +{"text": "<|endoftext|>👍🏽
́\"Z0عZꟲ'Re🙂…<٣٤٥٦'MDž㍿'D½<|fim_prefix|>(ⅣZd́　٣٤٥٦㋿ß'Ma'\r\n'ReⅣ㍿\r\n\r\n.", "tokens": 83, "pieces": ["<|", "endoftext", "|>👍🏽", "
", "́\"", "Z", "0", "عZꟲ", "'Re", "🙂", "…", "<", "٣٤٥", "٦", "'M", "Dž", "㍿'", "D", "½", "<|", "fim", "_prefix", "|>(", "Ⅳ", "Zd", "́", "　", "٣٤٥", "٦", "㋿ß", "'M", "a", "'\r\n", "'Re", "Ⅳ", "㍿\r\n\r\n", "."]} +{"text": "​9ſＡ㋿\r\n\r\n😀🏽Dž字'VE\tİ'reḍ̇#$%<|endoftext|>\"#$%‍\n", "tokens": 45, "pieces": ["​", "9", "ſＡ", "㋿\r\n\r\n", "😀🏽", "Dž字", "'VE", "\tİ", "'re", "ḋ", "̣#$%<|", "endoftext", "|><", "META", "_START", ">\"#$%‍\n"]} +{"text": "(<|fim_prefix|>", "tokens": 7, "pieces": ["(<|", "fim", "_prefix", "|>"]} +{"text": "ſ'ſ'T­Z. \n
<|endoftext|>!!३9m٣٤٥٦9 \r\n𐞁
ßſ's'३'ſ
'VEḍ̇", "tokens": 59, "pieces": ["ſ", "'ſ", "'T", "­Z", ".", " \n", "
", "<|", "endoftext", "|>!!", "३9", "m", "٣٤٥", "٦9", " \r\n", "𐞁", "
ßſ", "'s", "'", "३", "'ſ", "
", "'VE", "ḋ", "̣"]} +{"text": "'s३'S \n ſ!'D­🙂㋿\rḍ̇e#$% \n,ss '", "tokens": 28, "pieces": ["'s", "३", "'S", " \n", " ſ", "!'", "D", "­🙂㋿\r", "ḋ", "̣e", "#$%", " \n", ",ss", " '"]} +{"text": "'VE\u000b\t\t漢\u000b<|endoftext|>åꟲ😀🏽.'sm's!!'ſ́३", "tokens": 36, "pieces": ["'VE", "\u000b\t", "\t漢", "\u000b", "<|", "endoftext", "|>", "a", "̊ꟲ", "😀🏽.'", "sm", "'s", "!!'", "ſ", "́", "३"]} +{"text": ".\"<\t'S>　ſ!!eå👍🏽'M'Re!😀🏽 \n!!…ß", "tokens": 48, "pieces": [".\"<", "\t", "'S", ">", "　ſ", "!!", "ea", "̊👍🏽'", "M", "'Re", "!😀🏽", " \n", "!!", "…ß"]} +{"text": "!!'ſ'ſ\r\n->'ś'D0é0㍿<'ſ'VE漢\"é­", "tokens": 34, "pieces": ["!!<", "EOT", ">'", "ſ", "'ſ", "\r\n", "->'", "s", "́'", "D", "0", "e", "́", "0", "㍿<'", "ſ", "'VE", "漢", "\"é", "­"]} +{"text": "12345678ſ'M\n'T㍿d\t", "tokens": 13, "pieces": ["123", "456", "78", "ſ", "'M", "\n", "'T", "㍿d", "\t"]} +{"text": " ſfi🙂0
½0\u000b
\u000b ‍\r\nⅣ
𐞁12345678'VE😀🏽e'S>fi\r", "tokens": 48, "pieces": [" ", " ſfi", "🙂", "0", "
", "½0", "\u000b
\u000b ", " ‍\r\n", "Ⅳ", "
𐞁", "123", "456", "78", "'VE", "😀🏽", "e", "'S", ">fi", "\r"]} +{"text": "'VEå(ḍ̇0#$%
's Ａꟲ", "tokens": 22, "pieces": ["'VE", "a", "̊(", "ḋ", "̣", "0", "#$%", "
", "'s", " Ａꟲ"]} +{"text": "Dž're
漢ß(\" (é(åfi㍿'VE", "tokens": 27, "pieces": ["Dž", "'re", "
漢", "ß", "(\"", " ", "(e", "́(", "a", "̊fi", "㍿'", "VE"]} +{"text": "<|endoftext|>m​㋿", "tokens": 12, "pieces": ["<|", "endoftext", "|>", "m", "​㋿"]} +{"text": "ḍ̇#$% \nt'e㍿\n \n'", "tokens": 22, "pieces": ["ḋ", "̣#$%", " \n", "t", "'", "e", "㍿\n", " \n", "'"]} +{"text": "­́ع0'''ſsḍ̇", "tokens": 46, "pieces": ["­́<", "META", "_START", ">ع", "0", "'''", "ſsḋ", "̣"]} +{"text": "'T'D'SꟲaDž字字'MaDžmfim ́ß\"
eعm!…m!٣٤٥٦‍s\r\n\r\n0👍🏽Ａ", "tokens": 67, "pieces": ["'T", "'D", "'S", "ꟲaDž字字", "'M", "aDž", "mfim", " <", "META", "_START", ">́", "ß", "\"", "
eعm", "!", "…m", "!", "٣٤٥", "٦", "‍s", "\r\n\r\n", "0", "👍🏽", "Ａ"]} +{"text": "'Reḍ̇ß𐞁Ⅳ'ſ \n'De­ع0ḍ̇.'\r\nå\u000b　", "tokens": 34, "pieces": ["'Re", "ḋ", "̣ß𐞁", "Ⅳ", "'ſ", " \n", "'D", "e", "­ع", "0", "ḋ", "̣.'\r\n", "a", "̊", "\u000b　"]} +{"text": "İ's\nⅣ12345678Džd9\r\n
's<|fim_prefix|>(Dž
'Re'VE'll ß'M‍\n''s漢é३'ſaéⅣḍ̇EOT", "tokens": 62, "pieces": ["İ", "'s", "\n", "Ⅳ12", "345", "678", "Džd", "9", "\r\n", "
", "'s", "<|", "fim", "_prefix", "|>(", "Dž", "
", "'Re", "'VE", "'ll", " ß", "'M", "‍\n", "''", "s漢e", "́", "३", "'ſ", "ae", "́", "Ⅳ", "ḋ", "̣<", "EOT", ">EOT"]} +{"text": "amé'VEſZ𐞁'M🙂.e字", "tokens": 17, "pieces": ["ame", "́'", "VEſZ𐞁", "'M", "🙂.", "e字"]} +{"text": "ꟲ0s.​EOTꟲ\r\n\r\n\r\nع字 0Z　🙂ſ<|fim_prefix|> \nfi👍🏽İ,Ａ\nİt!å‍", "tokens": 52, "pieces": ["ꟲ", "0", "s", ".​", "EOTꟲ", "\r\n\r\n\r\n", "ع字", " ", "0", "Z", "　", "🙂ſ", "<|", "fim", "_prefix", "|>", " \n", "fi", "👍🏽", "İ", ",Ａ", "\n", "İt", "!a", "̊‍"]} +{"text": "'T9Ａ!'D𐞁's
\"٣٤٥٦\t٣٤٥٦'M'T<<'Så३'re­ .EOT­'　eꟲm<|endoftext|>　,'Så", "tokens": 78, "pieces": ["a", "̊", " ", " '", "D", "'re", "\r\n\r\n", "漢", "<|", "fim", "_prefix", "|>'", "s", "
", "\"", "٣٤٥", "٦", "\t", "٣٤٥", "٦", "'M", "'T", "<<'", "Sa", "̊", "३", "'re", "­", " ", ".EOT", "­<", "EOT", ">'", "　eꟲm", "<|", "endoftext", "|>", "　", ",'", "Sa", "̊"]} +{"text": "\r🙂'Tß字#$%\r\n\r\n\t㍿\r\n\r\n.\u000bİ㋿
12345678'T", "tokens": 27, "pieces": ["\r", "🙂'", "Tß字", "#$%\r\n\r\n", "\t", "㍿\r\n\r\n", ".", "\u000bİ", "㋿", "
", "123", "456", "78", "'T"]} +{"text": "'M
 ß!ſ'Red('ll­\n'S!!Ⅳs#$%.'ss'VE­
'''Ḿa're'M'D", "tokens": 43, "pieces": ["'M", "
", " ß", "!ſ", "'Re", "d", "('", "ll", "­\n", "'S", "!!", "Ⅳ", "s", "#$%.'", "s", "s", "'VE", "­", "
", "'''", "M", "́", "a", "'re", "'M", "'D"]} +{"text": "<|endoftext|><字fiŹ\n12345678㍿EOT#$%Džé👍🏽
\n'S…'D'M'ſ​ع\n<‍", "tokens": 49, "pieces": ["<|", "endoftext", "|><", "字fiZ", "́\n", "123", "456", "78", "㍿EOT", "#$%", "Džé", "👍🏽", "
\n", "'S", "…", "'D", "'M", "'ſ", "​ع", "\n", "<‍"]} +{"text": "<|endoftext|>ß\r\nİ#$%9ß'ſ'T½", "tokens": 19, "pieces": ["<|", "endoftext", "|>", "ß", "\r\n", "İ", "#$%", "9", "ß", "'ſ", "'T", "½"]} +{"text": "9,\tſé٣٤٥٦é'reſ'ſZDžsİDž'S-'s漢0!
're<|endoftext|>𐞁's
é٣٤٥٦㍿!!\r\n　's", "tokens": 71, "pieces": ["9", ",", "\tſé", "٣٤٥", "٦", "é", "'re", "ſ", "'ſ", "ZDžsİDž", "'S", "-'", "s漢", "", "0", "!", "
", "'re", "<|", "endoftext", "|>", "𐞁", "'s", "
é", "٣٤٥", "٦", "㍿!!\r\n", "　", "'s"]} +{"text": "½m½́🙂Ze \nt‍́ 
sꟲå‍'Reé", "tokens": 27, "pieces": ["½", "m", "½", "́🙂", "Ze", " \n", "t", "‍́", " ", "
sꟲa", "̊‍'", "Reé"]} +{"text": "12345678 \n'Re漢(.e字're‍#$%'VE!!aEOT😀🏽­'S🙂\"'VE\re漢\r\n\r\neZ", "tokens": 39, "pieces": ["123", "456", "78", " \n", "'Re", "漢", "(.", "e字", "'re", "‍#$%'", "VE", "!!", "aEOT", "😀🏽­'", "S", "🙂\"'", "VE", "\r", "e漢", "\r\n\r\n", "eZ"]} +{"text": "'ll字,", "tokens": 3, "pieces": ["'ll", "字", ","]} +{"text": "s,'M\rſå", "tokens": 9, "pieces": ["s", ",'", "M", "\r", "ſa", "̊"]} +{"text": "s'\"'ſeع …<'Reꟲ'T­é\r\n\r\n", "tokens": 20, "pieces": ["s", "'\"'", "ſeع", " ", "…", "<'", "Reꟲ", "'T", "­e", "́\r\n\r\n"]} +{"text": "😀🏽#$%\r\n…12345678'Re<\".漢m٣٤٥٦عå'ſe\u000bt12345678å", "tokens": 48, "pieces": ["😀🏽#$%\r\n", "…", "", "123", "456", "78", "'Re", "<\".", "漢m", "٣٤٥", "٦", "ع", "a", "̊'", "ſe", "\u000bt", "123", "456", "78", "a", "̊"]} +{"text": "'VE'lla　 \r\n\r\n'llt<", "tokens": 9, "pieces": ["'VE", "'ll", "a", "　 \r\n\r\n", "'ll", "t", "<"]} +{"text": "éé­İ'S‍<", "tokens": 9, "pieces": ["e", "́é", "­İ", "'S", "‍<"]} +{"text": "'ll'st漢EOT​Ａeİ𐞁0éḍ̇EOT­", "tokens": 27, "pieces": ["'ll", "'s", "t漢EOT", "​Ａeİ𐞁", "0", "e", "́ḋ", "̣EOT", "­"]} +{"text": ">\n𐞁sⅣ𐞁#$%t'VE#$%<|endoftext|>-<|endoftext|>ſ­>'D­३d'VE\t👍🏽𐞁tꟲ\n12345678'D#$%ⅣZ'll㍿ 漢a'M", "tokens": 82, "pieces": [">\n", "𐞁s", "Ⅳ", "𐞁", "#$%", "t", "'VE", "#$%<|", "endoftext", "|>-<|", "endoftext", "|>", "ſ", "­>'", "D", "­", "३", "d", "'VE", "\t", "👍🏽", "𐞁tꟲ", "\n", "123", "456", "78", "'D", "#$%", "Ⅳ", "Z", "'ll", "㍿", " 漢a", "'M"]} +{"text": "'Ḿ's \né! ,'T
's \"'D9ع'llDžEOT­Ａ ½漢…", "tokens": 35, "pieces": ["'M", "́'", "s", " \n", "é", "!", " ,'", "T", "
", "'s", " ", "\"'", "D", "9", "ع", "'ll", "DžEOT", "­Ａ", " ", "½", "漢", "…"]} +{"text": "ḍ̇ſe\t \n߅'re're'Sfi'عİ­​​'ll🙂ḍ̇㍿'lle.><|endoftext|>(0,漢ḍ̇½e​\u000b", "tokens": 57, "pieces": ["ḋ", "̣ſe", "\t \n", "ß", "…", "'re", "'re", "'S", "fi", "'عİ", "­​​'", "ll", "🙂ḋ", "̣㍿'", "lle", ".><|", "endoftext", "|>(", "0", ",漢ḋ", "̣", "½", "e", "​", "\u000b"]} +{"text": "३ßZ字0́'Re'T'ſſ12345678३>'remé\r\n­…fit \n!İ….\n'ſ#$% ", "tokens": 42, "pieces": ["३", "ßZ字", "0", "́'", "Re", "'T", "'ſ", "ſ", "123", "456", "78३", ">'", "remé", "\r\n", "­", "…fit", " \n", "!İ", "…", ".\n", "'ſ", "#$%", " "]} +{"text": "'VE字<|endoftext|><|endoftext|>\nＡع㋿", "tokens": 22, "pieces": ["'VE", "字", "<|", "endoftext", "|><|", "endoftext", "|>\n", "Ａع", "㋿"]} +{"text": "!é \nⅣa!­\"!m\r\n#$%'字𐞁'ع㍿<|endoftext|>'se( ꟲḍ̇Džd漢!. 'S!!d", "tokens": 57, "pieces": ["!e", "́", " \n", "Ⅳ", "a", "!­\"!", "m", "\r\n", "#$%'", "字𐞁", "'ع", "㍿<|", "endoftext", "|>'", "se", "(", " ꟲḋ", "̣Džd漢", "!.", " ", " '", "S", "!!", "d", ""]} +{"text": "!'SDžİⅣ\r\n\r\n\r!㋿-é\ntéé🙂İ​\r\nſ👍🏽!‍​㋿
३e‍ a", "tokens": 53, "pieces": ["!'", "SDžİ", "Ⅳ", "\r\n\r\n\r", "!㋿<", "META", "_START", ">-", "é", "\n", "tée", "́🙂", "İ", "​\r\n", "ſ", "👍🏽!‍​㋿", "
", "३", "e", "‍", " a"]} +{"text": "\u000b'S", "tokens": 2, "pieces": ["\u000b", "'S"]} +{"text": "<\"é ſ ­ꟲ Ⅳ🙂'ſ३ع٣٤٥٦aé𐞁'M
'T<|endoftext|>", "tokens": 45, "pieces": ["<\"", "e", "́", " ſ", " ­", "ꟲ", " ", "Ⅳ", "🙂'", "ſ", "३", "ع", "٣٤٥", "٦", "aé𐞁", "'M", "
", "'T", "<|", "endoftext", "|>"]} +{"text": "!.eعt\r\n\r\n\"<|fim_prefix|>fi EOT\r\n\r\nꟲ
'Reḍ̇　sfi,漢mtm\t", "tokens": 45, "pieces": ["!.<", "META", "_START", ">eعt", "\r\n\r\n", "\"<", "EOT", "><|", "fim", "_prefix", "|>", "fi", " EOT", "\r\n\r\n", "ꟲ", "
", "'Re", "ḋ", "̣", "　sfi", ",漢mtm", "\t"]} +{"text": "ßé're912345678t", "tokens": 7, "pieces": ["ßé", "'re", "912", "345", "678", "t"]} +{"text": "İ\"!!", "tokens": 3, "pieces": ["İ", "\"!!"]} +{"text": " .\u000bå🙂t😀🏽<|fim_prefix|>​​,0'(é'ſm'Re", "tokens": 32, "pieces": [" ", ".", "\u000ba", "̊🙂", "t", "😀🏽<|", "fim", "_prefix", "|>​​,", "0", "'(", "e", "́'", "ſm", "'Re"]} +{"text": "Ⅳ'ſİ😀🏽e0𐞁 😀🏽#$%t'S>३9㍿٣٤٥٦'㋿'De'Re'!<|endoftext|>e½​‍ \r\n\r\n'…​ß ' ", "tokens": 72, "pieces": ["Ⅳ", "'ſ", "İ", "😀🏽", "e", "0", "𐞁", " 😀🏽#$%", "t", "'S", ">", "३9", "㍿", "٣٤٥", "٦", "'㋿'", "De", "'Re", "'!<|", "endoftext", "|>", "e", "½", "​‍", " \r\n\r\n", "'<", "EOT", ">", "…", "​ß", " '", " "]} +{"text": "ſ", "tokens": 2, "pieces": ["ſ"]} +{"text": "​३ ('reḍ̇漢ḍ̇m-'𐞁​'D-ꟲ\r\n\r\nEOT字Ⅳ㋿𐞁 \n> \u000b", "tokens": 51, "pieces": ["​", "३", " ", " ('", "reḋ", "̣漢ḋ", "̣m", "-'", "𐞁", "​'", "D", "-ꟲ", "\r\n\r\n", "EOT字", "Ⅳ", "㋿𐞁", " \n", "><", "META", "_START", ">", " \u000b"]} +{"text": "­Ⅳ-ſ! 'sß", "tokens": 13, "pieces": ["­", "Ⅳ", "-", "ſ", "!", " '", "sß"]} +{"text": "漢­­9🙂🙂𐞁😀🏽0 d'TZ", "tokens": 23, "pieces": ["漢", "­­", "9", "🙂🙂", "𐞁", "😀🏽", "0", " ", " d", "'T", "Z"]} +{"text": "İé0<|endoftext|>İ'll🙂́fi　​s-efi<|endoftext|>'ll­́३,", "tokens": 45, "pieces": ["İe", "́", "0", "<|", "endoftext", "|>", "İ", "'ll", "🙂́", "fi", "　", "​s", "-efi", "<|", "endoftext", "|>'", "ll", "­́", "३", ","]} +{"text": "12345678're　漢́é\n0mßḍ̇!!Ze12345678're'VE\tm'M\r\né'll😀🏽Ⅳt½\t👍🏽ḍ̇fi'T<|endoftext|>­㍿Z'Re", "tokens": 64, "pieces": ["\u000b", "́'", "re", "-.‍", "eß", "Ze", "123", "456", "78", "'re", "'VE", "\tm", "'M", "\r\n", "e", "́'", "ll", "😀🏽", "Ⅳ", "t", "½", "\t", "👍🏽", "ḋ", "̣fi", "'T", "<|", "endoftext", "|>­㍿", "Z", "'Re"]} +{"text": "(ع👍🏽ſⅣ'e…t㋿fifi३'T🙂dＡém'ss\r\nꟲ㍿9 \n!…Z\" 're٣٤٥٦'re\u000bt!!", "tokens": 65, "pieces": ["(ع", "👍🏽", "ſ", "Ⅳ", "'e", "…t", "㋿fi", "fi", "३", "'T", "🙂dＡém", "'s", "s", "\r\n", "ꟲ", "㍿", "9", " \n", "!", "…Z", "\"", " '", "re", "٣٤٥", "٦", "'re", "\u000bt", "!!"]} +{"text": "𐞁漢å　<|endoftext|>'VE㋿ＡZ\u000bm🙂İ\r\n\r\n'sع12345678å🙂\t#$%'-ß'Reع> d…é", "tokens": 53, "pieces": ["𐞁漢a", "̊", "　", "<|", "endoftext", "|>'", "VE", "㋿ＡZ", "\u000bm", "🙂İ", "\r\n\r\n", "'s", "ع", "123", "456", "78", "a", "̊🙂", "\t", "#$%'-", "ß", "'Re", "ع", ">", " d", "…e", "́"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\n<|endoftext|>'VE½'re𐞁\u000b漢12345678eé<㍿漢Z'M'VE
'M👍🏽s'D ḍ̇-㍿<|fim_prefix|>", "tokens": 61, "pieces": ["\n", "<|", "endoftext", "|>'", "VE", "½", "'re", "𐞁", "\u000b漢", "123", "456", "78", "ee", "́<㍿", "漢Z", "'M", "'VE", "
", "'M", "👍🏽", "s", "'D", " ḋ", "̣-㍿<|", "fim", "_prefix", "|>"]} +{"text": "12345678\r\n\r\nſ<½12345678ḍ̇'ſع<'ReＡ'D(\n漢‍ #$%​٣٤٥٦🙂­!! ​\r\n\r\n<<|endoftext|>>!!ⅣDžſſ!字", "tokens": 71, "pieces": ["123", "456", "78", "\r\n\r\n", "ſ", "<", "½12", "345", "678", "ḋ", "̣'", "ſع", "<'", "ReＡ", "'D", "(\n", "漢", "‍", " ", " #$%​", "٣٤٥", "٦", "🙂­!!", " ​\r\n\r\n", "<<|", "endoftext", "|>>!!", "Ⅳ", "Džſſ", "!字"]} +{"text": "
s𐞁tⅣ(٣٤٥٦åZ½​'D ḍ̇<‍ ‍'VEeعfi\r\ń½'S9'T\tſ漢​'re'Ms漢㍿'M", "tokens": 70, "pieces": ["
s𐞁t", "Ⅳ", "(", "٣٤٥", "٦", "a", "̊Z", "½", "​'", "D", " ḋ", "̣<<", "META", "_START", ">‍", " ‍'", "VEeعfi", "\r\n", "́", "½", "'S", "9", "'T", "\tſ漢", "​'", "re", "'M", "s漢", "㍿'", "M"]} +{"text": "fiꟲḍ̇", "tokens": 10, "pieces": ["fiꟲḋ", "̣"]} +{"text": "字😀🏽
İⅣＡ​é<|fim_prefix|>DžEOT😀🏽!<\u000b㋿\"　", "tokens": 38, "pieces": ["字", "😀🏽", "
İ", "Ⅳ", "Ａ", "​é", "<|", "fim", "_prefix", "|>", "DžEOT", "😀🏽!<", "\u000b", "㋿\"", "　"]} +{"text": "0㍿<ḍ̇İ'T!…　\r\n \n'M٣٤٥٦İ́漢\rtꟲ…éİ٣٤٥٦é're 'M‍ß.", "tokens": 60, "pieces": ["0", "㍿<", "ḋ", "̣İ", "'T", "!", "…　\r\n \n", "'M", "٣٤٥", "٦", "İ", "́漢", "\r", "tꟲ", "…éİ", "٣٤٥", "٦", "é", "'re", " ", " '", "M", "‍ß", "."]} +{"text": "0'llt ", "tokens": 4, "pieces": ["0", "'ll", "t", " "]} +{"text": "🙂'9ß(å\u000ba'Re\r\nḍ̇.\rꟲ㋿ع9'VEsZDž…'M\r\n\r\n\rİＡ'Mfia.३ḍ̇", "tokens": 60, "pieces": ["🙂<", "META", "_START", ">'<", "META", "_START", ">", "9", "ß", "(a", "̊", "\u000ba", "'Re", "\r\n", "ḋ", "̣.\r", "ꟲ", "㋿ع", "9", "'VE", "sZDž", "…", "'M", "\r\n\r\n\r", "İＡ", "'M", "fia", ".", "३", "ḋ", "̣"]} +{"text": "   \n'VEt<|fim_prefix|>é㋿Džts,'lĺ'Re\r\n<,", "tokens": 29, "pieces": ["   \n", "'VE", "t", "<|", "fim", "_prefix", "|>", "e", "́㋿", "Džts", ",'", "ll", "́'", "Re", "\r\n", "<,"]} +{"text": "(- 'Té𐞁ǻ…EOTعꟲ'Re\u000b,٣٤٥٦㍿s\u000b", "tokens": 40, "pieces": ["(-", " ", " '", "Te", "́𐞁a", "̊́", "…EOTعꟲ", "'", "Re", "\u000b", ",", "٣٤٥", "٦", "㍿s", "\u000b"]} +{"text": "\"Ⅳ-𐞁-e!'llåİ 漢Zfi㋿", "tokens": 24, "pieces": ["\"", "Ⅳ", "-𐞁", "-e", "!'", "lla", "̊İ", " ", " 漢Zfi", "㋿"]} +{"text": "d<|endoftext|>'TⅣ\r😀🏽ſⅣd‍!㋿", "tokens": 28, "pieces": ["d", "<|", "endoftext", "|>'", "T", "Ⅳ", "\r", "😀🏽", "ſ", "Ⅳ", "d", "‍!㋿"]} +{"text": "👍🏽#$% EOT<|endoftext|>,عſéſ\t\r½'sd…😀🏽é½<\t\n'VEé👍🏽", "tokens": 51, "pieces": ["👍🏽#$%", " ", " EOT", "<|", "endoftext", "|>,", "عſe", "́ſ", "\t\r", "½", "'s", "d", "…", "😀🏽", "é", "½", "<", "\t\n", "'VE", "e", "́👍🏽"]} +{"text": "(åع🙂ع🙂Dž­'S!'s're­㋿'ſfi😀🏽½'D9", "tokens": 34, "pieces": ["(a", "̊ع", "🙂ع", "🙂Dž", "­'", "S", "!'", "s", "'re", "­㋿'", "ſfi", "😀🏽", "½", "'D", "9"]} +{"text": "\t٣٤٥٦'T३\rع…𐞁fi٣٤٥٦🙂'll'll
", "tokens": 37, "pieces": ["\t", "٣٤٥", "٦", "'T", "३", "\r", "ع", "…𐞁fi", "٣٤٥", "٦", "🙂'", "ll", "'ll", "
"]} +{"text": "'­\"́́'ll're🙂👍🏽　字

…0ꟲ'llⅣ\" \n३　,\u000b'Re \n", "tokens": 43, "pieces": ["'­\"́́'", "ll", "'re", "🙂👍🏽", "　字", "
", "", "
", "…", "0", "ꟲ", "'ll", "Ⅳ", "\"", " \n", "३", "　", ",", "\u000b", "'Re", " \n"]} +{"text": "
Z<|fim_prefix|>!!\n‍é\u000b㋿ \n'Re　　0d(d㍿Z'lls12345678 👍🏽'T ​́> >👍🏽　", "tokens": 67, "pieces": ["
Z", "<|", "fim", "_prefix", "|>!!\n", "‍<", "META", "_START", ">é", "\u000b", "㋿<", "EOT", ">", " \n", "'Re", "　", "　", "0", "d", "(d", "㍿Z", "'ll", "s", "123", "456", "78", " ", "👍🏽'", "T", " ", " ​́>", " >👍🏽", "　"]} +{"text": "'S漢​'Sİḍ̇‍\r\nＡ", "tokens": 20, "pieces": ["'S", "漢", "​<", "META", "_START", ">'", "Sİḋ", "̣‍\r\n", "Ａ"]} +{"text": "㍿ <<|fim_prefix|>å\r\n<|fim_prefix|>㍿'T\r\nſ!", "tokens": 31, "pieces": ["㍿", " ", " <<|", "fim", "_prefix", "|>", "a", "̊\r\n", "<|", "fim", "_prefix", "|>㍿'", "T", "\r\n", "ſ", "!"]} +{"text": ".a
'll'llm<|endoftext|>Z\r\"-'ſ😀🏽>字'sⅣ<|endoftext|>d\tİ­", "tokens": 40, "pieces": [".a", "
", "'ll", "'ll", "m", "<|", "endoftext", "|>", "Z", "\r", "\"-'", "ſ", "😀🏽>", "字", "'s", "Ⅳ", "<|", "endoftext", "|>", "d", "\tİ", "­"]} +{"text": "😀🏽'Re🙂'sعm'VE🙂's('re㋿daa'Dß🙂\u000bİ'll!! åع'så'St12345678\r\n\r\n\neé", "tokens": 51, "pieces": ["😀🏽'", "Re", "🙂'", "sعm", "'VE", "🙂'", "s", "('", "re", "㋿daa", "'D", "ß", "🙂", "\u000bİ", "'ll", "!!", " a", "̊ع", "'s", "a", "̊'", "St", "123", "456", "78", "\r\n\r\n\n", "eé"]} +{"text": "'D're\r\n123456789㍿t‍.​<|endoftext|>!ß㍿\r\nDž<|fim_prefix|>㋿ḍ̇́'M…", "tokens": 52, "pieces": ["'D", "'re", "\r\n", "123", "456", "789", "㍿t", "‍.​<|", "endoftext", "|>!", "ß", "㍿<", "EOT", ">\r\n", "Dž", "<|", "fim", "_prefix", "|>㋿", "ḋ", "̣́'", "M", "…"]} +{"text": "t́e9　\t'VE漢ꟲ12345678　Džİ'ZZ\rع𐞁é.😀🏽İ​\u000b­(0'\rZ…m!!EOT　́😀🏽", "tokens": 58, "pieces": ["t", "́e", "9", "　", "\t", "'VE", "漢ꟲ", "123", "456", "78", "　Džİ", "'ZZ", "\r", "ع𐞁e", "́.😀🏽", "İ", "​", "\u000b", "­(", "0", "'\r", "Z", "…m", "!!", "EOT", "　", "́😀🏽"]} +{"text": "EOT٣٤٥٦!!<|fim_prefix|>fifiaß́\n'VE𐞁\n­🙂 \n(t㋿'D\n\r\né", "tokens": 47, "pieces": ["EOT", "٣٤٥", "٦", "!!<|", "fim", "_prefix", "|>", "fifiaß", "́\n", "'VE", "𐞁", "\n", "­🙂", " \n", "(t", "㋿'", "D", "\n\r\n", "e", "́"]} +{"text": "! \n…#$%\r\nſ😀🏽३\"́!…😀🏽́é‍#$%é.ém'll'S", "tokens": 44, "pieces": ["!", " \n", "…", "#$%<", "META", "_START", ">\r\n", "ſ", "😀🏽", "३", "\"́!", "…", "😀🏽́", "é", "‍#$%", "e", "́.", "ém", "'ll", "'S"]} +{"text": "İ", "tokens": 1, "pieces": ["İ"]} +{"text": "'Md३🙂'llİ🙂\"!　‍Dž<½\r\n\r\n'ſ­s漢!! ع 12345678\"½\n👍🏽\r\n३½.𐞁३", "tokens": 56, "pieces": ["'M", "d", "३", "🙂'", "llİ", "🙂<", "META", "_START", ">\"!", "　", "‍Dž", "<", "½", "\r\n\r\n", "'ſ", "­s漢", "!!", " ع", " ", "123", "456", "78", "\"", "½", "\n", "👍🏽\r\n", "३½", ".𐞁", "३"]} +{"text": "\u000b(-<|endoftext|>.३　👍🏽#$%'VE'res\r\n", "tokens": 24, "pieces": ["\u000b", "(-<|", "endoftext", "|>.", "३", "　", "👍🏽#$%'", "VE", "'re", "s", "\r\n"]} +{"text": "🙂 \n😀🏽<|endoftext|>​('🙂mt'sm\"ع́é­m'S9!㍿𐞁é'ßＡm'ſ'S½'ſ
", "tokens": 60, "pieces": ["🙂", " \n", "😀🏽<|", "endoftext", "|>​('🙂", "m", "t", "'s", "m", "\"ع", "́e", "́­", "m", "'S", "9", "!㍿", "𐞁e", "́'", "ßＡm", "'ſ", "'S", "½", "'ſ", "
"]} +{"text": "-,漢<|endoftext|>漢\r\n𐞁<|fim_prefix|>𐞁𐞁<|endoftext|>​ع'D㋿'ReⅣ-\"'ſ'Z'll​'Tå　 ​Dž", "tokens": 77, "pieces": ["<|", "endoftext", "|>", "漢", "\r\n", "𐞁", "<|", "fim", "_prefix", "|>", "𐞁𐞁", "<|", "endoftext", "|>​", "ع", "'D", "㋿'", "Re", "Ⅳ", "-\"'", "ſ", "'Z", "'ll", "​'", "Ta", "̊", "　 ", " ​", "Dž", ""]} +{"text": "ع漢EOT३<|fim_prefix|>ع'llİ 　é<|endoftext|>sß!!🙂㋿- ꟲå0ع'Dfi३>٣٤٥٦ 'S\r\n\r\n\rfi", "tokens": 66, "pieces": ["ع漢EOT", "३", "<|", "fim", "_prefix", "|>", "ع", "'ll", "İ", " ", "　e", "́<|", "endoftext", "|>", "sß", "!!🙂㋿-", " ꟲa", "̊", "0", "ع", "'D", "fi", "३", ">", "٣٤٥", "٦", " ", "'S", "\r\n\r\n\r", "fi"]} +{"text": "'ſEOT\u000b​t́,漢字,ḍ̇🙂d'T'llİḍ̇‍'VE😀🏽12345678s漢٣٤٥٦漢İ 　", "tokens": 58, "pieces": ["'ſ", "EOT", "\u000b", "​t", "́,", "漢字", ",ḋ", "̣🙂", "d", "'T", "'ll", "İḋ", "̣‍'", "VE", "😀🏽", "123", "456", "78", "s漢", "٣٤٥", "٦", "漢İ", " 　"]} +{"text": "Z'Så(ḍ̇'VEe‍", "tokens": 16, "pieces": ["Z", "'S", "a", "̊(", "ḋ", "̣'", "VEe", "‍"]} +{"text": "sßꟲsa\u000baſtع㍿́ع३漢\r\n\tEOTßſé\"­\u000bع'S ́ß", "tokens": 41, "pieces": ["sßꟲsa", "\u000baſtع", "㍿́", "ع", "३", "漢", "\r\n", "\tEOTß", "ſe", "́\"­", "\u000bع", "'S", " ́", "ß"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Džd३\"İ12345678'll字'llfi\r\ńé३Dž'VE'VE.a<'lldt", "tokens": 34, "pieces": ["Džd", "३", "\"İ", "123", "456", "78", "'ll", "字", "'ll", "fi", "\r\n", "́", "e", "́", "३", "Dž", "'VE", "'VE", ".a", "<'", "lldt"]} +{"text": "'S <|endoftext|>é'S𐞁<|endoftext|>​ſ…­　''re", "tokens": 32, "pieces": ["'S", " ", " <|", "endoftext", "|>", "e", "́'", "S𐞁", "<|", "endoftext", "|>​", "ſ", "…", "­", "　", "''", "re"]} +{"text": "'s\t'ſZ \"​d👍🏽>!><İſ𐞁'İ<|fim_prefix|>'D-　🙂.'ll'S𐞁sEOT😀🏽dEOT'M　Z", "tokens": 61, "pieces": ["'s", "\t", "'ſ", "Z", " ", "\"​", "d", "👍🏽>!><", "İſ𐞁", "'İ", "<|", "fim", "_prefix", "|>'", "D", "-", "　", "🙂.'", "ll", "'S", "𐞁sEOT", "😀🏽", "dEOT", "'M", "　Z"]} +{"text": "　\u000be,ſs😀🏽字\"ع<|fim_prefix|>'re… \ns\r\r\n ", "tokens": 32, "pieces": ["　", "\u000be", ",ſ", "s", "😀🏽", "字", "\"ع", "<|", "fim", "_prefix", "|>'", "re", "… \n", "s", "\r\r\n "]} +{"text": " \t<<|fim_prefix|> \n>㋿ḍ̇½t🙂'T👍🏽\t<|endoftext|>\r\n\r\n 
🙂", "tokens": 44, "pieces": [" ", "\t", "<<|", "fim", "_prefix", "|>", " \n", ">㋿", "ḋ", "̣", "½", "t", "🙂'", "T", "👍🏽", "\t", "<|", "endoftext", "|>\r\n\r\n", " ", "
", "🙂"]} +{"text": " 0\t's'M<,👍🏽'ſ\t's…👍🏽0", "tokens": 28, "pieces": [" ", " ", "0", "\t", "'s", "'M", "<,👍🏽'", "ſ", "\t", "'s", "…", "👍🏽", "0"]} +{"text": "ḍ̇\nİ12345678३㋿", "tokens": 15, "pieces": ["ḋ", "̣\n", "İ", "123", "456", "78३", "㋿"]} +{"text": "'M　9<|endoftext|>fi('𐞁a<𐞁😀🏽½½a-'VE\tm\r é　½𐞁ⅣDžéd<|fim_prefix|>…\t'sZİع
Ⅳ", "tokens": 73, "pieces": ["'M", "　", "9", "<|", "endoftext", "|>", "fi", "('", "𐞁a", "<𐞁", "😀🏽", "½½", "a", "-'", "VE", "\tm", "\r", " ", " e", "́", "　", "½", "𐞁", "Ⅳ", "Dže", "́d", "<|", "fim", "_prefix", "|>", "…", "\t", "'s", "Z", "İع", "
", "Ⅳ"]} +{"text": "
m \n!!ſⅣ‍ß𐞁's#$%ع-d'́\t!!<|endoftext|>aDžfi㍿😀🏽#$%#$% ḍ̇‍,", "tokens": 58, "pieces": ["
m", " \n", "!!", "ſ", "Ⅳ", "‍ß𐞁", "'s", "#$%", "ع", "-d", "'́", "\t", "!!<|", "endoftext", "|>", "aDžfi", "㍿😀🏽#$%#$%", " ", " ḋ", "̣‍,"]} +{"text": "́a-　३ḍ̇'reꟲ㋿> 'll字ع😀🏽'M\t-'M\r", "tokens": 39, "pieces": ["́a", "-<", "EOT", ">", "　", "३", "ḋ", "̣'", "reꟲ", "㋿>", " '", "ll字ع", "😀🏽'", "M", "\t", "-'", "M", "\r"]} +{"text": "'S
('S🙂Z<'VE tm", "tokens": 12, "pieces": ["'S", "
", "('", "S", "🙂Z", "<'", "VE", " ", " tm"]} +{"text": "-0! \u000b'M0EOT(t'Re　e𐞁>dEOT\"​eꟲmm٣٤٥٦!!ḍ̇٣٤٥٦👍🏽12345678", "tokens": 61, "pieces": ["-", "0", "!", " ", "\u000b", "'M", "0", "EOT", "(t", "'Re", "　e", "𐞁", ">dEOT", "\"​", "eꟲmm", "٣٤٥", "٦", "!!", "ḋ", "̣", "٣٤٥", "٦", "👍🏽", "123", "456", "78"]} +{"text": "٣٤٥٦́a9><Ａ​\t㍿Ａ!!'<fi>é,0㋿漢Ⅳ'Re‍å字m🙂 'VE", "tokens": 49, "pieces": ["٣٤٥", "٦", "́a", "9", "><", "Ａ", "​", "\t", "㍿Ａ", "!!'<", "fi", ">e", "́,", "0", "㋿漢", "Ⅳ", "'Re", "‍a", "̊字m", "🙂", " ", " '", "VE"]} +{"text": "ع", "tokens": 1, "pieces": ["ع"]} +{"text": " \n0㍿𐞁Dž㋿é字'Re\tḍ̇\r\nꟲ\r　'S'D½", "tokens": 39, "pieces": [" \n", "0", "㍿𐞁Dž", "㋿é字", "'Re", "\tḋ", "̣<", "META", "_START", ">\r\n", "ꟲ", "\r", "　", "'S", "'D", "½", ""]} +{"text": "å'S dfi㍿fi　'0Z-'re३
Ａ'ſ<|endoftext|>Ⅳeå…(ſ\r\nm\r\n\r\n<|endoftext|>Dž漢عḍ̇\r\n\r\n-", "tokens": 77, "pieces": ["a", "̊'", "S", "", " ", " dfi", "㍿fi", "　", "'", "0", "Z", "-<", "EOT", ">'", "re", "३", "
Ａ", "'", "ſ", "<|", "endoftext", "|>", "Ⅳ", "ea", "̊", "…", "(ſ", "\r\n", "m", "\r\n\r\n", "<|", "endoftext", "|>", "Dž漢عḋ", "̣\r\n\r\n", "-"]} +{"text": "<'M'M!!(ꟲ½12345678!İ🙂'VEDž३#$%\"㍿'D 'Re>må'D\t
漢<|endoftext|><|endoftext|>'s<|fim_prefix|>'T", "tokens": 64, "pieces": ["<'", "M", "'M", "!!(", "ꟲ", "½12", "345", "678", "!İ", "🙂'", "VEDž", "३", "#$%\"㍿'", "D", " '", "Re", ">ma", "̊'", "D", "\t", "
漢", "<|", "endoftext", "|><|", "endoftext", "|>'", "s", "<|", "fim", "_prefix", "|>'", "T"]} +{"text": "Ⅳ'M.'Tḍ̇­-'ſt㍿㋿é­ḍ̇0'D9#$%<|fim_prefix|>\r\n\r\nEOT'Re 'Re😀🏽,dꟲ𐞁#$%é", "tokens": 66, "pieces": ["Ⅳ", "'", "M", ".'", "Tḋ", "̣­-'", "ſt", "㍿㋿", "é", "­ḋ", "̣", "0", "'D", "9", "#$%<|", "fim", "_prefix", "|>\r\n\r\n", "EOT", "'Re", " '", "Re", "😀🏽,", "dꟲ𐞁", "#$%", "é"]} +{"text": "'Mİ åＡ\r\n㋿'ſm \n\t>👍🏽!!!ḍ̇İ9𐞁12345678​EOT 字漢fi'ſ9३½ !!\nꟲ", "tokens": 65, "pieces": ["'M", "İ", " ", " a", "̊Ａ", "\r\n", "㋿'", "ſm", " \n", "\t", ">👍🏽!!!", "ḋ", "̣İ", "9", "𐞁", "123", "456", "78", "​EOT", " 字漢fi", "'ſ", "9३½", " ", " !!\n", "ꟲ"]} +{"text": "'Re　- 𐞁å'ſꟲeå\r\n's　🙂­ع🙂ع\r\n\r\nꟲ#$%
>ḍ̇é", "tokens": 45, "pieces": ["'Re", "　", "-", " 𐞁a", "̊'", "ſꟲea", "̊\r\n", "'s", "　", "🙂­", "ع", "🙂ع", "\r\n\r\n", "ꟲ", "#$%", "
", ">ḋ", "̣é"]} +{"text": "­(- sådＡ😀🏽éDž#$%<|endoftext|>字", "tokens": 27, "pieces": ["­(-", " sa", "̊dＡ", "😀🏽", "éDž", "#$%<|", "endoftext", "|>", "字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n\r\n-'Sİe \n'MⅣ́👍🏽‍字9🙂漢fi're­a12345678字'M<|endoftext|>\r\n\r\n漢 …,\"Džß", "tokens": 61, "pieces": ["\r\n\r\n", "-'", "Sİe", " \n", "'M", "", "Ⅳ", "́👍🏽‍", "字", "9", "🙂<", "EOT", ">漢fi", "'re", "­a", "123", "456", "78", "字", "'M", "<|", "endoftext", "|>\r\n\r\n", "漢", " ", "…", ",\"", "Džß"]} +{"text": "İs㋿‍fí👍🏽9‍>", "tokens": 20, "pieces": ["İs", "㋿‍", "fi", "́👍🏽", "9", "‍>"]} +{"text": "ḍ̇㋿é'ſ'ſ'D12345678 ß‍!", "tokens": 29, "pieces": ["ḋ", "̣㋿", "e", "́<", "EOT", ">'", "ſ", "'ſ", "'D", "123", "456", "78", " ", " ß", "‍!"]} +{"text": "Z.'ſ (
sd'ſ's\r\n\r\n\r<👍🏽'Re'D>'Re\n\r\n\r\nZ 'Res‍
\t.ع", "tokens": 39, "pieces": ["Z", ".'", "ſ", " (", "
sd", "'ſ", "'s", "\r\n\r\n\r", "<👍🏽'", "Re", "'D", ">'", "Re", "\n\r\n\r\n", "Z", " ", " '", "Res", "‍", "
", "\t", ".ع"]} +{"text": "\r\n\r\n\r\n\r\nＡd\t'Sع ", "tokens": 8, "pieces": ["\r\n\r\n\r\n\r\n", "Ａd", "\t", "'S", "ع", " "]} +{"text": "'ſꟲ​seaع#$%fi'll0'ſZꟲ<|fim_prefix|>9<ꟲ'D'D", "tokens": 36, "pieces": ["'ſ", "ꟲ", "​seaع", "#$%", "fi", "'ll", "0", "'ſ", "Zꟲ", "<|", "fim", "_prefix", "|>", "9", "<ꟲ", "'D", "'D"]} +{"text": "éß'll'T㋿　\r\n\r\n\n　<|fim_prefix|>‍́<|endoftext|>㍿ꟲ\"'字\t ­👍🏽\t'Re'D‍ \nß", "tokens": 59, "pieces": ["éß", "'ll", "'T", "㋿", "　\r\n\r\n\n", "　", "<|", "fim", "_prefix", "|>‍́<|", "endoftext", "|>㍿", "ꟲ", "\"'<", "EOT", ">字", "\t", " ­👍🏽", "\t", "'Re", "'", "D", "‍", " \n", "ß"]} +{"text": "0ſ9Ａß\rꟲéⅣ\r㋿\u000bß'Reé\t­<|endoftext|>9\r\n 's'­'M", "tokens": 47, "pieces": ["0", "ſ", "9", "Ａß", "\r", "ꟲe", "́<", "EOT", "><", "EOT", ">", "Ⅳ", "\r", "㋿", "\u000bß", "'Re", "é", "\t", "­<|", "endoftext", "|>", "9", "\r\n", " ", "'s", "'­'", "M"]} +{"text": "12345678​\r\n漢 \n'Reḍ̇9EOT('\r\n\r\nＡ'Re.ḍ̇　　　ſ9'D𐞁字\"e🙂12345678ꟲ'reſe३㋿!!<|fim_prefix|>,å", "tokens": 72, "pieces": ["123", "456", "78", "​\r\n", "漢", " \n", "'Re", "ḋ", "̣", "9", "EOT", "('\r\n\r\n", "Ａ", "'Re", ".ḋ", "̣", "　　", "　ſ", "", "9", "'D", "𐞁字", "\"e", "🙂", "123", "456", "78", "ꟲ", "'re", "ſe", "३", "㋿!!<|", "fim", "_prefix", "|>,", "a", "̊"]} +{"text": "३<|endoftext|>!́𐞁३漢#$% 漢>
a9‍", "tokens": 32, "pieces": ["३", "<|", "endoftext", "|>!́", "𐞁", "३", "漢", "#$%", " ", " 漢", ">", "
a", "9", "‍"]} +{"text": "­'Re-…de!!9…'T'D\n\r\néⅣm🙂\ts\ta <'ll", "tokens": 26, "pieces": ["­'", "Re", "-", "…de", "!!", "9", "…", "'T", "'D", "\n\r\n", "é", "Ⅳ", "m", "🙂", "\ts", "\ta", " <'", "ll"]} +{"text": "å\u000b́٣٤٥٦<|fim_prefix|>\"", "tokens": 46, "pieces": ["a", "̊<", "META", "_START", ">", "\u000b", "́<", "a", "­㋿<", "½", "!", " t", "\n", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\""]} +{"text": "ſ'ſ> d'ſ …İ'ſm's#$%. 9 ", "tokens": 25, "pieces": ["ſ", "'ſ", ">", " ", " d", "'ſ", " ", "…İ", "'ſ", "m", "'s", "#$%.", " ", "9", " "]} +{"text": "('M🙂\r½'ſdEOTDžＡfi　<#$%", "tokens": 50, "pieces": ["('", "M", "🙂\r", "½", "'ſ", "dEOTDžＡ", "", "fi", "　", "<#$%"]} +{"text": " \nd½!'ReDžİ'VEd٣٤٥٦🙂e \n\n'Tß　'S
dDž", "tokens": 35, "pieces": [" \n", "d", "½", "!'", "ReDžİ", "'VE", "d", "٣٤٥", "٦", "🙂e", " \n\n", "'T", "ß", "　", "'S", "
", "dDž"]} +{"text": ">\u000b㍿#$%'fi\t'll!!é9>,­'VE'​'Re  'll(\u000b'S字👍🏽\n \n're٣٤٥٦ !!\r ع", "tokens": 55, "pieces": [">", "\u000b", "㍿#$%'", "fi", "\t", "'ll", "!!", "é", "9", ">,­'", "VE", "'​'", "Re", "  ", " '", "ll", "(", "\u000b", "'S", "字", "👍🏽\n", " \n", "'re", "٣٤٥", "٦", " ", "!!\r", " ع"]} +{"text": "­३(!", "tokens": 4, "pieces": ["­", "३", "(!"]} +{"text": " t　‍Ⅳa'D'reåd㋿İdZ…½Ⅳ\nعꟲ\td!!", "tokens": 35, "pieces": [" t", "　", "‍", "Ⅳ", "a", "'D", "'re", "a", "̊d", "㋿İdZ", "…", "½", "", "Ⅳ", "\n", "عꟲ", "\td", "!!"]} +{"text": "…a's<'Mع😀🏽'M…9ꟲ'ſ\"Ａ\ta\u000bḍ̇('T-12345678e \n 🙂<|endoftext|>́'D", "tokens": 57, "pieces": ["…a", "'s", "<'", "Mع", "😀🏽'", "M", "…", "9", "ꟲ", "'ſ", "\"Ａ", "\ta", "\u000bḋ", "̣(<", "META", "_START", ">'", "T", "-", "123", "456", "78", "e", " \n", " ", "🙂<|", "endoftext", "|>́'", "D"]} +{"text": "d½t'll㋿\r\n
>字字mع,12345678d'Sé३'VE!!><|endoftext|>EOT́å½12345678'Ａ,Ａ­½\r\n👍🏽é", "tokens": 67, "pieces": ["d", "½", "t", "'ll", "㋿\r\n", "
", ">字字mع", ",", "123", "456", "78", "d", "'S", "é", "३", "'", "VE", "!!><|", "endoftext", "|>", "EOT", "́a", "̊", "½12", "345", "678", "'Ａ", ",Ａ", "­", "½", "\r\n", "👍🏽", "é", ""]} +{"text": "'ſDž…㍿#$%>'re,fi \ńß ( \n漢#$%.", "tokens": 27, "pieces": ["'ſ", "Dž", "…", "㍿#$%>'", "re", ",fi", " \n", "́ß", " ", " (", " \n", "漢", "#$%."]} +{"text": "e ३عm('s漢 \n \né漢m#$%Ａé'Sع12345678e\"ſ३'ll😀🏽Z'Re>👍🏽३…­\r\n\r\n😀🏽", "tokens": 63, "pieces": ["e", "", " ", " ", "३", "عm", "('", "s漢", " \n \n", "é漢m", "#$%", "Ａé", "'S", "ع", "123", "456", "78", "e", "\"ſ", "३", "'ll", "😀🏽", "Z", "'Re", ">👍🏽", "३", "…", "­\r\n\r\n", "😀🏽"]} +{"text": "\r\n\r\n'ſ\r\n\r\n!👍🏽\t \n㋿#$%.å\r\n're漢\"! ", "tokens": 34, "pieces": ["\r\n\r\n", "'ſ", "\r\n\r\n", "!👍🏽", "\t \n", "㋿#$%.", "a", "̊\r\n", "'re", "漢", "\"!", " "]} +{"text": "​fi漢.\"ع\nعa'lls'ſ", "tokens": 15, "pieces": ["​fi漢", ".\"", "ع", "\n", "عa", "'ll", "s", "'ſ"]} +{"text": "\r'S'D　a字fi­<9EOT½…>d,\r😀🏽٣٤٥٦😀🏽'M0ḿ𐞁med's𐞁​", "tokens": 53, "pieces": ["\r", "'S", "'D", "　a字fi", "­<", "9", "EOT", "½", "…", ">d", ",\r", "😀🏽", "٣٤٥", "٦", "😀🏽'", "M", "0", "m", "́𐞁med", "'s", "𐞁", "​"]} +{"text": "٣٤٥٦'VE\r\n\r\n　\r\n\r\n\r\n\r\ń\r\n𐞁(́'ſ​३!!\u000b Z漢,ع　­'EOT́😀🏽Ａſ­Dž <|fim_prefix|> >fi", "tokens": 66, "pieces": ["٣٤٥", "٦", "'VE", "\r\n\r\n　\r\n\r\n\r\n\r\n", "́\r\n", "𐞁", "(́'", "ſ", "​", "३", "!!", "\u000b", " Z漢", ",ع", "　", "­'", "EOT", "́<", "META", "_START", ">😀🏽", "Ａſ", "­Dž", " <|", "fim", "_prefix", "|>", " ", " >", "fi"]} +{"text": "(å \n‍(-12345678'D٣٤٥٦é<… İ　🙂字,٣٤٥٦EOT'VE,ſ>字'll'll<|endoftext|>fi", "tokens": 61, "pieces": ["(a", "̊", " \n", "‍(-", "123", "456", "78", "'D", "٣٤٥", "٦", "é", "<", "…", " İ", "　", "🙂字", ",", "٣٤٥", "٦", "EOT", "'VE", ",ſ", ">", "字", "'ll", "'ll", "<|", "endoftext", "|>", "fi"]} +{"text": "𐞁👍🏽.㋿('Sfi\nd\r\n<|fim_prefix|>…ꟲ字(​ḍ̇<|fim_prefix|>\u000b𐞁e漢", "tokens": 59, "pieces": ["𐞁", "👍🏽.㋿('", "Sfi", "\n", "d", "\r\n", "<|", "fim", "_prefix", "|><", "EOT", ">", "…ꟲ字", "(​", "ḋ", "̣<|", "fim", "_prefix", "|>", "\u000b𐞁e漢"]} +{"text": "\u000b漢😀🏽\t \r\n\r\n'T!½🙂Ａ9,éfie🙂(\r\n\r\n\r\n!Ⅳ٣٤٥٦>a\u000b'­!!㍿", "tokens": 46, "pieces": ["\u000b漢", "😀🏽", "\t \r\n\r\n", "'T", "!", "½", "🙂Ａ", "9", ",e", "́fie", "🙂(\r\n\r\n\r\n", "!", "Ⅳ٣٤", "٥٦", ">a", "\u000b", "'­!!㍿"]} +{"text": "<|fim_prefix|>🙂, 😀🏽<|endoftext|>　!!'llDž字㍿\u000b''M㋿'llⅣ#$%!! 漢 0", "tokens": 50, "pieces": ["<|", "fim", "_prefix", "|>🙂,", " ", " 😀🏽<|", "endoftext", "|>", "　", "!!'", "llDž字", "㍿", "\u000b", "''", "M", "㋿'", "ll", "Ⅳ", "#$%!!", " 漢", " ", "0"]} +{"text": "'T're#$%İ-!!Ź9'VE<|fim_prefix|>٣٤٥٦'ll,Ａs­d'ſ", "tokens": 40, "pieces": ["'T", "'re", "#$%", "İ", "-!!", "Z", "́", "9", "'VE", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "'", "ll", ",Ａs", "­d", "'ſ"]} +{"text": "٣٤٥٦ß٣٤٥٦­d'Dž　é >''ſm\t　<|fim_prefix|>fi½\t'T \nſ d", "tokens": 49, "pieces": ["٣٤٥", "٦", "ß", "٣٤٥", "٦", "­d", "'Dž", "　é", " ", ">''", "ſm", "\t", "　", "<|", "fim", "_prefix", "|>", "fi", "½", "\t", "'T", " \n", "ſ", " d"]} +{"text": "'Dع12345678­ſ㋿<|fim_prefix|>㍿. ‍½ꟲs. ꟲ'M'llß(
<|endoftext|>\ts!
漢", "tokens": 60, "pieces": ["'D", "ع", "123", "456", "78", "­ſ", "㋿<|", "fim", "_prefix", "|>㍿.", " ‍", "½", "ꟲs", ".", " ꟲ", "'M", "'ll", "ß", "(", "
", "<|", "endoftext", "|>", "\t", "<", "EOT", ">s", "!", "
漢"]} +{"text": "\tß　​𐞁<|endoftext|> ㋿\rⅣ>'s\"->,'D
​
Ａ\n Ⅳ!!s\r\n🙂's👍🏽(漢áع", "tokens": 62, "pieces": ["\tß", "　", "​𐞁", "<|", "endoftext", "|>", " ", "㋿<", "EOT", ">\r", "Ⅳ", ">'", "s", "\"->,'", "D", "
", "​", "
Ａ", "\n", " ", "Ⅳ", "!!", "s", "\r\n", "🙂'", "s", "👍🏽(", "漢a", "́ع"]} +{"text": "㋿ e👍🏽s'Refi(👍🏽<|endoftext|>'S½><#$% \r\nＡ \r­🙂🙂
fi 0ꟲ'D 👍🏽(<|endoftext|>漢", "tokens": 73, "pieces": ["㋿", " ", "e", "👍🏽", "s", "'Re", "fi", "(👍🏽<|", "endoftext", "|>'", "S", "½", "><#$%", " \r\n", "Ａ", " \r", "­🙂🙂", "
fi", " ", "0", "ꟲ", "'D", " ", "👍🏽(<|", "endoftext", "|>", "漢"]} +{"text": "🙂㋿½", "tokens": 6, "pieces": ["🙂㋿", "½"]} +{"text": " 😀🏽㍿ 𐞁'llß\rßé12345678aع'S-\r's\r'#$%!\"eß𐞁", "tokens": 38, "pieces": [" ", " 😀🏽㍿", " 𐞁", "'ll", "ß", "\r", "ße", "́", "123", "456", "78", "aع", "'S", "-\r", "'s", "\r", "'#$%!\"", "eß𐞁"]} +{"text": "‍㍿'Mſ㍿½ ​9s漢ß漢afi
ſå12345678\r\n\r\n­ꟲ…­ＡＡ字td漢​eع\"<|fim_prefix|>'ll", "tokens": 66, "pieces": ["‍㍿'", "Mſ", "㍿", "½", " ", " ​", "9", "s漢ß漢afi", "
ſa", "̊", "123", "456", "78", "\r\n\r\n", "­ꟲ", "…", "­ＡＡ字td漢", "​eع", "\"<|", "fim", "_prefix", "|>'", "ll"]} +{"text": "12345678'D́ſ\"عfi'llＡ ㋿'Me㋿'ſZع\r\n\r\n<
\"! ­'D\n é😀🏽ع字sé", "tokens": 52, "pieces": ["123", "456", "78", "'D", "́ſ", "\"عfi", "'ll", "Ａ", " ㋿'", "Me", "㋿'", "ſZع", "\r\n\r\n", "<", "
", "\"!<", "EOT", ">", " ", " ­'", "D", "\n", " é", "😀🏽", "ع字sé"]} +{"text": "å…9㍿ḍ̇Ａ<|endoftext|>­
İꟲmEOT'Mİét'VE'VE🙂\r字ſ", "tokens": 48, "pieces": ["a", "̊", "…", "9", "㍿ḋ", "̣Ａ", "<|", "endoftext", "|>­", "
İꟲmEOT", "'M", "İét", "'VE", "'VE", "🙂\r", "字ſ"]} +{"text": "½fi­9½​m㋿😀🏽\r\nḍ̇#$%字𐞁'ſ#$%字Ａe \nZ", "tokens": 40, "pieces": ["½", "fi", "­", "9½", "​m", "㋿😀🏽\r\n", "ḋ", "̣#$%", "字𐞁", "'ſ", "#$%", "字Ａe", " \n", "Z"]} +{"text": "🙂sé<|fim_prefix|>'ll 'M漢!!!!Dž㍿٣٤٥٦　Zḍ̇<|endoftext|>'ſ <|endoftext|>Ａ,İ#$%", "tokens": 58, "pieces": ["🙂sé", "<|", "fim", "_prefix", "|>'", "ll", " '", "M漢", "!!!!", "Dž", "㍿", "٣٤٥", "٦", "　Zḋ", "̣<|", "endoftext", "|>'", "ſ", " <|", "endoftext", "|>", "Ａ", ",İ", "#$%"]} +{"text": "('Re'ſ'Re", "tokens": 9, "pieces": ["('", "Re", "'ſ", "'", "Re"]} +{"text": " å\r\n\r\n<ßfifié<|fim_prefix|>'reＡ'DžⅣ́!!#$%'<|fim_prefix|>ع㍿", "tokens": 46, "pieces": [" a", "̊\r\n\r\n", "<ßfifie", "́<|", "fim", "_prefix", "|>'", "reＡ", "'Dž", "Ⅳ", "́!!#$%'<|", "fim", "_prefix", "|><", "META", "_START", ">ع", "㍿"]} +{"text": "\r\n\r\n­
३\n字'ſ'T𐞁\u000b's\rsd<|endoftext|>fifiDž!'s ", "tokens": 38, "pieces": ["\r\n\r\n", "­", "
", "३", "\n", "字", "'ſ", "'T", "𐞁", "\u000b", "'s", "\r", "sd", "<|", "endoftext", "|>", "fifiDž", "!'", "s", " "]} +{"text": "\u000b ,é\r\n\r\n\r 'S
😀🏽-'re𐞁\r\n\r\n\r\n\r\n", "tokens": 24, "pieces": ["\u000b ", " ,", "e", "́\r\n\r\n\r", " ", " '", "S", "
", "😀🏽-'", "re𐞁", "\r\n\r\n\r\n\r\n"]} +{"text": "‍-ß
te👍🏽…d-́\r\n<|endoftext|>🙂!!\n<|endoftext|>fi\r\n\r\n​Ⅳ\n", "tokens": 46, "pieces": ["‍-", "ß", "
", "te", "👍🏽", "…d", "-́\r\n", "<|", "endoftext", "|>🙂!!\n", "<|", "endoftext", "|>", "fi", "\r\n\r\n", "​", "Ⅳ", "\n"]} +{"text": "\ntſ\t'll३e'ſ'T'',🙂0Dž'TsDž'llḍ̇'D३½👍🏽‍-", "tokens": 46, "pieces": ["\n", "tſ", "\t", "'ll", "३", "e", "'ſ", "'T", "'',🙂", "0", "Dž", "'T", "sDž", "'ll", "ḋ", "̣'", "D", "३", "", "½", "👍🏽‍-"]} +{"text": "m \t<㋿#$%́12345678🙂'S漢 \n's‍㍿'VE​ßs12345678t'SZ0,,9'ſ<|endoftext|>!12345678'ſ३🙂<|fim_prefix|><>Ａ", "tokens": 71, "pieces": ["m", " ", "\t", "<㋿#$%́", "123", "456", "78", "🙂'", "S漢", " \n", "'s", "‍㍿'", "VE", "​ßs", "123", "456", "78", "t", "'S", "Z", "0", ",,", "9", "'ſ", "<|", "endoftext", "|>!", "123", "456", "78", "'ſ", "३", "🙂<|", "fim", "_prefix", "|><>", "Ａ"]} +{"text": "aḍ̇ꟲ!!é(\n'Re ㋿!!>!'\u000bé\r\n\r\n'ſ \nＡ,é\néfi9 👍🏽,<👍🏽ꟲd'Tḍ̇", "tokens": 64, "pieces": ["aḋ", "̣ꟲ", "!!", "é", "(\n", "'Re", " ㋿!!>!<", "META", "_START", ">'", "\u000bé", "\r\n\r\n", "'ſ", " \n", "Ａ", ",é", "\n", "éfi", "9", " ", " 👍🏽,<👍🏽", "ꟲ", "d", "'T", "ḋ", "̣"]} +{"text": "'s漢0'D 𐞁.>漢….\"\r½'M'sé<|endoftext|>>12345678ع12345678𐞁", "tokens": 40, "pieces": ["'s", "漢", "0", "'D", " 𐞁", ".>", "漢", "…", ".\"\r", "½", "'M", "'s", "é", "<|", "endoftext", "|>>", "123", "456", "78", "ع", "123", "456", "78", "𐞁"]} +{"text": "!!'S\"'ſfi.　'", "tokens": 14, "pieces": ["!!'", "S", "\"<", "META", "_START", ">'", "ſfi", ".", "　", "'"]} +{"text": "<ع's", "tokens": 3, "pieces": ["<ع", "'s"]} +{"text": "㍿漢te
'D　m \n३<|endoftext|>ⅣDž,.३ \nß字ſß㍿''>9ع.", "tokens": 49, "pieces": ["㍿漢t", "e", "
", "'D", "　m", " \n", "३", "<|", "endoftext", "|>", "Ⅳ", "Dž", ",.", "३", " \n", "ß字ſß", "㍿''>", "9", "ع", "."]} +{"text": ">🙂😀🏽'Rees're( '12345678‍9​\"'D👍🏽
", "tokens": 44, "pieces": [">🙂😀🏽'", "Rees", "'", "re", "(", " '", "123", "456", "78", "‍", "9", "​\"'", "D", "👍🏽", "
"]} +{"text": " ३'Re\r\n\r9‍
ſ\r\nع\r\n\r\nå'T'Z𐞁'VE ­‍.0!é👍🏽fiEOT\nt🙂EOT><­ꟲ ", "tokens": 68, "pieces": [" ", " ", "३", "'Re", "\r\n\r", "9", "‍", "
", "ſ", "\r\n", "ع", "\r\n\r\n", "a", "̊'", "T", "'Z𐞁", "'VE", " ", " ­‍.", "0", "!", "é", "👍🏽", "fiEOT", "\n", "t", "🙂EOT", "><­", "ꟲ", " "]} +{"text": "Z'VEEOT<|endoftext|>㋿'s😀🏽9#$%😀🏽d　'Re((Džꟲ>< ع'\u000bé'VE㍿#$%'ReEOT​'T!! ३-'M", "tokens": 64, "pieces": ["Z", "'VE", "EOT", "<|", "endoftext", "|>㋿'", "s", "😀🏽", "9", "#$%😀🏽", "d", "　", "'Re", "((", "Džꟲ", "><", " ع", "'", "\u000be", "́'", "VE", "㍿#$%'", "ReEOT", "​'", "T", "!!", " ", "३", "-'", "M"]} +{"text": "mꟲ\"aé,ḍ̇'VE'Ma‍d👍🏽a…(字\rd'VE9é'Re<|endoftext|> mꟲ're‍½Džꟲ‍,'s 漢", "tokens": 65, "pieces": ["mꟲ", "\"ae", "́,", "ḋ", "̣'", "VE", "'M", "a", "‍d", "👍🏽", "a", "…", "(字", "\r", "d", "'VE", "9", "é", "'Re", "<|", "endoftext", "|>", " mꟲ", "'re", "‍", "½", "Džꟲ", "‍,'", "s", " 漢"]} +{"text": "t<|fim_prefix|>9㍿½EOTå.Z'T‍漢Ⅳ'ſe𐞁", "tokens": 38, "pieces": ["t", "<|", "fim", "_prefix", "|>", "9", "㍿", "½", "EOTa", "̊<", "META", "_START", ">.", "Z", "'T", "‍漢", "Ⅳ", "'ſ", "e𐞁"]} +{"text": "\r\n\r\n0'S,!!aEOTéå're\r\n\r\n", "tokens": 16, "pieces": ["\r\n\r\n", "0", "'S", ",!!", "aEOTe", "́a", "̊'", "re", "\r\n\r\n"]} +{"text": "٣٤٥٦ \néåfiع🙂𐞁>'Reḍ̇'M'lleＡ'llＡ", "tokens": 41, "pieces": ["٣٤٥", "٦", " \n", "éa", "̊fi", "ع", "🙂𐞁", ">'", "Reḋ", "̣'", "M", "'ll", "eＡ", "'ll", "Ａ"]} +{"text": " \n­\u000b­!　ḍ̇​!Dž\"𐞁!!12345678­\u000bs٣٤٥٦é'så㍿\r\ńZ…", "tokens": 49, "pieces": [" \n", "­", "\u000b", "­!", "　ḋ", "̣​!", "Dž", "\"𐞁", "!!", "123", "456", "78", "­", "\u000bs", "٣٤٥", "٦", "e", "́'", "sa", "̊㍿\r\n", "́Z", "…"]} +{"text": "'s<|fim_prefix|>İ!漢<|fim_prefix|>(ꟲ'reé å𐞁", "tokens": 33, "pieces": ["'s", "<|", "fim", "_prefix", "|>", "İ", "!漢", "<|", "fim", "_prefix", "|>(", "ꟲ", "'re", "e", "́", " ", " a", "̊𐞁"]} +{"text": " 😀🏽<|endoftext|>‍👍🏽\".>\r\n𐞁Dž\n", "tokens": 32, "pieces": [" ", " 😀🏽<|", "endoftext", "|>‍👍🏽\".>\r\n", "𐞁Dž", "\n"]} +{"text": "\r\nḍ̇\r\n\r\nＡ'D\n….­", "tokens": 15, "pieces": ["\r\n", "ḋ", "̣\r\n\r\n", "Ａ", "'D", "\n", "…", ".­"]} +{"text": "'D ㍿'refi!!<|fim_prefix|>\n👍🏽…𐞁…'re0<|endoftext|>>𐞁'VE‍́ \n'll", "tokens": 52, "pieces": ["'D", " ", " ㍿'", "refi", "!!<|", "fim", "_prefix", "|>\n", "👍🏽", "…𐞁", "…", "'re", "0", "<|", "endoftext", "|>>", "𐞁", "'VE", "‍́", " \n", "'ll"]} +{"text": "\rå", "tokens": 4, "pieces": ["\r", "a", "̊"]} +{"text": "㍿İ'ſ#$%́\r\n\r\n\r\n\r\n\u000b字Z
.a'llEOT 9 <'D\"!é'D'Sm'S‍Z'Dꟲİa#$%<|fim_prefix|> \n", "tokens": 58, "pieces": ["㍿İ", "'ſ", "#$%́\r\n\r\n\r\n\r\n", "\u000b字Z", "
", ".a", "'ll", "EOT", " ", " ", "9", " ", "<'", "D", "\"!", "e", "́'", "D", "'S", "m", "'", "S", "‍Z", "'D", "ꟲİa", "#$%<|", "fim", "_prefix", "|>", " \n"]} +{"text": "'re>'ſ<|fim_prefix|>́
é ", "tokens": 17, "pieces": ["'re", ">'", "ſ", "<|", "fim", "_prefix", "|>́", "
e", "́", " "]} +{"text": "\r\n\r\n'Td(İ'T>dꟲ\r\nm", "tokens": 12, "pieces": ["\r\n\r\n", "'T", "d", "(İ", "'T", ">dꟲ", "\r\n", "m"]} +{"text": ".'Re're'S'Ss'ſＡⅣé٣٤٥٦#$%
!-㍿'D 'VEa's㋿'De\"
 　", "tokens": 52, "pieces": [".'", "Re", "'re", "'S", "'S", "s", "'ſ", "Ａ", "Ⅳ", "é", "٣٤٥", "٦", "#$%", "
", "!-㍿'", "D", " ", "'VE", "a", "'s", "㋿'", "D", "e", "\"", "
 　"]} +{"text": "३Ａ#$%0m", "tokens": 12, "pieces": ["३", "Ａ", "#$%", "0", "m"]} +{"text": ",\u000b-'ll \n'𐞁Ａée'll9fi㋿Dž​\rs字", "tokens": 33, "pieces": [",", "\u000b", "-'", "ll", " \n", "'<", "EOT", ">𐞁Ａée", "'ll", "9", "fi", "㋿", "Dž", "​\r", "s字"]} +{"text": "é<|fim_prefix|>\r\n\r\n㍿　‍é", "tokens": 19, "pieces": ["e", "́<", "META", "_START", "><|", "fim", "_prefix", "|>\r\n\r\n", "㍿", "　", "‍é"]} +{"text": "<|endoftext|>", "tokens": 7, "pieces": ["<|", "endoftext", "|>"]} +{"text": "ſ عfi\r‍-EOT🙂‍'S
 ­'ꟲ#$%\re\"ꟲ#$%<|endoftext|>👍🏽\"09
Ⅳ <|endoftext|>>🙂é", "tokens": 76, "pieces": ["ſ", " عfi", "\r", "‍-", "EOT", "🙂‍'", "S", "
 ", " ­'", "ꟲ", "#$%\r", "e", "\"<", "META", "_START", ">ꟲ", "#$%<|", "endoftext", "|><", "META", "_START", ">👍🏽\"", "09", "
", "Ⅳ", " ", "<|", "endoftext", "|>><", "EOT", ">🙂", "e", "́"]} +{"text": "㋿fi\tDž \n'Tḍ̇éa🙂 \n!!ß😀🏽\r<'reŹ0​\u000b­efi#$%'Sa'S٣٤٥٦
d", "tokens": 55, "pieces": ["㋿fi", "\tDž", " \n", "'T", "ḋ", "̣e", "́a", "🙂", " \n", "!!", "ß", "😀🏽\r", "<'", "reZ", "́", "0", "​", "\u000b", "­efi", "#$%'", "Sa", "'S", "٣٤٥", "٦", "
d"]} +{"text": "0a'såEOTEOTḍ̇'Tß's'VE'D,<|fim_prefix|>…'Dž'S<\r…fim👍🏽ḍ̇३", "tokens": 54, "pieces": ["0", "a", "'s", "a", "̊EOTEOTḋ", "̣'", "Tß", "'s", "'VE", "'D", ",<|", "fim", "_prefix", "|>", "…", "'Dž", "'S", "<\r", "…fim", "👍🏽", "ḋ", "̣", "३"]} +{"text": "㍿!!Ａ'D!'ſ\n٣٤٥٦‍é", "tokens": 23, "pieces": ["㍿!!", "Ａ", "'D", "!'", "ſ", "\n", "٣٤٥", "٦", "‍e", "́"]} +{"text": "m😀🏽½'T𐞁​ß🙂 \n,ß­ \n(́'ſDž­३'EOT>'ſ\rع'M㋿tfiZ'S🙂 ", "tokens": 54, "pieces": ["m", "😀🏽", "½", "'T", "𐞁", "​<", "EOT", ">ß", "🙂", " \n", ",ß", "­", " \n", "(́'", "ſDž", "­", "३", "'EOT", ">'", "ſ", "\r", "ع", "'M", "㋿tfiZ", "'S", "🙂", " "]} +{"text": "åé ½ é\t#$%\r\n'D're'D('st　å<|fim_prefix|>t'Td
d \n,㍿'s9'Mḍ̇", "tokens": 51, "pieces": ["a", "̊é", " ", "½", " ", " e", "́", "\t", "#$%\r\n", "'D", "'re", "'D", "('", "st", "　a", "̊<|", "fim", "_prefix", "|>", "t", "'T", "d", "
d", " \n", ",㍿'", "s", "", "9", "'M", "ḋ", "̣"]} +{"text": "<ꟲ<|endoftext|>å
'll'ſé字d<|endoftext|>\neeḍ̇\r\n\r\n\r\t\t　\"\"td \n're", "tokens": 44, "pieces": ["<ꟲ", "<|", "endoftext", "|>", "a", "̊", "
", "'ll", "'ſ", "é字d", "<|", "endoftext", "|>\n", "eeḋ", "̣\r\n\r\n\r", "\t\t", "　", "\"\"", "td", " \n", "'re"]} +{"text": ",‍e😀🏽
('M٣٤٥٦㋿fi㋿", "fi", ",fi👍🏽…e'Reع 'ſ'Re Ⅳ'M'S ", "tokens": 40, "pieces": [".漢", "
d", "㋿🙂<", "META", "_START", ">,", "fi", "👍🏽", "…e", "'Re", "ع", " ", "'ſ", "'Re", " ", " ", "Ⅳ", "'M", "'S", " "]} +{"text": "EOT d\r ſ‍
d३ \n㋿12345678Zé", "tokens": 23, "pieces": ["EOT", " d", "\r", " ſ", "‍", "
d", "३", " \n", "㋿", "123", "456", "78", "Ze", "́"]} +{"text": "-ＡEOT!!fi \n🙂<漢,½३é字'ś 字'ſtsd漢12345678", "tokens": 34, "pieces": ["-ＡEOT", "!!", "fi", " \n", "🙂<", "漢", ",", "½३", "e", "́字", "'s", "́", " 字", "'ſ", "tsd漢", "123", "456", "78"]} +{"text": "​(fi#$%­\n\r<|endoftext|><\r\n'VE漢　𐞁٣٤٥٦mDž'ſ́'TEOT<|fim_prefix|>ſ", "tokens": 56, "pieces": ["​(", "fi", "#$%­\n\r", "<|", "endoftext", "|><\r\n", "'VE", "漢", "　𐞁", "٣٤٥", "٦", "mDž", "'ſ", "́'", "TEOT", "<|", "fim", "_prefix", "|>", "ſ"]} +{"text": "9字٣٤٥٦t٣٤٥٦ ­​\r\n३", "tokens": 25, "pieces": ["9", "字", "٣٤٥", "٦", "t", "٣٤٥", "٦", " ", "­​\r\n", "३"]} +{"text": "ع'M!½<|fim_prefix|>EOTeꟲ \nſ!<👍🏽½३a!", "tokens": 34, "pieces": ["ع", "'M", "!", "½", "<|", "fim", "_prefix", "|><", "EOT", ">EOTeꟲ", " \n", "ſ", "!<👍🏽", "½३", "a", "!"]} +{"text": "'re!!İ٣٤٥٦İ( ,\tZ#$%
İ😀🏽…İ.'Re　'ReEOT#$%'T#$%字<|fim_prefix|>m­İ", "tokens": 55, "pieces": ["'re", "!!", "İ", "٣٤٥", "٦", "İ", "(", " ", ",", "\tZ", "#$%", "
İ", "😀🏽", "…İ", ".'", "Re", "　", "'Re", "EOT", "#$%'", "T", "#$%", "字", "<|", "fim", "_prefix", "|><", "EOT", ">m", "­İ"]} +{"text": "s\u000b‍'re 'ſfi0ma.​9́\t.\n字'Ré'M'll'Sİ,
EOT…👍🏽\t'VE​s,", "tokens": 60, "pieces": ["٣٤٥", "٦", "ß", "<|", "fim", "_prefix", "|>", " '", "ſfi", "0", "ma", ".​", "9", "́", "\t", ".\n", "字", "'Re", "́'", "M", "'", "ll", "'S", "İ", ",", "
EOT", "…", "👍🏽", "\t", "'VE", "​s", ","]} +{"text": "'S\"Ⅳ.\ta३ Ａ(㋿'VEß!!é'VE😀🏽-'VE٣٤٥٦🙂　e'VE'\t0ß👍🏽字#$% \r\n", "tokens": 61, "pieces": ["'S", "\"", "Ⅳ", ".", "\ta", "३", " Ａ", "(㋿'", "VEß", "!!", "é", "'VE", "😀🏽-'", "VE", "٣٤٥", "٦", "🙂", "　e", "'VE", "'", "\t", "0", "ß", "👍🏽", "字", "#$%", " \r\n"]} +{"text": "12345678ſ​ #$%ꟲ !'VEm", "tokens": 16, "pieces": ["123", "456", "78", "ſ", "​", " ", " #$%", "ꟲ", " !'", "VEm"]} +{"text": "!　 \n字'Ａ 'ſ\r\nå", "tokens": 66, "pieces": ["!", "　 \n", "字", "'<", "META", "_START", ">Ａ", "", " ", " '", "ſ", "\r\n", "a", "̊"]} +{"text": "'ſ\tꟲ𐞁m9\"…\u000b­‍éع'reⅣ,‍٣٤٥٦('sDžZ'm👍🏽👍🏽", "tokens": 64, "pieces": ["'ſ", "\tꟲ𐞁m", "9", "\"", "…", "\u000b", "­<", "META", "_START", ">‍", "éع", "'", "re", "Ⅳ", ",<", "EOT", ">‍", "٣٤٥", "٦", "('", "sDžZ", "'m", "👍🏽👍🏽"]} +{"text": "'sＡ 'T'sfi9­'Re ḍ̇e'M­\r\"　EOT\u000btDž", "tokens": 37, "pieces": ["'s", "Ａ", " <", "EOT", ">'", "T", "'s", "fi", "9", "­'", "Re", " ", " ḋ", "̣e", "'M", "­\r", "\"", "　EOT", "\u000btDž", ""]} +{"text": "'S'VE😀🏽'D,½㍿a <<|endoftext|>'reß㍿👍🏽!'re
é…-
½ع𐞁\"fi,
́é>'D-😀🏽0,㋿\t", "tokens": 75, "pieces": ["'S", "'VE", "😀🏽'", "D", ",", "½", "㍿a", " ", " <<|", "endoftext", "|>'", "reß", "㍿👍🏽!'", "re", "
e", "́", "…", "-", "
", "½", "ع𐞁", "\"fi", ",", "
", "́e", "́>'", "D", "-😀🏽", "0", ",㋿", "\t"]} +{"text": "'S‍𐞁٣٤٥٦३Džaé　t<|fim_prefix|>Ａ\"-'Rea\r\né\r\n\u000b \u000b9'S'S>", "tokens": 49, "pieces": ["'S", "‍𐞁", "٣٤٥", "٦३", "Džae", "́", "　t", "<|", "fim", "_prefix", "|>", "Ａ", "\"-<", "EOT", ">'", "Rea", "\r\n", "é", "\r\n", "\u000b ", "\u000b", "9", "'S", "'S", ">"]} +{"text": "e'VE'Då'Sḍ̇👍🏽", "tokens": 27, "pieces": ["e", "'VE", "'D", "a", "̊'", "Sḋ", "̣👍🏽<", "EOT", "><", "EOT", ">"]} +{"text": "㍿\r\n'll٣٤٥٦at㋿>Dž😀🏽\n ", "tokens": 27, "pieces": ["㍿\r\n", "'ll", "٣٤٥", "٦", "at", "㋿>", "Dž", "😀🏽\n", " "]} +{"text": "'śé0३😀🏽 \n#$%,#$%<|endoftext|>-İ
!३", "tokens": 30, "pieces": ["'s", "́e", "́", "0३", "😀🏽", " \n", "#$%,#$%<|", "endoftext", "|>-", "İ", "
", "!", "३"]} +{"text": "ſéⅣ\r\n🙂Dž㍿", "tokens": 14, "pieces": ["ſe", "́", "Ⅳ", "\r\n", "🙂Dž", "㍿"]} +{"text": ".́\", fi\r\n\r\n३9 é!!㋿aé👍🏽!!'s३éꟲ\r!!'ſḍ̇ß🙂", "tokens": 45, "pieces": [".́\",", " fi", "\r\n\r\n", "३9", " ", " é", "!!㋿", "ae", "́👍🏽!!'", "s", "३", "éꟲ", "\r", "!!'", "ſḋ", "̣ß", "🙂"]} +{"text": "m'VEs's'll<Ａ👍🏽㋿'ſ\r\r\n\r\n.12345678\tع\r\n\r\n字🙂ḍ̇½'reſ\"!!!!́Ⅳfiꟲ<|fim_prefix|>\r‍s", "tokens": 63, "pieces": ["m", "'VE", "s", "'s", "'ll", "<Ａ", "👍🏽㋿'", "ſ", "\r\r\n\r\n", ".", "123", "456", "78", "\tع", "\r\n\r\n", "字", "🙂ḋ", "̣", "½", "'re", "ſ", "\"!!!!́", "Ⅳ", "fiꟲ", "<|", "fim", "_prefix", "|>\r", "‍s"]} +{"text": " \n\tDžḍ̇EOTe ㋿'Re𐞁\r'VEaDž<|endoftext|>9#$%ꟲå字're
!\r\n😀🏽'reＡaḍ̇ Dž 'M漢\r\n'ſ're", "tokens": 81, "pieces": [" \n", "\tDžḋ", "̣EOTe", " ", " <", "META", "_START", ">㋿'", "Re𐞁", "\r", "'VE", "aDž", "<|", "endoftext", "|>", "9", "#$%", "ꟲa", "̊字", "'re", "
", "!\r\n", "😀🏽'", "reＡaḋ", "̣", " ", " Dž", " ", "'M", "漢", "\r\n", "'ſ", "'re"]} +{"text": "漢'Ré'reEOT​m", "tokens": 10, "pieces": ["漢", "'Re", "́'", "reEOT", "​m"]} +{"text": "‍é'Dd……!e३>.<Ⅳ३३", "tokens": 27, "pieces": ["‍", "é", "'D", "d", "…", "…", "!<", "EOT", ">e", "३", ">.<", "Ⅳ३३"]} +{"text": "'TDž're0're\r\n'Re
Ⅳḍ̇ ㋿‍ß-'TdEOTfi å!(fiḍ̇ \n", "tokens": 43, "pieces": ["'T", "Dž", "'re", "0", "'re", "\r\n", "'Re", "
", "Ⅳ", "ḋ", "̣", " ", "㋿‍", "ß", "-'", "TdEOTfi", " ", " a", "̊!(", "fiḋ", "̣", " \n"]} +{"text": "'s½,­३d‍ꟲ\t漢㋿𐞁.t… \r\n\r\n9fi字­٣٤٥٦👍🏽fi👍🏽('s12345678\r\n\r\nEOT \n'ع#$%'ll½", "tokens": 69, "pieces": ["'s", "½", ",­", "३", "d", "‍ꟲ", "\t漢", "㋿𐞁", ".t", "… \r\n\r\n", "9", "fi字", "­", "٣٤٥", "٦", "👍🏽", "fi", "👍🏽('", "s", "123", "456", "78", "\r\n\r\n", "EOT", " \n", "'ع", "#$%'", "ll", "½"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\nå,a字字'Re'S😀🏽12345678 ſfiḍ̇'Re#$%㋿'D", "tokens": 39, "pieces": ["\r\n", "a", "̊<", "META", "_START", ">,", "a字字", "'Re", "'S", "😀🏽", "123", "456", "78", " ſfiḋ", "̣'", "Re", "#$%㋿'", "D"]} +{"text": "\r\néDž👍🏽漢", "tokens": 12, "pieces": ["\r\n", "éDž", "👍🏽", "漢"]} +{"text": "Ａ\r\n\r\n\r\n㍿عtZⅣ​d'retſ!<|fim_prefix|> 'VE,", "tokens": 36, "pieces": ["Ａ", "\r\n\r\n\r\n", "㍿عtZ", "Ⅳ", "​<", "EOT", ">d", "'re", "tſ", "!<", "EOT", "><", "META", "_START", "><|", "fim", "_prefix", "|>", " '", "VE", ","]} +{"text": " \n", "tokens": 1, "pieces": [" \n"]} +{"text": "t-\t", "tokens": 3, "pieces": ["t", "-", "\t"]} +{"text": "'M12345678é<|endoftext|>'Sḍ̇㍿é!ſḍ̇a'ſfi're#$%́", "tokens": 41, "pieces": ["'M", "123", "456", "78", "é", "<|", "endoftext", "|>'", "Sḋ", "̣㍿", "e", "́!", "ſḋ", "̣a", "'ſ", "fi", "'re", "#$%́"]} +{"text": "'T　ع(㋿\"­İ", "tokens": 13, "pieces": ["'T", "　ع", "(㋿<", "META", "_START", ">\"­", "İ"]} +{"text": "字'lĺ­!㍿'ſ<|fim_prefix|>½a0\t<|endoftext|>३😀🏽\r\n\r\n.m​'ſfiåEOT'‍'M\r\n\r\n字\r\n\r\n㋿s're", "tokens": 70, "pieces": ["字", "'ll", "́­!㍿'", "ſ", "<|", "fim", "_prefix", "|>", "½", "a", "0", "", "\t", "<|", "endoftext", "|>", "३", "😀🏽\r\n\r\n", ".m", "​'", "ſfia", "̊EOT", "'‍'", "M", "\r\n\r\n", "字", "\r\n\r\n", "㋿s", "'", "re"]} +{"text": ",…<|fim_prefix|>­　'D­\r\ne\u000b'SⅣ<|fim_prefix|>.\u000b😀🏽s'VEa\t𐞁'Rea Ⅳ", "tokens": 50, "pieces": [",", "…", "<|", "fim", "_prefix", "|>­", "　", "'D", "­\r\n", "e", "\u000b", "'S", "Ⅳ", "<|", "fim", "_prefix", "|>.", "\u000b", "😀🏽", "s", "'VE", "a", "", "\t𐞁", "'Re", "a", " ", "Ⅳ"]} +{"text": "­'s#$%\u000bİ", "tokens": 7, "pieces": ["­'", "s", "#$%", "\u000bİ"]} +{"text": "㋿'M​", "tokens": 6, "pieces": ["㋿'", "M", "​"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "字12345678㋿ß\u000b\u000b \n½t字'S9½\r🙂å३0½< ſ\r'sſ'sع\r-d‍　'refi's", "tokens": 57, "pieces": ["字", "123", "456", "78", "㋿ß", "\u000b\u000b \n", "½", "t字", "'S", "9½", "\r", "🙂a", "̊", "३0½", "<", " ", " ſ", "\r", "'s", "ſ", "'s", "ع", "\r", "-d", "‍", "　", "'re", "fi", "'s"]} +{"text": "0‍٣٤٥٦
0­éß漢d漢
漢's-ꟲås\r字'll-.İsＡ ​́", "tokens": 46, "pieces": ["0", "‍", "٣٤٥", "٦", "
", "0", "­e", "́ß漢d漢", "
漢", "'s", "-ꟲa", "̊s", "\r", "字", "'ll", "-.", "İsＡ", " ​́"]} +{"text": ". é३'re.>३'Mḍ̇'re👍🏽<|fim_prefix|>'T́漢12345678'VE\r\n\r\n👍🏽­'ſDž", "tokens": 53, "pieces": [".", " é", "३", "'re", ".>", "३", "'M", "ḋ", "̣'", "re", "👍🏽<|", "fim", "_prefix", "|>'", "T", "́漢", "123", "456", "78", "'VE", "\r\n\r\n", "👍🏽­'", "ſDž"]} +{"text": "\"ⅣEOT'VE
'VE!!…\u000bd'S>\"", "tokens": 18, "pieces": ["\"", "Ⅳ", "EOT", "'VE", "
", "'VE", "!!", "…", "\u000bd", "'S", ">\""]} +{"text": "­\nŹ\r\n", "tokens": 5, "pieces": ["­\n", "Z", "́\r\n"]} +{"text": ">'T㍿fi#$%­字Ⅳ0'M'İmⅣEOT-👍🏽 Ａ!EOTⅣ\n0\r\n'VEe\u000b ع\u000b漢🙂­", "tokens": 59, "pieces": [">'", "T", "㍿", "fi", "#$%­", "字", "Ⅳ0", "'M", "'<", "EOT", ">İm", "Ⅳ", "EOT", "-👍🏽", " ", " Ａ", "!EOT", "Ⅳ", "\n", "0", "\r\n", "'VE", "e", "\u000b ", " ع", "\u000b漢", "🙂­"]} +{"text": "tＡ \"s's\n's　­<\n𐞁,ＡEOT㍿Ａ\r\n\r\n½漢\rEOT'EOTs.", "tokens": 36, "pieces": ["tＡ", " ", "\"s", "'s", "\n", "'s", "　", "­<\n", "𐞁", ",ＡEOT", "㍿Ａ", "\r\n\r\n", "½", "漢", "\r", "EOT", "'EOTs", "."]} +{"text": "Džſ😀🏽😀🏽<>​<‍é 'ſEOT<|endoftext|>‍­'Sfi
", "tokens": 46, "pieces": ["Džſ", "😀🏽😀🏽<>​<‍", "é", " ", " '", "ſEOT", "<|", "endoftext", "|>‍­'", "Sfi", "", "
"]} +{"text": " Zåİ12345678字", "tokens": 10, "pieces": [" Za", "̊İ", "123", "456", "78", "字"]} +{"text": "#$%٣٤٥٦İé'Se‍fi!!Dž<'ſ ½", "tokens": 32, "pieces": ["#$%", "٣٤٥", "٦", "İé", "'S", "e", "‍fi", "!!", "Dž", "<'", "ſ", " <", "META", "_START", ">", "½", ""]} +{"text": "👍🏽s", "tokens": 7, "pieces": ["👍🏽", "s"]} +{"text": "#$%\r\n\r\ns٣٤٥٦<|fim_prefix|>३\u000bm9'Re\ta½<|fim_prefix|>\"\nZ​!!\n#$%ß\rm's\"'T", "tokens": 49, "pieces": ["#$%\r\n\r\n", "s", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "३", "\u000bm", "9", "'Re", "\ta", "", "½", "<|", "fim", "_prefix", "|>\"\n", "Z", "​!!\n", "#$%", "ß", "\r", "m", "'s", "\"'", "T"]} +{"text": "'re'T'S\ń\r\n​fi<ع!字12345678\n9'M‍😀🏽́'re\u000b漢😀🏽́'", "re", "\u000b漢", "
'rea \néEOT😀🏽", "tokens": 37, "pieces": ["e字", "<字", "'re", "٣٤٥", "٦ⅣⅣ", "<|", "fim", "_prefix", "|>", "
", "'re", "a", " \n", "éEOT", "😀🏽"]} +{"text": "٣٤٥٦'ſ<|endoftext|>0EOT👍🏽0  ­…å\r\"'T\r\nḍ̇éa-'re<|endoftext|>'S\r\n\r\n", "tokens": 59, "pieces": ["٣٤٥", "٦", "'ſ", "<|", "endoftext", "|>", "0", "EOT", "👍🏽", "0", " ", " ", "­", "…a", "̊\r", "\"'", "T", "\r\n", "ḋ", "̣e", "́a", "-'", "re", "<|", "endoftext", "|>'", "S", "\r\n\r\n"]} +{"text": "Dž​fi're'll", "tokens": 7, "pieces": ["Dž", "​fi", "'re", "'ll"]} +{"text": "\r\nå…,漢
fiß\" \n\r-𐞁'VE>\tm㋿'s
>- \n㍿", "tokens": 38, "pieces": ["\r\n", "a", "̊", "…", ",漢", "
fiß", "\"", " \n\r", "-𐞁", "'VE", ">", "\tm", "㋿'", "s", "
", ">-", " \n", "㍿"]} +{"text": "ma'0½३'D'rea'ſ\u000b😀🏽漢'Ḿ'D!…𐞁 \nee٣٤٥٦ǻ>", "tokens": 46, "pieces": ["ma", "'", "0½३", "'D", "'re", "a", "'ſ", "\u000b", "😀🏽", "漢", "'M", "́'", "D", "!", "…𐞁", " \n", "ee", "٣٤٥", "٦", "a", "̊́>"]} +{"text": "​ع<|endoftext|>㍿ \nZ\u000b\tß'Re\r\n\r\n३EOT", "tokens": 23, "pieces": ["​ع", "<|", "endoftext", "|>㍿", " \n", "Z", "\u000b", "\tß", "'Re", "\r\n\r\n", "३", "EOT"]} +{"text": "'s\tḍ̇ t
 ㋿-٣٤٥٦ße🙂😀🏽ß12345678😀🏽m<|endoftext|>٣٤٥٦字ſ(\r\néfi0é'T", "tokens": 73, "pieces": ["'s", "\tḋ", "̣", " t", "
 ", " ㋿<", "META", "_START", ">-", "٣٤٥", "٦", "ße", "🙂😀🏽", "ß", "123", "456", "78", "😀🏽", "m", "<|", "endoftext", "|>", "٣٤٥", "٦", "字ſ", "(\r\n", "éfi", "0", "e", "́'", "T"]} +{"text": "Ａ'VE<|fim_prefix|>", "tokens": 11, "pieces": ["Ａ", "'VE", "<|", "fim", "_prefix", "|>"]} +{"text": "fi‍Ａa'S-9㍿!­EOT.'M>\r\né", "tokens": 25, "pieces": ["fi", "‍", "Ａa", "'S", "-", "9", "㍿!­", "EOT", ".'", "M", ">\r\n", "é"]} +{"text": "ſ'VEḍ̇३å漢Ⅳ0m🙂👍🏽ꟲ㍿", "tokens": 34, "pieces": ["ſ", "'VE", "ḋ", "̣", "३", "a", "̊漢", "Ⅳ0", "m", "🙂👍🏽", "ꟲ", "㍿"]} +{"text": "'ſ'll<|endoftext|>< tm३ Z-<|endoftext|>'St.m", "tokens": 27, "pieces": ["'ſ", "'ll", "<|", "endoftext", "|><", " ", " tm", "३", " Z", "-<|", "endoftext", "|>'", "St", ".m"]} +{"text": "\"-Z<|endoftext|>'rea\n  Zéé'Taé0'Méå　Dž<|fim_prefix|>ⅣDž9's", "tokens": 43, "pieces": ["\"-", "Z", "<|", "endoftext", "|>'", "rea", "\n", " ", " Zée", "́'", "Tae", "́", "0", "'M", "e", "́a", "̊", "　Dž", "<|", "fim", "_prefix", "|>", "Ⅳ", "Dž", "9", "'s"]} +{"text": "Ⅳ😀🏽\r\ndé½\r\n\r\né'D's(", "tokens": 16, "pieces": ["Ⅳ", "😀🏽\r\n", "de", "́", "½", "\r\n\r\n", "é", "'D", "'s", "("]} +{"text": " (🙂're漢İⅣ \nß'Tİ'T'M", "tokens": 65, "pieces": [" ", "(🙂'", "re漢İ", "Ⅳ", "", " \n", "ß", "'T", "İ", "'T", "'M"]} +{"text": "\r\n\r\n!!\"\t　́\tſEOT\t漢𐞁\"ع's㋿(­३'MDž\t\nع\té>d'Sİ\"\n", "tokens": 43, "pieces": ["\r\n\r\n", "!!\"", "\t", "　", "́", "\tſEOT", "\t漢𐞁", "\"<", "EOT", ">ع", "'s", "㋿(­", "३", "'M", "Dž", "\t\n", "ع", "\te", "́>", "d", "'S", "İ", "\"\n"]} +{"text": "\r\n\r\nİ字'T㍿å \n‍.fi\r!!!t!<,Z​́Z", "tokens": 25, "pieces": ["\r\n\r\n", "İ字", "'T", "㍿a", "̊", " \n", "‍.", "fi", "\r", "!!!", "t", "!<,", "Z", "​́", "Z"]} +{"text": "s 'D٣٤٥٦Z\" 's<|endoftext|>\t#$%\n \n'llé'MZm𐞁'D'T", "tokens": 49, "pieces": ["s", " ", " '", "D", "٣٤٥", "٦", "Z", "\"", " ", " '", "s", "<|", "endoftext", "|>", "\t", "#$%\n", " \n", "'ll", "e", "́<", "META", "_START", ">'", "MZm𐞁", "'D", "'", "T", ""]} +{"text": "🙂<|fim_prefix|>🙂'Re12345678\n>Dž#$%'s12345678!é字عa👍🏽Ａ字a漢\u000b0\t#$%", "tokens": 64, "pieces": ["🙂<", "EOT", "><", "me", "́!!'", "M", "<|", "endoftext", "|><|", "fim", "_prefix", "|>🙂'", "Re", "123", "456", "78", "\n", ">Dž", "#$%'", "s", "123", "456", "78", "!e", "́字عa", "👍🏽", "Ａ字a漢", "\u000b", "0", "\t", "#$%"]} +{"text": "‍½'D'D‍sDž㋿-'s\"́é'D d\u000bعEOT​㋿", "tokens": 29, "pieces": ["‍", "½", "'D", "'D", "‍sDž", "㋿-'", "s", "\"́", "é", "'D", " d", "\u000bعEOT", "​㋿"]} +{"text": "å'Tt<|fim_prefix|>字𐞁𐞁\u000b'TⅣ12345678é字", "tokens": 34, "pieces": ["a", "̊'", "T", "t", "<|", "fim", "_prefix", "|>", "字𐞁𐞁", "\u000b", "'T", "Ⅳ12", "345", "678", "e", "́字"]} +{"text": ">😀🏽ꟲ", "tokens": 9, "pieces": [">😀🏽", "ꟲ"]} +{"text": "ß🙂　!İ'ſ<|endoftext|>'re-Zß9ḍ̇\n\téع٣٤٥٦(éEOT>㍿é#$%-m12345678ſDžİ", "tokens": 57, "pieces": ["ß", "🙂", "　", "!İ", "'ſ", "<|", "endoftext", "|>'", "re", "-Zß", "9", "ḋ", "̣\n", "\te", "́ع", "٣٤٥", "٦", "(éEOT", ">㍿", "é", "#$%-", "m", "123", "456", "78", "ſDžİ"]} +{"text": "9'́<|fim_prefix|>🙂\"!​ꟲꟲ\r\n\r\n'reße0Z'D漢", "tokens": 29, "pieces": ["9", "'́<|", "fim", "_prefix", "|>🙂\"!​", "ꟲꟲ", "\r\n\r\n", "'re", "ße", "0", "Z", "'D", "漢"]} +{"text": "!0're,'s'D.'VE#$%👍🏽😀🏽İ​🙂'Sé 'MEOT<٣٤٥٦'s\u000bfi0<|endoftext|>عé", "tokens": 55, "pieces": ["!", "0", "'re", ",'", "s", "'D", ".'", "VE", "#$%👍🏽😀🏽", "İ", "​🙂'", "Sé", " ", " '", "MEOT", "<", "٣٤٥", "٦", "'s", "\u000bfi", "0", "<|", "endoftext", "|>", "عé"]} +{"text": "\u000be㋿👍🏽'D٣٤٥٦'reع'M'll
12345678Ⅳa漢㍿>Ⅳ.-'D'VEs ३­\n", "tokens": 53, "pieces": ["\u000be", "㋿👍🏽'", "D", "٣٤٥", "٦", "'re", "ع", "'M", "'ll", "
", "123", "456", "78Ⅳ", "a漢", "㍿>", "Ⅳ", ".-'", "D", "'VE", "s", " ", " ", "३", "­\n"]} +{"text": "é>'12345678㋿½٣٤٥٦३'Re\"t　!!‍ ́", "tokens": 32, "pieces": ["é", ">'", "123", "456", "78", "㋿", "½٣٤", "٥٦३", "'Re", "\"t", "　", "!!‍", " ", "́<", "EOT", ">"]} +{"text": "Ź's𐞁e字Ａ9­ ßＡ漢tå\rd'MDž<", "tokens": 29, "pieces": ["Z", "́'", "s𐞁e字Ａ", "9", "­", " ßＡ漢ta", "̊\r", "d", "'M", "Dž", "<"]} +{"text": "\r\n\u000bEOTa'sEOT½0​½'VE-\r\t(👍🏽\u000bꟲ\"İ३\néİ <|endoftext|>'M0", "tokens": 49, "pieces": ["\r\n", "\u000bEOTa", "'s", "EOT", "½0", "​", "½", "'VE", "-\r", "\t", "(👍🏽<", "META", "_START", ">", "\u000bꟲ", "\"İ", "३", "\n", "éİ", " ", "<|", "endoftext", "|>'", "M", "0"]} +{"text": "㋿'T ae🙂ſ0٣٤٥٦👍🏽m,‍\ré'VE Ⅳ'Så­>\n'Re३'Re<​.㍿'Tß'T👍🏽", "tokens": 63, "pieces": ["㋿'", "T", " ae", "🙂ſ", "0٣٤", "٥٦", "👍🏽", "m", ",‍\r", "e", "́'", "VE", " ", "Ⅳ", "'S", "a", "̊­>\n", "'Re", "३", "'Re", "<​.㍿'", "Tß", "'T", "👍🏽"]} +{"text": "<|fim_prefix|>", "tokens": 7, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "a!!EOT'", "tokens": 5, "pieces": ["a", "!!", "EOT", "'"]} +{"text": "ſ🙂å漢ſéfi ", "tokens": 20, "pieces": ["ſ", "🙂a", "̊漢ſe", "́<", "META", "_START", ">fi", " "]} +{"text": "́́
0Dž‍㍿sع'Re'sm'Re
<|fim_prefix|>३å३字👍🏽\n", "tokens": 42, "pieces": ["́́", "
", "0", "Dž", "‍㍿", "sع", "'Re", "'s", "m", "'Re", "
", "<|", "fim", "_prefix", "|>", "३", "a", "̊", "३", "字", "👍🏽\n"]} +{"text": "t\né\t'å \n​,'M\"\u000b½m㋿ \n<‍ß12345678d　Z'VE'ſＡ٣٤٥٦'M#$%éDž", "tokens": 51, "pieces": ["t", "\n", "e", "́", "\t", "'a", "̊", " \n", "​,'", "M", "\"", "\u000b", "½", "m", "㋿", " \n", "<‍", "ß", "123", "456", "78", "d", "　Z", "'VE", "'ſ", "Ａ", "٣٤٥", "٦", "'M", "#$%", "éDž"]} +{"text": "'T<'S!!\r\nZDž.㍿ås'em\n'T 12345678'T9🙂 \n's㍿\"'D9's\r.ſ12345678👍🏽", "tokens": 49, "pieces": ["'T", "<'", "S", "!!\r\n", "ZDž", ".㍿", "a", "̊s", "'em", "\n", "'T", " ", "123", "456", "78", "'T", "9", "🙂", " \n", "'s", "㍿\"'", "D", "9", "'s", "\r", ".ſ", "123", "456", "78", "👍🏽"]} +{"text": "me<|fim_prefix|>#$%é​", "tokens": 13, "pieces": ["me", "<|", "fim", "_prefix", "|>#$%", "e", "́​"]} +{"text": "\tst…\reßé 'ſé'S\tḍ̇", "tokens": 21, "pieces": ["\tst", "…\r", "eßé", " ", " '", "ſe", "́'", "S", "\tḋ", "̣"]} +{"text": "İaé<|endoftext|>#$%<'ſs\n", "tokens": 17, "pieces": ["İaé", "<|", "endoftext", "|>#$%<'", "ſs", "\n"]} +{"text": "é\n eZ's­👍🏽 ​'T.mm🙂d<|endoftext|> 's\"!é­ ‍\".", "tokens": 39, "pieces": ["e", "́\n", " eZ", "'s", "­👍🏽", " ", " ​'", "T", ".mm", "🙂d", "<|", "endoftext", "|>", " '", "s", "\"!", "é", "­", " ", " ‍\"."]} +{"text": "🙂​​\u000bⅣ-<|fim_prefix|>'VE½½'ſ…å'll\"😀🏽 \n३Ａta<½ \n<", "tokens": 43, "pieces": ["🙂​​", "\u000b", "Ⅳ", "-<|", "fim", "_prefix", "|>'", "VE", "½½", "'ſ", "…a", "̊'", "ll", "\"😀🏽", " \n", "३", "Ａta", "<", "½", " \n", "<"]} +{"text": " e'VE👍🏽­EOT<|fim_prefix|>漢ßå​.'re<|fim_prefix|>​😀🏽‍½ع𐞁,é　9­㍿
.…\n ٣٤٥٦aＡ<|fim_prefix|>s ", "tokens": 85, "pieces": [" ", " e", "'VE", "👍🏽­", "EOT", "<|", "fim", "_prefix", "|>", "漢ßa", "̊​.'", "re", "<|", "fim", "_prefix", "|>​😀🏽‍", "½", "ع𐞁", ",é", "　", "9", "­㍿", "
", ".", "…\n", " ", "٣٤٥", "٦", "aＡ", "<|", "fim", "_prefix", "|>", "s", " "]} +{"text": "é३!!-a12345678!!", "tokens": 10, "pieces": ["é", "३", "!!-", "a", "123", "456", "78", "!!"]} +{"text": " \n're.", "tokens": 7, "pieces": [" \n", "'", "re", "."]} +{"text": "'T'DＡ३𐞁'T", "tokens": 11, "pieces": ["'T", "'D", "Ａ", "३", "𐞁", "'T"]} +{"text": " <|endoftext|>", "tokens": 8, "pieces": [" ", "<|", "endoftext", "|>"]} +{"text": "👍🏽…­漢\r\nⅣ åZＡZ'T<|fim_prefix|>\n\nsZm\" EOTZ\r\n\r\n
'll 
Z👍🏽漢㋿٣٤٥٦😀🏽EOTꟲ0're½‍", "tokens": 79, "pieces": ["👍🏽", "…", "­漢", "\r\n", "Ⅳ", " a", "̊ZＡZ", "'T", "<|", "fim", "_prefix", "|>\n\n", "sZm", "\"", " ", " EOTZ", "\r\n\r\n", "
", "'ll", " ", "
Z", "👍🏽", "漢", "㋿", "٣٤٥", "٦", "😀🏽", "EOTꟲ", "0", "'re", "½", "‍"]} +{"text": "🙂as\nå\r.å​Dž(\t​!!Ⅳd‍>㋿#$%​\u000b字'll<|endoftext|>fi!,", "tokens": 52, "pieces": ["🙂<", "META", "_START", ">as", "\n", "a", "̊\r", ".a", "̊​", "Dž", "(", "\t", "​!!", "Ⅳ", "d", "‍<", "EOT", ">>㋿#$%​", "\u000b字", "'ll", "<|", "endoftext", "|>", "fi", "!,"]} +{"text": "\"٣٤٥٦å٣٤٥٦'s éZ!!👍🏽( 12345678İ́ <|endoftext|>\r\n\r\n< …\r\n\r\n'T'٣٤٥٦㋿😀🏽<|endoftext|>٣٤٥٦te'MmꟲＡ", "tokens": 89, "pieces": ["\"", "٣٤٥", "٦", "a", "̊", "٣٤٥", "٦", "'s", " ", " éZ", "!!👍🏽(", " ", "123", "456", "78", "İ", "́", " <|", "endoftext", "|>\r\n\r\n", "<", " …\r\n\r\n", "'T", "'", "٣٤٥", "٦", "㋿😀🏽<|", "endoftext", "|>", "٣٤٥", "٦", "te", "'M", "mꟲＡ"]} +{"text": ">\u000b😀🏽漢́s-'M'sEOT 漢9#$%ß👍🏽,", "tokens": 31, "pieces": [">", "\u000b", "😀🏽", "漢", "́s", "-'", "M", "'s", "EOT", " ", " 漢", "9", "#$%", "ß", "👍🏽,"]} +{"text": "́> DžDžEOTfi\r\n🙂a'Re >EOTＡEOT.Z0<|fim_prefix|><|endoftext|>.\"é ss'D0‍ å…", "tokens": 53, "pieces": ["́>", " ", " DžDžEOTfi", "\r\n", "🙂a", "'Re", " ", ">EOTＡEOT", ".Z", "0", "<|", "fim", "_prefix", "|><|", "endoftext", "|>.\"", "e", "́", " ss", "'D", "0", "‍", " ", " a", "̊", "…"]} +{"text": "İ'T'sd!å½ḍ̇㋿ع're👍🏽é漢ß\n'VE​Ⅳ\t㍿'VE!,٣٤٥٦'ſ", "tokens": 67, "pieces": ["İ", "'T", "'s", "d", "!<", "m漢ß", "<|", "endoftext", "|>", "a", "̊", "½", "ḋ", "̣㋿", "ع", "'re", "👍🏽", "é漢ß", "\n", "'VE", "​", "Ⅳ", "\t", "㍿'", "VE", "!,", "٣٤٥", "٦", "'ſ"]} +{"text": "EOT𐞁afi　 \taDžfí 字‍ß \n", "tokens": 22, "pieces": ["EOT𐞁afi", "　 ", "\taDžfi", "́", " 字", "‍ß", " \n"]} +{"text": "'VE\n'ſ'Deéḍ̇\"𐞁Z'ſ ,12345678 \n
Zꟲ 9😀🏽.…!!!eDž<-,　", "tokens": 51, "pieces": ["'VE", "\n", "'ſ", "'D", "eéḋ", "̣\"", "𐞁Z", "'ſ", " ,", "123", "456", "78", " \n", "
Zꟲ", " ", "9", "😀🏽.", "…", "!!!", "eDž", "<-,", "　"]} +{"text": "EOTe'T'ſ𐞁<|endoftext|>\n0ZZع(\u000b>½m ", "tokens": 30, "pieces": ["EOTe", "'T", "'ſ", "𐞁", "<|", "endoftext", "|>\n", "0", "Z", "Zع", "(", "\u000b", ">", "½", "m", " "]} +{"text": "sZ‍<|fim_prefix|>'Re!'Té　\r🙂ea'D'D<㍿é́ e'D'T'Tſ(EOT😀🏽\r\n\r\n", "tokens": 46, "pieces": ["sZ", "‍<|", "fim", "_prefix", "|>'", "Re", "!'", "Té", "　\r", "🙂ea", "'D", "'D", "<㍿", "e", "́́", " <", "EOT", ">e", "'D", "'T", "'T", "ſ", "(EOT", "😀🏽\r\n\r\n"]} +{"text": "'T<İ'S
​('sＡ३'lled(", "tokens": 44, "pieces": ["ſe", "-", "٣٤٥", "٦", "'s", "Ａ", "३", "'ll", "ed", "("]} +{"text": "''Re'D\t​eſ́.<12345678'S'VE\"ḍ̇'Re​'Reḍ̇s<fiDž''Te🙂ꟲſ'D(​'VEé9́", "tokens": 59, "pieces": ["''", "Re", "'D", "\t", "​eſ", "́.<", "123", "456", "78", "'S", "'VE", "\"ḋ", "̣'", "Re", "​'", "Reḋ", "̣s", "<fiDž", "''", "Te", "🙂ꟲſ", "'", "D", "(​'", "VEé", "9", "́"]} +{"text": ",\t\"\re>Ａé!!\r\n\r\n'ś#$%'reEOT9!'S'llEOT\u000b", "tokens": 25, "pieces": [",", "\t", "\"\r", "e", ">Ａé", "!!\r\n\r\n", "'s", "́#$%'", "reEOT", "9", "!'", "S", "'ll", "EOT", "\u000b"]} +{"text": "#$%İa'Dß \n
­
㋿…'Re ", "tokens": 19, "pieces": ["#$%", "İa", "'D", "ß", " \n", "
", "­", "
", "㋿", "…", "'Re", " "]} +{"text": "'s　'DEOTé0'll👍🏽mⅣ\u000b'Re'D\n're", "tokens": 25, "pieces": ["'s", "　", "'D", "EOTé", "0", "'ll", "👍🏽", "m", "Ⅳ", "\u000b", "'Re", "'", "D", "\n", "'re"]} +{"text": "a'Ds'VEⅣ're　'VE 9​'Re", "tokens": 21, "pieces": ["a", "'D", "s", "'VE", "", "Ⅳ", "'re", "　", "'VE", " ", " ", "9", "​'", "Re"]} +{"text": "'s­👍🏽…fi\u000b漢,\r\n'reع!!-fis", "tokens": 23, "pieces": ["'s", "­👍🏽", "…fi", "\u000b漢", ",\r\n", "'re", "ع", "!!-", "fis"]} +{"text": ".m　 \u000b㋿d'ſ́'D𐞁\r\n'Mså​9EOT㋿\"9.
عa", "tokens": 40, "pieces": [".m", "　 ", "\u000b", "㋿d", "'ſ", "́'", "D𐞁", "\r\n", "'M", "sa", "̊​", "9", "EOT", "㋿\"", "9", ".", "
عa"]} +{"text": " \nßＡ'M 9'S　­'Re're're漢漢's<|fim_prefix|>éé\rDž…ß#$%👍🏽­
'D\r\t#$%Z,'VE३#$%,'re-", "tokens": 65, "pieces": [" \n", "ßＡ", "'M", " ", " ", "9", "'S", "　", "­'", "Re", "'re", "'re", "漢漢", "'s", "<|", "fim", "_prefix", "|>", "éé", "\r", "Dž", "…ß", "#$%👍🏽­", "
", "'D", "\r", "\t", "#$%", "Z", ",'", "VE", "३", "#$%,'", "re", "-"]} +{"text": "㍿ß½m\u000b<㋿'ſ 'Ａ　ſ漢d\råtEOT
‍>", "tokens": 38, "pieces": ["㍿ß", "½", "m", "\u000b", "<㋿'", "ſ", " '", "Ａ", "　ſ", "漢d", "\r", "a", "̊tEOT", "
", "‍>"]} +{"text": "\"é३🙂'Reḍ̇!Ａ're", "tokens": 17, "pieces": ["\"é", "३", "🙂'", "Reḋ", "̣!", "Ａ", "'re"]} +{"text": "३Ａ\"m<|fim_prefix|><12345678t'M<|endoftext|>'s㋿٣٤٥٦…\r\n\r\n12345678ś", "tokens": 45, "pieces": ["३", "Ａ", "\"m", "<|", "fim", "_prefix", "|><", "123", "456", "78", "t", "'M", "<|", "endoftext", "|>'", "s", "㋿", "٣٤٥", "٦", "…\r\n\r\n", "123", "456", "78", "s", "́"]} +{"text": "'ll🙂'٣٤٥٦éEOTfi漢EOT<|fim_prefix|>\r\n\r\n \n'VEDž\n<|fim_prefix|>", "tokens": 42, "pieces": ["'ll", "🙂'", "٣٤٥", "٦", "e", "́EOTfi漢EOT", "<|", "fim", "_prefix", "|>\r\n\r\n", " \n", "'VE", "Dž", "\n", "<|", "fim", "_prefix", "|>"]} +{"text": "'re३Dž's!! \n𐞁'll'SEOT9!!㍿éDž\r0é", "tokens": 28, "pieces": ["'re", "३", "Dž", "'s", "!!", " \n", "𐞁", "'ll", "'S", "EOT", "9", "!!㍿", "e", "́Dž", "\r", "0", "é"]} +{"text": "'D-
'12345678́<|endoftext|>#$%٣٤٥٦㍿<|endoftext|>'VE\"åDž,é
\r…'𐞁​́Dž
字　Dž", "tokens": 64, "pieces": ["'D", "-", "
", "'", "123", "456", "78", "́<|", "endoftext", "|>#$%", "٣٤٥", "٦", "㍿<|", "endoftext", "|>'", "VE", "\"a", "̊Dž", ",e", "́", "
\r", "…", "'𐞁", "​́", "Dž", "
字", "　Dž"]} +{"text": "​‍\r\n\r\n\"EOT\n\r\n🙂 . ㋿ع́éعع>İsعİ́ 'ſ…́é字", "tokens": 39, "pieces": ["​‍\r\n\r\n", "\"EOT", "\n", "\r\n", "🙂", " ", ".", " ", "㋿ع", "́éعع", ">İsعİ", "́", " ", "'ſ", "…", "́e", "́字"]} +{"text": "\n\n#$% \n\r\tع \nⅣmعs३٣٤٥٦
<\r\n.ḍ̇👍🏽", "tokens": 39, "pieces": ["\n\n", "#$%", " \n\r", "\tع", " \n", "Ⅳ", "mعs", "३٣٤", "٥٦", "
", "<\r\n", ".ḋ", "̣👍🏽"]} +{"text": "('VE㋿fiḍ̇　", "tokens": 17, "pieces": ["('", "VE", "㋿<", "META", "_START", ">fiḋ", "̣", "　"]} +{"text": "m't", "tokens": 2, "pieces": ["m", "'t"]} +{"text": "9\r#$% 'Re𐞁\rt#$%'SEOTİ𐞁é'0sḍ̇!!'ſtꟲ9a<|fim_prefix|>\"㋿'Re㋿, ٣٤٥٦👍🏽-e👍🏽Ⅳ㋿​", "tokens": 86, "pieces": ["9", "\r", "#$%", " ", "'Re", "𐞁", "\r", "t", "#$%'", "SEOTİ𐞁e", "́'", "0", "sḋ", "̣!!'", "ſtꟲ", "9", "a", "<|", "fim", "_prefix", "|>\"㋿'", "Re", "㋿,", " ", "٣٤٥", "٦", "👍🏽-", "e", "👍🏽", "Ⅳ", "㋿​"]} +{"text": "𐞁​\n12345678.𐞁0d𐞁㍿३e½Ａ३'re", "tokens": 32, "pieces": ["𐞁", "​\n", "123", "456", "78", ".𐞁", "0", "d𐞁", "㍿", "३", "e", "½", "Ａ", "३", "'re"]} +{"text": "ßé'Re ́#$%'Sm
0'12345678's0\r\ńⅣEOT㍿ \naſİꟲ漢(", "tokens": 38, "pieces": ["ßé", "'Re", " ", " ́#$%'", "Sm", "
", "0", "'", "123", "456", "78", "'s", "0", "\r\n", "́", "Ⅳ", "EOT", "㍿", " \n", "aſİꟲ漢", "("]} +{"text": "t\rſ#$%'Mꟲ'ſ​!!EOT0", "tokens": 18, "pieces": ["t", "\r", "ſ", "#$%'", "Mꟲ", "'ſ", "​!!", "EOT", "0"]} +{"text": "‍afi\n0漢", "tokens": 9, "pieces": ["‍afi", "\n", "0", "漢"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'Smع'll\r\n'VE\r\n\r\n😀🏽\t\nſ漢㍿
'Re.ع\r\n\r\n", "😀🏽", "\t\n", "ſ漢", "㍿", "
", "'Re", ".ع", "½ḍ̇Dž​(ＡDž'T9𐞁.'Ts㋿<|fim_prefix|>‍Dž½>ſ,Z'Ms.٣٤٥٦d>'D're'ſ😀🏽9<|endoftext|>a", "tokens": 76, "pieces": ["", "½", "ḋ", "̣Dž", "​(", "ＡDž", "'T", "9", "𐞁", ".'", "Ts", "㋿<|", "fim", "_prefix", "|>‍", "Dž", "½", ">ſ", ",Z", "'M", "s", ".", "٣٤٥", "٦", "d", ">'", "D", "'re", "'ſ", "😀🏽", "9", "<|", "endoftext", "|>", "a"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "　İ­İſDž\u000b", "tokens": 9, "pieces": ["　İ", "­İſDž", "\u000b"]} +{"text": "३\r\n\r\n㋿Ａ٣٤٥٦eعs!!'s㋿½Ⅳ'Té\r\n\r\n\"'D!<|fim_prefix|>عſ\r\n\r\n\r\ń́\r,\n　\n<३", "tokens": 62, "pieces": ["३", "\r\n\r\n", "㋿Ａ", "٣٤٥", "٦", "eعs", "!!'", "s", "㋿", "½Ⅳ", "'T", "e", "́\r\n\r\n", "\"'", "D", "!<|", "fim", "_prefix", "|>", "عſ", "\r\n\r\n\r\n", "́́\r", ",\n", "　\n", "<", "३", ""]} +{"text": "Z>sé\t\r\nſß<|endoftext|>…!😀🏽0#$%😀🏽é <|endoftext|> \n99'VE
å٣٤٥٦-fi漢Ⅳ‍'D're㋿'Re,", "tokens": 79, "pieces": ["Z", ">sé", "\t\r\n", "ſß", "<|", "endoftext", "|>", "…", "!😀🏽", "0", "#$%😀🏽", "é", " <|", "endoftext", "|>", " \n", "99", "'VE", "
a", "̊<", "EOT", ">", "٣٤٥", "٦", "-fi漢", "Ⅳ", "‍'", "D", "'re", "㋿'", "Re", ","]} +{"text": "é 👍🏽عdꟲ'T'Re", "tokens": 15, "pieces": ["é", " ", "👍🏽", "عdꟲ", "'T", "'Re"]} +{"text": "'Da\u000b#$% \nZ're <|endoftext|>…​Z 0.\u000b㋿e", "tokens": 26, "pieces": ["'D", "a", "\u000b", "#$%", " \n", "Z", "'re", " <|", "endoftext", "|>", "…", "​Z", " ", "0", ".", "\u000b", "㋿e"]} +{"text": "\u000b३𐞁'M…­\n㋿", "tokens": 15, "pieces": ["\u000b", "३", "𐞁", "'M", "…", "­\n", "㋿"]} +{"text": "'s\n'SZ㍿", "tokens": 7, "pieces": ["'s", "\n", "'S", "Z", "㍿"]} +{"text": "👍🏽!!‍\"ß\"ḍ̇\r\n\r\n३fi", "tokens": 22, "pieces": ["👍🏽!!‍\"", "ß", "\"ḋ", "̣\r\n\r\n", "३", "fi"]} +{"text": "éé'll \nＡ !-", "tokens": 12, "pieces": ["e", "́e", "́'", "ll", " \n", "Ａ", " ", " !-"]} +{"text": "'Re 'ſtḍ̇… 9漢ß字!#$%𐞁…a12345678㋿9", "tokens": 40, "pieces": ["'Re", " ", " '", "ſtḋ", "̣", "… ", " ", "9", "漢ß字", "!#$%", "𐞁", "…a", "", "123", "456", "78", "㋿", "9"]} +{"text": "#$%字　Dž\r", "tokens": 7, "pieces": ["#$%", "字", "　Dž", "\r"]} +{"text": "\" \n㋿<|fim_prefix|>EOT\r\n\r\n　", "tokens": 16, "pieces": ["\"", " \n", "㋿<|", "fim", "_prefix", "|>", "EOT", "\r\n\r\n　"]} +{"text": "\r\n\r\n㋿👍🏽!'VEß12345678'll\r'ſ!!ß'DⅣ", "tokens": 30, "pieces": ["\r\n\r\n", "㋿👍🏽!'", "VEß", "123", "456", "78", "'ll", "\r", "'ſ", "!!<", "META", "_START", ">ß", "'D", "Ⅳ"]} +{"text": "EOT😀🏽👍🏽<|fim_prefix|>12345678t'D0!ꟲ0fi…12345678👍🏽é㋿12345678.", "tokens": 56, "pieces": ["EOT", "😀🏽👍🏽<|", "fim", "_prefix", "|>", "123", "456", "78", "t", "'D", "0", "!ꟲ", "", "0", "fi", "…", "123", "456", "78", "👍🏽", "e", "́㋿", "123", "456", "78", "."]} +{"text": "<|endoftext|> <|endoftext|>0　'reع'Ma½", "tokens": 21, "pieces": ["<|", "endoftext", "|>", " ", " <|", "endoftext", "|>", "0", "　", "'re", "ع", "'M", "a", "½"]} +{"text": "\r's‍(. .\n👍🏽0Ａ<|endoftext|>'D\ré­\r\n\r\ń​'D'T", "tokens": 37, "pieces": ["\r", "'s", "‍(.", " ", ".\n", "👍🏽", "0", "Ａ", "<|", "endoftext", "|>'", "D", "\r", "e", "́­\r\n\r\n", "́​'", "D", "'T"]} +{"text": "'ReDž-…🙂漢Z漢
　'll.\r३é​'VE\u000b\"'reé'S 'VE漢0ß👍🏽😀🏽ع\t‍😀🏽ꟲ", "tokens": 71, "pieces": ["'Re", "Dž", "-", "…", "🙂漢Z漢", "
", "　", "'", "ll", ".\r", "३", "é", "​'", "VE", "\u000b", "\"'", "ree", "́'", "S", " '", "VE漢", "0", "ß", "👍🏽😀🏽", "ع", "\t", "‍<", "EOT", ">😀🏽", "ꟲ"]} +{"text": "0<>'Tİ.ḍ̇åDž\r\n\r\nعＡ", "tokens": 20, "pieces": ["0", "<>'", "Tİ", ".ḋ", "̣a", "̊Dž", "\r\n\r\n", "عＡ"]} +{"text": "s9ḍ̇字𐞁'll'३Ａ é　Z‍½s>'Mİa'Sé#$%té'<|endoftext|>", "tokens": 44, "pieces": ["s", "9", "ḋ", "̣字𐞁", "'ll", "'", "३", "Ａ", " é", "　Z", "‍", "½", "s", ">'", "Mİa", "'S", "e", "́#$%", "té", "'<|", "endoftext", "|>"]} +{"text": "𐞁å
t'llZéDž…e'TعEOT#$%#$%\u000b​. EOTa> ́'(ß<|endoftext|>㋿'re😀🏽-.", "tokens": 64, "pieces": ["𐞁a", "̊", "
t", "'ll", "Ze", "́Dž", "…e", "'T", "عEOT", "#$%#$%", "\u000b", "​.", " EOTa", ">", " ", " ́'(", "ß", "<|", "endoftext", "|>㋿'", "re", "😀🏽<", "EOT", ">-."]} +{"text": "Dž٣٤٥٦EOT\r٣٤٥٦　字å𐞁👍🏽ſ0​ '㋿<|fim_prefix|>𐞁#$%字'llＡ\t#$%å \né<|fim_prefix|>㍿#$%s'VEع're", "tokens": 91, "pieces": ["Dž", "٣٤٥", "٦", "EOT", "\r", "٣٤٥", "٦", "　字a", "̊𐞁", "👍🏽", "ſ", "0", "​", " ", " '㋿<|", "fim", "_prefix", "|>", "𐞁", "#$%", "字", "'ll", "Ａ", "\t", "#$%", "a", "̊", " \n", "e", "́<|", "fim", "_prefix", "|>㍿#$%", "s", "'VE", "ع", "'re", ""]} +{"text": "…\n½ >'D\r\n\r\n\t­-é\t9​'ll٣٤٥٦mſ漢-Dž٣٤٥٦İEOT'½\u000b\r\n٣٤٥٦漢.ḍ̇ß½", "tokens": 69, "pieces": ["…\n", "½", " >'", "D", "\r\n\r\n", "\t", "­-", "e", "́", "\t", "9", "​'", "ll", "٣٤٥", "٦", "mſ漢", "-Dž", "٣٤٥", "٦", "İEOT", "'", "½", "\u000b\r\n", "٣٤٥", "٦", "漢", ".ḋ", "̣ß", "½"]} +{"text": " \n'Re㋿👍🏽éZ½aſ́-ſ'll'Re🙂(<|fim_prefix|>'\rfi'VEDžİ㋿d!é>", "tokens": 47, "pieces": [" \n", "'Re", "㋿👍🏽", "éZ", "½", "aſ", "́-", "ſ", "'ll", "'Re", "🙂(<|", "fim", "_prefix", "|>'\r", "fi", "'VE", "Džİ", "㋿d", "!é", ">"]} +{"text": " \n''VEḍ̇'T", "tokens": 14, "pieces": [" \n", "''", "VE", "ḋ", "̣'", "T"]} +{"text": "!!#$%\n0é٣٤٥٦å.(<
12345678'ſa\n漢Ａḍ̇e　t('ſå'll\"Z'VE \"\té#$%mt", "tokens": 63, "pieces": ["!!#$%<", "EOT", ">\n", "0", "e", "́", "٣٤٥", "٦", "a", "̊.(<", "
", "123", "456", "78", "'ſ", "a", "\n", "漢Ａḋ", "̣e", "　t", "('", "ſa", "̊'", "ll", "\"Z", "'VE", " \"", "\te", "́#$%", "mt"]} +{"text": "'e߅!å''MⅣ㋿'VE", "tokens": 17, "pieces": ["'eß", "…", "!a", "̊''", "M", "Ⅳ", "㋿'", "VE"]} +{"text": "🙂fi<|fim_prefix|>👍🏽é'D'Re\r'ſ", "tokens": 26, "pieces": ["🙂fi", "<|", "fim", "_prefix", "|>👍🏽", "e", "́'", "D", "'Re", "\r", "'ſ"]} +{"text": "　́\r\ń\n😀🏽'ſfi'Re0éß", "tokens": 19, "pieces": ["　", "́\r\n", "́\n", "😀🏽'", "ſfi", "'Re", "0", "éß"]} +{"text": "㍿'re'SＡ字", "tokens": 9, "pieces": ["㍿'", "re", "'S", "Ａ字"]} +{"text": "é漢\"m#$%!!\r\n\r\n३👍🏽EOT\t漢 9 'D\"٣٤٥٦\r\n\r\n\r\nꟲ#$%́'M- \nعDž \n😀🏽9!", "tokens": 67, "pieces": ["é漢", "\"m", "#$%!!<", "EOT", ">\r\n\r\n", "३", "👍🏽", "EOT", "\t", "漢", " ", " ", "9", " '", "D", "\"", "٣٤٥", "٦", "\r\n\r\n\r\n", "ꟲ", "#$%́'", "M", "-", " \n", "عDž", " \n", "😀🏽", "9", "!<", "EOT", ">"]} +{"text": "d-\te d👍🏽,½‍åå٣٤٥٦", "tokens": 29, "pieces": ["d", "-", "\te", " d", "👍🏽,", "½", "‍a", "̊a", "̊", "٣٤٥", "٦"]} +{"text": "\r\nEOTś٣٤٥٦'D0'S👍🏽字", "tokens": 25, "pieces": ["\r\n", "EOT", "s", "́", "٣٤٥", "٦", "'D", "0", "'S", "👍🏽", "字"]} +{"text": ",é½", "tokens": 3, "pieces": [",e", "́", "½"]} +{"text": "eEOT👍🏽'ſ <|endoftext|>", "tokens": 19, "pieces": ["eEOT", "👍🏽'", "ſ", " ", " <|", "endoftext", "|>"]} +{"text": "ḍ̇\tß𐞁\r'VE\r\n12345678'llſ\u000bé३🙂.­'VEꟲ<|fim_prefix|>\u000b㍿9Z㍿\r\n\r\n0ſ >́'Mfi,'S漢\u000b'<|fim_prefix|>", "tokens": 61, "pieces": ["-", "9", "'ll", "‍!", "漢tDž", "<|", "endoftext", "|>'", "VEꟲ", "<|", "fim", "_prefix", "|>", "\u000b", "㍿", "9", "Z", "㍿\r\n\r\n", "0", "ſ", " ", ">́'", "Mfi", ",'", "S漢", "\u000b", "'<|", "fim", "_prefix", "|>"]} +{"text": "'D!'ll㍿aZ>­EOT­", "EOT", "　\t,'ll<'SⅣ漢'!!(a½#$%é㋿eEOT😀🏽t#$%9 \t٣٤٥٦<|endoftext|>'Re　a", "tokens": 71, "pieces": ["EOTs", "'VE", "#$%<|", "fim", "_prefix", "|>", "　", "\t", ",'", "ll", "<'", "S", "Ⅳ", "漢", "'!!(", "a", "½", "#$%", "e", "́㋿", "eEOT", "😀🏽", "t", "#$%", "9", " ", "\t", "٣٤٥", "٦", "<|", "endoftext", "|>'", "Re", "　a"]} +{"text": "\"09\r\n'T", "tokens": 4, "pieces": ["\"", "09", "\r\n", "'T"]} +{"text": "́😀🏽EOT\r<|fim_prefix|>Z'T٣٤٥٦é㍿ß
 ½\"''VE
12345678عtZ<|endoftext|>", "tokens": 59, "pieces": ["́😀🏽", "EOT", "\r", "<|", "fim", "_prefix", "|>", "Z", "'T", "٣٤٥", "٦", "e", "́㍿", "ß", "
", " ", "½", "\"''", "VE", "
", "123", "456", "78", "عtZ", "<|", "endoftext", "|>"]} +{"text": "d!< \n㋿t😀🏽12345678㋿…e<(‍>ع'Tſ", "tokens": 33, "pieces": ["d", "!<", " \n", "㋿t", "😀🏽", "123", "456", "78", "㋿", "…e", "<(‍>", "ع", "'T", "ſ"]} +{"text": "🙂", "tokens": 2, "pieces": ["🙂"]} +{"text": "dfi​'DⅣéZ- \nſ", "tokens": 15, "pieces": ["dfi", "​'", "D", "Ⅳ", "e", "́Z", "-", " \n", "ſ"]} +{"text": "'Tm👍🏽0🙂12345678́!!½s \n٣٤٥٦\re#$%㋿漢EOT", "tokens": 38, "pieces": ["'T", "m", "👍🏽", "0", "🙂", "123", "456", "78", "́!!", "½", "s", " \n", "٣٤٥", "٦", "\r", "e", "#$%㋿", "漢EOT"]} +{"text": "㍿mméZ-ع,DžDž'S½\r\n\r\n'T㋿\"tm<ꟲ'Re\r\n\r\nEOT<|endoftext|>​", "tokens": 41, "pieces": ["㍿mméZ", "-ع", ",<", "EOT", ">DžDž", "'S", "½", "\r\n\r\n", "'T", "㋿\"", "tm", "<ꟲ", "'Re", "\r\n\r\n", "EOT", "<|", "endoftext", "|>​"]} +{"text": "d\t𐞁m\tém३½👍🏽 \"\tⅣ-'ſ'M", "tokens": 28, "pieces": ["d", "\t𐞁m", "\te", "́m", "३½", "👍🏽", " ", " \"", "\t", "Ⅳ", "-'", "ſ", "'M"]} +{"text": "  m३!!'Re　́e!!ßḍ̇ß İ\n'Té<|endoftext|>9 ", "tokens": 34, "pieces": [" ", " m", "३", "!!'", "Re", "　", "́<", "META", "_START", ">e", "!!", "ßḋ", "̣ß", " İ", "\n", "'T", "é", "<|", "endoftext", "|>", "9", " "]} +{"text": "ḍ̇s\r\r\n\r\n<\t\n🙂>0ß'Re\r\n\r\nＡEOTdＡt9'T.😀🏽!fi'llé<|endoftext|>'T‍ꟲ
", "tokens": 61, "pieces": ["ḋ", "̣s", "\r\r\n\r\n", "<", "\t\n", "🙂>", "0", "ß", "'Re", "\r\n\r\n", "ＡEOTdＡt", "9", "'T", ".😀🏽<", "META", "_START", ">!", "fi", "'ll", "e", "́<|", "endoftext", "|><", "EOT", ">'", "T", "‍ꟲ", "
"]} +{"text": "\n\r\n#$% ́\u000bm.\t<|endoftext|>　ꟲa Ａ­㍿́㋿(́'ll'😀🏽t​", "tokens": 50, "pieces": ["\n\r\n", "#$%", " ́", "\u000b", "m", ".", "\t", "<|", "endoftext", "|>", "　ꟲa", " ", " Ａ", "­㍿́㋿(́'", "ll", "'<", "META", "_START", ">😀🏽", "t", "​"]} +{"text": "ſ\"½é>‍#$%🙂're　½ \n<👍🏽am漢­.12345678 \n\t'ſ", "tokens": 37, "pieces": ["ſ", "\"", "½", "é", ">‍#$%🙂'", "re", "　", "½", " \n", "<👍🏽", "am漢", "­.", "123", "456", "78", " \n", "\t", "'ſ"]} +{"text": "㍿'VEå'VE \nEOT <|fim_prefix|>\nåaå'S\n\r‍", "tokens": 32, "pieces": ["㍿'", "VEa", "̊'", "VE", " \n", "EOT", " ", " <|", "fim", "_prefix", "|>\n", "a", "̊aa", "̊'", "S", "\n\r", "‍"]} +{"text": "'Re'll#$%!-'s<­d'Re​a0😀🏽 'S漢\t", "tokens": 27, "pieces": ["'Re", "'ll", "#$%!-<", "META", "_START", ">'", "s", "<­", "d", "'Re", "​a", "0", "😀🏽", " '", "S漢", "\t"]} +{"text": "fiDž😀🏽m''Re!! t>a'å\t(́ݽꟲ-DžＡt㍿İ's😀🏽9‍m\ta \"", "tokens": 49, "pieces": ["fiDž", "😀🏽", "m", "''", "Re", "!!", " t", ">a", "'a", "̊", "\t", "(́", "İ", "½", "ꟲ", "-DžＡt", "㍿İ", "'s", "😀🏽", "9", "‍m", "\ta", " ", "\""]} +{"text": "­'D're", "tokens": 4, "pieces": ["­'", "D", "'re"]} +{"text": "mḍ̇9ḍ̇fi'M", "tokens": 22, "pieces": ["m", "ḋ", "̣", "9", "ḋ", "̣fi", "'M", ""]} +{"text": "́ \n㍿\r\n\r\n👍🏽👍🏽!!Ａa㋿½Dže­🙂\r\n\r\n'T　!!dZt👍🏽\u000b>ع'S\"mt", "tokens": 58, "pieces": ["́", " \n", "㍿\r\n\r\n", "👍🏽👍🏽!!", "Ａa", "㋿", "½", "Dže", "­🙂\r\n\r\n", "'T", "　", "!!", "dZt", "👍🏽", "\u000b", ">", "ع", "'S", "\"mt"]} +{"text": "👍🏽#$%\t Z<|endoftext|>'Reḍ̇,0😀🏽! \n!́½\r\n\r\n\tſ'​ꟲ\"\u000bß‍EOT\"​(ß👍🏽", "tokens": 61, "pieces": ["👍🏽#$%", "\t ", " Z", "<|", "endoftext", "|>'", "Reḋ", "̣,", "0", "😀🏽!", " \n", "!́", "½", "\r\n\r\n", "\tſ", "'​", "ꟲ", "\"", "\u000bß", "‍EOT", "\"​(", "ß", "👍🏽"]} +{"text": "
'Sé‍Ⅳ<|fim_prefix|>漢9\r\n\r\n​ßs#$%- …​漢1234567812345678-Ⅳ#$%,,㍿é'saß(字'ſ\u000b", "tokens": 63, "pieces": ["
", "'", "Sé", "‍", "Ⅳ", "<|", "fim", "_prefix", "|>", "漢", "9", "\r\n\r\n", "​ßs", "#$%-", " ", "…", "​漢", "123", "456", "781", "234", "567", "8", "-", "Ⅳ", "#$%,,㍿<", "META", "_START", ">e", "́'", "saß", "(字", "'ſ", "\u000b"]} +{"text": "(d\r\n\r\n,'Re\u000bꟲ'VE　<|endoftext|>
ḍ̇'ſ 'Re\r\n\r\nİ字👍🏽'M漢 ٣٤٥٦'漢‍ſ​ \n<|fim_prefix|>", "tokens": 73, "pieces": ["(d", "\r\n\r\n", ",'", "Re", "\u000bꟲ", "'VE", "　", "<|", "endoftext", "|>", "
ḋ", "̣'", "ſ", " ", " <", "META", "_START", ">'", "Re", "\r\n\r\n", "İ字", "👍🏽'", "M漢", " ", " ", "٣٤٥", "٦", "'漢", "‍ſ", "​", " \n", "<|", "fim", "_prefix", "|>"]} +{"text": " ३", "tokens": 3, "pieces": [" ", "३"]} +{"text": "​'S<|fim_prefix|>s🙂0-'ſع­t\"㋿　\r\n", "tokens": 25, "pieces": ["​'", "S", "<|", "fim", "_prefix", "|>", "s", "🙂", "0", "-'", "ſع", "­t", "\"㋿", "　\r\n"]} +{"text": "ⅣEOT ", "tokens": 8, "pieces": ["Ⅳ", "EOT", "", " "]} +{"text": "EOT", "tokens": 2, "pieces": ["EOT"]} +{"text": "a'ſ12345678🙂漢​ſḍ̇Z-‍m\n'ſDž", "tokens": 30, "pieces": ["a", "'ſ", "123", "456", "78", "🙂漢", "​ſḋ", "̣Z", "-‍", "m", "\n", "'ſ", "Dž"]} +{"text": "EOT字'VE", "tokens": 9, "pieces": ["EOT", "字", "'VE"]} +{"text": "­'re\"\r\n𐞁ع.a'DEOT\"‍\u000b'Re", "tokens": 23, "pieces": ["­'", "re", "\"\r", "\n", "𐞁ع", ".a", "'D", "EOT", "\"‍", "\u000b", "'Re"]} +{"text": "…,e'ſd''VEＡ٣٤٥٦​DždEOT漢'ſ'll\rſ㍿", "tokens": 43, "pieces": ["…", ",e", "'ſ", "d", "''", "VE", "Ａ", "٣٤٥", "٦", "​", "DždEOT漢", "'ſ", "'ll", "\r", "ſ", "㍿"]} +{"text": "㋿,", "tokens": 4, "pieces": ["㋿,"]} +{"text": "ḍ̇㍿٣٤٥٦<|fim_prefix|>t \"İ''Tfí‍å‍'retd'İé\"d", "tokens": 49, "pieces": ["ḋ", "̣㍿", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "t", " \"", "İ", "''", "Tfi", "́‍", "a", "̊‍'", "ret", "d", "'İe", "́\"", "d"]} +{"text": "'s🙂'D 
عé 's'VE.İ'İ..12345678Ａ\r㋿é ½'ſ'VE-ꟲ #$%<|endoftext|>", "tokens": 73, "pieces": ["'s", "🙂'", "D", " ", "
", "عé", " ", " '", "s", "'VE", ".İ", "'İ", "..", "123", "456", "78", "Ａ", "\r", "㋿e", "́", " ", "½", "'ſ", "'VE", "-ꟲ", " ", " #$%<|", "endoftext", "|>"]} +{"text": "mＡ🙂​㍿漢'll( eＡß12345678\t#$%३t's　'T!!!!><😀🏽漢Dž're", "tokens": 45, "pieces": ["mＡ", "🙂​㍿", "漢", "'", "ll", "(", " eＡß", "123", "456", "78", "\t", "#$%", "३", "t", "'s", "　", "'T", "!!!!><😀🏽", "漢Dž", "'re"]} +{"text": "('S'å's\n‍Ⅳ½\r's'VE 字", "tokens": 19, "pieces": ["('", "S", "'a", "̊'", "s", "\n", "‍", "Ⅳ½", "\r", "'s", "'VE", " ", " 字"]} +{"text": "fi'D㋿", "tokens": 6, "pieces": ["fi", "'D", "㋿"]} +{"text": "'re", "tokens": 1, "pieces": ["'re"]} +{"text": "'D9…٣٤٥٦!!t
🙂字🙂漢-e'Re　'M​\u000b'ſ", "tokens": 32, "pieces": ["'D", "9", "…", "٣٤٥", "٦", "!!", "t", "
", "🙂字", "🙂漢", "-e", "'Re", "　", "'M", "​", "\u000b", "'ſ"]} +{"text": "\nḍ̇fiEOT12345678㍿👍🏽㋿Džع\u000b३<'re𐞁'VE'Mßḍ̇\r>𐞁 90'T½ع𐞁 ‍é<|endoftext|>'reDž३12345678.", "tokens": 81, "pieces": ["\n", "ḋ", "̣fiEOT", "123", "456", "78", "㍿👍🏽㋿", "Džع", "\u000b", "३", "<'", "re𐞁", "'VE", "'M", "ßḋ", "̣\r", ">𐞁", " ", "90", "'T", "½", "ع𐞁", " ", "‍é", "<|", "endoftext", "|>'", "reDž", "३12", "345", "678", "."]} +{"text": "　e'Re​é9㍿s​'Mİ,Ⅳ12345678ع'sＡs Ⅳ\t.12345678", "tokens": 36, "pieces": ["　e", "'Re", "​é", "9", "㍿<", "EOT", ">s", "​'", "Mİ", ",", "Ⅳ12", "345", "678", "ع", "'s", "Ａs", " ", "Ⅳ", "\t", ".", "123", "456", "78"]} +{"text": "'EOT\"…𐞁9é🙂're9,>a,'VEmꟲ-ß٣٤٥٦", "tokens": 35, "pieces": ["'EOT", "\"", "…𐞁", "9", "é", "🙂'", "re", "9", ",>", "a", ",'", "VEmꟲ", "-ß", "٣٤٥", "٦"]} +{"text": "ꟲ­　!!­'S's😀🏽…­Z#$%'re　sDž'll12345678'Sعs٣٤٥٦́३t­ ", "tokens": 47, "pieces": ["ꟲ", "­", "　", "!!­'", "S", "'s", "😀🏽", "…", "­Z", "#$%'", "re", "　sDž", "'ll", "123", "456", "78", "'S", "عs", "٣٤٥", "٦", "́", "३", "t", "­", " "]} +{"text": "s\"é<|endoftext|>👍🏽!漢>ḍ̇\u000b\tⅣ\u000b12345678𐞁\r\n字\u000bm!㍿('VE'ſ0 \n㍿​'M\r\n\r\n'M12345678fi㋿tⅣ", "tokens": 72, "pieces": ["s", "\"e", "́<|", "endoftext", "|>👍🏽!", "漢", ">ḋ", "̣", "\u000b", "\t", "Ⅳ", "\u000b", "123", "456", "78", "𐞁", "\r\n", "字", "\u000bm", "!㍿('", "VE", "'ſ", "0", " \n", "㍿​'", "M", "\r\n\r\n", "'M", "123", "456", "78", "fi", "㋿t", "Ⅳ"]} +{"text": "12345678\n(ß㋿Ａ́12345678'S½<|endoftext|>", "tokens": 26, "pieces": ["123", "456", "78", "\n", "(ß", "㋿Ａ", "́", "123", "456", "78", "'S", "½", "<|", "endoftext", "|>"]} +{"text": "‍㍿", "tokens": 5, "pieces": ["‍㍿"]} +{"text": "é#$%9s\r\nſ'VE<|endoftext|>'ll👍🏽sfi'Re'reå", "tokens": 33, "pieces": ["e", "́#$%", "9", "s", "\r\n", "ſ", "'VE", "<|", "endoftext", "|>'", "ll", "👍🏽", "sfi", "'Re", "'re", "a", "̊"]} +{"text": "Ⅳꟲåå(å<|fim_prefix|>!sİ!\t \n'ſ'ſ३　éDž
́é㍿Dž\r\ńs٣٤٥٦<|endoftext|><|fim_prefix|>", "tokens": 73, "pieces": ["Ⅳ", "ꟲa", "̊a", "̊(", "a", "̊<|", "fim", "_prefix", "|>!", "sİ", "!", "\t \n", "'ſ", "'ſ", "३", "　e", "́Dž", "
", "́é", "㍿Dž", "\r\n", "́s", "٣٤٥", "٦", "<|", "endoftext", "|><|", "fim", "_prefix", "|>"]} +{"text": "𐞁\n½٣٤٥٦Ａ㋿0\"å!fi字́'ll ", "tokens": 31, "pieces": ["𐞁", "\n", "½٣٤", "٥٦", "Ａ", "㋿", "0", "\"a", "̊!", "fi字", "́'", "ll", " "]} +{"text": "ꟲⅣ-ꟲaé漢­!'Sḍ̇EOT!İas", "tokens": 26, "pieces": ["ꟲ", "Ⅳ", "-ꟲaé漢", "­!'", "Sḋ", "̣EOT", "!İas"]} +{"text": "Dž'T<|fim_prefix|>字㋿́漢'Dm12345678 ", "tokens": 23, "pieces": ["Dž", "'T", "<|", "fim", "_prefix", "|>", "字", "㋿́", "漢", "'D", "m", "123", "456", "78", " "]} +{"text": ">", "tokens": 1, "pieces": [">"]} +{"text": "'re>'m#$%…!!𐞁\t\n9ſ㍿EOTİ'漢", "tokens": 25, "pieces": ["'re", ">'", "m", "#$%", "…", "!!", "𐞁", "\t\n", "9", "ſ", "㍿EOTİ", "'漢"]} +{"text": "s#$% fi<|fim_prefix|>\u000b 漢Ａ😀🏽ßfi'M'T'T'MZßＡꟲ>\n0 !#$%٣٤٥٦\r\nDž'VE٣٤٥٦é'll'ret", "tokens": 71, "pieces": ["s", "#$%", " ", " fi", "<|", "fim", "_prefix", "|>", "\u000b", " 漢Ａ", "😀🏽", "ßfi", "'M", "'T", "'T", "'M", "ZßＡꟲ", ">\n", "0", " !#$%", "٣٤٥", "٦", "\r\n", "Dž", "'VE", "٣٤٥", "٦", "é", "'ll", "'re", "t"]} +{"text": "0…'T's\u000bZfi", "tokens": 9, "pieces": ["0", "…", "'T", "'s", "\u000bZfi"]} +{"text": "e漢dé\r\n\r\n'M", "tokens": 11, "pieces": ["e漢", "de", "́\r\n\r\n", "'M"]} +{"text": "😀🏽", "tokens": 5, "pieces": ["😀🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "-é'S<Ⅳ३ ع\tſ!! \n", "tokens": 15, "pieces": ["-e", "́'", "S", "<", "Ⅳ३", " ع", "\tſ", "!!", " \n"]} +{"text": "<", "tokens": 1, "pieces": ["<"]} +{"text": "Z㋿0!!½'ll('Tt'remDžEOT9>'S \u000bDž. \u000bⅣfia३'re漢'M'Re", "tokens": 39, "pieces": ["Z", "㋿", "0", "!!", "½", "'ll", "('", "Tt", "'re", "mDžEOT", "9", ">'", "S", " ", "\u000bDž", ".", " ", "\u000b", "Ⅳ", "fia", "३", "'re", "漢", "'M", "'Re"]} +{"text": "😀🏽!‍>'D \na‍ Ａ", "tokens": 16, "pieces": ["😀🏽!‍>'", "D", " \n", "a", "‍", " Ａ"]} +{"text": "Z<|fim_prefix|>𐞁'llé", "tokens": 18, "pieces": ["Z", "<|", "fim", "_prefix", "|>", "𐞁", "'", "lle", "́"]} +{"text": "ＡEOT<Dž's👍🏽'VE'VEİ\r\n\r\nEOT! \n३9t", "tokens": 28, "pieces": ["ＡEOT", "<Dž", "'s", "👍🏽'", "VE", "'VE", "İ", "\r\n\r\n", "EOT", "!", " \n", "३9", "t"]} +{"text": "\r\n½\ne", "tokens": 4, "pieces": ["\r\n", "½", "\n", "e"]} +{"text": "'DEOTeꟲs\rEOT㋿­'ſ12345678'", "tokens": 27, "pieces": ["'", "DEOTeꟲs", "\r", "EOT", "㋿­<", "EOT", ">'", "ſ", "123", "456", "78", "'"]} +{"text": "­ſ٣٤٥٦å12345678", "tokens": 17, "pieces": ["­ſ", "٣٤٥", "٦", "a", "̊", "123", "456", "78"]} +{"text": "12345678'VEＡ½sDž字", "tokens": 12, "pieces": ["123", "456", "78", "'VE", "Ａ", "½", "sDž字"]} +{"text": "३'M
(…ZꟲⅣ'VE字½𐞁 ‍", "tokens": 28, "pieces": ["३", "'M", "
", "(", "…Zꟲ", "Ⅳ", "'VE", "字", "½", "𐞁", " ", "‍"]} +{"text": "३'(é𐞁३!½'Sfié!#$%'ſ㍿ß '(ſ'Refi,!𐞁'llt'ſ　🙂 ß😀🏽𐞁és#$%'", "tokens": 60, "pieces": ["३", "'(", "é𐞁", "३", "!", "½", "'S", "fie", "́!#$%'", "ſ", "㍿ß", " ", "'(", "ſ", "'Re", "fi", ",!", "𐞁", "'ll", "t", "'ſ", "　", "🙂", " ß", "😀🏽", "𐞁és", "#$%'"]} +{"text": "👍🏽𐞁é\r\n\r\n!!'ReſßعZ'T!!İ字ſEOT́-é३　漢'll字漢ſmⅣ,👍🏽m(", "tokens": 54, "pieces": ["👍🏽", "𐞁é", "\r\n\r\n", "!!'", "ReſßعZ", "'T", "!!", "İ字ſEOT", "́-", "é", "३", "　漢", "'ll", "字漢ſm", "Ⅳ", ",👍🏽", "m", "("]} +{"text": "　\t<|fim_prefix|>𐞁'll'Tdꟲḍ̇🙂'T\r\né́e​Ⅳ", "tokens": 36, "pieces": ["　", "\t", "<|", "fim", "_prefix", "|>", "𐞁", "'ll", "'T", "dꟲḋ", "̣🙂'", "T", "\r\n", "e", "́́", "e", "​", "Ⅳ"]} +{"text": "ſZß漢!!ꟲ!!㋿㋿ Ⅳ", "tokens": 24, "pieces": ["ſZ", "ß漢", "!!", "ꟲ", "!!㋿㋿", " ", "Ⅳ"]} +{"text": "!ḍ̇​dsåm\r\n\r\n><|endoftext|>,'Re<|fim_prefix|>'llfis\r\n\r\n½", "tokens": 37, "pieces": ["!ḋ", "̣​", "dsa", "̊m", "\r\n\r\n", "><|", "endoftext", "|>,<", "EOT", ">'", "Re", "<|", "fim", "_prefix", "|>'", "llfis", "\r\n\r\n", "½"]} +{"text": "tEOT'D('Re'M><|endoftext|>\u000b‍,0İ's're'T\tＡ", "tokens": 26, "pieces": ["tEOT", "'D", "('", "Re", "'M", "><|", "endoftext", "|>", "\u000b", "‍,", "0", "İ", "'s", "'re", "'T", "\tＡ"]} +{"text": "㋿'M'll-ḍ̇é
9Ⅳfi!!><|endoftext|>é字　#$%​0'Re!!İ'll9<字<…ſå", "tokens": 55, "pieces": ["㋿'", "M", "'ll", "-ḋ", "̣e", "́", "
", "", "9Ⅳ", "fi", "!!><|", "endoftext", "|>", "e", "́字", "　", "#$%​", "0", "'Re", "!!", "İ", "'ll", "9", "<字", "<", "…ſa", "̊"]} +{"text": "Z'\u000b٣٤٥٦ſ
\u000b''D'D‍m½\nZ……<'T >ém'MaＡ\u000b\r\n \n å\u000bå½\r#$%9", "tokens": 58, "pieces": ["Z", "'", "\u000b", "٣٤٥", "٦", "ſ", "
", "\u000b", "''", "D", "'D", "‍m", "½", "\n", "Z", "…", "…", "<'", "T", " ", " >", "ém", "'M", "aＡ", "\u000b\r\n \n", " ", " a", "̊", "\u000ba", "̊<", "META", "_START", ">", "½", "\r", "#$%", "9"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "😀🏽İDž٣٤٥٦\r're''S\r😀🏽é.0s🙂'ع½ \n's\"(😀🏽\u000b'ſ!!😀🏽İZ漢ſ عé-", "tokens": 66, "pieces": ["😀🏽", "İDž", "٣٤٥", "٦", "\r", "'", "re", "''", "S", "\r", "😀🏽", "e", "́.", "0", "s", "🙂'", "ع", "½", " \n", "'s", "\"(😀🏽", "\u000b", "'ſ", "!!😀🏽", "İZ漢ſ", " عé", "-"]} +{"text": "\r\nEOT
å(٣٤٥٦\t#$%'lld ٣٤٥٦9#$%㋿- ,
 \n\rtḍ̇>​Z", "tokens": 60, "pieces": ["\r\n", "EOT", "
a", "̊(", "٣٤٥", "٦", "\t", "#$%'", "lld", " ", "٣٤٥", "٦9", "#$%㋿-", " ", " ,", "
 \n\r", "tḋ", "̣>​", "Z", ""]} +{"text": "9​ ſ𐞁Ⅳ'ſꟲDž😀🏽عßZ㋿afi…'re'lla\t …字́३Ⅳ'st", "tokens": 49, "pieces": ["9", "​", " ", " ſ𐞁", "Ⅳ", "'ſ", "ꟲDž", "😀🏽", "عßZ", "㋿afi", "…", "'re", "'ll", "a", "\t ", "…字", "́", "३Ⅳ", "'s", "t"]} +{"text": "👍🏽!!\n\nⅣＡm,٣٤٥٦,Ａ,Zß'ſ 12345678\n9ḍ̇Ⅳß \n🙂é 👍🏽<|fim_prefix|><|endoftext|>", "tokens": 67, "pieces": ["👍🏽!!\n\n", "Ⅳ", "Ａm", ",", "٣٤٥", "٦", ",Ａ", ",Zß", "'ſ", " ", "123", "456", "78", "\n", "9", "ḋ", "̣", "Ⅳ", "ß", " \n", "🙂e", "́", " ", " 👍🏽<|", "fim", "_prefix", "|><|", "endoftext", "|>"]} +{"text": ".३'S'S\"a'D­٣٤٥٦
'llＡＡ𐞁0字9", "tokens": 30, "pieces": [".", "३", "'S", "'S", "\"a", "'D", "­", "٣٤٥", "٦", "
", "'ll", "ＡＡ𐞁", "0", "字", "9"]} +{"text": "\r\n'!!é'll<|endoftext|>\t'MⅣt漢‍‍𐞁Ａ'ſe>​ EOT'Reſ'-𐞁Z's", "tokens": 50, "pieces": ["\r\n", "'!!", "é", "'ll", "<|", "endoftext", "|>", "\t", "'M", "Ⅳ", "t漢", "‍‍", "𐞁Ａ", "'ſ", "e", ">​", " EOT", "'Re", "ſ", "'-", "𐞁Z", "'s"]} +{"text": "\t<|endoftext|>👍🏽ß'Re漢a9>éå'D12345678İDž…9\t㍿Ａt", "tokens": 43, "pieces": ["\t", "<|", "endoftext", "|>👍🏽", "ß", "'Re", "漢a", "9", ">e", "́a", "̊'", "D", "123", "456", "78", "İDž", "…", "9", "\t", "㍿Ａt"]} +{"text": "漢Dž!é'M‍'VEꟲ\t㋿'ſ…'så're㋿İ9fi!!<|endoftext|>", "tokens": 44, "pieces": ["漢Dž", "!é", "'M", "‍'", "VEꟲ", "\t", "㋿'", "ſ", "…", "'s", "a", "̊'", "re", "㋿İ", "9", "fi", "!!<|", "endoftext", "|>"]} +{"text": "́
fi'Rem\u000b!å'Reſ㋿ …", "tokens": 25, "pieces": ["́", "
fi", "'Re", "m", "\u000b", "!a", "̊'", "Re", "ſ", "㋿", " …"]} +{"text": "<|endoftext|>a(,'DDž\r\n 'T'Så‍　a ㍿\r<|endoftext|>", "tokens": 37, "pieces": ["<|", "endoftext", "|>", "a", "(,'", "DDž", "\r\n", " '", "T", "'S", "a", "̊‍", "　a", " ", " ㍿\r", "<|", "endoftext", "|>"]} +{"text": "‍<<|fim_prefix|>9", "tokens": 10, "pieces": ["‍<<|", "fim", "_prefix", "|>", "9"]} +{"text": "​‍ع٣٤٥٦ḍ̇👍🏽!३ Dž​
\n(字　٣٤٥٦t,\r\n'M'T's'Dfi𐞁-ḍ̇dß𐞁", "tokens": 70, "pieces": ["​‍", "ع", "٣٤٥", "٦", "ḋ", "̣👍🏽!", "३", " Dž", "​", "
\n", "(字", "　", "٣٤٥", "٦", "t", ",\r\n", "'M", "'T", "'s", "'D", "fi𐞁", "-", "ḋ", "̣dß𐞁"]} +{"text": "́'reⅣß'll👍🏽\r\n\r\n​👍🏽'\r\n\r\nß㍿", "tokens": 29, "pieces": ["́<", "EOT", ">'", "re", "Ⅳ", "ß", "'ll", "👍🏽\r\n\r\n", "​👍🏽'\r\n\r\n", "ß", "㍿"]} +{"text": "å…　'M,eع'Ḿ\r‍🙂\" å‍İع𐞁m'TtEOT'S　　'D", "tokens": 38, "pieces": ["a", "̊", "…", "　", "'M", ",eع", "'M", "́\r", "‍🙂\"", " a", "̊‍", "İع𐞁m", "'T", "tEOT", "'S", "　", "　", "'D"]} +{"text": "'ReＡ…,'s\n　", "tokens": 9, "pieces": ["'Re", "Ａ", "…", ",'", "s", "\n　"]} +{"text": "aé é're👍🏽a\r\n\r\n字ḍ̇‍12345678('Re's0‍>éé,", "tokens": 41, "pieces": ["aé", " ", " e", "́'", "re", "👍🏽", "a", "\r\n\r\n", "字ḋ", "̣‍", "123", "456", "78", "('", "Re", "'s", "0", "‍>", "e", "́e", "́,"]} +{"text": "0́…½ße‍a𐞁٣٤٥٦<|endoftext|>
", "tokens": 30, "pieces": ["0", "́", "…", "½", "ße", "‍a𐞁", "٣٤٥", "٦", "<|", "endoftext", "|>", "
"]} +{"text": "fimİ<|endoftext|>'M<|fim_prefix|>३ \n
.字Ａİt漢é'Re12345678", "tokens": 39, "pieces": ["fimİ", "<|", "endoftext", "|>'", "M", "<|", "fim", "_prefix", "|>", "३", " \n", "
", ".字Ａİt漢e", "́'", "Re", "123", "456", "78"]} +{"text": "٣٤٥٦ſ Ａ\u000bİſ're<|fim_prefix|>ع'll…\r\n\r\n!!'Dİ9'Mعåd३　0‍‍\r\n\r\nd'D­", "tokens": 57, "pieces": ["٣٤٥", "٦", "ſ", " ", " Ａ", "\u000bİſ", "'re", "<|", "fim", "_prefix", "|>", "ع", "'ll", "…\r\n\r\n", "!!'", "Dİ", "9", "'M", "عa", "̊d", "३", "　", "0", "‍‍\r\n\r\n", "d", "'D", "­"]} +{"text": "ꟲe0\r😀🏽 \nḍ̇'S
<|endoftext|>EOT\u000b!!Ａİ\n'Reſ", "tokens": 39, "pieces": ["ꟲe", "0", "\r", "😀🏽", " \n", "ḋ", "̣'", "S", "
", "<|", "endoftext", "|>", "EOT", "\u000b", "!!", "Ａİ", "\n", "'Re", "ſ"]} +{"text": "('T \n𐞁 \nå👍🏽aꟲ!!­'ſ\r\n\r\na'VE‍'Se㍿\"<|fim_prefix|>>İ'S,", "tokens": 47, "pieces": ["('", "T", " \n", "𐞁", " \n", "a", "̊👍🏽", "aꟲ", "!!­'", "ſ", "\r\n\r\n", "a", "'VE", "‍'", "Se", "㍿\"<|", "fim", "_prefix", "|>>", "İ", "'S", ","]} +{"text": "'T'T👍🏽…<|fim_prefix|>'VE\té㋿", "tokens": 23, "pieces": ["'T", "'T", "👍🏽", "…", "<|", "fim", "_prefix", "|>'", "VE", "\té", "㋿"]} +{"text": "­'re…'T,a…३½ \n-\r \na'VE-ꟲ", "tokens": 23, "pieces": ["­'", "re", "…", "'T", ",a", "…", "३½", " \n", "-\r", " \n", "a", "'VE", "-ꟲ"]} +{"text": "İ!!é \nⅣ\n…\r\n\r\n
m　", "tokens": 19, "pieces": ["İ", "!!", "e", "́", " \n", "", "Ⅳ", "\n…\r\n\r\n", "
m", "　"]} +{"text": "ع😀🏽'ſ㋿.t åfiZ", "tokens": 20, "pieces": ["ع", "😀🏽'", "ſ", "㋿.", "t", " a", "̊fiZ"]} +{"text": "'s\r\n!td'M'D!​́ \u000b \n'll😀🏽'Re!!>́9́", "9", "EOT'sع\r\n\r\ndꟲé<㍿efi 🙂'T…'Re", "tokens": 46, "pieces": [".-", "İe", "́#$%", "e", "́-!!", "e", "　漢", "­", "　", "\u000b", ">EOT", "'s", "ع", "\r\n\r\n", "dꟲe", "́<㍿", "efi", " ", "🙂'", "T", "…", "'", "Re"]} +{"text": "\"\u000b're!!\nsſ👍🏽e
…ſ<|endoftext|>'D́s!!é9३\"㍿,
é😀🏽Ⅳ\t<\net-漢fiDžt", "tokens": 63, "pieces": ["\"", "\u000b", "'re", "!!\n", "sſ", "👍🏽", "e", "
", "…ſ", "<|", "endoftext", "|>'", "D", "́s", "!!", "e", "́", "9३", "\"㍿,", "
e", "́😀🏽", "Ⅳ", "\t", "<\n", "et", "-漢fiDžt"]} +{"text": "Ⅳ३㋿'ll'S𐞁 fi\u000b<|fim_prefix|>漢'ſ#$% \nm t‍'VE<|endoftext|> ३ 
", "tokens": 52, "pieces": ["Ⅳ३", "㋿'", "ll", "'S", "𐞁", " fi", "\u000b", "<|", "fim", "_prefix", "|>", "漢", "'ſ", "#$%", " \n", "m", " t", "‍'", "VE", "<|", "endoftext", "|>", " ", "३", " 
"]} +{"text": "< ꟲ…'s#$%\u000b🙂İs\r\n\r\n㋿dßm>\"​\r\n\r\nⅣ'D𐞁-​'Śtß\r­!漢d…३👍🏽", "tokens": 62, "pieces": ["<", " ", " ꟲ", "…", "'s", "#$%", "\u000b", "🙂İs", "\r\n\r\n", "㋿", "dßm", ">\"​\r\n\r\n", "Ⅳ", "'D", "𐞁", "-​'", "S", "́tß", "\r", "­<", "EOT", ">!", "漢d", "…", "३", "👍🏽"]} +{"text": "٣٤٥٦㋿0am<|fim_prefix|>åé
#$%Ⅳ9eⅣ12345678 \n…mßعe㍿ع'", "tokens": 54, "pieces": ["٣٤٥", "٦", "㋿", "0", "am", "<|", "fim", "_prefix", "|>", "a", "̊é", "", "
", "#$%", "Ⅳ9", "e", "Ⅳ12", "345", "678", " \n", "…mßعe", "㍿ع", "'"]} +{"text": "<½'Re'm12345678\nm \n漢Džſ", "tokens": 16, "pieces": ["<", "½", "'Re", "'m", "123", "456", "78", "\n", "m", " \n", "漢Džſ"]} +{"text": "漢<|endoftext|>　'S'sꟲZꟲas<|endoftext|>Ａ㍿́fiſ0'M​912345678ſ", "tokens": 45, "pieces": ["漢", "<|", "endoftext", "|>", "　", "'S", "'s", "ꟲZꟲas", "<|", "endoftext", "|>", "Ａ", "㍿́", "fiſ", "0", "'M", "​", "912", "345", "678", "ſ"]} +{"text": " 'D३éDžİ'Sd
EOT. t", "tokens": 17, "pieces": [" '", "D", "३", "e", "́Džİ", "'S", "d", "
EOT", ".", " t"]} +{"text": "'ll😀🏽'DfiZ'VE 　-ꟲ
\rꟲe…
", "tokens": 30, "pieces": ["'ll", "😀🏽'", "DfiZ", "'VE", " ", "　", "-ꟲ", "
\r", "ꟲe", "…
"]} +{"text": "३!\u000bds'll'M\r\n", "tokens": 11, "pieces": ["३", "!", "\u000bds", "'ll", "'M", "\r\n"]} +{"text": "𐞁'll½ḍ̇", "tokens": 11, "pieces": ["𐞁", "'ll", "½", "ḋ", "̣"]} +{"text": "!!ꟲß ß'D(12345678👍🏽\td \r\n\r\nİＡſ½dd㋿😀🏽 é0éå<|endoftext|>é३", "tokens": 59, "pieces": ["!!", "ꟲß", " ", " ß", "'D", "(", "123", "456", "78", "👍🏽", "\td", " ", " <", "META", "_START", ">\r\n\r\n", "İＡſ", "½", "dd", "㋿😀🏽", " e", "́", "0", "e", "́a", "̊<|", "endoftext", "|>", "é", "३"]} +{"text": "''llma㍿'s<|fim_prefix|>ꟲDž\n!​(-#$%…Z👍🏽eé tꟲ.\"\r…Dž😀🏽'll ", "tokens": 56, "pieces": ["''", "llma", "㍿'", "s", "<|", "fim", "_prefix", "|>", "ꟲDž", "\n", "!​(-#$%", "…Z", "👍🏽", "ee", "́", " tꟲ", ".\"\r", "…Dž", "😀🏽'", "ll", " "]} +{"text": "\r\nt'Re३'re!!\"㍿éß'M\r\n\r\n'Rea½㍿'M<|endoftext|>'ſ'D'Mß🙂Zd字'Re9e \nmé'!!'Re's", "tokens": 51, "pieces": ["\r\n", "t", "'Re", "३", "'re", "!!\"㍿", "éß", "'M", "\r\n\r\n", "'Re", "a", "½", "㍿'", "M", "<|", "endoftext", "|>'", "ſ", "'D", "'M", "ß", "🙂Zd字", "'Re", "9", "e", " \n", "me", "́'!!'", "Re", "'s"]} +{"text": "ß>'Refi12345678ßꟲé🙂\"'S٣٤٥٦!३!𐞁\u000b 👍🏽", "tokens": 41, "pieces": ["ß", ">'", "Refi", "123", "456", "78", "ßꟲe", "́🙂\"'", "S", "٣٤٥", "٦", "!", "३", "!𐞁", "\u000b ", " 👍🏽"]} +{"text": "'SZ­t. (𐞁 \n'Reé
EOT\r'll…\r\n㍿ſ!!!İ ſ,\r\nfi\r㍿㍿\"<|fim_prefix|>ع", "tokens": 54, "pieces": ["'S", "Z", "­t", ".", " ", "(𐞁", " \n", "'Re", "é", "
EOT", "\r", "'ll", "…\r\n", "㍿ſ", "!!!", "İ", " ", " ſ", ",\r\n", "fi", "\r", "㍿㍿\"<|", "fim", "_prefix", "|>", "ع", ""]} +{"text": "#$%>é!!", "tokens": 6, "pieces": ["#$%>", "e", "́!!"]} +{"text": "İḍ̇́9m'T12345678'S12345678s½३́é\r\n\r\n👍🏽'ſ㋿ꟲd 'Reḍ̇‍́😀🏽'Dß're EOT! \r\n\r\n", "tokens": 68, "pieces": ["İḋ", "̣́", "9", "m", "'T", "123", "456", "78", "'S", "123", "456", "78", "s", "½३", "́é", "\r\n\r\n", "👍🏽'", "ſ", "㋿ꟲd", " ", "'Re", "ḋ", "̣‍́<", "META", "_START", ">😀🏽'", "Dß", "'re", " EOT", "!", " \r\n\r\n"]} +{"text": ">漢ß🙂're‍'VE!!<|fim_prefix|>ḍ̇", "tokens": 25, "pieces": [">漢ß", "🙂'", "re", "‍'", "VE", "!!<|", "fim", "_prefix", "|>", "ḋ", "̣"]} +{"text": "'lld'Re!𐞁>!漢Dž12345678\t漢٣٤٥٦㍿", "tokens": 42, "pieces": ["'ll", "d", "'Re", "!𐞁", ">!", "漢", "Dž", "123", "456", "78", "\t漢", "٣٤٥", "٦", "㍿"]} +{"text": "é<|fim_prefix|>Dž٣٤٥٦½👍🏽½'Re9'll𐞁d12345678́ſa ß>\",'TaＡ \n😀🏽ß!<\r\n\r\nå \n漢<|endoftext|>'llＡ", "tokens": 78, "pieces": ["e", "́<|", "fim", "_prefix", "|>", "Dž", "٣٤٥", "٦½", "👍🏽", "½", "'Re", "9", "'ll", "𐞁d", "123", "456", "78", "́ſa", " ", " ß", ">\",'", "TaＡ", " \n", "😀🏽", "ß", "!<\r\n\r\n", "a", "̊", " \n", "漢", "<|", "endoftext", "|>'", "llＡ"]} +{"text": "­\t　Z<|endoftext|>EOTé(amd'M㋿ 字a \nḍ̇　é'VEſＡḍ̇'ſß'ſ'　<'s'D<", "tokens": 55, "pieces": ["­", "\t", "　Z", "<|", "endoftext", "|>", "EOTe", "́(", "amd", "'M", "㋿", " 字a", " \n", "ḋ", "̣", "　é", "'VE", "ſＡḋ", "̣'", "ſß", "'ſ", "'", "　", "<'", "s", "'D", "<"]} +{"text": "!!m𐞁", "tokens": 6, "pieces": ["!!", "m𐞁"]} +{"text": "Z३\u000b\r'am! \n,<|endoftext|>㋿🙂EOT㋿字#$%!…٣٤٥٦字<|fim_prefix|>!\rſ'S'D½​½ſ>\u000b
\r\n\r\n0", "tokens": 63, "pieces": ["Z", "३", "\u000b\r", "'am", "!", " \n", ",<|", "endoftext", "|>㋿🙂", "EOT", "㋿字", "#$%!", "…", "٣٤٥", "٦", "字", "<|", "fim", "_prefix", "|>!\r", "ſ", "'S", "'D", "½", "​", "½", "ſ", ">", "\u000b
\r\n\r\n", "0"]} +{"text": "\r🙂'D!!'lla-­🙂\u000b.'S ​dfi\rꟲİ㋿-\nⅣ\n é 0", "tokens": 36, "pieces": ["\r", "🙂'", "D", "!!'", "lla", "-­🙂", "\u000b", ".'", "S", " ", "​dfi", "\r", "ꟲİ", "㋿-\n", "Ⅳ", "\n", " é", " ", "0"]} +{"text": "\n'll\r'ſ\n're0👍🏽d", "tokens": 19, "pieces": ["\n", "'ll", "\r", "'ſ", "\n", "'re", "0", "👍🏽", "d"]} +{"text": "!!Ａ

😀🏽\r\n३'M'Reꟲ㍿éDž, Ａ<|endoftext|>", "tokens": 37, "pieces": ["!!", "Ａ", "
", "
", "😀🏽\r\n", "३", "'M", "'Re", "ꟲ", "㍿éDž", ",", " ", " Ａ", "<|", "endoftext", "|>"]} +{"text": "𐞁0-'ſ'VE<|fim_prefix|>a", "tokens": 18, "pieces": ["𐞁", "0", "-'", "ſ", "'VE", "<|", "fim", "_prefix", "|>", "a"]} +{"text": "éＡ😀🏽
EOT㍿　😀🏽'VE! ½å'lléⅣ,　<\u000b#$%漢.\tts\"'T<|fim_prefix|>\r,ßé'SédＡDž", "tokens": 67, "pieces": ["e", "́Ａ", "😀🏽", "
EOT", "㍿", "　", "😀🏽'", "VE", "!", " ", " ", "½", "a", "̊'", "lle", "́", "Ⅳ", ",", "　", "<", "\u000b", "#$%", "漢", ".", "\tts", "\"'", "T", "<|", "fim", "_prefix", "|>\r", ",ße", "́'", "Se", "́dＡDž"]} +{"text": "'T३​漢'll0٣٤٥٦ꟲ ", "tokens": 20, "pieces": ["'T", "३", "​漢", "'ll", "0٣٤", "٥٦", "ꟲ", " "]} +{"text": "㍿m,㍿t\t
<|endoftext|>0tt123456780t'ſß're\u000be㍿٣٤٥٦'llⅣ'ſ½dåꟲ३!!9ḍ̇🙂", "tokens": 67, "pieces": ["㍿m", ",㍿", "t", "\t", "
", "<|", "endoftext", "|>", "0", "tt", "123", "456", "780", "t", "'ſ", "ß", "'re", "\u000be", "㍿", "٣٤٥", "٦", "'ll", "Ⅳ", "'ſ", "½", "da", "̊ꟲ", "३", "!!", "9", "ḋ", "̣🙂"]} +{"text": "!!t,'reDž12345678,s \r\n\r\n<|endoftext|>'Sḍ̇Ａ'S,🙂é'VE12345678éß
\r\n\r\n\r\n\r\n'S\r\nDž'T<|fim_prefix|>é३ſⅣ!!३
İ🙂", "tokens": 71, "pieces": ["!!", "t", ",'", "reDž", "123", "456", "78", ",s", " \r\n\r\n", "<|", "endoftext", "|>'", "Sḋ", "̣Ａ", "'S", ",🙂", "e", "́'", "VE", "123", "456", "78", "e", "́ß", "
\r\n\r\n\r\n\r\n", "'S", "\r\n", "Dž", "'T", "<|", "fim", "_prefix", "|>", "é", "३", "ſ", "Ⅳ", "!!", "३", "
İ", "🙂"]} +{"text": "'M0\r\n\r\n\r0ßEOT'reＡ
", "tokens": 13, "pieces": ["'M", "0", "\r\n\r\n\r", "0", "ßEOT", "'re", "Ａ", "
"]} +{"text": "å‍Ａ's9'ſ🙂漢's
\u000b\t
a\"👍🏽عع-0Džfi\u000bt", "tokens": 41, "pieces": ["a", "̊‍", "Ａ", "'s", "9", "'ſ", "🙂漢", "'s", "
\u000b\t", "
a", "\"👍🏽", "عع", "-", "0", "Džfi", "\u000bt"]} +{"text": "'set", "tokens": 2, "pieces": ["'s", "et"]} +{"text": "…𐞁fiꟲ", "tokens": 11, "pieces": ["…𐞁fiꟲ"]} +{"text": "'re漢\r\n\u000b'Re㍿ee!!fisḍ̇​३\n<|endoftext|>😀🏽,t'ſ", "tokens": 40, "pieces": ["'re", "漢", "\r\n", "\u000b", "'Re", "㍿ee", "!!", "fisḋ", "̣​", "३", "\n", "<|", "endoftext", "|>😀🏽,", "t", "'ſ"]} +{"text": "🙂 \n𐞁'T­'re(İ'reDž-'reDžs½\r\n>🙂😀🏽ḍ̇'re\r漢́", "tokens": 42, "pieces": ["🙂", " \n", "𐞁", "'T", "­'", "re", "(İ", "'re", "Dž", "-'", "reDžs", "½", "\r\n", ">🙂😀🏽", "ḋ", "̣'", "re", "\r", "漢", "́"]} +{"text": "'lĺ -d㋿\n!!'Re\r\n.é'D­å𐞁\u000bꟲ\r\n\r\n'S!!<|fim_prefix|>­e٣٤٥٦", "tokens": 49, "pieces": ["'ll", "́", " ", " -", "d", "㋿\n", "!!'", "Re", "\r\n", ".e", "́'", "D", "­a", "̊𐞁", "\u000bꟲ", "\r\n\r\n", "'S", "!!<|", "fim", "_prefix", "|>­", "e", "٣٤٥", "٦"]} +{"text": "m'T­>d\u000bＡ'sⅣ fia🙂\u000b👍🏽fi👍🏽ſtع", "tokens": 36, "pieces": ["m", "'T", "­>", "d", "\u000bＡ", "'s", "Ⅳ", " fia", "🙂", "\u000b", "👍🏽", "fi", "👍🏽", "ſtع"]} +{"text": "'!\u000b㋿.9३t\u000ba.'Te 'D٣٤٥٦Ⅳ​ İ\"m 'Re'Re𐞁　<|endoftext|> ㋿\rꟲ😀🏽", "tokens": 64, "pieces": ["'!", "\u000b", "㋿.", "9३", "t", "\u000ba", ".'", "Te", " ", "'D", "٣٤٥", "٦Ⅳ", "​", " İ", "\"m", " ", " '", "Re", "'Re", "𐞁", "　", "<|", "endoftext", "|>", " ㋿\r", "ꟲ", "😀🏽"]} +{"text": "\n'Re㍿\nm‍'½ḍ̇Dž \n…  ३.é", "tokens": 31, "pieces": ["\n", "'Re", "㍿\n", "m", "‍'", "½", "ḋ", "̣Dž", " \n", "…  ", " ", "३", ".e", "́"]} +{"text": "ḍ̇㍿'S🙂>\"<|endoftext|>Ａ'३é!!　<|fim_prefix|>'३\u000b'T\r\" e🙂", "tokens": 52, "pieces": ["ḋ", "̣㍿'", "S", "🙂>\"<|", "endoftext", "|>", "Ａ", "'", "३", "e", "́!!<", "EOT", ">", "　", "<|", "fim", "_prefix", "|>'", "३", "\u000b", "'T", "\r", "\"", " ", " e", "🙂"]} +{"text": "३𐞁m㋿ＡⅣ", "tokens": 14, "pieces": ["३", "𐞁m", "㋿Ａ", "Ⅳ"]} +{"text": "fi ", "tokens": 3, "pieces": ["fi", " "]} +{"text": " s're​EOT'reꟲ½'sfié́", "tokens": 20, "pieces": [" ", " s", "'re", "​EOT", "'re", "ꟲ", "½", "'s", "fie", "́́"]} +{"text": " \t­'D🙂'ſ漢d", "tokens": 42, "pieces": [" ", "\t", "­'", "D", "🙂'", "ſ", "<", "EOT", ">漢d"]} +{"text": "字ꟲDž́🙂<|endoftext|>tＡ's12345678­<|fim_prefix|>'ſ́Z👍🏽mＡś's'D'VE#$%'VE\téß㍿ß'M", "tokens": 68, "pieces": ["字ꟲDž", "́🙂<|", "endoftext", "|>", "tＡ", "'s", "123", "456", "78", "­<|", "fim", "_prefix", "|>'", "ſ", "́Z", "👍🏽", "mＡs", "́'", "s", "'", "D", "'VE", "#$%'", "VE", "\téß", "㍿ß", "'M"]} +{"text": "0
३'ll🙂'll", "tokens": 10, "pieces": ["0", "
", "३", "'ll", "🙂'", "ll"]} +{"text": "\r!!'T!!ꟲe٣٤٥٦fi<|fim_prefix|>👍🏽-#$%́𐞁\r\n#$%㍿'sé㍿", "tokens": 52, "pieces": ["\r", "!!'", "T", "!!", "ꟲe", "٣٤٥", "٦", "fi", "<|", "fim", "_prefix", "|>👍🏽-#$%́", "𐞁", "\r\n", "#$%㍿'", "se", "́㍿"]} +{"text": "'D", "tokens": 4, "pieces": ["'", "D"]} +{"text": "ꟲ.­Dž's\r\n 'VE \n٣٤٥٦é!!9fi-𐞁\r\u000b12345678\r\n.EOT12345678İ'ſaſＡ-㍿ \n0", "tokens": 67, "pieces": ["ꟲ", ".­", "Dž", "'s", "\r\n", " '", "VE", " \n", "٣٤٥", "٦", "é", "!!", "9", "fi", "-𐞁", "\r", "\u000b", "123", "456", "78", "\r\n", ".EOT", "", "123", "456", "78", "İ", "'ſ", "a", "ſＡ", "-㍿", " \n", "0"]} +{"text": "😀🏽s.​㍿,…å-e'VE9
fi  ", "tokens": 28, "pieces": ["😀🏽", "s", ".​㍿,", "…a", "̊-", "e", "'VE", "9", "
fi", "  "]} +{"text": "\u000b12345678<|endoftext|>", "tokens": 11, "pieces": ["\u000b", "123", "456", "78", "<|", "endoftext", "|>"]} +{"text": "ḍ̇Dž's.\n,㍿𐞁㍿s9'Ta \n<|endoftext|>́👍🏽e'Re'lls \n'll.d
.…m", "tokens": 52, "pieces": ["ḋ", "̣Dž", "'s", ".\n", ",㍿", "𐞁", "㍿s", "9", "'T", "a", " \n", "<|", "endoftext", "|>́👍🏽", "e", "'Re", "'ll", "s", " \n", "'ll", ".d", "
", ".", "…m"]} +{"text": "(㋿#$%'M\r 'DmEOT're'reeå­\u000b,å字aİ👍🏽‍‍fiaḍ̇'re'VE \n,'T!!'så\r", "tokens": 59, "pieces": ["(㋿#$%'", "M", "\r", " ", "'D", "mEOT", "'re", "'re", "ea", "̊­", "\u000b", ",a", "̊字", "aİ", "👍🏽‍‍", "fiaḋ", "̣'", "re", "'VE", " \n", ",'", "T", "!!'", "sa", "̊\r"]} +{"text": ".mİéå😀🏽\r½𐞁ßİ३…<عé'Re\r\r\n\r\n ß​åddع0", "tokens": 39, "pieces": [".mİéa", "̊😀🏽\r", "½", "𐞁ßİ", "३", "…", "<عé", "'Re", "\r\r\n\r\n", " ", " ß", "​a", "̊ddع", "0"]} +{"text": "#$%\nſd३!!३ḍ̇㋿12345678'll, \n'ſ!! <|fim_prefix|>'ſé㍿,0…'ll३'TDž!٣٤٥٦Z😀🏽00ＡZ", "tokens": 73, "pieces": ["#$%<", "META", "_START", ">\n", "ſd", "३", "!!", "३", "ḋ", "̣㋿", "123", "456", "78", "'ll", ",", " \n", "'ſ", "!!", " <|", "fim", "_prefix", "|>'", "ſé", "㍿,", "0", "…", "'ll", "३", "'T", "Dž", "!", "٣٤٥", "٦", "Z", "😀🏽", "00", "ＡZ"]} +{"text": "\t'DZ(><,!!عdDžm\nat-👍🏽\tß😀🏽३­́½'Sfi漢mع", "tokens": 49, "pieces": ["\t", "'D", "Z", "(><,!!", "عdDžm", "\n", "at", "-👍🏽", "\tß", "😀🏽", "३", "­́", "½", "'S", "fi", "漢", "mع"]} +{"text": "ß٣٤٥٦ꟲ fi
a", "tokens": 17, "pieces": ["ß", "٣٤٥", "٦", "ꟲ", " fi", "
a"]} +{"text": "　'Ḿ​t　é㋿Ⅳ𐞁 ſ漢d‍fim<|fim_prefix|>", "tokens": 35, "pieces": ["　", "'M", "́​", "t", "　e", "́㋿", "Ⅳ", "𐞁", " ſ漢d", "‍fim", "<|", "fim", "_prefix", "|>"]} +{"text": "'Re('S\n", "tokens": 4, "pieces": ["'Re", "('", "S", "\n"]} +{"text": "Dž👍🏽Ⅳ\u000bfi'VE㍿at 字eḍ̇\"字…\t字　é'Reſ", "tokens": 41, "pieces": ["Dž", "👍🏽", "Ⅳ", "\u000bfi", "'VE", "㍿at", "", " ", " 字eḋ", "̣\"", "字", "…", "\t字", "　é", "'Re", "ſ"]} +{"text": "'D<|fim_prefix|>d𐞁\r'll'Re٣٤٥٦३㍿'ll('ſ́\u000bs…'S\r\n\r\n👍🏽'D'<'VE'SZ09'D'S㋿<|endoftext|>", "tokens": 67, "pieces": ["'D", "<|", "fim", "_prefix", "|>", "d𐞁", "\r", "'ll", "'Re", "٣٤٥", "٦३", "㍿'", "ll", "('", "ſ", "́", "\u000bs", "…", "'S", "\r\n\r\n", "👍🏽'", "D", "'<'", "VE", "'S", "Z", "09", "'D", "'S", "㋿<|", "endoftext", "|>"]} +{"text": "ḍ̇0‍🙂​​'Re0㍿>", "tokens": 18, "pieces": ["ḋ", "̣", "0", "‍🙂​​'", "Re", "0", "㍿>"]} +{"text": "é'M\n\r'M'DＡ\"", "tokens": 11, "pieces": ["e", "́'", "M", "\n\r", "'M", "'D", "Ａ", "\""]} +{"text": "e㍿' EOTéåDž fiß's'll🙂\u000b'Tİ\r\nİ!", "tokens": 30, "pieces": ["e", "㍿'", " EOTéa", "̊Dž", " fiß", "'", "s", "'ll", "🙂", "\u000b", "'T", "İ", "\r\n", "İ", "!"]} +{"text": "\r0'VE
'VEZعs　\t'VE㋿İ　ḍ̇́-té😀🏽Ａ'ſ\n\r \n0é'S​", "tokens": 50, "pieces": ["\r", "0", "'VE", "", "
", "'VE", "Zعs", "　", "\t", "'VE", "㋿İ", "　ḋ", "̣́-", "te", "́😀🏽", "Ａ", "'ſ", "\n\r \n", "0", "é", "'S", "​"]} +{"text": "　ſt \r\nś'M'D", "tokens": 11, "pieces": ["　ſt", " \r\n", "s", "́'", "M", "'D"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "9'M漢漢३!!é½''Dß字'll''ſ\r㍿Ⅳ", "tokens": 25, "pieces": ["9", "'M", "漢漢", "३", "!!", "é", "½", "''", "Dß字", "'ll", "''", "ſ", "\r", "㍿", "Ⅳ"]} +{"text": "́", "tokens": 1, "pieces": ["́"]} +{"text": "->\r\n\r\n­'Re​ꟲ<|endoftext|>‍m…‍㋿0
eDž'S字\"<|endoftext|>Ⅳé㍿‍ \nع'M' >́'VEfi", "tokens": 60, "pieces": ["->\r\n\r\n", "­'", "Re", "​ꟲ", "<|", "endoftext", "|>‍", "m", "…", "‍㋿", "0", "
eDž", "'S", "字", "\"<|", "endoftext", "|>", "Ⅳ", "é", "㍿‍", " \n", "ع", "'M", "'", " ", ">́'", "VEfi"]} +{"text": "0㍿'é're.!!ß0", "tokens": 13, "pieces": ["0", "㍿'", "e", "́'", "re", ".!!", "ß", "0"]} +{"text": "'ſ'T🙂 'M\u000b½a'ſ‍字!!m(!!😀🏽ع", "tokens": 38, "pieces": ["'ſ", "'T", "🙂", " ", " '", "M", "", "\u000b", "½", "a", "'ſ", "‍<", "EOT", "><", "EOT", ">字", "!!", "m", "(!!😀🏽", "ع"]} +{"text": "\r\ń12345678Ⅳ😀🏽'Re😀🏽<|fim_prefix|>漢", "tokens": 31, "pieces": ["\r\n", "́", "123", "456", "78", "", "Ⅳ", "😀🏽'", "Re", "😀🏽<|", "fim", "_prefix", "|>", "漢"]} +{"text": "\u000b'Re-<|endoftext|>𐞁-👍🏽'D\u000b!!🙂\u000b\r", "tokens": 32, "pieces": ["\u000b", "'Re", "-<|", "endoftext", "|>", "𐞁", "-👍🏽'", "D", "\u000b", "!!🙂", "\u000b\r"]} +{"text": "'s\t#$%EOT0漢. .(t‍s>-'re\rİ12345678EOT-'", "re", "\r", "İ", "123", "456", "78", "EOT", "'ll👍🏽>,", "tokens": 21, "pieces": [",t", "㍿‍\r\n\r\n", "m", "…", "'", "ll", "👍🏽>,"]} +{"text": "'ll>३\r\r\n\r\ń", "tokens": 7, "pieces": ["'ll", ">", "३", "\r\r\n\r\n", "́"]} +{"text": "​", "tokens": 1, "pieces": ["​"]} +{"text": "'D३😀🏽<|fim_prefix|><'T\r\n\r\nDž!\"tDž\r\n漢.　(a<|fim_prefix|>\r\n\r\n-㋿\n.\né<|fim_prefix|>DžⅣ字ad!!٣٤٥٦", "tokens": 66, "pieces": ["'D", "३", "😀🏽<|", "fim", "_prefix", "|><'", "T", "\r\n\r\n", "Dž", "!\"", "tDž", "\r\n", "漢", ".", "　", "(a", "<|", "fim", "_prefix", "|>\r\n\r\n", "-㋿\n", ".\n", "é", "<|", "fim", "_prefix", "|>", "Dž", "Ⅳ", "字ad", "!!", "٣٤٥", "٦"]} +{"text": "\r", "tokens": 1, "pieces": ["\r"]} +{"text": "!!", "tokens": 1, "pieces": ["!!"]} +{"text": ".𐞁​㍿\"e३'M \ń \nfifi\t 😀🏽mt <|fim_prefix|>", "tokens": 37, "pieces": [".𐞁", "​㍿\"", "e", "३", "'M", " \n", "́<", "META", "_START", ">", " \n", "fifi", "\t ", " 😀🏽", "mt", " <|", "fim", "_prefix", "|>"]} +{"text": "Z'MamＡ　'ſd½㋿㋿a\"d字 Ⅳ<|endoftext|>́İ'ſ'VE'ſ\n12345678㍿　👍🏽𐞁İ", "tokens": 63, "pieces": ["Z", "'M", "amＡ", "　", "'ſ", "d", "½", "㋿㋿<", "META", "_START", ">a", "\"d字", " ", " ", "Ⅳ", "<|", "endoftext", "|>́", "İ", "'ſ", "'VE", "'ſ", "\n", "123", "456", "78", "㍿", "　", "👍🏽", "𐞁İ"]} +{"text": "٣٤٥٦!Zſꟲ-…'ll ", "tokens": 20, "pieces": ["٣٤٥", "٦", "!Zſꟲ", "-", "…", "'ll", " "]} +{"text": "'ll‍ İdİ\naEOTd!\r\n\r\n\r\n\r\n\u000b'M'VEꟲ㍿<|endoftext|>0 \n<|endoftext|>'ſ­㋿'ſ", "tokens": 49, "pieces": ["'ll", "‍", " ", " İdİ", "\n", "aEOTd", "!\r\n\r\n\r\n\r\n", "\u000b", "'M", "'VE", "ꟲ", "㍿<|", "endoftext", "|>", "0", " \n", "<|", "endoftext", "|>'", "ſ", "­㋿'", "ſ"]} +{"text": "fi\r\n\r\n…", "tokens": 5, "pieces": ["fi", "\r\n\r\n…"]} +{"text": "ſ>'s'Sa9ع٣٤٥٦\r\n\r\n-Ⅳ\nß", "tokens": 22, "pieces": ["ſ", ">'", "s", "'S", "a", "9", "ع", "٣٤٥", "٦", "\r\n\r\n", "-", "Ⅳ", "\n", "ß"]} +{"text": "٣٤٥٦<|fim_prefix|>0\u000b​(-३é字'S!!!!字fi'ſ<|endoftext|>0 's\u000bZ㍿'Red\r>👍🏽s\n👍🏽'll'll
e", "tokens": 75, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "0", "\u000b", "​(-", "३", "e", "́字", "'S", "!!!!", "字fi", "'ſ", "<|", "endoftext", "|>", "0", " ", "'s", "\u000bZ", "㍿'", "Red", "\r", ">👍🏽", "s", "\n", "👍🏽'", "ll", "'ll", "
e"]} +{"text": "­t漢 (\nfi >t\rs!!'T're!!½'refi　­­!!\nḍ̇😀🏽#$%­EOT'T>", "tokens": 40, "pieces": ["­t漢", " (\n", "fi", " >", "t", "\r", "s", "!!'", "T", "'re", "!!", "½", "'re", "fi", "　", "­­!!\n", "ḋ", "̣😀🏽#$%­", "EOT", "'T", ">"]} +{"text": "'S<|fim_prefix|>.㋿\"👍🏽½​ḍ̇\u000b>9", "tokens": 32, "pieces": ["'S", "<|", "fim", "_prefix", "|>.㋿\"👍🏽", "½", "​ḋ", "̣<", "EOT", ">", "\u000b", ">", "9"]} +{"text": "٣٤٥٦😀🏽㍿-0!!'VE''S!!Z(-0'!!́३
", "tokens": 37, "pieces": ["٣٤٥", "٦", "😀🏽㍿-", "0", "!!'", "VE", "''", "S", "!!<", "EOT", ">Z", "(-", "0", "'!!́", "३", "
"]} +{"text": "'ſ㍿!İ're　t\r\n‍½ſfi\u000b'Dé!漢­½‍'ſ\"ḍ̇éＡ<'Ss,", "tokens": 52, "pieces": ["'ſ", "㍿!", "İ", "'re", "　t", "\r\n", "‍<", "EOT", ">", "½", "ſfi", "\u000b", "'D", "é", "!漢", "­", "½", "‍'", "ſ", "\"ḋ", "̣éＡ", "<'", "Ss", ","]} +{"text": ".ḍ̇åİ\"!!ḍ̇ꟲ漢  ­Ａ\nſ>३12345678#$%a३\u000b'ſ9,", "tokens": 49, "pieces": [".ḋ", "̣a", "̊İ", "\"!!", "ḋ", "̣ꟲ漢", " ", " ­", "Ａ", "\n", "ſ", ">", "३12", "345", "678", "#$%", "a", "३", "\u000b", "'ſ", "9", ","]} +{"text": "㋿12345678>٣٤٥٦<|fim_prefix|>,'Re<|fim_prefix|>>́😀🏽ＡDž9'VE \n're", "tokens": 46, "pieces": ["㋿", "123", "456", "78", ">", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>,'", "Re", "<|", "fim", "_prefix", "|>>́😀🏽", "ＡDž", "9", "'VE", " \n", "'re"]} +{"text": "Džſ­< \n　\rme12345678dعſ­#$%-ḿ…\"　(Dž 's‍fi!٣٤٥٦<|endoftext|>\r\n'll
… \n're", "tokens": 62, "pieces": ["Džſ", "­<", " \n　\r", "me", "123", "456", "78", "dعſ", "­#$%-", "m", "́", "…", "\"", "　", "(Dž", " ", " '", "s", "‍fi", "!", "٣٤٥", "٦", "<|", "endoftext", "|>\r\n", "'ll", "
… \n", "'re"]} +{"text": "İſ,㍿!a0'M‍'ll漢'VE<|endoftext|>Dž३…!Dž ḍ̇'Dé'll🙂ع३aåß \n,½'Re\r\n\r\n漢'll'ß", "tokens": 65, "pieces": ["İſ", ",㍿!", "a", "0", "'M", "‍'", "ll漢", "'", "VE", "<|", "endoftext", "|>", "Dž", "३", "…", "!Dž", " ḋ", "̣'", "Dé", "'ll", "🙂ع", "३", "aa", "̊ß", " \n", ",", "½", "'Re", "\r\n\r\n", "漢", "'ll", "'ß"]} +{"text": "٣٤٥٦Ａ㋿EOT​\r\n\r\n\"‍'Sa'VE٣٤٥٦Z!ꟲ🙂 'ſ'll", "tokens": 43, "pieces": ["٣٤٥", "٦", "Ａ", "㋿EOT", "​\r\n\r\n", "\"‍'", "Sa", "'VE", "٣٤٥", "٦", "Z", "!ꟲ", "🙂", " '", "ſ", "'ll"]} +{"text": "\t(s(", "tokens": 3, "pieces": ["\t", "(s", "("]} +{"text": "ＡDž𐞁a-😀🏽é\r 漢
​'\u000b𐞁EOTḍ̇'VE…>㍿\t
字🙂!'s­Ａ#$%'VE漢0ß'M're㍿", "tokens": 71, "pieces": ["ＡDž𐞁a", "-😀🏽", "e", "́\r", " 漢", "
", "​'", "\u000b", "𐞁EOTḋ", "̣'", "VE", "…", ">㍿", "\t", "
字", "🙂!'", "s", "­Ａ", "#$%'", "VE漢", "0", "ß", "'M", "'re", "㍿"]} +{"text": "-!🙂'll٣٤٥٦ '.㍿Dž!!eſe", "tokens": 26, "pieces": ["-!🙂'", "ll", "٣٤٥", "٦", " ", "'.㍿", "Dž", "!!", "eſe"]} +{"text": " 😀🏽9İ\t㋿…ß m\t", "tokens": 15, "pieces": [" 😀🏽", "9", "İ", "\t", "㋿", "…ß", " m", "\t"]} +{"text": "…㋿🙂tm🙂字>EOT 漢'T­\r\r\n\r\n ٣٤٥٦éé", "tokens": 31, "pieces": ["…", "㋿🙂", "tm", "🙂字", ">EOT", " 漢", "'T", "­\r\r\n\r\n", " ", "٣٤٥", "٦", "éé"]} +{"text": "'ſ\r漢
<|endoftext|>😀🏽\rå😀🏽<|endoftext|>a‍<|fim_prefix|>　ع\u000bå", "tokens": 55, "pieces": ["'ſ", "\r", "漢", "
", "<|", "endoftext", "|>😀🏽\r", "a", "̊😀🏽<", "EOT", "><|", "endoftext", "|>", "a", "‍<|", "fim", "_prefix", "|>", "　ع", "\u000ba", "̊"]} +{"text": "ß'T'🙂EOTZdEOT🙂 ½!!'D<|endoftext|>fi0\" \n㋿٣٤٥٦-é\"'VE　'S", "tokens": 48, "pieces": ["ß", "'T", "'🙂", "EOTZdEOT", "🙂", " ", " ", "½", "!!'", "D", "<|", "endoftext", "|>", "fi", "0", "\"", " \n", "㋿", "٣٤٥", "٦", "-e", "́\"'", "VE", "　", "'S"]} +{"text": "İ\n's12345678ß<|fim_prefix|>\r\n\r\ns漢 -٣٤٥٦#$% 'Re0!!!㋿\tDž", "tokens": 50, "pieces": ["İ", "\n", "'s", "123", "456", "78", "ß", "<|", "fim", "_prefix", "|>\r\n\r\n", "s漢", " -", "٣٤٥", "٦", "#$%", " ", " '", "Re", "0", "!!!㋿", "\t", "Dž", ""]} +{"text": "\"́ع'T\rḍ̇é㍿👍🏽ḍ̇\r\n\r\n​🙂's#$%,½'T<|endoftext|>ßDž ३'M​å<|endoftext|>''T", "tokens": 66, "pieces": ["\"́", "ع", "'T", "\r", "ḋ", "̣e", "́㍿👍🏽", "ḋ", "̣<", "META", "_START", ">\r\n\r\n", "​🙂'", "s", "#$%,", "½", "'T", "<|", "endoftext", "|>", "ßDž", " ", "३", "'M", "​a", "̊<|", "endoftext", "|>''", "T"]} +{"text": "12345678ééİ Dž<|endoftext|>!!𐞁३㋿👍🏽漢‍<|fim_prefix|><|endoftext|>s", "tokens": 56, "pieces": ["123", "456", "78", "e", "́e", "́İ", " ", " Dž", "<|", "endoftext", "|>!!", "𐞁", "", "३", "㋿👍🏽", "漢", "‍<|", "fim", "_prefix", "|><|", "endoftext", "|>", "s"]} +{"text": "㍿🙂Ａ<­> éع123456780\r\n𐞁12345678'VE'S漢<|fim_prefix|>éDžİ'D're…Zeé>٣٤٥٦ \t'VE>fiéß", "tokens": 69, "pieces": ["㍿🙂", "Ａ", "<­>", " e", "́ع", "123", "456", "780", "\r\n", "𐞁", "123", "456", "78", "'VE", "'S", "漢", "<|", "fim", "_prefix", "|><", "EOT", ">e", "́Džİ", "'D", "'re", "…Zee", "́>", "٣٤٥", "٦", " ", "\t", "'VE", ">fie", "́ß"]} +{"text": "👍🏽a'VE9ſعDž ٣٤٥٦­Dž\r\n३fiß‍‍'De<|fim_prefix|>ſع漢<|fim_prefix|>\"­Ａ", "tokens": 69, "pieces": ["👍🏽", "a", "'VE", "9", "ſ", "عDž", " ", " ", "٣٤٥", "٦", "­Dž", "\r\n", "३", "fiß", "‍‍'", "De", "<|", "fim", "_prefix", "|>", "ſع漢", "<|", "fim", "_prefix", "|>\"<", "META", "_START", ">­", "Ａ"]} +{"text": "å,\r\n\r\nع½㋿\r\né३-'M<|fim_prefix|>'M­ 'Téßİ'st३'s\"tＡ\r\n\r\nd\u000b'sḍ̇ſ٣٤٥٦\r\n", "tokens": 59, "pieces": ["a", "̊,\r\n\r\n", "ع", "½", "㋿\r\n", "e", "́", "३", "-'", "M", "<|", "fim", "_prefix", "|>'", "M", "­", " '", "Téßİ", "'s", "t", "३", "'s", "\"tＡ", "\r\n\r\n", "d", "\u000b", "'s", "ḋ", "̣ſ", "٣٤٥", "٦", "\r\n"]} +{"text": "ß 'M#$%'s ३㍿　sdDž'll'sEOT
\r\n\r\n<|endoftext|> ‍\u000bEOT\u000b\n㍿Dž éEOT'S㋿ \né", "tokens": 56, "pieces": ["ß", " '", "M", "#$%'", "s", " ", "३", "㍿", "　sdDž", "'ll", "'s", "EOT", "
\r\n\r\n", "<|", "endoftext", "|>", " ", "‍", "\u000bEOT", "\u000b\n", "㍿Dž", " <", "EOT", ">éEOT", "'S", "㋿", " \n", "é"]} +{"text": "٣٤٥٦åſ ‍és'ſ'lĺ12345678'T#$%٣٤٥٦!ع ́­", "tokens": 46, "pieces": ["٣٤٥", "٦", "a", "̊ſ", " ", "‍és", "'ſ", "'ll", "́", "123", "456", "78", "'T", "#$%<", "META", "_START", ">", "٣٤٥", "٦", "!ع", " ", " ́­"]} +{"text": ">EOT'reꟲß\nſ'S's'llm\u000b👍🏽'㋿'lĺ\t㋿Ⅳ>漢👍🏽 Ａ", "tokens": 46, "pieces": [">EOT", "'re", "ꟲß", "\n", "ſ", "'S", "'s", "'ll", "m", "\u000b", "👍🏽'㋿'", "ll", "́", "\t", "㋿", "Ⅳ", ">漢", "👍🏽", " ", " Ａ"]} +{"text": "'re٣٤٥٦EOTEOT<३😀🏽're-Ⅳ'D­ß\té३​漢٣٤٥٦-'S>ḍ̇ſ0sſ9-d0İ", "tokens": 61, "pieces": ["'re", "٣٤٥", "٦", "EOTEOT", "<", "३", "😀🏽'", "re", "-", "Ⅳ", "'D", "­ß", "\te", "́", "३", "​漢", "٣٤٥", "٦", "-'", "S", ">ḋ", "̣ſ", "0", "sſ", "9", "-d", "0", "İ"]} +{"text": "12345678\tſ", "tokens": 6, "pieces": ["123", "456", "78", "\tſ"]} +{"text": " EOT\"Ⅳ😀🏽'ſ", "tokens": 14, "pieces": [" ", " EOT", "\"", "Ⅳ", "😀🏽'", "ſ"]} +{"text": "½ßs\u000b\"😀🏽'llDžＡ .9İ'Re#$%12345678é", "tokens": 27, "pieces": ["½", "ßs", "\u000b", "\"😀🏽'", "llDžＡ", " ", ".", "9", "İ", "'Re", "#$%", "123", "456", "78", "é"]} +{"text": " 👍🏽\u000b३ !!​!!漢<|fim_prefix|>\n
fi a½\t12345678'Re' \u000b12345678", "tokens": 63, "pieces": ["", " ", " 👍🏽", "\u000b", "", "३", " ", " !!​!!", "漢", "<|", "fim", "_prefix", "|>\n", "
fi", " a", "½", "\t", "123", "456", "78", "'Re", "'", " ", "\u000b", "123", "456", "78", "<", "EOT", ">"]} +{"text": "'re😀🏽EOTß\ns'll\n#$%0<́é>ع<Ⅳ'ع", "tokens": 48, "pieces": ["'re", "😀🏽", "EOTß", "\n", "s", "'ll", "\n", "#$%", "0", "<<", "EOT", ">́", "e", "́>", "ع", "<", "Ⅳ", "'ع"]} +{"text": "(字as'ſ!ꟲ
9m👍🏽\"ⅣZḿ\"'Tİá12345678Dž'll😀🏽0\nDž!㋿ḍ̇Ａa'll'M٣٤٥٦ꟲ 
12345678", "tokens": 66, "pieces": ["Dž", "123", "456", "78", "'re", "'ll", "㋿d", "9", "\"👍🏽<", "META", "_START", ">Dž", "'ll", "😀🏽", "0", "\n", "Dž", "!㋿", "ḋ", "̣Ａa", "'ll", "'M", "٣٤٥", "٦", "ꟲ", " ", "
", "123", "456", "78"]} +{"text": "aa‍!Z㍿👍🏽!!>٣٤٥٦ ꟲDžＡ! ½ꟲéé.Dždafi𐞁Dž­ \n…Ａ-👍🏽, 😀🏽𐞁👍🏽", "tokens": 84, "pieces": ["aa", "‍!", "Z", "㍿👍🏽!!>", "٣٤٥", "٦", " ꟲDžＡ", "!", " ", "½", "ꟲéé", ".Dždafi", "𐞁Dž", "­", " \n", "…Ａ", "-👍🏽,", " ", " 😀🏽", "𐞁", "👍🏽"]} +{"text": "<|endoftext|>½३㋿#$%a9,.åDžDž'S ß漢12345678….'TDž", "tokens": 40, "pieces": ["<|", "endoftext", "|>", "½३", "㋿#$%", "a", "9", ",.", "a", "̊DžDž", "'S", " ", " ß漢", "123", "456", "78", "…", ".'", "TDž"]} +{"text": "dⅣfifi'MⅣ३‍'s\n­'ll (!!'s é\"-", "tokens": 29, "pieces": ["d", "Ⅳ", "fifi", "'M", "Ⅳ३", "‍'", "s", "\n", "­'", "ll", " ", "(!!'", "s", " ", " e", "́\"-"]} +{"text": "㋿🙂 ḍ̇#$%\r\n ٣٤٥٦½ꟲ🙂'ſß\u000b\rعa‍<|endoftext|> Ａ \na­Z'll\r\n'T\r\n\r\nsⅣ'ſſEOT #$%​", "tokens": 77, "pieces": ["㋿🙂", " ḋ", "̣#$%\r\n", " ", " ", "٣٤٥", "٦½", "ꟲ", "🙂'", "ſß", "\u000b\r", "عa", "‍<|", "endoftext", "|>", " <", "EOT", ">Ａ", " \n", "a", "­Z", "'ll", "\r\n", "'T", "\r\n\r\n", "s", "Ⅳ", "'ſ", "ſEOT", " <", "EOT", ">#$%​"]} +{"text": "漢ع Ⅳ🙂​'ll<|endoftext|> 'ḍ̇​İ…३\r\n٣٤٥٦", "tokens": 40, "pieces": ["漢ع", " ", "Ⅳ", "🙂​'", "ll", "<|", "endoftext", "|>", " ", " '", "ḋ", "̣​", "İ", "…", "३", "\r\n", "٣٤٥", "٦"]} +{"text": "s​🙂9ſ0\r's👍🏽'VE 9(Ａ", "tokens": 23, "pieces": ["s", "​🙂", "9", "ſ", "0", "\r", "'s", "👍🏽'", "VE", " ", "9", "(Ａ"]} +{"text": ".'S'VE字漢'lĺaDž!!éEOT\u000b<　Ａ㋿aådع'SEOT\u000b​­<'Re", "tokens": 40, "pieces": [".'", "S", "'VE", "字漢", "'ll", "́aDž", "!!", "e", "́EOT", "\u000b", "<", "　Ａ", "㋿aa", "̊dع", "'", "SEOT", "\u000b", "​­<'", "Re"]} +{"text": "fi!! ", "tokens": 4, "pieces": ["fi", "!!", " "]} +{"text": "\"‍'ſ'Ma9'VE>\r\n'llfi", "tokens": 20, "pieces": ["\"‍'", "ſ", "'M", "a", "9", "'", "VE", ">\r\n", "'ll", "fi"]} +{"text": "mß'll'M 9عd9e'retEOT\r\n\r\n३'ſ9
㋿", "tokens": 26, "pieces": ["mß", "'ll", "'M", " ", "9", "عd", "9", "e", "'re", "tEOT", "\r\n\r\n", "३", "'ſ", "9", "
", "㋿"]} +{"text": "🙂0!!!‍½'a½½>s字‍dfi-'\r'ſ'reⅣZß'Re…", "tokens": 36, "pieces": ["🙂", "0", "!!!‍", "½", "'a", "½½", ">s字", "‍dfi", "-'\r", "'ſ", "'", "re", "Ⅳ", "Zß", "'Re", "…"]} +{"text": "!!eⅣ!<|fim_prefix|>", "tokens": 11, "pieces": ["!!", "e", "Ⅳ", "!<|", "fim", "_prefix", "|>"]} +{"text": "a'ſ ", "tokens": 5, "pieces": ["a", "'ſ", " "]} +{"text": "<|endoftext|>½és're‍", "tokens": 28, "pieces": ["<|", "endoftext", "|>", "½", "és", "'re", "‍"]} +{"text": "'M's​\r­　're'S'sdfi're'VE(​Ⅳ", "tokens": 19, "pieces": ["'M", "'s", "​\r", "­", "　", "'re", "'S", "'s", "dfi", "'re", "'VE", "(​", "Ⅳ"]} +{"text": "
३EOT éfi'll\r!…İ", "tokens": 16, "pieces": ["
", "३", "EOT", " ", " éfi", "'ll", "\r", "!", "…İ"]} +{"text": "'Ts,m", "tokens": 3, "pieces": ["'T", "s", ",m"]} +{"text": "𐞁Z12345678é\"\u000b'ſ ́٣٤٥٦ꟲZ", "tokens": 36, "pieces": ["𐞁Z", "123", "456", "78", "e", "́\"<", "EOT", ">", "\u000b", "'ſ", " ", " <", "EOT", ">́", "٣٤٥", "٦", "ꟲZ"]} +{"text": "'M're Ⅳ𐞁­\"", "tokens": 11, "pieces": ["'M", "'re", " ", "Ⅳ", "𐞁", "­\""]} +{"text": "
㋿\r\n \nés…", "tokens": 10, "pieces": ["
", "㋿\r\n", " \n", "és", "…"]} +{"text": "<(٣٤٥٦.m0字İ're٣٤٥٦m\r\n\r\n\r\n　00Z'M \n'ſZ­Z's㋿ \n🙂 \n\nDž<|fim_prefix|>mİ's<|endoftext|>", "tokens": 62, "pieces": ["<(", "٣٤٥", "٦", ".m", "0", "字İ", "'re", "٣٤٥", "٦", "m", "\r\n\r\n\r\n", "　", "00", "Z", "'M", " \n", "'ſ", "Z", "­Z", "'s", "㋿", " \n", "🙂", " \n\n", "Dž", "<|", "fim", "_prefix", "|>", "mİ", "'s", "<|", "endoftext", "|>"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'ll३\u000b<|endoftext|>🙂\"!…Z字'T㋿s 'T字!👍🏽…🙂…㍿,fi\"ḍ̇!ع\nå ​Ⅳ　½\"t", "tokens": 71, "pieces": ["'ll", "३", "\u000b", "<|", "endoftext", "|>🙂\"!", "…Z字", "'T", "㋿s", "", " ", "'T", "字", "!👍🏽", "…", "🙂", "…", "㍿,", "fi", "\"ḋ", "̣!", "ع", "\n", "a", "̊", " ​", "Ⅳ", "　", "½", "\"t"]} +{"text": " ꟲ \u000b12345678 \r
👍🏽𐞁", "tokens": 23, "pieces": [" ꟲ", " ", "\u000b", "123", "456", "78", " \r", "
", "👍🏽", "𐞁"]} +{"text": "½9🙂ß", "tokens": 5, "pieces": ["½9", "🙂ß"]} +{"text": "'Re9́<\n漢t\"½#$%\u000b\u000b字٣٤٥٦('M'M#$%'‍　\"'S字​<|fim_prefix|> 'VE\u000b👍🏽'TZ​d", "tokens": 54, "pieces": ["'Re", "9", "́<\n", "漢t", "\"", "½", "#$%", "\u000b", "\u000b字", "٣٤٥", "٦", "('", "M", "'M", "#$%'‍", "　", "\"'", "S字", "​<|", "fim", "_prefix", "|>", " '", "VE", "\u000b", "👍🏽'", "TZ", "​d"]} +{"text": "!\r\n​　　0ſ'T'reḍ̇​ ㋿#$%9EOT​", "tokens": 26, "pieces": ["!\r\n", "​", "　", "　", "0", "ſ", "'T", "'re", "ḋ", "̣​", " ", " ㋿#$%", "9", "EOT", "​"]} +{"text": "'VE'S.\r\n\r\n٣٤٥٦<|fim_prefix|>\r'VEＡ\nＡZ\u000b.", "tokens": 34, "pieces": ["'VE", "'S", ".\r\n\r\n", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\r", "'VE", "Ａ", "\n", "ＡZ", "\u000b", "."]} +{"text": "'VE!.12345678éＡ'fiea \nḍ̇\t'Re0'm#$%𐞁!!‍EOT<|endoftext|>　Ａ,३Dž", "tokens": 53, "pieces": ["'VE", "!.", "123", "456", "78", "e", "́Ａ", "'<", "EOT", ">fiea", " \n", "ḋ", "̣", "\t", "'Re", "0", "'m", "#$%", "𐞁", "!!‍", "EOT", "<|", "endoftext", "|>", "　Ａ", ",", "३", "Dž"]} +{"text": ".9aＡ!!åé字#$%Ⅳé 
EOT​mm(…‍'字字EOT \n👍🏽\r\n'.'VE🙂", "tokens": 49, "pieces": [".", "9", "aＡ", "!!", "a", "̊é字", "#$%", "Ⅳ", "é", " ", "
EOT", "​mm", "(", "…", "‍'", "字字EOT", " \n", "👍🏽\r\n", "'.'", "VE", "🙂"]} +{"text": "\r\n👍🏽.sesEOT㋿\"'De!'VE9", "tokens": 21, "pieces": ["\r\n", "👍🏽.", "sesEOT", "㋿\"'", "De", "!'", "VE", "9"]} +{"text": "\t!!…'Re<|endoftext|> å12345678Ⅳ漢<'re!!mEOT\tEOTß　<|endoftext|>EOTm-𐞁Ａ ḍ̇漢ⅣZ!é'ſ😀🏽EOT", "tokens": 77, "pieces": ["\t", "!!", "…", "'Re", "<|", "endoftext", "|>", " a", "̊", "123", "456", "78Ⅳ", "漢", "<'", "re", "!!", "mEOT", "\tEOT", "ß", "　", "<|", "endoftext", "|>", "EOTm", "-𐞁Ａ", " ḋ", "̣漢", "Ⅳ", "Z", "!e", "́'", "ſ", "😀🏽", "EOT"]} +{"text": "\u000b\rm­s'T𐞁Ⅳ're…३é>Ⅳ‍'s'ſ(\r\nfi<字tſEOT½éd́-'reDž\"<|endoftext|>m㋿'M", "tokens": 60, "pieces": ["\u000b\r", "m", "­s", "'T", "𐞁", "Ⅳ", "'re", "…", "३", "e", "́>", "Ⅳ", "‍'", "s", "'ſ", "(\r\n", "fi", "<字tſEOT", "½", "e", "́d", "́-'", "reDž", "\"<|", "endoftext", "|>", "m", "㋿'", "M"]} +{"text": "Ⅳ#$%'Ree\u000b½<|endoftext|>ß​½Ａ½s 'sEOT𐞁\r\u000b½ḍ̇'٣٤٥٦", "tokens": 47, "pieces": ["Ⅳ", "#$%'", "Ree", "\u000b", "½", "<|", "endoftext", "|>", "ß", "​", "½", "Ａ", "½", "s", " ", "'s", "EOT𐞁", "\r", "\u000b", "½", "ḋ", "̣'", "٣٤٥", "٦"]} +{"text": "ſé٣٤٥٦0‍t­Ⅳ½ꟲ'llⅣe\"½ ", "tokens": 30, "pieces": ["ſe", "́", "٣٤٥", "٦0", "‍t", "­", "Ⅳ½", "ꟲ", "'ll", "Ⅳ", "e", "\"", "½", " "]} +{"text": "
>'T½émå9e>😀🏽ém İéEOT \n#$%Ⅳ'Rea<|endoftext|>å12345678𐞁ZEOT　字t'T", "tokens": 57, "pieces": ["
", ">'", "T", "½", "e", "́ma", "̊", "9", "e", ">😀🏽", "e", "́m", " ", " İe", "́EOT", " \n", "#$%", "Ⅳ", "'Re", "a", "<|", "endoftext", "|>", "a", "̊", "123", "456", "78", "𐞁ZEOT", "　字t", "'T"]} +{"text": "'D.é㋿!\r12345678٣٤٥٦\"fi\ts'M३'Dع㋿\r\u000bⅣm\n-𐞁㍿", "tokens": 62, "pieces": ["字ß", "'Re", "\n", "𐞁a", "̊ḋ", "̣漢a", "̊", "\t", "'S", "!!'", "ſ", "9", "<|", "fim", "_prefix", "|>", "\ts", "'M", "३", "'D", "ع", "㋿<", "EOT", ">\r", "\u000b", "Ⅳ", "m", "\n", "-𐞁", "㍿"]} +{"text": "\n're'reDžfi
Dž  \r\n", "tokens": 13, "pieces": ["\n", "'re", "'re", "Džfi", "
Dž", "  \r\n"]} +{"text": "!EOT.'ſ \n'T're'VE'Dſ  字 ", "tokens": 24, "pieces": ["!EOT", ".'", "ſ", " \n", "'T", "'", "re", "'VE", "'D", "ſ", " ", " 字", " <", "EOT", ">"]} +{"text": "'sm!　 t\"٣٤٥٦漢Zé'll 'll!٣٤٥٦ -00㋿́\u000b \n m　́'Ddع .Dž𐞁", "tokens": 56, "pieces": ["'s", "m", "!", "　 ", " t", "\"", "٣٤٥", "٦", "漢Ze", "́'", "ll", " '", "ll", "!", "٣٤٥", "٦", " ", " -", "00", "㋿́", "\u000b \n", " m", "　", "́'", "Ddع", " .", "Dž𐞁"]} +{"text": "'S 'ſ \nt
​fi\nſ㍿9'VE12345678Z㍿<|endoftext|>Ⅳ👍🏽's<\n\r\n\r\n", "tokens": 53, "pieces": ["'S", " ", " '", "ſ", " \n", "t", "
", "​", "fi", "\n", "ſ", "㍿", "9", "'VE", "123", "456", "78", "Z", "㍿<|", "endoftext", "|>", "Ⅳ", "👍🏽'", "s", "<<", "META", "_START", ">\n\r\n\r\n"]} +{"text": "漢👍🏽.Džſ0!!\r\n'T9å åaåEOTİꟲ㍿‍㋿\t\tḍ̇", "tokens": 52, "pieces": ["漢", "👍🏽.", "Džſ", "0", "!!\r\n", "'T", "9", "a", "̊", " ", "a", "̊aa", "̊EOTİꟲ", "㍿‍㋿", "\t", "\tḋ", "̣"]} +{"text": "Z\r\n\r\n\r'ſ'MZ\r,12345678㍿㍿", "tokens": 19, "pieces": ["Z", "\r\n\r\n\r", "'ſ", "'M", "Z", "\r", ",", "123", "456", "78", "㍿㍿"]} +{"text": "　\rd'ſa😀🏽're३<㋿\r\n\r\nꟲZ<🙂's <|fim_prefix|>mꟲ👍🏽字", "tokens": 48, "pieces": ["　\r", "d", "'ſ", "a", "😀🏽'", "re", "३", "<㋿\r\n\r\n", "ꟲZ", "<🙂'", "s", " ", " <|", "fim", "_prefix", "|>", "mꟲ", "👍🏽", "字"]} +{"text": "ßİm(\r\n'lléＡ're\rs'DDž!fi३عdＡ'D-
​e㍿!!‍", "tokens": 36, "pieces": ["ßİm", "(\r\n", "'ll", "e", "́Ａ", "'re", "\r", "s", "'D", "Dž", "!fi", "३", "عdＡ", "'D", "-", "
", "​e", "㍿!!‍"]} +{"text": "㍿ꟲ0dßt's漢 \n🙂😀🏽­ꟲ's👍🏽'Dtm<|endoftext|>é٣٤٥٦#$%!字éع'll👍🏽Ａ", "tokens": 73, "pieces": ["㍿ꟲ", "0", "dßt", "'s", "漢", " \n", "🙂😀🏽­", "ꟲ", "'s", "👍🏽'", "Dtm", "<|", "endoftext", "|>", "e", "́", "٣٤٥", "٦", "#$%!<", "EOT", ">字éع", "'", "ll", "👍🏽", "Ａ"]} +{"text": "\u000b<|endoftext|>Ａ‍ß0#$%ⅣtZ>
're　Dž-ع!e\tß'ſ　é عß,<　
ḍ̇", "tokens": 54, "pieces": ["\u000b", "<|", "endoftext", "|>", "Ａ", "‍ß", "0", "#$%", "Ⅳ", "tZ", ">", "
", "'re", "　Dž", "-ع", "!e", "\tß", "'ſ", "　e", "́", " ", " ع", "ß", ",<", "　", "
ḋ", "̣"]} +{"text": "éḍ̇\r\n\t0!'T\ts字㋿Dž\u000bع'reEOT'Re!.Z'ſ", "tokens": 30, "pieces": ["e", "́ḋ", "̣\r\n", "\t", "0", "!'", "T", "\ts字", "㋿Dž", "\u000bع", "'re", "EOT", "'Re", "!.", "Z", "'ſ"]} +{"text": "ḍ̇ée!㍿\"'D👍🏽\"fidＡ'M'VE'!!🙂½\r\n\r\n>'s's
Zḍ̇'ll9漢m٣٤٥٦Z#$%(­", "tokens": 66, "pieces": ["ḋ", "̣é", "e", "!㍿\"'", "D", "👍🏽\"", "fidＡ", "'M", "'VE", "'!!🙂", "½", "\r\n\r\n", ">'", "s", "'s", "
Zḋ", "̣'", "ll", "9", "漢m", "٣٤٥", "٦", "Z", "#$%(­"]} +{"text": "0'lléa😀🏽 \r\n\r\n字… e'Tt> e 'Té​12345678EOTZ㋿
.½0Ａ㍿字𐞁́漢m'sa", "tokens": 54, "pieces": ["0", "'ll", "éa", "😀🏽", " \r\n\r\n", "字", "…", " e", "'T", "t", ">", " e", " ", " '", "Te", "́​", "123", "456", "78", "EOTZ", "㋿", "
", ".", "½0", "Ａ", "㍿字𐞁", "́漢m", "'s", "a"]} +{"text": "<|fim_prefix|>漢　fi𐞁ع#$%.😀🏽#$%#$%12345678½ꟲ'Re!å'Reع😀🏽\n'sfi\ńＡ½\t.>　'M's🙂", "tokens": 65, "pieces": ["<|", "fim", "_prefix", "|>", "漢", "　fi𐞁ع", "#$%.😀🏽#$%#$%", "123", "456", "78½", "ꟲ", "'Re", "!a", "̊'", "Reع", "😀🏽\n", "'s", "fi", "\n", "́Ａ", "½", "\t", ".>", "　", "'M", "'s", "🙂"]} +{"text": "½'s
　　12345678 \nſEOT\r㍿Zḍ̇!EOTſ'T\r\nß 𐞁", "tokens": 40, "pieces": ["½", "'s", "
　", "　", "123", "456", "78", " \n", "ſEOT", "\r", "㍿Zḋ", "̣!", "EOTſ", "'T", "\r\n", "ß", " 𐞁"]} +{"text": "'ſ<|fim_prefix|>'Rea12345678'll­٣٤٥٦Z0३9'Res", "tokens": 36, "pieces": ["'ſ", "<|", "fim", "_prefix", "|>'", "Rea", "123", "456", "78", "'ll", "­", "٣٤٥", "٦", "Z", "0३9", "'Re", "s", ""]} +{"text": "३12345678e-Zİe Ⅳ'(\r\n\r\n‍\r\n\r\né \n漢㍿\nＡ㍿å<
 \nEOT­Ⅳ,\"9<|endoftext|>\t9\u000b\r", "tokens": 56, "pieces": ["३12", "345", "678", "e", "-Zİe", " ", " ", "Ⅳ", "'(\r\n\r\n", "‍\r\n\r\n", "é", " \n", "漢", "㍿\n", "Ａ", "㍿a", "̊<", "
 \n", "EOT", "­", "Ⅳ", ",\"", "9", "<|", "endoftext", "|>", "\t", "9", "\u000b\r"]} +{"text": "é'VE''Ta 
<.­👍🏽\nİ\u000b#$%d٣٤٥٦'ſ'll '漢'Re‍½😀🏽'VE٣٤٥٦", "tokens": 58, "pieces": ["e", "́'", "VE", "''", "Ta", " ", "
", "<.­👍🏽\n", "İ", "\u000b", "#$%", "d", "٣٤٥", "٦", "'ſ", "'ll", " '", "漢", "'Re", "‍", "½", "😀🏽'", "VE", "٣٤٥", "٦"]} +{"text": "'Mfi \r\n12345678EOT  ٣٤٥٦éZİſ …,👍🏽!", "tokens": 36, "pieces": ["'M", "fi", " \r\n", "123", "456", "78", "EOT", "  ", " ", "٣٤٥", "٦", "éZİſ", " ", "…", ",👍🏽!"]} +{"text": "'🙂😀🏽a\"- 0 '𐞁'VEßé'D३👍🏽İ'D👍🏽ßＡ'VEt ㋿ \tİé", "tokens": 61, "pieces": ["'🙂😀🏽", "a", "\"-", " ", " ", "0", " ", " <", "EOT", ">'", "𐞁", "'VE", "ße", "́'", "D", "३", "👍🏽", "İ", "'D", "👍🏽", "ßＡ", "'VE", "t", " ", "㋿", " ", "\tİé"]} +{"text": "👍🏽aßa\" .å", "tokens": 14, "pieces": ["👍🏽", "aßa", "\"", " .", "a", "̊"]} +{"text": " \nꟲ½\te\u000b३ع'Re漢'T9Ⅳ'>३-<|fim_prefix|><­-'D'll'D'sİ<|endoftext|>12345678EOT
", "tokens": 52, "pieces": [" \n", "ꟲ", "½", "\te", "\u000b", "३", "ع", "'Re", "漢", "'T", "9Ⅳ", "'>", "३", "-<", "EOT", "><|", "fim", "_prefix", "|><­-'", "D", "'ll", "'D", "'s", "İ", "<|", "endoftext", "|>", "123", "456", "78", "EOT", "
"]} +{"text": "İåeEOT\n‍👍🏽‍😀🏽 \r\n\r\n12345678", "tokens": 27, "pieces": ["İa", "̊eEOT", "\n", "‍👍🏽‍😀🏽", " \r\n\r\n", "123", "456", "78"]} +{"text": "#$%'ſ\r\n 90😀🏽<|fim_prefix|>'ſ \n \n­㋿då", "tokens": 31, "pieces": ["#$%'", "ſ", "\r\n", " ", " ", "90", "😀🏽<|", "fim", "_prefix", "|>'", "ſ", " \n \n", "­㋿", "da", "̊"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'s'<|endoftext|>EOTعdİ\"å\ta", "tokens": 17, "pieces": ["'s", "'<|", "endoftext", "|>", "EOTعdİ", "\"a", "̊", "\ta"]} +{"text": "'Ａ🙂 \n's\r12345678're", "tokens": 12, "pieces": ["'Ａ", "🙂", " \n", "'s", "\r", "123", "456", "78", "'re"]} +{"text": "ꟲ<'M>", "tokens": 6, "pieces": ["ꟲ", "<'", "M", ">"]} +{"text": "👍🏽fißå<İ㍿#$%'Re٣٤٥٦0́٣٤٥٦Z<|endoftext|>‍<|endoftext|>ß👍🏽ßfi,", "tokens": 70, "pieces": ["👍🏽<", "META", "_START", ">fißa", "̊<", "İ", "㍿#$%'", "Re", "٣٤٥", "٦0", "́", "٣٤٥", "٦", "Z", "<|", "endoftext", "|>‍<|", "endoftext", "|>", "ß", "👍🏽", "ßfi", ","]} +{"text": "​<|fim_prefix|>\r'T㋿'VE字.​EOT,'T३", "tokens": 24, "pieces": ["​<|", "fim", "_prefix", "|>\r", "'T", "㋿'", "VE字", ".​", "EOT", ",'", "T", "३"]} +{"text": "\u000b", "tokens": 5, "pieces": ["", "\u000b"]} +{"text": "12345678㋿Džé½>\"!!e'Re…'M٣٤٥٦EOTİ😀🏽", "tokens": 37, "pieces": ["123", "456", "78", "㋿Džé", "½", ">\"!!", "e", "'Re", "…", "'M", "٣٤٥", "٦", "EOTİ", "😀🏽"]} +{"text": "\"å\u000bⅣ.e é३", "tokens": 11, "pieces": ["\"a", "̊", "\u000b", "Ⅳ", ".e", " ", " é", "३"]} +{"text": "👍🏽ZDž<|endoftext|> ", "tokens": 17, "pieces": ["👍🏽", "ZDž", "<|", "endoftext", "|>", " "]} +{"text": "​\r\nḍ̇́å'T　Z漢0

'Re\r\nå٣٤٥٦\t½㋿ꟲ'T㋿　㍿😀🏽 'VEß'refi>\nḍ̇m's12345678-ß\r\n", "tokens": 76, "pieces": ["​\r\n", "ḋ", "̣́", "a", "̊'", "T", "　Z漢", "0", "
", "
", "'Re", "\r\n", "a", "̊", "٣٤٥", "٦", "\t", "½", "㋿ꟲ", "'T", "㋿", "　", "㍿😀🏽", " '", "VEß", "'re", "fi", ">\n", "ḋ", "̣m", "'s", "123", "456", "78", "-ß", "\r\n"]} +{"text": "½ 12345678('ſ😀🏽<|endoftext|>𐞁😀🏽\r\n\r\n(!'Sſ<|fim_prefix|>Z's👍🏽 👍🏽,", "tokens": 62, "pieces": ["½", " ", " ", "123", "456", "78", "('", "ſ", "😀🏽<|", "endoftext", "|>", "𐞁", "😀🏽\r\n\r\n", "(!'", "Sſ", "<|", "fim", "_prefix", "|>", "Z", "'s", "👍🏽", " ", "👍🏽,"]} +{"text": "'llⅣ३>½12345678ḍ̇́é>a>'D(\r\n\r\n,12345678'M", "tokens": 29, "pieces": ["'ll", "Ⅳ३", ">", "½12", "345", "678", "ḋ", "̣́", "e", "́>", "a", ">'", "D", "(\r\n\r\n", ",", "123", "456", "78", "'M"]} +{"text": "İ字㋿㋿Z'Sع
ḍ̇EOT.'D
㋿e 🙂12345678\u000b,ßå'Reé🙂, \ne'ſ३'s>‍\"ꟲ's'", "tokens": 65, "pieces": ["İ字", "㋿㋿", "Z", "'S", "ع", "
ḋ", "̣EOT", ".'", "D", "
", "㋿e", " 🙂", "123", "456", "78", "\u000b", ",<", "META", "_START", ">ßa", "̊'", "Ree", "́🙂,", " \n", "e", "'ſ", "३", "'s", ">‍\"", "ꟲ", "'s", "'"]} +{"text": "m👍🏽", "tokens": 7, "pieces": ["m", "👍🏽"]} +{"text": "字ⅣDž​ -0漢12345678", "tokens": 14, "pieces": ["字", "Ⅳ", "Dž", "​", " ", " -", "0", "漢", "123", "456", "78"]} +{"text": "'DEOT' \n 0字Z३", "tokens": 11, "pieces": ["'D", "EOT", "'", " \n", " ", "0", "字Z", "३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "字Ⅳm​d‍½'re👍🏽,\nDžs㍿𐞁Z12345678\r\n", "tokens": 44, "pieces": ["字", "Ⅳ", "m", "​d", "‍", "½", "'re", "👍🏽,\n", "Džs", "㍿𐞁Z", "123", "456", "78", "\r\n"]} +{"text": "'ſ<|endoftext|>t'Tdaع👍🏽字\rZ'Mꟲ\r 漢\t0fi<|endoftext|>'MⅣe',३­Ａt­​Ⅳé½><|fim_prefix|>", "tokens": 68, "pieces": ["'ſ", "<|", "endoftext", "|>", "t", "'T", "daع", "👍🏽", "字", "\r", "Z", "'M", "ꟲ", "\r", " ", " 漢", "\t", "0", "fi", "<|", "endoftext", "|>'", "M", "Ⅳ", "e", "',", "३", "­Ａt", "­​", "Ⅳ", "e", "́", "½", "><|", "fim", "_prefix", "|>"]} +{"text": "İ'Ré　Ⅳ ſ'reZsm\ń 'Tḍ̇'reEOT　​  \n𐞁\r'Sa'Té٣٤٥٦­́٣٤٥٦DžⅣＡ", "tokens": 63, "pieces": ["İ", "'Re", "́", "　", "Ⅳ", " ſ", "'re", "Zsm", "\n", "́", " '", "Tḋ", "̣'", "reEOT", "　", "​", "  \n", "𐞁", "\r", "'", "Sa", "'T", "é", "٣٤٥", "٦", "­́", "٣٤٥", "٦", "Dž", "Ⅳ", "Ａ"]} +{"text": "…字\u000btå>'T字'VEḍ̇½'", "T字", "'VE", "ḋ", "̣", "½", "İ'sé\r\n\r\n­ßs\r\n\r\nfi<9're\n'MéⅣ<|fim_prefix|>#$%.\"'Te३ \r\n\r\n>　e٣٤٥٦\u000bed", "tokens": 60, "pieces": ["Z", "́'", "s", "123", "456", "78", "Z", "İ", "'s", "e", "́\r\n\r\n", "­ßs", "\r\n\r\n", "fi", "<", "9", "'re", "\n", "'M", "e", "́", "Ⅳ", "<|", "fim", "_prefix", "|>#$%.\"'", "Te", "३", " \r\n\r\n", ">", "　e", "٣٤٥", "٦", "\u000bed"]} +{"text": "'s'll ३Džad'll-　 ㍿'T'ſ\"Ⅳ 字漢 ३ !!İa\r\n\r\n३0s", "tokens": 40, "pieces": ["'s", "'ll", " ", " ", "३", "Džad", "'ll", "-", "　", " ", "㍿'", "T", "'ſ", "\"", "Ⅳ", " ", " 字漢", " ", "३", " ", "!!", "İa", "\r\n\r\n", "३0", "s"]} +{"text": " ‍㍿'VEEOT-\t🙂́12345678\r\n\r\n'Re#$%😀🏽​🙂-\r\n\r\n😀🏽're#$%ع,'VEEOT㍿👍🏽é𐞁m'D　👍🏽٣٤٥٦\"(ſ", "tokens": 80, "pieces": [" ", "‍㍿'", "VEEOT", "-", "\t", "🙂́", "123", "456", "78", "\r\n\r\n", "'Re", "#$%😀🏽​🙂-\r\n\r\n", "😀🏽'", "re", "#$%", "ع", ",'", "VEEOT", "㍿👍🏽", "é𐞁m", "'D", "　", "👍🏽", "٣٤٥", "٦", "\"(", "ſ"]} +{"text": ".a\r\n
'Sꟲ㋿fiDž\n9fi 9🙂'Re👍🏽 d……३", "tokens": 38, "pieces": [".a", "\r\n", "
", "'S", "ꟲ", "㋿fiDž", "\n", "9", "fi", " ", "9", "🙂'", "Re", "👍🏽", " d", "…", "…", "३"]} +{"text": "d👍🏽fi!m漢漢!!‍EOT\r\n<|fim_prefix|>'Re­'ll㍿　🙂\r\n\r\n,'Dḍ̇e­　­étع́'llß<|endoftext|>\r　-", "tokens": 71, "pieces": ["d", "👍🏽", "fi", "!m漢", "漢", "!!‍", "EOT", "\r\n", "<|", "fim", "_prefix", "|>'", "Re", "­'", "ll", "㍿", "　", "🙂\r\n\r\n", ",'", "Dḋ", "̣e", "­", "　", "­e", "́tع", "́'", "llß", "<|", "endoftext", "|>\r", "　", "-"]} +{"text": "<|endoftext|>'S!eⅣ\r\naéEOT'ſ\r\n\r\n­<|endoftext|>sß　👍🏽ⅣéEOT​", "tokens": 46, "pieces": ["<|", "endoftext", "|>'", "S", "!e", "Ⅳ", "\r\n", "aéEOT", "'ſ", "\r\n\r\n", "­<|", "endoftext", "|>", "sß", "　", "👍🏽", "Ⅳ", "éEOT", "​"]} +{"text": "\ŕ<|fim_prefix|>'D'M 
ḍ̇s \n😀🏽#$%Ａéİ　㋿\n ", "tokens": 38, "pieces": ["\r", "́<|", "fim", "_prefix", "|>'", "D", "'M", " ", "
ḋ", "̣s", " \n", "😀🏽#$%", "Ａéİ", "　", "㋿\n", " "]} +{"text": "ꟲ\"Zḍ̇ååfiꟲ", "tokens": 21, "pieces": ["ꟲ", "\"Zḋ", "̣a", "̊a", "̊fiꟲ"]} +{"text": "\r\n\r\nm'll're'VE\u000bEOTd\t🙂're٣٤٥٦ꟲ漢\t字#$%ḍ̇\"de", "tokens": 42, "pieces": ["\r\n\r\n", "m", "'ll", "'re", "'VE", "\u000b", "EOTd", "\t", "🙂'", "re", "٣٤٥", "٦", "ꟲ漢", "\t字", "#$%", "ḋ", "̣\"", "de"]} +{"text": ".漢\t'㋿12345678ßZEOT'VE <\r \n…㍿(fis㍿'VEDž's-ſ12345678٣٤٥٦", "tokens": 51, "pieces": [".漢", "\t", "'㋿", "123", "456", "78", "ßZEOT", "'VE", " ", "<\r", " \n", "…", "㍿(", "fis", "㍿'", "VEDž", "'s", "-ſ", "123", "456", "78٣", "٤٥٦"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'TEOT\"-ſ㋿ \n㋿", "tokens": 13, "pieces": ["'T", "EOT", "\"-", "ſ", "㋿", " \n", "㋿"]} +{"text": ".'llåm0'T­‍'VE're m½\ŕ\t", "tokens": 19, "pieces": [".'", "lla", "̊m", "0", "'T", "­‍'", "VE", "'re", " ", " m", "½", "\r", "́", "\t"]} +{"text": "'Re😀🏽३'Mt", "tokens": 14, "pieces": ["'Re", "😀🏽", "३", "'M", "t", ""]} +{"text": "ⅣZå\r!('llDž\r\n
ßſ're 𐞁'sꟲ", "tokens": 28, "pieces": ["Ⅳ", "Za", "̊\r", "!('", "llDž", "\r\n", "
ßſ", "'re", " 𐞁", "'s", "ꟲ"]} +{"text": "👍🏽d́.\r\n३ \n\"　\t😀🏽e Z'ḍ̇EOT👍🏽>", "tokens": 38, "pieces": ["👍🏽", "d", "́.\r\n", "३", " \n", "\"", "　", "\t", "😀🏽", "e", " ", " Z", "'ḋ", "̣EOT", "👍🏽>"]} +{"text": "9m㋿12345678's'S'Ts'Re'Re\t<|fim_prefix|>.\"\r\n", "tokens": 23, "pieces": ["9", "m", "㋿", "123", "456", "78", "'s", "'S", "'T", "s", "'Re", "'Re", "\t", "<|", "fim", "_prefix", "|>.\"\r\n"]} +{"text": "EOT!字,'ſ字\r😀🏽éİ#$%٣٤٥٦३m🙂\".é'S🙂ḍ̇ſ \n \n\" 9EOTe", "tokens": 59, "pieces": ["EOT", "!字", ",'", "ſ字", "\r", "😀🏽", "e", "́<", "EOT", ">İ", "#$%", "٣٤٥", "٦३", "m", "🙂\"<", "META", "_START", ">.", "é", "'S", "🙂ḋ", "̣ſ", " \n \n", "\"", " ", "9", "EOTe"]} +{"text": " \na\r\n\r\n9'll<'VE'Re漢
­\r\nⅣ\u000b㋿字🙂(>Ａ", "tokens": 31, "pieces": [" \n", "a", "\r\n\r\n", "9", "'", "ll", "<'", "VE", "'Re", "漢", "
", "­\r\n", "Ⅳ", "\u000b", "㋿字", "🙂(>", "Ａ"]} +{"text": "㋿ſ Ⅳ ><|fim_prefix|>
Z🙂
d\t", "tokens": 28, "pieces": ["㋿", "ſ", " ", "Ⅳ", " ", "><|", "fim", "_prefix", "|>", "
Z", "🙂", "
d", "\t"]} +{"text": "
((½Ⅳ 𐞁‍#$%(é'T<#$%éEOTZ12345678's\n‍\u000b.ſe<|endoftext|>㋿​́>㋿<|fim_prefix|>
字Dž٣٤٥٦", "tokens": 76, "pieces": ["
", "((", "½Ⅳ", " 𐞁", "‍#$%(", "é", "'T", "<#$%", "éEOTZ", "123", "456", "78", "'s", "\n", "‍", "\u000b", ".ſe", "<|", "endoftext", "|>㋿​́>㋿<|", "fim", "_prefix", "|>", "
字", "Dž", "٣٤٥", "٦"]} +{"text": "­漢EOT'D३é \r\n\r\n‍'s㍿ \n'll\n12345678ßEOTtd İع‍e#$% İ'VEßＡ", "tokens": 42, "pieces": ["­漢EOT", "'D", "३", "é", " \r\n\r\n", "‍'", "s", "㍿", " \n", "'ll", "\n", "123", "456", "78", "ßEOTtd", " İع", "‍e", "#$%", " ", " İ", "'VE", "ßＡ"]} +{"text": "fi'S,\r\n\"​<😀🏽", "tokens": 12, "pieces": ["fi", "'S", ",\r\n", "\"​<😀🏽"]} +{"text": "ſ­ EOT'ſ
'Re👍🏽😀🏽Dž\u000bad🙂s", "tokens": 33, "pieces": ["ſ", "­", " EOT", "'ſ", "", "
", "'Re", "👍🏽😀🏽", "Dž", "\u000bad", "🙂s"]} +{"text": "<|endoftext|>İ\n…-👍🏽㍿m's　­Džt <|fim_prefix|>12345678'T<|fim_prefix|>dḍ̇a", "tokens": 54, "pieces": ["<|", "endoftext", "|>", "İ", "\n", "…", "-👍🏽㍿", "m", "'s", "　", "­Džt", " ", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "<|", "fim", "_prefix", "|>", "dḋ", "̣a"]} +{"text": "('D\r'ſ", "tokens": 6, "pieces": ["('", "D", "\r", "'ſ"]} +{"text": "!!aée\"漢 漢 .㍿　", "tokens": 19, "pieces": ["!!", "ae", "́<", "EOT", ">e", "\"漢", " ", " 漢", " .㍿", "　"]} +{"text": "å½­12345678\t,ع,as'VE\r ٣٤٥٦ḍ̇३ \n㍿٣٤٥٦　ſå½𐞁ßḍ̇\n12345678ǻ", "tokens": 70, "pieces": ["a", "̊", "½", "­", "123", "456", "78", "\t", ",ع", ",as", "'VE", "\r", " ", " ", "٣٤٥", "٦", "ḋ", "̣", "३", " \n", "㍿", "٣٤٥", "٦", "　ſa", "̊", "½", "𐞁ßḋ", "̣\n", "123", "456", "78", "a", "̊́"]} +{"text": "é'Tm
.́字
're​ \né'll", "tokens": 19, "pieces": ["e", "́'", "Tm", "
", ".́", "字", "
", "'re", "​", " \n", "e", "́'", "ll"]} +{"text": "㋿9EOT½,'re'S!'ſ<|endoftext|>🙂३İ're'T\r\n…ꟲ#$%'Sḍ̇😀🏽12345678'ſḍ̇\r's𐞁!!\n
漢#$%", "tokens": 70, "pieces": ["㋿", "9", "EOT", "½", ",'", "re", "'S", "!'", "ſ", "<|", "endoftext", "|>🙂", "३", "İ", "'re", "'T", "\r\n", "…ꟲ", "#$%'", "Sḋ", "̣😀🏽", "123", "456", "78", "'ſ", "ḋ", "̣\r", "'s", "𐞁", "!!\n", "
漢", "#$%"]} +{"text": ".fi　ꟲt12345678\n㍿ḍ̇\r\n\r\ń\n", "㍿ḋ", "̣\r\n\r\n", "́<", "td", "\r\n", "́(", "t", "!"]} +{"text": "İ<|endoftext|>EOTＡ\"e>　 \"㍿", "tokens": 20, "pieces": ["İ", "<|", "endoftext", "|>", "EOTＡ", "\"e", ">", "　", " \"㍿"]} +{"text": "ſ<|endoftext|>9Ａ-'re>½>́ ", "tokens": 19, "pieces": ["ſ", "<|", "endoftext", "|>", "9", "Ａ", "-'", "re", ">", "½", ">́", " "]} +{"text": "Ａ \n(Ⅳ'VEſ(ZZå\r\n\r\nå\" s㍿ḍ̇㍿9e.!!́\n", "tokens": 39, "pieces": ["Ａ", " \n", "(", "Ⅳ", "'VE", "ſ", "(ZZa", "̊\r\n\r\n", "a", "̊\"", " ", " s", "㍿ḋ", "̣㍿", "9", "e", ".!!́\n"]} +{"text": "ꟲDž !<\r\n're -", "tokens": 11, "pieces": ["ꟲDž", " !<\r\n", "'re", " ", " -"]} +{"text": "ß9<|fim_prefix|>'sⅣ'sḍ̇éſ\t#$%Ａ'Mé­9\u000bés>　​漢's😀🏽'll ½!!\r\n'M'T‍\n", "tokens": 61, "pieces": ["ß", "9", "<|", "fim", "_prefix", "|>'", "s", "Ⅳ", "'s", "ḋ", "̣e", "́ſ", "\t", "#$%", "Ａ", "'M", "é", "­", "9", "\u000be", "́s", ">", "　", "​<", "EOT", ">漢", "'s", "😀🏽'", "ll", " ", "½", "!!\r\n", "'M", "'T", "‍\n"]} +{"text": "́,字́\"d's'S\r\n　½9'D ḍ̇½🙂ꟲ'́漢!!🙂", "tokens": 32, "pieces": ["́,", "字", "́\"", "d", "'s", "'S", "\r\n", "　", "½9", "'D", " ", " ḋ", "̣", "½", "🙂ꟲ", "'́", "漢", "!!🙂"]} +{"text": "#$%ZꟲＡam!9Ⅳß½́12345678!!", "tokens": 20, "pieces": ["#$%", "ZꟲＡam", "!", "9Ⅳ", "ß", "½", "́", "123", "456", "78", "!!"]} +{"text": "
\r\n\r\n㍿<0\r\nß'9<|endoftext|>ſ", "tokens": 21, "pieces": ["
\r\n\r\n", "㍿<", "0", "\r\n", "ß", "'", "9", "<|", "endoftext", "|>", "ſ"]} +{"text": "
ع…-å'S9👍🏽EOT́\"'ḍ̇.'é­!́EOT<|fim_prefix|>字åå ", "tokens": 52, "pieces": ["
ع", "…", "-a", "̊'", "S", "9", "👍🏽", "EOT", "́\"'", "ḋ", "̣.'", "é", "­<", "META", "_START", ">!́", "EOT", "<|", "fim", "_prefix", "|>", "字a", "̊a", "̊", " "]} +{"text": "<|fim_prefix|>𐞁'ſ9ḍ̇
字ꟲع éEOT𐞁 #$%!!'ſ​३mEOT\r\nZ(​'s㍿#$%!!\"", "tokens": 64, "pieces": ["<|", "fim", "_prefix", "|>", "𐞁", "'ſ", "9", "ḋ", "̣", "
字ꟲع", " éEOT𐞁", " ", "#$%!!<", "EOT", ">'", "ſ", "​", "३", "mEOT", "\r\n", "Z", "(​'", "s", "㍿#$%!!\""]} +{"text": "Ⅳ½​m\r\nＡ'T…\u000bdDžaİ　<|fim_prefix|>‍'llⅣ‍\"\u000b'S-", "tokens": 44, "pieces": ["Ⅳ½", "​m", "\r\n", "Ａ", "'T", "…", "\u000bdDžaİ", "　", "<|", "fim", "_prefix", "|>‍'", "ll", "", "Ⅳ", "‍\"", "\u000b", "'S", "-"]} +{"text": "ſe… EOT", "tokens": 8, "pieces": ["ſe", "…", " EOT"]} +{"text": "३('VE漢\r\n\r\n 👍🏽-́a‍#$%<\u000b>😀🏽𐞁ſꟲ'Re
ꟲte'llİ", "tokens": 46, "pieces": ["३", "('", "VE漢", "\r\n\r\n", " ", " 👍🏽-́", "a", "‍#$%<", "\u000b", ">😀🏽", "𐞁ſꟲ", "'Re", "
ꟲte", "'ll", "İ"]} +{"text": "½‍\r­'reåⅣİé㍿,'s'S", "tokens": 19, "pieces": ["½", "‍\r", "­'", "rea", "̊", "Ⅳ", "İé", "㍿,'", "s", "'S"]} +{"text": "((
Dž'… 0d\r>aé're'll<|endoftext|>'D'llع<|fim_prefix|>'>عé're9\t \nＡa", "tokens": 50, "pieces": ["((", "
Dž", "'", "…", " ", "0", "d", "\r", ">aé", "'re", "'ll", "<|", "endoftext", "|>'", "D", "'ll", "ع", "<|", "fim", "_prefix", "|>'>", "عe", "́'", "re", "", "9", "\t \n", "Ａa"]} +{"text": "ع'll😀🏽're'Re㋿'M字 ", "tokens": 21, "pieces": ["ع", "'ll", "😀🏽'", "re", "'Re", "㋿'", "M", "字", " "]} +{"text": "fi㋿(d< 0\"Z漢ß‍😀🏽漢''ſé<|fim_prefix|>\r\n!9\"#$%…­EOT👍🏽\t👍🏽'TDž-", "tokens": 63, "pieces": ["fi", "㋿(", "d", "<", " ", "0", "\"Z漢ß", "‍😀🏽", "漢", "''", "ſé", "<|", "fim", "_prefix", "|>\r\n", "!", "9", "\"#$%", "…", "­EOT", "👍🏽", "\t", "👍🏽'", "TDž", "-"]} +{"text": "ſm🙂…Ⅳ𐞁½́st!'re", "tokens": 18, "pieces": ["ſm", "🙂", "…", "Ⅳ", "𐞁", "½", "́st", "!'", "re"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n\r\n😀🏽ع́字🙂ß३'Re\"𐞁\"'ll \n㋿12345678🙂\n's\n #$%!!ꟲ'ſ#$%!!ſ\r\n👍🏽's're'T", "tokens": 60, "pieces": ["\r\n\r\n", "😀🏽", "ع", "́字", "🙂ß", "३", "'Re", "\"𐞁", "\"'", "ll", " \n", "㋿", "123", "456", "78", "🙂\n", "'s", "\n", " ", " #$%!!", "ꟲ", "'ſ", "#$%!!", "ſ", "\r\n", "👍🏽'", "s", "'re", "'T"]} +{"text": "㍿㍿<👍🏽\"🙂 déEOTåḍ̇\r\n\r\n's\t<|fim_prefix|>><'S漢㋿å're \n½'D-", "tokens": 62, "pieces": ["㍿㍿<👍🏽\"🙂", " ", " déEOTa", "̊<", "EOT", ">ḋ", "̣\r\n\r\n", "'s", "\t", "<|", "fim", "_prefix", "|>><'", "S漢", "㋿", "a", "̊'", "re", " \n", "½", "'D", "-"]} +{"text": "'VE.́　…\rⅣİ'Re\"é٣٤٥٦'VE\u000b字٣٤٥٦ḍ̇👍🏽0!!Ａ३é12345678mat\r\n'S👍🏽å漢‍'sDžＡ'S", "tokens": 83, "pieces": ["'VE", ".́", "　…\r", "Ⅳ", "İ", "'Re", "\"é", "٣٤٥", "٦", "'VE", "\u000b字", "٣٤٥", "٦", "ḋ", "̣👍🏽", "0", "!!", "Ａ", "३", "e", "́", "123", "456", "78", "mat", "\r\n", "'S", "👍🏽", "a", "̊漢", "‍'", "sDžＡ", "'S"]} +{"text": "!!字é<|fim_prefix|>'D\r(Ⅳ'Dꟲ,🙂…EOT३'D( \n \n­ \n'lléas
'Ree\r\n- ", "tokens": 45, "pieces": ["!!", "字é", "<|", "fim", "_prefix", "|>'", "D", "\r", "(", "Ⅳ", "'D", "ꟲ", ",🙂", "…EOT", "३", "'D", "(", " \n \n", "­", " \n", "'ll", "e", "́as", "
", "'Re", "e", "\r\n", "-", " "]} +{"text": "…", "tokens": 2, "pieces": ["…"]} +{"text": "'re#$%Dž'😀🏽㋿ع \nå0­
'lla字''s'S😀🏽'ſ \u000bZ-é \"'ll­👍🏽<|fim_prefix|>ꟲ!!\n", "tokens": 67, "pieces": ["'re", "#$%", "Dž", "'😀🏽㋿", "ع", " \n", "a", "̊", "0", "­", "
", "'ll", "a字", "''", "s", "'S", "😀🏽'", "ſ", " ", "\u000bZ", "-e", "́", " ", "\"'", "ll", "­👍🏽<|", "fim", "_prefix", "|>", "ꟲ", "!!\n"]} +{"text": "'D…\u000b🙂a🙂 EOT!!٣٤٥٦12345678\r'Té३\r\n\r\n9'VE", "tokens": 32, "pieces": ["'D", "…", "\u000b", "🙂a", "🙂", " EOT", "!!", "٣٤٥", "٦12", "345", "678", "\r", "'T", "é", "३", "\r\n\r\n", "9", "'VE"]} +{"text": "'Så\u000be𐞁'ſ㍿'s㋿İ'S㍿d\t㋿'S'ſ're's😀🏽३#$%İⅣfi　fifidꟲ'", "ſ", "㍿'", "s", "㋿İ", "'S", "㍿d", "\t", "㋿'", "S", "'ſ", "'re", "'s", "😀🏽", "३", "#$%", "İ", "Ⅳ", "fi", "　fifidꟲ", "éå㍿'s\té\r٣٤٥٦å­å'Re\r\n🙂٣٤٥٦12345678­Ⅳ", "tokens": 53, "pieces": ["t", "-a", "'D", ">éa", "̊㍿'", "s", "\te", "́\r", "٣٤٥", "٦", "a", "̊­", "a", "̊'", "Re", "\r\n", "🙂", "٣٤٥", "٦12", "345", "678", "­", "Ⅳ"]} +{"text": "½'Re'ſ9>d \n<'llé0dß12345678'D­ſ#$%'s", "tokens": 27, "pieces": ["½", "'", "Re", "'ſ", "9", ">d", " \n", "<'", "llé", "0", "dß", "123", "456", "78", "'D", "­ſ", "#$%'", "s"]} +{"text": "'VE", "tokens": 6, "pieces": ["'VE", ""]} +{"text": "'re's३'Re12345678eＡ'VE\nＡ,'s#$%\n𐞁\"Dž", "tokens": 31, "pieces": ["'re", "'s", "३", "'Re", "123", "456", "78", "eＡ", "'VE", "\n", "Ａ", ",'", "s", "#$%\n", "𐞁", "\"Dž"]} +{"text": "\n'VE🙂½<|endoftext|><\n'M!'Mꟲ­𐞁ꟲ're\u000bfis! 9'Da\r\na👍🏽😀🏽're( 12345678'", "tokens": 64, "pieces": ["\n", "'VE", "🙂", "½", "<|", "endoftext", "|><\n", "'M", "!'", "Mꟲ", "­<", "EOT", ">𐞁ꟲ", "'re", "\u000bfis", "!", " ", "9", "'D", "a", "\r\n", "a", "👍🏽😀🏽'", "re", "(", " ", " ", "123", "456", "78", "'"]} +{"text": "fi \t👍🏽12345678're​ꟲ👍🏽EOT٣٤٥٦'ll٣٤٥٦ å('.", "tokens": 47, "pieces": ["fi", " ", "\t", "👍🏽", "123", "456", "78", "'re", "​ꟲ", "👍🏽", "EOT", "٣٤٥", "٦", "'ll", "٣٤٥", "٦", " a", "̊('."]} +{"text": "\"\rd㋿\t \t　'S\tså'VE🙂", "tokens": 17, "pieces": ["\"\r", "d", "㋿", "\t \t", "　", "'S", "\tsa", "̊'", "VE", "🙂"]} +{"text": "m३\r\n 𐞁👍🏽\r's,s😀🏽́­ \r\n\r\n🙂𐞁", "tokens": 32, "pieces": ["m", "३", "\r\n", " 𐞁", "👍🏽\r", "'s", ",s", "😀🏽́­", " \r\n\r\n", "🙂𐞁"]} +{"text": "0ſ\r!EOT​é'Ḿ\r'M'S​et'reſ\u000b", "tokens": 24, "pieces": ["", "0", "ſ", "\r", "!EOT", "​é", "'M", "́\r", "'M", "'S", "​et", "'re", "ſ", "\u000b"]} +{"text": "#$%,d
EOT漢\n​Ⅳꟲe'M", "tokens": 18, "pieces": ["#$%,", "d", "
EOT漢", "\n", "​", "Ⅳ", "ꟲe", "'M"]} +{"text": "><|endoftext|>!ḍ̇'Ré.漢>'M", "tokens": 24, "pieces": ["><|", "endoftext", "|>!", "ḋ", "̣'", "Re", "́.", "漢", ">'", "M", ""]} +{"text": "Zéſ--Dž's\r\n\r\nꟲ́'VE字漢're!!s\r\n'-<", "EOT", ">'", "s", "\r\n\r\n", "ꟲ", "́'", "VE字漢", "'re", "!!", "s", "\r\n", "'-<", "EOT", "🙂", " ", "　", "#$%"]} +{"text": " 👍🏽
!!\t!!'sꟲ́<ꟲ🙂é'VEＡ\"Z𐞁'-İ<|endoftext|>", "
", "!!", "\t", "!!'", "sꟲ", "́<", "ꟲ", "🙂é", "'VE", "Ａ", "\"Z𐞁", "'-", "İ", "<|", "endoftext", "|><", "s", "9", "​"]} +{"text": ",tع'D \nḍ̇㍿'Re\r\n 𐞁'VEd👍🏽​'ll'ReＡ𐞁 \n🙂", "tokens": 42, "pieces": [",tع", "'D", " \n", "ḋ", "̣㍿'", "Re", "\r\n", " 𐞁", "'VE", "d", "👍🏽​'", "ll", "'Re", "Ａ𐞁", " \n", "🙂"]} +{"text": "́'VEDž\nⅣé𐞁‍'M", "tokens": 17, "pieces": ["́'", "VEDž", "\n", "Ⅳ", "é𐞁", "‍'", "M"]} +{"text": "é \nEOT
<'‍'Ttå!", "tokens": 16, "pieces": ["e", "́", " \n", "EOT", "
", "<'‍'", "Tta", "̊!"]} +{"text": "!!字\r\n\r\n'ſ'D　Z㋿'T( Ⅳ0eé㋿a're \n", "tokens": 28, "pieces": ["!!", "字", "\r\n\r\n", "'ſ", "'D", "　Z", "㋿'", "T", "(", " ", " ", "Ⅳ0", "eé", "㋿a", "'re", " \n"]} +{"text": "aſ'D👍🏽­㍿aḍ̇\u000b​! \n'S<|endoftext|>s're9fi>", "tokens": 38, "pieces": ["aſ", "'D", "👍🏽­㍿", "aḋ", "̣", "\u000b", "​!", " \n", "'S", "<|", "endoftext", "|>", "s", "'re", "9", "fi", ">"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Ⅳ३Ａ٣٤٥٦0éſ\u000b\r𐞁 Džé½ḍ̇sd,'Sa\r\n\r\n३é", "tokens": 42, "pieces": ["Ⅳ३", "Ａ", "٣٤٥", "٦0", "éſ", "\u000b\r", "𐞁", " Džé", "½", "ḋ", "̣sd", ",'", "Sa", "\r\n\r\n", "३", "e", "́"]} +{"text": "t d​ß'ſfi-!😀🏽a३'sḍ̇12345678Z'll­'Re#$%
 ,s'Så'Té'ſ(\r\n\r\ntſm'\r\n", "tokens": 58, "pieces": ["t", " ", " d", "​ß", "'ſ", "fi", "-!😀🏽", "a", "३", "'s", "ḋ", "̣", "123", "456", "78", "Z", "'ll", "­'", "Re", "#$%", "
 ", " ,", "s", "'S", "a", "̊'", "Te", "́'", "ſ", "(\r\n\r\n", "tſm", "'\r\n"]} +{"text": "​漢<'DtEOT­ḍ̇<𐞁<字́ 'M're0", "tokens": 28, "pieces": ["​", "漢", "<'", "DtEOT", "­ḋ", "̣<", "𐞁", "<字", "́", " '", "M", "'re", "0"]} +{"text": ">dé,!!-\rEOT- \nt Ⅳ \n\tḍ̇Z字", "tokens": 31, "pieces": [">", "de", "́,!!-\r", "EOT", "-", " \n", "t", " ", " ", "Ⅳ", " \n", "", "\tḋ", "̣Z字"]} +{"text": "\r\n
'Sſ٣٤٥٦!!…DžZ३'Tع#$%e", "tokens": 31, "pieces": ["\r\n", "
", "'S", "ſ", "٣٤٥", "٦", "!!", "…", "DžZ", "३", "'T", "ع", "#$%", "e"]} +{"text": "fi🙂'👍🏽'M \n'Ta🙂t \u000b　>Ａfi", "🙂'👍🏽'", "M", " \n", "'T", "a", "🙂t", " \u000b", "　", ">Ａ", "#$%å'ſ𐞁<|fim_prefix|>ß!!🙂….ꟲ\"", "tokens": 54, "pieces": [".", "123", "456", "78", "'𐞁", "३", "ع", "'M", "(,", "½", "́\n", "३", ">#$%", "a", "̊'", "ſ𐞁", "<|", "fim", "_prefix", "|>", "ß", "!!🙂", "…", ".ꟲ", "\""]} +{"text": "<|fim_prefix|>> \n𐞁e", "tokens": 13, "pieces": ["<|", "fim", "_prefix", "|>>", " \n", "𐞁e"]} +{"text": "'Dt-ḍ̇३'ſ'dZ.m\r\n‍\r\n\r\n,Dž\r''M'VE", "tokens": 28, "pieces": ["'D", "t", "-ḋ", "̣", "३", "'ſ", "'d", "Z", ".m", "\r\n", "‍\r\n\r\n", ",Dž", "\r", "''", "M", "'VE"]} +{"text": "'VE'TＡ<\r\n'S'VEfi漢-#$%\u000b½sééd.½　'S.'ll>🙂Ａ's('T​漢'T", "tokens": 42, "pieces": ["'VE", "'T", "Ａ", "<\r\n", "'S", "'VE", "fi漢", "-#$%", "\u000b", "½", "se", "́e", "́d", ".", "½", "　", "'S", ".'", "ll", ">🙂", "Ａ", "'s", "('", "T", "​漢", "'T"]} +{"text": "'S\"𐞁\r\n\r\n字‍,
\"🙂‍.aa!9'Dta'M​\t\nZ\t　…m­İ𐞁,Za㋿'re", "tokens": 53, "pieces": ["'S", "\"", "𐞁", "\r\n\r\n", "字", "‍,", "
", "\"🙂‍.", "aa", "!", "9", "'D", "ta", "'M", "​", "\t\n", "Z", "\t　", "…m", "­İ", "𐞁", ",Za", "㋿'", "re"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㋿å㍿12345678(-<|endoftext|>>!!\r\n'ſ'VEſ
'sZ<|endoftext|>😀🏽9ßİ‍…EOTⅣع's(ſ <|fim_prefix|>", "tokens": 77, "pieces": ["㋿a", "̊㍿", "123", "456", "78", "(-<|", "endoftext", "|>><", "EOT", ">!!\r\n", "'ſ", "'", "VEſ", "
", "'s", "Z", "<|", "endoftext", "|>😀🏽", "9", "ßİ", "‍", "…EOT", "Ⅳ", "ع", "'s", "(ſ", " <|", "fim", "_prefix", "|>"]} +{"text": " '‍\r\n9-'ree'rea\r\n\r\n३𐞁(́Dž…'Ree😀🏽\"🙂'D0,fi", "tokens": 43, "pieces": [" ", " '‍\r\n", "9", "-'", "ree", "'re", "a", "\r\n\r\n", "३", "𐞁", "(́", "Dž", "", "…", "'Re", "e", "😀🏽\"🙂'", "D", "0", ",fi"]} +{"text": "🙂0漢(Dž'VE<|fim_prefix|> \"ḍ̇­Z👍🏽٣٤٥٦漢(å字EOT\"12345678's\n'D", "tokens": 57, "pieces": ["🙂", "0", "漢", "(Dž", "'VE", "<|", "fim", "_prefix", "|>", " ", "\"ḋ", "̣­", "Z", "👍🏽", "٣٤٥", "٦", "漢", "(a", "̊字", "EOT", "\"", "123", "456", "78", "'s", "\n", "'D"]} +{"text": "<|fim_prefix|><|endoftext|>'Re…'T#$% d<|endoftext|><|fim_prefix|>12345678", "tokens": 36, "pieces": ["<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "Re", "…", "'T", "#$%", " d", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "123", "456", "78"]} +{"text": "　㋿𐞁İ\nmḍ̇'re<|fim_prefix|>'Sḍ̇mfi9<٣٤٥٦'Dß're \n ḍ̇㍿\r‍'👍🏽<|endoftext|>🙂ſ'S", "tokens": 78, "pieces": ["　", "㋿𐞁İ", "\n", "mḋ", "̣'", "re", "<|", "fim", "_prefix", "|>'", "Sḋ", "̣mfi", "9", "<", "٣٤٥", "٦", "'D", "ß", "'re", " \n", " ḋ", "̣㍿\r", "‍'👍🏽<|", "endoftext", "|>🙂", "ſ", "'S"]} +{"text": "'D're½½Ａs9>'M,😀🏽.'Re　́🙂'Ms's३­\r\n­9'", "re", "½½", "Ａs", "9", ">'", "M", ",😀🏽.'", "Re", "　", "́🙂'", "Ms", "'s", "३", "­\r\n", "­", "9", "é́t𐞁Ⅳ#$%'VE'sfi,ſḍ̇'M-,EOT \nＡ's", "tokens": 55, "pieces": ["İ", "‍​", "
", "㋿", "　漢", "‍\r", "३", "!!́<", "EOT", ">é", "́t𐞁", "Ⅳ", "#$%'", "VE", "'s", "fi", ",ſḋ", "̣'", "M", "-,", "EOT", " \n", "Ａ", "'s"]} +{"text": "'re…0!!٣٤٥٦ßdZ‍'sZ", "tokens": 21, "pieces": ["'re", "…", "0", "!!", "٣٤٥", "٦", "ßdZ", "‍'", "sZ"]} +{"text": "fiＡ \nEOT\t½'ll,s Ａ9", "tokens": 14, "pieces": ["fiＡ", " \n", "EOT", "\t", "½", "'ll", ",s", " Ａ", "9"]} +{"text": "𐞁!!漢٣٤٥٦　㋿'S\r\n\r\nſ", "tokens": 24, "pieces": ["𐞁", "!!", "漢", "٣٤٥", "٦", "　", "㋿'", "S", "\r\n\r\n", "ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "!\r\n\r\n.\u000b­ſ.åe", "tokens": 11, "pieces": ["!\r\n\r\n", ".", "\u000b", "­ſ", ".a", "̊e"]} +{"text": " \t9'Mſ9!!\"\"9'llꟲ\u000bİ㋿!!fiZع'ſⅣⅣſ", "tokens": 33, "pieces": [" ", "\t", "9", "'M", "ſ", "9", "!!\"\"", "9", "'ll", "ꟲ", "\u000bİ", "㋿!!", "fiZع", "'ſ", "ⅣⅣ", "ſ"]} +{"text": "EOT३ 12345678'T'Reḍ̇ \n'Dm'sd😀🏽\t㍿,t 'Tع\rå", "tokens": 47, "pieces": ["EOT", "३", "", " ", "123", "456", "78", "'", "T", "'Re", "ḋ", "̣", " \n", "'D", "m", "'s", "d", "😀🏽", "\t", "㍿,", "t", "", " ", "'T", "ع", "\r", "a", "̊"]} +{"text": "\u000bＡé
e\r'T <|endoftext|>t!!e'D٣٤٥٦𐞁👍🏽Ⅳ\rſ'lltDž('", "tokens": 52, "pieces": ["\u000bＡe", "́", "
e", "\r", "'T", " ", " <|", "endoftext", "|>", "t", "!!<", "EOT", ">e", "'D", "٣٤٥", "٦", "𐞁", "👍🏽", "Ⅳ", "\r", "ſ", "'ll", "tDž", "('"]} +{"text": "\nḍ̇㋿㍿😀🏽漢a", "tokens": 20, "pieces": ["\n", "ḋ", "̣㋿㍿😀🏽", "漢a"]} +{"text": "s ㍿'reＡt!!𐞁½d㋿EOT…d!!㍿㍿e
éİ.​‍ßZ
३0Ⅳ,　é'S'VE漢३0's", "tokens": 62, "pieces": ["s", " ", "㍿'", "reＡt", "!!", "𐞁", "½", "d", "㋿EOT", "…d", "!!㍿㍿", "e", "
éİ", ".​‍", "ßZ", "
", "३0Ⅳ", ",", "　é", "'S", "'VE", "漢", "३0", "'s"]} +{"text": " <㍿ع😀🏽Z𐞁ſ
mm​dß'Re漢\r\n\r\n \n\"…\"", "tokens": 32, "pieces": [" <㍿", "ع", "😀🏽", "Z𐞁ſ", "
mm", "​dß", "'Re", "漢", "\r\n\r\n \n", "\"", "…", "\""]} +{"text": " \ném'S½'s
字­'ſ\r\n\r\n‍ſ٣٤٥٦\r\n\r\n
'VE​éDž㍿\u000bååع\r\n'se\rعt''ll", "tokens": 59, "pieces": [" \n", "e", "́m", "'S", "½", "'s", "
字", "­<", "EOT", ">'", "ſ", "\r\n\r\n", "‍ſ", "٣٤٥", "٦", "\r\n\r\n", "
", "'VE", "​e", "́Dž", "㍿", "\u000ba", "̊a", "̊ع", "\r\n", "'s", "e", "\r", "عt", "''", "ll"]} +{"text": "字mſ🙂'", "tokens": 7, "pieces": ["字mſ", "🙂'"]} +{"text": "🙂 ", "tokens": 3, "pieces": ["🙂", " "]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'s<|endoftext|>㋿\"👍🏽\rEOT́!e", "tokens": 24, "pieces": ["'s", "<|", "endoftext", "|>㋿\"👍🏽\r", "EOT", "́!", "e"]} +{"text": ".㋿at字<'Sm'M9字e
\u000bع!👍🏽ß㍿..-d ḍ̇🙂.'S\r'ſ\r\n\r\nＡ'ſ12345678'VE
é'ſ\n", "tokens": 63, "pieces": [".㋿", "at字", "<'", "Sm", "'M", "9", "字e", "
", "\u000bع", "!👍🏽", "ß", "㍿..-", "d", " ", " ḋ", "̣🙂.'", "S", "\r", "'ſ", "\r\n\r\n", "Ａ", "'ſ", "123", "456", "78", "'VE", "
e", "́'", "ſ", "\n"]} +{"text": "ḍ̇e'ſ👍🏽<|fim_prefix|><|endoftext|>s!­ é😀🏽ſ½…'Mꟲ\u000b -㋿ 'S\r#$%fi字-<\u000b\r(-\reé㋿", "tokens": 78, "pieces": ["ḋ", "̣e", "'ſ", "👍🏽<|", "fim", "_prefix", "|><|", "endoftext", "|>", "s", "!­", " ", "e", "́😀🏽", "ſ", "½", "…", "'M", "ꟲ", "\u000b", " -㋿", " ", " '", "S", "\r", "#$%", "fi字", "-<", "\u000b\r", "(-\r", "eé", "㋿"]} +{"text": "fid <|endoftext|>!­ 'D३Z🙂<|endoftext|>#$%字'ſé", "tokens": 37, "pieces": ["fid", " ", " <|", "endoftext", "|><", "META", "_START", ">!­", " ", "'D", "३", "Z", "🙂<|", "endoftext", "|>#$%", "字", "'ſ", "e", "́"]} +{"text": "'ll !!​<|fim_prefix|>ع‍'ll‍‍''S.EOT#$%'
\u000b👍🏽Dž'\r\n\r\nİ́", "tokens": 46, "pieces": ["'ll", " ", " !!​<", "EOT", "><|", "fim", "_prefix", "|>", "ع", "‍'", "ll", "‍‍''", "S", ".EOT", "#$%'", "
", "\u000b", "👍🏽", "Dž", "'\r\n\r\n", "İ", "́"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "<|fim_prefix|>a­fi\"'S'ſ३<0'Sé>'VE㋿a.'Re'T\r\nſDža🙂0\"ſ", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>", "a", "­fi", "\"'", "S", "'ſ", "३", "<", "0", "'S", "e", "́>'", "VE", "㋿a", ".'", "Re", "'T", "\r\n", "ſDža", "🙂", "0", "\"ſ"]} +{"text": "㍿12345678'S!!#$%fi'ß\t㋿åe.­😀🏽'VE\r\n\r\nDž‍漢٣٤٥٦İ㍿ >9'Re\u000b9!\u000b0 \nß‍'Re'ſſ", "tokens": 70, "pieces": ["㍿", "123", "456", "78", "'S", "!!#$%", "fi", "'ß", "\t", "㋿a", "̊e", ".­😀🏽'", "VE", "\r\n\r\n", "Dž", "‍漢", "٣٤٥", "٦", "İ", "㍿", " ", " >", "9", "'Re", "\u000b", "9", "!", "\u000b", "0", " \n", "ß", "‍'", "Re", "'ſ", "ſ"]} +{"text": "tm'll\t12345678<,३Ａ🙂12345678Dž'll𐞁İå𐞁\u000b\u000b
", "tokens": 36, "pieces": ["tm", "'ll", "\t", "123", "456", "78", "<,", "३", "Ａ", "🙂", "123", "456", "78", "Dž", "'ll", "𐞁İa", "̊𐞁", "\u000b\u000b
"]} +{"text": "'Reßå>12345678å­m😀🏽", "tokens": 19, "pieces": ["'Re", "ßa", "̊>", "123", "456", "78", "a", "̊­", "m", "😀🏽"]} +{"text": "​ ſå㍿ꟲ<|fim_prefix|>-EOT<|fim_prefix|>'T,(㍿ 
ßعé'Mḍ̇.", "tokens": 50, "pieces": ["​", " ſa", "̊㍿", "ꟲ", "<|", "fim", "_prefix", "|>-", "EOT", "<|", "fim", "_prefix", "|>'", "T", ",(㍿", " ", "", "
ßعé", "'M", "ḋ", "̣."]} +{"text": "　㋿㋿\u000bꟲ", "tokens": 11, "pieces": ["　", "㋿㋿", "\u000bꟲ"]} +{"text": "　ꟲ३'ſ'ſ\n'll!!'M'S1234567812345678🙂\r\nZ\r", "tokens": 29, "pieces": ["　ꟲ", "३", "'ſ", "'ſ", "\n", "'ll", "!!'", "M", "'S", "123", "456", "781", "234", "567", "8", "🙂\r\n", "Z", "\r"]} +{"text": "fi​'D\"Džmſ 
>३ꟲ\r\n'ſa", "tokens": 25, "pieces": ["fi", "​'", "D", "\"Džmſ", " ", "
", ">", "३", "ꟲ", "\r\n", "'ſ", "a"]} +{"text": "é>…‍́漢́ꟲ09…12345678d-　<|endoftext|> \n\r'' EOTå- ,'sEOT#$% \n<|endoftext|>9!!…😀🏽", "tokens": 71, "pieces": ["é", ">", "…", "‍́", "漢", "́ꟲ", "09", "…", "123", "456", "78", "d", "-", "　", "<|", "endoftext", "|>", " \n\r", "''", " ", " EOTa", "̊-<", "META", "_START", ">", " ", ",'", "sEOT", "#$%", " \n", "<|", "endoftext", "|>", "9", "!!", "…", "😀🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "å'lls<|fim_prefix|>'M'-12345678 \n'D é", "tokens": 23, "pieces": ["a", "̊'", "lls", "<|", "fim", "_prefix", "|>'", "M", "'-", "123", "456", "78", " \n", "'D", " ", " e", "́"]} +{"text": "३漢t!!!!es\u000b\u000b\r\n​'ſ<|fim_prefix|>'TZ\"å!!!!", "es", "\u000b\u000b\r\n", "​'", "ſ", "<|", "fim", "_prefix", "|>'", "TZ", "\"a", "̊<", "eꟲ", "　", "9", "\r", "ſ", "(字", "‍​", "字", " '", "M", " \n", "👍🏽", "0"]} +{"text": "🙂　99 ㍿'D🙂-", "tokens": 16, "pieces": ["🙂", "　", "", "99", " ㍿'", "D", "🙂-"]} +{"text": "'T \n'T‍mZ'llmＡ!\r\n\r\n(ds\t", "tokens": 15, "pieces": ["'T", " \n", "'T", "‍mZ", "'ll", "mＡ", "!\r\n\r\n", "(ds", "\t"]} +{"text": "­
Dž\"ßİ#$%'M'sꟲ(\r\n\r\n<ꟲ'S'M😀🏽d३….​½ꟲ٣٤٥٦", "tokens": 50, "pieces": ["­", "
Dž", "\"", "ßİ", "#$%'", "M", "'s", "ꟲ", "(\r\n\r\n", "<ꟲ", "'S", "'M", "😀🏽", "d", "३", "…", ".​", "½", "ꟲ", "٣٤٥", "٦"]} +{"text": "'Da‍㍿,'Reß12345678'DeZ-'re'D's,-12345678'VE'll३(", "tokens": 30, "pieces": ["'D", "a", "‍㍿,'", "Reß", "123", "456", "78", "'D", "eZ", "-'", "re", "'D", "'s", ",-", "123", "456", "78", "'VE", "'ll", "३", "("]} +{"text": "​ \n'VEſ", "tokens": 6, "pieces": ["​", " \n", "'VE", "ſ"]} +{"text": "-ß'Re३å .", "tokens": 9, "pieces": ["-ß", "'Re", "३", "a", "̊", " ."]} +{"text": "åḍ̇!!<|fim_prefix|>\r's'S‍'sſ٣٤٥٦'D12345678.#$%-!fi　fi< 'é‍😀🏽३å> ", "tokens": 64, "pieces": ["a", "̊ḋ", "̣!!<|", "fim", "_prefix", "|>\r", "'s", "'S", "‍'", "sſ", "٣٤٥", "٦", "'D", "123", "456", "78", ".#$%-!", "fi", "　fi", "<", " ", " '", "é", "‍😀🏽", "३", "a", "̊>", " "]} +{"text": "ß😀🏽ſ'llt'T''T'M0́!a'S,㍿٣٤٥٦>
…s>́'re\n's", "tokens": 43, "pieces": ["ß", "😀🏽", "ſ", "'ll", "t", "'T", "''", "T", "'M", "0", "́!", "a", "'S", ",㍿", "٣٤٥", "٦", ">", "
", "…s", ">́'", "re", "\n", "'s"]} +{"text": "
\r\n
İ0'Red#$% 12345678åe \nع'VE'Re d'T漢> 'll'Tİ́㍿३𐞁ß12345678 ㋿", "tokens": 57, "pieces": ["
\r\n", "
İ", "0", "'Re", "d", "#$%<", "EOT", ">", " ", "123", "456", "78", "a", "̊e", " \n", "ع", "'VE", "'Re", " d", "'T", "漢", ">", " '", "ll", "'T", "İ", "́㍿", "३", "𐞁ß", "123", "456", "78", " ", " ㋿"]} +{"text": "\u000bعع<|endoftext|>㍿ ​<|endoftext|>EOT'ſtd㍿'s‍'M\r(#$%9\n<|fim_prefix|>! \n'ſ9٣٤٥٦'T\tⅣ", "tokens": 68, "pieces": ["\u000bعع", "<|", "endoftext", "|>㍿", " ", " ​<|", "endoftext", "|>", "EOT", "'ſ", "td", "㍿'", "s", "‍'", "M", "\r", "(#$%", "9", "\n", "<|", "fim", "_prefix", "|>!", " \n", "'ſ", "9٣٤", "٥٦", "'T", "\t", "Ⅳ"]} +{"text": "eEOT''ress'VEm>'S- 'M字\r\n\r\n'Sḍ̇İ0're'Re
́dEOT", "''", "ress", "'VE", "m", ">'", "S", "-", " '", "M字", "\r\n\r\n", "'S", "ḋ", "̣İ", "0", "'re", "'Re", "
", "́d", "EOT‍ß\r
\n字<\u000bꟲḍ̇Dž'D́ésⅣ!\u000b<…(<|endoftext|>0's'Re 're'Sſa", "tokens": 67, "pieces": ["'re", "0", "Ａ", ".㋿<", "EOT", ">EOT", "‍ß", "\r
\n", "字", "<", "\u000bꟲḋ", "̣Dž", "'D", "́e", "́s", "Ⅳ", "!", "\u000b", "<", "…", "(<|", "endoftext", "|>", "0", "'", "s", "'Re", " <", "META", "_START", ">'", "re", "'S", "ſa"]} +{"text": "\" é", "tokens": 4, "pieces": ["\"", " ", " e", "́"]} +{"text": "😀🏽's…½\t३#$%ḍ̇'T\r'Reé-0İ㍿<|fim_prefix|>Dž \n　", "tokens": 62, "pieces": ["ꟲع", " ßＡe", "(\r\n\r\n", "eß", "㍿", " ", " ḋ", "̣'", "sꟲ", "<|", "endoftext", "|>", "ḋ", "̣'", "T", "\r", "'Re", "e", "́-", "0", "İ", "㍿<|", "fim", "_prefix", "|>", "Dž", " \n　"]} +{"text": "عe 'D३'s'reéa#$%9字\r!'s", "tokens": 18, "pieces": ["عe", " ", "'D", "३", "'s", "'re", "e", "́a", "#$%", "9", "字", "\r", "!'", "s"]} +{"text": "'ſⅣⅣ12345678>Dž​٣٤٥٦३", "tokens": 25, "pieces": ["'ſ", "ⅣⅣ1", "234", "567", "8", ">Dž", "​", "٣٤٥", "٦३"]} +{"text": "\rſ'ſ.ſ", "tokens": 9, "pieces": ["\r", "ſ", "'ſ", ".ſ"]} +{"text": "​(ꟲ'M🙂'VE\re'VE\n12345678İ's!!>٣٤٥٦Z\r\n\r\n\tＡ३\r\nZ㍿\r,­٣٤٥٦-\r !!'VE<|endoftext|><|endoftext|>", "tokens": 72, "pieces": ["​(", "ꟲ", "'M", "🙂'", "VE", "\r", "e", "'VE", "\n", "123", "456", "78", "İ", "'s", "!!>", "٣٤٥", "٦", "Z", "\r\n\r\n", "\tＡ", "३", "\r\n", "Z", "㍿\r", ",­", "٣٤٥", "٦", "-\r", " ", "!!'", "VE", "<|", "endoftext", "|><|", "endoftext", "|>"]} +{"text": "ⅣⅣZé'T　\tm…'T<|fim_prefix|>ḍ̇- \n㋿Z'sé漢३​m \nd", "tokens": 42, "pieces": ["ⅣⅣ", "Ze", "́'", "T", "　", "\tm", "…", "'T", "<|", "fim", "_prefix", "|>", "ḋ", "̣-", " \n", "㋿Z", "'s", "e", "́漢", "३", "​m", " \n", "d"]} +{"text": "!!ꟲ'Ts, #$%,…🙂>​aİ\r\n\r\n
ée

é‍,a'VE​aß\r\n\t9t<|fim_prefix|>\rſ٣٤٥٦<|fim_prefix|>Džae", "tokens": 74, "pieces": ["ſ", "\n", "😀🏽'", "re", "<|", "fim", "_prefix", "|>🙂>​", "aİ", "\r\n\r\n", "
ée", "
", "
e", "́‍,", "a", "'VE", "​aß", "\r\n", "\t", "9", "t", "<|", "fim", "_prefix", "|>\r", "ſ", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "Džae"]} +{"text": "<|endoftext|>㍿­ås½ ع\r\n(ع𐞁👍🏽-'ll!'llm㋿Z", "tokens": 40, "pieces": ["<|", "endoftext", "|>㍿­", "a", "̊s", "½", " ", " ع", "\r\n", "(ع𐞁", "👍🏽-'", "ll", "!'", "llm", "㋿Z"]} +{"text": "<|fim_prefix|>İd㍿ß ­🙂!>,\r\n\r\n𐞁٣٤٥٦\" ,ꟲ…tå　a½é<|endoftext|>,٣٤٥٦é>", "tokens": 67, "pieces": ["<|", "fim", "_prefix", "|>", "İd", "㍿ß", " ­🙂!>,\r\n\r\n", "𐞁", "٣٤٥", "٦", "\"", " ,", "ꟲ", "…ta", "̊", "　a", "½", "e", "́<|", "endoftext", "|>,", "٣٤٥", "٦", "e", "́>"]} +{"text": "> ́!\t're'T㍿ſ㍿EOT\r\n\r\n­'ReEOTⅣ'M", "tokens": 29, "pieces": [">", " ", "́!<", "META", "_START", ">", "\t", "'re", "'T", "㍿ſ", "㍿EOT", "\r\n\r\n", "­'", "ReEOT", "Ⅳ", "'M"]} +{"text": "m-½𐞁Zꟲ漢㋿'D<|fim_prefix|>.<|fim_prefix|>'re'T0''lle\ńmsd12345678!!'Reſ(Ⅳ", "tokens": 52, "pieces": ["m", "-", "½", "𐞁Zꟲ漢", "㋿'", "D", "<|", "fim", "_prefix", "|>.<|", "fim", "_prefix", "|>'", "re", "'T", "0", "''", "lle", "\n", "́msd", "123", "456", "78", "!!'", "Reſ", "(", "Ⅳ"]} +{"text": "ꟲ👍🏽٣٤٥٦　Dž‍9'M㋿fi<|endoftext|>'ſs\té(㋿", "tokens": 53, "pieces": ["ꟲ", "👍🏽<", "META", "_START", ">", "٣٤٥", "٦", "　Dž", "‍<", "META", "_START", ">", "9", "'M", "㋿fi", "<|", "endoftext", "|>'", "ſs", "\té", "(㋿"]} +{"text": "𐞁'VE\r\n🙂‍字字(", "tokens": 14, "pieces": ["𐞁", "'VE", "\r\n", "🙂‍", "字字", "("]} +{"text": "́­'Re>Ａ́å.'VE½\n'Tt9åⅣ\"fi🙂Ａ", "tokens": 33, "pieces": ["́­'", "Re", "><", "EOT", ">Ａ", "́a", "̊.'", "VE", "½", "\n", "'T", "t", "9", "a", "̊", "Ⅳ", "\"fi", "🙂Ａ"]} +{"text": "<ſå½EOTeDžé ", "tokens": 14, "pieces": ["<ſa", "̊", "½", "EOTeDžé", " "]} +{"text": "éİ(\r㍿ \né३Z!!9\u000bⅣ\u000b<|fim_prefix|>'ſ　ḍ̇漢-\tm'Té", "tokens": 43, "pieces": ["éİ", "(\r", "㍿", " \n", "e", "́", "३", "Z", "!!", "9", "", "\u000b", "Ⅳ", "\u000b", "<|", "fim", "_prefix", "|>'", "ſ", "　ḋ", "̣漢", "-", "\tm", "'T", "é"]} +{"text": "'ll \"\u000b 🙂३<‍🙂12345678Dž <㋿e漢'ſⅣ0ع<|endoftext|>a㍿", "tokens": 44, "pieces": ["'ll", " ", " \"", "\u000b ", " 🙂", "३", "<‍🙂", "123", "456", "78", "Dž", " ", " <㋿", "e漢", "'ſ", "Ⅳ0", "ع", "<|", "endoftext", "|>", "a", "㍿"]} +{"text": "!!'ſa'Re\rＡ!Ａ9m", "tokens": 14, "pieces": ["!!'", "ſa", "'Re", "\r", "Ａ", "!Ａ", "9", "m"]} +{"text": "ع12345678", "tokens": 4, "pieces": ["ع", "123", "456", "78"]} +{"text": "​字ع're\n ㍿'VEé'llfi", "tokens": 15, "pieces": ["​字ع", "'re", "\n", " ㍿'", "VEé", "'ll", "fi"]} +{"text": " \n.عİ12345678d'fia\n​३ḍ̇m!>३'re#$%(​👍🏽𐞁
'S­'llſſ\n👍🏽 \nDž'Re 'sfi", "tokens": 71, "pieces": [" \n", ".عİ", "123", "456", "78", "d", "'fia", "\n", "​", "३", "ḋ", "̣m", "!><", "META", "_START", ">", "३", "'", "re", "#$%(​👍🏽", "𐞁", "
", "'S", "­'", "llſſ", "\n", "👍🏽", " \n", "Dž", "'Re", " ", "'s", "fi"]} +{"text": "d", "tokens": 1, "pieces": ["d"]} +{"text": "\t㋿", "tokens": 4, "pieces": ["\t", "㋿"]} +{"text": ">  ㋿'llZ\t<|endoftext|><|endoftext|>'VE­ḍ̇!!'s­-\r'", "tokens": 37, "pieces": [">", " ", " ", "㋿'", "llZ", "\t", "<|", "endoftext", "|><|", "endoftext", "|>'", "VE", "­ḋ", "̣!!'", "s", "­-\r", "'"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ꟲDž㍿𐞁d<|endoftext|><|fim_prefix|>< (عＡ字‍‍'T👍🏽's's#$%!!<Ⅳß ٣٤٥٦>'Dع　", "tokens": 67, "pieces": ["ꟲDž", "㍿𐞁d", "<|", "endoftext", "|><|", "fim", "_prefix", "|><", " ", " (", "عＡ字", "‍‍'", "T", "👍🏽'", "s", "'s", "#$%!!<", "Ⅳ", "ß", " ", "٣٤٥", "٦", ">'", "Dع", "　"]} +{"text": "m \n's9İé(!>३㋿­<|fim_prefix|> !漢𐞁ḍ̇s#$%åß12345678'D", "tokens": 52, "pieces": ["m", " \n", "'s", "9", "İé", "(!>", "३", "㋿­<|", "fim", "_prefix", "|>", " ", " <", "META", "_START", ">!", "漢𐞁", "ḋ", "̣s", "#$%", "a", "̊ß", "123", "456", "78", "'D"]} +{"text": "!é🙂\r\n३s𐞁'llEOT>
㋿>,́<𐞁عḍ̇\"عté0sam", "tokens": 40, "pieces": ["!é", "🙂\r\n", "३", "s𐞁", "'ll", "EOT", ">", "
", "㋿>,́<", "𐞁عḋ", "̣\"", "عte", "́", "0", "sam"]} +{"text": "fiZ'ſ!\nß\u000bsea0'ſ
Ⅳ🙂👍🏽 \n‍
('M　t('DＡ!éⅣ‍ m\té9d😀🏽'ſ", "tokens": 63, "pieces": ["fiZ", "'ſ", "!\n", "ß", "\u000bsea", "0", "'ſ", "
", "Ⅳ", "🙂👍🏽", " \n", "‍", "
", "('", "M", "　t", "('", "DＡ", "!e", "́", "Ⅳ", "‍", " m", "\té", "9", "d", "😀🏽'", "ſ"]} +{"text": "12345678\n'…><|fim_prefix|>Ａ(d'ſ!'ll<|fim_prefix|>'s", "tokens": 30, "pieces": ["123", "456", "78", "\n", "'", "…", "><|", "fim", "_prefix", "|>", "Ａ", "(d", "'ſ", "!'", "ll", "<|", "fim", "_prefix", "|>'", "s"]} +{"text": "\r\n Z>'ſåſ\"ſ½­-(<　ع\r\n\r\n
", "tokens": 23, "pieces": ["\r\n", " Z", ">'", "ſa", "̊ſ", "\"ſ", "½", "­-(<", "　ع", "\r\n\r\n
"]} +{"text": "!😀🏽🙂EOT'VEſ", "tokens": 14, "pieces": ["!😀🏽🙂", "EOT", "'VE", "ſ"]} +{"text": "é#$%m\r\n\r\nß𐞁>\t(‍Ａ\r\n\r\n'ſḍ̇३'S🙂İ(#$%", "tokens": 39, "pieces": ["e", "́#$%", "m", "\r\n\r\n", "ß", "𐞁", ">", "\t", "(‍", "Ａ", "\r\n\r\n", "'ſ", "ḋ", "̣", "३", "'S", "🙂İ", "(#$%"]} +{"text": ".'Sḍ̇'S…", "tokens": 15, "pieces": [".'", "Sḋ", "̣'", "S", "", "…"]} +{"text": "ع …Ａ🙂'T. \n!'Re🙂é12345678Z\r\n", "tokens": 27, "pieces": ["ع", "", " ", "…Ａ", "🙂'", "T", ".", " \n", "!'", "Re", "🙂e", "́", "123", "456", "78", "Z", "\r\n"]} +{"text": "\t३‍Ａ🙂Ⅳ㋿İḍ̇
<\r\n\r\n½<|fim_prefix|>ꟲ'DعＡDž'D'VE<|endoftext|>'ſ", "tokens": 53, "pieces": ["\t", "३", "‍Ａ", "🙂", "Ⅳ", "㋿İḋ", "̣", "
", "<\r\n\r\n", "½", "<|", "fim", "_prefix", "|>", "ꟲ", "'D", "عＡDž", "'D", "'VE", "<|", "endoftext", "|>'", "ſ"]} +{"text": "maſ\r\n\r\n'TDž'D Dž\rⅣ'T12345678å\t
Dž漢(!ع(!! ½EOT\r\nḍ̇!!　<|fim_prefix|>", "tokens": 57, "pieces": ["maſ", "\r\n\r\n", "'T", "Dž", "'D", " ", " Dž", "\r", "Ⅳ", "'T", "123", "456", "78", "a", "̊", "\t", "
Dž漢", "(!", "ع", "(!!", " ", " ", "½", "EOT", "\r\n", "ḋ", "̣!!<", "EOT", ">", "　", "<|", "fim", "_prefix", "|>"]} +{"text": "​<|fim_prefix|>\t'lls漢Ⅳ >'T٣٤٥٦! Z#$%\"'re>
ع'M'ſEOT's👍🏽'\r\n\r\n‍​s\n­<|endoftext|>​!!ḍ̇0,", "tokens": 73, "pieces": ["​<|", "fim", "_prefix", "|>", "\t", "'ll", "s漢", "Ⅳ", " ", ">'", "T", "٣٤٥", "٦", "!", " Z", "#$%\"'", "re", ">", "
ع", "'M", "'ſ", "EOT", "'s", "👍🏽'\r\n\r\n", "‍​", "s", "\n", "­<|", "endoftext", "|>​!!", "ḋ", "̣", "0", ","]} +{"text": "½!t३\n .d㍿'Res​12345678 \n\r\n.漢…ع…ß'Re漢'VE<|endoftext|>٣٤٥٦ß
.\u000b'Séd
漢", "tokens": 70, "pieces": ["½", "!t", "३", "\n", " ", " .", "d", "㍿'", "Res", "​", "123", "456", "78", " \n\r\n", ".", "漢", "…ع", "…ß", "'Re", "漢", "'VE", "<|", "endoftext", "|>", "٣٤٥", "٦", "ß", "", "
", ".", "\u000b", "'S", "éd", "
漢"]} +{"text": "…s👍🏽e३'VE''reZ　 \n", "tokens": 18, "pieces": ["٣٤٥", "٦", "<|", "endoftext", "|>", "Z", "　 \n"]} +{"text": "m>'S12345678\r\nḍ̇m‍ḍ̇漢d9<|fim_prefix|>ds, ( EOT!'D9é, a
#$%𐞁d́🙂", "tokens": 60, "pieces": ["m", ">'", "S", "123", "456", "78", "\r\n", "ḋ", "̣m", "‍ḋ", "̣漢d", "9", "<|", "fim", "_prefix", "|>", "ds", ",", " (", " ", " EOT", "!<", "META", "_START", ">'", "D", "9", "e", "́,", " a", "
", "#$%", "𐞁d", "́🙂"]} +{"text": "(", "tokens": 1, "pieces": ["("]} +{"text": ">​-<|fim_prefix|>㍿'Tt(<　'D🙂 Z\r\n\r\n
(s'VE.#$%Dž​'T㋿\t​
Dž'-", "tokens": 51, "pieces": [">​-<|", "fim", "_prefix", "|>㍿'", "Tt", "(<", "　", "'D", "🙂", " Z", "\r\n\r\n", "
", "(s", "'VE", ".#$%", "Dž", "​'", "T", "㋿", "\t", "​", "
Dž", "'-"]} +{"text": "'Ḿ9", "tokens": 3, "pieces": ["'M", "́", "9"]} +{"text": "'s'S>", "tokens": 3, "pieces": ["'s", "'S", ">"]} +{"text": "‍㋿Ⅳ", "tokens": 7, "pieces": ["‍㋿", "Ⅳ"]} +{"text": "٣٤٥٦\r\n\r\n­å ­0​½½9字e \u000b", "tokens": 26, "pieces": ["٣٤٥", "٦", "\r\n\r\n", "­a", "̊", " ­", "0", "​", "½½9", "字e", " \u000b"]} +{"text": "ßⅣ> m'VEfi　ḍ̇", "tokens": 16, "pieces": ["ß", "Ⅳ", ">", " ", " m", "'VE", "fi", "　ḋ", "̣"]} +{"text": "'M<|fim_prefix|>ع\t<ꟲ'S><|endoftext|>‍'ſ\u000b́
EOTé­
 \n9", "tokens": 42, "pieces": ["'M", "<|", "fim", "_prefix", "|>", "ع", "\t", "<", "ꟲ", "'S", "><|", "endoftext", "|>‍'", "ſ", "\u000b", "́", "
EOTé", "­", "
 \n", "9"]} +{"text": "\rß\rZa\tm　👍🏽0　🙂漢EOT'll're''Ａ\u000b< \naa-'llå\r\nḍ̇t­<12345678'Mſ", "tokens": 51, "pieces": ["\r", "ß", "\r", "Za", "\tm", "　", "👍🏽", "0", "　", "🙂漢", "EOT", "'ll", "'re", "''", "Ａ", "\u000b", "<", " \n", "aa", "-'", "lla", "̊\r\n", "ḋ", "̣t", "­<", "123", "456", "78", "'M", "ſ"]} +{"text": "　ßd٣٤٥٦EOT\r\n\r\n9-🙂'ſEOT\r\u000b<|fim_prefix|>'🙂 !!字mZꟲ<|endoftext|>!!٣٤٥٦'ſ!'ſ!!ſ", "tokens": 66, "pieces": ["　ßd", "٣٤٥", "٦", "EOT", "\r\n\r\n", "9", "-🙂'", "ſEOT", "\r", "\u000b", "<|", "fim", "_prefix", "|>'🙂", " !!", "字mZꟲ", "<|", "endoftext", "|>!!", "٣٤٥", "٦", "'ſ", "!'", "ſ", "!!", "ſ"]} +{"text": "0're<…,12345678­Džå'Tḍ̇'Re👍🏽\nDž漢𐞁", "tokens": 43, "pieces": ["0", "'re", "<", "…", ",", "123", "456", "78", "­", "Dža", "̊'", "Tḋ", "̣'", "Re", "👍🏽\n", "Dž漢𐞁"]} +{"text": "-'VE…<|fim_prefix|>'́́'0e‍s,''ll'ſ\t…s0👍🏽é", "tokens": 40, "pieces": ["-'", "VE", "…", "<|", "fim", "_prefix", "|>'́́'", "0", "e", "‍s", ",''", "ll", "'ſ", "\t", "…s", "0", "👍🏽", "e", "́"]} +{"text": "ḍ̇ 　३'Tḍ̇漢 ſ's…'D… 　<\rعİétḍ̇å🙂0é\u000bع<|endoftext|>'T\u000b३", "tokens": 65, "pieces": ["ḋ", "̣", " <", "EOT", ">", "　", "३", "'T", "ḋ", "̣漢", " ſ", "'s", "…", "'D", "… ", "　", "<\r", "عİétḋ", "̣a", "̊🙂", "0", "e", "́", "\u000bع", "<|", "endoftext", "|>'", "T", "\u000b", "३", ""]} +{"text": "٣٤٥٦(é👍🏽e9 \n(😀🏽0\r'Sſ're漢é\r\né👍🏽,0 ſds's㍿9ḍ̇'re(🙂ſa　", "tokens": 66, "pieces": ["٣٤٥", "٦", "(é", "👍🏽", "e", "9", " \n", "(😀🏽", "0", "\r", "'S", "ſ", "'re", "漢é", "\r\n", "é", "👍🏽,", "0", " ſds", "'s", "㍿", "9", "ḋ", "̣'", "re", "(🙂", "ſa", "　"]} +{"text": "ßİ0​'ſ#$%​\"㋿́'Sꟲ<ع-٣٤٥٦", "tokens": 31, "pieces": ["ßİ", "0", "​'", "ſ", "#$%​\"㋿́'", "Sꟲ", "<ع", "-", "٣٤٥", "٦"]} +{"text": "😀🏽<|fim_prefix|>s!'M!!😀🏽\u000bea", "tokens": 23, "pieces": ["😀🏽<|", "fim", "_prefix", "|>", "s", "!'", "M", "!!😀🏽", "\u000bea"]} +{"text": ".½\r\n𐞁ſ㋿½👍🏽Dž''ll㋿'S's'ſ½0 mfi　Z,s'Redİ", "tokens": 44, "pieces": [".", "½", "\r\n", "𐞁ſ", "㋿", "½", "👍🏽", "Dž", "''", "ll", "㋿'", "S", "'s", "'ſ", "½0", " mfi", "　Z", ",s", "'Re", "dİ"]} +{"text": "㍿", "tokens": 3, "pieces": ["㍿"]} +{"text": "é12345678fi 𐞁'ſ<|fim_prefix|>३!!İ\t0Z!d.<|fim_prefix|>🙂#$%>…!", "tokens": 49, "pieces": ["é", "123", "456", "78", "fi", " ", " 𐞁", "'ſ", "<|", "fim", "_prefix", "|><", "META", "_START", ">", "३", "!!", "İ", "\t", "0", "Z", "!d", ".<|", "fim", "_prefix", "|>🙂#$%>", "…", "!"]} +{"text": "#$%㍿'Mm
,!!0'S'ſ'VE'D", "tokens": 23, "pieces": ["#$%㍿'", "M", "m", "
", ",!!", "0", "'S", "'ſ", "'VE", "'D"]} +{"text": "<|endoftext|>'S字m'VEt́9#$%''ret\r'll­", "tokens": 22, "pieces": ["<|", "endoftext", "|>'", "S字m", "'VE", "t", "́", "9", "#$%''", "ret", "\r", "'ll", "­"]} +{"text": "mⅣé", "tokens": 4, "pieces": ["m", "Ⅳ", "é"]} +{"text": "DžEOT ('Re'll½🙂㋿ⅣＡ́(👍🏽>́", "tokens": 34, "pieces": ["DžEOT", " ", " (<", "EOT", ">'", "Re", "'ll", "½", "🙂㋿", "Ⅳ", "Ａ", "́(👍🏽><", "META", "_START", ">́"]} +{"text": "३ Ａ(­'D　s　Dž🙂Dž \n㍿ 'ſ　\t🙂Ａ ३#$%Dž0'T'0\u000b", "tokens": 45, "pieces": [" ", "😀🏽", "Ａ", "<|", "endoftext", "|>", " \n", "㍿", " ", "'ſ", "　", "\t", "🙂Ａ", " ", "३", "#$%", "Dž", "", "0", "'T", "'", "0", "\u000b"]} +{"text": "Ａ<|endoftext|>fiß9ꟲ<|endoftext|>e ½ >㋿İ😀🏽' ㍿😀🏽́\u000b३\n\u000b9d'M'Tße", "tokens": 60, "pieces": ["Ａ", "<|", "endoftext", "|>", "fiß", "9", "ꟲ", "<|", "endoftext", "|>", "e", " ", "½", " ", ">㋿", "İ", "😀🏽'", " ", "㍿😀🏽́", "\u000b", "३", "\n", "\u000b", "9", "d", "'M", "'T", "ße"]} +{"text": "…½é\r\ne\t9 #$%'ll(s漢'ſ<|endoftext|>Ａ<|fim_prefix|>'S<|endoftext|>'D>EOT㋿٣٤٥٦🙂<½'M½té>'T", "tokens": 67, "pieces": ["…", "½", "e", "́\r\n", "e", "\t", "9", " ", " #$%'", "ll", "(s漢", "'ſ", "<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>'", "S", "<|", "endoftext", "|>'", "D", ">EOT", "㋿", "٣٤٥", "٦", "🙂<", "½", "'M", "½", "te", "́>'", "T"]} +{"text": "　Zma<‍és'VE'sm'S \n👍🏽ꟲ9😀🏽", "tokens": 31, "pieces": ["　Zma", "<‍", "és", "'VE", "'s", "m", "'S", " \n", "👍🏽", "ꟲ", "9", "😀🏽"]} +{"text": "ß'VE<|fim_prefix|>d\t!!ḍ̇å\ńſ\"́é​　'D'VE㋿!'sé😀🏽EOT 'lld😀🏽́'s", "tokens": 61, "pieces": ["ß", "'VE", "<|", "fim", "_prefix", "|>", "d", "\t", "!!", "ḋ", "̣a", "̊\n", "́ſ", "\"́", "e", "́​", "　", "'D", "'VE", "㋿!'", "se", "́😀🏽", "EOT", " ", "'ll", "d", "😀🏽́'", "s", ""]} +{"text": "é<|endoftext|>EOT́<|fim_prefix|> \n( fiß-㋿'ſ<|fim_prefix|>,'s \n#$%-
\r", "tokens": 46, "pieces": ["e", "́<|", "endoftext", "|>", "EOT", "́<|", "fim", "_prefix", "|>", " \n", "(", " fiß", "-㋿'", "ſ", "<|", "fim", "_prefix", "|>,'", "s", " \n", "#$%-", "
\r"]} +{"text": "
é٣٤٥٦'D \n'll\u000bå'MEOT 'll'ſⅣ'\rfifid😀🏽­…字😀🏽३'ll'Dd,ḍ̇", "tokens": 62, "pieces": ["
e", "́", "٣٤٥", "٦", "'D", " \n", "'ll", "\u000ba", "̊'", "MEOT", " ", " '", "ll", "'ſ", "Ⅳ", "'\r", "fifid", "😀🏽­", "…字", "😀🏽", "३", "'ll", "'D", "d", ",ḋ", "̣"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿½<ſ'll \n(½'ll字'Re!!!!<…ꟲ½a\tＡ'VE'lltⅣDžm'Re ", "tokens": 37, "pieces": ["㍿", "½", "<ſ", "'ll", " \n", "(", "½", "'ll", "字", "'Re", "!!!!<", "…ꟲ", "½", "a", "\tＡ", "'VE", "'ll", "t", "Ⅳ", "Džm", "'Re", " "]} +{"text": "'s…\u000b\r-'s'll's'ſ​\r\n\r\n'Sm\u000b½ !\ré9😀🏽s　ſ", "tokens": 34, "pieces": ["'s", "…\u000b\r", "-'", "s", "'ll", "'s", "'ſ", "​\r\n\r\n", "'S", "m", "\u000b", "½", " !\r", "é", "9", "😀🏽<", "EOT", ">s", "　ſ"]} +{"text": "😀🏽ſ'ſİ
字ém'Re \n​<́३ 'T", "tokens": 29, "pieces": ["😀🏽", "ſ", "'ſ", "İ", "
字", "e", "́m", "'Re", " \n", "​<́", "३", " ", "'T"]} +{"text": "'VE<#$%'re<|fim_prefix|>!fié٣٤٥٦'s>fis\"<|fim_prefix|>　३'ll\r\n12345678𐞁>,", "tokens": 51, "pieces": ["'VE", "<#$%'", "re", "<|", "fim", "_prefix", "|>!", "fie", "́", "٣٤٥", "٦", "'s", ">fis", "\"<|", "fim", "_prefix", "|>", "　", "३", "'ll", "\r\n", "123", "456", "78", "𐞁", ">,"]} +{"text": " 
a éꟲ👍🏽
ꟲ'ſ 'VEtZ🙂½漢", "tokens": 38, "pieces": [" ", "
a", " e", "́ꟲ", "👍🏽", "
ꟲ", "'ſ", " '", "VEt", "<", "EOT", ">Z", "🙂", "½", "漢"]} +{"text": "Ⅳİ''ll\t…å😀🏽 \n\u000b!عDž", "tokens": 22, "pieces": ["Ⅳ", "İ", "''", "ll", "\t", "…a", "̊😀🏽", " \n", "\u000b", "!عDž"]} +{"text": "\r\nfi 👍🏽!!,😀🏽İ字EOT\r\n\r\nⅣⅣ.ḍ̇…🙂Ⅳ
(('Ss t́Ⅳ \nZ\t 's\r\n\r\n!!", "tokens": 54, "pieces": ["\r\n", "fi", " ", "👍🏽!!,😀🏽", "İ字EOT", "\r\n\r\n", "ⅣⅣ", ".ḋ", "̣", "…", "🙂", "Ⅳ", "
", "(('", "Ss", " ", " t", "́", "Ⅳ", " \n", "Z", "\t ", " '", "s", "\r\n\r\n", "!!"]} +{"text": "é", "tokens": 1, "pieces": ["é"]} +{"text": "­-👍🏽Ａ😀🏽s\t.12345678½ \r\nḍ̇''Re​‍.
sḍ̇a", "tokens": 43, "pieces": ["­-👍🏽", "Ａ", "😀🏽", "s", "\t", ".", "123", "456", "78½", " \r\n", "ḋ", "̣''", "Re", "​‍.", "
sḋ", "̣a"]} +{"text": "Ａ‍👍🏽\t​ås字 ع'Ds", "tokens": 20, "pieces": ["Ａ", "‍👍🏽", "\t", "​a", "̊s字", " ع", "'D", "s"]} +{"text": "9", "tokens": 1, "pieces": ["9"]} +{"text": "Dž", "tokens": 2, "pieces": ["Dž"]} +{"text": "́,İt'VE!!å'ſⅣ🙂'ſ's9\r\nſ𐞁ḍ̇　<|fim_prefix|>Ź٣٤٥٦'Dİfiꟲß,s0a 字", "tokens": 68, "pieces": ["́,", "İt", "'VE", "!!", "a", "̊'", "ſ", "Ⅳ", "🙂'", "ſ", "'s", "9", "\r\n", "ſ", "𐞁ḋ", "̣", "　", "<|", "fim", "_prefix", "|>", "Z", "́", "٣٤٥", "٦", "'D", "İfiꟲß", ",s", "0", "a", " ", " 字"]} +{"text": "'S<|fim_prefix|><<|fim_prefix|>…-\r\n\"é\rZ'llع(fi", "tokens": 31, "pieces": ["'S", "<|", "fim", "_prefix", "|><<|", "fim", "_prefix", "|><", "META", "_START", ">", "…", "-\r\n", "\"e", "́\r", "Z", "'ll", "ع", "(fi"]} +{"text": "s'½ \nééſ'll👍🏽'ſ😀🏽fi\r\n9é‍'VE\r…　𐞁\r'Dİſ\"é 字İé'M'DＡ🙂,ſ", "tokens": 66, "pieces": ["s", "'", "½", " \n", "ééſ", "'ll", "👍🏽'", "ſ", "😀🏽", "fi", "\r\n", "9", "é", "‍'", "VE", "\r", "…", "", "　𐞁", "\r", "'D", "İſ", "\"é", " ", " 字İe", "́'", "M", "'D", "Ａ", "🙂,", "ſ"]} +{"text": "'re fi e'D('D", "tokens": 9, "pieces": ["'re", " ", " fi", " e", "'D", "('", "D"]} +{"text": "9're字­'D字'🙂12345678å>🙂's३ſ \n", "tokens": 26, "pieces": ["9", "'re", "字", "­'", "D字", "'🙂", "123", "456", "78", "a", "̊>🙂'", "s", "३", "ſ", " \n"]} +{"text": "12345678😀🏽!字\r\n'Tḍ̇٣٤٥٦­½'T'VEⅣ\"", "tokens": 37, "pieces": ["123", "456", "78", "😀🏽!", "字", "\r\n", "'T", "ḋ", "̣", "٣٤٥", "٦", "­", "½", "'T", "'VE", "", "Ⅳ", "\""]} +{"text": ".👍🏽\r\n\r\nd­‍<|endoftext|>😀🏽　Ⅳ<|endoftext|>12345678m\r\n\r\n'S漢ꟲ", "tokens": 45, "pieces": [".👍🏽\r\n\r\n", "d", "­‍<|", "endoftext", "|>😀🏽", "　", "Ⅳ", "<|", "endoftext", "|>", "123", "456", "78", "m", "\r\n\r\n", "'S", "漢ꟲ"]} +{"text": ".>#$%Z12345678…㍿", "tokens": 13, "pieces": [".>#$%", "Z", "123", "456", "78", "…", "㍿"]} +{"text": "#$% \n'Re \n<́(ꟲ㍿​fiéfi\t​Ⅳ'S ́ßemd\t0\n", "tokens": 35, "pieces": ["#$%", " \n", "'Re", " \n", "<́(<", "EOT", ">ꟲ", "㍿​", "fiéfi", "\t", "​", "Ⅳ", "'S", " ", "́ßemd", "\t", "0", "\n"]} +{"text": "(…'lleá#$%EOT!! ​9ß­'res\"ⅣⅣ-'M३e#$%'s12345678'll<|fim_prefix|>…Džfi‍åZ<|endoftext|>", "tokens": 64, "pieces": ["(", "…", "'ll", "ea", "́#$%", "EOT", "!!", " ", "​", "9", "ß", "­'", "res", "\"", "ⅣⅣ", "-'", "M", "३", "e", "#$%'", "s", "123", "456", "78", "'ll", "<|", "fim", "_prefix", "|>", "…Džfi", "‍a", "̊Z", "<|", "endoftext", "|>"]} +{"text": "
‍३'re0\r\nDž­🙂a👍🏽­㍿-ḍ̇Dž", "tokens": 37, "pieces": ["
", "‍", "३", "'re", "0", "\r\n", "Dž", "­🙂", "a", "👍🏽­<", "EOT", ">㍿-", "ḋ", "̣Dž"]} +{"text": "\té'ſ \nḍ̇aꟲ''VE'VEé 漢…' Ａ'Re's.", "tokens": 37, "pieces": ["\té", "'ſ", " \n", "ḋ", "̣aꟲ", "''", "VE", "'VE", "e", "́", " ", " 漢", "…", "'", " Ａ", "'Re", "'s", ".<", "META", "_START", ">"]} +{"text": "12345678'stⅣع.… ,EOT字İ漢㋿ !!'M🙂㋿
", "tokens": 35, "pieces": ["123", "456", "78", "'s", "t", "Ⅳ", "ع", ".", "…", " ", ",EOT字İ漢", "㋿", " ", "!!<", "META", "_START", ">'", "M", "🙂㋿", "
"]} +{"text": " \n.Ａİ,½👍🏽're\t'VE𐞁عéDžſ(#$%'reſ ㋿😀🏽ع!å𐞁‍s👍🏽å­Z'", "tokens": 72, "pieces": [" \n", ".Ａ", "İ", ",", "½", "👍🏽'", "re", "\t", "'VE", "𐞁عéDžſ", "(#$%'", "reſ", " ", " ㋿😀🏽", "ع", "!a", "̊𐞁", "‍s", "👍🏽", "a", "̊­", "Z", "'"]} +{"text": "ḍ̇ꟲms12345678e'S9٣٤٥٦字ḍ̇\"ع㍿İ'M'llſ\r\n 🙂a<|endoftext|>fi漢!३<|fim_prefix|>　\u000b٣٤٥٦>", "tokens": 79, "pieces": ["ḋ", "̣ꟲms", "123", "456", "78", "e", "'S", "9", "", "٣٤٥", "٦", "字ḋ", "̣\"", "ع", "㍿İ", "'M", "'ll", "ſ", "\r\n", " ", " 🙂", "a", "<|", "endoftext", "|>", "fi漢", "!", "३", "<|", "fim", "_prefix", "|>", "　", "\u000b", "٣٤٥", "٦", ">"]} +{"text": "ßåع0ع12345678½㋿　Džꟲtſ-sꟲ#$%'ſ'ſt''D sDž\"a‍Ⅳ字12345678 \n", "tokens": 55, "pieces": ["ßa", "̊<", "EOT", ">ع", "0", "ع", "123", "456", "78½", "㋿", "　Džꟲtſ", "-sꟲ", "#$%'", "ſ", "'ſ", "t", "''", "D", " ", " sDž", "\"a", "‍", "Ⅳ", "字", "123", "456", "78", " \n"]} +{"text": "m<|endoftext|>Ⅳ\r\n\r\n!\r\n\r\n'漢  'S'Ses'D're𐞁'VE.aDžm(", "tokens": 37, "pieces": ["m", "<|", "endoftext", "|>", "Ⅳ", "\r\n\r\n", "!\r\n\r\n", "'漢", " ", " ", "'S", "'S", "es", "'D", "'re", "𐞁", "'VE", ".a", "Džm", "("]} +{"text": "\téa\t0t漢ꟲEOT\u000bEOT́e­-\u000b​", "tokens": 25, "pieces": ["\téa", "\t", "0", "t漢ꟲEOT", "\u000bEOT", "́e", "­<", "EOT", ">-", "\u000b", "​"]} +{"text": "<|endoftext|>½EOTfiḍ̇​½́
\"ḍ̇#$%Dž​Ａꟲ\"<", "tokens": 39, "pieces": ["<|", "endoftext", "|>", "½", "EOTfiḋ", "̣​", "½", "́", "
", "\"ḋ", "̣#$%", "Dž", "​Ａꟲ", "\"<"]} +{"text": "٣٤٥٦0(😀🏽'lld0\r.<İ'Re'Re#$%<|endoftext|>#$%<|endoftext|>ß­'Re>sa", "tokens": 50, "pieces": ["٣٤٥", "٦0", "(😀🏽'", "lld", "0", "\r", ".<", "EOT", "><", "İ", "'Re", "'Re", "#$%<|", "endoftext", "|>#$%<|", "endoftext", "|>", "ß", "­'", "Re", ">sa"]} +{"text": "😀🏽12345678㋿ḍ̇.𐞁\r\n<|endoftext|>,ß -😀🏽é.'S9\r\n\r\n　!!漢", "tokens": 45, "pieces": ["😀🏽", "123", "456", "78", "㋿ḋ", "̣.", "𐞁", "\r\n", "<|", "endoftext", "|>,", "ß", " -😀🏽", "é", ".'", "S", "9", "\r\n\r\n", "　", "!!", "漢"]} +{"text": "👍🏽\u000b ſ>‍!!字٣٤٥٦\"ḍ̇'३ß9é३\"e\nß0
!Ａd  Ａ9\n", "tokens": 55, "pieces": ["👍🏽", "\u000b ", " ſ", ">‍!!", "字", "٣٤٥", "٦", "\"ḋ", "̣'", "३", "ß", "9", "e", "́", "३", "\"e", "\n", "ß", "0", "
", "!Ａd", " ", " Ａ", "9", "\n"]} +{"text": "t12345678 'ſfiḍ̇ſfiⅣ㍿>", "tokens": 24, "pieces": ["t", "123", "456", "78", " '", "ſfiḋ", "̣ſfi", "Ⅳ", "㍿>"]} +{"text": "('Mꟲ字'Dß-<|endoftext|>٣٤٥٦\u000bd\r\nſ😀🏽\n…aémＡ…㍿½👍🏽३s
 ", "tokens": 80, "pieces": [" ", " <'", "sZ", "　", "‍", "½", "e", "'ſ", "e", "
e", "9", " \n", "<|", "fim", "_prefix", "|>'", "Dß", "-<|", "endoftext", "|>", "٣٤٥", "٦", "\u000bd", "\r\n", "ſ", "😀🏽\n", "…ae", "́mＡ", "…", "㍿", "½", "👍🏽", "३", "s", "
 "]} +{"text": " ſ>!'Re㋿'M's'T,İ!ع9-(­å㍿ع-'ll𐞁'Re​d😀🏽ꟲ !!😀🏽", "tokens": 54, "pieces": [" ſ", ">!'", "Re", "㋿'", "M", "'s", "'T", ",İ", "!ع", "9", "-(­", "a", "̊㍿", "ع", "-'", "ll", "𐞁", "'Re", "​d", "😀🏽", "ꟲ", " ", "!!😀🏽"]} +{"text": "३- \n٣٤٥٦é ſ!!'ll'ſ \n>'å­'ll½'M𐞁́Dž​<|endoftext|>!fißſa12345678<|endoftext|>́9'", "a", "̊­'", "ll", "½", "'M", "𐞁", "́Dž", "​<|", "endoftext", "|>!", "fißſa", "123", "456", "78", "<|", "endoftext", "|>́", "9", "字<|endoftext|>(0​\rＡ　0İ㍿'D㋿ḍ̇㋿\"Z", "tokens": 51, "pieces": [" \n", "9", "👍🏽", "é", "'re", "字", "<|", "endoftext", "|>(", "0", "​\r", "Ａ", "　", "0", "İ", "㍿'", "D", "㋿ḋ", "̣<", "EOT", ">㋿\"", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'😀🏽ع!𐞁　s\"😀🏽字٣٤٥٦٣٤٥٦İ,٣٤٥٦ꟲ#$%,\r\n\r\n字a", "tokens": 55, "pieces": ["'😀🏽", "ع", "!𐞁", "　s", "\"😀🏽", "字", "٣٤٥", "٦٣٤", "٥٦", "İ", ",", "٣٤٥", "٦", "ꟲ", "#$%,\r\n\r\n", "字a"]} +{"text": "\rDž'ſ're12345678‍'Re'T\"\r'M
ع\u000b\tt", "tokens": 26, "pieces": ["Z漢", "-'", "VEDž", " ", " 🙂,", "9", "字", "9", "…", "\"\r", "'M", "
ع", "\u000b", "\tt"]} +{"text": "<ḍ̇漢ꟲ'T<('Reꟲع\r\n\r\nDžⅣ'll㍿!!Dž \n'T'Dعfi<|fim_prefix|>漢é­m'ſ㋿ \n㋿0\u000b'refi", "tokens": 71, "pieces": ["<ḋ", "̣漢", "ꟲ", "'T", "<('", "Reꟲع", "\r\n\r\n", "Dž", "Ⅳ", "'ll", "㍿!!", "Dž", " \n", "'T", "'D", "عfi", "<|", "fim", "_prefix", "|>", "漢é", "­m", "'ſ", "㋿", " \n", "㋿", "0", "\u000b", "'", "refi"]} +{"text": "0'Tt漢漢 <|endoftext|><|fim_prefix|><|fim_prefix|>'T\"m'S½\u000bt'ſḍ̇>ßſ'Tt>s éZ 12345678'Ma­'Re😀🏽", "tokens": 66, "pieces": ["0", "'T", "t漢漢", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "T", "\"m", "'S", "½", "\u000bt", "'ſ", "ḋ", "̣>", "ßſ", "'T", "t", ">s", " e", "́Z", " ", "123", "456", "78", "'M", "a", "­'", "Re", "😀🏽"]} +{"text": "'VE'll\r\n\r\n's'M're­'ss'D३ß㍿", "tokens": 17, "pieces": ["'VE", "'ll", "\r\n\r\n", "'s", "'M", "'re", "­'", "ss", "'D", "३", "ß", "㍿"]} +{"text": "​㍿㍿ 字३३Dž!12345678s'VEt漢­EOT", "tokens": 28, "pieces": ["​㍿㍿", " ", " 字", "३३", "Dž", "!", "123", "456", "78", "s", "'VE", "t漢", "­EOT"]} +{"text": "'S‍s😀🏽'Re ㋿émfi0", "tokens": 22, "pieces": ["'S", "‍s", "😀🏽'", "Re", " ", " ㋿", "e", "́mfi", "0"]} +{"text": "#$%dé'ſ́Z'MEOT.'VE>", "tokens": 15, "pieces": ["#$%", "de", "́'", "ſ", "́Z", "'M", "EOT", ".'", "VE", ">"]} +{"text": "t>Dž'Re aEOT­\r\n\r\n字9EOT-!(\u000b'Mm-", "tokens": 23, "pieces": ["t", ">Dž", "'Re", " aEOT", "­\r\n\r\n", "字", "9", "EOT", "-!(", "\u000b", "'M", "m", "-"]} +{"text": "字㍿　!('T \r9'll😀🏽'D EOT‍<|endoftext|>(Dž-'D​\r\nEOT", "tokens": 46, "pieces": ["字", "㍿", "　", "!('", "T", " \r", "9", "'ll", "😀🏽'", "D", "", " EOT", "‍<|", "endoftext", "|><", "EOT", ">(", "Dž", "-'", "D", "​\r\n", "EOT"]} +{"text": ">漢㋿d t😀🏽‍Dž \n'TsdEOT\"<|endoftext|>ع🙂\r👍🏽(ß9's,", "tokens": 44, "pieces": [">漢", "㋿d", " t", "😀🏽‍", "Dž", " \n", "'T", "sdEOT", "\"<|", "endoftext", "|>", "ع", "🙂\r", "👍🏽(", "ß", "9", "'s", ","]} +{"text": "㋿\u000b‍EOTåt字\r\né\r #$%ꟲ>㋿é'S 'm.𐞁", "tokens": 37, "pieces": ["㋿", "\u000b", "‍EOTa", "̊t字", "\r\n", "é", "\r", " ", " #$%", "ꟲ", ">㋿", "e", "́'", "S", " '", "m", ".𐞁"]} +{"text": "\r\n 'ſḍ̇​'Re,", "tokens": 16, "pieces": ["\r\n", " <", "META", "_START", ">'", "ſḋ", "̣​'", "Re", ","]} +{"text": "'ſ!
\"'S>'sſ'Re.
　½<|fim_prefix|> \nd'<'ſé'll", "tokens": 38, "pieces": ["'ſ", "!", "
", "\"'", "S", ">'", "s", "ſ", "'Re", ".", "
", "　", "½", "<|", "fim", "_prefix", "|>", " \n", "d", "'<'", "ſe", "́'", "ll"]} +{"text": "(Ź\n<|fim_prefix|>́'VEＡ<|fim_prefix|>", "tokens": 22, "pieces": ["(Z", "́\n", "<|", "fim", "_prefix", "|>́'", "VEＡ", "<|", "fim", "_prefix", "|>"]} +{"text": "😀🏽'VEfiḍ̇\r\n\r\nßéſ#$%Ａm!'re½ 'Re­३\nⅣ (>'reعe\r漢", "tokens": 46, "pieces": ["😀🏽'", "VEfiḋ", "̣\r\n\r\n", "ßéſ", "#$%", "Ａm", "!'", "re", "½", " ", "'Re", "­", "३", "\n", "Ⅳ", " (>'", "reعe", "\r", "漢"]} +{"text": "'s㍿m'VE>ßß'ſ (Ⅳ‍'Dßß", "'", "ſ", " ", "(", "Ⅳ", "‍'", "D", "字", "tokens": 41, "pieces": ["३", "𐞁", "-", "\t", "…", "㍿Dž字", "!!👍🏽", "ꟲ", "'VE", "é", "#$%\r\n", "<'", "s", "<|", "endoftext", "|>", "字"]} +{"text": "㋿9e,🙂å\r\n­'M>.eꟲ.'st'ſ३0Z'M.", "tokens": 37, "pieces": ["㋿", "9", "e", ",🙂", "a", "̊<", "EOT", ">\r\n", "­'", "M", ">.", "eꟲ", ".'", "st", "'ſ", "३", "", "0", "Z", "'M", "."]} +{"text": " \n'Dž.'ssé
9́(#$%'M'S㍿Ⅳ漢éé's \n­\r\u000b'M😀🏽", "tokens": 49, "pieces": [" \n", "'Dž", ".'", "ssé", "
", "9", "́(#$%'", "M", "'S", "㍿", "Ⅳ", "漢ée", "́'", "s", " \n", "­\r", "", "\u000b", "'", "M", "😀🏽"]} +{"text": "ḍ̇é漢", "tokens": 8, "pieces": ["ḋ", "̣é漢"]} +{"text": "Z12345678\u000b('D,é>'<|endoftext|>'\r\n漢'sm<|endoftext|>Zt", "tokens": 32, "pieces": ["Z", "123", "456", "78", "\u000b", "('", "D", ",é", ">'<|", "endoftext", "|>'\r\n", "漢", "'s", "m", "<|", "endoftext", "|>", "Zt"]} +{"text": "!!'re'VE'll're　ع'D　\u000b9!!", "tokens": 14, "pieces": ["!!'", "re", "'VE", "'ll", "'re", "　ع", "'D", "　", "\u000b", "9", "!!"]} +{"text": "
å å‍>👍🏽\r\n<|endoftext|>ݽe­ꟲDž-ḍ̇!!ꟲfi<|fim_prefix|>\t'VEs <é\r\n\r\n'll٣٤٥٦", "tokens": 70, "pieces": ["
a", "̊", " ", " a", "̊‍>👍🏽\r\n", "<|", "endoftext", "|>", "İ", "½", "e", "­ꟲDž", "-ḋ", "̣!!", "ꟲfi", "<|", "fim", "_prefix", "|>", "\t", "'VE", "s", " <", "é", "\r\n\r\n", "'ll", "٣٤٥", "٦"]} +{"text": "\r\n'VE>dd́字'Re'lle's\né  t३ 漢'ſ!!#$%漢३\n\"
'👍🏽😀🏽EOT", "tokens": 49, "pieces": ["\r\n", "'VE", ">dd", "́字", "'Re", "'ll", "e", "'s", "\n", "é", "  ", " t", "३", " 漢", "'ſ", "!!#$%", "漢", "३", "\n", "\"", "
", "'👍🏽😀🏽", "EOT"]} +{"text": "etd12345678 ", "tokens": 9, "pieces": ["etd", "123", "456", "78", "", " "]} +{"text": "\n½'reİİ\"\n 'M>'ſ's\u000b'VE㋿d　🙂a\nEOTé​", "tokens": 40, "pieces": ["\n", "½", "'re", "İİ", "\"\n", " '", "M", ">'", "ſ", "'s", "\u000b", "'VE", "㋿d", "", "　", "🙂a", "\n", "EOTe", "́<", "META", "_START", ">​"]} +{"text": ".\r\n\r\n٣٤٥٦", "tokens": 9, "pieces": [".\r\n\r\n", "٣٤٥", "٦"]} +{"text": "å字( å .s-　m३9( ́!​<'Tm", "tokens": 42, "pieces": ["a", "̊字", "(", " a", "̊", " ", " .", "s", "-", "　m", "३9", "(<", "EOT", ">", " ́!​<<", "META", "_START", "><", "EOT", ">'", "Tm"]} +{"text": "#$%ع\"\n", "tokens": 4, "pieces": ["#$%", "ع", "\"\n"]} +{"text": " \nſꟲ-👍🏽(mm#$%!!(\t
'D\" 
'Ré#$%>", "tokens": 42, "pieces": [" \n", "ſ", "ꟲ", "-👍🏽(", "mm", "#$%!!(", "\t", "
", "'D", "\"", " ", " <", "EOT", ">", "
", "'Re", "́#$%>"]} +{"text": "!! \nßåå", "tokens": 9, "pieces": ["!!", " \n", "ßa", "̊a", "̊"]} +{"text": "ꟲ'VE­́Ⅳte!-'M#$%", "tokens": 15, "pieces": ["ꟲ", "'VE", "­́", "Ⅳ", "te", "!-'", "M", "#$%"]} +{"text": "​㋿0\r'M漢é́Dždåe s \nå😀🏽'Resd \n३>'ll\u000b 'refi'lla'VE½\n", "tokens": 47, "pieces": ["​㋿", "0", "\r", "'M", "漢é", "́Džda", "̊e", " s", " \n", "a", "̊😀🏽'", "Resd", " \n", "३", ">'", "ll", "\u000b ", " '", "refi", "'ll", "a", "'VE", "½", "\n"]} +{"text": "e'Re9٣٤٥٦​٣٤٥٦'T", "tokens": 21, "pieces": ["e", "'Re", "9٣٤", "٥٦", "​", "٣٤٥", "٦", "'T"]} +{"text": "🙂m", "tokens": 3, "pieces": ["🙂m"]} +{"text": " 
́'ll字İ", "tokens": 8, "pieces": [" ", "
", "́'", "ll字İ"]} +{"text": "å'Mꟲ'S\t٣٤٥٦३…e­　­å३é<|endoftext|>　　\r\t.", "tokens": 45, "pieces": ["a", "̊'", "Mꟲ", "'S", "\t", "٣٤٥", "٦३", "…e", "­", "　", "­a", "̊", "३", "é", "<|", "endoftext", "|>", "　　\r", "\t", "."]} +{"text": "٣٤٥٦t𐞁 \n\r\n!éDž>'T'S.'s0'ſ ", "tokens": 29, "pieces": ["٣٤٥", "٦", "t𐞁", " \n\r\n", "!éDž", ">'", "T", "'S", ".'", "s", "0", "'ſ", " "]} +{"text": "٣٤٥٦'reſ \n\r\n<Ⅳ　\r३0
s½'VE \n#$%٣٤٥٦\u000b𐞁", "tokens": 52, "pieces": ["٣٤٥", "٦", "'re", "ſ", " \n\r\n", "<", "Ⅳ", "　", "\r", "३0", "
s", "½", "'VE", " \n", "#$%", "٣٤٥", "٦", "\u000b", "𐞁"]} +{"text": "'　ḍ̇…d'é<|endoftext|>EOT \u000b㋿'Re'll­ \n😀🏽½𐞁<|endoftext|>#$%é9'VE9's(İ", "tokens": 59, "pieces": ["'", "　ḋ", "̣", "…d", "'é", "<|", "endoftext", "|>", "EOT", " ", "\u000b", "㋿'", "Re", "'ll", "­", " \n", "😀🏽", "½", "𐞁", "<|", "endoftext", "|>#$%", "é", "9", "'", "VE", "9", "'s", "(İ"]} +{"text": "​'漢00
…㍿!😀🏽‍-'𐞁 😀🏽\r\n\r\n👍🏽Ａ漢字<<𐞁m\r\n", "tokens": 53, "pieces": ["​'", "漢", "00", "
", "", "…", "㍿!😀🏽‍-'", "𐞁", " ", " 😀🏽\r\n\r\n", "👍🏽", "Ａ漢字", "<<", "𐞁m", "\r\n"]} +{"text": "fi‍(", "tokens": 5, "pieces": ["fi", "‍("]} +{"text": "­!ꟲ字é㍿m　 ㍿éＡⅣEOTfi　'ſ𐞁a \n‍ſ é\r\n\r\n\"å'S", "tokens": 55, "pieces": ["­!", "ꟲ字e", "́㍿", "m", "　", " ", "㍿éＡ", "Ⅳ", "EOTfi", "　", "'ſ", "𐞁a", " \n", "‍<", "EOT", ">ſ", " é", "\r\n\r\n", "\"a", "̊'", "S"]} +{"text": "Z \n!!ß(<|endoftext|>EOT३　ḍ̇>åEOTß!字ꟲ½字.㍿!!é 'VE\t", "tokens": 48, "pieces": ["Z", " \n", "!!", "ß", "(<|", "endoftext", "|>", "EOT", "३", "　", "ḋ", "̣>", "a", "̊EOTß", "!字ꟲ", "½", "字", ".㍿!!", "é", " '", "VE", "\t"]} +{"text": "'s\t#$%‍‍  \n　Dž ٣٤٥٦ 0.fiꟲ<\u000b'D…'re12345678ꟲ…㍿ع'T<|fim_prefix|>\tå­'T'D>!!'ſm", "tokens": 69, "pieces": ["'s", "\t", "#$%‍‍", "  \n", "　Dž", " ", "٣٤٥", "٦", " ", "0", ".fiꟲ", "<", "\u000b", "'D", "…", "'re", "123", "456", "78", "ꟲ", "…", "㍿ع", "'T", "<|", "fim", "_prefix", "|>", "\ta", "̊­'", "T", "'D", ">!!'", "ſm"]} +{"text": "　'VEع'reeⅣ \n\r\n٣٤٥٦३ ‍ EOT-.　😀🏽'sa", "tokens": 37, "pieces": ["　", "'VE", "ع", "'re", "e", "Ⅳ", " \n", "\r\n", "٣٤٥", "٦३", " ‍", " ", " EOT", "-.", "　", "😀🏽'", "sa"]} +{"text": "㍿EOTſ<12345678𐞁\tßa .​t漢<|fim_prefix|><ع😀🏽\u000b\n
　字Z0Dž<|fim_prefix|>", "tokens": 54, "pieces": ["㍿EOTſ", "<", "123", "456", "78", "𐞁", "\tßa", " ", " .​", "t漢", "<|", "fim", "_prefix", "|><", "ع", "😀🏽", "\u000b\n", "
", "　字Z", "0", "Dž", "<|", "fim", "_prefix", "|>"]} +{"text": "\"'T'D'Re ́'sİ<|fim_prefix|>\r३\r\nꟲaßſ½́½'ll", "tokens": 35, "pieces": ["\"'", "T", "'D", "'Re", " ", " <", "META", "_START", ">́'", "sİ", "<|", "fim", "_prefix", "|>\r", "३", "\r\n", "ꟲaßſ", "½", "́", "½", "'ll"]} +{"text": "<'s\nİ३#$%İd ‍ſ字㋿㍿ſ-字> 😀🏽́ß\na", "tokens": 35, "pieces": ["<'", "s", "\n", "İ", "३", "#$%", "İd", " ‍", "ſ字", "㋿㍿", "ſ", "-字", ">", " ", " 😀🏽́", "ß", "\n", "a"]} +{"text": "字 ", "tokens": 2, "pieces": ["字", " "]} +{"text": "-'lltİ12345678ꟲd½İ0ßḍ̇t-< 👍🏽", "tokens": 33, "pieces": ["-'", "lltİ", "123", "456", "78", "ꟲd", "½", "İ", "0", "ß", "ḋ", "̣t", "-<", " ", " 👍🏽"]} +{"text": " ㋿\r<|fim_prefix|> 0é!!.㋿'ſḍ̇é'll 'T#$%\t\r\n\r\n12345678fi\"\"Džé'Reع<", "tokens": 51, "pieces": [" ", "㋿\r", "<|", "fim", "_prefix", "|>", " ", "0", "é", "!!.㋿'", "ſḋ", "̣e", "́'", "ll", " ", " '", "T", "#$%", "\t\r\n\r\n", "123", "456", "78", "fi", "\"\"", "Dže", "́'", "Reع", "<"]} +{"text": "'llå,\r­ \u000b㍿\r\n\r\n\r'👍🏽\r\n\r\né", "tokens": 24, "pieces": ["'ll", "a", "̊,\r", "­", " ", "\u000b", "㍿\r\n\r\n\r", "'👍🏽\r\n\r\n", "e", "́"]} +{"text": "'ſ", "tokens": 3, "pieces": ["'ſ"]} +{"text": " 're'éé㍿ḍ̇DžZ(m😀🏽
عé\rⅣ'M㋿ḍ̇İé½\u000b", "tokens": 44, "pieces": [" '", "re", "'e", "́é", "㍿ḋ", "̣DžZ", "(m", "😀🏽", "
عe", "́\r", "Ⅳ", "'M", "㋿ḋ", "̣İe", "́", "½", "\u000b"]} +{"text": "'ſⅣt𐞁­\n👍🏽🙂a🙂'M<|fim_prefix|>字Dž𐞁ع́㋿", "Ⅳ", "t𐞁", "­\n", "👍🏽🙂", "a", "🙂'", "M", "<|", "fim", "_prefix", "|>", "字Dž𐞁ع", "́㋿<", "e", "́漢", "…", ",e", "(𐞁"]} +{"text": "​.'ś‍­.'ll9m٣٤٥٦ßⅣ'ſḍ̇'Re\tmß>𐞁#$%Ⅳ(\t !ꟲＡZ 0d0", "tokens": 56, "pieces": ["​.'", "s", "́‍­.'", "ll", "9", "m", "٣٤٥", "٦", "ß", "Ⅳ", "'ſ", "ḋ", "̣'", "Re", "\tmß", ">𐞁", "#$%", "Ⅳ", "(", "\t ", " !", "ꟲＡZ", " ", "0", "d", "0"]} +{"text": "́!'DZ Dž<|fim_prefix|> \n'Re", "tokens": 16, "pieces": ["́!'", "DZ", " Dž", "<|", "fim", "_prefix", "|>", " \n", "'Re"]} +{"text": "Ⅳ\r🙂'M9-́ \n!!İ­\t!字'D<|fim_prefix|>ꟲe'VE \n's😀🏽mعꟲet　👍🏽'D字🙂'Re", "tokens": 61, "pieces": ["Ⅳ", "\r", "🙂'", "M", "9", "-́", " \n", "!!", "İ", "­", "\t", "!字", "'D", "<|", "fim", "_prefix", "|>", "ꟲe", "'VE", " \n", "'s", "😀🏽", "mعꟲet", "　", "👍🏽'", "D字", "🙂<", "EOT", ">'", "Re"]} +{"text": "　,> Z٣٤٥٦(‍'s…\r'Re>…é\u000bİꟲ'S'Re!\ndḍ̇'DéZ字 ſ'Re0ع'S \u000bİ", "tokens": 56, "pieces": ["　", ",>", " ", " Z", "٣٤٥", "٦", "(‍'", "s", "…\r", "'Re", ">", "…é", "\u000bİꟲ", "'", "S", "'Re", "!\n", "dḋ", "̣'", "DéZ字", " ſ", "'Re", "0", "ع", "'S", " ", "\u000bİ"]} +{"text": "'MZ‍!!'S३ß<|fim_prefix|>(<|fim_prefix|>e'red٣٤٥٦'VE \n'VEe0👍🏽 ", "tokens": 51, "pieces": ["'M", "Z", "‍!!'", "S", "३", "ß", "<|", "fim", "_prefix", "|>(<|", "fim", "_prefix", "|>", "e", "'re", "d", "٣٤٥", "٦", "'VE", " \n", "'VE", "e", "0", "👍🏽", " "]} +{"text": "#$%‍ å𐞁é㍿ <EOTd'ſ12345678\r!漢­'Tfi\n.'Sm#$%", "tokens": 49, "pieces": ["#$%‍", " a", "̊𐞁é", "㍿", " ", "<<", "META", "_START", ">EOTd", "'ſ", "123", "456", "78", "\r", "!漢", "<", "META", "_START", ">­'", "Tfi", "\n", ".'", "Sm", "#$%"]} +{"text": ",漢½m'T३👍🏽Dž\r\n
́-s\r\n\r\n<|fim_prefix|>a३éİ­🙂عfi­ſ'Mfi٣٤٥٦\n", "tokens": 56, "pieces": [",漢", "½", "m", "'T", "३", "👍🏽", "Dž", "\r\n", "
", "́-", "s", "\r\n\r\n", "<|", "fim", "_prefix", "|>", "a", "३", "éİ", "­🙂", "عfi", "­ſ", "'M", "fi", "٣٤٥", "٦", "\n"]} +{"text": "…#$%", "tokens": 4, "pieces": ["…", "#$%"]} +{"text": "<|fim_prefix|>", "tokens": 7, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "👍🏽fi \nes'T　é0å😀🏽ع\t12345678İd́Ａ­ḍ̇<|fim_prefix|>\rＡß! ", "tokens": 55, "pieces": ["👍🏽", "fi", " \n", "es", "'T", "　e", "́", "0", "a", "̊😀🏽", "ع", "\t", "", "123", "456", "78", "İd", "́Ａ", "­ḋ", "̣<|", "fim", "_prefix", "|>\r", "Ａß", "!", " "]} +{"text": " ­<", "tokens": 3, "pieces": [" ", "­<"]} +{"text": "\"e🙂>ع-é12345678Džḍ̇.t👍🏽!३", "tokens": 29, "pieces": ["\"e", "🙂>", "ع", "-e", "́", "123", "456", "78", "Džḋ", "̣.", "t", "👍🏽!", "३"]} +{"text": "३\r\nå!\nZ㋿ſ'Re,𐞁'll(\n'D!'re'ſ 字m", "tokens": 33, "pieces": ["३", "\r\n", "a", "̊!\n", "Z", "㋿ſ", "'Re", ",𐞁", "'ll", "(\n", "'D", "!<", "META", "_START", ">'", "re", "'ſ", " ", " 字m"]} +{"text": "<|fim_prefix|>字𐞁'VEİs \nعⅣ'ReéfiⅣDž\u000bfi", "tokens": 35, "pieces": ["<|", "fim", "_prefix", "|>", "字", "𐞁", "'VE", "İs", " \n", "ع", "Ⅳ", "'Re", "e", "́fi", "Ⅳ", "Dž", "\u000bfi"]} +{"text": "m-<\r\n\r\n'ſ\u000b\"", "tokens": 9, "pieces": ["m", "-<\r\n\r\n", "'ſ", "\u000b", "\""]} +{"text": "'ſ're㍿m'S'Reꟲ\r
🙂.12345678sfi,'<|fim_prefix|>9ꟲ\u000b漢\nt>字\r\nsſ'Red'TmZEOT\t'T", "tokens": 63, "pieces": ["'ſ", "'re", "㍿m", "'S", "'Re", "ꟲ", "\r", "
", "🙂.", "123", "456", "78", "sfi", ",'<|", "fim", "_prefix", "|>", "9", "ꟲ", "\u000b漢", "\n", "t", ">字", "\r\n", "sſ", "'Re", "d", "'T", "m", "ZEOT", "\t", "'T"]} +{"text": ",　", "tokens": 2, "pieces": [",", "　"]} +{"text": "EOT🙂!EOT're,ß'Ms\r\n9\r\n\r\nZ#$%\"'sEOT'ſ \n'll😀🏽㍿'M- ½'ſ!a'D'll0\n", "tokens": 50, "pieces": ["EOT", "🙂!", "EOT", "'re", ",ß", "'M", "s", "\r\n", "9", "\r\n\r\n", "Z", "#$%\"'", "sEOT", "'ſ", " \n", "'ll", "😀🏽㍿'", "M", "-", " ", " ", "½", "'ſ", "!a", "'D", "'ll", "0", "\n"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " \n Ⅳ0e 🙂", "tokens": 11, "pieces": [" \n", " ", "Ⅳ", "", "0", "e", " 🙂"]} +{"text": "d<|endoftext|>'sİ!'VEß,́Dž!tⅣ'M<ꟲ字 \n\u000b!!㍿٣٤٥٦　!!<\rDž'T''VE<|endoftext|>(漢㋿", "tokens": 65, "pieces": ["d", "<|", "endoftext", "|>'", "sİ", "!'", "VEß", ",́", "Dž", "!t", "Ⅳ", "'M", "<ꟲ字", " \n", "\u000b", "!!㍿", "٣٤٥", "٦", "　", "!!<\r", "Dž", "'T", "''", "VE", "<|", "endoftext", "|>(", "漢", "㋿"]} +{"text": "#$%\r\n​aé　 !३'é\r­\rZ字!!é0٣٤٥٦'Re'M ", "tokens": 35, "pieces": ["#$%\r\n", "​aé", "", "　 ", " !", "३", "'e", "́\r", "­\r", "Z字", "!!", "e", "́", "0٣٤", "٥٦", "'Re", "'M", " "]} +{"text": "e!EOTع­<|endoftext|>0t½½ع́'ſ", "tokens": 25, "pieces": ["e", "!EOTع", "­<|", "endoftext", "|>", "0", "t", "½½", "ع", "́'", "ſ"]} +{"text": "éＡ㍿ḍ̇e'VEDž\r 12345678½é're'Re0\t\r\n\r\ń½!!\r\n\r\n👍🏽'Dfi'S<|endoftext|>s\r\n-ꟲa<|fim_prefix|>'ſ 'res", "tokens": 73, "pieces": ["e", "́Ａ", "㍿ḋ", "̣e", "'VE", "Dž", "\r", " ", "123", "456", "78½", "é", "'re", "'Re", "0", "\t\r\n\r\n", "́", "½", "!!\r\n\r\n", "👍🏽'", "Dfi", "'S", "<|", "endoftext", "|>", "s", "\r\n", "-ꟲa", "<|", "fim", "_prefix", "|>'", "ſ", "", " ", "'re", "s"]} +{"text": "\"‍٣٤٥٦é0!!ꟲＡ", "tokens": 20, "pieces": ["\"‍", "٣٤٥", "٦", "e", "́", "0", "!!", "ꟲＡ"]} +{"text": "és", "tokens": 3, "pieces": ["e", "́s"]} +{"text": "Ⅳع­𐞁#$%\t😀🏽fi ꟲfi'ſ!!t'D'MZꟲ", "tokens": 46, "pieces": ["Ⅳ", "ع", "­<", "META", "_START", ">𐞁", "#$%", "\t", "😀🏽", "fi", " ꟲfi", "'ſ", "!!", "t", "'", "D", "'M", "Zꟲ"]} +{"text": "🙂DžＡ9fi'D𐞁EOT.0‍<漢,'M>३㍿‍漢'fi>'S", "tokens": 47, "pieces": ["🙂DžＡ", "9", "<", "META", "_START", ">fi", "'D", "𐞁EOT", ".", "0", "‍<", "漢", ",'", "M", ">", "३", "㍿‍", "漢", "'fi", ">'", "S"]} +{"text": "​ <|endoftext|>字(​ 'ſEOT'Mß‍㍿<|endoftext|>字ſ　'sꟲ12345678Ⅳ'Dꟲ३'reⅣs'll'll\n ㋿Ａ　a
㍿३ \n", "tokens": 72, "pieces": ["​", " ", " <|", "endoftext", "|>", "字", "(​", " '", "ſEOT", "'M", "ß", "‍㍿<|", "endoftext", "|>", "字ſ", "　", "'s", "ꟲ", "123", "456", "78Ⅳ", "'D", "ꟲ", "३", "'re", "Ⅳ", "s", "'ll", "'ll", "\n", " ㋿", "Ａ", "　a", "
", "㍿", "३", " \n"]} +{"text": "𐞁's㍿dſ'ſ　👍🏽\t#$%Ⅳſ<
🙂\"m‍🙂ſ", "tokens": 41, "pieces": ["𐞁", "'s", "㍿dſ", "'ſ", "　", "👍🏽", "\t", "#$%", "Ⅳ", "ſ", "<", "
", "🙂\"", "m", "‍🙂", "ſ"]} +{"text": "\r", "tokens": 1, "pieces": ["\r"]} +{"text": "m\"'T<　　👍🏽'T ꟲ字㋿é! #$%\u000b!!٣٤٥٦漢'M", "tokens": 43, "pieces": ["m", "\"'", "T", "<<", "META", "_START", ">", "　", "　", "👍🏽'", "T", " ꟲ字", "㋿e", "́!", " ", " #$%", "\u000b", "!!", "٣٤٥", "٦", "漢", "'M"]} +{"text": "<'så漢'M'ſ🙂٣٤٥٦\nfi😀🏽d #$%(", "tokens": 32, "pieces": ["<'", "sa", "̊漢", "'M", "'ſ", "🙂", "٣٤٥", "٦", "\n", "fi", "😀🏽", "d", " ", " #$%("]} +{"text": "d'S
 ", "tokens": 5, "pieces": ["d", "'S", "
 "]} +{"text": " EOT́😀🏽­d\tＡ123456780.'VE😀🏽éåé9<|endoftext|>­ß'S'EOTDž'llßİ", "tokens": 50, "pieces": [" EOT", "́😀🏽­", "d", "\tＡ", "123", "456", "780", ".'", "VE", "😀🏽", "é", "a", "̊e", "́", "9", "<|", "endoftext", "|>­", "ß", "'S", "'EOTDž", "'ll", "ßİ"]} +{"text": "👍🏽İ<|fim_prefix|>", "tokens": 14, "pieces": ["👍🏽", "İ", "<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|>\r\n-d\r\n\r\n<|endoftext|> \n'D>½…\t🙂", "tokens": 25, "pieces": ["<|", "endoftext", "|>\r\n", "-d", "\r\n\r\n", "<|", "endoftext", "|>", " \n", "'D", ">", "½", "…", "\t", "🙂"]} +{"text": "ع'reå", "tokens": 5, "pieces": ["ع", "'re", "a", "̊"]} +{"text": "EOT'll're　åt\r\nZ 9<|fim_prefix|>'M…>­㍿!!fiعEOT\u000b'Re", "tokens": 36, "pieces": ["EOT", "'ll", "'re", "　a", "̊t", "\r\n", "Z", " ", "9", "<|", "fim", "_prefix", "|>'", "M", "…", ">­㍿!!", "fiعEOT", "\u000b", "'Re"]} +{"text": "㋿ß漢…Z㍿\"ß<#$%
\r\n\tZEOTZ9́>ＡZ'M­'Re　\r é字!!é", "tokens": 46, "pieces": ["㋿ß漢", "…Z", "㍿\"", "ß", "<#$%", "
\r\n", "\tZEOTZ", "9", "́><", "EOT", ">ＡZ", "'M", "­'", "Re", "　\r", " é字", "!!", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿'ſ'VE'VE'Re'Mḍ̇　­​\u000b(ḍ̇​'Re㍿㋿　\u000ba'reé\r㍿\n,12345678 𐞁", "tokens": 64, "pieces": ["㍿'", "ſ", "'VE", "'VE", "'Re", "'", "Mḋ", "̣", "　", "­​", "\u000b", "(<", "EOT", ">ḋ", "̣​'", "Re", "㍿㋿", "　", "\u000ba", "'re", "e", "́\r", "㍿\n", ",", "123", "456", "78", " ", " 𐞁"]} +{"text": "​
字>㍿😀🏽\"'D'reéDž 're‍é'Re\"'T٣٤٥٦12345678٣٤٥٦Z㍿😀🏽\"'", "D", "'re", "éDž", " ", " '", "re", "‍e", "́'", "Re", "\"'", "T", "٣٤٥", "٦12", "345", "678", "٣٤٥", "٦", "Z", "'ſ　s漢́\t>'Re<­", "tokens": 24, "pieces": ["ß", "#$%🙂", "३", "a", "'VE", "\r\n", ">'", "ſ", "　s漢", "́", "\t", ">'", "Re", "<­"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "é\rDž12345678　'll\tḍ̇'T\"!'llعs'll'll ㋿", "tokens": 28, "pieces": ["é", "\r", "Dž", "123", "456", "78", "　", "'ll", "\tḋ", "̣'", "T", "\"!'", "llعs", "'ll", "'ll", " ", "㋿"]} +{"text": ">Ａ#$%åEOT's字ß‍ \t9'ſ>'M 'S'Res<|endoftext|><|endoftext|>İ㍿é9Džé's漢'", "tokens": 59, "pieces": [">Ａ", "#$%", "a", "̊EOT", "'s", "字ß", "‍", " ", "\t", "9", "'ſ", ">'", "M", " ", " <", "META", "_START", ">'", "S", "'Re", "s", "<|", "endoftext", "|><|", "endoftext", "|>", "İ", "㍿é", "9", "Dže", "́'", "s漢", "'"]} +{"text": "Ⅳ!!0\u000bé\r\n\r\nＡ!!Džꟲ'9…9'D㍿\n0Z12345678'M#$%
\r\nfi'reß字", "tokens": 62, "pieces": ["Ⅳ", "!!", "0", "\u000be", "́\r\n\r\n", "Ａ", "!!", "Džꟲ", "'", "9", "…", "9", "'D", "㍿\n", "", "0", "Z", "123", "456", "78", "'M", "#$%", "
\r\n", "fi", "'re", "ß", "字"]} +{"text": "عſ३es9​㍿\r\n'D\r\n\r\n", "tokens": 24, "pieces": ["عſ", "३", "es", "9", "​㍿\r\n", "'D", "\r\n\r\n"]} +{"text": "éDž.­ …>…\r\n­", "tokens": 14, "pieces": ["e", "́Dž", ".­", " ", "…", ">", "…\r\n", "­"]} +{"text": "​㋿'T'D㋿㍿>字 ㍿", "tokens": 19, "pieces": ["​㋿'", "T", "'D", "㋿㍿>", "字", " ", "㍿"]} +{"text": "\r!!'re's‍!m(字­
\téع́d😀🏽EOTaDž\"e'VE\rå0\nꟲＡ漢>\n9… 'Déعع", "tokens": 56, "pieces": ["\r", "!!'", "re", "'s", "‍!", "m", "(字", "­", "
", "\téع", "́d", "😀🏽", "EOTaDž", "\"e", "'VE", "\r", "a", "̊", "0", "\n", "ꟲＡ漢", ">\n", "9", "…", " ", "'D", "e", "́عع"]} +{"text": "Ａ\r​a\t🙂　'Dع​t𐞁'llfiİ", "tokens": 21, "pieces": ["Ａ", "\r", "​a", "\t", "🙂", "　", "'D", "ع", "​t𐞁", "'ll", "fiİ"]} +{"text": "EOT😀🏽<|endoftext|>‍\r\n\"Džꟲ'\rd.<9\r\r\n\r\n!\n…9　٣٤٥٦", "tokens": 46, "pieces": ["EOT", "😀🏽<|", "endoftext", "|>‍\r\n", "\"Dž", "ꟲ", "'\r", "d", ".<", "9", "\r\r\n\r\n", "!\n", "…", "9", "　", "٣٤٥", "٦"]} +{"text": "'re 👍🏽字\r!!<|endoftext|>sⅣ'T(　Z\r\n😀🏽٣٤٥٦0🙂👍🏽'M<|endoftext|>\r漢ḍ̇\r\n\r\n İ<9Ⅳ'Ds", "tokens": 74, "pieces": ["'re", " ", "👍🏽", "字", "\r", "!!<|", "endoftext", "|>", "s", "Ⅳ", "'T", "(", "　Z", "\r\n", "😀🏽", "٣٤٥", "٦0", "🙂👍🏽'", "M", "<|", "endoftext", "|>\r", "漢ḋ", "̣\r\n\r\n", " İ", "<", "9Ⅳ", "'D", "s"]} +{"text": "\u000b\tſ漢é", "tokens": 8, "pieces": ["\u000b", "\tſ漢e", "́"]} +{"text": "Dž#$% t'Re‍ ꟲ", "tokens": 11, "pieces": ["Dž", "#$%", " t", "'Re", "‍", " ꟲ"]} +{"text": "Zꟲ½e#$%e", "tokens": 9, "pieces": ["Zꟲ", "½", "e", "#$%", "e"]} +{"text": "　…‍​'s'ſ🙂'll <|fim_prefix|>ع\t!é٣٤٥٦ḍ̇e…漢Dž", "tokens": 69, "pieces": ["　", "…", "‍​'", "s", "'ſ", "🙂'", "ll", " <|", "fim", "_prefix", "|>", "ع", "", "\t", "!e", "́", "٣٤٥", "٦", "ḋ", "̣e", "…漢Dž"]} +{"text": "'T<|fim_prefix|>12345678'\r\n
á İ\" İ 's", "tokens": 23, "pieces": ["'T", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'\r\n", "
a", "́", " İ", "\"", " ", " İ", " ", "'s"]} +{"text": "'s\t9EOT'ſ<😀🏽​>0漢sſ ꟲ'VEǻ‍", "tokens": 33, "pieces": ["'s", "\t", "9", "EOT", "'ſ", "<😀🏽​>", "0", "漢sſ", " ꟲ", "'VE", "a", "̊́‍"]} +{"text": ">'M", "tokens": 5, "pieces": ["><", "META", "_START", ">'", "M"]} +{"text": ".!<|endoftext|> ३#$%", "tokens": 13, "pieces": [".!<|", "endoftext", "|>", " ", "३", "#$%"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'T​å३t𐞁å½­​0a", "tokens": 20, "pieces": ["'T", "​a", "̊", "३", "t𐞁a", "̊", "½", "­​", "0", "a"]} +{"text": "‍", "tokens": 2, "pieces": ["‍"]} +{"text": "\u000bétＡ\r\nå \n​0'T EOT9\r\n\r\n \nt\"'Sa", "tokens": 23, "pieces": ["\u000be", "́tＡ", "\r\n", "a", "̊", " \n", "​", "0", "'T", " EOT", "9", "\r\n\r\n \n", "t", "\"'", "Sa"]} +{"text": "(-‍
'S­Z漢Z🙂12345678 'VEs'Re漢👍🏽​>　 ​́s😀🏽Ⅳ'D
Z'Z", "tokens": 59, "pieces": ["(-‍", "
", "'", "S", "­<", "META", "_START", ">Z漢Z", "🙂<", "EOT", ">", "123", "456", "78", " ", "'VE", "s", "'Re", "漢", "👍🏽​>", "　", " ​́", "s", "😀🏽", "Ⅳ", "'D", "
Z", "'Z"]} +{"text": "…fi'll's　#$%㋿std#$%.Ⅳ😀🏽👍🏽'T㍿​ß'M", "tokens": 47, "pieces": ["…fi", "'ll", "'s", "　", "#$%㋿", "s", "td", "#$%.", "Ⅳ", "😀🏽👍🏽'", "T", "㍿​", "ß", "'M"]} +{"text": "㍿é0\r\n\r\n㋿d　'…12345678漢㋿Z ſ 'Re ( 'reDž\n\r\n\r\n", "tokens": 42, "pieces": ["㍿", "e", "́", "0", "\r\n\r\n", "㋿d", "　", "'", "…", "123", "456", "78", "漢", "㋿Z", " ſ", " ", " '", "Re", " ", " (", " ", " '", "reDž", "\n\r\n\r\n"]} +{"text": "'llDž12345678aé<'T-'S ३eEOTſ\r'ſ\u000bꟲEOT‍ .DžⅣZ!!-'", "S", " ", "३", "eEOTſ", "\r", "'ſ", "\u000bꟲEOT", "‍", " .", "Dž", "Ⅳ", "Z", "!!<", "EOTtع", "'D", "　e", "́,", "a", "̊́", "Ａ", "३"]} +{"text": "#$%‍½İ'T㋿‍漢e ", "tokens": 16, "pieces": ["#$%‍", "½", "İ", "'T", "㋿‍", "漢e", " "]} +{"text": "'VE!Z३fi३9Dž　'T
'T>‍'Re'Re ,.㋿'s#$%'T\r\n㍿9½<|endoftext|>'lle \n漢 字​", "tokens": 54, "pieces": ["'VE", "!Z", "३", "fi", "३9", "Dž", "　", "'T", "
", "'T", ">‍'", "Re", "'Re", " ", ",.㋿'", "s", "#$%'", "T", "\r\n", "㍿", "9½", "<|", "endoftext", "|>'", "lle", " \n", "漢", " 字", "​"]} +{"text": "é👍🏽ꟲ𐞁 漢Z'ſſ'll!!<|fim_prefix|>\r\nſ­ \n­\r\n\r\n'll٣٤٥٦\tfi12345678ḍ̇é \n\"12345678İt'Sḍ̇🙂-", "tokens": 75, "pieces": ["é", "👍🏽", "ꟲ𐞁", " 漢Z", "'ſ", "ſ", "'ll", "!!<|", "fim", "_prefix", "|>\r\n", "ſ", "­", " \n", "­\r\n\r\n", "'ll", "٣٤٥", "٦", "\tfi", "123", "456", "78", "ḋ", "̣é", " \n", "\"", "123", "456", "78", "İt", "'S", "ḋ", "̣🙂-"]} +{"text": "-㍿́#$%ꟲ-㍿\r\n\r\n\r\n'D　
‍ꟲ ꟲعé#$%
​🙂ßع👍🏽'ſ'D​<|endoftext|>!'ll\r\n\r\nDž½ßd", "tokens": 65, "pieces": ["-㍿́#$%", "ꟲ", "-㍿\r\n\r\n\r\n", "'D", "　", "
", "‍ꟲ", " ꟲعe", "́#$%", "
", "​🙂", "ßع", "👍🏽'", "ſ", "'D", "​<|", "endoftext", "|>!'", "ll", "\r\n\r\n", "Dž", "½", "ßd"]} +{"text": "'sſ 's
…ع-٣٤٥٦>e!　٣٤٥٦
㍿<|endoftext|>
<fi😀🏽'VE>İé\n<ꟲ­ßſ🙂12345678'DZ ㋿ Ⅳ", "tokens": 72, "pieces": ["\n\n", ">-", "٣٤٥", "٦", ">e", "!", "　", "٣٤٥", "٦", "
", "㍿<|", "endoftext", "|>", "
", "<fi", "😀🏽'", "VE", ">İe", "́\n", "<ꟲ", "­ßſ", "🙂", "123", "456", "78", "'D", "Z", " ", "㋿", " ", "Ⅳ"]} +{"text": "\td#$%#$%>\r\n,t\t0<|fim_prefix|>\nع३½\r\n", "tokens": 27, "pieces": ["\td", "#$%#$%><", "EOT", ">\r\n", ",t", "\t", "0", "<|", "fim", "_prefix", "|>\n", "ع", "", "३½", "\r\n"]} +{"text": "́-­'T'lld0.e'M,'Se\r\n\r\n\r\n!!㍿'reDž字", "tokens": 25, "pieces": ["́-­'", "T", "'ll", "d", "0", ".e", "'M", ",'", "Se", "\r\n", "\r\n\r\n", "!!㍿'", "reDž字"]} +{"text": ".'sEOT\r\n\r\nⅣå!s", "tokens": 12, "pieces": [".'", "sEOT", "\r\n\r\n", "Ⅳ", "a", "̊!", "s"]} +{"text": "३İ'll३漢", "tokens": 8, "pieces": ["३", "İ", "'ll", "३", "漢"]} +{"text": "Z \n‍Ａ'VE\r\nméå\t㋿\té​‍\"é'ſt㋿!", "tokens": 34, "pieces": ["Z", " \n", "‍Ａ", "'VE", "\r\n", "me", "́a", "̊", "\t", "㋿", "\te", "́​‍\"", "e", "́'", "ſt", "㋿!"]} +{"text": "fi'ſ\r\n\r\n!!. \"ع", "tokens": 10, "pieces": ["fi", "'ſ", "\r\n\r\n", "!!.", " ", " \"", "ع"]} +{"text": "𐞁ꟲ're\n<\"İém­'re", "tokens": 17, "pieces": ["𐞁ꟲ", "'re", "\n", "<\"", "İe", "́m", "­'", "re"]} +{"text": "\r\nå0​👍🏽Ａ're'TEOT", "tokens": 18, "pieces": ["\r\n", "a", "̊", "0", "​👍🏽", "Ａ", "'re", "'T", "EOT"]} +{"text": "\n>'Sfí㋿ed
'Re<|fim_prefix|>aZ<|fim_prefix|>漢
", "tokens": 33, "pieces": ["\n", ">'", "Sfi", "́㋿", "ed", "
", "'Re", "<|", "fim", "_prefix", "|>", "aZ", "<|", "fim", "_prefix", "|>", "漢", "
"]} +{"text": "123456789
ꟲ­<|fim_prefix|>㋿٣٤٥٦­'ſ'S!ß'Smḍ̇ḍ̇", "tokens": 46, "pieces": ["123", "456", "789", "
ꟲ", "­<|", "fim", "_prefix", "|>㋿", "٣٤٥", "٦", "­'", "ſ", "'S", "!ß", "'S", "mḋ", "̣ḋ", "̣"]} +{"text": "३", "tokens": 2, "pieces": ["३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿​٣٤٥٦12345678'Dع🙂9ſ😀🏽ß㋿İ'Ret३Zع9İ'Reſ‍字‍!३​'s<|fim_prefix|><|fim_prefix|>𐞁12345678​\n'llm", "tokens": 82, "pieces": ["㍿​", "٣٤٥", "٦12", "345", "678", "'D", "ع", "🙂", "9", "ſ", "😀🏽", "ß", "㋿İ", "'Re", "t", "३", "Zع", "9", "İ", "'Re", "ſ", "‍字", "‍!", "३", "​'", "s", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "𐞁", "123", "456", "78", "​\n", "'ll", "m"]} +{"text": "字as'ś'sＡ漢12345678å", "tokens": 16, "pieces": ["字as", "'s", "́'", "sＡ漢", "123", "456", "78", "a", "̊"]} +{"text": "<|endoftext|>s012345678<|endoftext|>0>\"\r\n\r\n㋿İ𐞁.漢,'Ta'Dd\n \n㋿'S½漢\r\n \n漢", "tokens": 49, "pieces": ["<|", "endoftext", "|>", "s", "012", "345", "678", "<|", "endoftext", "|>", "0", ">\"\r\n\r\n", "㋿İ𐞁", ".漢", ",'", "Ta", "'D", "d", "\n \n", "㋿'", "S", "½", "漢", "\r\n \n", "漢"]} +{"text": " ㋿३ßda'T!!", "tokens": 10, "pieces": [" ㋿", "३", "ßda", "'T", "!!"]} +{"text": "<Dž 𐞁㋿\u000bع‍", "tokens": 15, "pieces": ["<Dž", " 𐞁", "㋿", "\u000bع", "‍"]} +{"text": "🙂e٣٤٥٦'llZ<|fim_prefix|>'T", "tokens": 21, "pieces": ["🙂e", "٣٤٥", "٦", "'ll", "Z", "<|", "fim", "_prefix", "|>'", "T"]} +{"text": "m,'\"e字㋿字\u000b'll\u000b0å\r'M9ds𐞁Ａ\r\n 're12345678ḍ̇éaEOT\r½dZ\r\n\r\n \n>e,9é", "tokens": 52, "pieces": ["m", ",'\"", "e字", "㋿字", "\u000b", "'ll", "\u000b", "0", "a", "̊\r", "'M", "9", "ds𐞁Ａ", "\r\n", " ", " '", "re", "123", "456", "78", "ḋ", "̣éaEOT", "\r", "½", "dZ", "\r\n\r\n \n", ">e", ",", "9", "e", "́"]} +{"text": "12345678'M😀🏽.٣٤٥٦'T'Re३e(<㍿åd('T'll\r\n\r\n'VE㋿🙂m​,ḍ̇ \n#$%ß \n‍<|endoftext|>ß'T½'
Dž#$%", "tokens": 78, "pieces": ["123", "456", "78", "'M", "😀🏽.", "٣٤٥", "٦", "'T", "'Re", "३", "e", "(<㍿", "a", "̊d", "('", "T", "'", "ll", "\r\n\r\n", "'VE", "㋿🙂", "m", "​,", "ḋ", "̣", " \n", "#$%", "ß", " \n", "‍<|", "endoftext", "|>", "ß", "'T", "½", "'", "
Dž", "#$%"]} +{"text": "a\r\n\r\nEOT \nḍ̇>12345678's<|fim_prefix|>'ſ.EOT12345678'Rema'D𐞁'T<|endoftext|>fi.", "tokens": 47, "pieces": ["a", "\r\n\r\n", "EOT", " \n", "ḋ", "̣>", "123", "456", "78", "'s", "<|", "fim", "_prefix", "|>'", "ſ", ".EOT", "123", "456", "78", "'Re", "ma", "'D", "𐞁", "'T", "<|", "endoftext", "|>", "fi", "."]} +{"text": "!!!!fi漢İＡe.ḍ̇t漢㍿​!!'reda'M.'ſ‍é> 'VÉ", "tokens": 47, "pieces": ["!!<", "EOT", ">!!", "fi漢İ", "Ａe", ".ḋ", "̣t漢", "㍿​!!'", "reda", "'M", ".'", "ſ", "‍e", "́>", " ", " '", "VE", "́"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "­>\"ådſ　 ", "tokens": 10, "pieces": ["­>\"", "a", "̊dſ", "　 "]} +{"text": "'D'D 'Re12345678'Re'T ZEOT ع㍿\t's㍿­ſ, ", "tokens": 27, "pieces": ["'D", "'D", " ", " '", "Re", "123", "456", "78", "'Re", "'T", " ZEOT", " ع", "㍿", "\t", "'s", "㍿­", "ſ", ",", " "]} +{"text": "'Re'S,-👍🏽 'T
é　#$%", "tokens": 19, "pieces": ["'Re", "'S", ",-👍🏽", " ", " '", "T", "
e", "́", "　", "#$%"]} +{"text": "😀🏽\u000b'llＡ\r\n\r\n\r\n<|endoftext|>(㍿.'s👍🏽½dDž\"", "tokens": 33, "pieces": ["😀🏽", "\u000b", "'ll", "Ａ", "\r\n\r\n\r\n", "<|", "endoftext", "|>(㍿.'", "s", "👍🏽", "½", "dDž", "\""]} +{"text": "9'Re0İ0'M\n<|endoftext|>'T😀🏽٣٤٥٦0", "tokens": 29, "pieces": ["9", "'Re", "0", "İ", "0", "'M", "\n", "<|", "endoftext", "|>'", "T", "😀🏽", "٣٤٥", "٦0"]} +{"text": "<|endoftext|>s㋿\r\n\r\n👍🏽éad.'sſ'S", "tokens": 32, "pieces": ["<|", "endoftext", "|><", "EOT", ">s", "㋿\r\n\r\n", "👍🏽", "e", "́ad", ".'", "sſ", "'", "S"]} +{"text": "a \u000b'ſꟲfi<'VE'Re\r\n", "tokens": 18, "pieces": ["a", " ", "\u000b", "'ſ", "ꟲfi", "<'", "VE", "'Re", "\r\n"]} +{"text": "\r\n\r\n­Dž's'M'S<́'VE'Re\t　'३'🙂é👍🏽\u000b<|endoftext|>Ⅳ३‍㋿'re", "tokens": 46, "pieces": ["\r\n\r\n", "­Dž", "'s", "'M", "'S", "<́'", "VE", "'Re", "\t", "　", "'", "३", "'🙂", "é", "👍🏽", "\u000b", "<|", "endoftext", "|>", "Ⅳ३", "‍㋿'", "re"]} +{"text": "é漢a t'Z\tع> #$%\tİ…'ReDž…٣٤٥٦'D!!é\nⅣmＡt sZ😀🏽'Mfi'll", "tokens": 59, "pieces": ["e", "́漢a", " ", " t", "'Z", "\tع", ">", " ", " #$%", "\tİ", "…", "'Re", "Dž", "…", "٣٤٥", "٦", "'D", "!!", "e", "́\n", "Ⅳ", "mＡt", " sZ", "😀🏽'", "Mfi", "'ll"]} +{"text": "٣٤٥٦ß'Re…!'re\tZmé字9fi!'TEOT -'ſ's(漢a", "tokens": 33, "pieces": ["٣٤٥", "٦", "ß", "'Re", "…", "!'", "re", "\tZme", "́字", "9", "fi", "!'", "TEOT", " -'", "ſ", "'s", "(漢a"]} +{"text": "<|endoftext|>m½!!‍0　<|fim_prefix|>-\r12345678'T'Zs12345678.　'sZ‍<|endoftext|>d'VEd漢'S'VEⅣ
\u000bé漢'Re \n", "tokens": 69, "pieces": ["<|", "endoftext", "|>", "m", "½", "!!‍", "0", "　", "<|", "fim", "_prefix", "|>-\r", "", "123", "456", "78", "'T", "'Zs", "123", "456", "78", ".", "　", "'s", "Z", "‍<|", "endoftext", "|>", "d", "'VE", "d漢", "'S", "'VE", "Ⅳ", "
", "\u000be", "́漢", "'Re", " \n"]} +{"text": "ſ‍Ａ(ſ३\t'D \tⅣ's\r\"EOT.٣٤٥٦ \nİ \"", "tokens": 33, "pieces": ["ſ", "‍Ａ", "(ſ", "३", "\t", "'D", " ", "\t", "Ⅳ", "'s", "\r", "\"EOT", ".", "٣٤٥", "٦", " \n", "İ", " \""]} +{"text": "　(!'Re'Re<|fim_prefix|>\rع\u000bDž're　ß‍'T99d!!åⅣ,‍'D!!㍿
a'll\nŹ'D　", "tokens": 57, "pieces": ["　", "(!'", "Re", "'", "Re", "<|", "fim", "_prefix", "|>\r", "ع", "\u000bDž", "'re", "　ß", "‍'", "T", "99", "d", "!!", "a", "̊", "Ⅳ", ",‍'", "D", "!!㍿", "
a", "'ll", "\n", "Z", "́'", "D", "　"]} +{"text": "'D're㋿0!", "tokens": 7, "pieces": ["'D", "'re", "㋿", "0", "!"]} +{"text": "Ⅳ'ſ‍<|fim_prefix|>9t'Re\rḍ̇🙂'D\r\n\r\nſ'Re字! ع'D\"…　e'll\n!İ !<|fim_prefix|>　tＡ\"fi", "tokens": 60, "pieces": ["Ⅳ", "'ſ", "‍<|", "fim", "_prefix", "|>", "9", "t", "'Re", "\r", "ḋ", "̣🙂'", "D", "\r\n\r\n", "ſ", "'Re", "字", "!", " ع", "'D", "\"", "…", "　e", "'ll", "\n", "!İ", " !<|", "fim", "_prefix", "|>", "　tＡ", "\"fi"]} +{"text": "🙂\u000bİ12345678\r\n12345678", "tokens": 11, "pieces": ["🙂", "\u000bİ", "123", "456", "78", "\r\n", "123", "456", "78"]} +{"text": "åe\u000b0m <\t'T㍿…'VE𐞁e9<|fim_prefix|>'sſ!!\n\r\n㋿Ⅳ'D　<0é", "tokens": 53, "pieces": ["a", "̊e", "\u000b", "0", "m", " <", "\t", "'T", "㍿", "…", "'VE", "𐞁e", "9", "<|", "fim", "_prefix", "|>'", "sſ", "!!<", "EOT", ">\n\r\n", "㋿", "Ⅳ", "'D", "　", "<", "0", "e", "́"]} +{"text": "'ll'Ddm>㋿'İ're'S-'S>𐞁 …Ａ", "tokens": 26, "pieces": ["'ll", "'D", "dm", ">㋿'", "İ", "'re", "'S", "-'", "S", ">", "𐞁", " ", "…Ａ"]} +{"text": "12345678'M\r,Ａ 字'sm\r\n\r\n m>𐞁 \n عſe٣٤٥٦", "tokens": 37, "pieces": ["123", "456", "78", "'M", "\r", ",Ａ", " ", " 字", "'s", "m", "\r\n\r\n", " m", ">𐞁", " \n", " عſe", "٣٤٥", "٦"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿　'DZ. \n!\r\n\r\n\n-👍🏽'ſ\"'D\"㍿ \n'D,'ſ", "tokens": 34, "pieces": ["㍿", "　", "'", "DZ", ".", " \n", "!\r\n\r\n\n", "-👍🏽'", "ſ", "\"'", "D", "\"㍿", " \n", "'D", ",'", "ſ"]} +{"text": "ZaعeéEOT🙂(!!'T٣٤٥٦ع 0éⅣ\n'ſ<|endoftext|>'re👍🏽\r\n\r\nå.< 字", "tokens": 61, "pieces": ["Zaعee", "́EOT", "🙂(!!<", "META", "_START", ">'", "T", "٣٤٥", "٦", "ع", " ", "0", "e", "́", "Ⅳ", "\n", "'ſ", "<", "EOT", "><|", "endoftext", "|>'", "re", "👍🏽\r\n\r\n", "a", "̊.<", " ", " 字"]} +{"text": "İm12345678\r''12345678'T-t　ß12345678
<|fim_prefix|>‍", "tokens": 31, "pieces": ["İm", "123", "456", "78", "\r", "''", "123", "456", "78", "'T", "-t", "　ß", "123", "456", "78", "
", "<|", "fim", "_prefix", "|>‍"]} +{"text": "ḍ̇ea\" 字ع\r
👍🏽
👍🏽😀🏽åḍ̇३Ａ\u000b<|endoftext|>'M­'TZ'ſ👍🏽(‍㋿३0", "tokens": 74, "pieces": ["ḋ", "̣ea", "\"", " 字ع", "\r", "
", "👍🏽", "
", "👍🏽😀🏽", "a", "̊ḋ", "̣", "३", "Ａ", "\u000b", "<|", "endoftext", "|>'", "M", "­'", "TZ", "'ſ", "👍🏽(‍㋿", "३0"]} +{"text": "'D\n½½㍿\r\nꟲ́12345678 a\t#$%٣٤٥٦\n!d0'ſ12345678٣٤٥٦㍿ßé😀🏽.'", "tokens": 59, "pieces": ["'D", "\n", "½½", "㍿<", "META", "_START", ">\r\n", "ꟲ", "́", "123", "456", "78", " a", "\t", "#$%", "٣٤٥", "٦", "\n", "!d", "0", "'ſ", "123", "456", "78٣", "٤٥٦", "㍿ßé", "😀🏽.'"]} +{"text": "Dž>Ａa㍿́<(ḍ̇İ#$%\r' \n　e!!!!9<'T", "tokens": 31, "pieces": ["Dž", ">Ａa", "㍿́<(", "ḋ", "̣İ", "#$%\r", "'<", "META", "_START", ">", " \n", "　e", "!!!!", "9", "<'", "T"]} +{"text": "#$%!字\"👍🏽㋿a'Re\t​Ａ​İ'M\rſ'T\u000bİ字<'D!!
", "tokens": 34, "pieces": ["#$%!", "字", "\"👍🏽㋿", "a", "'Re", "\t", "​Ａ", "​İ", "'M", "\r", "ſ", "'T", "\u000bİ字", "<'", "D", "!!", "
"]} +{"text": "ꟲ're 😀🏽\"'", "tokens": 10, "pieces": ["ꟲ", "'re", " ", " 😀🏽\"'"]} +{"text": "　…'s \nßEOTé", "tokens": 10, "pieces": ["　", "…", "'s", " \n", "ßEOTe", "́"]} +{"text": "́(", "tokens": 2, "pieces": ["́("]} +{"text": "\r\n\r\n
å<|fim_prefix|>EOT'Ss \r\nt…㋿'ll!!'VE're 0Zt👍🏽ع'D", "tokens": 43, "pieces": ["\r\n\r\n", "
a", "̊<|", "fim", "_prefix", "|>", "EOT", "'S", "s", " \r\n", "t", "…", "㋿'", "ll", "!!'", "VE", "'re", " ", "0", "Zt", "👍🏽", "ع", "'D"]} +{"text": "'Dß字9\r\n\r\n३…d'VEfi👍🏽Z…ſ(\tDže३", "tokens": 35, "pieces": ["'D", "ß字", "9", "\r\n\r\n", "३", "…d", "'VE", "fi", "👍🏽", "Z", "…ſ", "(", "\tDže", "", "३"]} +{"text": "\"ḍ̇ع\"!,!!　''T　12345678漢́\"漢 <|endoftext|>🙂'ſ's㋿👍🏽>‍", "tokens": 63, "pieces": ["\"ḋ", "̣ع", "\"!,!!", "　", "''", "T", "　", "123", "456", "78", "漢", "́\"", "漢", " <|", "endoftext", "|>🙂'", "ſ", "'s", "㋿👍🏽>‍"]} +{"text": "'\r\nع'Dé'<|endoftext|>'Re12345678\r\n\r\nfi#$%t\r\n\r\n字EOT<<|fim_prefix|> \n-'VE'Re​ \n", "tokens": 41, "pieces": ["'\r\n", "ع", "'D", "é", "'<|", "endoftext", "|>'", "Re", "123", "456", "78", "\r\n\r\n", "fi", "#$%", "t", "\r\n\r\n", "字EOT", "<<|", "fim", "_prefix", "|>", " \n", "-'", "VE", "'Re", "​", " \n", ""]} +{"text": "'ReEOT𐞁ḍ̇<|fim_prefix|>e½İ\u000b½å", "tokens": 27, "pieces": ["'Re", "EOT𐞁ḋ", "̣<|", "fim", "_prefix", "|>", "e", "½", "İ", "\u000b", "½", "a", "̊"]} +{"text": "ſe​", "tokens": 4, "pieces": ["ſe", "​"]} +{"text": "'Re😀🏽'\r😀🏽12345678\tß\r\n\r\n　<|fim_prefix|>́ſ<|endoftext|>#$%('s\r\n\r\n,!!ꟲ𐞁👍🏽­́ß ", "tokens": 61, "pieces": ["'Re", "😀🏽'\r", "😀🏽", "123", "456", "78", "\tß", "\r\n\r\n", "　", "<|", "fim", "_prefix", "|>́", "ſ", "<|", "endoftext", "|>#$%('", "s", "\r\n\r\n", ",!!", "ꟲ𐞁", "👍🏽­́", "ß", " "]} +{"text": "'D👍🏽́'s'", "s", "#$%m", "tokens": 55, "pieces": ["\u000b", "#$%<", "d字", "́'", "ſß", "m"]} +{"text": "å 's　\"<|fim_prefix|>12345678'll. \n -　\u000bé½ß'Dſ\"!!ꟲm😀🏽'Rea12345678Ⅳ", "tokens": 48, "pieces": ["a", "̊", " ", " '", "s", "　", "\"<|", "fim", "_prefix", "|>", "123", "456", "78", "'ll", ".", " \n", " -", "　", "\u000bé", "½", "ß", "'D", "ſ", "\"!!", "ꟲm", "😀🏽'", "Rea", "123", "456", "78Ⅳ"]} +{"text": "'ḍ̇ ३'ReＡ३\r\n
s( .​9
-\n½…½­😀🏽'D(fi", "tokens": 42, "pieces": ["'ḋ", "̣", " ", " ", "३", "'Re", "Ａ", "३", "\r\n", "
s", "(", " ", " .​", "9", "
", "-\n", "½", "…", "½", "­😀🏽'", "D", "(fi"]} +{"text": "'Mḍ̇s(sfiZ٣٤٥٦d'D‍漢\r\n\r\n!12345678३EOT'S0́㋿漢　½", "tokens": 44, "pieces": ["'M", "ḋ", "̣s", "(sfiZ", "٣٤٥", "٦", "d", "'D", "‍漢", "\r\n\r\n", "!", "123", "456", "78३", "EOT", "'S", "0", "́㋿", "漢", "　", "½"]} +{"text": "漢! \r\ń😀🏽'😀🏽 'SZ\"t,ع'ſ𐞁<|endoftext|>Z#$%\"", "tokens": 42, "pieces": ["漢", "!", " \r\n", "́😀🏽'😀🏽", " <", "META", "_START", ">'", "SZ", "\"t", ",ع", "'ſ", "𐞁", "<|", "endoftext", "|>", "Z", "#$%\""]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ſ
s👍🏽㍿­å<|fim_prefix|>ßa", "tokens": 30, "pieces": ["ſ", "
s", "👍🏽㍿­", "a", "̊<|", "fim", "_prefix", "|>", "ß", "a"]} +{"text": "Ⅳſİ'S'll'D'llé-ß'S ​́­m", "tokens": 18, "pieces": ["Ⅳ", "ſİ", "'S", "'ll", "'D", "'ll", "e", "́-", "ß", "'S", " ​́­", "m"]} +{"text": "0EOT-३s𐞁 \nḍ̇'M\r\n㋿fi!𐞁a'D''se<|fim_prefix|>\"İ<|fim_prefix|>Ａ­ḍ̇", "tokens": 57, "pieces": ["0", "EOT", "-", "३", "s𐞁", " \n", "ḋ", "̣'", "M", "\r\n", "㋿fi", "!𐞁a", "'D", "''", "se", "<|", "fim", "_prefix", "|>\"", "İ", "<|", "fim", "_prefix", "|>", "Ａ", "­ḋ", "̣"]} +{"text": "­\neḍ̇fi\"s.'S漢  'VE😀🏽é\r\n\r\n!!s Z-ع(mé\r\ń's.　t३!", "tokens": 44, "pieces": ["­\n", "eḋ", "̣fi", "\"s", ".'", "S漢", " ", " ", "'VE", "😀🏽", "é", "\r\n\r\n", "!!", "s", " ", " Z", "-ع", "(mé", "\r\n", "́'", "s", ".", "　t", "३", "!"]} +{"text": "漢#$%9fiⅣ½㍿s😀🏽fi\r", "tokens": 22, "pieces": ["漢", "#$%", "9", "fi", "Ⅳ½", "㍿s", "😀🏽", "fi", "\r"]} +{"text": "<|endoftext|>\r\n\r\nꟲ\u000b'VE\n…Ⅳ𐞁 \n👍🏽9'll12345678​", "tokens": 41, "pieces": ["<|", "endoftext", "|>\r\n\r\n", "ꟲ", "\u000b", "'VE", "\n", "", "…", "Ⅳ", "𐞁", " \n", "👍🏽", "9", "'ll", "123", "456", "78", "​"]} +{"text": "'sDž'Re", "tokens": 7, "pieces": ["'s", "Dž", "'", "Re"]} +{"text": "🙂 ­'M d'ſ 😀🏽㍿eå<0'll\"'Dm", "tokens": 29, "pieces": ["🙂", " ", " ­'", "M", " d", "'ſ", " 😀🏽㍿", "e", "a", "̊<", "0", "'ll", "\"'", "Dm"]} +{"text": "\n…'Re'll👍🏽maé٣٤٥٦åİ'VE㋿🙂٣٤٥٦<|endoftext|>ée'Re\t\r\n!'Mfi
字🙂", "tokens": 74, "pieces": ["e", "́'", "ll", "'ll", "eİ", " <|", "fim", "_prefix", "|>", "…", "'Re", "'ll", "👍🏽", "mae", "́", "٣٤٥", "٦", "a", "̊İ", "'VE", "㋿🙂", "٣٤٥", "٦", "<|", "endoftext", "|>", "e", "́e", "'Re", "\t\r\n", "!'", "Mfi", "
字", "🙂"]} +{"text": "Džå0‍.'M\u000b'Se‍漢!!'T\u000bⅣ́", "tokens": 28, "pieces": ["Dža", "̊", "0", "‍.'", "M", "\u000b", "'S", "e", "‍漢", "!!'", "T", "\u000b", "Ⅳ", "́"]} +{"text": "Z\"", "tokens": 2, "pieces": ["Z", "\""]} +{"text": "aſ'ſ(e>'T Z12345678
<'Reé👍🏽s!!٣٤٥٦'D\r\ns👍🏽dDž\"ß'ع\rꟲa", "tokens": 62, "pieces": ["aſ", "'ſ", "(e", ">'", "T", " Z", "123", "456", "78", "
", "<'", "Reé", "👍🏽", "s", "!!", "٣٤٥", "٦", "'D", "\r\n", "s", "👍🏽", "d", "Dž", "\"ß", "'ع", "\r", "ꟲa"]} +{"text": ".t(ta'M‍😀🏽'ré🙂'Sé12345678　é", "tokens": 28, "pieces": [".t", "(ta", "'", "M", "‍😀🏽'", "re", "́🙂'", "Se", "́", "123", "456", "78", "　é"]} +{"text": "'M å​'ſ漢é!", "tokens": 13, "pieces": ["'M", " a", "̊​'", "ſ漢é", "!"]} +{"text": "e…d\r\n\r\n½　'S'M‍\"\tİe- \n!!'re३(\n字 ​fi 's ,\n9ꟲ­", "tokens": 39, "pieces": ["e", "…d", "\r\n\r\n", "½", "　", "'S", "'M", "‍\"", "\tİe", "-", " \n", "!!'", "re", "३", "(<", "META", "_START", ">\n", "字", " ", "​fi", " ", "'s", " ,\n", "9", "ꟲ", "­"]} +{"text": "#$%\r \n½ a'S­'D", "tokens": 11, "pieces": ["#$%\r", " \n", "½", " a", "'S", "­'", "D"]} +{"text": "🙂aß㍿<|fim_prefix|>'VE 𐞁#$%ſ'll#$%\r\nＡ<|endoftext|>", "tokens": 37, "pieces": ["🙂aß", "㍿<|", "fim", "_prefix", "|>'", "VE", " ", " 𐞁", "#$%", "ſ", "'ll", "#$%\r\n", "Ａ", "<|", "endoftext", "|>"]} +{"text": "𐞁 å!ae(\r\ns\t字'M'll'TＡ😀🏽٣٤٥٦ع\t́😀🏽éZİ\u000b", "tokens": 44, "pieces": ["𐞁", " a", "̊!", "ae", "(\r\n", "s", "\t字", "'M", "'ll", "'T", "Ａ", "😀🏽", "٣٤٥", "٦", "ع", "\t", "́😀🏽", "e", "́Zİ", "\u000b"]} +{"text": "0m٣٤٥٦Ⅳ'T", "tokens": 13, "pieces": ["0", "m", "٣٤٥", "٦Ⅳ", "'T"]} +{"text": "å'M", "tokens": 5, "pieces": ["a", "̊'", "M"]} +{"text": "å'Ret'Retꟲ𐞁
m🙂🙂\n𐞁e", "tokens": 27, "pieces": ["a", "̊'", "Ret", "'Re", "tꟲ𐞁", "
m", "🙂🙂\n", "𐞁e"]} +{"text": "'re're>åſ㍿​\u000bé'M0'M\r", "tokens": 19, "pieces": ["'re", "'re", ">a", "̊ſ", "㍿​", "\u000be", "́'", "M", "0", "'M", "\r"]} +{"text": "e　\u000b'S.a", "tokens": 5, "pieces": ["e", "　", "\u000b", "'S", ".a"]} +{"text": " <  \n漢d're​eİe\"\t㋿
ع́\t(", "tokens": 23, "pieces": [" ", "<", "  \n", "漢d", "'re", "​eİe", "\"", "\t", "㋿", "
ع", "́", "\t", "("]} +{"text": "!!'s'S'Tt", "tokens": 9, "pieces": ["!!'", "s", "'S", "'T", "t"]} +{"text": ",'ſḍ̇<|fim_prefix|>d é字'ſ㋿㋿e", "tokens": 37, "pieces": [",'", "ſḋ", "̣<|", "fim", "_prefix", "|>", "d", " ", " é", "字", "'ſ", "㋿<", "META", "_START", ">㋿", "e"]} +{"text": "Dž 'D𐞁
👍🏽#$%𐞁!٣٤٥٦Z!!t", "tokens": 37, "pieces": ["Dž", " '", "D𐞁", "
", "👍🏽#$%", "𐞁", "!<", "EOT", ">", "٣٤٥", "٦", "Z", "!!", "t"]} +{"text": "\t,Ⅳ", "tokens": 4, "pieces": ["\t", ",", "Ⅳ"]} +{"text": " ​'re \nعt𐞁'Dta \nåé'D漢👍🏽så㍿́DžDžt Z'S< ", "tokens": 49, "pieces": [" ", " ​'", "re", " \n", "عt𐞁", "'D", "ta", " \n", "a", "̊e", "́'", "D漢", "👍🏽", "sa", "̊㍿́", "DžDžt", " ", " Z", "'S", "<", " "]} +{"text": "<|fim_prefix|>…'s.𐞁عfi字 >\r\n're", "tokens": 25, "pieces": ["<|", "fim", "_prefix", "|>", "…", "'s", ".𐞁عfi字", "", " ", " >\r\n", "'re"]} +{"text": "'VE9👍🏽", "tokens": 9, "pieces": ["'VE", "9", "👍🏽"]} +{"text": "Ⅳfi(\"'D\r\nİ>Ａ漢😀🏽'S　.mEOTZ
……!!9字ḍ̇𐞁<ꟲ\r\n<", "tokens": 56, "pieces": ["Ⅳ", "fi", "(\"'", "D", "\r\n", "İ", ">Ａ漢", "😀🏽'", "S", "　", ".mEOTZ", "
", "", "…", "…", "!!", "9", "字ḋ", "̣𐞁", "<ꟲ", "\r\n", "<"]} +{"text": "'M#$%٣٤٥٦(#$%.m㍿㋿9#$%'T㋿\r\n\r\n!!Ａḍ̇", "tokens": 37, "pieces": ["'M", "#$%", "٣٤٥", "٦", "(#$%.", "m", "㍿㋿", "9", "#$%'", "T", "㋿\r\n\r\n", "!!", "Ａḋ", "̣"]} +{"text": "'s<|fim_prefix|>'ReZ😀🏽㋿漢s𐞁Ⅳḍ̇t İ ́e👍🏽…ḍ̇fi \n　漢ß'VE", "tokens": 60, "pieces": ["'s", "<|", "fim", "_prefix", "|>'", "ReZ", "😀🏽㋿", "漢s𐞁", "Ⅳ", "ḋ", "̣t", " ", " İ", " ́", "e", "👍🏽", "…ḋ", "̣fi", " \n", "　漢ß", "'VE"]} +{"text": "'re'Dİ字9​.🙂​㋿\t-
İé12345678‍
<|endoftext|>㍿12345678👍🏽\r㍿Zfi'VE>　é", "tokens": 62, "pieces": ["'re", "'D", "İ字", "9", "​.🙂​㋿", "\t", "-", "
İé", "123", "456", "78", "‍", "
", "<|", "endoftext", "|>㍿", "123", "456", "78", "👍🏽\r", "㍿Zfi", "'VE", ">", "　e", "́<", "EOT", ">"]} +{"text": "0 \n­  …'re漢­'T\r٣٤٥٦9s9d. 're\n'sⅣ\r\n'T<|fim_prefix|>", "tokens": 47, "pieces": ["0", " \n", "­", "  ", "…", "'re", "漢", "­'", "T", "\r", "٣٤٥", "٦9", "s", "", "9", "d", ".", " '", "re", "\n", "'s", "Ⅳ", "\r\n", "'T", "<|", "fim", "_prefix", "|>"]} +{"text": "🙂㋿'re字'll,s\r½　ḍ̇𐞁漢'D<|fim_prefix|>٣٤٥٦٣٤٥٦'re'Re,👍🏽!!­\r9", "tokens": 64, "pieces": ["🙂㋿'", "re字", "'ll", ",<", "EOT", ">s", "\r", "½", "　ḋ", "̣𐞁漢", "'D", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦٣٤", "٥٦", "'re", "'Re", ",👍🏽!!­\r", "9"]} +{"text": "'VE", "tokens": 2, "pieces": ["'VE"]} +{"text": "
ß 's'Re#$% \r\n㍿́m 'M12345678!!
🙂<|fim_prefix|>ß>é'M'D!!DžݽEOT", "tokens": 36, "pieces": ["!ß", "🙂", "٣٤٥", "٦", "!", "
", "'T", "'M", "ß", ">e", "́'", "M", "'D", "!!", "Džİ", "", "½", "EOT"]} +{"text": "mZas　!Ⅳ>
'S㍿0\r­ꟲ,#$%漢'T'll
-½'ſ'D!!Zté\"漢9字字's
½", "tokens": 49, "pieces": ["mZas", "　", "!", "Ⅳ", ">", "
", "'S", "㍿", "0", "\r", "­ꟲ", ",#$%", "漢", "'T", "'ll", "
", "-", "½", "'ſ", "'D", "!!", "Zté", "\"漢", "9", "字字", "'s", "
", "½"]} +{"text": "<'Re­ 'T㋿'VE0 \nEOT\n　mééİ.\r\n\r\né<|endoftext|>å<|endoftext|>🙂!12345678\"İ'S-Dž", "tokens": 52, "pieces": ["<'", "Re", "­", " ", " '", "T", "㋿'", "VE", "0", " \n", "EOT", "\n", "　mééİ", ".\r\n\r\n", "é", "<|", "endoftext", "|>", "a", "̊<|", "endoftext", "|>🙂!", "123", "456", "78", "\"İ", "'S", "-Dž"]} +{"text": "­'Tå'D're\r\n\r\n-é,12345678.<,٣٤٥٦ſ٣٤٥٦\t́\"'VE½m\u000bꟲ", "tokens": 51, "pieces": ["­'", "T", "a", "̊'", "D", "'re", "\r\n\r\n", "-e", "́,", "123", "456", "78", ".<,", "٣٤٥", "٦", "ſ", "", "٣٤٥", "٦", "\t", "́\"'", "VE", "½", "m", "\u000bꟲ"]} +{"text": "'M🙂12345678e\r\n9,12345678İ'reعＡ", "tokens": 18, "pieces": ["'M", "🙂", "123", "456", "78", "e", "\r\n", "9", ",", "123", "456", "78", "İ", "'re", "عＡ"]} +{"text": " \"-m'३㍿字🙂 \tꟲ‍ḍ̇éḍ̇#$%­å,'s", "tokens": 45, "pieces": [" ", " \"-", "m", "'<", "EOT", ">", "३", "㍿字", "🙂", " ", "\tꟲ", "‍", "ḋ", "̣éḋ", "̣#$%­", "a", "̊,'", "s"]} +{"text": "㍿\rEOTḍ̇'S ßİ​d,½٣٤٥٦Ⅳ' ٣٤٥٦漢'Dé　'S𐞁ḍ̇mfi\u000b\rm'Re ½<|fim_prefix|><|endoftext|>> \n'", "tokens": 87, "pieces": ["㍿\r", "EOTḋ", "̣'", "S", " ßİ", "​d", ",<", "EOT", ">", "½٣٤", "٥٦Ⅳ", "'", " ", " ", "٣٤٥", "٦", "漢", "'D", "é", "　", "'S", "𐞁ḋ", "̣mfi", "\u000b\r", "m", "'Re", " ", " ", "½", "<|", "fim", "_prefix", "|><|", "endoftext", "|>>", " \n", "'"]} +{"text": "\r\n\r\né<　(ḍ̇ ", "tokens": 12, "pieces": ["\r\n\r\n", "e", "́<", "　", "(ḋ", "̣", " "]} +{"text": "t,!㍿😀🏽éé", "tokens": 22, "pieces": ["t", "Ⅳ", "İ𐞁", "'T", "Ａ", ".🙂'", "ll", "Ⅳ", "́>", "éé"]} +{"text": "é́٣٤٥٦Džmſ\r𐞁<|fim_prefix|>­s\r\n9ß \n å'Re…", "tokens": 40, "pieces": ["é", "́", "٣٤٥", "٦", "Džmſ", "\r", "𐞁", "<|", "fim", "_prefix", "|>­", "s", "\r\n", "9", "ß", " \n", " a", "̊'", "Re", "…"]} +{"text": "!!­d(ſe<|endoftext|>Ⅳſ'D\r\n\r\n#$%dd漢㋿(ع(漢EOT.'s½字", "tokens": 39, "pieces": ["!!­", "d", "(ſe", "<|", "endoftext", "|>", "Ⅳ", "ſ", "'D", "\r\n\r\n", "#$%", "dd漢", "㋿(", "ع", "(漢EOT", ".'", "s", "½", "字"]} +{"text": "ꟲ'Re fi漢!!å>'D", "tokens": 19, "pieces": ["ꟲ", "'Re", " ", " fi漢", "!!", "a", "̊>'", "D"]} +{"text": "12345678­ꟲ!! 漢́'reéZ's", "tokens": 22, "pieces": ["123", "456", "78", "­ꟲ", "!!", " 漢", "́'", "reéZ", "'s", ""]} +{"text": "́éé.㋿İé३​><…漢­🙂'VE\t \n\n!Ⅳ", "tokens": 30, "pieces": ["́e", "́e", "́.㋿", "İe", "́", "३", "​><", "…漢", "­🙂'", "VE", "\t \n\n", "!", "Ⅳ"]} +{"text": "eⅣ😀🏽'D're३'s's9-0!", "tokens": 19, "pieces": ["e", "Ⅳ", "😀🏽'", "D", "'re", "३", "'s", "'s", "9", "-", "0", "!"]} +{"text": "0 \n(½å<|fim_prefix|> m عd'llé!Z字EOTå😀🏽㍿é.İ　Ａ \nfi
.'ßfi​Z'VEfi \n", "tokens": 60, "pieces": ["0", " \n", "(", "½", "a", "̊<|", "fim", "_prefix", "|>", " m", " عd", "'ll", "e", "́!", "Z字EOTa", "̊😀🏽㍿", "e", "́.", "İ", "　Ａ", " \n", "fi", "
", ".'", "ßfi", "​Z", "'VE", "fi", " \n"]} +{"text": "<|endoftext|> \n…Ａİ‍ ,", "tokens": 17, "pieces": ["<|", "endoftext", "|>", " \n", "…Ａİ", "‍", " ", " ,"]} +{"text": "a'S!", "tokens": 3, "pieces": ["a", "'S", "!"]} +{"text": "👍🏽\r\n\r\n<|fim_prefix|>字'M.ḍ̇ꟲ", "tokens": 25, "pieces": ["👍🏽\r\n\r\n", "<|", "fim", "_prefix", "|>", "字", "'M", ".ḋ", "̣ꟲ"]} +{"text": " ½字\nm\r\n\r\n'T<>>mſaå㋿ 'T!𐞁ع<|endoftext|>\r\n\r\na're,123456789'S \n-", "tokens": 47, "pieces": ["", " ", "½", "字", "\n", "m", "\r\n\r\n", "'T", "<>>", "mſaa", "̊㋿", " ", "'T", "!𐞁ع", "<|", "endoftext", "|>\r\n\r\n", "a", "'re", ",", "123", "456", "789", "'S", " \n", "-"]} +{"text": "'Re<|endoftext|>Dž\n٣٤٥٦ḍ̇ta!fi…a٣٤٥٦　-Ⅳ😀🏽é\n#$%漢👍🏽  　😀🏽'T'Mſ'ſé", "tokens": 81, "pieces": ["'Re", "<|", "endoftext", "|>", "Dž", "\n", "٣٤٥", "٦", "ḋ", "̣ta", "!fi", "…a", "٣٤٥", "٦", "　", "-", "Ⅳ", "😀🏽", "é", "\n", "#$%", "漢", "👍🏽", "  ", "　", "😀🏽'", "T", "'", "Mſ", "'ſ", "é"]} +{"text": "́㍿'Re9d#$%#$%ꟲＡꟲ<|fim_prefix|>\r\n\n<|endoftext|>EOTZ㍿m", "tokens": 42, "pieces": ["́㍿'", "Re", "9", "d", "#$%#$%", "ꟲＡꟲ", "<|", "fim", "_prefix", "|>\r\n\n", "<|", "endoftext", "|>", "EOTZ", "㍿m"]} +{"text": "'Tt㍿😀🏽efi'T12345678sİ 'S​(​'S'T", "tokens": 27, "pieces": ["'T", "t", "㍿😀🏽", "efi", "'T", "123", "456", "78", "sİ", " '", "S", "​(​'", "S", "'T"]} +{"text": "9'<ⅣⅣ 0'll　d'ſDž", "tokens": 20, "pieces": ["9", "'<", "ⅣⅣ", " ", " ", "0", "'", "ll", "　d", "'ſ", "Dž"]} +{"text": "\r\n.d<|fim_prefix|>字­,ꟲꟲ!EOT's", "tokens": 22, "pieces": ["\r\n", ".d", "<|", "fim", "_prefix", "|>", "字", "­,", "ꟲꟲ", "!EOT", "'s"]} +{"text": "ſ.'ſ<|endoftext|>👍🏽EOT👍🏽Ａ Ⅳ👍🏽9'ſ　å…\"m'M>\u000b.<'VEm", "tokens": 61, "pieces": ["ſ", ".'", "ſ", "<|", "endoftext", "|>👍🏽", "EOT", "👍🏽", "Ａ", " ", "Ⅳ", "👍🏽", "9", "'ſ", "　", "<", "EOT", ">a", "̊", "…", "\"m", "'M", ">", "\u000b", ".<'", "VEm"]} +{"text": " 'll12345678'D\u000bſ\t'Re𐞁…㍿e'ſ­EOT𐞁.Ａ'ReEOT\n", "tokens": 43, "pieces": [" ", " '", "ll", "123", "456", "78", "'", "D", "\u000bſ", "\t", "'Re", "𐞁", "…", "㍿e", "'ſ", "­EOT𐞁", ".Ａ", "'Re", "EOT", "\n"]} +{"text": "s३漢㍿\r\n\r\nꟲع
DžEOT\"ſ३ḍ̇mDž", "tokens": 35, "pieces": ["s", "३", "漢", "㍿\r\n\r\n", "ꟲع", "
DžEOT", "\"ſ", "३", "ḋ", "̣m", "Dž"]} +{"text": " ſ…'MåDžḍ̇\t'Re,<|endoftext|>ⅣＡ字 \n İfi.\tétſsꟲ<|endoftext|>㍿>ꟲ", "tokens": 59, "pieces": [" ", " ſ", "…", "'M", "a", "̊Džḋ", "̣", "\t", "'Re", ",<|", "endoftext", "|>", "Ⅳ", "Ａ字", " \n", " ", " İfi", ".", "\te", "́tſsꟲ", "<|", "endoftext", "|>㍿>", "ꟲ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "<|endoftext|> ", "tokens": 8, "pieces": ["<|", "endoftext", "|>", " "]} +{"text": "12345678're'½'ſ😀🏽漢‍(ſ m𐞁漢->Ⅳ😀🏽'ſ(字\u000bm9Džعḍ̇漢ḍ̇ \n½ß ", "tokens": 63, "pieces": ["123", "456", "78", "'re", "'", "½", "'ſ", "😀🏽", "漢", "‍(", "ſ", " m𐞁漢", "->", "Ⅳ", "😀🏽'", "ſ", "(字", "\u000bm", "9", "Džعḋ", "̣漢ḋ", "̣", " \n", "½", "ß", " "]} +{"text": "'Re -e𐞁EOT9'VE<|endoftext|>\r'D㋿e!\r\n😀🏽‍́ḍ̇…ع字é<|fim_prefix|>'ſ", "tokens": 61, "pieces": ["'Re", " ", "-e", "<", "META", "_START", ">𐞁EOT", "9", "'VE", "<|", "endoftext", "|>\r", "'D", "㋿e", "!\r\n", "😀🏽‍́", "ḋ", "̣", "…ع字é", "<|", "fim", "_prefix", "|>'", "ſ"]} +{"text": "<|endoftext|>>a#$%dZEOT m\naع", "tokens": 17, "pieces": ["<|", "endoftext", "|>>", "a", "#$%", "dZEOT", " m", "\n", "aع"]} +{"text": "🙂‍DžDž㍿字\u000b's字'é're", "tokens": 19, "pieces": ["🙂‍", "DžDž", "㍿字", "\u000b", "'s", "字", "'e", "́'", "re"]} +{"text": "ع,<|fim_prefix|>'e \n>ḍ̇\u000bd'D12345678're a#$%…Dž𐞁\nå㍿\u000bⅣ!!\"½\n\"​#$%​Ⅳ½́́t", "tokens": 60, "pieces": ["ع", ",<|", "fim", "_prefix", "|>'", "e", " \n", ">ḋ", "̣", "\u000bd", "'D", "123", "456", "78", "'re", " ", " a", "#$%", "…Dž𐞁", "\n", "a", "̊㍿", "\u000b", "Ⅳ", "!!\"", "½", "\n", "\"​#$%​", "Ⅳ½", "́́", "t"]} +{"text": "-.'Sfi🙂9İ­#$%Ⅳ9\"m\r'll३s\r\n\r\nfiZe३'ll", "tokens": 37, "pieces": ["-<", "META", "_START", ">.'", "Sfi", "🙂", "9", "İ", "­#$%", "Ⅳ9", "\"m", "\r", "'", "ll", "३", "s", "\r\n\r\n", "fiZe", "३", "'ll"]} +{"text": "​\rＡ(٣٤٥٦d👍🏽<ع'DعＡé'Re\r\n\r\n \t'ſt#$%Z\"Dž‍.'Sa\r　𐞁 \n\r\n", "tokens": 55, "pieces": ["​\r", "Ａ", "(", "٣٤٥", "٦", "d", "👍🏽<", "ع", "'D", "عＡe", "́'", "Re", "\r\n\r\n", " ", "\t", "'ſ", "t", "#$%", "Z", "\"Dž", "‍.'", "Sa", "\r", "　𐞁", " \n\r\n"]} +{"text": "ꟲ,'M", "tokens": 5, "pieces": ["ꟲ", ",'", "M"]} +{"text": "㋿　fi,ع'll12345678'reé\r\n,\ns'", "tokens": 19, "pieces": ["㋿", "　fi", ",ع", "'ll", "123", "456", "78", "'re", "e", "́\r\n", ",\n", "s", "'"]} +{"text": "'re-Ⅳ\"ḍ̇ع!!s", "tokens": 13, "pieces": ["'re", "-", "Ⅳ", "\"ḋ", "̣ع", "!!", "s"]} +{"text": "'lléꟲ\n12345678ḍ̇Ⅳ0㍿'VE'D12345678e㍿ß㋿'sß-½s\u000b\t'll…#$%\r Z'TtEOT 𐞁,ꟲd", "tokens": 67, "pieces": ["'ll", "e", "́ꟲ", "\n", "123", "456", "78", "ḋ", "̣", "Ⅳ0", "㍿'", "VE", "'D", "123", "456", "78", "e", "㍿ß", "㋿'", "sß", "-", "½", "s", "\u000b", "\t", "'ll", "…", "#$%\r", " <", "EOT", ">Z", "'T", "tEOT", " 𐞁", ",ꟲd"]} +{"text": "\u000b
åZ‍ \r\n\r\n\r\n\r\nEOT \n‍tع…'Re\r\n!ⅣſⅣ!!😀🏽9<(\r\n>'​fi\u000b🙂", "tokens": 45, "pieces": ["\u000b", "
a", "̊Z", "‍", " \r\n\r\n\r\n\r\n", "EOT", " \n", "‍tع", "…", "'Re", "\r\n", "!", "Ⅳ", "ſ", "Ⅳ", "!!😀🏽", "9", "<(\r\n", ">'​", "fi", "\u000b", "🙂"]} +{"text": " …३\r'M
!漢- 
12345678t<|endoftext|>
'D,\na'Red ٣٤٥٦漢\"é㋿ s‍>\r\n\r\ns9DžZ½😀🏽", "tokens": 66, "pieces": [" ", "…", "३", "\r", "'M", "
", "!漢", "-", " ", "
", "123", "456", "78", "t", "<|", "endoftext", "|>", "
", "'D", ",\n", "a", "'Re", "d", " ", "٣٤٥", "٦", "漢", "\"e", "́㋿", " s", "‍>\r\n\r\n", "s", "9", "DžZ", "½", "😀🏽"]} +{"text": "'ll𐞁\r…'ll!Ⅳ-<|endoftext|>ß \r\n\r\n\r\n\r\nét㋿\n'Tt12345678", "tokens": 35, "pieces": ["'ll", "𐞁", "\r", "…", "'ll", "!", "Ⅳ", "-<|", "endoftext", "|>", "ß", " \r\n\r\n\r\n\r\n", "e", "́t", "㋿\n", "'T", "t", "123", "456", "78"]} +{"text": "'s\u000b-'s‍EOT#$%ſ\u000bßEOT\u000b(åعfi", "tokens": 23, "pieces": ["'s", "\u000b", "-'", "s", "‍EOT", "#$%", "ſ", "\u000bßEOT", "\u000b", "(a", "̊عfi"]} +{"text": "\t< '漢e漢12345678'Re'D'D912345678\u000b éad　ḍ̇İmm!!12345678३㋿\r\n", "tokens": 41, "pieces": ["\t", "<", " ", " '", "漢e漢", "123", "456", "78", "'Re", "'D", "'D", "912", "345", "678", "\u000b ", " e", "́ad", "　ḋ", "̣İmm", "!!", "123", "456", "78३", "㋿\r\n"]} +{"text": "Z09٣٤٥٦0é'T're 'T", "tokens": 16, "pieces": ["Z", "09٣", "٤٥٦", "0", "é", "'T", "'re", " ", "'T"]} +{"text": "㍿٣٤٥٦\nt㍿字\r\n\r\n漢٣٤٥٦\n<'re‍>\t㍿㍿Ⅳ\r\n!'VE.9e'.ع", "tokens": 51, "pieces": ["㍿", "٣٤٥", "٦", "\n", "t", "㍿字", "\r\n\r\n", "漢", "٣٤٥", "٦", "\n", "<'", "re", "‍>", "\t", "㍿㍿", "Ⅳ", "\r\n", "!'", "VE", ".", "9", "e", "'.", "ع"]} +{"text": "!! ‍é½३Ａ'VEſ<|fim_prefix|>ع​é😀🏽'T\r<|fim_prefix|> d!!a'll٣٤٥٦½Ａ<字s😀🏽#$%d½㋿e#$%", "tokens": 77, "pieces": ["!!", " ", "‍é", "½३", "Ａ", "'VE", "ſ", "<|", "fim", "_prefix", "|>", "ع", "​e", "́😀🏽'", "T", "\r", "<|", "fim", "_prefix", "|>", " d", "!!", "a", "'", "ll", "٣٤٥", "٦½", "Ａ", "<字s", "😀🏽#$%", "d", "½", "㋿e", "#$%"]} +{"text": "ع​Ａ(t12345678>'S'T😀🏽a'Re٣٤٥٦\r\nd", "tokens": 34, "pieces": ["ع", "​Ａ", "(<", "META", "_START", ">t", "123", "456", "78", ">'", "S", "'T", "😀🏽", "a", "'Re", "٣٤٥", "٦", "\r\n", "d"]} +{"text": "!!\"
#$%\"'lléß
ß　12345678->t👍🏽\r\n\r\n're漢　‍<|fim_prefix|>d0 \n 0's㋿e#$%0😀🏽", "tokens": 65, "pieces": ["!!\"", "
", "#$%\"'", "lléß", "
ß", "　", "123", "456", "78", "->", "t", "👍🏽\r\n\r\n", "'re", "漢", "　", "‍<|", "fim", "_prefix", "|>", "d", "0", " \n", " ", "0", "'s", "㋿e", "#$%<", "META", "_START", ">", "0", "😀🏽"]} +{"text": "'Da<३éſd'Ret'Re𐞁字字'DEOT\r\n\r\nEOT\"½'S­ -\r.tå\r\n\r\n​​ع㋿😀🏽", "tokens": 46, "pieces": ["'D", "a", "<", "३", "e", "́ſd", "'Re", "t", "'Re", "𐞁字字", "'D", "EOT", "\r\n\r\n", "EOT", "\"", "½", "'S", "­", " ", "-\r", ".ta", "̊\r\n\r\n", "​​", "ع", "㋿😀🏽"]} +{"text": "\r\n'llＡ", "tokens": 4, "pieces": ["\r\n", "'ll", "Ａ"]} +{"text": ">🙂\"'Rea", "tokens": 6, "pieces": [">🙂\"'", "Rea"]} +{"text": "'D'ſ'٣٤٥٦ <\u000b漢㋿0'llＡ.\t'D🙂'T٣٤٥٦ ٣٤٥٦-<|endoftext|>!! <|fim_prefix|><|endoftext|>字Džé㍿ع\r\nd\u000b're㍿​ḍ̇", "tokens": 95, "pieces": ["'D", "'ſ", "'", "٣٤٥", "٦", " ", " <", "\u000b漢", "㋿", "0", "'ll", "Ａ", ".", "\t", "'D", "🙂'", "T", "٣٤٥", "٦", " ", " ", "٣٤٥", "٦", "-<|", "endoftext", "|>!!", " ", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "字Dže", "́㍿", "ع", "\r\n", "d", "\u000b", "'re", "㍿​", "ḋ", "̣"]} +{"text": "'　字'D٣٤٥٦dſZ \u000b­　字é'reéé
ḍ̇fi😀🏽İḍ̇é‍३md𐞁<|endoftext|>­👍🏽​'VE'‍\r", "tokens": 80, "pieces": ["'", "　字", "'D", "٣٤٥", "٦", "dſZ", " ", "\u000b", "­", "　字é", "'re", "e", "́é", "
", "ḋ", "̣fi", "😀🏽", "İḋ", "̣é", "‍", "३", "md𐞁", "<|", "endoftext", "|>­👍🏽​'", "VE", "'‍\r"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "😀🏽 ㍿'ſ😀🏽👍🏽ḍ̇'Dİ'S字\r9😀🏽漢字fi<|endoftext|>s(\t­\r(", "tokens": 58, "pieces": ["😀🏽", " ", "㍿'", "ſ", "😀🏽👍🏽", "ḋ", "̣'", "Dİ", "'S", "字", "\r", "9", "😀🏽", "漢字fi", "<|", "endoftext", "|>", "s", "(", "\t", "­\r", "("]} +{"text": "åm½s\"'Re'D字\r\n''ſ字ع0s12345678😀🏽Ａ\u000b<|endoftext|>\"
EOT're", "tokens": 41, "pieces": ["a", "̊m", "½", "s", "\"'", "Re", "'D", "字", "\r\n", "''", "ſ字ع", "0", "s", "123", "456", "78", "😀🏽", "Ａ", "\u000b", "<|", "endoftext", "|>\"", "
EOT", "'re"]} +{"text": " \ń👍🏽1234567812345678", "tokens": 14, "pieces": [" \n", "́👍🏽", "123", "456", "781", "234", "567", "8"]} +{"text": "\r\"\"é\r\n\r\n#$%😀🏽-𐞁…\r\n de…t­s>EOT!!", "tokens": 29, "pieces": ["\r", "\"\"", "e", "́\r\n\r\n", "#$%😀🏽-", "𐞁", "…\r\n", " ", " de", "…t", "­s", ">EOT", "!!"]} +{"text": "́ḍ̇", "tokens": 6, "pieces": ["́ḋ", "̣"]} +{"text": "''字9ꟲ 'VE\u000b😀🏽\n'ſ!0Zß٣٤٥٦m　😀🏽<<|fim_prefix|>字å 'Re\r\n३'T'D<", "字a", "̊", " ", " '", "Re", "\r\n", "३", "'T", "'D", "<<", "d", "👍🏽", "İ", "…", "("]} +{"text": "ع<", "tokens": 2, "pieces": ["ع", "<"]} +{"text": "
\t\u000bḍ̇'\"", "tokens": 10, "pieces": ["
\t", "\u000bḋ", "̣'\""]} +{"text": "\"'M३ds😀🏽'T​\r\n.\t d🙂٣٤٥٦
३'Ss\r\n\r\nfi'T\r\n\r\n\u000b", "tokens": 39, "pieces": ["\"'", "M", "३", "ds", "😀🏽'", "T", "​\r\n", ".", "\t ", " d", "🙂", "٣٤٥", "٦", "
", "३", "'S", "s", "\r\n\r\n", "fi", "'T", "\r\n\r\n\u000b"]} +{"text": "0‍'SZ(12345678㋿🙂'D12345678
\"ⅣEOT'T\r\n\r\n字<|endoftext|>#$%<|endoftext|>'D\u000b<|fim_prefix|>​　Ａ'Refi­Dž0'<ſfi-,🙂", "tokens": 73, "pieces": ["0", "‍'", "SZ", "(", "123", "456", "78", "㋿🙂'", "D", "123", "456", "78", "
", "\"", "Ⅳ", "EOT", "'T", "\r\n\r\n", "字", "<|", "endoftext", "|>#$%<|", "endoftext", "|>'", "D", "\u000b", "<|", "fim", "_prefix", "|>​", "　Ａ", "'Re", "fi", "­Dž", "0", "'<", "ſfi", "-,🙂"]} +{"text": "< \"'Re<|fim_prefix|>", "tokens": 16, "pieces": ["<", " ", "\"'", "Re", "<|", "fim", "_prefix", "|><", "META", "_START", ">"]} +{"text": "½\u000b'ſＡDž.\r­><|fim_prefix|>!!½s　ß🙂", "tokens": 26, "pieces": ["½", "\u000b", "'ſ", "ＡDž", ".\r", "­><|", "fim", "_prefix", "|>!!", "½", "s", "　ß", "🙂"]} +{"text": "0.३é!90🙂d9'VE​字 t.'S३😀🏽0ß字<\rＡ٣٤٥٦🙂字m<<|fim_prefix|>👍🏽", "tokens": 63, "pieces": ["0", ".", "३", "e", "́!", "90", "🙂d", "9", "'VE", "​字", " t", ".'", "S", "३", "😀🏽", "0", "ß字", "<\r", "Ａ", "٣٤٥", "٦", "🙂字m", "<<|", "fim", "_prefix", "|>👍🏽"]} +{"text": "ḍ̇漢'Dé​#$%㋿'re''re<>\r\né😀🏽12345678", "tokens": 32, "pieces": ["ḋ", "̣漢", "'D", "e", "́​#$%㋿'", "re", "''", "re", "<>\r\n", "e", "́😀🏽", "123", "456", "78"]} +{"text": "'ll३-té½e'…\n​ß\u000bⅣ😀🏽Ⅳ'M9!३'", "tokens": 29, "pieces": ["'ll", "३", "-te", "́", "½", "e", "'", "…\n", "​ß", "\u000b", "Ⅳ", "😀🏽", "Ⅳ", "'M", "9", "!", "३", "'"]} +{"text": "'llع😀🏽 -'s Džع99( 'Re9até9٣٤٥٦㍿漢->\"EOTéEOT½!!ßa12345678​
'VE'Re", "tokens": 59, "pieces": ["'ll", "ع", "😀🏽", " -'", "s", " Džع", "99", "(", " '", "Re", "9", "ate", "́", "9٣٤", "٥٦", "㍿漢", "->\"", "EOTéEOT", "½", "!!", "ßa", "123", "456", "78", "​", "
", "'VE", "'Re"]} +{"text": "ع'VE字", "tokens": 4, "pieces": ["ع", "'VE", "字"]} +{"text": "٣٤٥٦𐞁12345678'…​'VE𐞁(ß", "tokens": 30, "pieces": ["٣٤٥", "٦", "𐞁", "123", "456", "78", "'<", "META", "_START", ">", "…", "​'", "VE𐞁", "(ß"]} +{"text": "12345678字(İ 字😀🏽३\u000bm𐞁<|fim_prefix|>👍🏽 \"fi'T'Msꟲ́½​\"漢fi\u000b<|endoftext|>㍿fi", "tokens": 68, "pieces": ["123", "456", "78", "字", "(İ", " ", " 字", "😀🏽", "३", "\u000bm𐞁", "<|", "fim", "_prefix", "|>👍🏽", " ", " \"", "fi", "'T", "'M", "sꟲ", "́", "½", "​\"<", "EOT", ">漢fi", "\u000b", "<|", "endoftext", "|>㍿", "fi"]} +{"text": "㍿12345678­ꟲ'VE'DꟲDž'Sḍ̇ḍ̇.'T'\r\n\r\n'll\rDž \n'llZåſ<|endoftext|>𐞁.", "tokens": 56, "pieces": ["㍿", "123", "456", "78", "­ꟲ", "'VE", "'D", "ꟲDž", "'S", "ḋ", "̣ḋ", "̣.'", "T", "'\r\n\r\n", "'ll", "\r", "Dž", " \n", "'ll", "Za", "̊ſ", "<|", "endoftext", "|>", "𐞁", "."]} +{"text": "<ꟲ३é<|fim_prefix|>0(​😀🏽", "tokens": 23, "pieces": ["<ꟲ", "३", "e", "́<|", "fim", "_prefix", "|>", "0", "(​😀🏽"]} +{"text": " 's\"'ll\r'M… 'D'\t­　‍Ａ㋿\r\n\r\n \n.", "tokens": 29, "pieces": [" '", "s", "\"'", "ll", "\r", "'M", "… ", " '", "D", "'", "\t", "­", "　", "‍Ａ", "㋿\r\n\r\n", " \n", "."]} +{"text": " 'ReEOTİ́é­s𐞁'Dع٣٤٥٦'M0½㋿\n'VEa", "tokens": 32, "pieces": [" '", "ReEOTİ", "́é", "­s𐞁", "'D", "ع", "٣٤٥", "٦", "'M", "0½", "㋿\n", "'VE", "a"]} +{"text": "­\"Ａ", "tokens": 4, "pieces": ["­\"", "Ａ"]} +{"text": "!!\r\na㋿'reéDž\ré!!!!t字tḍ̇!!'DDž .\r\u000be漢,", "tokens": 43, "pieces": ["!!\r\n", "a", "㋿'", "reéDž", "\r", "é", "!!!!", "t字tḋ", "̣!!<", "m", "'", "DDž", " ", ".\r", "\u000be漢", ","]} +{"text": " 12345678!((\rm‍e ½ſ'VE 'S३\t‍.9<", "tokens": 28, "pieces": [" ", "123", "456", "78", "!((\r", "m", "‍e", " ", "½", "ſ", "'VE", " ", " '", "S", "३", "\t", "‍.", "9", "<"]} +{"text": "!<|endoftext|>12345678\ns½0<\"٣٤٥٦​
Ａ㍿ \n
İ字é​", "tokens": 38, "pieces": ["!<|", "endoftext", "|>", "123", "456", "78", "\n", "s", "½0", "<\"", "٣٤٥", "٦", "​", "
Ａ", "㍿", " \n", "
İ字é", "​"]} +{"text": "🙂ꟲ\u000bt\n
​㋿㍿.\r\"​12345678㋿字e\"'ſ漢'T\t!ſ​
're<…​", "tokens": 56, "pieces": ["🙂ꟲ", "", "\u000bt", "\n", "
", "​㋿㍿.\r", "\"​", "123", "456", "78", "㋿字e", "\"'", "ſ漢", "'T", "\t", "!ſ", "​", "
", "'re", "<", "…", "​"]} +{"text": "ßEOT½ 0½عZ　'‍‍'VEé\u000b", "tokens": 20, "pieces": ["ßEOT", "½", " ", "0½", "عZ", "　", "'‍‍'", "VEe", "́", "\u000b"]} +{"text": "åİa\t字9㋿9…<|fim_prefix|>,　‍'re…\t'12345678-'D<|fim_prefix|>漢 \n!'M　'D\t字…åe", "tokens": 57, "pieces": ["a", "̊İa", "\t字", "9", "㋿", "9", "…", "<|", "fim", "_prefix", "|>,", "　", "‍'", "re", "…", "\t", "'", "123", "456", "78", "-'", "D", "<|", "fim", "_prefix", "|>", "漢", " \n", "!'", "M", "　", "'D", "\t字", "…a", "̊e"]} +{"text": "ḍ̇\nḍ̇​٣٤٥٦", "tokens": 20, "pieces": ["ḋ", "̣\n", "ḋ", "̣​", "٣٤٥", "٦"]} +{"text": "fi ,ß\"İZ09\u000b \né\"'Reİꟲ\"ß'T́!!('ree-'S'reḍ̇Z
'VE", "tokens": 39, "pieces": ["fi", " ", " ,", "ß", "\"İZ", "09", "\u000b \n", "e", "́\"'", "Reİꟲ", "\"ß", "'T", "́!!('", "ree", "-'", "S", "'re", "ḋ", "̣Z", "
", "'VE"]} +{"text": "漢ꟲ🙂İ👍🏽\r\n\r\n字' Ａ😀🏽(ꟲ𐞁<|endoftext|>ＡZéa👍🏽٣٤٥٦\r", "tokens": 63, "pieces": ["漢ꟲ", "🙂İ", "👍🏽\r\n\r\n", "字", "'", " Ａ", "😀🏽(", "ꟲ𐞁", "<|", "endoftext", "|>", "ＡZéa", "👍🏽", "٣٤٥", "٦", "\r"]} +{"text": "12345678​
\n‍­9½­'re<|endoftext|>İ👍🏽fi'ſé½'ſ​\u000b㋿0
字é<|endoftext|>d३0Ａ㍿­٣٤٥٦12345678>EOT-🙂", "tokens": 83, "pieces": ["123", "456", "78", "​", "
\n", "‍­", "9½", "­'", "re", "<|", "endoftext", "|>", "İ", "👍🏽", "fi", "'ſ", "e", "́", "½", "'ſ", "​", "\u000b", "㋿", "0", "
字é", "<|", "endoftext", "|>", "d", "३0", "Ａ", "㍿­", "٣٤٥", "٦12", "345", "678", ">EOT", "-🙂"]} +{"text": "عt\"é<|fim_prefix|>Dž!!é㋿'VEé½\n-٣٤٥٦", "tokens": 35, "pieces": ["عt", "\"e", "́<|", "fim", "_prefix", "|>", "Dž", "!!", "e", "́㋿'", "VEe", "́", "½", "\n", "-", "٣٤٥", "٦"]} +{"text": "İ㋿'T'VE'SⅣ0EOT٣٤٥٦,.٣٤٥٦Ａ½", "tokens": 34, "pieces": ["İ", "㋿'", "T", "'VE", "'S", "Ⅳ0", "EOT", "٣٤٥", "٦", ",.", "٣٤٥", "٦", "Ａ", "½"]} +{"text": "ße'D३Z‍", "tokens": 7, "pieces": ["ße", "'D", "३", "Z", "‍"]} +{"text": "é<|fim_prefix|>9dåſ!!Z ", "tokens": 18, "pieces": ["e", "́<|", "fim", "_prefix", "|>", "9", "da", "̊ſ", "!!", "Z", " "]} +{"text": " 'SEOTDž\r\n\u000b
‍ع😀🏽\r\nع'sḍ̇å
", "tokens": 33, "pieces": [" '", "SEOTDž", "\r\n", "\u000b", "
", "‍ع", "😀🏽\r\n", "ع", "'", "sḋ", "̣a", "̊", "
"]} +{"text": "𐞁㋿ḍ̇\n　½ſḍ̇m漢", "tokens": 28, "pieces": ["𐞁", "㋿ḋ", "̣\n", "　", "½", "ſḋ", "̣m漢", ""]} +{"text": "'T३!!9'T'MⅣ\r\n\r\n🙂½.ꟲ-Z…'s0漢عع\r'Re(ع'll123456780㋿(<|endoftext|>\r\n३é", "tokens": 47, "pieces": ["'T", "३", "!!", "9", "'T", "'M", "Ⅳ", "\r\n\r\n", "🙂", "½", ".ꟲ", "-Z", "…", "'s", "0", "漢عع", "\r", "'Re", "(ع", "'ll", "123", "456", "780", "㋿(<|", "endoftext", "|>\r\n", "३", "é"]} +{"text": "(
\"
>", "tokens": 7, "pieces": ["(", "
", "\"", "
", ">"]} +{"text": "漢́\r\n\r\n\t́字字a 'Dé", "tokens": 16, "pieces": ["漢", "́\r\n\r\n", "\t", "́", "字字a", " ", " '", "Dé"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "　Dž 'refi\r'\u000b<|endoftext|>Z\t0", "tokens": 20, "pieces": ["　Dž", " '", "refi", "\r", "'", "\u000b", "<|", "endoftext", "|>", "Z", "\t", "0"]} +{"text": "'Re㋿İ-éEOTＡſa‍字\"㍿…İع​'VE'VE㋿Ａḍ̇ꟲ 's.👍🏽>\r\n\r\n३ \n\r\n…12345678ad \n㍿", "tokens": 70, "pieces": ["'", "Re", "㋿İ", "-e", "́EOTＡſa", "‍字", "\"㍿", "…İع", "​'", "VE", "'VE", "㋿Ａḋ", "̣ꟲ", " '", "s", ".👍🏽>\r\n\r\n", "३", " \n\r\n", "…", "123", "456", "78", "ad", " \n", "㍿"]} +{"text": "\",\r\n\r\nḍ̇\r\n\r\n<|endoftext|>(tİ<|fim_prefix|>'re\r\n\r\n🙂'llea\u000bＡe", "tokens": 39, "pieces": ["\",\r\n\r\n", "ḋ", "̣\r\n\r\n", "<|", "endoftext", "|>(", "tİ", "<|", "fim", "_prefix", "|>'", "re", "\r\n\r\n", "🙂'", "llea", "\u000b", "Ａe"]} +{"text": "sZ0's😀🏽
३", "tokens": 13, "pieces": ["sZ", "0", "'s", "😀🏽", "
", "३"]} +{"text": "\rs字 \n漢'D​m𐞁e0,Z0½", "tokens": 18, "pieces": ["\r", "s字", " \n", "漢", "'D", "​m𐞁e", "0", ",Z", "0½"]} +{"text": "12345678　\r\n'Re>ſ\ne Ａ\t­ \n…mＡfi#$%éå'", "tokens": 30, "pieces": ["123", "456", "78", "　\r\n", "'Re", ">ſ", "\n", "e", " Ａ", "\t", "­", " \n", "…mＡfi", "#$%", "éa", "̊'"]} +{"text": "<|fim_prefix|> \na\r\n\r\n'.'𐞁🙂३'re.0\r\n\r\n👍🏽,<|endoftext|>< éDžé'T𐞁㍿\r'll.𐞁👍🏽\u000bİ‍", "tokens": 67, "pieces": ["<|", "fim", "_prefix", "|>", " \n", "a", "\r\n\r\n", "'.'", "𐞁", "🙂", "३", "'re", ".", "0", "\r\n\r\n", "👍🏽,<|", "endoftext", "|><", " e", "́Džé", "'T", "𐞁", "㍿\r", "'ll", ".𐞁", "👍🏽", "\u000bİ", "‍"]} +{"text": "‍'ſ
 'ſ​!!.EOT عİ#$%\r\n\r\n漢's'Sfi", "tokens": 27, "pieces": ["‍'", "ſ", "
 ", " '", "ſ", "​!!.", "EOT", " ", " عİ", "#$%\r\n\r\n", "漢", "'s", "'S", "fi"]} +{"text": "! \n#$%ع'llmꟲDž'VE\t\r\n\r\n-½­'VE𐞁EOT're's🙂", "tokens": 30, "pieces": ["!", " \n", "#$%", "ع", "'ll", "mꟲDž", "'VE", "\t\r\n\r\n", "-", "½", "­'", "VE𐞁EOT", "'re", "'s", "🙂"]} +{"text": "fi\nⅣ'Re㍿!!­", "tokens": 11, "pieces": ["fi", "\n", "Ⅳ", "'Re", "㍿!!­"]} +{"text": "'M!!a0'VE\r\n\r\nß漢(Ⅳm ع>(!", "tokens": 18, "pieces": ["'M", "!!", "a", "0", "'VE", "\r\n\r\n", "ß漢", "(", "Ⅳ", "m", " ", " ع", ">(!"]} +{"text": "'VE#$%efi're.'re'ſ'T'VEḍ̇㋿'S㋿<|endoftext|>٣٤٥٦👍🏽", "tokens": 50, "pieces": ["'VE", "#$%", "efi", "'re", ".'", "re", "'ſ", "'T", "'VE", "ḋ", "̣㋿'", "S", "㋿<|", "endoftext", "|>", "٣٤٥", "٦", "👍🏽"]} +{"text": "<|endoftext|>'ſ 'T0'll𐞁<|endoftext|>'S<|endoftext|>'s👍🏽🙂👍🏽9d🙂ßعZ, 's<|endoftext|>😀🏽'T's\u000bŹ \n<<<\ré𐞁", "tokens": 86, "pieces": ["<|", "endoftext", "|>'", "ſ", " ", " '", "T", "0", "'ll", "𐞁", "<|", "endoftext", "|>'", "S", "<|", "endoftext", "|>'", "s", "👍🏽🙂👍🏽", "9", "d", "🙂ßعZ", ",", " ", " '", "s", "<|", "endoftext", "|>😀🏽'", "T", "'s", "\u000bZ", "́", " \n", "<<<\r", "e", "́𐞁"]} +{"text": "fi'ſ.#$%\r\n\r\n'Md‍åsfié !!å\r12345678! <|fim_prefix|>é\"漢字漢\né½> \r㍿", "tokens": 55, "pieces": ["fi", "'ſ", ".#$%\r\n\r\n", "'M", "d", "‍a", "̊sfié", " ", "!!", "a", "̊\r", "123", "456", "78", "!", " ", "<|", "fim", "_prefix", "|>", "é", "\"漢字漢", "\n", "e", "́", "½", ">", " \r", "㍿"]} +{"text": "e'ReⅣm　'T#$%👍🏽\u000b,", "tokens": 17, "pieces": ["e", "'Re", "Ⅳ", "m", "　", "'T", "#$%👍🏽", "\u000b", ","]} +{"text": "'ſع9ꟲfi", "tokens": 10, "pieces": ["'ſ", "ع", "9", "ꟲfi"]} +{"text": "(å字téعⅣé,", "tokens": 12, "pieces": ["(a", "̊字téع", "Ⅳ", "e", "́,"]} +{"text": "<́'Re9'㍿ ½\"<|fim_prefix|>\r\n.<|endoftext|>  're\r\n\r\nms٣٤٥٦é😀🏽'VE漢ßdd́>sḍ̇é'Sm", "tokens": 61, "pieces": ["<́'", "Re", "9", "'㍿", " ", "½", "\"<|", "fim", "_prefix", "|>\r\n", ".<|", "endoftext", "|>", " ", " ", "'re", "\r\n\r\n", "ms", "٣٤٥", "٦", "é", "😀🏽'", "VE漢ßdd", "́>", "sḋ", "̣é", "'S", "m"]} +{"text": ">‍\n'ſ-　Z字'ſſ12345678åꟲعs'llDž\r\n\r\nfiⅣß'Mé👍🏽Dž漢㋿👍🏽'S\u000b'Så", "tokens": 65, "pieces": [">‍\n", "'ſ", "-", "　Z字", "'ſ", "ſ", "123", "456", "78", "a", "̊ꟲعs", "'ll", "Dž", "\r\n\r\n", "fi", "Ⅳ", "ß", "'M", "e", "́👍🏽", "Dž漢", "㋿👍🏽'", "S", "\u000b", "'S", "a", "̊"]} +{"text": "漢ꟲ…\r\n\r\nés'Sİ12345678ſaع'MEOTⅣ
'Tꟲ", "tokens": 29, "pieces": ["漢ꟲ", "…\r\n\r\n", "és", "'S", "İ", "123", "456", "78", "ſaع", "'M", "EOT", "Ⅳ", "
", "'T", "ꟲ"]} +{"text": "Ⅳ 'VE>ḍ̇\n'S\n🙂<|endoftext|>.Dž''MZ㍿'ll\r\n\r\né‍12345678٣٤٥٦😀🏽'llé'Mt㋿'ll\n\tEOTß", "tokens": 66, "pieces": ["Ⅳ", " ", " '", "VE", ">ḋ", "̣\n", "'S", "\n", "🙂<|", "endoftext", "|>.", "Dž", "''", "MZ", "㍿'", "ll", "\r\n\r\n", "é", "‍", "123", "456", "78٣", "٤٥٦", "😀🏽'", "lle", "́'", "Mt", "㋿'", "ll", "\n", "\tEOTß"]} +{"text": "ḍ̇fi", "tokens": 7, "pieces": ["ḋ", "̣fi"]} +{"text": "åDžßDžⅣ.'Ré-\"12345678fifi!!<|endoftext|>", "tokens": 32, "pieces": ["a", "̊DžßDž", "Ⅳ", ".'", "Re", "́-\"", "123", "456", "78", "fifi", "!!<|", "endoftext", "|>"]} +{"text": "½<|fim_prefix|>​12345678a", "tokens": 13, "pieces": ["½", "<|", "fim", "_prefix", "|>​", "123", "456", "78", "a"]} +{"text": "e EOTß👍🏽<'VE𐞁(<|fim_prefix|>#$%<ḍ̇'Re're#$%#$%å'ſſ́\rå字 \n>'M're\t", "tokens": 64, "pieces": ["e", " ", " EOTß", "👍🏽<'", "VE𐞁", "<", "EOT", ">(<|", "fim", "_prefix", "|>#$%<", "ḋ", "̣'", "Re", "'re", "#$%#$%", "a", "̊'", "ſſ", "́\r", "a", "̊字", " \n", ">'", "M", "'re", "\t"]} +{"text": "
ḍ̇'s\r\n0 \n㍿'T!'T,'ll🙂<|endoftext|>Dž!'s🙂's'll'VE's'\u000b", "tokens": 44, "pieces": ["
ḋ", "̣'", "s", "\r\n", "0", " \n", "㍿'", "T", "!'", "T", ",'", "ll", "🙂<|", "endoftext", "|>", "Dž", "!'", "s", "🙂'", "s", "'ll", "'VE", "'s", "'", "\u000b"]} +{"text": "\u000b\t👍🏽'll#$%m<٣٤٥٦‍\t 'D😀🏽<\n​­EOT३.㋿!㋿Z", "tokens": 48, "pieces": ["\u000b", "\t", "👍🏽'", "ll", "#$%", "m", "<", "٣٤٥", "٦", "‍", "\t ", " '", "D", "😀🏽<\n", "​­", "EOT", "३", ".㋿!㋿", "Z"]} +{"text": "(9'll", "tokens": 3, "pieces": ["(", "9", "'ll"]} +{"text": "🙂
…ſd#$%\u000b'VEé'Re\u000ba(<|endoftext|>mＡ́🙂字​'S　\r\n'llß'llⅣ", "tokens": 44, "pieces": ["🙂", "
", "…ſd", "#$%", "\u000b", "'VE", "e", "́'", "Re", "\u000ba", "(<|", "endoftext", "|>", "mＡ", "́🙂", "字", "​'", "S", "　\r\n", "'ll", "ß", "'ll", "Ⅳ"]} +{"text": " EOT'Re\u000b0字", "tokens": 6, "pieces": [" EOT", "'Re", "\u000b", "0", "字"]} +{"text": "Z. 
9عt 'Rea🙂字m!<㍿ꟲḍ̇fi'D　ḍ̇9're('ſß\r\n\r\nss", "tokens": 51, "pieces": ["Z", ".<", "EOT", ">", " ", "
", "9", "عt", " '", "Rea", "🙂字m", "!<㍿", "ꟲḋ", "̣fi", "'D", "　ḋ", "̣", "9", "'re", "('", "ſß", "\r\n\r\n", "ss", ""]} +{"text": "ⅣEOT<|endoftext|>fi'll<|fim_prefix|>e \tZ 'T\"ꟲ字,EOT漢Dž\"\r\n'ſ\r\nſ><\r\n\r\n'S­👍🏽\råé#$%", "tokens": 64, "pieces": ["Ⅳ", "EOT", "<|", "endoftext", "|>", "fi", "'ll", "<|", "fim", "_prefix", "|>", "e", " ", "\tZ", " ", "'T", "\"ꟲ字", ",EOT漢", "Dž", "\"\r\n", "'ſ", "\r\n", "ſ", "><\r\n\r\n", "'S", "­👍🏽\r", "a", "̊é", "#$%"]} +{"text": "0'ſ㋿'T　>\n㍿'re字!!'reDž\t👍🏽 dd-​‍0İ ", "tokens": 37, "pieces": ["0", "'ſ", "㋿'", "T", "　", ">\n", "㍿'", "re字", "!!'", "reDž", "\t", "👍🏽", " dd", "-​‍", "0", "İ", " "]} +{"text": "\r-'>12345678\"!!漢㍿٣٤٥٦­ＡmＡع", "tokens": 28, "pieces": ["\r", "-'>", "123", "456", "78", "\"!!", "漢", "㍿", "٣٤٥", "٦", "­ＡmＡع"]} +{"text": "\u000b٣٤٥٦\n'D😀🏽Z'S\"'ſ٣٤٥٦​9\r\n\r\n(#$%!!🙂ع<>İt३'ḍ̇　　́Ⅳ㍿㍿", "tokens": 65, "pieces": ["\u000b", "٣٤٥", "٦", "\n", "'D", "😀🏽", "Z", "'S", "\"'", "ſ", "٣٤٥", "٦", "​", "9", "\r\n\r\n", "(#$%<", "EOT", ">!!🙂", "ع", "<>", "İt", "३", "'ḋ", "̣", "　", "　", "́", "Ⅳ", "㍿㍿"]} +{"text": "'s\r\n", "tokens": 5, "pieces": ["'", "s", "\r\n"]} +{"text": "… 'ſ​>Dž,'s<'Re<|fim_prefix|>fißß<'S'VE<|fim_prefix|>t🙂Ⅳ\u000b\r'VE<|fim_prefix|>e,a<|endoftext|>字ＡİDž", "tokens": 75, "pieces": ["… ", " '", "ſ", "​>", "Dž", ",'", "s", "<'", "Re", "<|", "fim", "_prefix", "|>", "fißß", "<'", "S", "'VE", "<|", "fim", "_prefix", "|>", "t", "🙂<", "EOT", ">", "Ⅳ", "\u000b\r", "'VE", "<|", "fim", "_prefix", "|>", "e", ",a", "<|", "endoftext", "|>", "字ＡİDž"]} +{"text": "漢's'T𐞁\u000b\t12345678'D9ſḍ̇9'VE'S㋿Z👍🏽d­漢éd", "tokens": 46, "pieces": ["漢", "'s", "'T", "𐞁", "\u000b", "\t", "123", "456", "78", "'", "D", "9", "ſḋ", "̣", "9", "'VE", "'S", "㋿Z", "👍🏽", "d", "­漢e", "́d"]} +{"text": "é٣٤٥٦­#$%ḍ̇½ḍ̇ ㍿ḍ̇\t­½٣٤٥٦'Re𐞁EOTع<|fim_prefix|><|fim_prefix|>!Ⅳfié<|fim_prefix|>\r…0#$%​'M \nt're", "tokens": 95, "pieces": ["é", "٣٤٥", "٦", "­#$%", "ḋ", "̣", "½", "ḋ", "̣", " ", " ㍿", "ḋ", "̣", "\t", "­", "½٣٤", "٥٦", "'Re", "𐞁EOTع", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>!", "Ⅳ", "fie", "́<|", "fim", "_prefix", "|>\r", "…", "0", "#$%​'", "M", " \n", "t", "'re"]} +{"text": "\n­Ⅳ\r\n\r\n 'M,
\r\n\r\n,é e'Sfi👍🏽", "tokens": 25, "pieces": ["\n", "­", "Ⅳ", "\r\n\r\n", " ", "'M", ",", "
", "\r\n\r\n", ",é", " e", "'S", "fi", "👍🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b(́'res ꟲm\r\néé\r\n\r\nḍ̇!.'S字
\r\nfia'ſDž'T's­'VEm३d字'll", "tokens": 42, "pieces": ["👍🏽", "123", "456", "78३", "­​<", "META", "_START", ">.'", "S字", "
\r\n", "fia", "'ſ", "Dž", "'T", "'s", "­'", "VEm", "३", "d字", "'ll"]} +{"text": "<\r\n\r\nå<'Tfi9#$%Dž\r\n😀🏽🙂‍😀🏽\t,'T.'s\u000b字e#$% \u000b t9٣٤٥٦\t'ſ \n", "tokens": 63, "pieces": ["<<", "META", "_START", ">\r\n\r\n", "a", "̊<'", "Tfi", "9", "#$%<", "EOT", ">Dž", "\r\n", "😀🏽🙂‍😀🏽", "\t", ",'", "T", ".'", "s", "\u000b字e", "#$%", " \u000b ", " t", "9٣٤", "٥٦", "\t", "'ſ", " \n"]} +{"text": ">\ń٣٤٥٦-a‍<|endoftext|>\u000bé'T12345678Dž0eß\r\nfi\r\n\r\n　ß́́ ⅣDž<|fim_prefix|>t​…åZ​a<|fim_prefix|>", "tokens": 74, "pieces": [">\n", "́", "٣٤٥", "٦", "-a", "‍<|", "endoftext", "|>", "\u000be", "́'", "T", "123", "456", "78", "Dž", "0", "eß", "\r\n", "fi", "\r\n\r\n", "　ß", "́<", "META", "_START", ">́", " ", "Ⅳ", "Dž", "<|", "fim", "_prefix", "|>", "t", "​", "…a", "̊Z", "​a", "<|", "fim", "_prefix", "|>"]} +{"text": "­..😀🏽're漢\t\t\r\n😀🏽\u000bß'VE𐞁é漢<|endoftext|>​Ⅳ'Ś…<|fim_prefix|>(Ⅳ\tm३", "tokens": 57, "pieces": ["­..😀🏽'", "re漢", "\t\t\r\n", "😀🏽", "\u000bß", "'VE", "𐞁é漢", "<|", "endoftext", "|>​", "Ⅳ", "'", "S", "́", "…", "<|", "fim", "_prefix", "|>(", "Ⅳ", "\tm", "३"]} +{"text": "😀🏽漢😀🏽 's-\"\r'ſDž0#$%!!👍🏽! Dž㋿'<|endoftext|>'VEع m\r字Ⅳ٣٤٥٦>Dž½<​<|fim_prefix|>", "tokens": 84, "pieces": ["😀🏽", "漢", "😀🏽", " ", " '", "s", "-\"\r", "'ſ", "Dž", "0", "#$%!!👍🏽!<", "META", "_START", ">", " Dž", "㋿'<|", "endoftext", "|>'", "VEع", " ", " m", "\r", "字", "Ⅳ٣٤", "٥٦", ">Dž", "½", "<​<", "EOT", "><|", "fim", "_prefix", "|><", "EOT", ">"]} +{"text": "‍#$%éİ
३>0\"'VEé", "tokens": 16, "pieces": ["‍#$%", "éİ", "
", "३", ">", "0", "\"'", "VEe", "́"]} +{"text": "‍,'res'VE‍DžⅣ's٣٤٥٦\r\n\r\n'Re 🙂'M\r\n\r\n\rdſ \nZ½é", "tokens": 36, "pieces": ["‍,'", "res", "'VE", "‍Dž", "Ⅳ", "'s", "٣٤٥", "٦", "\r\n\r\n", "'Re", " ", " 🙂'", "M", "\r\n\r\n\r", "dſ", " \n", "Z", "½", "é"]} +{"text": "'re!ع\rmDž😀🏽#$%…é
\nİ\"é'VEß(sd'VE <漢­ꟲ\r\n\r\nså'M", "tokens": 44, "pieces": ["'re", "!ع", "\r", "mDž", "😀🏽#$%", "…é", "
\n", "İ", "\"e", "́'", "VEß", "(sd", "'VE", " ", "<漢", "­ꟲ", "\r\n\r\n", "sa", "̊'", "M"]} +{"text": "'D, \r\n\r\nعꟲ'ſ", "tokens": 10, "pieces": ["'D", ",", " \r\n\r\n", "عꟲ", "'ſ"]} +{"text": "漢<|fim_prefix|>aéع \na'VE­‍ſDž字­t'\te𐞁0́'VEsİ
ß字'll!İ(-ꟲ", "tokens": 56, "pieces": ["漢", "<|", "fim", "_prefix", "|>", "ae", "́ع", " \n", "a", "'VE", "­‍", "ſDž", "字", "­t", "'", "\te𐞁", "0", "́'", "VEsİ", "
ß字", "'ll", "!İ", "(-<", "EOT", ">ꟲ"]} +{"text": "ßİꟲ😀🏽'\r\ne<|fim_prefix|>Dž٣٤٥٦t(ſ'ReＡ", "tokens": 36, "pieces": ["ßİꟲ", "😀🏽'\r\n", "e", "<|", "fim", "_prefix", "|>", "Dž", "٣٤٥", "٦", "t", "(ſ", "'Re", "Ａ"]} +{"text": "'ll字,aß'Re½ Ａ", "tokens": 8, "pieces": ["'ll", "字", ",aß", "'Re", "½", " Ａ"]} +{"text": "Ⅳ's漢İ
(𐞁0ée\r\n\r\nd½\rḍ̇9!9
​,٣٤٥٦\r\nع'ſ३'Re", "tokens": 49, "pieces": ["Ⅳ", "'s", "漢İ", "
", "(𐞁", "0", "e", "́e", "\r\n\r\n", "d", "½", "\r", "ḋ", "̣", "9", "!", "9", "
", "​,", "٣٤٥", "٦", "\r\n", "ع", "'ſ", "३", "'Re"]} +{"text": "e'M　9　(…'Re!Z<|fim_prefix|>'D", "tokens": 22, "pieces": ["e", "'M", "　", "9", "　", "(", "…", "'Re", "!", "Z", "<|", "fim", "_prefix", "|>'", "D"]} +{"text": "Z \r😀🏽‍'VE字Ａ😀🏽-\t#$%'>́Dž\r-\r\n're漢'ſ", "tokens": 36, "pieces": ["Z", " \r", "😀🏽‍'", "VE字Ａ", "😀🏽-", "\t", "#$%'>́", "Dž", "\r", "-\r\n", "'re", "漢", "'ſ"]} +{"text": "é
٣٤٥٦ 'M🙂fim 'S<", "tokens": 22, "pieces": ["é", "
", "٣٤٥", "٦", " ", " '", "M", "🙂fim", " '", "S", "<"]} +{"text": "å
🙂ⅣDž\n👍🏽s's'res \n's'll!fi😀🏽​a're", "tokens": 36, "pieces": ["a", "̊", "
", "🙂", "Ⅳ", "Dž", "\n", "👍🏽", "s", "'s", "'re", "s", " \n", "'s", "'ll", "!fi", "😀🏽​", "a", "'re"]} +{"text": "9EOT‍ 're\"<|endoftext|>३t'T\n.İ\n'T́s🙂<'D\ta", "tokens": 32, "pieces": ["9", "EOT", "‍", " ", "'re", "\"<|", "endoftext", "|>", "३", "t", "'T", "\n", ".İ", "\n", "'T", "́s", "🙂<'", "D", "\ta"]} +{"text": "'ſ'llⅣ", "tokens": 9, "pieces": ["'ſ", "'ll", "Ⅳ", ""]} +{"text": "🙂ß𐞁'll<|endoftext|> \n<|endoftext|>Ａ<|fim_prefix|>'M字㍿'Re's!!!fiEOTⅣ", "tokens": 51, "pieces": ["🙂ß𐞁", "'ll", "<|", "endoftext", "|>", " \n", "<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>'", "M字", "㍿'", "Re", "'s", "!<", "META", "_START", ">!!", "fiEOT", "Ⅳ"]} +{"text": "\u000bé‍'re
é́漢 　t<|endoftext|>'Re12345678\"­ ́㋿\u000b字e\r'll", "tokens": 50, "pieces": ["\u000be", "́‍'", "re", "
é", "́漢", " ", "　t", "<|", "endoftext", "|>'", "Re", "123", "456", "78", "\"­", " ́<", "META", "_START", "><", "EOT", ">㋿<", "META", "_START", ">", "\u000b字e", "\r", "'ll"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi'Dİ'reéåع\"Ⅳ(téİ", "tokens": 20, "pieces": ["fi", "'D", "İ", "'re", "e", "́<", "EOT", ">a", "̊ع", "\"", "Ⅳ", "(téİ"]} +{"text": "9å'reⅣ're­\u000b'Téd ,　字😀🏽'Såع٣٤٥٦ḍ̇\r,½ꟲ漢‍ꟲ…Ⅳ's", "tokens": 72, "pieces": ["9", "a", "̊'", "re", "Ⅳ", "'re", "­", "\u000b", "'T", "e", "́d", " ", ",", "　字", "😀🏽'", "Sa", "̊ع", "", "٣٤٥", "٦", "ḋ", "̣\r", ",", "½", "ꟲ漢", "‍ꟲ", "…", "Ⅳ", "'s"]} +{"text": "㋿‍!!​'re-EOT\r\n\r\n-​,'ll​éd\n!ß­ m'Rea'ſ \r'Red…0", "tokens": 44, "pieces": ["㋿‍!!​'", "re", "-EOT", "\r\n\r\n", "-​,'", "ll", "​e", "́d", "\n", "!", "ß", "­", " m", "'Re", "a", "'ſ", " \r", "'Re", "d", "…", "0"]} +{"text": "'M'ſ'D½\t…𐞁….\n12345678İ
.'M‍㍿<|fim_prefix|>", "tokens": 39, "pieces": ["'M", "'ſ", "'D", "½", "\t", "…𐞁", "…", ".\n", "123", "456", "78", "İ", "
", ".'", "M", "‍㍿<|", "fim", "_prefix", "|>"]} +{"text": "ß\ra
9😀🏽'ſ>…😀🏽Ⅳ­'T<㋿İꟲ‍'ſ 0'll.t", "tokens": 44, "pieces": ["ß", "\r", "a", "
", "9", "😀🏽'", "ſ", ">", "…", "😀🏽", "Ⅳ", "­'", "T", "<㋿", "İꟲ", "‍'", "ſ", " ", "0", "'ll", ".t"]} +{"text": "'sſ'Re\"\n123456780'ſa ", "tokens": 13, "pieces": ["'s", "ſ", "'Re", "\"\n", "123", "456", "780", "'ſ", "a", " "]} +{"text": "\r\n 漢t ḍ̇😀🏽sꟲ'Dt\rfi\" 漢td ", "tokens": 32, "pieces": ["\r\n", " 漢t", " ḋ", "̣😀🏽", "sꟲ", "'D", "t", "\r", "fi", "\"", " ", " 漢td", " "]} +{"text": "ع㋿Z'M𐞁 \n…'ll'VE,e.Džs", "tokens": 21, "pieces": ["ع", "㋿Z", "'M", "𐞁", " \n", "…", "'ll", "'VE", ",e", ".Džs"]} +{"text": " 'DéＡ<\r\n\r\n>Dž!d字#$%'D!½e‍­s!<\réＡعfi'Re", "tokens": 32, "pieces": [" '", "DéＡ", "<\r\n\r\n", ">Dž", "!d字", "#$%'", "D", "!", "½", "e", "‍­", "s", "!<\r", "e", "́Ａعfi", "'Re"]} +{"text": "(aſİ'ſ
字Ａ\"३e9!\r ½
's​\r\n\r\n'VE", "tokens": 29, "pieces": ["(aſİ", "'ſ", "
字Ａ", "\"", "३", "e", "9", "!\r", " ", " ", "½", "
", "'s", "​\r\n\r\n", "'VE"]} +{"text": "Ａ \nꟲ'T\"fis
's𐞁🙂å\r\na\r\n\r\n", "tokens": 26, "pieces": ["Ａ", " \n", "ꟲ", "'T", "\"fis", "
", "'s", "𐞁", "🙂a", "̊\r\n", "a", "\r\n\r\n"]} +{"text": "m 'VE-'.👍🏽EOTé\r\nétå", "tokens": 21, "pieces": ["m", " ", "'VE", "-'.👍🏽", "EOTé", "\r\n", "e", "́ta", "̊"]} +{"text": "'S'D-ſ‍#$%'Dß́'S s!'MZé>é", "tokens": 21, "pieces": ["'S", "'D", "-ſ", "‍#$%'", "Dß", "́'", "S", " ", " s", "!'", "MZé", ">é"]} +{"text": "'ll,'VEs'VE're​a​\"-Zع'Séå…ꟲ字", "tokens": 25, "pieces": ["'ll", ",'", "VEs", "'VE", "'re", "​a", "​\"-", "Zع", "'S", "e", "́a", "̊", "…ꟲ字"]} +{"text": "\n \n s mfi !\t'Dt!!ꟲ,dİ12345678㍿…'Reع<|endoftext|>​漢ع😀🏽'ſ 　m", "tokens": 49, "pieces": ["\n \n", " ", " s", " ", " mfi", " !", "\t", "'D", "t", "!!", "ꟲ", ",dİ", "123", "456", "78", "㍿", "…", "'Re", "ع", "<|", "endoftext", "|>​", "漢ع", "😀🏽'", "ſ", " ", "　m"]} +{"text": "㍿ßßm'DDž ٣٤٥٦'Re
 𐞁(½\n \n٣٤٥٦<|endoftext|>å'Mß́0\"…,'Re'M(fi<|fim_prefix|>EOT㋿Ⅳ٣٤٥٦", "tokens": 86, "pieces": ["㍿ßßm", "'D", "Dž", " ", " ", "٣٤٥", "٦", "'Re", "
", " 𐞁", "(", "½", "\n \n", "٣٤٥", "٦", "<|", "endoftext", "|>", "a", "̊'", "Mß", "́", "0", "\"", "…", ",'", "Re", "'M", "(", "fi", "<|", "fim", "_prefix", "|>", "EOT", "㋿", "Ⅳ٣٤", "٥٦"]} +{"text": "(ꟲé　<|fim_prefix|> \n<|fim_prefix|> 🙂😀🏽fi0a>\n٣٤٥٦éꟲ😀🏽́é", " \n", "<|", "fim", "_prefix", "|>", " ", " 🙂😀🏽", "fi", "0", "a", ">\n", "٣٤٥", "٦", "éꟲ", "😀🏽́", "é", "½12345678…😀🏽Ⅳ<|endoftext|>\"३ع㍿té
d​m½", "tokens": 53, "pieces": ["…", "'ll", "éع", " \n", "<'", "re", "Ⅳ", "<ß", "<|", "fim", "_prefix", "|>", "½12", "345", "678", "…", "😀🏽", "Ⅳ", "<|", "endoftext", "|>\"", "३", "ع", "㍿te", "́", "
d", "​m", "½"]} +{"text": "٣٤٥٦Ⅳ ​Ⅳ'reſ́'Re٣٤٥٦éé<<|endoftext|> <|fim_prefix|>عåⅣ😀🏽 ḍ̇'D'D
٣٤٥٦㍿'s'VEs'Re👍🏽0'VEå\r\n>字", "tokens": 98, "pieces": ["٣٤٥", "٦Ⅳ", " ", " ​", "Ⅳ", "'re", "ſ", "́'", "Re", "٣٤٥", "٦", "ée", "́<<|", "endoftext", "|>", " ", " <|", "fim", "_prefix", "|>", "عa", "̊", "Ⅳ", "😀🏽", " ", " ḋ", "̣'", "D", "'D", "
", "٣٤٥", "٦", "㍿'", "s", "'VE", "s", "'Re", "👍🏽", "0", "'VE", "a", "̊\r\n", ">字"]} +{"text": "#$%!!㋿Z½٣٤٥٦ḍ̇\tＡtعd㋿漢'Téع9Ａ'é\né­#$%Dž…", "tokens": 53, "pieces": ["#$%!!㋿", "Z", "½٣٤", "٥٦", "ḋ", "̣", "\tＡtعd", "㋿<", "META", "_START", ">漢", "'T", "e", "́ع", "9", "Ａ", "'é", "\n", "é", "­#$%", "Dž", "…"]} +{"text": "m
ḍ̇ع're're漢 \n\r\nＡⅣ\n'res<|fim_prefix|>‍عé12345678", "tokens": 37, "pieces": ["m", "
ḋ", "̣ع", "'re", "'re", "漢", " \n\r\n", "Ａ", "Ⅳ", "\n", "'re", "s", "<|", "fim", "_prefix", "|>‍", "عe", "́", "123", "456", "78"]} +{"text": "é'SDž'sعß'T.", "tokens": 9, "pieces": ["é", "'S", "Dž", "'s", "عß", "'T", "."]} +{"text": "\"　--Dž-‍mＡ\r\n\n\" \n're", "tokens": 17, "pieces": ["\"", "　", "--", "Dž", "-‍", "mＡ", "\r\n\n", "\"", " \n", "'re"]} +{"text": "\u000bmé0‍­'S㍿­३𐞁Dž‍", "tokens": 23, "pieces": ["\u000bmé", "0", "‍­'", "S", "㍿­", "३", "𐞁Dž", "‍"]} +{"text": "!!字㍿ Ⅳ #$% !!漢<|fim_prefix|>😀🏽,
\"ꟲEOT  Z\"'Tعå٣٤٥٦0>😀🏽-tfi\t㋿😀🏽", "tokens": 76, "pieces": ["!!", "字", "㍿", " ", "Ⅳ", " ", " #$%", " ", "!!", "漢", "<|", "fim", "_prefix", "|>😀🏽,", "
", "\"ꟲEOT", " ", " Z", "\"'", "Tعa", "̊", "٣٤٥", "٦0", "><", "EOT", ">😀🏽-", "tfi", "\t", "㋿😀🏽"]} +{"text": "EOT'RedⅣZ🙂('Re'D\rDž'VE\t\t.'T㋿\u000b​㍿​😀🏽é0٣٤٥٦\r\nZ-'re🙂fi漢½'re.ع", "tokens": 60, "pieces": ["EOT", "'Re", "d", "Ⅳ", "Z", "🙂('", "Re", "'D", "\r", "Dž", "'VE", "\t", "\t", ".'", "T", "㋿", "\u000b", "​㍿​😀🏽", "e", "́", "0٣٤", "٥٦", "\r\n", "Z", "-'", "re", "🙂fi漢", "½", "'re", ".ع"]} +{"text": "#$%½sé", "tokens": 5, "pieces": ["#$%", "½", "sé"]} +{"text": "\u000bat<İ…ß🙂عꟲ字're३,'T٣٤٥٦'Re…Ａ\r\n ́t>\r\n's​
éDž9'Dع‍", "tokens": 57, "pieces": ["\u000bat", "<İ", "", "…ß", "🙂عꟲ字", "'re", "३", ",'", "T", "٣٤٥", "٦", "'Re", "…Ａ", "\r\n", " ", " ́", "t", ">\r\n", "'s", "​<", "EOT", ">", "
éDž", "9", "'D", "ع", "‍"]} +{"text": "a漢\u000bİ​\n\r's!'Re'D👍🏽İ\r\n'VEİ'St ", "tokens": 26, "pieces": ["a漢", "\u000bİ", "​\n\r", "'s", "!'", "Re", "'D", "👍🏽", "İ", "\r\n", "'VE", "İ", "'S", "t", " "]} +{"text": "𐞁d'Sع㋿éݽ'S\u000bſ'T\"9ſ123456780\r\nß'smＡ,½　", "tokens": 34, "pieces": ["𐞁d", "'S", "ع", "㋿éİ", "½", "'S", "\u000bſ", "'T", "\"", "9", "ſ", "123", "456", "780", "\r\n", "ß", "'s", "mＡ", ",", "½", "　"]} +{"text": "́m#$%'T　('VE\r,Ⅳ😀🏽!!½ ́\t-　0'D३å‍>eſd", "tokens": 43, "pieces": ["́m", "#$%'", "T", "　", "('", "VE", "\r", ",", "Ⅳ", "😀🏽!!", "½", " ", " ́", "\t", "-", "　", "0", "'D", "३", "a", "̊‍>", "eſd"]} +{"text": " \n ", "tokens": 2, "pieces": [" \n "]} +{"text": "\t \n<|endoftext|>\r\n9,\r\n12345678EOT", "tokens": 15, "pieces": ["\t \n", "<|", "endoftext", "|>\r\n", "9", ",\r\n", "123", "456", "78", "EOT"]} +{"text": "'Dſ'S9३İ ,'T🙂\u000b're\t'e'Re<fi​🙂<|fim_prefix|>'D,‍<|fim_prefix|>　-12345678字12345678é　ꟲ", "tokens": 60, "pieces": ["'D", "ſ", "'S", "9", "", "३", "İ", " ,'", "T", "🙂", "\u000b", "'re", "\t", "'e", "'Re", "<fi", "​🙂<|", "fim", "_prefix", "|>'", "D", ",‍<|", "fim", "_prefix", "|>", "　", "-", "123", "456", "78", "字", "123", "456", "78", "e", "́", "　ꟲ"]} +{"text": "(㋿\u000b0\r\n\r\n'M", "tokens": 8, "pieces": ["(㋿", "\u000b", "0", "\r\n\r\n", "'M"]} +{"text": "\t
'ſⅣ㍿ 字'D　>", "tokens": 20, "pieces": ["\t", "
", "'ſ", "Ⅳ", "㍿", " ", "字", "'D", "　", ">"]} +{"text": "m\r㍿( Dž\n३-­\"\t's", "tokens": 20, "pieces": ["m", "\r", "㍿(", " Dž", "\n", "३", "-­\"<", "EOT", ">", "\t", "'s"]} +{"text": "३!İ🙂(,字d0​漢a", "tokens": 15, "pieces": ["३", "!İ", "🙂(,", "字d", "0", "​漢a"]} +{"text": "!!e \n<#$% a >ع \n​३\u000b'VE­\r", "tokens": 19, "pieces": ["!!", "e", " \n", "<#$%", " a", " >", "ع", " \n", "​", "३", "\u000b", "'VE", "­\r"]} +{"text": ",'Dž", "tokens": 3, "pieces": [",'", "Dž"]} +{"text": "Ａ'ſ \n", "tokens": 6, "pieces": ["Ａ", "'ſ", " \n"]} +{"text": "ſEOT0‍-éZع\r", "tokens": 12, "pieces": ["ſEOT", "0", "‍-", "éZع", "\r"]} +{"text": "'re EOTŹ'३عś\r.ع
… ß👍🏽", "tokens": 27, "pieces": ["'re", " EOTZ", "́'", "३", "عs", "́\r", ".ع", "
…", " ß", "👍🏽"]} +{"text": "'DEOTＡ'Re\r\n\u000b \n😀🏽ae,ſDžꟲ.㍿'llİ३𐞁Ａ'M'VE…é'll9<|endoftext|><|endoftext|>'", "tokens": 59, "pieces": ["'D", "EOTＡ", "'Re", "\r\n\u000b \n", "😀🏽", "ae", ",ſDžꟲ", ".㍿'", "llİ", "३", "𐞁Ａ", "'M", "'VE", "…é", "'ll", "9", "<|", "endoftext", "|><|", "endoftext", "|>'"]} +{"text": "Ⅳ('s\"!!'S#$%'llé'll\u000b ㋿!a's<|fim_prefix|>('re　<|fim_prefix|>\r\né
½'Re\t½½'s!́'ll३ \n ", "tokens": 60, "pieces": ["Ⅳ", "('", "s", "\"!!'", "S", "#$%'", "ll", "e", "́'", "ll", "\u000b", " ㋿!", "a", "'s", "<|", "fim", "_prefix", "|>('", "re", "　", "<|", "fim", "_prefix", "|>\r\n", "e", "́", "
", "½", "'Re", "\t", "½½", "'s", "!́'", "ll", "३", " \n "]} +{"text": "e12345678", "tokens": 4, "pieces": ["e", "123", "456", "78"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi0‍ḍ̇<|endoftext|>ḍ̇>9 '<|fim_prefix|>\nß́'VE\r\n​(𐞁𐞁", "tokens": 50, "pieces": ["fi", "0", "‍ḋ", "̣<", "META", "_START", "><|", "endoftext", "|>", "ḋ", "̣>", "9", " ", " '<|", "fim", "_prefix", "|>\n", "ß", "́'", "VE", "\r\n", "​(", "𐞁𐞁"]} +{"text": "­-'D9👍🏽👍🏽é\"👍🏽e's<|fim_prefix|>12345678İt åé३'D >('VE\n", "tokens": 55, "pieces": ["­-<", "EOT", ">'", "D", "9", "👍🏽👍🏽", "e", "́\"👍🏽", "e", "'s", "<|", "fim", "_prefix", "|>", "123", "456", "78", "İt", " a", "̊e", "́", "३", "'D", " ", ">('", "VE", "\n"]} +{"text": "'lls\r\n\r\n \"'TＡ'Re٣٤٥٦sé 😀🏽", "tokens": 24, "pieces": ["'ll", "s", "\r\n\r\n", " \"'", "TＡ", "'Re", "٣٤٥", "٦", "se", "́", " ", "😀🏽"]} +{"text": "s\n'VE\rEOT\n", "tokens": 8, "pieces": ["s", "\n", "'VE", "\r", "EOT", "\n"]} +{"text": "fiⅣ
>", "tokens": 7, "pieces": ["fi", "Ⅳ", "
", ">"]} +{"text": "½!!عDžⅣⅣİ😀🏽'T \néß'Dm'Tعß!, ع>", "tokens": 32, "pieces": ["½", "!!", "عDž", "ⅣⅣ", "İ", "😀🏽'", "T", " \n", "éß", "'D", "m", "'T", "عß", "!,", " ع", ">"]} +{"text": " \r\n\r\nⅣ'Rea", "tokens": 5, "pieces": [" \r\n\r\n", "Ⅳ", "'Re", "a"]} +{"text": "#$%å's🙂㍿'Re\t😀🏽Z　­e‍'s漢're>!-'sß٣٤٥٦'s<|fim_prefix|>㍿0!! \ń<|endoftext|>👍🏽9m­Ａ", "tokens": 84, "pieces": ["#$%", "a", "̊'", "s", "🙂㍿'", "Re", "\t", "😀🏽", "Z", "", "　", "­e", "‍'", "s漢", "'re", ">!-'", "sß", "٣٤٥", "٦", "'s", "<|", "fim", "_prefix", "|>㍿", "0", "!!", " \n", "́<|", "endoftext", "|>👍🏽", "9", "m", "­<", "META", "_START", ">Ａ"]} +{"text": "Ａ>㋿Ⅳ#$%­­'VEs\rİ'Md　 \r\n…<'s.ſ٣٤٥٦EOT\"9​<　t\u000b", "tokens": 50, "pieces": ["Ａ", ">㋿", "Ⅳ", "#$%­­'", "VEs", "\r", "İ", "'M", "d", "　 \r\n", "…", "<'", "s", ".", "ſ", "٣٤٥", "٦", "EOT", "\"", "9", "​<", "　t", "\u000b"]} +{"text": "Ａé-'\n", "tokens": 9, "pieces": ["Ａé", "-'\n"]} +{"text": "<|fim_prefix|>a'Re<𐞁漢
½", "tokens": 22, "pieces": ["<|", "fim", "_prefix", "|>", "a", "'Re", "<<", "META", "_START", ">𐞁漢", "
", "½"]} +{"text": "<|endoftext|>٣٤٥٦Džéś9Ａ　🙂s!<,㍿12345678\n'M'VE<|endoftext|>😀🏽", "tokens": 54, "pieces": ["<|", "endoftext", "|>", "٣٤٥", "٦", "Džés", "́", "9", "Ａ", "　", "🙂s", "!<,㍿", "123", "456", "78", "\n", "'", "M", "'VE", "<|", "endoftext", "|>😀🏽"]} +{"text": "é🙂漢𐞁Ⅳfi'VEd.­ \n…\u000b㋿ \n𐞁㋿d\"EOT--'ſZ\tm", "tokens": 54, "pieces": ["e", "́🙂", "漢", "𐞁", "Ⅳ", "fi", "'VE", "d", ".­", " \n", "…", "\u000b", "㋿", " \n", "𐞁", "㋿d", "\"EOT", "--'", "ſZ", "\tm"]} +{"text": "'M'ret👍🏽ꟲ\u000bt12345678'M -٣٤٥٦!'re", "tokens": 30, "pieces": ["'M", "'re", "t", "👍🏽", "ꟲ", "\u000bt", "123", "456", "78", "'M", " ", " -", "٣٤٥", "٦", "!'", "re"]} +{"text": "\r\n\r\nİ'VE٣٤٥٦­éZ\t \n", "tokens": 20, "pieces": ["\r\n\r\n", "İ", "'VE", "٣٤٥", "٦", "­e", "́<", "EOT", ">Z", "\t \n"]} +{"text": "<|endoftext|>'s🙂𐞁​‍d", "tokens": 18, "pieces": ["<|", "endoftext", "|>'", "s", "🙂𐞁", "​‍", "d"]} +{"text": "'sßꟲ ,12345678㍿🙂're<|fim_prefix|>'MＡſ'll㋿½e!,'ll‍ꟲⅣ>", "tokens": 50, "pieces": ["'", "sßꟲ", " ,", "123", "456", "78", "㍿🙂'", "re", "<|", "fim", "_prefix", "|>'", "MＡſ", "'ll", "㋿", "½", "e", "!,'", "ll", "‍ꟲ", "Ⅳ", ">"]} +{"text": "<|endoftext|>\"'sfi", "tokens": 11, "pieces": ["<|", "endoftext", "|>\"'", "sfi"]} +{"text": "(m12345678㍿३'Re(İ'D( #$%sZ'llé0🙂‍­9", "tokens": 28, "pieces": ["(m", "123", "456", "78", "㍿", "३", "'Re", "(İ", "'D", "(", " ", "#$%", "sZ", "'ll", "é", "0", "🙂‍­", "9"]} +{"text": "ſ(字\t​!ſＡ٣٤٥٦😀🏽　 ३'TfiＡ…'s㋿<'VE", "tokens": 44, "pieces": ["ſ", "(字", "\t", "​!", "ſＡ", "٣٤٥", "٦", "😀🏽", "　", " ", "३", "'T", "fiＡ", "…", "'s", "㋿<'", "VE"]} +{"text": "' d字😀🏽­Z<|fim_prefix|>Ⅳ
fi", "tokens": 24, "pieces": ["'", " ", " d字", "😀🏽­", "Z", "<|", "fim", "_prefix", "|>", "Ⅳ", "
fi"]} +{"text": "😀🏽㍿", "tokens": 8, "pieces": ["😀🏽㍿"]} +{"text": "\t\n<|endoftext|>m,\r\n\r\n\u000b🙂\"<|endoftext|>\na", "tokens": 23, "pieces": ["\t\n", "<|", "endoftext", "|><", "EOT", ">m", ",\r\n\r\n", "\u000b", "🙂\"<|", "endoftext", "|>\n", "a"]} +{"text": "s,\"…!!'res‍Dž ḍ̇漢ⅣDž12345678'D'M😀🏽👍🏽 \n'D 
", "tokens": 44, "pieces": ["s", ",\"", "…", "!!'", "res", "‍Dž", " ", " ḋ", "̣漢", "Ⅳ", "Dž", "123", "456", "78", "'D", "'M", "😀🏽👍🏽", " \n", "'D", " 
"]} +{"text": "ḍ̇㋿ع'M''M m漢\r　'D0s'TZEOTfi­", "tokens": 30, "pieces": ["ḋ", "̣㋿", "ع", "'M", "''", "M", " m漢", "\r", "　", "'D", "0", "s", "'T", "ZEOTfi", "­"]} +{"text": "🙂åe㋿'D….ḍ̇́\n.DžⅣḍ̇😀🏽'VE'S漢😀🏽Ａ!!d(a𐞁 \nZ<|fim_prefix|>٣٤٥٦EOT", "tokens": 77, "pieces": ["🙂<", "EOT", ">a", "̊e", "㋿'", "D", "…", ".ḋ", "̣́\n", ".Dž", "Ⅳ", "ḋ", "̣😀🏽'", "VE", "'S", "漢", "😀🏽", "Ａ", "!!", "d", "(a𐞁", " \n", "Z", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "EOT"]} +{"text": "‍ aé<|endoftext|>Ａ٣٤٥٦ſ\u000bꟲ\nt'll㋿ß\"!!\nséå- ع'M漢\r\n\r\n 'DEOT㍿12345678½Ａ", "tokens": 60, "pieces": ["‍", " aé", "<|", "endoftext", "|>", "Ａ", "٣٤٥", "٦", "ſ", "\u000bꟲ", "\n", "t", "'ll", "㋿ß", "\"!!\n", "se", "́a", "̊-", " ع", "'M", "漢", "\r\n\r\n", " '", "DEOT", "㍿", "123", "456", "78½", "Ａ"]} +{"text": "dét\u000b \n#$%\t#$%'S12345678\r\n𐞁\u000b'T‍ 'ree㍿㍿\"İＡ. \nd\r\n\r\nééaß\n", "tokens": 47, "pieces": ["dét", "\u000b \n", "#$%", "\t", "#$%'", "S", "123", "456", "78", "\r\n", "𐞁", "\u000b", "'T", "‍", " ", " '", "ree", "㍿㍿\"", "İＡ", ".", " \n", "d", "\r\n\r\n", "ée", "́<", "META", "_START", ">aß", "\n"]} +{"text": "12345678İ\u000b'VE\u000b<|fim_prefix|>99,👍🏽Ａ'D😀🏽½ 👍🏽'll३-á'Reß'll", "tokens": 48, "pieces": ["123", "456", "78", "İ", "\u000b", "'VE", "\u000b", "<|", "fim", "_prefix", "|>", "99", ",👍🏽", "Ａ", "'D", "😀🏽", "½", " ", " 👍🏽'", "ll", "३", "-a", "́'", "Reß", "'ll"]} +{"text": " ㋿\"", "tokens": 6, "pieces": [" ", " ㋿\""]} +{"text": "!!é's.m>'M🙂İ'<|fim_prefix|>\r\n(　\u000b's ́,٣٤٥٦.'T'll
½m<'VEd'ſ", "tokens": 46, "pieces": ["!!", "e", "́'", "s", ".m", ">'", "M", "🙂İ", "'<|", "fim", "_prefix", "|>\r\n", "(", "　", "\u000b", "'s", " ", "́,", "٣٤٥", "٦", ".'", "T", "'ll", "
", "½", "m", "<'", "VEd", "'ſ"]} +{"text": "-\n😀🏽(dd३!!'ſå'T\r\n\u000bſ'VE'VEa…\u000b​ 're", "tokens": 34, "pieces": ["-\n", "😀🏽(", "dd", "३", "!!'", "ſa", "̊'", "T", "\r\n", "\u000bſ", "'VE", "'VE", "a", "…", "\u000b", "​", " '", "re"]} +{"text": "\r\n\r\n<|endoftext|>12345678­‍\r\nع'sé'T>Ⅳ
ꟲ!́<|endoftext|>", "tokens": 38, "pieces": ["\r\n\r\n", "<|", "endoftext", "|>", "123", "456", "78", "­‍\r\n", "ع", "'s", "e", "́'", "T", ">", "Ⅳ", "
ꟲ", "!́<|", "endoftext", "|>"]} +{"text": "'D'VEt\r㋿\r", "tokens": 12, "pieces": ["'D", "'VE", "t", "\r", "㋿\r"]} +{"text": "'Reå", "tokens": 11, "pieces": ["'Re", "a", "̊<", "EOT", ">"]} +{"text": "t'll٣٤٥٦­\ra　Ａ ḍ̇🙂-𐞁'reⅣDžåé㍿\r\n\r\n'Re!\n👍🏽\r\n\r\n\r'S👍🏽å३‍\u000b㍿
­", "tokens": 74, "pieces": ["t", "'ll", "٣٤٥", "٦", "­\r", "a", "　Ａ", " ḋ", "̣🙂-", "𐞁", "'re", "Ⅳ", "Dža", "̊e", "́㍿\r\n\r\n", "'Re", "!\n", "👍🏽\r\n\r\n\r", "'S", "👍🏽", "a", "̊", "३", "‍", "\u000b", "㍿", "
", "­"]} +{"text": "!!Ⅳ🙂Dž\r\n\r\n're \ńd漢½'S🙂må\"'S٣٤٥٦!🙂ß\t‍é", "tokens": 40, "pieces": ["!!", "Ⅳ", "🙂Dž", "\r\n\r\n", "'re", " \n", "́d漢", "½", "'S", "🙂ma", "̊\"'", "S", "٣٤٥", "٦", "!🙂", "ß", "\t", "‍e", "́"]} +{"text": ".­é 12345678'Re½👍🏽ꟲ字,!ſ'ſDž
👍🏽🙂३\n ", "tokens": 42, "pieces": [".­", "e", "́", " ", "123", "456", "78", "'Re", "½", "👍🏽", "ꟲ字", ",!", "ſ", "'ſ", "Dž", "
", "👍🏽🙂", "३", "\n "]} +{"text": "eعé٣٤٥٦🙂 >­é ꟲ🙂\r\n\r\n\r", "tokens": 26, "pieces": ["eعé", "٣٤٥", "٦", "🙂", " ", " >­", "e", "́", " ", " ꟲ", "🙂\r\n\r\n\r"]} +{"text": "åd<|endoftext|>'́\r\n\r\n's㋿😀🏽ḍ̇😀🏽'Re🙂å0'D‍#$%'VEḍ̇🙂　(🙂sd#$%éa\"\u000b­ſİ\r\n\r\n", "tokens": 69, "pieces": ["a", "̊d", "<|", "endoftext", "|>'́\r\n\r\n", "'s", "㋿😀🏽", "ḋ", "̣😀🏽'", "Re", "🙂a", "̊", "0", "'D", "‍#$%'", "VEḋ", "̣🙂", "　", "(🙂", "sd", "#$%", "éa", "\"", "\u000b", "­ſİ", "\r\n\r\n"]} +{"text": "\rDžZmZ'Dt!'s!㍿", "tokens": 14, "pieces": ["\r", "DžZmZ", "'D", "t", "!'", "s", "!㍿"]} +{"text": "ݽ#$%ß👍🏽'll­'ſ\n字'Dé㍿a'३'D 😀🏽\t <|endoftext|>Dž३fiꟲEOTtdé…>'ſ", "tokens": 59, "pieces": ["İ", "½", "#$%", "ß", "👍🏽'", "ll", "­'", "ſ", "\n", "字", "'D", "e", "́㍿", "a", "'", "३", "'D", " 😀🏽", "\t ", " <|", "endoftext", "|>", "Dž", "३", "fiꟲEOTtdé", "…", ">'", "ſ"]} +{"text": "EOT ́<|fim_prefix|>a'llßع \n!,😀🏽é😀🏽-İ'sß#$%㋿é", "tokens": 39, "pieces": ["EOT", " ", "́<|", "fim", "_prefix", "|>", "a", "'ll", "ßع", " \n", "!,😀🏽", "é", "😀🏽-", "İ", "'s", "ß", "#$%㋿", "e", "́"]} +{"text": "½<😀🏽'Re", "tokens": 9, "pieces": ["½", "<😀🏽'", "Re"]} +{"text": "३'Re<|endoftext|>😀🏽!३ḍ̇!!.😀🏽\u000b'M>㍿", "tokens": 35, "pieces": ["३", "'Re", "<|", "endoftext", "|>😀🏽!", "३", "ḋ", "̣!!.😀🏽", "\u000b", "'M", ">㍿"]} +{"text": "((", "tokens": 1, "pieces": ["(("]} +{"text": "'sée 'S‍\t.,Džm½….‍㍿Ⅳ　𐞁'll٣٤٥٦عa🙂
-ꟲ\r\n\r\n३'ſ<'T's!'-EOT
", "tokens": 63, "pieces": ["'s", "e", "́e", " '", "S", "‍", "\t", ".,", "Džm", "½", "…", ".‍㍿", "Ⅳ", "　𐞁", "'ll", "٣٤٥", "٦", "عa", "🙂", "
", "-ꟲ", "\r\n\r\n", "३", "'ſ", "<'", "T", "'s", "!'-", "EOT", "
"]} +{"text": "\t字…'s字em sae\" 'ſ12345678é'reİ'ss㋿<|fim_prefix|>#$%　e漢'VE.fi>…é½ꟲ…", "tokens": 60, "pieces": ["Ⅳ", "'Re", "\u000b\n", "'ſ", "\tt", "", " sae", "\"", " ", " '", "ſ", "123", "456", "78", "é", "'re", "İ", "'s", "s", "㋿<|", "fim", "_prefix", "|>#$%", "　e漢", "'VE", ".fi", ">", "…e", "́", "½", "ꟲ", "…"]} +{"text": "ḍ̇Dž<|endoftext|>'MEOTＡ ſ'lls٣٤٥٦<|endoftext|>Zꟲ,a,\t\r\n\r\n\r\n", "tokens": 46, "pieces": ["ḋ", "̣Dž", "<|", "endoftext", "|>'", "MEOTＡ", " ", " ſ", "'ll", "s", "٣٤٥", "٦", "<|", "endoftext", "|>", "Zꟲ", ",a", ",", "\t\r\n\r\n\r\n"]} +{"text": "EOT…㋿'T,t9'Me\u000b( Ａ!!('ſⅣfi🙂're!0(ḍ̇'T'D㍿
's'll", "tokens": 47, "pieces": ["EOT", "…", "㋿'", "T", ",t", "9", "'M", "e", "\u000b", "(", " Ａ", "!!('", "ſ", "Ⅳ", "fi", "🙂'", "re", "!", "0", "(ḋ", "̣'", "T", "'D", "㍿", "
", "'s", "'ll"]} +{"text": "m<<|fim_prefix|>!!å🙂!!<|fim_prefix|>fi😀🏽éꟲſ'Re३ \n0😀🏽\ńå½
\r\n\r\n!!a'VE-ع>", "tokens": 62, "pieces": ["m", "<<|", "fim", "_prefix", "|>!!", "a", "̊🙂!!<|", "fim", "_prefix", "|>", "fi", "😀🏽", "e", "́ꟲſ", "'Re", "३", " \n", "0", "😀🏽\n", "́a", "̊", "½", "
\r\n\r\n", "!!", "a", "'VE", "-ع", ">"]} +{"text": "<|endoftext|>ß \nfi'Re9­'T\t 'Må\"㋿", "tokens": 25, "pieces": ["<|", "endoftext", "|>", "ß", " \n", "fi", "'Re", "9", "­'", "T", "\t", " '", "Ma", "̊\"㋿"]} +{"text": "Ⅳ(d­\r>éⅣ", "tokens": 9, "pieces": ["Ⅳ", "(d", "­\r", ">é", "Ⅳ"]} +{"text": "ßİZ,Ａé\"t 'VE‍'>'T're½!㍿\tm.<|endoftext|>ḍ̇,'VEd0!Zm!!e漢😀🏽t­å.­", "tokens": 58, "pieces": ["ßİZ", ",Ａé", "\"t", " '", "VE", "‍'>'", "T", "'re", "½", "!㍿", "\tm", ".<|", "endoftext", "|>", "ḋ", "̣,'", "VEd", "0", "!Zm", "!!", "e漢", "😀🏽", "t", "­a", "̊.­"]} +{"text": "'se'reeDžZ-　e　Ⅳ½Ⅳ👍🏽🙂tat𐞁ḍ̇'VE9<ꟲſİ𐞁(‍étꟲ", "tokens": 57, "pieces": ["'s", "e", "'re", "eDžZ", "-", "　e", "　", "Ⅳ½Ⅳ", "👍🏽🙂", "tat𐞁ḋ", "̣'", "VE", "9", "<ꟲſİ𐞁", "(‍", "e", "́tꟲ"]} +{"text": "ꟲ'ſ<|fim_prefix|>'M½ß\r\n\r\né½å​٣٤٥٦\u000b…'T​Ⅳsع🙂0", "tokens": 43, "pieces": ["ꟲ", "'ſ", "<|", "fim", "_prefix", "|>'", "M", "½", "ß", "\r\n\r\n", "é", "½", "a", "̊​", "٣٤٥", "٦", "\u000b", "…", "'T", "​", "Ⅳ", "sع", "🙂", "0"]} +{"text": "é're ३.‍dßع½𐞁<　<\r're'T's(ḍ̇𐞁 \r\n\r\n\n\r'S12345678'\r\n\r\né\r\n", "tokens": 43, "pieces": ["é", "'re", " ", "३", ".‍", "dßع", "½", "𐞁", "<", "　", "<\r", "'re", "'T", "'s", "(ḋ", "̣𐞁", " \r\n\r\n\n\r", "'S", "123", "456", "78", "'\r\n\r\n", "é", "\r\n"]} +{"text": "a
. 漢- -<|fim_prefix|>ع字٣٤٥٦fiꟲ'ſ漢'T<㋿'T#$%a#$%dⅣ#$%😀🏽", "tokens": 59, "pieces": ["a", "
", ".", " 漢", "-", " ", " -<|", "fim", "_prefix", "|>", "ع字", "٣٤٥", "٦", "fiꟲ", "'ſ", "漢", "'T", "<㋿'", "T", "#$%", "a", "#$%", "d", "Ⅳ", "#$%😀🏽"]} +{"text": "㋿d's'Dž'sDž<|endoftext|>'ll'VE,(­åſ𐞁ßDž\u000bfi>-‍t å", "tokens": 50, "pieces": ["㋿d", "'s", "'Dž", "'s", "Dž", "<|", "endoftext", "|>'", "ll", "'VE", ",(­", "a", "̊ſ𐞁ßDž", "\u000bfi", ">-‍", "t", " ", " a", "̊<", "META", "_START", ">"]} +{"text": "'re#$%½a'S\t-< \n12345678'VE\r\n\r\n
…,\"t　 ", "tokens": 26, "pieces": ["'re", "#$%", "½", "a", "'S", "\t", "-<", " \n", "123", "456", "78", "'", "VE", "\r\n\r\n", "
", "…", ",\"", "t", "　 "]} +{"text": "३(漢'ſ­㍿ >", "tokens": 14, "pieces": ["३", "(漢", "'ſ", "­㍿", " ", ">"]} +{"text": "
'T'ſ٣٤٥٦\"\u000bꟲ<|fim_prefix|>'M'llſ\r\n'D\r\n\r\n'ſ٣٤٥٦", "tokens": 44, "pieces": ["
", "'T", "'ſ", "٣٤٥", "٦", "\"", "\u000bꟲ", "<|", "fim", "_prefix", "|>'", "M", "'ll", "ſ", "\r\n", "'D", "\r\n\r\n", "'ſ", "٣٤٥", "٦"]} +{"text": "İ​é'S\r𐞁\r\n\r\n<…​!!9'Re", "tokens": 23, "pieces": ["İ", "​", "é", "'S", "\r", "𐞁", "\r\n\r\n", "<", "…", "​!!", "9", "'Re"]} +{"text": "\r\n\r\n0fi0-\u000b", "tokens": 9, "pieces": ["", "0", "fi", "0", "-", "\u000b"]} +{"text": "'ſꟲZ<|endoftext|>\u000b", "tokens": 15, "pieces": ["'ſ", "ꟲZ", "<|", "endoftext", "|>", "\u000b"]} +{"text": "dt!!Z \n'T'D0\rEOT'Re<|fim_prefix|>aß\u000baⅣ !字İ'DＡſ's\"ḍ̇'S漢Ａe३a\r\n\r\nå'Re", "tokens": 58, "pieces": ["dt", "!!", "Z", " \n", "'T", "'D", "0", "\r", "EOT", "'Re", "<|", "fim", "_prefix", "|>", "aß", "\u000ba", "Ⅳ", " !", "字İ", "'D", "Ａſ", "'s", "\"ḋ", "̣'", "S漢Ａe", "३", "a", "\r\n\r\n", "a", "̊'", "Re"]} +{"text": "'S　'T‍\t12345678m(\"字", "tokens": 12, "pieces": ["'S", "　", "'T", "‍", "\t", "123", "456", "78", "m", "(\"", "字"]} +{"text": "\r\n'ſ<漢'VE>,'Re\u000btfié漢
ḍ̇
'Mḍ̇ß's#$%\rع \nＡ!!", "tokens": 49, "pieces": ["\r\n", "'ſ", "<漢", "'VE", ">,'", "Re", "\u000btfie", "́漢", "
ḋ", "̣", "
", "'M", "ḋ", "̣<", "META", "_START", ">ß", "'s", "#$%\r", "ع", " \n", "Ａ", "!!"]} +{"text": "!字㋿字ß<|fim_prefix|><🙂ḍ̇\r漢 ſfi…\r\n\r\né​EOT'Mḍ̇m
ḿDžé­½<|endoftext|>字9", "tokens": 68, "pieces": ["!", "字", "㋿字ß", "<|", "fim", "_prefix", "|><🙂", "ḋ", "̣\r", "漢", " ſfi", "…\r\n\r\n", "é", "​EOT", "'M", "ḋ", "̣m", "
m", "́Dže", "́­", "½", "<|", "endoftext", "|>", "字", "", "9"]} +{"text": "'M👍🏽字'ḍ̇d'D\r\n\r\nfi \n​åİZ'S<|fim_prefix|>'ſ'ſ\r\n\r\neＡ㍿漢're漢d<|fim_prefix|>9\n#$%😀🏽İ", "tokens": 72, "pieces": ["'M", "👍🏽", "字", "'ḋ", "̣d", "'D", "\r\n\r\n", "fi", " \n", "​a", "̊İ", "Z", "'S", "<|", "fim", "_prefix", "|>'", "ſ", "'ſ", "\r\n\r\n", "eＡ", "㍿漢", "'re", "漢d", "<|", "fim", "_prefix", "|>", "9", "\n", "#$%😀🏽", "İ"]} +{"text": "'reع\r\n\nＡ'T😀🏽", "tokens": 37, "pieces": ["'", "reع", "\r\n", "\n", "Ａ", "'T", "😀🏽"]} +{"text": "　!!12345678\t'MZ'T.\t'VE𐞁\r!!٣٤٥٦!! ", "tokens": 32, "pieces": ["　", "!!", "123", "456", "78", "\t", "'M", "Z", "'", "T", ".", "\t", "'VE", "𐞁", "\r", "!!", "٣٤٥", "٦", "!!", " "]} +{"text": "!!ꟲ漢 𐞁½👍🏽9'!!漢,'re漢　​é\r字!'VEefi\n<|fim_prefix|>d\r\nꟲ", "tokens": 51, "pieces": ["!!", "ꟲ漢", " 𐞁", "½", "👍🏽", "9", "'!!", "漢", ",'", "re漢", "　", "​e", "́\r", "字", "!'", "VEefi", "\n", "<|", "fim", "_prefix", "|>", "d", "\r\n", "ꟲ"]} +{"text": " \n\u000b", "tokens": 2, "pieces": [" \n\u000b"]} +{"text": "<'Reé<|endoftext|>.'ReEOT \n", "tokens": 21, "pieces": ["<<", "EOT", ">'", "Reé", "<|", "endoftext", "|><", "META", "_START", ">.'", "ReEOT", " \n"]} +{"text": "m\n­ع é́'S'Mt!!''ll\r\n\r\n<'ſ,'", "tokens": 21, "pieces": ["m", "\n", "­ع", " ", " é", "́'", "S", "'M", "t", "!!''", "ll", "\r\n\r\n", "<'", "ſ", ",'"]} +{"text": "t 'M
fi.EOTꟲḍ̇\t​\u000be'SＡ'Re\u000b,", "tokens": 27, "pieces": ["t", " '", "M", "
fi", ".EOTꟲḋ", "̣", "\t", "​", "\u000be", "'S", "Ａ", "'Re", "\u000b", ","]} +{"text": "ßm'D .🙂\u000b<|endoftext|>Ａ'VE'T३9<|fim_prefix|>ḍ̇ḍ̇‍s<|fim_prefix|><|fim_prefix|> #$%12345678å<|endoftext|>\r0tع(字\r", "tokens": 82, "pieces": ["ßm", "'D", " ", ".🙂", "\u000b", "<|", "endoftext", "|>", "Ａ", "'VE", "'T", "३9", "<|", "fim", "_prefix", "|>", "ḋ", "̣ḋ", "̣‍", "s", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", " ", "#$%", "123", "456", "78", "a", "̊<|", "endoftext", "|><", "EOT", ">\r", "0", "tع", "(字", "\r"]} +{"text": "0'VE'll 'VE ,'sEOT12345678字<字㍿㋿!'Mꟲ!!Dž\r.åm'S0\tⅣDž#$%s½'ll🙂㍿'ll…㋿", "tokens": 61, "pieces": ["0", "'VE", "'ll", " ", "'VE", " ", " ,'", "sEOT", "123", "456", "78", "字", "<字", "㍿㋿!'", "Mꟲ", "!!", "Dž", "\r", ".a", "̊m", "'S", "0", "\t", "Ⅳ", "Dž", "#$%", "s", "½", "'ll", "🙂㍿'", "ll", "…", "㋿"]} +{"text": "<-EOT​ \n㍿漢'VEdꟲ123456789", "tokens": 23, "pieces": ["<-", "EOT", "​", " \n", "㍿", "漢", "'VE", "dꟲ", "123", "456", "789"]} +{"text": "\r .٣٤٥٦漢㋿'ll👍🏽t🙂'D\t㋿,'VE..'ReＡ'ſ!!<|fim_prefix|>　\u000b <(", "tokens": 57, "pieces": ["\r", " ", ".", "٣٤٥", "٦", "漢", "㋿'", "ll", "👍🏽", "t", "🙂'", "D", "\t", "㋿,'", "VE", "..'", "ReＡ", "'ſ", "!!<|", "fim", "_prefix", "|>", "　\u000b", " <("]} +{"text": "…då!('T👍🏽,字漢㋿'ll12345678½\u000båå,👍🏽\"", "tokens": 42, "pieces": ["…da", "̊!('", "T", "👍🏽,", "字漢", "㋿'", "ll", "123", "456", "78½", "\u000ba", "̊a", "̊,👍🏽\""]} +{"text": "ꟲéꟲ<|endoftext|>😀🏽s<|fim_prefix|>'Daéعt­'12345678 ३d​\t३'M#$%#$%\tDž09​👍🏽字<|fim_prefix|>漢fiꟲ", "tokens": 75, "pieces": ["ꟲéꟲ", "<|", "endoftext", "|>😀🏽", "s", "<|", "fim", "_prefix", "|>'", "Daéعt", "­'", "123", "456", "78", " ", "३", "d", "​", "\t", "३", "'M", "#$%#$%", "\tDž", "09", "​👍🏽", "字", "<|", "fim", "_prefix", "|>", "漢fiꟲ"]} +{"text": "éfiꟲß'ſ\r\n\r\n'VE\u000b\r ­'llḍ̇𐞁12345678ßꟲ½12345678İＡ漢'S12345678fi9🙂'Mt!!s", "tokens": 64, "pieces": ["e", "́fiꟲ", "ß", "'ſ", "\r\n\r\n", "'VE", "\u000b\r", " ­'", "llḋ", "̣𐞁", "123", "456", "78", "ßꟲ", "½12", "345", "678", "İＡ漢", "'S", "123", "456", "78", "fi", "9", "🙂'", "Mt", "!!", "s"]} +{"text": "#$%\"Džé\r\nEOTⅣ\u000b \n.éaEOTḍ̇㍿ꟲd𐞁㍿Džſ‍s\"İ㋿\"fi<|fim_prefix|>'D\r\n\r\n", "tokens": 63, "pieces": ["#$%\"", "Dž", "é", "\r\n", "EOT", "Ⅳ", "\u000b \n", ".éaEOTḋ", "̣㍿", "ꟲd𐞁", "㍿Džſ", "‍s", "\"İ", "㋿\"", "fi", "<|", "fim", "_prefix", "|>'", "D", "\r\n\r\n"]} +{"text": "\tt'S'SEOT​Z.eꟲ'll'D½İ३字ḿ'!!tå é'll'Dİ(#$%­𐞁'll٣٤٥٦​t'Re\t …", "tokens": 56, "pieces": ["\tt", "'S", "'S", "EOT", "​Z", ".eꟲ", "'ll", "'D", "½", "İ", "३", "字m", "́'!!", "ta", "̊", " ", " e", "́'", "ll", "'D", "İ", "(#$%­", "𐞁", "'ll", "٣٤٥", "٦", "​t", "'Re", "\t …"]} +{"text": "'VE>'ll٣٤٥٦", "tokens": 12, "pieces": ["'VE", ">'", "ll", "٣٤٥", "٦"]} +{"text": "­漢عe𐞁>ts e'!!'Red漢're!d!!ع\rsfi👍🏽", "tokens": 39, "pieces": ["­漢", "عe𐞁", ">", "ts", " ", " e", "'!!'", "Red漢", "'re", "!d", "!!", "ع", "\r", "sfi", "👍🏽"]} +{"text": "㋿३\r\n<|fim_prefix|>Ⅳ'T'Mḍ̇ZZ\r\t \n\",'S'VE", "tokens": 34, "pieces": ["㋿", "३", "\r\n", "<", "META", "_START", "><|", "fim", "_prefix", "|>", "Ⅳ", "'T", "'M", "ḋ", "̣ZZ", "\r\t \n", "\",'", "S", "'VE"]} +{"text": "…'ſ\r\n\r\n\r\rⅣ　​'S>'T‍<|fim_prefix|>9ß\r'ſ''VE.!!'VEm'Dé…\r\n㍿ſꟲé\n\raEOT", "tokens": 64, "pieces": ["…", "'ſ", "\r\n\r\n\r\r", "", "Ⅳ", "　", "​'", "S", ">'", "T", "‍<|", "fim", "_prefix", "|><", "EOT", ">", "9", "ß", "\r", "'ſ", "''", "VE", ".!!'", "VEm", "'D", "e", "́", "…\r\n", "㍿ſꟲé", "\n\r", "aEOT"]} +{"text": "#$%\r\n字'St‍'VE", "tokens": 9, "pieces": ["#$%\r\n", "字", "'S", "t", "‍'", "VE"]} +{"text": "Z​m'sß٣٤٥٦'ſ㋿ \n'́", "tokens": 22, "pieces": ["Z", "​m", "'s", "ß", "٣٤٥", "٦", "'ſ", "㋿", " \n", "'́"]} +{"text": "ſع\u000bⅣ㋿!! -'re३fi<fi", "tokens": 21, "pieces": ["ſع", "\u000b", "Ⅳ", "㋿!!", " ", " -'", "re", "३", "fi", "<fi"]} +{"text": "😀🏽\"ع-́", "tokens": 9, "pieces": ["😀🏽\"", "ع", "-́"]} +{"text": "ꟲ'sⅣ 0́́'T!!㋿#$%३Ａt<", "tokens": 24, "pieces": ["ꟲ", "'s", "Ⅳ", " ", "0", "́́'", "T", "!!㋿#$%", "३", "Ａt", "<"]} +{"text": "> 'S\r\n\r\n'll''VEꟲꟲ\r\n­ＡZ'Re\r\n漢9<½\r\n<‍İ\r\n­‍\n12345678'M.<|fim_prefix|>\t
tt", "tokens": 50, "pieces": [">", " '", "S", "\r\n\r\n", "'ll", "''", "VEꟲꟲ", "\r\n", "­ＡZ", "'Re", "\r\n", "漢", "9", "<", "½", "\r\n", "<‍", "İ", "\r\n", "­‍\n", "123", "456", "78", "'M", ".<|", "fim", "_prefix", "|>", "\t", "
tt"]} +{"text": "𐞁", "tokens": 4, "pieces": ["𐞁"]} +{"text": "\t​", "tokens": 2, "pieces": ["\t", "​"]} +{"text": "\u000b́\r\naع'S\r\n'S\u000bḍ̇'S'Re㋿　'S'ſ,👍🏽'll12345678s 😀🏽\t\u000bm 'ḍ̇#$%", "tokens": 58, "pieces": ["\u000b", "́\r\n", "aع", "'S", "\r\n", "'S", "\u000bḋ", "̣'", "S", "'Re", "㋿", "　", "'S", "'ſ", ",👍🏽'", "ll", "123", "456", "78", "s", " ", " 😀🏽", "\t", "\u000bm", " ", "'<", "EOT", ">ḋ", "̣#$%"]} +{"text": "\"é're<|endoftext|>", "tokens": 10, "pieces": ["\"é", "'re", "<|", "endoftext", "|>"]} +{"text": "👍🏽ꟲ字Ⅳ𐞁9e!!㋿'Mfi㍿'M9!", "tokens": 33, "pieces": ["👍🏽", "ꟲ字", "Ⅳ", "𐞁", "9", "e", "!!㋿'", "Mfi", "㍿'", "M", "9", "!"]} +{"text": "İé́'S'S'VE‍\"'D\r\n\r\n👍🏽 daＡḍ̇ \nＡ'DDž㍿", "tokens": 40, "pieces": ["İé", "́'", "S", "'S", "'VE", "‍\"'", "D", "\r\n\r\n", "👍🏽", " ", " daＡḋ", "̣", " \n", "Ａ", "'", "DDž", "㍿"]} +{"text": "Dž\r\n\r\n.é　Ａ<|fim_prefix|>es.ع漢'ſ<|endoftext|>ß<|fim_prefix|>ß<|endoftext|><́>(🙂ع<|fim_prefix|>'ll字", "tokens": 63, "pieces": ["Dž", "\r\n\r\n", ".é", "　Ａ", "<|", "fim", "_prefix", "|>", "es", ".ع", "漢", "'ſ", "<|", "endoftext", "|>", "ß", "<|", "fim", "_prefix", "|>", "ß", "<|", "endoftext", "|><́>(🙂", "ع", "<|", "fim", "_prefix", "|>'", "ll字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'Re字\n#$%!
\r\n\r\n<|fim_prefix|>㍿ꟲＡ9'…
<|fim_prefix|>\r\n\r\n३EOT 
 \né'D0'M‍", "tokens": 50, "pieces": ["'Re", "字", "\n", "#$%!", "
\r\n\r\n", "<|", "fim", "_prefix", "|>㍿", "ꟲＡ", "9", "'", "…", "
", "<|", "fim", "_prefix", "|>\r\n\r\n", "३", "EOT", " 
 \n", "é", "'D", "0", "'M", "‍"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ſ#$%'SＡ!! \n\"‍!!ḍ̇\r\n­å(,#$%😀🏽-…e \r(!! 9ß　m\tꟲ('Re0é🙂㍿-", "tokens": 62, "pieces": ["ſ", "#$%'", "SＡ", "!!", " \n", "\"‍!!", "ḋ", "̣\r\n", "­a", "̊(,#$%😀🏽<", "EOT", ">-", "…e", " \r", "(!!", " ", "9", "ß", "　m", "\tꟲ", "('", "Re", "0", "é", "🙂㍿-"]} +{"text": "ꟲé>\"'ll'S( ḍ̇\r<|endoftext|>　-<🙂", "tokens": 54, "pieces": ["ꟲé", ">\"'", "ll", "'S", "(", " ", " ḋ", "̣\r", "<|", "endoftext", "|>", "　", "-<", "ta", "…", "‍\r\n", "́a", "'S", "a", "̊", "Ⅳ", "a", "!!\r", "🙂<|", "endoftext", "|><🙂"]} +{"text": "\"\nsDžte\u000b\u000bé㋿½
", "tokens": 15, "pieces": ["\"\n", "sDžte", "\u000b", "\u000be", "́㋿", "½", "
"]} +{"text": "<|endoftext|>>em\u000b\ŕßé(Ⅳ㋿'M'MＡ éḍ̇!fi!\r\n\r\n'T's.🙂'ReZ'Re…#$%", "tokens": 48, "pieces": ["<|", "endoftext", "|>>", "em", "\u000b\r", "́ßé", "(", "Ⅳ", "㋿'", "M", "'M", "Ａ", " éḋ", "̣!", "fi", "!\r\n\r\n", "'T", "'s", ".🙂'", "ReZ", "'Re", "…", "#$%"]} +{"text": "Z字9#$%!!efim123456780<|endoftext|>", "tokens": 23, "pieces": ["Z字", "9", "#$%!!", "efi", "m", "123", "456", "780", "<|", "endoftext", "|>"]} +{"text": "👍🏽'Re㍿ſß߅'re'reꟲ\t\"字ſ\"12345678", "tokens": 24, "pieces": ["\r", "<|", "endoftext", "|>", "…", "'re", "'re", "ꟲ", "\t", "\"字ſ", "\"", "123", "456", "78"]} +{"text": "'T'll'VE (‍ſ३½👍🏽𐞁'VE😀🏽Ⅳ'\n👍🏽㍿​'D'lla#$% ", "tokens": 49, "pieces": ["'T", "'ll", "'VE", " (‍", "ſ", "३½", "👍🏽", "𐞁", "'VE", "😀🏽", "Ⅳ", "'\n", "👍🏽㍿​'", "D", "'ll", "a", "#$%", " "]} +{"text": "d…-\r\n\r\n'S!漢's…!m­­'T12345678'M'Reع
\u000b'S<|fim_prefix|>ع\r", "tokens": 37, "pieces": ["d", "…", "-\r\n\r\n", "'S", "!漢", "'s", "…", "!m", "­­'", "T", "123", "456", "78", "'M", "'Re", "ع", "
", "\u000b", "'S", "<|", "fim", "_prefix", "|>", "ع", "\r"]} +{"text": "'D👍🏽𐞁<|fim_prefix|>Ⅳḍ̇'M字<|endoftext|>३", "tokens": 39, "pieces": ["'D", "👍🏽", "𐞁", "<|", "fim", "_prefix", "|>", "Ⅳ", "ḋ", "̣'", "M字", "<|", "endoftext", "|>", "३"]} +{"text": "9㋿e'TEOTafi😀🏽!!<字漢'M<'VE‍åꟲ fi\u000b😀🏽é​'M#$%", "tokens": 52, "pieces": ["9", "㋿e", "'T", "EOTafi", "😀🏽<", "EOT", ">!!<", "字漢", "'M", "<'", "VE", "‍a", "̊ꟲ", " ", " fi", "\u000b", "😀🏽", "e", "́​'", "M", "#$%"]} +{"text": "\"ḍ̇!ع٣٤٥٦EOT ,s>­'VE-İ'llⅣ", "tokens": 34, "pieces": ["\"ḋ", "̣!", "ع", "٣٤٥", "٦", "EOT", " ", " ,", "s", ">­'", "VE", "-<", "META", "_START", ">İ", "'ll", "Ⅳ"]} +{"text": "(\r\n\r\n'MfiⅣ!!ſ\rfim‍9Ａ<|fim_prefix|>'VEＡ　", "tokens": 30, "pieces": ["(\r\n\r\n", "'M", "fi", "Ⅳ", "!!", "ſ", "\r", "fim", "‍", "9", "Ａ", "<|", "fim", "_prefix", "|>'", "VEＡ", "　"]} +{"text": "e­-\n12345678字", "tokens": 7, "pieces": ["e", "­-\n", "123", "456", "78", "字"]} +{"text": "Ａé \nß9'TEOT𐞁#$%>\"912345678
12345678\n\"'s", "tokens": 27, "pieces": ["Ａé", " \n", "ß", "9", "'T", "EOT𐞁", "#$%>\"", "912", "345", "678", "
", "123", "456", "78", "\n", "\"'", "s"]} +{"text": "'VE's \n\r\nİ　३ \nm<|endoftext|>㋿'llZ'll­İ\"9e-ꟲ'Re", "tokens": 38, "pieces": ["'VE", "'", "s", " \n\r\n", "İ", "　", "३", " \n", "m", "<|", "endoftext", "|>㋿'", "llZ", "'ll", "­İ", "\"", "9", "e", "-ꟲ", "'Re"]} +{"text": "漢\r\n\r\n\r\n\r\n<|endoftext|>EOT
<|endoftext|>\n٣٤٥٦字́'ſ'S< …\u000b\t'a \n'Re　'T𐞁'Re", "tokens": 57, "pieces": ["漢", "\r\n\r\n\r\n\r\n", "<|", "endoftext", "|>", "EOT", "", "
", "<|", "endoftext", "|>\n", "٣٤٥", "٦", "字", "́'", "ſ", "'S", "<", " …\u000b", "\t", "'a", "", " \n", "'Re", "　", "'T", "𐞁", "'Re"]} +{"text": "́…'D\"s>'lla", "tokens": 7, "pieces": ["́", "…", "'D", "\"s", ">'", "lla"]} +{"text": "s\nét-'M \n'De'ſfiḍ̇'ſ(İa½ EOT🙂ed㋿😀🏽12345678a\té­​\rZe…'", "tokens": 59, "pieces": ["s", "\n", "ét", "-'", "M", " \n", "'D", "e", "'ſ", "fiḋ", "̣'", "ſ", "(İa", "½", " EOT", "🙂ed", "㋿😀🏽", "123", "456", "78", "a", "\té", "­​\r", "Ze", "…", "'"]} +{"text": "½字
字<|endoftext|>-dZ'Re 'ſ'VE‍字#$%0EOT'İ <|endoftext|>>(…Ａ漢‍‍٣٤٥٦.!!.a're", "tokens": 60, "pieces": ["½", "字", "
字", "<|", "endoftext", "|>-", "dZ", "'Re", " ", "'ſ", "'VE", "‍字", "#$%", "0", "EOT", "'İ", " ", " <|", "endoftext", "|>>(", "…Ａ漢", "‍‍", "٣٤٥", "٦", ".!!.", "a", "'re"]} +{"text": "Z\r\n\r\né<|fim_prefix|>Z字㍿
å'M'Tſm½'Re\rⅣ \n's,9DžéDž字EOT漢ḍ̇ a12345678're ‍0\u000b漢", "tokens": 62, "pieces": ["Z", "\r\n\r\n", "e", "́<|", "fim", "_prefix", "|>", "Z字", "㍿", "
a", "̊'", "M", "'T", "ſm", "½", "'Re", "\r", "Ⅳ", " \n", "'s", ",", "9", "DžéDž字EOT漢ḋ", "̣", " a", "123", "456", "78", "'re", " ‍", "0", "\u000b漢"]} +{"text": "‍　'ꟲ'T'Re<#$%'S<٣٤٥٦sDž\t \né\n's\r\nm0👍🏽😀🏽🙂　\r\n\r\nⅣ12345678ßt", "tokens": 54, "pieces": ["‍", "　", "'ꟲ", "'T", "'Re", "<#$%'", "S", "<", "٣٤٥", "٦", "sDž", "\t \n", "é", "\n", "'s", "\r\n", "m", "0", "👍🏽😀🏽🙂", "　\r\n\r\n", "Ⅳ12", "345", "678", "ßt"]} +{"text": "aae 're", "tokens": 4, "pieces": ["aae", " '", "re"]} +{"text": "s'ſع \r\n", "tokens": 6, "pieces": ["s", "'ſ", "ع", " \r\n"]} +{"text": "३\n🙂.a‍ 　012345678🙂 .…　'VE
½fiſ😀🏽", "tokens": 35, "pieces": ["३", "\n", "🙂.", "a", "‍", " ", "　", "012", "345", "678", "🙂", " ", " .", "…", "　", "'VE", "
", "½", "fiſ", "😀🏽"]} +{"text": "<|fim_prefix|>́.…😀🏽Ⅳ漢👍🏽éḍ̇字-EOTZ'ſḍ̇'#$%s😀🏽३!(ع 　́𐞁👍🏽(ZZ½-", "tokens": 77, "pieces": ["<|", "fim", "_prefix", "|>́.<", "EOT", ">", "…", "😀🏽", "Ⅳ", "漢", "👍🏽", "éḋ", "̣字", "-EOTZ", "'ſ", "ḋ", "̣'#$%", "s", "😀🏽", "३", "!(", "ع", " ", "　", "́𐞁", "👍🏽(", "ZZ", "½", "-"]} +{"text": ">­ ‍Džḍ̇ 　\t٣٤٥٦,\t12345678'S\u000b\tEOT'S\"‍ \nt'll　Dž", "tokens": 42, "pieces": [">­", " ", " ‍", "Džḋ", "̣", " 　", "\t", "٣٤٥", "٦", ",", "\t", "123", "456", "78", "'S", "\u000b", "\tEOT", "'S", "\"‍", " \n", "t", "'ll", "　Dž"]} +{"text": "å　mع'S're're!!#$%'VEé\n‍", "tokens": 18, "pieces": ["a", "̊", "　mع", "'S", "'re", "'re", "!!#$%'", "VEe", "́\n", "‍"]} +{"text": "٣٤٥٦३　ꟲe.…Zd​Dž\n㍿ ß́٣٤٥٦'re　㋿'㋿ ś\nḍ̇", "tokens": 58, "pieces": ["٣٤٥", "٦३", "　ꟲe", ".", "…Zd", "​Dž", "\n", "㍿", " ß", "́", "٣٤٥", "٦", "'re", "　", "㋿'㋿", " <", "META", "_START", ">s", "́\n", "ḋ", "̣"]} +{"text": " \n\t'MZ", "tokens": 7, "pieces": ["", " \n", "\t", "'M", "Z"]} +{"text": "ß٣٤٥٦㍿ ‍'s><㍿ع's漢
 İꟲ­\t!漢‍'Re \nß'VE", "tokens": 45, "pieces": ["ß", "٣٤٥", "٦", "㍿", " ", " ‍'", "s", "><㍿", "ع", "'s", "漢", "
 ", " İꟲ", "­", "\t", "!漢", "‍'", "Re", " \n", "ß", "'VE"]} +{"text": "İ३", "tokens": 3, "pieces": ["İ", "३"]} +{"text": " \ń字aİ!'re \r\n\r\né-\rmm,\u000b'S\r\n\u000b​!½'D\"'ſꟲs३\n0​ع　ꟲEOTé🙂", "tokens": 48, "pieces": [" \n", "́字aİ", "!'", "re", " \r\n\r\n", "é", "-\r", "mm", ",", "\u000b", "'S", "\r\n", "\u000b", "​!", "½", "'D", "\"'", "ſꟲs", "३", "\n", "0", "​ع", "　ꟲEOTe", "́🙂"]} +{"text": "३'D '　åZ12345678ع'S\"३ḍ̇🙂>\tḍ̇Za\"å \n're-é‍𐞁", "tokens": 51, "pieces": ["३", "'D", " ", "'", "　a", "̊Z", "", "123", "456", "78", "ع", "'S", "\"", "३", "ḋ", "̣🙂>", "\tḋ", "̣Za", "\"a", "̊", " \n", "'re", "-é", "‍𐞁"]} +{"text": "😀🏽İⅣ½…ꟲm　ꟲ", "tokens": 22, "pieces": ["😀🏽", "İ", "Ⅳ½", "…ꟲm", "　", "ꟲ"]} +{"text": "🙂0EOTḍ̇漢(,\réꟲ٣٤٥٦éＡ<<|fim_prefix|> \n\r\n\r\n!!", "tokens": 40, "pieces": ["🙂", "0", "EOTḋ", "̣漢", "(,\r", "éꟲ", "٣٤٥", "٦", "éＡ", "<<|", "fim", "_prefix", "|>", " \n\r\n\r\n", "!!"]} +{"text": "…é(٣٤٥٦é<|endoftext|>‍>fiſ\r\n0
('D<'D!!!!s're!!🙂\r\n'ſ'VE½३\r\n\r\né!'VE", "tokens": 55, "pieces": ["…é", "(", "٣٤٥", "٦", "é", "<|", "endoftext", "|>‍>", "fiſ", "\r\n", "0", "
", "('", "D", "<'", "D", "!!!!", "s", "'re", "!!🙂\r\n", "'ſ", "'VE", "½३", "\r\n\r\n", "e", "́!'", "VE"]} +{"text": "d!㍿<|endoftext|>‍12345678d­​漢😀🏽é<|fim_prefix|>  ſs\"ḍ̇‍", "tokens": 49, "pieces": ["d", "!㍿<|", "endoftext", "|>‍", "123", "456", "78", "d", "­​", "漢", "😀🏽", "e", "́<|", "fim", "_prefix", "|>", "  ", " ſs", "\"ḋ", "̣‍"]} +{"text": "㋿​<|endoftext|>½'ReꟲEOTs", "tokens": 19, "pieces": ["㋿​<|", "endoftext", "|>", "½", "'Re", "ꟲEOTs"]} +{"text": "㍿.ZDž𐞁'VE…½", "tokens": 16, "pieces": ["㍿.", "ZDž𐞁", "'VE", "…", "½"]} +{"text": "'SＡd­EOT😀🏽ع!a/b,", "tokens": 17, "pieces": ["'S", "Ａd", "­EOT", "😀🏽", "ع", "!a", "/b", ","]} +{"text": "İ'Sé㍿…0ſ'M", "tokens": 12, "pieces": ["İ", "'S", "é", "㍿", "…", "0", "ſ", "'M"]} +{"text": "㍿İ　­🙂Ⅳ…!!('s/\r\n½٣٤٥٦𐞁字", "tokens": 30, "pieces": ["㍿İ", "　", "­🙂", "Ⅳ", "…", "!!('", "s", "/\r\n", "½٣٤", "٥٦", "𐞁字"]} +{"text": "#$%EOT!mİſ!!aB́😀🏽/🙂>aB\r字𐞁aa", "tokens": 31, "pieces": ["#$%", "EOT", "!mİſ", "!!", "aB", "́😀🏽/🙂>", "aB", "\r", "字𐞁aa"]} +{"text": "HTTPServer's,", "tokens": 4, "pieces": ["HTTPServer", "'s", ","]} +{"text": "\u000b9-ßß'T३㍿\n/𐞁camelCaset'Ａ/ \n
😀🏽é'Re 0Z😀🏽漢字Z'M0\r\n'T\u000b‍<😀🏽", "tokens": 61, "pieces": ["\u000b", "9", "-ßß", "'T", "३", "㍿\n", "/𐞁camelCaset", "'Ａ", "/", " \n", "
", "😀🏽", "e", "́'", "Re", " ", "0", "Z", "😀🏽", "漢字Z", "'M", "0", "\r\n", "'T", "\u000b", "‍<😀🏽"]} +{"text": "\r\n\r\n \n ३Ze", "tokens": 6, "pieces": ["\r\n\r\n \n", " ", "३", "Ze"]} +{"text": "eＡABC<\r#$%\r \n EOTꟲ0aBᵃ字< ㍿
<|endoftext|>👍🏽ḍ̇!!fiaİ\r\n\r\nEOTᵃaBABC\r\n\r\n#$%'retBßHTTPServer", "tokens": 69, "pieces": ["eＡABC", "<\r", "#$%\r", " \n", " EOTꟲ", "0", "aBᵃ字", "<", " ", "㍿", "
", "<|", "endoftext", "|>👍🏽", "ḋ", "̣!!", "fiaİ", "\r\n\r\n", "EOTᵃaBABC", "\r\n\r\n", "#$%'", "retBßHTTPServer"]} +{"text": "ꟲ٣٤٥٦t", "tokens": 12, "pieces": ["ꟲ", "٣٤٥", "٦", "t"]} +{"text": "́åm'll12345678👍🏽ḍ̇12345678́fi字HTTPServer　mDžungla\r\naBABC12345678٣٤٥٦/-'Sé字́<|endoftext|>Z#$%<|fim_prefix|>", "tokens": 73, "pieces": ["́a", "̊m", "'ll", "123", "456", "78", "👍🏽", "ḋ", "̣", "123", "456", "78", "́fi字HTTPServer", "　mDžungla", "\r\n", "aBABC", "123", "456", "78٣", "٤٥٦", "/-'", "Se", "́字", "́<|", "endoftext", "|>", "Z", "#$%<|", "fim", "_prefix", "|>"]} +{"text": "'re\t \n!! \nå…!!ḍ̇<|fim_prefix|>\r\n\r\naB'Mds\n/>\t　…ع<ſꟲiOS𐞁camelCase/\r\n٣٤٥٦>,' 　camelCaseDžunglaé😀🏽", "tokens": 71, "pieces": ["'re", "\t \n", "!!", " \n", "a", "̊", "…", "!!", "ḋ", "̣<|", "fim", "_prefix", "|>\r\n\r\n", "aB", "'M", "ds", "\n", "/>", "\t　", "…ع", "<ſꟲiOS𐞁camelCase", "/\r\n", "٣٤٥", "٦", ">,'", " ", "　camelCaseDžunglaé", "😀🏽"]} +{"text": "camelCase३字iOS/!!-Ａſß-!!'ResDžunglaDž​́<\t", "tokens": 28, "pieces": ["camelCase", "३", "字iOS", "/!!-", "Ａſß", "-!!'", "ResDžunglaDž", "​́<", "\t"]} +{"text": "fiiOS…'re३HTTPServer\n­'é", "tokens": 15, "pieces": ["fiiOS", "…", "'re", "३", "HTTPServer", "\n", "­'", "e", "́"]} +{"text": "'re\n \nſ!!'re \n 
Džungla'ſ-'Tå maB", "tokens": 28, "pieces": ["'re", "\n \n", "ſ", "!!'", "re", "", " \n", " ", "
Džungla", "'ſ", "-'", "Ta", "̊", " ", " maB"]} +{"text": "aB\r/.㍿字'SDžungla漢(''Re\u000b/a/bDžEOTe<|fim_prefix|>a/b12345678<|fim_prefix|>\r\na/b㍿<\r\n\r\n\n\n/\t#$%'T'<|endoftext|>é㍿\r\n ", "tokens": 75, "pieces": ["aB", "\r", "/.㍿", "字", "'S", "Džungla漢", "(''", "Re", "\u000b", "/a", "/b", "DžEOTe", "<|", "fim", "_prefix", "|>", "a", "/b", "123", "456", "78", "<|", "fim", "_prefix", "|>\r\n", "a", "/b", "㍿<\r\n\r\n\n\n", "/", "\t", "#$%'", "T", "'<|", "endoftext", "|>", "é", "㍿\r\n", " "]} +{"text": "ḍ̇a٣٤٥٦\ta/b,,iOS/\r\n😀🏽Z0/\r\nİDžé,<|endoftext|>e'D\" \n 
३ꟲ", "tokens": 74, "pieces": ["ḋ", "̣a", "٣٤٥", "٦", "\ta", "/b", ",,", "iOS", "/\r\n", "😀🏽", "Z", "0", "/\r\n", "İDže", "́,<", "a", "/b", "‍\n", "\t", "㍿!!", "sAb", "
", "!!", "t", "<<|", "endoftext", "|><|", "endoftext", "|>", "e", "'D", "\"", " \n", " ", "
", "३", "ꟲ"]} +{"text": "㍿Ａ<.camelCaseİ\r\n\r\n éZ/\r\n\r\n\r\nع'll½\u000ba/b\naDžungla<|endoftext|>'reAb漢\n/(HTTPServer　　
<|fim_prefix|>'re, ", "tokens": 55, "pieces": ["㍿Ａ", "<.", "camelCaseİ", "\r\n\r\n", " éZ", "/\r\n\r\n\r\n", "ع", "'ll", "½", "\u000ba", "/b", "\n", "aDžungla", "<|", "endoftext", "|>'", "reAb漢", "\n", "/(", "HTTPServer", "　　", "
", "<|", "fim", "_prefix", "|>'", "re", ",", " "]} +{"text": "<|endoftext|>Ⅳ \n å字\td're漢😀🏽ᵃ#$%ᵃ\r\nع<9
", "tokens": 37, "pieces": ["<|", "endoftext", "|>", "Ⅳ", " \n", " a", "̊字", "\td", "'re", "漢", "😀🏽", "ᵃ", "#$%", "ᵃ", "\r\n", "ع", "<", "9", "
"]} +{"text": "Džunglaꟲ\n<|endoftext|>ß㍿Džungla\"0,　👍🏽ᵃ12345678\na/bḍ̇\u000bdİع'T/\r\ń🙂👍🏽dＡ　Z🙂㋿m!", "tokens": 77, "pieces": ["Džunglaꟲ", "\n", "<|", "endoftext", "|>", "ß", "㍿Džungla", "\"", "0", ",<", "EOT", ">", "　", "👍🏽", "ᵃ", "123", "456", "78", "\n", "a", "/bḋ", "̣", "\u000bdİع", "'T", "/\r\n", "́🙂👍🏽", "dＡ", "　Z", "🙂㋿", "m", "!"]} +{"text": "ḍ̇", "tokens": 5, "pieces": ["ḋ", "̣"]} +{"text": "👍🏽>\u000b>😀🏽é
Dž,'M漢'D\n\"ꟲ Z\r<|fim_prefix|>'ſⅣ　Džunglaḍ̇½", "tokens": 55, "pieces": ["👍🏽>", "\u000b", ">😀🏽", "e", "́", "
Dž", ",'", "M漢", "'D", "\n", "\"ꟲ", " ", " Z", "\r", "<|", "fim", "_prefix", "|>'", "ſ", "Ⅳ", "　Džunglaḋ", "̣", "½"]} +{"text": "camelCased-m٣٤٥٦ＡHTTPServer", "tokens": 20, "pieces": ["camelCased", "-m", "٣٤٥", "٦", "Ａ", "HTTPServer"]} +{"text": "㋿漢'M'VEiOS<|endoftext|>", "tokens": 16, "pieces": ["㋿漢", "'M", "'VE", "iOS", "<|", "endoftext", "|>"]} +{"text": "/,HTTPServeŕ ḍ̇iOS٣٤٥٦'DHTTPServer'M\r\n're'reꟲ👍🏽'reaBB<|endoftext|>\u000be​ſ'måꟲſſd👍🏽m½", "tokens": 69, "pieces": ["/,", "HTTPServer", "́", " ḋ", "̣iOS", "٣٤٥", "٦", "'D", "HTTPServer", "'M", "\r\n", "'re", "'re", "ꟲ", "👍🏽'", "reaBB", "<|", "endoftext", "|>", "\u000be", "​ſ", "'m", "a", "̊ꟲſſd", "👍🏽", "m", "½"]} +{"text": "Abḍ̇\r\n", "tokens": 7, "pieces": ["Abḋ", "̣\r\n"]} +{"text": "ḍ̇\r\nİ​m#$% ३­HTTPServeré'Re", "tokens": 19, "pieces": ["ḋ", "̣\r\n", "İ", "​m", "#$%", " ", "३", "­HTTPServeré", "'Re"]} +{"text": "​३0.‍ZⅣ'Td'sßſs㋿ſ㋿ABC🙂𐞁\"", "tokens": 39, "pieces": ["​", "३0", ".‍", "Z", "Ⅳ", "'T", "d", "'s", "ßſs", "㋿ſ", "㋿ABC", "🙂𐞁", "\"<", "EOT", ">"]} +{"text": "s12345678!'ll \n 'ReⅣs　B(㍿　 'Tß'DABCAb>HTTPServer\r#$%\r\nDžunglaEOT\r\n\r\n12345678字", "tokens": 45, "pieces": ["s", "123", "456", "78", "!'", "ll", " \n", " '", "Re", "Ⅳ", "s", "　B", "(㍿", "　", " ", "'T", "ß", "'D", "ABCAb", ">HTTPServer", "\r", "#$%\r\n", "DžunglaEOT", "\r\n\r\n", "123", "456", "78", "字"]} +{"text": "Ａ'D's /ḍ̇ABCEOTſḍ̇s", "tokens": 21, "pieces": ["Ａ", "'D", "'s", " /", "ḋ", "̣ABCEOTſḋ", "̣s"]} +{"text": "<|endoftext|>Ⅳm're0a/b
méé's\n/Dž'Dd\n漢漢<|endoftext|>　é \naB🙂३'ReB‍m'ſ㍿.camelCase", "tokens": 66, "pieces": ["<|", "endoftext", "|>", "Ⅳ", "m", "'re", "0", "a", "/b", "
m", "éé", "'s", "\n", "/Dž", "'D", "d", "\n", "漢漢", "<|", "endoftext", "|>", "　e", "́", " \n", "aB", "🙂", "३", "'Re", "B", "‍m", "'ſ", "㍿.", "camelCase"]} +{"text": "éAb½dḍ̇d'T\rꟲᵃ>HTTPServera/b­'VEé३<…\n/<|endoftext|>Ａ<|fim_prefix|>", "tokens": 52, "pieces": ["éAb", "½", "dḋ", "̣<", "EOT", ">d", "'T", "\r", "ꟲᵃ", ">HTTPServera", "/b", "­'", "VEé", "३", "<", "…\n", "/<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>"]} +{"text": "㍿\ré ſéſ́ \r\n\r\n!ع(fi-ع
㋿camelCase#$%HTTPServerİ é…字 \n \"", "tokens": 42, "pieces": ["㍿\r", "e", "́", " ſe", "́ſ", "́", " \r\n\r\n", "!ع", "(fi", "-ع", "
", "㋿camelCase", "#$%", "HTTPServerİ", " é", "", "…字", " \n", " \""]} +{"text": "Ａ　Bé ꟲBAb,de🙂
", "tokens": 17, "pieces": ["Ａ", "　Be", "́", " ", " ꟲBAb", ",de", "🙂", "
"]} +{"text": "\u000b<𐞁\"ᵃå,'T㍿㋿ABC\n/<|endoftext|>​'T👍🏽ꟲꟲİ'VE'\r\n\r\n,
a/b", "tokens": 57, "pieces": ["\u000b", "<𐞁", "\"ᵃa", "̊,'", "T", "㍿㋿", "ABC", "\n", "/<|", "endoftext", "|><", "META", "_START", ">​'", "T", "👍🏽", "ꟲꟲİ", "'VE", "'\r\n\r\n", ",", "
a", "/b"]} +{"text": "
'll/㍿ع<|fim_prefix|>/\r\niOS\n/…\r\n\r\n　éå­ḍ̇'SDžungla𐞁9", "tokens": 48, "pieces": ["
", "'ll", "/㍿", "ع", "<|", "fim", "_prefix", "|><", "META", "_START", ">/\r\n", "iOS", "\n", "/", "…\r\n\r\n", "　e", "́a", "̊­", "ḋ", "̣'", "SDžungla𐞁", "9"]} +{"text": "a'll9Z ", "tokens": 5, "pieces": ["a", "'ll", "9", "Z", " "]} +{"text": "Dž'M字fi.'reİ'ſ　 #$%", "tokens": 16, "pieces": ["Dž", "'M", "字fi", ".'", "reİ", "'ſ", "　 ", " #$%"]} +{"text": "/ß'T½㍿<\rcamelCase<|endoftext|>'re", "tokens": 19, "pieces": ["/ß", "'T", "½", "㍿<\r", "camelCase", "<|", "endoftext", "|>'", "re"]} +{"text": "ß\u000ba/b<|fim_prefix|>½'T…!İ(𐞁漢‍\ta👍🏽'M\r<|endoftext|>t", "tokens": 44, "pieces": ["ß", "\u000ba", "/b", "<|", "fim", "_prefix", "|>", "½", "'T", "…", "!İ", "(𐞁漢", "‍", "\ta", "👍🏽'", "M", "\r", "<|", "endoftext", "|>", "t"]} +{"text": "👍🏽\n/!!iOS12345678", "tokens": 16, "pieces": ["👍🏽\n", "/!!", "iOS", "123", "456", "78", ""]} +{"text": "'VE 'll🙂ḍ̇\rs", "tokens": 18, "pieces": ["'VE", " ", " '", "ll", "🙂ḋ", "̣\r", "s", ""]} +{"text": "camelCase½Z…HTTPServer-'llſB's<|fim_prefix|>\n/\r\n \n fifi'.٣٤٥٦at'ſ", "tokens": 40, "pieces": ["camelCase", "½", "Z", "…HTTPServer", "-'", "llſB", "'s", "<|", "fim", "_prefix", "|>\n", "/\r\n", " \n", " fifi", "'.", "٣٤٥", "٦", "at", "'ſ"]} +{"text": "e'S#$%👍🏽iOSⅣ😀🏽/\r\n­'DžunglaEOT#$%'DiOS/#$%sB\r'res", "tokens": 41, "pieces": ["e", "'S", "#$%👍🏽", "iOS", "Ⅳ", "😀🏽/\r\n", "­'", "DžunglaEOT", "#$%'", "DiOS", "/#$%", "sB", "\r", "'re", "s"]} +{"text": "　9㋿a/b㍿'Re३字\n/#$%B
<|fim_prefix|>…a", "/b", "㍿'", "Re", "३", "字", "\n", "/#$%", "B", "
", "<|", "fim", "_prefix", "|>", "…", "mع㍿EOT\n<|fim_prefix|> \nEOT\r\nDžungla\"\r\nⅣ😀🏽Ａ'Re.<'ll<|endoftext|> eDžungla👍🏽漢12345678B\n'Re e​ \n ", "tokens": 69, "pieces": ["mع", "㍿EOT", "\n", "<|", "fim", "_prefix", "|>", " \n", "EOT", "\r\n", "Džungla", "\"\r\n", "Ⅳ", "😀🏽", "Ａ", "'Re", ".<'", "ll", "<|", "endoftext", "|>", " eDžungla", "👍🏽", "漢", "123", "456", "78", "B", "\n", "'Re", " e", "​", " \n "]} +{"text": "́٣٤٥٦ \n ‍\u000béaBiOS", "tokens": 17, "pieces": ["́", "٣٤٥", "٦", " \n", " ‍", "\u000béaBiOS"]} +{"text": "t🙂.#$%> camelCase ＡHTTPServerté…!Ⅳ𐞁12345678Zsm!‍ ⅣB\n\n/", "tokens": 41, "pieces": ["t", "🙂.#$%>", " ", " camelCase", " ", " ＡHTTPServerte", "́", "…", "!", "Ⅳ", "𐞁", "123", "456", "78", "Zsm", "!‍", " ", " ", "Ⅳ", "B", "\n\n", "/"]} +{"text": "́e-👍🏽Ab'Re'ſ'Re.𐞁­𐞁ß's//\r\n٣٤٥٦(🙂0/ \n \r‍
", "tokens": 52, "pieces": ["́e", "-👍🏽<", "EOT", ">Ab", "'Re", "'ſ", "'Re", ".𐞁", "­𐞁ß", "'s", "//\r\n", "٣٤٥", "٦", "(🙂", "0", "/", " \n \r", "‍", "
"]} +{"text": "d<'D", "tokens": 3, "pieces": ["d", "<'", "D"]} +{"text": "fi12345678ABC/\r\nᵃAb#$%İ12345678BiOS\n/
/½́
Ａ <|fim_prefix|>́", "tokens": 37, "pieces": ["fi", "123", "456", "78", "ABC", "/\r\n", "ᵃAb", "#$%", "İ", "123", "456", "78", "BiOS", "\n", "/", "
", "/", "½", "́", "
Ａ", " <|", "fim", "_prefix", "|>́"]} +{"text": "Ⅳ́0é'́\r\nDž🙂​\r\n\r\n<|fim_prefix|>\n👍🏽​", "tokens": 36, "pieces": ["Ⅳ", "́<", "EOT", ">", "0", "é", "'́\r\n", "Dž", "🙂<", "EOT", ">​\r\n\r\n", "<|", "fim", "_prefix", "|>\n", "👍🏽​"]} +{"text": " \n é/\n\r9'M
 \n ㍿ddḍ̇é\u000bع\r\n\r\n", "tokens": 24, "pieces": [" \n", " é", "/\n\r", "9", "'M", "
 \n", " ㍿", "ddḋ", "̣e", "́", "\u000bع", "\r\n\r\n"]} +{"text": "'VE<|fim_prefix|>½t>\u000bİd­é㍿\n/漢å", "tokens": 31, "pieces": ["'VE", "<|", "fim", "_prefix", "|>", "½", "t", ">", "\u000bİd", "­e", "́㍿\n", "/漢a", "̊"]} +{"text": "\r\n\r\nᵃcamelCaseZ12345678'D'Tm字㋿å\r\n\r\n'Re!!'ll\r\n12345678३🙂\nABCＡ漢-​㍿\n#$%HTTPServer!!'Tß​<|fim_prefix|>漢å!!/\r\n", "tokens": 71, "pieces": ["\r\n\r\n", "ᵃcamelCaseZ", "123", "456", "78", "'D", "'T", "m字", "㋿a", "̊\r\n\r\n", "'Re", "!!'", "ll", "\r\n", "123", "456", "78३", "🙂\n", "ABCＡ漢", "-​㍿\n", "#$%", "HTTPServer", "!!'", "Tß", "​<|", "fim", "_prefix", "|>", "漢a", "̊!!/\r\n"]} +{"text": "'s\n/", "tokens": 3, "pieces": ["'s", "\n", "/"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'re漢", "tokens": 3, "pieces": ["'re", "漢"]} +{"text": "ſ'a9Džungla", "tokens": 8, "pieces": ["ſ", "'a", "9", "Džungla"]} +{"text": "\n/\r\nm/'ſ​d'M٣٤٥٦ABC d٣٤٥٦ع\r!\r!ᵃ𐞁 (Dž'\r\n\r\nſ> ​é　", "tokens": 54, "pieces": ["\n", "/\r\n", "m", "/'", "ſ", "​", "d", "'M", "٣٤٥", "٦", "ABC", " d", "٣٤٥", "٦", "ع", "\r", "!\r", "!ᵃ𐞁", " ", "(Dž", "'\r\n\r\n", "ſ", ">", " ", "​é", "　"]} +{"text": ",'reZꟲᵃ/", "tokens": 10, "pieces": [",'", "reZꟲᵃ", "/"]} +{"text": "\rⅣ'ſDžHTTPServer'Reå12345678㋿9ſ㋿ABC🙂BåaDžungla‍!camelCase12345678<|endoftext|>/' ㋿\"'Ts", "tokens": 59, "pieces": ["\r", "Ⅳ", "'ſ", "DžHTTPServer", "'Re", "a", "̊", "123", "456", "78", "㋿", "9", "ſ", "㋿ABC", "🙂Ba", "̊aDžungla", "‍!", "camelCase", "123", "456", "78", "<|", "endoftext", "|>/'", " ", "㋿\"'", "Ts"]} +{"text": "\n'a/b‍<|fim_prefix|>½字EOT३", "tokens": 18, "pieces": ["\n", "'a", "/b", "‍<|", "fim", "_prefix", "|>", "½", "字EOT", "३"]} +{"text": " \n ع\r\n\r\nå😀🏽#$%  'M‍İ ㍿\r\n​'S­/\r\ncamelCase'S>…#$%a/b\n/Bé", "tokens": 46, "pieces": [" \n", " ع", "\r\n\r\n", "a", "̊😀🏽#$%", " ", " ", "'M", "‍İ", " ", " ㍿\r\n", "​'", "S", "­/\r\n", "camelCase", "'S", ">", "…", "#$%", "a", "/b", "\n", "/Bé"]} +{"text": " (#$%\"é.camelCase(-t　\r\n\r\n>㍿\n/\r\n \n 's㋿a/b\t<|fim_prefix|>٣٤٥٦ᵃt\t > ", "tokens": 50, "pieces": [" ", " (#$%\"", "e", "́.", "camelCase", "(-", "t", "　\r\n\r\n", ">㍿\n", "/\r\n", " \n", " '", "s", "㋿a", "/b", "\t", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ᵃt", "\t", " >", " "]} +{"text": "Z㋿EOT㋿'sdZⅣ're(dꟲ㍿'½عß", "tokens": 26, "pieces": ["Z", "㋿EOT", "㋿'", "sdZ", "Ⅳ", "'re", "(dꟲ", "㍿'", "½", "عß"]} +{"text": "ABC/\r\na👍🏽ᵃcamelCase½mß́camelCase👍🏽t \n<|fim_prefix|>a 12345678/漢Za/b́m\r>aB
m ́'M'ſ
　>…\u000b'll", "tokens": 69, "pieces": ["ABC", "/\r\n", "a", "👍🏽", "ᵃcamelCase", "½", "mß", "́camelCase", "👍🏽", "t", " \n", "<|", "fim", "_prefix", "|>", "a", " ", "123", "456", "78", "/漢Za", "/b", "́m", "\r", ">aB", "
m", " ́'", "M", "'ſ", "
", "　", ">", "…", "\u000b", "'ll"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n३㋿é㋿", "tokens": 11, "pieces": ["\r\n", "३", "㋿e", "́㋿"]} +{"text": " \r\n\r\n!ḍ̇ABC'VEå𐞁<|fim_prefix|>'s٣٤٥٦ꟲé<|endoftext|>a/btHTTPServeŕ'VEḍ̇!!\"t\t \n ,camelCase'VE>camelCase\n👍🏽", "tokens": 83, "pieces": [" \r\n\r\n", "!ḋ", "̣ABC", "'VE", "a", "̊𐞁", "<|", "fim", "_prefix", "|>'", "s", "٣٤٥", "٦", "ꟲ", "e", "́<|", "endoftext", "|>", "a", "/btHTTPServer", "́'", "VEḋ", "̣!!\"<", "EOT", ">t", "\t \n", " ,", "camelCase", "'VE", ">camelCase", "\n", "👍🏽"]} +{"text": ".㋿!a \n !aBḍ̇‍👍🏽12345678Dž", "tokens": 31, "pieces": [".<", "EOT", ">㋿!", "a", " \n", " !", "aBḋ", "̣‍👍🏽", "123", "456", "78", "Dž"]} +{"text": "عZ \n\nİ\u000b. …'s/İ,👍🏽", "tokens": 19, "pieces": ["عZ", " \n\n", "İ", "\u000b", ".", " ", "…", "'s", "/İ", ",👍🏽"]} +{"text": "s​", "tokens": 2, "pieces": ["s", "​"]} +{"text": "३EOT٣٤٥٦d\t're0iOSABCAbſaB\r\n\r\nİ\t!<|endoftext|>㍿ ع'llꟲaB३\"aB​.ᵃ9 t", "tokens": 59, "pieces": ["३", "EOT", "٣٤٥", "٦", "d", "\t", "'re", "0", "iOSABCAbſaB", "\r\n\r\n", "İ", "\t", "!<|", "endoftext", "|>㍿<", "EOT", ">", " ع", "'ll", "ꟲaB", "३", "\"aB", "​.", "ᵃ", "9", " t"]} +{"text": "…‍ß\t字​…aB<|endoftext|>　'ſعAb　/😀🏽", "tokens": 28, "pieces": ["漢", "<|", "endoftext", "|><|", "endoftext", "|>", "　", "'ſ", "عAb", "　", "/😀🏽"]} +{"text": "iOS\r\n\r\n　'Džungla<|endoftext|>'D
EOTſ", "tokens": 22, "pieces": ["iOS", "\r\n\r\n", "　", "'Džungla", "<|", "endoftext", "|>'", "D", "
EOTſ"]} +{"text": "!😀🏽字'M‍Dž0 12345678fi½…a", "tokens": 23, "pieces": ["!😀🏽", "字", "'M", "‍Dž", "0", " ", "123", "456", "78", "fi", "½", "…a"]} +{"text": "'S'Mm'\"#$%'re\t-👍🏽㍿\r\n\r\n.Z'ſEOT AbAb<|fim_prefix|>/. \n 字iOS‍ḍ̇字9", "tokens": 52, "pieces": ["'S", "'M", "m", "'\"#$%'", "re", "\t", "-<", "META", "_START", ">👍🏽㍿\r\n\r\n", ".Z", "'ſ", "EOT", " ", " AbAb", "<|", "fim", "_prefix", "|>/.", " \n", " 字iOS", "‍ḋ", "̣字", "9"]} +{"text": "　's\n/㋿'Re👍🏽/\r\n'　fi'S/DžunglacamelCase \n's(́m<|fim_prefix|>ᵃ👍🏽\r,'Re'M­𐞁\r\n\r\n\"字­🙂e<|endoftext|>漢Z12345678", "tokens": 78, "pieces": ["　", "'s", "\n", "/㋿'", "Re", "👍🏽/\r\n", "'", "　fi", "'S", "/DžunglacamelCase", " \n", "'s", "(́", "m", "<|", "fim", "_prefix", "|>", "ᵃ", "👍🏽\r", ",'", "Re", "'M", "­𐞁", "\r\n\r\n", "\"字", "­🙂", "e", "<|", "endoftext", "|>", "漢Z", "123", "456", "78"]} +{"text": "camelCase<ꟲ㍿\"ع😀🏽😀🏽", "tokens": 24, "pieces": ["camelCase", "<ꟲ", "㍿\"", "ع", "😀🏽😀🏽"]} +{"text": "é'D‍<|fim_prefix|>ḍ̇å½ſ​…ḍ̇'S'VE\r\n\r\n'll字a0ᵃ0Dž३0 \n ß \u000b𐞁ABCDž", "tokens": 60, "pieces": ["é", "'D", "‍<|", "fim", "_prefix", "|>", "ḋ", "̣a", "̊", "½", "ſ", "​", "…ḋ", "̣'", "S", "'VE", "\r\n\r\n", "'ll", "字a", "0", "ᵃ", "0", "Dž", "३0", " \n", " ß", " ", "\u000b𐞁ABCDž"]} +{"text": "!/\r\n‍s­Dža/b😀🏽", "tokens": 15, "pieces": ["!/\r\n", "‍s", "­Dža", "/b", "😀🏽"]} +{"text": "é's‍…٣٤٥٦t<|fim_prefix|>'VE٣٤٥٦,a/bDžungla\nAb/\r\n/\r\n\tacamelCase9éABC", "tokens": 50, "pieces": ["e", "́'", "s", "‍", "…", "٣٤٥", "٦", "t", "<|", "fim", "_prefix", "|>'", "VE", "٣٤٥", "٦", ",a", "/bDžungla", "\n", "Ab", "/\r\n", "/\r\n", "\tacamelCase", "9", "e", "́ABC"]} +{"text": " ⅣéZİ­'ſå\r\n\r\nAbAb🙂٣٤٥٦'ſᵃꟲ😀🏽ᵃ/ 12345678/\r\nABC𐞁a𐞁ع字12345678😀🏽ع", "tokens": 77, "pieces": [" ", "Ⅳ", "éZİ", "­'", "ſa", "̊\r\n\r\n", "AbAb", "🙂", "٣٤٥", "٦", "'ſ", "ᵃꟲ", "😀🏽", "ᵃ", "/<", "META", "_START", ">", " ", " ", "123", "456", "78", "/\r\n", "ABC𐞁a𐞁ع字", "123", "456", "78", "😀🏽", "ع", ""]} +{"text": ">'t", "tokens": 2, "pieces": [">'", "t"]} +{"text": "ꟲꟲe!aB­\r漢,é​éHTTPServerDžungla ‍ #$%< #$%ᵃ👍🏽/\r\nHTTPServer🙂　-9­DžZ𐞁", "tokens": 58, "pieces": ["ꟲꟲe", "!aB", "­\r", "漢", ",e", "́​", "e", "́HTTPServerDžungla", " ‍", " #$%<", " ", " #$%", "ᵃ", "👍🏽/\r\n", "HTTPServer", "🙂", "　", "-", "9", "­DžZ𐞁"]} +{"text": "!'s'>'Re9>३🙂Džunglaß字字aB\r<|endoftext|>ＡDžDž'll<|endoftext|>ᵃaB!<|fim_prefix|>ḍ̇́å \n٣٤٥٦", "tokens": 72, "pieces": ["!'", "s", "'>'", "Re", "9", ">", "३", "🙂Džunglaß字字aB", "\r", "<|", "endoftext", "|>", "ＡDžDž", "'ll", "<|", "endoftext", "|>", "ᵃaB", "!<|", "fim", "_prefix", "|>", "ḋ", "̣́", "a", "̊", " \n", "٣٤٥", "٦"]} +{"text": "aⅣ<|endoftext|>åß㋿/ \n  \rß-\r\n! \nع'reſDžungla　<|fim_prefix|>İ,‍字", "tokens": 49, "pieces": ["a", "Ⅳ", "<|", "endoftext", "|>", "a", "̊ß", "㋿/<", "EOT", ">", " \n  \r", "ß", "-\r\n", "!", " \n", "ع", "'re", "ſDžungla", "　", "<|", "fim", "_prefix", "|>", "İ", ",‍", "字"]} +{"text": "'ſ", "tokens": 3, "pieces": ["'ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ABCß/\n/>'aB㋿e0\nع'\n𐞁 \n😀🏽.é𐞁/\r\n　é­ İ'll<|endoftext|>!", "tokens": 47, "pieces": ["ABCß", "/\n", "/>'", "aB", "㋿e", "0", "\n", "ع", "'\n", "𐞁", " \n", "😀🏽.", "é𐞁", "/\r\n", "　e", "́­", " İ", "'ll", "<|", "endoftext", "|>!"]} +{"text": ".és٣٤٥٦İfi'D\n", "tokens": 15, "pieces": [".és", "٣٤٥", "٦", "İfi", "'D", "\n"]} +{"text": "/\r\n'a/b'ſZDž \n𐞁ꟲ'T<,,<|endoftext|>㍿'DABCꟲ👍🏽ᵃ\u000b३ \nＡ", "tokens": 55, "pieces": ["/\r\n", "'", "a", "/b", "'ſ", "ZDž", " \n", "𐞁ꟲ", "'T", "<,,<|", "endoftext", "|>㍿'", "DABCꟲ", "👍🏽", "ᵃ", "\u000b", "३", " \n", "Ａ"]} +{"text": "😀🏽ꟲtAb/\r\n/\r\n३٣٤٥٦Ⅳ​>/EOTcamelCaseＡ٣٤٥٦", "tokens": 40, "pieces": ["😀🏽", "ꟲtAb", "/\r\n", "/\r\n", "३٣٤", "٥٦Ⅳ", "​>/", "EOTcamelCaseＡ", "٣٤٥", "٦"]} +{"text": "å \n　å　字Ａ㋿\ra/bß'Re!­\n/!㋿", "tokens": 28, "pieces": ["a", "̊", " \n", "　a", "̊", "　字Ａ", "㋿\r", "a", "/bß", "'Re", "!­\n", "/!㋿"]} +{"text": "a/bé9㍿عå'M👍🏽 iOS/ᵃ", "tokens": 27, "pieces": ["a", "/be", "́", "9", "㍿عa", "̊'", "M", "👍🏽", " iOS", "/<", "META", "_START", ">ᵃ"]} +{"text": "ß/\r\nt(‍'S!'TiOS0a/bİDž㋿ \n ,　å'EOT<|endoftext|>a/bm'\r½", "tokens": 57, "pieces": ["ß", "/\r\n", "t", "(‍'", "S", "!'", "TiOS", "0", "a", "/bİDž", "㋿", " \n", " ,", "　", "a", "̊'", "EOT", "<|", "endoftext", "|>", "a", "/bm", "'\r", "½"]} +{"text": "'ſ'sß'SsABC😀🏽\t'T'll\r\n३'٣٤٥٦ḍ̇\r\n'T漢/İ12345678aBEOT", "tokens": 45, "pieces": ["'ſ", "'s", "ß", "'S", "sABC", "😀🏽", "\t", "'T", "'ll", "\r\n", "३", "'", "٣٤٥", "٦", "ḋ", "̣\r\n", "'T", "漢", "/İ", "123", "456", "78", "aBEOT"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " 字0a　'VE\t'VE　𐞁'ſ\u000b𐞁​", "tokens": 23, "pieces": [" 字", "0", "a", "　", "'VE", "\t", "'VE", "　𐞁", "'ſ", "\u000b𐞁", "​"]} +{"text": "\r\n<|endoftext|>", "tokens": 8, "pieces": ["\r\n", "<|", "endoftext", "|>"]} +{"text": "fiEOT🙂.,aB\n/\n/ 🙂, <|fim_prefix|>'Mdſ camelCaseEOT🙂…👍🏽\u000b٣٤٥٦a!!EOT'Re👍🏽fiå'll<|endoftext|>‍m", "tokens": 76, "pieces": ["fiEOT", "🙂.,", "aB", "\n", "/\n", "/", " ", "🙂,", " <|", "fim", "_prefix", "|>'", "Mdſ", " camelCaseEOT", "🙂", "…", "👍🏽", "\u000b", "٣٤٥", "٦", "a", "!!", "EOT", "'Re", "👍🏽", "fia", "̊'", "ll", "<|", "endoftext", "|>‍", "m"]} +{"text": "​12345678m!!éİéiOSABC'll　>Ⅳmİ<|fim_prefix|>\r!Ab /camelCase're'M12345678½'M \n /\r\nm-'ſ HTTPServer🙂", "tokens": 53, "pieces": ["​", "123", "456", "78", "m", "!!", "éİéiOSABC", "'ll", "　", ">", "Ⅳ", "mİ", "<|", "fim", "_prefix", "|>\r", "!Ab", " ", " /", "camelCase", "'re", "'", "M", "123", "456", "78½", "'M", " \n", " /\r\n", "m", "-'", "ſ", " HTTPServer", "🙂"]} +{"text": "ABC're\r\n\r\n'TEOT'VEa/b(​字\r\n\t", "tokens": 15, "pieces": ["ABC", "'re", "\r\n\r\n", "'T", "EOT", "'VE", "a", "/b", "(​", "字", "\r\n\t"]} +{"text": "㍿字EOTİfi'ſ(", "tokens": 13, "pieces": ["㍿字EOTİfi", "'ſ", "("]} +{"text": "( Džunglá​㋿İ'VE'!", "tokens": 20, "pieces": ["(<", "META", "_START", ">", " ", " Džungla", "́​㋿", "İ", "'VE", "'!"]} +{"text": " \nfi‍字é‍\"HTTPServer­-é'fi　\n/\n're\u000b㋿d'Ree", "tokens": 34, "pieces": [" <", "EOT", ">", " \n", "fi", "‍字é", "‍\"", "HTTPServer", "­-", "e", "́'", "fi", "　\n", "/\n", "'re", "\u000b", "㋿d", "'Re", "e"]} +{"text": "/Ａt'T㋿/\r\n
0HTTPServerEOT½ \tß‍𐞁\" ABC‍…'M'D/\r\nAb,​a/b'Mع-𐞁\t", "tokens": 52, "pieces": ["/Ａt", "'T", "㋿/\r\n", "
", "0", "HTTPServerEOT", "", "½", " ", "\tß", "‍𐞁", "\"", " ", " ABC", "‍", "…", "'M", "'D", "/\r\n", "Ab", ",​", "a", "/b", "'M", "ع", "-𐞁", "\t"]} +{"text": "'ſＡé\r\n\r\na'S👍🏽<|fim_prefix|>‍漢​'ll", "tokens": 33, "pieces": ["'ſ", "Ａé", "\r\n\r\n", "a", "'S", "👍🏽<|", "fim", "_prefix", "|>‍<", "EOT", ">漢", "​'", "ll"]} +{"text": " ㍿>'VE\t'Té'VEZعDžsＡ\t>…<Džungla>", "tokens": 28, "pieces": [" ㍿>'", "VE", "\t", "'T", "é", "'VE", "ZعDžsＡ", "\t", ">", "…", "<Džungla", ">"]} +{"text": "HTTPServer('ſHTTPServer३camelCaseİBDž\rå>tＡ'M#$%/å٣٤٥٦9ꟲé　ta/\r\n<|fim_prefix|>s.ꟲ\r\n/", "tokens": 63, "pieces": ["HTTPServer", "('", "ſHTTPServer", "३", "camelCaseİBDž", "\r", "a", "̊>", "tＡ", "'M", "#$%/", "a", "̊", "٣٤٥", "٦9", "ꟲe", "́", "　ta", "/\r\n", "<|", "fim", "_prefix", "|>", "s", ".ꟲ", "\r\n", "/"]} +{"text": "a\n/Ab", "tokens": 48, "pieces": ["a", "\n", "/Ab", ""]} +{"text": "
/(em'T's(漢'Re　 \n e　're\n/Z'ſå\r\n\r\n#$%#$%٣٤٥٦'Re
camelCase­'VE🙂", "tokens": 49, "pieces": ["
", "/(", "em", "'T", "'s", "(漢", "'Re", "　 \n", " e", "　", "'re", "\n", "/Z", "'ſ", "a", "̊\r\n\r\n", "#$%#$%", "٣٤٥", "٦", "'Re", "
camelCase", "­'", "VE", "🙂"]} +{"text": "<|fim_prefix|>0 <|fim_prefix|>Ⅳ́ZcamelCaseⅣ‍'Re-<|endoftext|> ſ0a/bmᵃ", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>", "0", " <|", "fim", "_prefix", "|>", "Ⅳ", "́ZcamelCase", "Ⅳ", "‍'", "Re", "-<|", "endoftext", "|>", " ", " ſ", "0", "a", "/bmᵃ"]} +{"text": "字éİ#$%ſZꟲᵃa iOS,camelCase👍🏽a​a0a0\u000bEOT😀🏽é", "tokens": 42, "pieces": ["字éİ", "#$%", "ſZꟲᵃa", " ", " iOS", ",camelCase", "👍🏽", "a", "​a", "0", "a", "0", "\u000bEOT", "😀🏽", "e", "́"]} +{"text": "\u000ba/bＡdeᵃᵃ(漢½
fi٣٤٥٦a/b", "tokens": 57, "pieces": ["\u000ba", "/bＡdeᵃᵃ", "(漢", "", "½", "
fi", "٣٤٥", "٦", "a", "/b"]} +{"text": "İ'VEİdDžungla​!!Abd\r'll'S'D'VE\n/", "tokens": 21, "pieces": ["İ", "'VE", "İdDžungla", "​!!", "Abd", "\r", "'ll", "'S", "'D", "'VE", "\n", "/"]} +{"text": "Ab½#$%'ll­(.𐞁३\r'll­😀🏽-d'D३½ع漢­'😀🏽
ꟲ./\r\n \naB\n\n0'Re \nd<|endoftext|>", "tokens": 63, "pieces": ["Ab", "½", "#$%'", "ll", "­(.<", "EOT", ">𐞁", "३", "\r", "'ll", "­😀🏽-", "d", "'D", "३½", "ع漢", "­'😀🏽", "
ꟲ", "./\r\n", " \n", "aB", "\n\n", "0", "'Re", " \n", "d", "<|", "endoftext", "|>"]} +{"text": "fi😀🏽\r\n\r\n<ع #$%\u000b \n 
", "tokens": 20, "pieces": ["fi", "😀🏽\r\n\r\n", "<", "ع", " ", "#$%", "\u000b \n 
"]} +{"text": "#$%'Me.Džunglaa'T٣٤٥٦a…fi\r\n\r\na'Tḍ̇字éAbAbå'Té're're-'\r\n\r\nscamelCasefi", "tokens": 51, "pieces": ["#$%'", "Me", ".Džunglaa", "'T", "٣٤٥", "٦", "a", "…fi", "\r\n\r\n", "a", "'T", "ḋ", "̣字e", "́AbAba", "̊'", "Té", "'re", "'re", "-'\r\n\r\n", "scamelCasefi"]} +{"text": "#$% \nḍ̇ZsⅣ<|endoftext|>Ⅳ㍿😀🏽", "tokens": 29, "pieces": ["#$%", " \n", "ḋ", "̣Zs", "Ⅳ", "<|", "endoftext", "|>", "Ⅳ", "㍿😀🏽"]} +{"text": "a/b­😀🏽'TＡع㍿>'re٣٤٥٦a/b\"𐞁a/b0 😀🏽ß'Re<|fim_prefix|>", "tokens": 49, "pieces": ["a", "/b", "­😀🏽'", "TＡع", "㍿>'", "re", "٣٤٥", "٦", "a", "/b", "\"𐞁a", "/b", "0", " 😀🏽", "ß", "'Re", "<|", "fim", "_prefix", "|>"]} +{"text": "sB,'D", "tokens": 4, "pieces": ["sB", ",'", "D"]} +{"text": "dt漢m#$%B𐞁fiḍ̇٣٤٥٦\n'sꟲDžunglaa👍🏽'VEſ字Ａ㍿ſſB\u000be>\r\n'DİſtiOS ᵃcamelCase>iOS", "tokens": 77, "pieces": ["dt漢m", "#$%<", "EOT", ">B𐞁fiḋ", "̣", "٣٤٥", "٦", "\n", "'s", "ꟲDžunglaa", "👍🏽'", "VEſ字Ａ", "㍿ſſB", "\u000be", ">\r\n", "'D", "İſtiOS", " ", " ᵃcamelCase", ">iOS"]} +{"text": "'T-Ａ😀🏽ſ'TEOT \n \r\n\r\n\n/\nİꟲ漢>fimaBſ३", "tokens": 76, "pieces": ["'T", "-Ａ", "😀🏽", "ſ", "'T", "EOT", "", " \n \r\n\r\n\n", "/\n", "İꟲ漢", ">fimaB", "", "ſ", "३"]} +{"text": "d\r\nꟲⅣ", "tokens": 7, "pieces": ["d", "\r\n", "ꟲ", "Ⅳ"]} +{"text": "'MiOS(\r\n\r\na/b'Mḍ̇éé\nİ‍fi\r\n\r\nßḍ̇ſⅣ'\"-İ(\u000b'VE(", "tokens": 63, "pieces": ["'M", "iOS", "(\r\n\r\n", "a", "/b", "'M", "ḋ", "̣<", "e", "'T", "AbcamelCase", "<|", "endoftext", "|>", "e", "́é", "\n", "İ", "‍fi", "\r\n\r\n", "ßḋ", "̣<", "EOT", ">ſ", "Ⅳ", "'\"-", "İ", "(", "\u000b", "'VE", "("]} +{"text": ",…\r‍Dž​'reé0\rDžungla'Re'Dfi/\r\n३­t'Rett'T'TDž字
٣٤٥٦>'VEa/b'Re\u000béé½Džİ'M0", "tokens": 57, "pieces": [",", "…\r", "‍Dž", "​'", "reé", "0", "\r", "Džungla", "'Re", "'D", "fi", "/\r\n", "३", "­t", "'Re", "tt", "'T", "'T", "Dž字", "
", "٣٤٥", "٦", ">'", "VEa", "/b", "'Re", "\u000béé", "½", "Džİ", "'M", "0"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "t#$%\t㋿EOT", "tokens": 9, "pieces": ["t", "#$%", "\t", "㋿EOT"]} +{"text": "90're \n(#$%-Dž　.ᵃ
'll'ſſ­'DDžungla.漢åaBed👍🏽 \n ع!!0\"/字", "tokens": 52, "pieces": ["90", "'re", "", " \n", "(#$%-", "Dž", "　", ".ᵃ", "
", "'ll", "'ſ", "ſ", "­'", "DDžungla", ".漢a", "̊aBed", "👍🏽", " \n", " ع", "!!", "0", "\"/", "字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "0>
Ａ😀🏽'ſ٣٤٥٦!/\r\n!\r\n\r\n >é'VEⅣB'Ss😀🏽'sAb…-ᵃḍ̇iOS…/\r\ncamelCase\r\n#$%", "tokens": 64, "pieces": ["0", ">", "
Ａ", "😀🏽'", "ſ", "٣٤٥", "٦", "!/\r\n", "!\r\n\r\n", " ", " >", "é", "'VE", "Ⅳ", "B", "'S", "s", "😀🏽'", "sAb", "…", "-ᵃḋ", "̣iOS", "…", "/\r\n", "camelCase", "\r\n", "#$%"]} +{"text": "Džfi0é\n/½'T#$%Ⅳ>…é\t‍d𐞁 <|endoftext|>!ß#$%tİİa‍/ع>…'S'D🙂", "tokens": 59, "pieces": ["Džfi", "0", "e", "́\n", "/", "½", "'T", "#$%", "Ⅳ", ">", "…", "e", "́", "\t", "‍d𐞁", " ", "<|", "endoftext", "|>!", "ß", "#$%", "tİİa", "‍/", "ع", ">", "…", "'S", "'D", "🙂"]} +{"text": "'M\r\n\u000b\ta/b\n//\r\n\r\nع㍿,Abᵃ\r", "tokens": 17, "pieces": ["'M", "\r\n", "\u000b", "\ta", "/b", "\n", "//\r\n\r\n", "ع", "㍿,", "Abᵃ", "\r"]} +{"text": "‍ 　ſABCå\"\naé👍🏽 a", "̊\"\n", "ae", "́👍🏽", " ", " éꟲ😀🏽camelCase,åDž\n/sé👍🏽Ab\rع'llcamelCase ,-iOS\rDžungla३…ꟲ㍿'re/aB", "tokens": 68, "pieces": ["9", "漢", "<|", "endoftext", "|>", "e", "́<", "EOT", ">ꟲ", "😀🏽", "camelCase", ",a", "̊Dž", "\n", "/se", "́👍🏽", "Ab", "\r", "ع", "'ll", "camelCase", " ", " ,-", "iOS", "\r", "Džungla", "३", "…ꟲ", "㍿'", "re", "/aB"]} +{"text": "0<|fim_prefix|>👍🏽'VE'Re!'re,Džunglaſ\r 'ſ>'Reſt
㋿Z!!'S", "tokens": 45, "pieces": ["0", "<|", "fim", "_prefix", "|>👍🏽'", "VE", "'Re", "!'", "re", ",Džunglaſ", "\r", " ", " '", "ſ", ">'", "Reſt", "
", "㋿Z", "!!'", "S"]} +{"text": "<\" \n 👍🏽HTTPServer'llꟲeém😀🏽👍🏽.<ᵃⅣ\u000b>𐞁s('M\u000bcamelCaseå字éDžd're a/b-㍿'D''T", "tokens": 70, "pieces": ["<\"<", "META", "_START", ">", " \n", " 👍🏽", "HTTPServer", "'ll", "ꟲee", "́m", "😀🏽👍🏽.<", "ᵃ", "Ⅳ", "\u000b", ">𐞁s", "('", "M", "\u000bcamelCasea", "̊字e", "́Džd", "'re", " ", " a", "/b", "-㍿'", "D", "''", "T"]} +{"text": "𐞁🙂'T<|endoftext|>\nDžungla \n'ReHTTPServer\raHTTPServerB ABC< <|endoftext|>aBⅣ​ écamelCase \n \r\n\r\n9Ａ漢Dž½.t<
fi
\n", "tokens": 66, "pieces": ["𐞁", "🙂'", "T", "<|", "endoftext", "|>\n", "Džungla", " \n", "'Re", "HTTPServer", "\r", "aHTTPServerB", " ", " ABC", "<", " ", " <|", "endoftext", "|>", "aB", "Ⅳ", "​", " e", "́camelCase", " \n \r\n\r\n", "9", "Ａ漢Dž", "½", ".t", "<", "
fi", "
\n"]} +{"text": "t​ée ", "tokens": 4, "pieces": ["t", "​ée", " "]} +{"text": "iOS \n­< !! \n<㋿ 　\rſcamelCaseⅣꟲ३camelCase- عiOS'S 　㋿é", "tokens": 38, "pieces": ["iOS", " \n", "­<", " ", "!!", " \n", "<㋿", " 　\r", "ſcamelCase", "Ⅳ", "ꟲ", "३", "camelCase", "-", " عiOS", "'S", " ", "　", "㋿e", "́"]} +{"text": "<|fim_prefix|>ḍ̇camelCase'se'TeHTTPServerEOT \r👍🏽d😀🏽'VEDž३", "tokens": 53, "pieces": ["<|", "fim", "_prefix", "|>", "ḋ", "̣camelCase", "'s", "e", "'T", "e", "HTTPServerEOT", "", " \r", "👍🏽", "d", "😀🏽'", "VE", "Dž", "३"]} +{"text": "HTTPServer'reeé\r\n \nß", "tokens": 8, "pieces": ["HTTPServer", "'re", "ee", "́\r\n", " \n", "ß"]} +{"text": ">'VE0ᵃ>", "tokens": 11, "pieces": [">'", "VE", "", "0", "ᵃ", ">"]} +{"text": "/\r\né'Dß<|fim_prefix|>🙂'T00\r\n\r\n\"ع'TcamelCase-\u000b12345678ꟲa/b", "tokens": 37, "pieces": ["/\r\n", "e", "́'", "Dß", "<|", "fim", "_prefix", "|>🙂'", "T", "00", "\r\n\r\n", "\"ع", "'", "TcamelCase", "-", "\u000b", "123", "456", "78", "ꟲa", "/b"]} +{"text": "Džungla12345678>HTTPServer/\r\n​'re'VE.㋿12345678​ B'll- Zm​'é'SعⅣ'DⅣ/\r\nعB́B'D'Re\u000b٣٤٥٦", "tokens": 57, "pieces": ["Džungla", "123", "456", "78", ">HTTPServer", "/\r\n", "​'", "re", "'VE", ".㋿", "123", "456", "78", "​", " B", "'ll", "-", " Zm", "​'", "e", "́'", "Sع", "Ⅳ", "'D", "Ⅳ", "/\r\n", "عB", "́B", "'D", "'Re", "\u000b", "٣٤٥", "٦"]} +{"text": "\t𐞁🙂Dž\nDž
#$%́ ‍ \n \r\n\r\n'MAbmdABC-", "tokens": 35, "pieces": ["\t𐞁", "🙂Dž", "\n", "Dž", "
", "#$%́<", "META", "_START", ">", " ", " ‍", " \n \r\n\r\n", "'M", "Ab", "mdABC", "-"]} +{"text": "m½'D\n😀🏽\"🙂‍'Re　'D…éＡ́३ꟲEOT\n/\r'M\u000b\n<́", "tokens": 42, "pieces": ["m", "½", "'D", "\n", "😀🏽\"🙂‍'", "Re", "　", "'D", "…e", "́Ａ", "́", "३", "ꟲEOT", "\n", "/\r", "'M", "\u000b", "\n", "<́"]} +{"text": "<éBABC…👍🏽 'Ds'S0tHTTPServerABC​\r\n'll' 0<|endoftext|>", "tokens": 35, "pieces": ["<éBABC", "…", "👍🏽", " ", " '", "Ds", "'S", "0", "tHTTPServerABC", "​\r\n", "'ll", "'", " ", " ", "0", "<|", "endoftext", "|>"]} +{"text": "ßm\t\r\n\r\n
iOSDžungla \n a/b", "tokens": 13, "pieces": ["ßm", "\t\r\n\r\n", "
iOSDžungla", " \n", " a", "/b"]} +{"text": "‍\r\n\r\naBع'M㍿t\n 'SZ/\r\n/\r\n½é \n're. (
字", "tokens": 27, "pieces": ["‍\r\n\r\n", "aBع", "'M", "㍿t", "\n", " ", " '", "SZ", "/\r\n", "/\r\n", "½", "é", " \n", "'re", ".", " ", "(", "
字"]} +{"text": "漢ſå\r\n\r\n<|fim_prefix|>\r\n😀🏽EOT<|fim_prefix|>½ iOS \n", "tokens": 33, "pieces": ["漢ſa", "̊\r\n\r\n", "<|", "fim", "_prefix", "|>\r\n", "😀🏽", "EOT", "<|", "fim", "_prefix", "|>", "½", " ", " iOS", " \n"]} +{"text": "'sᵃ㋿å'DABC'Teå<-'D½½9å", "tokens": 31, "pieces": ["'s", "ᵃ", "㋿a", "̊'", "DABC", "'T", "ea", "̊<-'", "D", "½", "", "½9", "a", "̊"]} +{"text": "İ<|fim_prefix|>éeaBa/b\n/iOS'D \n EOT,Ab'll!🙂d\r\n\r\n\r\n\r\nᵃcamelCase \n ABC٣٤٥٦", "tokens": 43, "pieces": ["İ", "<|", "fim", "_prefix", "|>", "e", "́eaBa", "/b", "\n", "/iOS", "'D", " \n", " EOT", ",Ab", "'ll", "!🙂", "d", "\r\n\r\n\r\n\r\n", "ᵃcamelCase", " \n", " ABC", "٣٤٥", "٦"]} +{"text": "ⅣaBİ<|endoftext|>'M…#$%'VEDž ", "tokens": 21, "pieces": ["Ⅳ", "aBİ", "<|", "endoftext", "|>'", "M", "…", "#$%'", "VEDž", " "]} +{"text": "a/b३t0/\r\n \n\r\n\r\n­\tḍ̇ᵃ'Re><|fim_prefix|>'THTTPServer字­camelCase \n'M ३'T'T", "tokens": 42, "pieces": ["a", "/b", "३", "t", "0", "/\r\n", " \n\r\n\r\n", "­", "\tḋ", "̣ᵃ", "'Re", "><|", "fim", "_prefix", "|>'", "THTTPServer字", "­camelCase", " \n", "'M", " ", " ", "३", "'T", "'T"]} +{"text": "​12345678😀🏽<|endoftext|>Dž \n ", "tokens": 30, "pieces": ["iOS", "🙂", " ", " B", "/\r\n", "\t", "㋿>", "123", "456", "78", "😀🏽<|", "endoftext", "|>", "Dž", " \n "]} +{"text": "ᵃ-\u000b12345678ſiOSa/b9<|endoftext|>'D'll'Dž'll'sſ#$%éſ<|endoftext|>9(EOTfi é#$%ée­­𐞁'llZ ㋿", "tokens": 67, "pieces": ["ᵃ", "-", "\u000b", "123", "456", "78", "ſiOSa", "/b", "9", "<|", "endoftext", "|>'", "D", "'ll", "'Dž", "'ll", "'s", "ſ", "#$%", "e", "́ſ", "<|", "endoftext", "|>", "9", "(EOTfi", " é", "#$%", "e", "́e", "­­", "𐞁", "'ll", "Z", " ", " ㋿"]} +{"text": "a/b(Dž३<|fim_prefix|>…😀🏽9éᵃ", "tokens": 26, "pieces": ["a", "/b", "(Dž", "३", "<|", "fim", "_prefix", "|>", "…", "😀🏽", "9", "éᵃ"]} +{"text": "-𐞁/ḍ̇!İ'VEDžunglaéfi'Td
(́(ع/", "tokens": 31, "pieces": ["-𐞁", "/ḋ", "̣!", "İ", "'VE", "Džunglaéfi", "'T", "d", "
", "(́(", "ع", "/"]} +{"text": "12345678-\n/aB­/!<|endoftext|>", "tokens": 15, "pieces": ["123", "456", "78", "-\n", "/aB", "­/!<|", "endoftext", "|>"]} +{"text": "å㋿d…é'/　 \n İ\"é\u000b漢-😀🏽s　<|endoftext|>½é­ABC,
 ḍ̇\n HTTPServerHTTPServer", "tokens": 58, "pieces": ["a", "̊㋿", "d", "", "…e", "́'/", "　 \n", " İ", "\"e", "́", "\u000b漢", "-😀🏽", "s", "　", "<|", "endoftext", "|>", "½", "e", "́­", "ABC", ",", "
", " ḋ", "̣\n", " ", " HTTPServerHTTPServer"]} +{"text": "İßs
ꟲa/b", "tokens": 10, "pieces": ["İßs", "
ꟲa", "/b"]} +{"text": "'ll'sé<|endoftext|>३å'M>!", "tokens": 19, "pieces": ["'ll", "'s", "é", "<|", "endoftext", "|>", "३", "a", "̊'", "M", ">!"]} +{"text": "sZAb  \n0ſB(\u000bd𐞁\r\n\r\n>'SDž /३عⅣd'M(aBé,…\n<|endoftext|>'S'll0Ab👍🏽
", "tokens": 56, "pieces": ["sZAb", "  \n", "0", "ſB", "(", "\u000bd𐞁", "\r\n\r\n", ">'", "SDž", " ", "/", "३", "ع", "Ⅳ", "d", "'M", "(aBé", ",", "…\n", "<|", "endoftext", "|>'", "S", "'ll", "0", "Ab", "👍🏽", "
"]} +{"text": "‍e'śع\"\r 12345678ß-́,9#$%\n/ſ \n -d‍字'SaB\na/bcamelCaseİ\n/", "tokens": 43, "pieces": ["‍e", "'s", "́ع", "\"\r", " ", " ", "123", "456", "78", "ß", "-́,", "9", "#$%\n", "/ſ", " \n", " -", "d", "‍字", "'S", "aB", "\n", "a", "/bcamelCase", "İ", "\n", "/"]} +{"text": "Dž \n 𐞁🙂<|endoftext|>'s9<|endoftext|>\u000b字'T́(\"9\r\n\r\n'siOS字<|fim_prefix|><|endoftext|>'Re'M\r\nd<|fim_prefix|>'reé🙂", "tokens": 64, "pieces": ["Dž", " \n", " 𐞁", "🙂<|", "endoftext", "|>'", "s", "9", "<|", "endoftext", "|>", "\u000b字", "'T", "́(\"", "9", "\r\n\r\n", "'s", "iOS字", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "Re", "'M", "\r\n", "d", "<|", "fim", "_prefix", "|>'", "ree", "́🙂"]} +{"text": "　EOT\u000bfidZ#$%HTTPServer/iOSᵃ㍿́\r\"ݽ…\nm.Ab'M'S٣٤٥٦Džungla(éꟲ'll", "tokens": 50, "pieces": ["　EOT", "\u000bfidZ", "#$%", "HTTPServer", "/iOSᵃ", "㍿́\r", "\"İ", "½", "…\n", "m", ".Ab", "'M", "'S", "٣٤٥", "٦", "Džungla", "(e", "́ꟲ", "'ll"]} +{"text": "ᵃ\r\n.#$%!!٣٤٥٦\ndＡB0\n/́ꟲ !!𐞁B(ſABC字HTTPServer<|fim_prefix|>\r \nABCé\n😀🏽HTTPServera/b\t'12345678", "tokens": 72, "pieces": ["ᵃ", "\r\n", ".#$%!!", "٣٤٥", "٦", "\n", "dＡB", "0", "\n", "/́", "ꟲ", " !!<", "META", "_START", ">𐞁B", "(ſABC字HTTPServer", "<|", "fim", "_prefix", "|>\r", " \n", "ABCe", "́\n", "😀🏽", "HTTPServera", "/b", "\t", "'", "123", "456", "78"]} +{"text": "( \n㋿td🙂å'så'M\n/camelCase½\n,ABCå\n0/", "tokens": 31, "pieces": ["(", " \n", "㋿td", "🙂a", "̊'", "sa", "̊'", "M", "\n", "/camelCase", "½", "\n", ",ABCa", "̊\n", "0", "/"]} +{"text": "\n/>-HTTPServer\nAb'VE'S#$%<|endoftext|>camelCaseß'VEعDžungla>ꟲéß\r'(", "tokens": 50, "pieces": ["\n", "/>-", "HTTPServer", "\n", "Ab", "'VE", "'S", "#$%<|", "endoftext", "|>", "camelCaseß", "'VE", "ع", "Džungla", ">ꟲe", "́ß", "\r", "'(<", "META", "_START", ">"]} +{"text": "ꟲa/b", "tokens": 5, "pieces": ["ꟲa", "/b"]} +{"text": "09HTTPServer's㋿iOS#$%ᵃ,ⅣaB'siOS", "tokens": 20, "pieces": ["09", "HTTPServer", "'s", "㋿iOS", "#$%", "ᵃ", ",", "Ⅳ", "aB", "'s", "iOS"]} +{"text": "é", "tokens": 1, "pieces": ["é"]} +{"text": "Džungla㋿'s", "tokens": 9, "pieces": ["Džungla", "㋿'", "s"]} +{"text": "字…‍HTTPServerＡ‍' \n 'D\n\r\n　a/b…\n😀🏽12345678Bfi'Re🙂s\"㋿!😀🏽\r\n­", "tokens": 58, "pieces": ["字", "", "…", "‍HTTPServer", "Ａ", "‍'", " \n", " '", "D", "\n\r\n", "　a", "/b", "…\n", "😀🏽", "123", "456", "78", "Bfi", "'Re", "🙂s", "\"㋿!😀🏽\r\n", "­"]} +{"text": " #$%éḍ̇\t३́afi, Ab\r\n\r\na٣٤٥٦\",­'re!!ABC\r\né!\r٣٤٥٦'re\r\r\n\r\n­ᵃ'VE\té<|fim_prefix|>", "tokens": 64, "pieces": [" ", "#$%", "éḋ", "̣", "\t", "३", "́afi", ",", " Ab", "\r\n\r\n", "a", "٣٤٥", "٦", "\",­'", "re", "!!", "ABC", "\r\n", "é", "!\r", "٣٤٥", "٦", "'re", "\r\r\n\r\n", "­ᵃ", "'VE", "\te", "́<|", "fim", "_prefix", "|>"]} +{"text": "İＡé,", "tokens": 5, "pieces": ["İＡé", ","]} +{"text": "‍ <|fim_prefix|>,'D.'ſ/\r.a0m\n!'reAb\"👍🏽/Ab!!Ab३​\tABC", "tokens": 42, "pieces": ["‍", " ", " <|", "fim", "_prefix", "|>,'", "D", ".'", "ſ", "/\r", ".a", "0", "m", "\n", "!'", "reAb", "\"👍🏽<", "META", "_START", ">/", "Ab", "!!", "Ab", "३", "​", "\tABC"]} +{"text": "ⅣcamelCase<|endoftext|>\" \r\n<|fim_prefix|>'rea/bᵃtfi'reᵃB👍🏽٣٤٥٦ABCe'DBᵃ\rm\"\r\r\n　d عDžungla\r\n0'Re𐞁 \n!ſ", "tokens": 76, "pieces": ["Ⅳ", "camelCase", "<|", "endoftext", "|>\"", " \r\n", "<|", "fim", "_prefix", "|>'", "rea", "/bᵃtfi", "'re", "ᵃB", "👍🏽", "٣٤٥", "٦", "ABCe", "'D", "Bᵃ", "\r", "m", "\"\r\r\n", "　d", " عDžungla", "\r\n", "0", "'Re", "𐞁", " \n", "!ſ"]} +{"text": "İ漢\"aZ½‍<|fim_prefix|>d12345678B're", "tokens": 21, "pieces": ["İ漢", "\"aZ", "½", "‍<|", "fim", "_prefix", "|>", "d", "123", "456", "78", "B", "'re"]} +{"text": "　0字…½\n'ſ/\r\né½t12345678漢ḍ̇.漢漢>é-/\r\n'ḍ̇\"Ⅳ'D👍🏽\n#$%<|endoftext|>", "tokens": 60, "pieces": ["　", "0", "字", "…", "½", "\n", "'ſ", "/\r\n", "e", "́", "½", "t", "123", "456", "78", "漢ḋ", "̣.", "漢漢", ">é", "-/\r\n", "'ḋ", "̣\"", "Ⅳ", "'D", "👍🏽\n", "#$%<|", "endoftext", "|>"]} +{"text": "
å'VE‍'D \n
ß'VE \n👍🏽٣٤٥٦​𐞁'TAb𐞁…tHTTPServer👍🏽", "tokens": 54, "pieces": ["
a", "̊'", "VE", "‍'", "D", " \n", "
ß", "'VE", " \n", "👍🏽", "٣٤٥", "٦", "​𐞁", "'T", "Ab𐞁", "…tHTTPServer", "👍🏽"]} +{"text": "'ſDžes\n'ſHTTPServer'S㋿e
  \n HTTPServer…\n
iOSZ㍿ß\n\u000b/👍🏽'reع漢 \n ḍ̇-'VE½", "tokens": 57, "pieces": ["'ſ", "Džes", "\n", "'ſ", "HTTPServer", "'S", "㋿e", "
  \n", " HTTPServer", "…\n", "
iOSZ", "㍿ß", "\n", "\u000b", "/👍🏽'", "reع漢", " \n", " ḋ", "̣-'", "VE", "½"]} +{"text": "'T½\"😀🏽'TBd/'re\r\n\r\nſ é٣٤٥٦'Re…<
", "tokens": 32, "pieces": ["'T", "½", "\"😀🏽'", "TBd", "/'", "re", "\r\n\r\n", "ſ", " ", " é", "٣٤٥", "٦", "'Re", "…", "<", "
"]} +{"text": "-\u000ba0Z9é '½.", "tokens": 11, "pieces": ["-", "\u000ba", "0", "Z", "9", "e", "́", " '", "½", "."]} +{"text": "३'S\nİa/b​Ａ. s \n ㋿/å.e!'Mḍ̇'DiOS", "tokens": 34, "pieces": ["३", "'S", "\n", "İa", "/b", "​Ａ", ".", " s", " \n", " ㋿/", "a", "̊.", "e", "!'", "Mḋ", "̣'", "DiOS"]} +{"text": "…/Džunglaİ
é\te \né \n eB <\rＡᵃ'ßBa\n/\n/½\t३sAb𐞁camelCase're/ 🙂", "tokens": 54, "pieces": ["…", "/<", "EOT", ">Džunglaİ", "
e", "́", "\te", " \n", "é", " \n", " eB", "", " ", " <\r", "Ａᵃ", "'ßBa", "\n", "/\n", "/", "½", "\t", "३", "sAb𐞁camelCase", "'re", "/", " 🙂"]} +{"text": "­'ReZ
字Z", "tokens": 8, "pieces": ["­'", "ReZ", "
字Z"]} +{"text": "ABC\n/漢'll\n\r\n\r\n🙂́ᵃＡꟲ'll'Réd>字-#$%😀🏽e½/\r\naBa/b'Re'", "ll", "'Re", "́d", ">字", "-#$%😀🏽", "e", "½", "/\r\n", "aBa", "/b", "'Re", "d9a\r\u000bꟲ'Re
漢éAb're<|endoftext|>\n0
EOTＡd٣٤٥٦😀🏽", "tokens": 75, "pieces": ["a", "'D", "a", "̊‍", " ", "'VE", "a", "/b", "'ll", "漢", " ", "\"", "…", "㋿ꟲ", "\n", "Dž", "d", "9", "a", "\r", "\u000bꟲ", "'Re", "
漢éAb", "'re", "<|", "endoftext", "|>\n", "0", "
EOTＡd", "٣٤٥", "٦", "😀🏽"]} +{"text": "12345678İ \nꟲEOTaعZ İ'ſ‍ \n12345678Džunglaé㍿eDž<|fim_prefix|>‍‍٣٤٥٦s\téⅣ12345678Z字aB(㋿fiAbé", "tokens": 75, "pieces": ["123", "456", "78", "İ", " \n", "ꟲEOTaعZ", " İ", "'ſ", "‍", " \n", "123", "456", "78", "Džunglaé", "㍿eDž", "<|", "fim", "_prefix", "|>‍‍", "٣٤٥", "٦", "s", "\te", "́", "Ⅳ12", "345", "678", "Z字aB", "(㋿", "fiAbe", "́"]} +{"text": "DžABC\r\n<|fim_prefix|>🙂‍Ab㍿٣٤٥٦Dž <|fim_prefix|>B#$%HTTPServer", "tokens": 41, "pieces": ["DžABC", "\r\n", "<|", "fim", "_prefix", "|>🙂‍", "Ab", "㍿", "٣٤٥", "٦", "Dž", " ", " <|", "fim", "_prefix", "|>", "B", "#$%", "HTTPServer"]} +{"text": "'sDžunglaHTTPServer  \n \n/漢'VE👍🏽ⅣAb0éⅣaå३12345678🙂'M,\u000b\r…٣٤٥٦Džungla३'Re(ſ'D", "tokens": 69, "pieces": ["'s", "DžunglaHTTPServer", "  \n \n", "/漢", "'VE", "👍🏽", "Ⅳ", "Ab", "0", "é", "Ⅳ", "aa", "̊", "३12", "345", "678", "🙂'", "M", ",", "\u000b\r", "…", "٣٤٥", "٦", "Džungla", "३", "'", "Re", "(ſ", "'D"]} +{"text": "'Mİ‍'sＡ.-½　ꟲ'VEḍ̇éᵃ \nAb12345678", "tokens": 30, "pieces": ["'M", "İ", "‍'", "sＡ", ".-", "½", "　ꟲ", "'VE", "ḋ", "̣éᵃ", " \n", "Ab", "123", "456", "78"]} +{"text": "Ａ　Ａ𐞁å㋿aß#$%a/\r\n,
s½>😀🏽(½
漢EOTABCع\u000b'S'-İ
m👍🏽<|endoftext|>\r\naᵃ", "tokens": 66, "pieces": ["Ａ", "　Ａ𐞁a", "̊㋿", "aß", "#$%", "a", "/\r\n", ",", "
s", "½", ">😀🏽(", "½", "
漢EOTABCع", "\u000b", "'S", "'-", "İ", "
m", "👍🏽<|", "endoftext", "|>\r\n", "aᵃ"]} +{"text": "#$%'VE(İ \n Ab­ 0a'Re　- 'S'Re­/\r\nB", "tokens": 22, "pieces": ["#$%'", "VE", "(İ", " \n", " Ab", "­", " ", " ", "0", "a", "'Re", "　", "-", " ", " '", "S", "'Re", "­/\r\n", "B"]} +{"text": "!!٣٤٥٦'Re \n👍🏽ꟲDžunglaa/b'ſßcamelCase(\r.'DaZs", "tokens": 41, "pieces": ["!!", "٣٤٥", "٦", "'Re", " \n", "👍🏽", "ꟲDžunglaa", "/b", "'ſ", "ßcamelCase", "(\r", ".'", "DaZs"]} +{"text": "sⅣ\n/ꟲa🙂½'M\n", "tokens": 20, "pieces": ["s", "", "Ⅳ", "\n", "/ꟲa", "🙂", "½", "'M", "\n"]} +{"text": "­​字Dž\t'Tß\r\niOS's'MAbİ >é'll/\r\nAbiOSa.​DžB😀🏽́", "tokens": 37, "pieces": ["­​", "字Dž", "\t", "'T", "ß", "\r\n", "iOS", "'s", "'M", "Abİ", " ", ">e", "́'", "ll", "/\r\n", "AbiOSa", ".​", "DžB", "😀🏽́"]} +{"text": "B\t/\r\n𐞁a/b<|fim_prefix|>Dž#$%\u000b\n/mcamelCase", "tokens": 28, "pieces": ["B", "\t", "/\r\n", "𐞁a", "/b", "<|", "fim", "_prefix", "|>", "Dž", "#$%", "\u000b\n", "/mcamelCase"]} +{"text": "!#$%'VEع<|fim_prefix|>HTTPServer\n<­\r\n\r\n‍字😀🏽é­,B<|fim_prefix|>३ \ne👍🏽\r\n\r\nd>", "tokens": 66, "pieces": ["!#$%'", "VEع", "<|", "fim", "_prefix", "|>", "HTTPServer", "\n", "<­\r\n\r\n", "‍字", "😀🏽", "e", "́<", "sꟲdß", "<𐞁", "
", ">­,", "B", "<|", "fim", "_prefix", "|>", "३", " \n", "e", "👍🏽\r\n\r\n", "d", ">"]} +{"text": "½tİ\u000b
", "tokens": 6, "pieces": ["½", "tİ", "\u000b
"]} +{"text": "'VE!!\n/ع😀🏽", "tokens": 10, "pieces": ["'VE", "!!\n", "/ع", "😀🏽"]} +{"text": "
 \n ABC\tfi\"'VEAb㍿/s🙂ABCa/b‍　'M0t\r\n'sEOT", "tokens": 32, "pieces": ["
 \n", " ABC", "\tfi", "\"'", "VEAb", "㍿/", "s", "🙂ABCa", "/b", "‍", "　", "'M", "0", "t", "\r\n", "'s", "EOT"]} +{"text": "\r\n\r\n#$%!!​½Ａ'M 'TiOS", "tokens": 13, "pieces": ["\r\n\r\n", "#$%!!​", "½", "Ａ", "'M", " ", " '", "TiOS"]} +{"text": "9éDžunglaEOTtda/bİ12345678<|fim_prefix|>#$%ß𐞁m\r\n/\r\nع9camelCases\u000b'SDžunglaḍ̇éꟲEOT<|fim_prefix|>aſ,/", "tokens": 70, "pieces": ["9", "e", "́DžunglaEOTt", "da", "/bİ", "123", "456", "78", "<|", "fim", "_prefix", "|>#$%", "ß𐞁m", "\r\n", "/\r\n", "ع", "9", "camelCases", "\u000b", "'S", "Džunglaḋ", "̣e", "́ꟲEOT", "<|", "fim", "_prefix", "|>", "aſ", ",/"]} +{"text": "EOT0　0Ⅳꟲ#$%字
Z'ſ'Ta/bḍ̇'llß", "tokens": 30, "pieces": ["EOT", "0", "　", "0Ⅳ", "ꟲ", "#$%", "字", "
Z", "'ſ", "'T", "a", "/bḋ", "̣'", "llß"]} +{"text": " \nDžunglam12345678\n/Ⅳ12345678EOT'lliOS'ſß", "tokens": 23, "pieces": [" \n", "Džunglam", "123", "456", "78", "\n", "/", "Ⅳ12", "345", "678", "EOT", "'ll", "iOS", "'ſ", "ß"]} +{"text": " \nEOTDž‍字\u000b👍🏽😀🏽字🙂", "tokens": 23, "pieces": [" \n", "EOTDž", "‍字", "\u000b", "👍🏽😀🏽", "字", "🙂"]} +{"text": "ꟲé12345678aB \n 12345678EOT \n<३'VE00…Ⅳ字iOSé'Reꟲ're's'siOSt 0aeعé's", "tokens": 56, "pieces": ["ꟲé", "123", "456", "78", "aB", " \n", " ", "123", "456", "78", "EOT", " \n", "<", "३", "'VE", "", "00", "…", "Ⅳ", "字iOSe", "́'", "Reꟲ", "'re", "'s", "'s", "iOSt", " ", "0", "aeعé", "'s", ""]} +{"text": "'ll'Sé/a/b(d eꟲ/\r\nＡå​'reAb'M#$%'sss\rḍ̇ ㋿/\r\nm", "tokens": 37, "pieces": ["'ll", "'S", "é", "/a", "/b", "(d", " eꟲ", "/\r\n", "Ａa", "̊​'", "reAb", "'M", "#$%'", "sss", "\r", "ḋ", "̣", " ", " ㋿/\r\n", "m"]} +{"text": "'ll \n/­", "tokens": 8, "pieces": ["'ll", " \n", "/­<", "EOT", ">"]} +{"text": "\"­t'll\"ééfi0Z's\n '㍿aB\"'re #$%…'ll\n/m漢३\u000b'D𐞁.ᵃ!!\t㍿Džungla😀🏽𐞁", "tokens": 64, "pieces": ["\"­<", "META", "_START", ">t", "'ll", "\"ééfi", "0", "Z", "'s", "\n", " ", " '㍿", "aB", "\"'", "re", " #$%", "…", "'ll", "\n", "/m漢", "३", "\u000b", "'D", "𐞁", ".ᵃ", "!!", "\t", "㍿Džungla", "😀🏽", "𐞁"]} +{"text": "sßAb'M\n\"\r\n\r\n<漢\tAb🙂DžＡ\t㍿\n/ ", "tokens": 24, "pieces": ["sßAb", "'M", "\n", "\"\r\n\r\n", "<漢", "\tAb", "🙂DžＡ", "\t", "㍿\n", "/", " "]} +{"text": "'s \n 👍🏽㍿ḍ̇🙂", "tokens": 17, "pieces": ["'s", " \n", " 👍🏽㍿", "ḋ", "̣🙂"]} +{"text": "(½　Z9s\r#$%12345678m'S
­\nDžunglafi𐞁ᵃmHTTPServer 'Re㍿㍿ \n B!!!'reZعİ<|endoftext|>", "tokens": 57, "pieces": ["(", "½", "　Z", "9", "s", "\r", "#$%", "123", "456", "78", "m", "'S", "
", "­\n", "Džunglafi𐞁ᵃmHTTPServer", " '", "Re", "㍿㍿", " \n", " B", "!!!'", "reZعİ", "<|", "endoftext", "|>"]} +{"text": "\réＡ a/ba­…٣٤٥٦\n12345678漢", "tokens": 25, "pieces": ["\r", "éＡ", " a", "/ba", "­", "…", "٣٤٥", "٦", "\n", "123", "456", "78", "漢"]} +{"text": "\r\n\r\n'T#$%< \n 'VE ", "tokens": 9, "pieces": ["\r\n\r\n", "'T", "#$%<", " \n", " '", "VE", " "]} +{"text": "\n/字iOSع½\r\n\r\n
camelCase#$%٣٤٥٦å<|fim_prefix|>­\r'SfiⅣ", "tokens": 41, "pieces": ["\n", "/字iOSع", "½", "\r\n\r\n", "
", "camelCase", "#$%", "٣٤٥", "٦", "a", "̊<|", "fim", "_prefix", "|>­\r", "'S", "fi", "Ⅳ"]} +{"text": "'s漢𐞁İiOScamelCasedİe #$%d'MABCs \n 'reİa/b-ع'reHTTPServer \n!३camelCase<|fim_prefix|>😀🏽é.", "tokens": 54, "pieces": ["'s", "漢𐞁İiOScamelCasedİe", " ", " #$%", "d", "'M", "ABCs", " \n", " '", "reİa", "/b", "-", "ع", "'re", "HTTPServer", " \n", "!", "३", "camelCase", "<|", "fim", "_prefix", "|>😀🏽", "é", "."]} +{"text": "12345678Džungla'D ½a \n …<12345678ᵃ/aBDžungla\u000b㍿camelCase🙂mع\u000b<|fim_prefix|>'Tm字iOS<|endoftext|><|endoftext|>iOSſABC\tꟲ😀🏽\n/Ab\t", "tokens": 83, "pieces": ["123", "456", "78", "Džungla", "'D", " ", "½", "a", " \n", " ", "…", "<", "123", "456", "78", "ᵃ", "/aBDžungla", "\u000b", "㍿camelCase", "🙂mع", "\u000b", "<|", "fim", "_prefix", "|>'", "Tm字iOS", "<|", "endoftext", "|><|", "endoftext", "|>", "iOSſABC", "\tꟲ", "😀🏽\n", "/Ab", "\t"]} +{"text": "漢'ſ", "tokens": 5, "pieces": ["漢", "'ſ"]} +{"text": "… camelCasefiİ<३#$%㍿ḍ̇dB😀🏽", "tokens": 27, "pieces": ["… ", " camelCasefiİ", "<", "३", "#$%㍿", "ḋ", "̣dB", "😀🏽"]} +{"text": "\u000b\n/!\r\n\r\niOS'ſ12345678/\r\n ㍿'M<|fim_prefix|>Ab\r😀🏽ſ'll­", "tokens": 37, "pieces": ["\u000b\n", "/!\r\n\r\n", "iOS", "'ſ", "123", "456", "78", "/\r\n", " ㍿'", "M", "<|", "fim", "_prefix", "|>", "Ab", "\r", "😀🏽", "ſ", "'ll", "­"]} +{"text": "'Taſ​'ll0३'S'ſ'Sſ!!😀🏽,!!\u000bHTTPServer/​mé
9𐞁ß𐞁\r\n\r\n's'ᵃ🙂s", "tokens": 53, "pieces": ["'T", "aſ", "​'", "ll", "0३", "'S", "'ſ", "'S", "ſ", "!!😀🏽,!!", "\u000bHTTPServer", "/​", "mé", "
", "9", "𐞁ß𐞁", "\r\n\r\n", "'s", "'ᵃ", "🙂s"]} +{"text": "/\r\n👍🏽­​İİß're\n/'llZmB\n/́ſ́aBİع😀🏽're …'ſ '　.㋿camelCaseHTTPServerfit'sm", "tokens": 58, "pieces": ["/\r\n", "👍🏽­​", "İİß", "'re", "\n", "/'", "llZmB", "\n", "/́", "ſ", "́aBİع", "😀🏽'", "re", " ", "…", "'ſ", " ", " '", "　", ".㋿", "camelCaseHTTPServerfit", "'s", "m"]} +{"text": "ꟲ!!m'D,.,ع🙂!!iOS#$%\r\n…\"\rAb'SaBABC👍🏽>\u000b👍🏽t#$%Dž½", "tokens": 44, "pieces": ["ꟲ", "!!", "m", "'D", ",.,", "ع", "🙂!!", "iOS", "#$%\r\n", "…", "\"\r", "Ab", "'S", "aBABC", "👍🏽>", "\u000b", "👍🏽", "t", "#$%", "Dž", "½"]} +{"text": "Ab0\r\nſ🙂é,B-,fi'll'VE'BEOT
12345678\n/HTTPServerEOT👍🏽ع \n ⅣcamelCase🙂字🙂ßEOT\u000beiOS…🙂\"٣٤٥٦", "tokens": 68, "pieces": ["Ab", "0", "\r\n", "ſ", "🙂e", "́,", "B", "-,", "fi", "'ll", "'VE", "'BEOT", "
", "123", "456", "78", "\n", "/HTTPServerEOT", "👍🏽", "ع", " \n", " ", "Ⅳ", "camelCase", "🙂字", "🙂ßEOT", "\u000beiOS", "…", "🙂\"", "٣٤٥", "٦"]} +{"text": "EOT'reİ😀🏽 ㋿aB
HTTPServer　<३'D", "tokens": 24, "pieces": ["EOT", "'re", "İ", "😀🏽", " ㋿", "aB", "
HTTPServer", "　", "<", "३", "'D"]} +{"text": "Ⅳ\r\u000b漢
<|endoftext|> \n 'll½­å Džungla　åⅣᵃiOSḍ̇́ſ!‍aB/\r\nAbع ́aBe\u000b'ſ㋿
", "tokens": 71, "pieces": ["Ⅳ", "\r", "\u000b漢", "
", "<|", "endoftext", "|><", "META", "_START", ">", " \n", " '", "ll", "½", "­a", "̊", " ", " Džungla", "　a", "̊", "Ⅳ", "ᵃiOSḋ", "̣́", "ſ", "!‍", "aB", "/\r\n", "Abع", " ́", "aBe", "\u000b", "'ſ", "㋿", "
"]} +{"text": "\nZ!!é#$%٣٤٥٦ ㋿\t​­/
ABC㋿", "tokens": 28, "pieces": ["\n", "Z", "!!", "é", "#$%", "٣٤٥", "٦", " ", "㋿", "\t", "​­/", "
ABC", "㋿"]} +{"text": " \n Zm\r\n'sé漢!!#$%a/bEOTꟲ<|endoftext|>'sß👍🏽😀🏽\n/dcamelCase'T'D9s>\niOS\n/ꟲ'VE's/\rᵃDž…iOS", "tokens": 70, "pieces": [" \n", " Zm", "\r\n", "'s", "e", "́漢", "!!#$%<", "META", "_START", ">a", "/bEOTꟲ", "<|", "endoftext", "|>'", "sß", "👍🏽😀🏽\n", "/dcamelCase", "'T", "'D", "9", "s", ">\n", "iOS", "\n", "/ꟲ", "'VE", "'s", "/\r", "ᵃDž", "…iOS"]} +{"text": "\rsＡعsEOT
　\tſ \n \n　 \n a/b\t'VEe>'VEDžungla.e'ReZ…'<|fim_prefix|>0BⅣ\"\ta", "tokens": 51, "pieces": ["\r", "sＡعsEOT", "
　", "\t", "ſ", " \n \n　 \n", " a", "/b", "\t", "'VE", "e", ">'", "VEDžungla", ".e", "'Re", "Z", "…", "'<|", "fim", "_prefix", "|>", "0", "B", "Ⅳ", "\"", "\ta"]} +{"text": "Ⅳ", "tokens": 6, "pieces": ["Ⅳ", ""]} +{"text": "ſiOSéEOT  \nsß\r\n\r\n字½!!㋿(é😀🏽
>", "tokens": 27, "pieces": ["ſiOSéEOT", "  \n", "sß", "\r\n\r\n", "字", "½", "!!㋿(", "é", "😀🏽", "
", ">"]} +{"text": "​½ꟲ㋿.dⅣ'M३1234567812345678iOS‍…'Refi'llİ", "tokens": 31, "pieces": ["​", "½", "ꟲ", "㋿.", "d", "Ⅳ", "'M", "३12", "345", "678", "123", "456", "78", "iOS", "‍", "…", "'Re", "fi", "'ll", "İ"]} +{"text": "㋿é
'Mſ#$%s#$%
👍🏽 😀🏽३İ‍'VEع-HTTPServer字­", "tokens": 40, "pieces": ["㋿é", "
", "'M", "ſ", "#$%", "s", "#$%", "
", "👍🏽", " ", " 😀🏽", "३", "İ", "‍'", "VEع", "-HTTPServer字", "­"]} +{"text": "३fi9­HTTPServer", "tokens": 8, "pieces": ["३", "fi", "9", "­HTTPServer"]} +{"text": " Abꟲ\"ABC", "tokens": 7, "pieces": [" Abꟲ", "\"ABC"]} +{"text": "字́d/Ⅳḍ̇'a/bZ,ᵃ
'St​'ll‍३s𐞁İ", "tokens": 36, "pieces": ["字", "́d", "/", "Ⅳ", "ḋ", "̣'", "a", "/bZ", ",ᵃ", "
", "'S", "t", "​'", "ll", "‍", "३", "s𐞁İ"]} +{"text": "漢ꟲéİſfiZ\u000b㍿.\rDž३iOS>\n
EOT㍿ \n'T", "tokens": 34, "pieces": ["漢ꟲe", "́İſfiZ", "\u000b", "㍿.\r", "Dž", "३", "iOS", ">\n", "
EOT", "㍿", " \n", "'T"]} +{"text": ">😀🏽's🙂>'re\r\ń/\r\n<|endoftext|>'DB\r\n9…'sＡ\n/(eß'D/a…- \n m\r\nem
ABC😀🏽'", "s", "🙂>'", "re", "\r\n", "́/\r\n", "<|", "endoftext", "|>'", "DB", "\r\n", "9", "…", "'s", "Ａ", "\n", "/(", "eß", "'D", "/", "a", "…", "-", " \n", " ", " m", "\r\n", "em", "
ABC", "12345678'T\n'VE𐞁𐞁\u000b<ß<|fim_prefix|>diOSaBABC…Ⅳa\r\n​", "tokens": 51, "pieces": ["　", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "\n", "'VE", "𐞁𐞁", "\u000b", "<ß", "<|", "fim", "_prefix", "|>", "d", "iOSaBABC", "…", "Ⅳ", "a", "\r\n", "​"]} +{"text": "å😀🏽عcamelCase /\r\n ḍ̇ \n Ab漢", "tokens": 28, "pieces": ["a", "̊😀🏽", "ع", "camelCase", " ", " /\r\n", " ", " ḋ", "̣", " \n", " Ab漢"]} +{"text": "ḍ̇\u000b­Z/Džungla٣٤٥٦!\n　…　>EOT<|fim_prefix|>
'ReAbéعABC\n/'VE½Ａ\u000be(👍🏽", "tokens": 57, "pieces": ["ḋ", "̣", "\u000b", "­Z", "/Džungla", "٣٤٥", "٦", "!\n", "　…", "　", ">EOT", "<|", "fim", "_prefix", "|>", "
", "'Re", "AbéعABC", "\n", "/'", "VE", "½", "Ａ", "\u000be", "(👍🏽"]} +{"text": "'S>m!B>#$%!!0ſAbABCİ👍🏽é'D ​ B \n\u000bDžungla\n/ aBB \n/　\r'M", "tokens": 41, "pieces": ["'S", ">m", "!B", ">#$%!!", "0", "ſAbABCİ", "👍🏽", "é", "'D", " ", "​", " B", " \n", "\u000bDžungla", "\n", "/", " ", " aBB", " \n", "/", "　\r", "'M"]} +{"text": "/😀🏽-İ-åع'ſ\r\n dcamelCase😀🏽's㋿(㍿㍿'et…a<|endoftext|>/\r\n!", "tokens": 56, "pieces": ["/😀🏽-", "İ", "-a", "̊ع", "'ſ", "\r\n", " dcamelCase", "😀🏽'", "s", "㋿(㍿㍿'<", "META", "_START", ">et", "…a", "<|", "endoftext", "|>/\r\n", "!<", "META", "_START", ">"]} +{"text": "Zع'fi\r\n\r\n\r\nABCe0\rZ३ficamelCase'DDžungla'MsEOTd\r0ſꟲ'ReDžungla9…Ab½DžunglaAbDžungla½", "tokens": 54, "pieces": ["Zع", "'fi", "\r\n\r\n\r\n", "ABCe", "0", "\r", "Z", "३", "ficamelCase", "'D", "Džungla", "'M", "sEOTd", "\r", "0", "ſꟲ", "'Re", "Džungla", "9", "…Ab", "½", "DžunglaAbDžungla", "½"]} +{"text": "ABCaBß　'VEe'sDžunglaᵃ", "tokens": 16, "pieces": ["ABCaBß", "　", "'VE", "e", "'s", "Džunglaᵃ"]} +{"text": "Z 🙂é!!漢‍\r\n漢👍🏽B", "tokens": 19, "pieces": ["Z", " ", " 🙂", "é", "!!", "漢", "‍\r\n", "漢", "👍🏽", "B"]} +{"text": "EOTDžungla (iOS'😀🏽 \n /\r\n,\rt\r\n\r\n🙂B'Ⅳ0EOTa/b\r\n\r\n!‍", "tokens": 41, "pieces": ["EOTDžungla", " ", " (", "iOS", "'😀🏽", " \n", " /\r\n", ",\r", "t", "\r\n\r\n", "🙂B", "'", "Ⅳ0", "EOTa", "/b", "\r\n\r\n", "!‍<", "EOT", ">"]} +{"text": "\u000bEOT.s
ſ👍🏽👍🏽İع. \n >🙂㍿,s'T𐞁iOSḍ̇‍BDžungla😀🏽Džungla>İ🙂DžunglacamelCase'D'ſ!d㋿", "tokens": 83, "pieces": ["\u000bEOT", ".s", "
ſ", "👍🏽👍🏽", "İع", ".", " \n", " >🙂㍿,", "s", "'", "T𐞁iOSḋ", "̣‍", "BDžungla", "😀🏽", "Džungla", ">İ", "🙂DžunglacamelCase", "'D", "'", "ſ", "!d", "㋿"]} +{"text": "''S9ABC㋿,\n/9\r'Ś(ABCⅣ́", "tokens": 21, "pieces": ["''", "S", "", "9", "ABC", "㋿,\n", "/", "9", "\r", "'S", "́(", "ABC", "Ⅳ", "́"]} +{"text": "漢㋿​'VE​ßḍ̇<㋿iOS…İ㍿ fiＡ  ́㋿ \n Ab'VE", "tokens": 41, "pieces": ["漢", "㋿​'", "VE", "​ßḋ", "̣<㋿", "iOS", "…İ", "㍿", " fiＡ", " ", " ", "́㋿", " \n", " Ab", "'VE"]} +{"text": "d字\r\n\r\n𐞁camelCaseHTTPServer'ſ́9aåå<|fim_prefix|>< .éḍ̇dⅣßfi", "tokens": 43, "pieces": ["d字", "\r\n\r\n", "𐞁camelCaseHTTPServer", "'ſ", "́", "9", "aa", "̊a", "̊<|", "fim", "_prefix", "|><", " ", " .", "éḋ", "̣d", "Ⅳ", "ßfi"]} +{"text": "'VEⅣåst \n fi\"( \nİ٣٤٥٦0camelCase٣٤٥٦'D'TＡ/\r\nBdt字३", "tokens": 43, "pieces": ["'VE", "Ⅳ", "a", "̊st", " \n", " fi", "\"(", " \n", "İ", "٣٤٥", "٦0", "camelCase", "٣٤٥", "٦", "'D", "'T", "Ａ", "/\r\n", "Bdt字", "३"]} +{"text": "字\rcamelCase\r\n\r\néåt#$%fi\n/,!HTTPServer#$% 'ss", "tokens": 28, "pieces": ["字", "\r", "camelCase", "\r\n\r\n", "e", "́a", "̊t", "#$%", "fi", "\n", "/<", "EOT", ">,!", "HTTPServer", "#$%", " ", " '", "ss"]} +{"text": "\t𐞁'll….\rⅣ9\u000bᵃ٣٤٥٦𐞁iOS\n\t…​'M　३\t'S👍🏽　d'", "tokens": 51, "pieces": ["\t𐞁", "'ll", "…", ".\r", "Ⅳ9", "\u000bᵃ", "٣٤٥", "٦", "𐞁iOS", "\n", "\t", "…", "​'", "M", "　", "३", "\t", "'S", "👍🏽", "　d", "'"]} +{"text": "fi漢👍🏽", "tokens": 10, "pieces": ["fi漢", "👍🏽"]} +{"text": "字<|endoftext|>\r\n>𐞁\t\r\nß AbZ \n 'Sd\r­/\r\n​ \n ", "tokens": 30, "pieces": ["字", "<|", "endoftext", "|>\r\n", ">𐞁", "\t\r\n", "ß", " AbZ", "", " \n", " '", "Sd", "\r", "­/\r\n", "​", " \n "]} +{"text": "​Džungla", "tokens": 5, "pieces": ["​Džungla"]} +{"text": "\r\n  éfi㍿Z३Dž", "tokens": 14, "pieces": ["\r\n", " ", " éfi", "㍿Z", "३", "Dž"]} +{"text": "9\u000b'VE'll", "tokens": 5, "pieces": ["9", "\u000b", "'VE", "'ll"]} +{"text": " 🙂!​\n'Mm\n/Bé'D>s\r\n字'M㋿\nEOT's/\r\n　fi('Ta/bmd", "tokens": 31, "pieces": [" 🙂!​\n", "'M", "m", "\n", "/Be", "́'", "D", ">s", "\r\n", "字", "'M", "㋿\n", "EOT", "'s", "/\r\n", "　fi", "('", "Ta", "/bmd"]} +{"text": "/\r\n‍å>B's字'VE'𐞁iOS-ꟲå३Džunglae‍ḍ̇​fi>.", "tokens": 47, "pieces": ["/\r\n", "‍a", "̊>", "B", "'s", "字", "'VE", "'𐞁iOS", "-ꟲa", "̊", "३", "Džunglae", "‍ḋ", "̣​", "fi", ">."]} +{"text": " \nABCiOS३İ\nABCDžs", "tokens": 11, "pieces": [" \n", "ABCiOS", "३", "İ", "\n", "ABCDžs"]} +{"text": "e字,\r\n\r\ncamelCase'llcamelCaseåHTTPServereعtEOT'ReABCé>", "tokens": 23, "pieces": ["e字", ",\r\n\r\n", "camelCase", "'ll", "camelCasea", "̊HTTPServereعtEOT", "'Re", "ABCe", "́>"]} +{"text": "009<|endoftext|>a'll\" 'reḍ̇Džungla　/\r\n émİꟲ'M", "tokens": 37, "pieces": ["009", "<|", "endoftext", "|>", "a", "'ll", "\"<", "META", "_START", ">", " ", " '", "reḋ", "̣Džungla", "　", "/\r\n", " ", " e", "́mİꟲ", "'M"]} +{"text": "\n漢'VE \n ", "tokens": 7, "pieces": ["\n", "漢", "'VE", " \n "]} +{"text": "\n/\t\nEOTꟲ0㋿édéB㋿,", "tokens": 24, "pieces": ["\n", "/<", "META", "_START", ">", "\t\n", "EOTꟲ", "0", "㋿e", "́déB", "㋿,"]} +{"text": "ᵃ漢 \n­\n/aBédéaB \n#$%fiZ\r\n!!HTTPServer'ſ'llcamelCase", "tokens": 30, "pieces": ["ᵃ漢", " \n", "­\n", "/aBédéaB", " \n", "#$%", "fiZ", "\r\n", "!!", "HTTPServer", "'ſ", "'ll", "camelCase"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'llİᵃ9😀🏽s‍'ſⅣ㋿<|endoftext|>ᵃ're🙂𐞁'ſ…\u000beعßB're", "tokens": 50, "pieces": ["'ll", "İᵃ", "9", "😀🏽", "s", "‍'", "ſ", "Ⅳ", "㋿<|", "endoftext", "|>", "ᵃ", "'re", "🙂𐞁", "'ſ", "…", "\u000beعßB", "'re"]} +{"text": "'Re!camelCase字👍🏽(å-!\"٣٤٥٦\u000b/½/​ع<ᵃ \n <|endoftext|>'Se 12345678'll🙂<|fim_prefix|>éDžunglaé'MHTTPServer!!‍!", "tokens": 73, "pieces": ["'Re", "!camelCase字", "👍🏽(", "a", "̊-!\"", "٣٤٥", "٦", "\u000b", "/", "½", "/​", "ع", "<ᵃ", " \n", " <|", "endoftext", "|>'", "Se", " ", "123", "456", "78", "'ll", "🙂<|", "fim", "_prefix", "|>", "e", "́Džunglae", "́'", "MHTTPServer", "!!‍!"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'VE㋿", "tokens": 5, "pieces": ["'VE", "㋿"]} +{"text": "
iOS''Refi👍🏽\nfi👍🏽\r\n\r\n!! \n ٣٤٥٦'ll٣٤٥٦t!!…😀🏽ᵃmİⅣ", "tokens": 59, "pieces": ["
iOS", "''", "Refi", "👍🏽\n", "fi", "👍🏽\r\n\r\n", "!!", " \n", " ", "٣٤٥", "٦", "'ll", "٣٤٥", "٦", "t", "!!", "…", "😀🏽", "ᵃmİ", "Ⅳ"]} +{"text": "…'Me<|fim_prefix|>\r\n\r\n 𐞁camelCase३\u000biOS12345678d('D", "tokens": 28, "pieces": ["…", "'M", "e", "<|", "fim", "_prefix", "|>\r\n\r\n", " 𐞁camelCase", "३", "\u000biOS", "123", "456", "78", "d", "('", "D"]} +{"text": "Džungla\rEOT Džungla-…aB\r\n ", "tokens": 20, "pieces": ["Džungla", "\r", "EOT", " ", " Džungla", "-", "…aB", "\r\n "]} +{"text": "İå'Sm-!fi'S\u000bſBiOS​ \n >camelCase\"912345678\r\n\r\n\tZ", "tokens": 27, "pieces": ["İa", "̊'", "Sm", "-!", "fi", "'S", "\u000bſBiOS", "​", " \n", " >", "camelCase", "\"", "912", "345", "678", "\r\n\r\n", "\tZ"]} +{"text": "‍9‍­/\r\n-sé,…!!d(\r\n \nſ#$%漢é#$%!㋿ᵃ½ABC\r\t\r\nZed\n/𐞁 ​ ", "tokens": 49, "pieces": ["‍", "9", "‍­/\r\n", "-sé", ",", "…", "!!", "d", "(\r\n", " \n", "ſ", "#$%", "漢é", "#$%!㋿", "ᵃ", "½", "ABC", "\r\t\r\n", "Zed", "\n", "/𐞁", "", " ​", " "]} +{"text": "HTTPServer'DßAb漢m'll9Dž!0漢'ſḍ̇\u000b-ſa/bfiB\r\n\r\n‍'TDž (Ⅳ/\r\n", "tokens": 49, "pieces": ["HTTPServer", "'D", "ßAb漢m", "'ll", "9", "Dž", "!", "0", "漢", "'ſ", "ḋ", "̣", "\u000b", "-ſa", "/bfiB", "\r\n\r\n", "‍'", "TDž", " ", " (", "Ⅳ", "/\r\n"]} +{"text": "!å('T ½a/b​0\r\n.DžunglaHTTPServer'ſ㍿fi㋿!字!​漢😀🏽\tᵃZ!Z字 ", "tokens": 51, "pieces": ["!a", "̊('", "T", " ", "½", "a", "/b", "​", "0", "\r\n", ".DžunglaHTTPServer", "'ſ", "㍿fi", "㋿!", "字", "!​", "漢", "😀🏽", "\tᵃZ", "!Z字", " "]} +{"text": "ع EOT!!0ſ'VEＡ", "tokens": 11, "pieces": ["ع", " EOT", "!!", "0", "ſ", "'VE", "Ａ"]} +{"text": "\r<|fim_prefix|>'s \n DžåAb\t­\n३B'sAbs\n😀🏽…", "tokens": 33, "pieces": ["\r", "<|", "fim", "_prefix", "|>'", "s", " \n", " Dža", "̊Ab", "\t", "­\n", "३", "B", "'s", "Abs", "\n", "😀🏽", "…"]} +{"text": "m👍🏽e\r\n\r\n9‍a/bⅣ<🙂́(ع(<|fim_prefix|>­- \n \"m<|endoftext|>>㋿\t
­-", " \n", " \"", "m", "<|", "endoftext", "|>>㋿", "\t", "
", "㍿ⅣHTTPServer \nABC", "tokens": 29, "pieces": [" \n", "!!", "İ", "\r\n\r\n", "٣٤٥", "٦", " ", "<|", "fim", "_prefix", "|>㍿", "Ⅳ", "HTTPServer", " \n", "ABC"]} +{"text": "'ll,🙂camelCase…­😀🏽-m'D'D ⅣB'Sᵃ😀🏽'HTTPServer", "tokens": 34, "pieces": ["'ll", ",🙂", "camelCase", "…", "­😀🏽-", "m", "'D", "'D", " ", "Ⅳ", "B", "'S", "ᵃ", "😀🏽'", "HTTPServer"]} +{"text": "é.Ⅳé
Ⅳ> \n12345678a/b👍🏽Dž\r\n\r\n\r\nBs𐞁<'T>…9,9字 'reᵃᵃ!>­ſ\"Džungla漢", "tokens": 64, "pieces": ["e", "́.", "Ⅳ", "e", "́", "
", "Ⅳ", ">", " \n", "123", "456", "78", "a", "/b", "👍🏽", "Dž", "\r\n\r\n\r\n", "Bs𐞁", "<'", "T", ">", "…", "9", ",", "9", "字", " '", "reᵃᵃ", "!>­", "ſ", "\"Džungla漢"]} +{"text": "aEOT'ſ'ſ s \n iOS \n ᵃ'refiABC३\n'D", "tokens": 24, "pieces": ["aEOT", "'ſ", "'ſ", " s", " \n", " iOS", " \n", " ᵃ", "'re", "fiABC", "३", "\n", "'D"]} +{"text": "tiOS'SＡ\r\n漢9‍　\t12345678'M٣٤٥٦>𐞁३٣٤٥٦\n/!!d漢'VEHTTPServer'reＡABC", "tokens": 62, "pieces": ["tiOS", "'S", "Ａ", "\r\n", "漢", "9", "‍", "　", "\t", "123", "456", "78", "'M", "٣٤٥", "٦", ">𐞁", "३٣٤", "٥٦", "\n", "/!!", "d漢", "'VE", "HTTPServer", "'re", "ＡABC"]} +{"text": "'S'T‍12345678字ABC", "tokens": 9, "pieces": ["'S", "'T", "‍", "123", "456", "78", "字ABC"]} +{"text": "㍿​'DB!!<Dž😀🏽'D漢\n//tAb　　ᵃ'ReHTTPServerfi\r\n字saBa字iOS'll#$%Z½㋿", "tokens": 50, "pieces": ["㍿​'", "DB", "!!<", "Dž", "😀🏽'", "D漢", "\n", "/<", "META", "_START", ">/", "tAb", "　", "　ᵃ", "'Re", "HTTPServerfi", "\r\n", "字saBa字iOS", "'ll", "#$%", "Z", "½", "㋿"]} +{"text": "s,\niOS!#$%Ａ\r…😀🏽,a/bZ's­mZḍ̇३'VE\r\n/'ll\u000b.HTTPServereعİm㋿'S'M'Re", "tokens": 50, "pieces": ["s", ",\n", "iOS", "!#$%", "Ａ", "\r", "…", "😀🏽,", "a", "/bZ", "'s", "­mZḋ", "̣", "३", "'VE", "\r\n", "/'", "ll", "\u000b", ".HTTPServereعİm", "㋿'", "S", "'M", "'Re"]} +{"text": "ßt 👍🏽́ع'VE's٣٤٥٦!!12345678<|endoftext|>\r/\r\n½ ḍ̇", "tokens": 40, "pieces": ["ßt", " 👍🏽́", "ع", "'VE", "'s", "٣٤٥", "٦", "!!", "123", "456", "78", "<|", "endoftext", "|>\r", "/\r\n", "½", " ḋ", "̣"]} +{"text": "\n/12345678camelCase/٣٤٥٦EOT'ſ's\n/<\ra/biOSİ!!😀🏽a\r\n\r\n\r\n!!åééa㍿a/b", "tokens": 51, "pieces": ["\n", "/", "123", "456", "78", "camelCase", "/", "٣٤٥", "٦", "EOT", "'ſ", "'s", "\n", "/<\r", "a", "/biOSİ", "!!😀🏽", "a", "\r\n\r\n\r\n", "!!", "a", "̊e", "́e", "́a", "㍿a", "/b"]} +{"text": "👍🏽½Ⅳ
'M \n字 'T ,½́/é'ſ'", "tokens": 27, "pieces": ["👍🏽", "½Ⅳ", "
", "'M", " \n", "字", " ", "'T", " ", " ,", "½", "́/", "e", "́'", "ſ", "'"]} +{"text": "\rZ'sEOTHTTPServerfi0½AbHTTPServera/b<|endoftext|>…'Re\n<|fim_prefix|>-", "tokens": 38, "pieces": ["\r", "Z", "'s", "EOTHTTPServer", "fi", "0½", "AbHTTPServera", "/b", "<|", "endoftext", "|>", "…", "'Re", "\n", "<|", "fim", "_prefix", "|>-"]} +{"text": "s'D'rét漢é<|endoftext|>…HTTPServerHTTPServerABC9!!'re½'D😀🏽,ß漢Z", "tokens": 39, "pieces": ["s", "'D", "'re", "́t漢e", "́<|", "endoftext", "|>", "…HTTPServerHTTPServerABC", "9", "!!'", "re", "½", "'D", "😀🏽,", "ß漢Z"]} +{"text": "😀🏽'VEt३ \t9'Reé́!,𐞁́<|endoftext|><|fim_prefix|>㋿9>­́'ReZ're/\r\n 👍🏽'ſ
é'T㍿\r\n\r\n", "tokens": 71, "pieces": ["😀🏽'", "VEt", "३", " ", "\t", "9", "'Re", "e", "́́!,", "𐞁", "́<|", "endoftext", "|><|", "fim", "_prefix", "|>㋿", "9", ">­́'", "ReZ", "'re", "/\r\n", " ", "👍🏽'", "ſ", "
e", "́'", "T", "㍿\r\n\r\n"]} +{"text": "
<|endoftext|>", "tokens": 9, "pieces": ["
", "<|", "endoftext", "|>"]} +{"text": "s.👍🏽é'M漢ꟲ'ß \n #$%0\r\n‍(", "tokens": 29, "pieces": ["s", ".👍🏽", "é", "'M", "漢ꟲ", "'ß", " \n", " #$%", "0", "\r\n", "‍("]} +{"text": "å漢EOT👍🏽🙂12345678\n/#$% \n'S　Džungla'D🙂Ⅳ12345678fi🙂EOT'", "tokens": 44, "pieces": ["a", "̊漢EOT", "👍🏽🙂", "123", "456", "78", "\n", "/#$%", " \n", "'S", "　Džungla", "'D", "🙂", "Ⅳ12", "345", "678", "fi", "🙂EOT", "'"]} +{"text": "३ſ'VEع\"'VE𐞁­'s٣٤٥٦'SZ/!!'ſꟲ<'M 
/\r\nHTTPServerfi", "tokens": 44, "pieces": ["३", "ſ", "'VE", "ع", "\"'", "VE𐞁", "­'", "s", "٣٤٥", "٦", "'S", "Z", "/!!'", "ſꟲ", "<'", "M", " ", "
", "/\r\n", "HTTPServerfi"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b're'SＡ'Re𐞁9\r\n\n/,m\r\n\r\ncamelCase𐞁३é\r\n\r\nع­​…\t'SＡ
aBſ…ḍ̇Z/㍿Z‍a/b", "tokens": 61, "pieces": ["\u000b", "'re", "'S", "Ａ", "'Re", "𐞁", "9", "\r\n\n", "/,", "m", "\r\n\r\n", "camelCase𐞁", "३", "e", "́\r\n\r\n", "ع", "­​", "…", "\t", "'S", "Ａ", "
", "aBſ", "…ḋ", "̣Z", "/㍿", "Z", "‍a", "/b"]} +{"text": "-‍.<|endoftext|>́Ⅳꟲa/b字'S#$%m'T
½", "tokens": 27, "pieces": ["-‍.<|", "endoftext", "|>́", "Ⅳ", "ꟲa", "/b字", "'S", "#$%", "m", "'T", "
", "½"]} +{"text": "'T,,e0/ \n' B'S'SDžunglaABCZ\u000b.\r\n9\ts\r\n\r\n'M㋿㋿12345678mſ!!
aB́>-
/\r\n👍🏽", "tokens": 54, "pieces": ["'T", ",,", "e", "0", "/", " \n", "'<", "EOT", ">", " ", " B", "'S", "'S", "DžunglaABCZ", "\u000b", ".\r\n", "9", "\ts", "\r\n\r\n", "'M", "㋿㋿", "123", "456", "78", "mſ", "!!", "
aB", "́>-", "
", "/\r\n", "👍🏽"]} +{"text": "ſe­'VE½'re\"́㍿'.0>d,٣٤٥٦Ab'\r\n ZZs're漢Ⅳ­Ａ \n\u000bZ0aB​/\r\nᵃEOT", "tokens": 51, "pieces": ["ſe", "­'", "VE", "½", "'re", "\"́㍿'.", "0", ">d", ",", "٣٤٥", "٦", "Ab", "'\r\n", " ", " ZZs", "'re", "漢", "Ⅳ", "­Ａ", " \n", "\u000bZ", "0", "aB", "​/\r\n", "ᵃEOT"]} +{"text": "…,😀🏽\n//\r\nHTTPServer!㋿Z'ſ-", "tokens": 21, "pieces": ["…", ",😀🏽\n", "//\r\n", "HTTPServer", "!㋿", "Z", "'ſ", "-"]} +{"text": "Dž<|endoftext|>…HTTPServer-ꟲ'VE　字　ßaBDžع9\u000bEOT9'.ꟲ\r\nssaB  Dž", "tokens": 44, "pieces": ["Dž", "<|", "endoftext", "|>", "…HTTPServer", "-ꟲ", "'VE", "　字", "　ßaBDžع", "9", "\u000bEOT", "9", "'.", "ꟲ", "\r\n", "ssaB", " ", " Dž"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "/\r\nİe‍#$%'ScamelCase'llEOT/'Dé
字字fi\tＡ\"'MABC/ع'reꟲ­ꟲ­aEOT𐞁字Džꟲßſ", "tokens": 58, "pieces": ["/\r\n", "İe", "‍#$%'", "ScamelCase", "'ll", "EOT", "/'", "De", "́", "
", "字字fi", "\tＡ", "\"'", "MABC", "/ع", "'re", "ꟲ", "­ꟲ", "­aEOT𐞁字Džꟲßſ"]} +{"text": "'VEᵃİ're.'s, !'ſ\r\r\n\r\n \n 9'T-\r\nm​camelCase/\r\n \n \r\n㋿٣٤٥٦字'ſ'VE>'s \n", "tokens": 52, "pieces": ["'VE", "ᵃİ", "'re", ".'", "s", ",", " ", "!'", "ſ", "\r\r\n\r\n \n", " ", "9", "'T", "-\r\n", "m", "​camelCase", "/\r\n", " \n \r\n", "㋿", "٣٤٥", "٦", "字", "'ſ", "'VE", ">'", "s", " \n", ""]} +{"text": "'M\rå<|fim_prefix|> \n9ḍ̇e/ᵃᵃcamelCase\t‍'D<|endoftext|>漢(", "tokens": 51, "pieces": ["'M", "\r", "a", "̊<", "EOT", "><|", "fim", "_prefix", "|>", " \n", "9", "ḋ", "̣e", "/ᵃᵃcamelCase", "\t", "‍'", "D", "<|", "endoftext", "|>", "漢", "("]} +{"text": "d-Ab \nḍ̇­aB漢'　…㋿>", "tokens": 22, "pieces": ["d", "-Ab", " \n", "ḋ", "̣­", "aB漢", "'", "　", "…", "㋿>"]} +{"text": "𐞁camelCase<|fim_prefix|>é'll ", "tokens": 16, "pieces": ["𐞁camelCase", "<|", "fim", "_prefix", "|>", "é", "'ll", " "]} +{"text": "<👍🏽camelCasecamelCase", "tokens": 11, "pieces": ["<👍🏽", "camelCasecamelCase"]} +{"text": "EOT-.́½ꟲ!!漢ABCd/\r\n!Džungla'lléABC'Reſſ'Mfim\"'Z'saBEOT'll‍Z", "tokens": 44, "pieces": ["EOT", "-.́", "½", "ꟲ", "!!", "漢ABCd", "/\r\n", "!Džungla", "'ll", "éABC", "'Re", "ſſ", "'M", "fi", "m", "\"'", "Z", "'s", "aBEOT", "'ll", "‍Z"]} +{"text": "å 'lla/b\n/\r\n㍿\r'　 \n Dž<|fim_prefix|>BAb字😀🏽Džungla\n/d-'Då\r'sᵃcamelCaseDž,\n'MfiZ\rＡ'ſå", "tokens": 70, "pieces": ["a", "̊", " '", "lla", "/b", "\n", "/\r\n", "㍿<", "META", "_START", ">\r", "'", "　 \n", " Dž", "<|", "fim", "_prefix", "|>", "BAb字", "😀🏽", "Džungla", "\n", "/d", "-'", "Da", "̊\r", "'s", "ᵃcamelCaseDž", ",\n", "'M", "fiZ", "\r", "Ａ", "'ſ", "a", "̊"]} +{"text": "👍🏽's", "tokens": 8, "pieces": ["👍🏽'", "s"]} +{"text": ".'Re😀🏽Dž", "tokens": 9, "pieces": [".'", "Re", "😀🏽", "Dž"]} +{"text": "camelCaseᵃé🙂a'll mᵃABC<|fim_prefix|>", "tokens": 22, "pieces": ["camelCaseᵃé", "🙂a", "'ll", " mᵃABC", "<|", "fim", "_prefix", "|>"]} +{"text": "عع\r\naBs\r\n\r\n'Re", "tokens": 10, "pieces": ["عع", "\r\n", "aBs", "\r\n\r\n", "'Re"]} +{"text": "
漢a　ß\r\n\r\n9㋿Ⅳ'Re'sABC‍\tABC<'Re㍿㋿漢,'re\"", "tokens": 34, "pieces": ["
漢a", "　ß", "\r\n\r\n", "9", "㋿", "Ⅳ", "'Re", "'s", "ABC", "‍", "\tABC", "<'", "Re", "㍿㋿", "漢", ",'", "re", "\""]} +{"text": "'ſ३fié- ́!B'T𐞁\n/!<👍🏽0'T0DžcamelCase\"!'​ \n'S　.\r<|fim_prefix|>åmABC😀🏽　", "tokens": 67, "pieces": ["'ſ", "३", "fie", "́-", " ", "́!<", "META", "_START", ">B", "'T", "𐞁", "\n", "/!<👍🏽", "0", "'T", "0", "DžcamelCase", "\"!'​", " \n", "'S", "　", ".\r", "<|", "fim", "_prefix", "|>", "a", "̊mABC", "😀🏽", "　"]} +{"text": "B㍿İ/s𐞁
", "tokens": 12, "pieces": ["B", "㍿İ", "/s𐞁", "
"]} +{"text": "…<|fim_prefix|>HTTPServer'Re३' \n 'sABC<|fim_prefix|>s \n ',", "tokens": 32, "pieces": ["…", "<|", "fim", "_prefix", "|>", "HTTPServer", "'Re", "३", "'", " \n", " '", "sABC", "<|", "fim", "_prefix", "|>", "s", " \n", " '<", "META", "_START", ">,"]} +{"text": "!!é\u000ba/bß'llfi字漢(Abé\u000b\r…‍d٣٤٥٦ \r'S.é'llEOT.AbcamelCase-\u000b'Re", "tokens": 47, "pieces": ["!!", "e", "́", "\u000ba", "/bß", "'ll", "fi字漢", "(Abe", "́", "\u000b\r", "…", "‍d", "٣٤٥", "٦", " \r", "'S", ".e", "́'", "llEOT", ".AbcamelCase", "-", "\u000b", "'Re"]} +{"text": "㋿.Dž
 /\r\n\t३/\r\niOS‍𐞁㍿́\r\n\r\n٣٤٥٦'0😀🏽<|fim_prefix|>㋿ß's㍿'ll ", "tokens": 66, "pieces": ["㋿.", "Dž", "
", " ", "/\r\n", "\t", "३", "/\r\n", "iOS", "‍𐞁", "㍿́\r\n\r\n", "٣٤٥", "٦", "'", "0", "😀🏽<|", "fim", "_prefix", "|>㋿", "ß", "'", "s", "㍿'", "ll", " "]} +{"text": ">,iOSHTTPServer\"(­ᵃt😀🏽", "tokens": 15, "pieces": [">,", "iOSHTTPServer", "\"(­", "ᵃt", "😀🏽"]} +{"text": "'ll", "tokens": 1, "pieces": ["'ll"]} +{"text": " \n…'MaBt'T'M字'lld\tEOT!! ‍Ⅳ\r\n\r\nEOT😀🏽", "tokens": 32, "pieces": [" \n", "…", "'M", "aBt", "'T", "'M", "字", "'ll", "d", "\tEOT", "!!", " ", " ‍", "Ⅳ", "\r\n\r\n", "EOT", "😀🏽<", "META", "_START", ">"]} +{"text": "iOS½,a/b',EOT\"㋿
'sEOT", "tokens": 16, "pieces": ["iOS", "½", ",a", "/b", "',", "EOT", "\"㋿", "
", "'s", "EOT"]} +{"text": "ABC<|fim_prefix|>#$% <|endoftext|>İéſAb'VEZ㍿0㋿㍿t9'VE😀🏽a/b🙂Džungla0'ReDžungla漢#$%'T🙂😀🏽\t", "tokens": 74, "pieces": ["ABC", "<|", "fim", "_prefix", "|>#$%", " ", " <|", "endoftext", "|>", "İéſAb", "'VE", "Z", "㍿", "0", "㋿㍿<", "META", "_START", ">t", "9", "'VE", "😀🏽", "a", "/b", "🙂Džungla", "0", "'Re", "Džungla漢", "#$%'", "T", "🙂😀🏽", "\t"]} +{"text": "㋿<|endoftext|>0ḿs\"-<|endoftext|>mᵃ #$%\"\taB're㍿s/'Re漢👍🏽Ⅳ'Re३. Źᵃ", "tokens": 57, "pieces": ["㋿<|", "endoftext", "|>", "0", "m", "́s", "\"-<|", "endoftext", "|>", "mᵃ", " ", "#$%\"", "\taB", "'re", "㍿s", "/'", "Re漢", "👍🏽", "Ⅳ", "'Re", "३", ".", " Z", "́ᵃ"]} +{"text": "ḍ̇\t\n/'re!!　 å😀🏽åmt!\né<|fim_prefix|>ع-\"''T\t'T́/\r\n!/#$%😀🏽-㋿㋿　d", "tokens": 62, "pieces": ["ḋ", "̣", "\t\n", "/'", "re", "!!", "　", " a", "̊😀🏽", "a", "̊mt", "!\n", "e", "́<|", "fim", "_prefix", "|>", "ع", "-\"''", "T", "\t", "'T", "́/\r\n", "!/#$%😀🏽-㋿㋿", "　d", ""]} +{"text": " ḍ̇…fiBe\n/\r\nAbⅣ\n/'ll", "tokens": 18, "pieces": [" ḋ", "̣", "…fiBe", "\n", "/\r\n", "Ab", "Ⅳ", "\n", "/'", "ll"]} +{"text": "HTTPServer\n/\n-éaB #$%9½.>'T \n\r\n\r\n", "tokens": 20, "pieces": ["HTTPServer", "\n", "/\n", "-e", "́aB", " #$%", "9½", ".>'", "T", " \n", "\r\n\r\n"]} +{"text": "ع-३٣٤٥٦\r\n\r\n012345678t३Ⅳ,漢ſiOS𐞁'Tİå'S12345678EOT'D𐞁ma/béⅣ'D😀🏽İ0漢/\r\n a/b
å", "tokens": 72, "pieces": ["ع", "-", "३٣٤", "٥٦", "\r\n\r\n", "012", "345", "678", "t", "३Ⅳ", ",漢ſiOS𐞁", "'T", "İa", "̊'", "S", "123", "456", "78", "EOT", "'D", "𐞁ma", "/be", "́", "Ⅳ", "'D", "😀🏽", "İ", "0", "漢", "/\r\n", " ", " a", "/b", "
a", "̊"]} +{"text": "Džİ ᵃ‍'MEOT'll!>…'sAba/bع \n0٣٤٥٦­/ \nß'sＡ　‍.\r,‍", "tokens": 47, "pieces": ["Džİ", " ᵃ", "‍'", "MEOT", "'ll", "!>", "…", "'s", "Aba", "/bع", " \n", "0٣٤", "٥٦", "­/", " \n", "ß", "'s", "Ａ", "　", "‍.\r", ",‍"]} +{"text": " e'M'Stſ'ſ ­Džungla​<|endoftext|>ꟲHTTPServer'D/\r\n​<|fim_prefix|>🙂ꟲ(.>/\r\n'Re'll\té𐞁字 \nABC0're", "tokens": 63, "pieces": [" e", "'M", "'S", "tſ", "'ſ", " ", "­Džungla", "​<|", "endoftext", "|>", "ꟲHTTPServer", "'D", "/\r\n", "​<|", "fim", "_prefix", "|>🙂", "ꟲ", "(.>/\r\n", "'", "Re", "'ll", "\té𐞁字", " \n", "ABC", "0", "'re"]} +{"text": "'ll(
> \n t🙂\n/>\r\n,éꟲ　aHTTPServer>éAb'VE…", "tokens": 27, "pieces": ["'ll", "(", "
", ">", " \n", " t", "🙂\n", "/>\r\n", ",e", "́ꟲ", "　aHTTPServer", ">éAb", "'VE", "…"]} +{"text": "!!½𐞁<­ᵃ\"ع ", "tokens": 14, "pieces": ["!!", "½", "𐞁", "<­", "ᵃ", "\"ع", " "]} +{"text": "\"(­­‍‍漢#$%'D\r\n\r\n \n 'llfi0eİ!aB\rå", "tokens": 28, "pieces": ["\"(­­‍‍", "漢", "#$%'", "D", "\r\n\r\n \n", " '", "llfi", "0", "eİ", "!aB", "\r", "a", "̊"]} +{"text": "'M'T漢ABĆ㍿a/bHTTPServer½a/bém#$%Ａ㍿ABC>
Ⅳ३'ſ𐞁'>aBABCB㍿ \n tABC　sa/bm'reᵃ12345678", "tokens": 64, "pieces": ["'M", "'T", "漢ABC", "́㍿", "a", "/bHTTPServer", "½", "a", "/be", "́m", "#$%", "Ａ", "㍿ABC", ">", "
", "Ⅳ३", "'ſ", "𐞁", "'>", "aBABCB", "㍿", " \n", " tABC", "　sa", "/bm", "'re", "ᵃ", "123", "456", "78"]} +{"text": "ع 
é́ꟲ-'M'S\"\t12345678>sİå", "tokens": 22, "pieces": ["ع", " ", "
é", "́ꟲ", "-'", "M", "'S", "\"", "\t", "123", "456", "78", ">sİa", "̊"]} +{"text": " \n /\r\n­'Dß👍🏽t\n/Ⅳ#$% B9​B\"Ⅳ'Re½Ａ<|endoftext|>字́", "tokens": 41, "pieces": [" \n", " /\r\n", "­'", "Dß", "👍🏽", "t", "\n", "/", "Ⅳ", "#$%", " B", "9", "​B", "\"", "Ⅳ", "'Re", "½", "Ａ", "<|", "endoftext", "|>", "字", "́"]} +{"text": "e/\r\n \n ३sᵃsDžunglaDžß \n 　'Re!ḍ̇́'ſ'D'sABCABC<|endoftext|>ABC0\r\n\r\n\r\n…!'VE👍🏽𐞁'Re'ſé,", "tokens": 71, "pieces": ["e", "/\r\n", " \n", " ", "३", "sᵃsDžunglaDžß", " \n", " ", "　", "'Re", "!ḋ", "̣́'", "ſ", "'D", "'s", "ABCABC", "<|", "endoftext", "|>", "ABC", "0", "\r\n\r\n\r\n", "…", "!'", "VE", "👍🏽", "𐞁", "'", "Re", "'ſ", "e", "́,"]} +{"text": "\n/B'VE३'a/b\rİ\r­字　　Džunglaſ
ꟲé/\r\nHTTPServer>\">", "tokens": 33, "pieces": ["\n", "/B", "'VE", "३", "'a", "/b", "\r", "İ", "\r", "­字", "　", "　Džunglaſ", "
ꟲe", "́/\r\n", "HTTPServer", ">\">"]} +{"text": "'MHTTPServer\r\n\r\n㍿téᵃt㍿ع!se0\r\nᵃ/\r\nße'Mé😀🏽‍\t́Dž😀🏽,#$%", "tokens": 50, "pieces": ["'M", "HTTPServer", "\r\n\r\n", "㍿téᵃt", "㍿ع", "!", "se", "0", "\r\n", "ᵃ", "/\r\n", "ße", "'M", "é", "😀🏽‍", "\t", "́Dž", "😀🏽,#$%"]} +{"text": " EOT'll'ſ́ⅣDžungla
 \n(.a/b\n/字s", "tokens": 27, "pieces": [" EOT", "'ll", "'ſ", "́<", "EOT", ">", "Ⅳ", "Džungla", "
 \n", "(.", "a", "/b", "\n", "/字s"]} +{"text": "‍'re\r\n\r\n字'lla-'s<12345678'D'MiOSé­ \n", "tokens": 24, "pieces": ["‍'", "re", "\r\n\r\n", "字", "'ll", "a", "-'", "s", "<", "123", "456", "78", "'D", "'M", "iOSé", "­", " \n"]} +{"text": "Ａ'VE'M/ 👍🏽Džungla12345678'ſ9m \n \n\n/daDžungla'T'S<|fim_prefix|>ſaB<|endoftext|>m's𐞁‍Z🙂é'S'reBfi", "tokens": 81, "pieces": ["Ａ", "'VE", "'", "M", "/", " ", "👍🏽", "Džungla", "123", "456", "78", "'ſ", "", "9", "m", " \n \n\n", "/daDžungla", "'T", "'S", "<|", "fim", "_prefix", "|>", "ſ", "aB", "<|", "endoftext", "|>", "m", "'s", "𐞁", "‍Z", "🙂é", "'S", "'re", "Bfi"]} +{"text": "……''re\r\n٣٤٥٦-B<|fim_prefix|>😀🏽(-­'ll(
iOS<|fim_prefix|>'s👍🏽
\r\ne\r\n\r\n​a/b 𐞁,", "tokens": 64, "pieces": ["…", "…", "''", "re", "\r\n", "٣٤٥", "٦", "-B", "<|", "fim", "_prefix", "|>😀🏽(-­'", "ll", "(", "
iOS", "<|", "fim", "_prefix", "|>'", "s", "👍🏽", "
\r\n", "e", "\r\n\r\n", "​a", "/b", " 𐞁", ","]} +{"text": "­", "tokens": 1, "pieces": ["­"]} +{"text": "ß\r\n🙂m'T9camelCase'Re\u000bAb'M ", "tokens": 14, "pieces": ["ß", "\r\n", "🙂m", "'T", "9", "camelCase", "'Re", "\u000bAb", "'M", " "]} +{"text": "åⅣ<|endoftext|>", "tokens": 12, "pieces": ["a", "̊", "Ⅳ", "<|", "endoftext", "|>"]} +{"text": " \n camelCaseß\rḍ̇.٣٤٥٦ſ३३EOT", "tokens": 27, "pieces": [" \n", " camelCaseß", "\r", "ḋ", "̣.", "٣٤٥", "٦", "ſ", "३३", "EOT"]} +{"text": "٣٤٥٦aB/\r\n½½'VEDž/\r\nm字Ａm👍🏽9dᵃ\n,\"camelCase'M漢'sß,Ａ'Tع𐞁Z's fiꟲå'D", "tokens": 69, "pieces": ["٣٤٥", "٦", "aB", "/\r\n", "½½", "'VE", "Dž", "/\r\n", "m字Ａm", "👍🏽", "9", "dᵃ", "\n", ",\"", "camelCase", "'M", "漢", "'s", "ß", ",Ａ", "'T", "ع𐞁Z", "'s", " ", " fiꟲa", "̊'", "D"]} +{"text": "/\r\nfi,0", "tokens": 9, "pieces": ["/\r\n", "fi", ",", "0"]} +{"text": "ᵃ-İe­🙂s'Tå're\r\n\r\nＡ\r\n\r\n👍🏽ḍ̇>", "tokens": 32, "pieces": ["ᵃ", "-İe", "­🙂", "s", "'T", "a", "̊'", "re", "\r\n\r\n", "Ａ", "\r\n\r\n", "👍🏽", "ḋ", "̣>"]} +{"text": " \n B ٣٤٥٦👍🏽👍🏽,'Dᵃ<|endoftext|><|fim_prefix|>iOS'll/fi'D'VEDžungla \n ꟲsDžZ👍🏽𐞁", "tokens": 72, "pieces": [" \n", " B", " ", "٣٤٥", "٦", "👍🏽👍🏽,'", "Dᵃ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "iOS", "'ll", "/fi", "'D", "'VE", "Džungla", " \n", " ꟲsDžZ", "👍🏽", "𐞁"]} +{"text": "d½\n/iOS .B-\n'́字iOS<|fim_prefix|>😀🏽camelCase‍å३½ꟲ'Sfi a/b,e'reAb/\r\ńDžcamelCase'S", "tokens": 57, "pieces": ["d", "½", "\n", "/iOS", " ", " <", "META", "_START", ">.", "B", "-\n", "'́", "字iOS", "<|", "fim", "_prefix", "|>😀🏽", "camelCase", "‍a", "̊", "३½", "ꟲ", "'S", "fi", " a", "/b", ",e", "'re", "Ab", "/\r\n", "́DžcamelCase", "'S"]} +{"text": "'M''T(\u000bé\r\n dⅣ're‍'a(𐞁㍿३a/b\n(m\r\n\r\n/\r\n\r.‍>/\r\n \n /\r\n'ReDžZ", "tokens": 47, "pieces": ["'M", "''", "T", "(", "\u000bé", "\r\n", " d", "Ⅳ", "'re", "‍'", "a", "(", "𐞁", "㍿", "३", "a", "/b", "\n", "(m", "\r\n\r\n", "/\r\n\r", ".‍>/\r\n", " \n", " /\r\n", "'Re", "DžZ"]} +{"text": "́ \r\n", "tokens": 2, "pieces": ["́", " \r\n"]} +{"text": "#$%\n\n.ع́'Rea٣٤٥٦\r\né-/\r\né ", "tokens": 23, "pieces": ["#$%\n\n", ".ع", "́'", "Rea", "٣٤٥", "٦", "\r\n", "e", "́-/\r\n", "é", " "]} +{"text": "<|fim_prefix|>", "tokens": 7, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "ABCDžunglaaB9camelCase
å>s👍🏽 'TＡDžungla­\r.'T👍🏽,", "tokens": 46, "pieces": ["ABCDžunglaaB", "9", "camelCase", "
a", "̊>", "s", "👍🏽", " '", "TＡDžungla", "­\r", ".'", "T", "👍🏽,"]} +{"text": "　\t'D#$%Ab/…", "tokens": 9, "pieces": ["　", "\t", "'D", "#$%", "Ab", "/", "…"]} +{"text": "'DHTTPServermḍ̇<Džungla \"åＡ漢𐞁㋿Džungla>ſEOT٣٤٥٦ⅣＡ'sＡsfiḍ̇sZ\ts0", "tokens": 73, "pieces": ["'D", "HTTPServermḋ", "̣<", "Džungla", " ", " \"", "a", "̊Ａ漢𐞁", "㋿Džungla", ">ſEOT", "٣٤٥", "٦Ⅳ", "Ａ", "'s", "Ａs", "fiḋ", "̣sZ", "\ts", "", "0"]} +{"text": "mİ漢 🙂㍿\n/e'ſ …
camelCaseaBḍ̇Ⅳ", "tokens": 30, "pieces": ["mİ漢", " ", "🙂㍿\n", "/e", "'ſ", " …", "
camelCaseaBḋ", "̣", "Ⅳ"]} +{"text": "𐞁(👍🏽漢Dž…　's>ꟲåfi३'T
d!!>\n/t\tß½ⅣＡås\na/b'M३!漢a/b", "tokens": 67, "pieces": ["𐞁", "(👍🏽", "漢Dž", "…", "", "　", "'s", ">ꟲa", "̊<", "META", "_START", ">fi", "३", "'T", "
d", "!!>\n", "/t", "\tß", "½Ⅳ", "Ａa", "̊s", "\n", "a", "/b", "'M", "३", "!漢a", "/b"]} +{"text": "12345678 \n ḍ̇'S'", "S", "'ḍ̇\r\n'M0字!!(🙂'reEOTAba/bḍ̇m\n/　٣٤٥٦\n/-a/b<|fim_prefix|>ß👍🏽åDžungla\r\n\r\n#$%a/b<", "tokens": 74, "pieces": ["<|", "fim", "_prefix", "|>'", "ḋ", "̣\r\n", "'M", "0", "字", "!!(🙂'", "reEOTAba", "/bḋ", "̣m", "\n", "/", "　", "٣٤٥", "٦", "\n", "/-", "a", "/b", "<|", "fim", "_prefix", "|>", "ß", "👍🏽", "a", "̊Džungla", "\r\n\r\n", "#$%", "a", "/b", "<"]} +{"text": "𐞁<|endoftext|>'½\r\n\r\nḍ̇　ABC字!!ᵃ😀🏽12345678\ré!aB\nAb'T🙂\t\"Dž\t字Džع३\nDž<|endoftext|>t🙂camelCase㋿ḍ̇", "tokens": 77, "pieces": ["𐞁", "<|", "endoftext", "|>'", "½", "\r\n\r\n", "ḋ", "̣", "　ABC字", "!!", "ᵃ", "😀🏽", "123", "456", "78", "\r", "é", "!aB", "\n", "Ab", "'T", "🙂", "\t", "\"Dž", "\t字Džع", "३", "\n", "Dž", "<|", "endoftext", "|>", "t", "🙂camelCase", "㋿ḋ", "̣"]} +{"text": "'VE'Re٣٤٥٦. \n 'THTTPServerＡ'll३ m🙂#$%𐞁fiiOS
漢\nEOT㍿éiOSa/b0a'
 \n 
", "tokens": 61, "pieces": ["'VE", "'Re", "٣٤٥", "٦", ".", " \n", " '", "THTTPServerＡ", "'ll", "३", " ", " m", "🙂#$%", "𐞁fiiOS", "", "
漢", "\n", "EOT", "㍿éiOSa", "/b", "0", "a", "'", "
 \n 
"]} +{"text": "३Z 9/<|endoftext|>", "tokens": 16, "pieces": ["", "३", "Z", " ", "9", "/<|", "endoftext", "|>"]} +{"text": "'T.½AbİABCḍ̇३iOS'ᵃ'M", "tokens": 19, "pieces": ["'T", ".", "½", "AbİABCḋ", "̣", "३", "iOS", "'ᵃ", "'M"]} +{"text": "tⅣ'ReiOSḍ̇'M<|endoftext|>,㍿Ａ#$%0\tZ.'‍½a/b9… /\r\n　HTTPServerḍ̇/\r\n're''VEHTTPServer \n漢　Ⅳ", "tokens": 64, "pieces": ["t", "Ⅳ", "'Re", "iOSḋ", "̣'", "M", "<|", "endoftext", "|>,㍿", "Ａ", "#$%", "0", "", "\tZ", ".'‍", "½", "a", "/b", "9", "… ", " /\r\n", "　HTTPServerḋ", "̣/\r\n", "'re", "''", "VEHTTPServer", " \n", "漢", "　", "Ⅳ"]} +{"text": "'D\tiOSt'\u000b\reſa 👍🏽 \n .ᵃ'TeＡḍ̇a/b!!", "tokens": 37, "pieces": ["'D", "\tiOSt", "'", "\u000b\r", "eſa", " ", " 👍🏽", " \n", " <", "META", "_START", ">.", "ᵃ", "'T", "eＡḋ", "̣a", "/b", "!!"]} +{"text": "३'s<ḍ̇ ſ0åß𐞁 🙂camelCase12345678", "tokens": 32, "pieces": ["३", "'", "s", "<ḋ", "̣", " ", " ſ", "0", "a", "̊ß𐞁", " ", "🙂camelCase", "123", "456", "78"]} +{"text": "عEOT<|endoftext|>'Re\r­½a'ReDžungla0ḍ̇aå  'S‍́ Dž🙂", "tokens": 41, "pieces": ["عEOT", "<|", "endoftext", "|>'", "Re", "\r", "­", "½", "a", "'Re", "Džungla", "0", "ḋ", "̣aa", "̊", " ", " ", "'S", "‍́", " ", " Dž", "🙂"]} +{"text": "‍㍿ᵃé\"<|endoftext|><|fim_prefix|>\ttᵃ​'Ta<|endoftext|>'½ \té字ᵃEOTABC<|fim_prefix|>'s", "tokens": 59, "pieces": ["‍㍿", "ᵃe", "́\"<|", "endoftext", "|><|", "fim", "_prefix", "|>", "\ttᵃ", "​'", "Ta", "<|", "endoftext", "|>'", "½", " ", "\té字ᵃEOTABC", "<|", "fim", "_prefix", "|>'", "s"]} +{"text": "'Bd\"ßsa/b .BAb…>é‍es\n/ \nAb9ḍ̇\r\nBd㍿٣٤٥٦EOT/\r\nABC", "tokens": 48, "pieces": ["'Bd", "\"ßsa", "/b", " ", " .", "BAb", "…", ">e", "́‍", "es", "\n", "/", " \n", "Ab", "9", "ḋ", "̣\r\n", "Bd", "㍿", "٣٤٥", "٦", "EOT", "/\r\n", "ABC"]} +{"text": "
fiåꟲ  ꟲ12345678ᵃ…-‍B'Dm>å#$%­/İ ß'ſ'llＡ.!! /", "tokens": 53, "pieces": ["", "
fia", "̊ꟲ", " ", " ꟲ", "123", "456", "78", "ᵃ", "…", "-‍", "B", "'D", "m", ">a", "̊#$%­/", "İ", " ß", "'ſ", "'ll", "Ａ", ".!!", " ", " /"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "عꟲ​/\r\n　s🙂'VE👍🏽٣٤٥٦0Džungla㋿/\r\nAb㋿'TDž/0Ａ३٣٤٥٦漢ß9ꟲ", "tokens": 64, "pieces": ["عꟲ", "​/\r\n", "　s", "🙂'", "VE", "👍🏽", "٣٤٥", "٦0", "Džungla", "㋿/\r\n", "Ab", "㋿'", "TDž", "/", "0", "Ａ", "३٣٤", "٥٦", "漢ß", "9", "ꟲ"]} +{"text": "'ſ/!!漢/\r\nm👍🏽sfi9ſ\r\n👍🏽's\t!!३camelCaseſfi\rꟲ😀🏽>ś㍿éAb \n ,\t㋿", "tokens": 63, "pieces": ["'ſ", "/!!", "漢", "/\r\n", "m", "👍🏽", "sfi", "9", "ſ", "\r\n", "👍🏽'", "s", "\t", "!!", "३", "camelCaseſfi", "\r", "ꟲ", "😀🏽>", "s", "́㍿", "éAb", " \n", " ,", "\t", "㋿"]} +{"text": "camelCase'llⅣe　>12345678\r\n \n ㍿㍿'VEcamelCase​ABC'res𐞁漢ḍ̇ ", "tokens": 40, "pieces": ["camelCase", "'ll", "Ⅳ", "e", "　", ">", "123", "456", "78", "\r\n \n", " ㍿㍿'", "VEcamelCase", "​ABC", "'re", "s𐞁漢ḋ", "̣", " "]} +{"text": "eDžungla‍<|fim_prefix|><", "ta", "̊e", "́Džß", "३", "́t", "-a", "\t", "…ḋ", "̣𐞁", "9", "/\r\n", "ḋ", "̣👍🏽", "\u000b", "…İ", "
"]} +{"text": "'ll　dm/\r\nDž́𐞁<|fim_prefix|>\n/😀🏽́Dž字\n字\r(Z", "tokens": 32, "pieces": ["'ll", "　dm", "/\r\n", "Dž", "́𐞁", "<|", "fim", "_prefix", "|>\n", "/😀🏽́", "Dž字", "\n", "字", "\r", "(Z"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "٣٤٥٦'T​🙂s​!
'DžHTTPServerZḍ̇\u000b'ſ!'M　\u000ba/b​👍🏽-​éDžunglaB \n,/ᵃ३", "tokens": 60, "pieces": ["٣٤٥", "٦", "'T", "​🙂", "s", "​!", "
", "'DžHTTPServerZḋ", "̣", "\u000b", "'ſ", "!'", "M", "　", "\u000ba", "/b", "​👍🏽-​", "éDžunglaB", " \n", ",/", "ᵃ", "३"]} +{"text": "🙂é\r(
'T>12345678­ 'll Ａ́\reaBḍ̇ \n 'VEdé(Z<‍Ⅳꟲ/😀🏽😀🏽'VE\u000bDž\r\tå𐞁", "tokens": 68, "pieces": ["🙂é", "\r", "(", "
", "'T", ">", "123", "456", "78", "­", " '", "ll", " ", " Ａ", "́\r", "eaBḋ", "̣", " \n", " '", "VEdé", "(Z", "<‍", "Ⅳ", "ꟲ", "/😀🏽😀🏽'", "VE", "\u000bDž", "\r", "\ta", "̊𐞁"]} +{"text": "\u000b\n/'Mſ'ſ'd", "tokens": 10, "pieces": ["\u000b\n", "/'", "Mſ", "'ſ", "'d"]} +{"text": "<|fim_prefix|>Dž><|fim_prefix|> \n/\r\naiOŚİ\r\n\r'M/\rå'ſ'Tع\n ᵃعABC'Re", "tokens": 42, "pieces": ["<|", "fim", "_prefix", "|>", "Dž", "><|", "fim", "_prefix", "|>", " \n", "/\r\n", "aiOS", "́İ", "\r\n\r", "'M", "/\r", "a", "̊'", "ſ", "'T", "ع", "\n", " ᵃعABC", "'Re"]} +{"text": "HTTPServerB३EOTⅣtm12345678'll!!", "tokens": 18, "pieces": ["HTTPServerB", "", "३", "EOT", "Ⅳ", "tm", "123", "456", "78", "'ll", "!!"]} +{"text": "EOTcamelCase
३å🙂m-\u000b🙂9Ab٣٤٥٦\u000bDžungla㍿!!iOS\n/<|endoftext|>'TaB٣٤٥٦/\r\n\r\n'sa/b…fi<|fim_prefix|>\n‍EOT", "tokens": 75, "pieces": ["EOTcamelCase", "
", "३", "a", "̊🙂", "m", "-", "\u000b", "🙂", "9", "Ab", "٣٤٥", "٦", "\u000bDžungla", "㍿!!", "iOS", "\n", "/<|", "endoftext", "|>'", "TaB", "٣٤٥", "٦", "/\r\n\r\n", "'s", "a", "/b", "…fi", "<|", "fim", "_prefix", "|>\n", "‍EOT"]} +{"text": "DžunglaABC𐞁\r /\r\n𐞁 \n t👍🏽Ab'sdméé9'M!!é…ꟲ\t-/\r\n'éé", "tokens": 44, "pieces": ["DžunglaABC𐞁", "\r", " /\r\n", "𐞁", " \n", " t", "👍🏽", "Ab", "'s", "dméé", "9", "'M", "!!", "é", "…ꟲ", "\t", "-/\r\n", "'ée", "́"]} +{"text": "ß,ᵃs\r\n
B<9'll \n 🙂é9Z
>iOS<|endoftext|>\r漢ß<|endoftext|>👍🏽٣٤٥٦㋿", "tokens": 63, "pieces": ["ß", ",ᵃs", "\r\n", "
B", "<", "9", "'ll", " \n", " <", "EOT", ">🙂", "e", "́", "9", "Z", "
", ">iOS", "<|", "endoftext", "|>\r", "漢ß", "<|", "endoftext", "|>👍🏽", "٣٤٥", "٦", "㋿"]} +{"text": "camelCase'Re,t'rem'ſꟲ🙂\r\nſ　👍🏽fi\r/iOSZ(", "tokens": 38, "pieces": ["camelCase", "'Re", ",t", "'re", "m", "'ſ", "ꟲ", "🙂<", "META", "_START", ">\r\n", "ſ", "　", "👍🏽", "fi", "\r", "/iOSZ", "("]} +{"text": "'re'VE'/\r\n#$%Ⅳ\r\n'DZfiعEOT'Re
t　字😀🏽<|endoftext|>", "tokens": 35, "pieces": ["'re", "'VE", "'/\r\n", "#$%", "Ⅳ", "\r\n", "'D", "ZfiعEOT", "'Re", "
t", "　字", "😀🏽<|", "endoftext", "|>"]} +{"text": "éHTTPServerᵃ👍🏽\r\n\r\n<Ⅳ're\"0\u000b9👍🏽​ع! ­'T\r\n\r\n ३'VE<|fim_prefix|>​\n/‍\n0\u000b٣٤٥٦Džungla", "tokens": 71, "pieces": ["e", "́HTTPServerᵃ", "👍🏽\r\n\r\n", "<", "Ⅳ", "'re", "\"", "0", "\u000b", "9", "👍🏽​", "ع", "!", " ", "­'", "T", "\r\n\r\n", " ", "३", "'VE", "<|", "fim", "_prefix", "|>​\n", "/‍\n", "0", "\u000b", "٣٤٥", "٦", "Džungla"]} +{"text": "'　..!0eſB \n ㋿'ſ's'T'漢\r\nZ\n/Abaé dAbEOT<|endoftext|>́👍🏽", "tokens": 48, "pieces": ["'", "　", "..!", "0", "eſB", " \n", " ㋿'", "ſ", "'s", "'T", "'漢", "\r\n", "Z", "\n", "/Abae", "́", " ", " dAbEOT", "<|", "endoftext", "|>́👍🏽"]} +{"text": "'M/\r\nZHTTPServer　<ᵃ \n e\n/fi.d9fi㋿'M字camelCasemd/👍🏽Ⅳ㍿字é😀🏽\r\r\t'T'll", "tokens": 54, "pieces": ["'M", "/\r\n", "ZHTTPServer", "　", "<ᵃ", " \n", " e", "\n", "/fi", ".d", "9", "fi", "㋿'", "M字camelCasemd", "/👍🏽", "Ⅳ", "㍿字e", "́😀🏽\r\r", "\t", "'T", "'ll"]} +{"text": "🙂ᵃ9ḍ̇‍AbⅣ३߅ſ\"", "tokens": 48, "pieces": ["🙂", "ᵃ", "9", "ḋ", "̣‍", "Ab", "Ⅳ३", "ß", "…ſ", "\""]} +{"text": "­𐞁EOTDžHTTPServerEOTعa\r\n\r\n", "tokens": 16, "pieces": ["­𐞁EOTDžHTTPServerEOTعa", "\r\n\r\n"]} +{"text": "éع!", "tokens": 3, "pieces": ["éع", "!"]} +{"text": "åaDžunglaacamelCase \n'D㋿/\r\n'sfí\n(\r!!EOTfi#$%iOS's'👍🏽(eEOT漢camelCase'S<\"㋿\r\n\u000b'T ", "tokens": 61, "pieces": ["a", "̊aDžungla", "acamelCase", " \n", "'D", "㋿/\r\n", "'s", "fi", "́\n", "(\r", "!!", "EOTfi", "#$%", "iOS", "'s", "'👍🏽(", "eEOT漢camelCase", "'S", "<\"㋿\r\n", "\u000b", "'T", " "]} +{"text": "\n\r\na/bAb\r
字­ſEOT漢0Ａ", "tokens": 19, "pieces": ["\n\r\n", "a", "/bAb", "\r", "
字", "­ſEOT漢", "0", "Ａ"]} +{"text": "ꟲs​ſ\t ́'ReaBaDž/'ReéⅣAbficamelCase𐞁aå\r\n\r\n<|endoftext|>İ\t \n m\u000b", "tokens": 46, "pieces": ["ꟲs", "​ſ", "\t", " ́'", "ReaBaDž", "/'", "Ree", "́", "Ⅳ", "AbficamelCase𐞁aa", "̊\r\n\r\n", "<|", "endoftext", "|>", "İ", "\t \n", " m", "\u000b"]} +{"text": "㍿(iOS\n/ᵃ㋿ #$%aİétaB 👍🏽Ａ!!('ll#$%!!HTTPServerᵃ́　HTTPServer‍'re", "tokens": 48, "pieces": ["㍿(", "iOS", "\n", "/ᵃ", "㋿", " ", "#$%", "aİétaB", " ", " 👍🏽", "Ａ", "!!('", "ll", "#$%!!", "HTTPServerᵃ", "́", "　HTTPServer", "‍'", "re"]} +{"text": "३Ⅳ!!camelCase'reaAbå​\r👍🏽 😀🏽́ḍ̇ⅣⅣ", "tokens": 37, "pieces": ["३Ⅳ", "!!", "camelCase", "'re", "aAba", "̊​\r", "👍🏽", " ", "😀🏽́", "ḋ", "̣", "ⅣⅣ"]} +{"text": "İ9mＡ㍿　\u000b\r\n\r\n<|endoftext|>Z'M'Reİ \n mZ🙂m12345678Ⅳ😀🏽٣٤٥٦\"12345678EOT(EOTB0\n𐞁😀🏽t'llå", "tokens": 75, "pieces": ["İ", "9", "mＡ", "㍿", "　\u000b\r\n\r\n", "<|", "endoftext", "|>", "Z", "'M", "'Re", "İ", " \n", " mZ", "🙂m", "123", "456", "78Ⅳ", "😀🏽<", "EOT", ">", "٣٤٥", "٦", "\"", "123", "456", "78", "EOT", "(EOTB", "0", "\n", "𐞁", "😀🏽", "t", "'ll", "a", "̊"]} +{"text": "'re'res!'s'll'D'Reé漢''ſ 'Re \n aB\r", "tokens": 20, "pieces": ["'re", "'re", "s", "!'", "s", "'ll", "'D", "'Re", "é漢", "''", "ſ", " '", "Re", " \n", " aB", "\r"]} +{"text": "> \ń­\r\n\r\n12345678/…HTTPServerᵃm\n 'T…,ſ\n/ꟲ/\r\n \nå㋿.DžunglaHTTPServerᵃ\t", "tokens": 57, "pieces": [">", " \n", "́­\r\n\r\n", "123", "456", "78", "/", "…HTTPServerᵃm", "\n", " ", " '", "T", "…", ",ſ", "\n", "/ꟲ", "/\r\n", " \n", "a", "̊㋿.", "DžunglaHTTPServerᵃ", "\t", ""]} +{"text": ">å,camelCasesＡ字Z<|fim_prefix|>s🙂fi're'iOS\n/𐞁­m'VE'D'ſ🙂½", "tokens": 42, "pieces": [">a", "̊,", "camelCasesＡ字Z", "<|", "fim", "_prefix", "|>", "s", "🙂fi", "'re", "'iOS", "\n", "/𐞁", "­m", "'VE", "'D", "'ſ", "🙂", "½"]} +{"text": "\r\n\r\n\"!!　fi'S🙂0İ0\u000b👍🏽('S\"EOT", "tokens": 23, "pieces": ["\r\n\r\n", "\"!!", "　fi", "'S", "🙂", "0", "İ", "0", "\u000b", "👍🏽('", "S", "\"EOT"]} +{"text": "ꟲ🙂/\r\nå912345678㍿३ABCſع\n'Tİ'T/\r\n/", "tokens": 30, "pieces": ["ꟲ", "🙂/\r\n", "a", "̊", "912", "345", "678", "㍿", "३", "ABCſع", "\n", "'T", "İ", "'T", "/\r\n", "/<", "EOT", ">"]} +{"text": "(tB(<|fim_prefix|>'D\"'re㍿\rDž\r\n'reB.'ſ'M", "tokens": 25, "pieces": ["(tB", "(<|", "fim", "_prefix", "|>'", "D", "\"'", "re", "㍿\r", "Dž", "\r\n", "'re", "B", ".'", "ſ", "'M"]} +{"text": "'漢iOS\n/ \néd漢,é\r\"m\rſDžunglaaBéß(İ漢!!ſع\r, 'SDžungla camelCase", "tokens": 46, "pieces": ["'漢iOS", "\n", "/", " \n", "éd漢", ",é", "\r", "\"m", "\r", "ſDžunglaaBéß", "(İ漢", "!!", "ſع", "\r", ",", " ", " '", "SDžungla", " camelCase"]} +{"text": "t(12345678dEOT👍🏽", "tokens": 14, "pieces": ["t", "(", "123", "456", "78", "dEOT", "👍🏽"]} +{"text": "ꟲ字🙂s", "tokens": 7, "pieces": ["ꟲ字", "🙂s"]} +{"text": "Abå\r\n\r\nDžungla<|endoftext|>Ａ're'DⅣéaB!aBZiOS,\rDžungla#$%Ａ0e\n ́Ⅳ\r\n\n/🙂.㋿iOS", "tokens": 57, "pieces": ["Aba", "̊\r\n\r\n", "Džungla", "<|", "endoftext", "|>", "Ａ", "'re", "'D", "Ⅳ", "e", "́aB", "!aBZiOS", ",\r", "Džungla", "#$%", "Ａ", "0", "e", "\n", " ", "́", "Ⅳ", "\r\n\n", "/🙂.㋿", "iOS"]} +{"text": "'T½漢>éḍ̇'re#$%a/bdB'S", "tokens": 22, "pieces": ["'T", "½", "漢", ">e", "́ḋ", "̣'", "re", "#$%<", "EOT", ">a", "/bdB", "'S"]} +{"text": "0'\r\n\r\n", "tokens": 2, "pieces": ["0", "'\r\n\r\n"]} +{"text": "'T٣٤٥٦12345678/ ,㍿\"'TBⅣ㍿", "tokens": 25, "pieces": ["'T", "٣٤٥", "٦12", "345", "678", "/", " ", " ,㍿\"'", "TB", "Ⅳ", "㍿"]} +{"text": "m!<|endoftext|>.é🙂> 'ſ \n !!å", "tokens": 21, "pieces": ["m", "!<|", "endoftext", "|>.", "e", "́🙂>", " '", "ſ", " \n", " !!", "a", "̊"]} +{"text": "t-\r\n\r\nDž‍", "tokens": 7, "pieces": ["t", "-\r\n\r\n", "Dž", "‍"]} +{"text": "<|endoftext|>\r\n\r\nſ　'D(​Džungla‍EOTİ 9B. \n camelCase!!t…>…'ll'Mḍ̇­EOT.½\t\u000b", "tokens": 55, "pieces": ["<|", "endoftext", "|>\r\n\r\n", "ſ", "　", "'D", "(​", "Džungla", "‍EOT", "İ", " ", " ", "9", "B", ".", " \n", " camelCase", "!!", "t", "…", ">", "…", "'ll", "'M", "ḋ", "̣­", "EOT", ".", "½", "\t\u000b"]} +{"text": "iOS٣٤٥٦\tt/\r\nᵃ<|fim_prefix|>.Ⅳ👍🏽🙂12345678\u000bmDžB/\r\n", "tokens": 43, "pieces": ["iOS", "٣٤٥", "٦", "\tt", "/\r\n", "ᵃ", "<|", "fim", "_prefix", "|>.", "Ⅳ", "👍🏽🙂", "123", "456", "78", "\u000bmDžB", "/\r\n"]} +{"text": ".\r\nＡ<́9d's　'll😀🏽é\r\n\r\n aB\rZcamelCase'sm>0<|endoftext|>s'reABC'Reḍ̇‍\r\n\r\n字Džungla\t\u000ba/bAb", "tokens": 56, "pieces": [".\r\n", "Ａ", "<́", "9", "d", "'s", "　", "'ll", "😀🏽", "é", "\r\n\r\n", " aB", "\r", "ZcamelCase", "'s", "m", ">", "0", "<|", "endoftext", "|>", "s", "'re", "ABC", "'Re", "ḋ", "̣‍\r\n\r\n", "字Džungla", "\t", "\u000ba", "/bAb"]} +{"text": " !漢fiEOT㍿a/bm'ree\r\n\r\n!!<|endoftext|>\nع…", "tokens": 27, "pieces": [" !", "漢fiEOT", "㍿a", "/bm", "'re", "e", "\r\n\r\n", "!!<|", "endoftext", "|>\n", "ع", "…"]} +{"text": "s​\n/å,漢\u000b", "tokens": 10, "pieces": ["s", "​\n", "/a", "̊,", "漢", "\u000b"]} +{"text": "字٣٤٥٦'ſßſ/\r\na/bꟲ!!'T<|endoftext|>🙂å\r\n\r\naå", "tokens": 40, "pieces": ["字", "٣٤٥", "٦", "'ſ", "ßſ", "/\r\n", "a", "/bꟲ", "!!'", "T", "<|", "endoftext", "|>🙂", "a", "̊\r\n\r\n", "aa", "̊"]} +{"text": "ZaB >٣٤٥٦<|endoftext|>'llB\rZ(<|endoftext|>HTTPServerEOT>0😀🏽'D'T(HTTPServer,ß \n　a/bⅣ\u000baB B\r\n/'llße", "tokens": 63, "pieces": ["ZaB", " ", " >", "٣٤٥", "٦", "<|", "endoftext", "|>'", "llB", "\r", "Z", "(<|", "endoftext", "|>", "HTTPServerEOT", ">", "0", "😀🏽'", "D", "'T", "(HTTPServer", ",ß", " \n", "　a", "/b", "Ⅳ", "\u000baB", " B", "\r\n", "/'", "llße"]} +{"text": "s'", "tokens": 2, "pieces": ["s", "'"]} +{"text": "<|endoftext|>e\n­'llé!12345678ſ'S're-𐞁٣٤٥٦\r\n\n ('sᵃ", "tokens": 40, "pieces": ["<|", "endoftext", "|>", "e", "\n", "­'", "lle", "́!", "123", "456", "78", "ſ", "'S", "'re", "-𐞁", "٣٤٥", "٦", "\r\n\n", " ('", "sᵃ"]} +{"text": "DžunglaDžungla(- a/b👍🏽\n .👍🏽0ع\n!!'ReéiOS ­ \naB　", "tokens": 42, "pieces": ["DžunglaDžungla", "(-", " ", " a", "/b", "👍🏽\n", " ", ".👍🏽", "0", "ع", "\n", "!!'", "Ree", "́iOS", " ", " ­", " \n", "aB", "　"]} +{"text": "å#$%a ḍ̇å'VE'S‍<漢‍<😀🏽\r\n'S\t. \tiOSAb \ns's٣٤٥٦'ll/\r\nع.ḍ̇'Re 9a/bAbABC'S", "tokens": 68, "pieces": ["a", "̊#$%", "a", " ḋ", "̣a", "̊'", "VE", "'S", "‍<", "漢", "‍<😀🏽\r\n", "'S", "\t", ".", " ", "\tiOSAb", " \n", "s", "'s", "٣٤٥", "٦", "'ll", "/\r\n", "ع", ".ḋ", "̣'", "Re", " ", "9", "a", "/bAbABC", "'S"]} +{"text": "t!'Refis\t'ſ#$%\r\na Dž㍿0\r\n\r\nİ12345678ABCß字㋿HTTPServer𐞁Dž!!(", "tokens": 42, "pieces": ["t", "!'", "Refis", "\t", "'ſ", "#$%\r\n", "a", " ", " Dž", "㍿", "0", "\r\n\r\n", "İ", "123", "456", "78", "ABCß字", "㋿HTTPServer𐞁Dž", "!!("]} +{"text": "٣٤٥٦३", "tokens": 10, "pieces": ["٣٤٥", "٦३"]} +{"text": "ß Ⅳ\n/'VE\r\nZ-#$%㋿🙂 \n ㍿iOS \n 'T㋿ع'ReAb!́é", "tokens": 38, "pieces": ["ß", " ", " ", "Ⅳ", "\n", "/'", "VE", "\r\n", "Z", "-#$%㋿🙂", " \n", " ", " ㍿", "iOS", " \n", " '", "T", "㋿ع", "'Re", "Ab", "!́", "e", "́"]} +{"text": "'ſ-½\n \niOSé　　>0/\r\n /'sa'Re'\t\"!! B­عDž \n9\té \n", "tokens": 34, "pieces": ["'ſ", "-", "½", "\n \n", "iOSe", "́", "　", "　", ">", "0", "/\r\n", " ", " /'", "sa", "'Re", "'", "\t", "\"!!", " B", "­عDž", " \n", "9", "\té", " \n"]} +{"text": "0'TåAb", "tokens": 6, "pieces": ["0", "'T", "a", "̊Ab"]} +{"text": "漢é\r­Ab'ReᵃiOS!!ᵃſ<|fim_prefix|>𐞁 éß
\r", "­Ab", "'Re", "ᵃiOS", "!!", "ᵃſ", "<|", "fim", "_prefix", "|>", "𐞁", " ", " e", "́ß", "
", ".\r0'VE㍿'T­<|endoftext|>/!!", "tokens": 45, "pieces": [" '", "S", "!!", " ", " '", "M", ",𐞁ß", "
", "'VE", "ＡiOS", "‍", "0", "𐞁", ">.\r", "0", "'VE", "㍿'", "T", "­<|", "endoftext", "|>/!!"]} +{"text": "HTTPServer!!a/bDž😀🏽DžⅣ<\tét\n/ \n <|fim_prefix|>AbaBé٣٤٥٦iOS'M𐞁३9! \n /
s", "tokens": 55, "pieces": ["HTTPServer", "!!", "a", "/bDž", "😀🏽", "Dž", "Ⅳ", "<", "\tét", "\n", "/", " \n", " <|", "fim", "_prefix", "|>", "AbaBé", "٣٤٥", "٦", "iOS", "'M", "𐞁", "३9", "!", " \n", " /", "
s"]} +{"text": "HTTPServerＡ>'reḍ̇　\"'S\r\n\r\n漢aBＡ< \n B/​­iOS­漢-a/baB​㍿㋿'ſ'S'VEß", "tokens": 52, "pieces": ["HTTPServerＡ", ">'", "reḋ", "̣", "　", "\"'", "S", "\r\n\r\n", "漢aBＡ", "<", " \n", " <", "META", "_START", ">B", "/​­", "iOS", "­漢", "-a", "/baB", "​㍿㋿'", "ſ", "'S", "'VE", "ß"]} +{"text": "‍9ꟲ \ńᵃfiİ㍿ \nfi'VEå​Ａᵃ0漢camelCase'reſAbEOT", "tokens": 42, "pieces": ["‍", "9", "ꟲ", " \n", "́ᵃfiİ", "㍿", " \n", "fi", "'VE", "a", "̊​", "Ａᵃ", "0", "漢camelCase", "'re", "ſAbEOT"]} +{"text": "a/b'Tꟲ/\r\nABC0字", "tokens": 14, "pieces": ["a", "/b", "'T", "ꟲ", "/\r\n", "ABC", "", "0", "字"]} +{"text": "'VE'VEᵃ𐞁'ſcamelCase‍<|endoftext|>så½İ", "tokens": 33, "pieces": ["'VE", "'VE", "ᵃ𐞁", "'ſ", "camelCase", "‍<|", "endoftext", "|>", "s", "a", "̊", "½", "İ"]} +{"text": "Dž'ſ!!ABC'lladé\r\n\r\n㍿Ab're", "tokens": 16, "pieces": ["Dž", "'ſ", "!!", "ABC", "'ll", "ade", "́\r\n\r\n", "㍿Ab", "'re"]} +{"text": "ḍ̇", "tokens": 5, "pieces": ["ḋ", "̣"]} +{"text": "HTTPServers\nꟲ🙂>a㍿", "tokens": 16, "pieces": ["HTTPServers", "\n", "ꟲ", "🙂>", "a", "㍿"]} +{"text": "😀🏽Ab/\r\n!ß㍿ 'ſ \n/.", " '", "ſ", " \n", "/.<", "m", " HTTPServer", "😀🏽", " "]} +{"text": "'ſ'llaBᵃ\n/", "tokens": 11, "pieces": ["'ſ", "'ll", "aBᵃ", "\n", "/"]} +{"text": "½ \n ​-0字å \n.'ſ٣٤٥٦Dž9 \n'ſ'Rea😀🏽 \t…t漢#$%Ab🙂ḍ̇", "tokens": 52, "pieces": ["½", " \n", " ​-", "0", "字a", "̊", " \n", ".'", "ſ", "٣٤٥", "٦", "Dž", "9", " \n", "'ſ", "'Re", "a", "😀🏽", " \t", "…t漢", "#$%", "Ab", "🙂ḋ", "̣"]} +{"text": "EOTⅣ9", "tokens": 8, "pieces": ["EOT", "Ⅳ9"]} +{"text": "EOT,\r<|endoftext|>😀🏽३!Bİ‍ᵃiOSعAb　ᵃ'ſ/<|endoftext|>　ḍ̇İ'ſᵃ'M/ꟲ😀🏽ع\r'red!<|endoftext|>㍿aB", "tokens": 85, "pieces": ["EOT", ",\r", "<|", "endoftext", "|>😀🏽", "३", "!Bİ", "‍ᵃiOSعAb", "　ᵃ", "'ſ", "/<|", "endoftext", "|>", "　ḋ", "̣İ", "'ſ", "ᵃ", "'M", "/ꟲ", "😀🏽", "ع", "\r", "'re", "d", "!<|", "endoftext", "|><", "EOT", ">㍿", "aB"]} +{"text": " éſd,<|endoftext|>å12345678Džt!!\nعa😀🏽EOT­\"'s½…'Ⅳ­'D \n ,'ſ\n-fi0", "tokens": 55, "pieces": [" éſd", ",<|", "endoftext", "|>", "a", "̊", "123", "456", "78", "Džt", "!!\n", "عa", "😀🏽", "EOT", "­\"'", "s", "½", "…", "'", "Ⅳ", "­'", "D", " \n", " ,'", "ſ", "\n", "-fi", "0"]} +{"text": "<|fim_prefix|>", "tokens": 7, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "/mEOT​ ABC", "tokens": 9, "pieces": ["/m", "EOT", "​", " ", " ABC"]} +{"text": "\r\n\r\nfi ½a/b㋿\" \u000bZt's漢३é", "tokens": 23, "pieces": ["\r\n\r\n", "fi", " ", " ", "½", "a", "/b", "㋿\"", " ", "\u000bZt", "'s", "漢", "३", "e", "́"]} +{"text": "t…٣٤٥٦Z'Reİ", "tokens": 14, "pieces": ["t", "…", "٣٤٥", "٦", "Z", "'Re", "İ"]} +{"text": " 'Re㍿😀🏽\" \n㋿iOS(12345678İＡ½,‍🙂‍'S's.　…a/b-'sİ­<|fim_prefix|>İＡ", "tokens": 55, "pieces": [" ", " '", "Re", "㍿😀🏽\"", " \n", "㋿iOS", "(", "123", "456", "78", "İＡ", "½", ",‍🙂‍'", "S", "'s", ".", "　", "…a", "/b", "-'", "sİ", "­<|", "fim", "_prefix", "|>", "İＡ"]} +{"text": "a\r\n\r\nＡ​<|endoftext|>'s's'T३'M.sＡABCDž👍🏽Džungla\r\n𐞁Džunglaع\u000bEOT​'ll\r
HTTPServer🙂½", "tokens": 58, "pieces": ["a", "\r\n\r\n", "Ａ", "​<|", "endoftext", "|>'", "s", "'s", "'T", "३", "'M", ".sＡABCDž", "👍🏽", "Džungla", "\r\n", "𐞁Džunglaع", "\u000bEOT", "​'", "ll", "\r", "
HTTPServer", "🙂", "½"]} +{"text": "Ⅳ٣٤٥٦'VE0#$%🙂9-#$%é!EOT𐞁ḍ̇e　fiAb", "tokens": 39, "pieces": ["Ⅳ٣٤", "٥٦", "'VE", "0", "#$%🙂", "9", "-#$%", "é", "!EOT𐞁ḋ", "̣e", "　fiAb"]} +{"text": "👍🏽字/\r\nBDž \n#$%\"eßß½\"", "tokens": 19, "pieces": ["👍🏽", "字", "/\r\n", "BDž", " \n", "#$%\"", "eßß", "½", "\""]} +{"text": "'T-'Re𐞁", "tokens": 7, "pieces": ["'T", "-'", "Re𐞁"]} +{"text": "!!\r\n'Ret \n\r\ns#$%Ⅳ", "tokens": 14, "pieces": ["!!\r\n", "'Re", "t", " \n\r\n", "s", "#$%", "Ⅳ", ""]} +{"text": "!!ſ𐞁👍🏽ABC\nḍ̇a/b<", "tokens": 23, "pieces": ["!!", "ſ𐞁", "👍🏽", "ABC", "\n", "ḋ", "̣a", "/b", "<"]} +{"text": "å　DžunglaEOT!!ḍ̇-Ab>.'sdaB'reḍ̇\"d\r\n\r\n #$%Ａ\n/", "tokens": 38, "pieces": ["a", "̊", "　DžunglaEOT", "!!", "ḋ", "̣-", "Ab", ">.'", "sdaB", "'re", "ḋ", "̣\"", "d", "\r\n\r\n", " #$%", "Ａ", "\n", "/"]} +{"text": "'M'ſABC'​👍🏽 ", "tokens": 14, "pieces": ["'M", "'ſ", "ABC", "'​👍🏽", " "]} +{"text": "\n'D'ſ/ \n'M­dḍ̇>㋿<|fim_prefix|>ᵃ漢\" \n\r\n0​ſſ​-", "tokens": 42, "pieces": ["\n", "'D", "'ſ", "/", " \n", "'M", "­dḋ", "̣>㋿<|", "fim", "_prefix", "|>", "ᵃ漢", "\"", " \n\r\n", "0", "​ſſ", "​-"]} +{"text": "Z'ſ'عᵃfi", "tokens": 11, "pieces": ["Z", "'ſ", "'عᵃfi"]} +{"text": "ſ!'ſ/\r\n \n𐞁eaB👍🏽́ da", "tokens": 22, "pieces": ["ſ", "!'", "ſ", "/\r\n", " \n", "𐞁eaB", "👍🏽́", " da"]} +{"text": "Ａ漢 12345678ḍ̇HTTPServer㍿\n/'Re\r\n\r\n‍B9ßaB'S!!👍🏽'M!!'llＡꟲ­👍🏽Dž9m½ع.字m\u000bＡ", "tokens": 66, "pieces": ["Ａ漢", " ", "123", "456", "78", "ḋ", "̣HTTPServer", "㍿\n", "/'", "Re", "\r\n\r\n", "‍B", "9", "ßaB", "'S", "!!👍🏽'", "M", "!!'", "llＡꟲ", "­👍🏽", "Dž", "9", "m", "½", "ع", ".字m", "\u000bＡ"]} +{"text": "camelCasee \n ٣٤٥٦\u000b　ABC'T", "tokens": 17, "pieces": ["camelCasee", " \n", " ", "٣٤٥", "٦", "\u000b", "　ABC", "'T"]} +{"text": "'s'Må\u000b ́Ab'T(‍", "tokens": 13, "pieces": ["'s", "'M", "a", "̊", "\u000b", " ́", "Ab", "'T", "(‍"]} +{"text": " \n\n/#$%ḍ̇'Re'Ta/b㋿>ḍ̇ \u000bع'll'ſ
 \t!३éＡ \n camelCasea/bå'ſB0B…< \n ,,a", "tokens": 67, "pieces": [" \n\n", "/#$%", "ḋ", "̣'", "Re", "'", "Ta", "/b", "㋿>", "ḋ", "̣", " ", "\u000bع", "'ll", "'ſ", "
 ", "\t", "!", "३", "éＡ", " \n", " camelCasea", "/ba", "̊'", "ſB", "0", "B", "…", "<", " \n", " ,,", "a"]} +{"text": " \n …<|fim_prefix|>\"𐞁!㍿'ss", "tokens": 21, "pieces": [" \n", " ", "…", "<|", "fim", "_prefix", "|>\"", "𐞁", "!㍿'", "ss"]} +{"text": "ᵃ😀🏽 é,…fi", "tokens": 16, "pieces": ["ᵃ", "😀🏽", " e", "́,", "…fi"]} +{"text": "é're-\r\n", "tokens": 3, "pieces": ["é", "'re", "-\r\n"]} +{"text": "12345678e½'M\rABC's㍿<\"𐞁éſ㋿👍🏽Ab　iOS'T\rⅣ('ll'llABCABCBiOS­>Ⅳ", "tokens": 47, "pieces": ["123", "456", "78", "e", "½", "'M", "\r", "ABC", "'s", "㍿<\"", "𐞁éſ", "㋿👍🏽", "Ab", "　iOS", "'T", "\r", "Ⅳ", "('", "ll", "'ll", "ABCABCBiOS", "­>", "Ⅳ"]} +{"text": "\r<٣٤٥٦\"İ́", "tokens": 13, "pieces": ["\r", "<", "٣٤٥", "٦", "\"İ", "́"]} +{"text": "<|fim_prefix|>.aBＡ𐞁é‍mAbABCåé'reꟲaaBåDž字‍9!é'llＡ!\rDžᵃ'Ret", "tokens": 61, "pieces": ["<|", "fim", "_prefix", "|>.", "aBＡ𐞁e", "́‍", "mAbABCa", "̊<", "META", "_START", ">e", "́'", "reꟲaaBa", "̊Dž字", "‍", "9", "!e", "́'", "llＡ", "!\r", "Džᵃ", "'Re", "t"]} +{"text": "fi0\n/🙂camelCase<ᵃ𐞁", "tokens": 23, "pieces": ["fi", "0", "\n", "/<", "META", "_START", ">🙂", "camelCase", "<ᵃ𐞁", ""]} +{"text": "​ß\r\nå漢å\n/>< \n 'T३EOT'٣٤٥٦ABC漢t9s !!३", "tokens": 42, "pieces": ["​ß", "\r\n", "a", "̊漢a", "̊<", "META", "_START", ">\n", "/><", " \n", " '", "T", "३", "EOT", "'", "٣٤٥", "٦", "ABC漢t", "9", "s", " ", "!!", "३"]} +{"text": "(d㍿　e.'s‍fiåİ\nB/\r\n \n­　(ß", "tokens": 27, "pieces": ["(d", "㍿", "　e", ".'", "s", "‍fi", "a", "̊İ", "\n", "B", "/\r\n", " \n", "­", "　", "(ß"]} +{"text": "'VE\r\n!!🙂\r\nⅣB​Ab12345678#$%'TdDžſABC", "tokens": 23, "pieces": ["'VE", "\r\n", "!!🙂\r\n", "Ⅳ", "B", "​Ab", "123", "456", "78", "#$%'", "TdDžſABC"]} +{"text": "𐞁're'sDžungladᵃ9\r­t-…/𐞁ḍ̇<\n(𐞁aB字, \nå\u000b<|fim_prefix|><|endoftext|>'​‍ꟲ", "tokens": 66, "pieces": ["𐞁", "'re", "'s", "Džungladᵃ", "9", "\r", "­t", "-", "…", "/𐞁ḋ", "̣<\n", "(<", "META", "_START", ">𐞁aB字", ",", " \n", "a", "̊", "\u000b", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'​‍", "ꟲ"]} +{"text": "İ'VE", "tokens": 6, "pieces": ["İ", "'", "VE"]} +{"text": "Ab …ꟲ٣٤٥٦­HTTPServer…<|fim_prefix|>ſ", "tokens": 29, "pieces": ["Ab", " ", "…ꟲ", "٣٤٥", "٦", "­HTTPServer", "…", "<|", "fim", "_prefix", "|>", "ſ"]} +{"text": ">å\u000bＡABC,HTTPServerǻ-(½EOTZ'D🙂Džungla字ᵃİ👍🏽\t\n३🙂👍🏽m 'll'S\u000bſ/\r\ncamelCase
", "tokens": 64, "pieces": [">a", "̊", "\u000bＡABC", ",HTTPServera", "̊́-(", "½", "EOTZ", "'D", "🙂Džungla字ᵃİ", "👍🏽", "\t\n", "३", "🙂👍🏽", "m", " ", "'ll", "'S", "\u000bſ", "/\r\n", "camelCase", "
"]} +{"text": " 'T\"'T<|endoftext|>9Dž/\r\nABC😀🏽éDž字>!!!HTTPServer ㍿\r\n\r\nHTTPServer٣٤٥٦'Re12345678Džungla½", "tokens": 59, "pieces": [" '", "T", "\"'", "T", "<|", "endoftext", "|>", "9", "Dž", "/\r\n", "ABC", "😀🏽", "e", "́Dž字", ">!!!", "HTTPServer", " ", " ㍿\r\n\r\n", "HTTPServer", "٣٤٥", "٦", "'Re", "123", "456", "78", "Džungla", "½"]} +{"text": "<|endoftext|>عAbⅣ\r\n\r\nméßⅣ\n/ABC𐞁ßſcamelCase'M𐞁(ficamelCaseᵃiOSB\r\n\r\nſ½é'Saa", "tokens": 54, "pieces": ["<|", "endoftext", "|>", "عAb", "Ⅳ", "\r\n\r\n", "méß", "Ⅳ", "\n", "/ABC𐞁ßſ", "camelCase", "'M", "𐞁", "(ficamelCaseᵃiOSB", "\r\n\r\n", "ſ", "½", "é", "'S", "aa"]} +{"text": "a\r🙂'ree字<|endoftext|>aB­./漢", "tokens": 22, "pieces": ["a", "\r", "🙂'", "ree字", "<|", "endoftext", "|><", "EOT", ">aB", "­./", "漢"]} +{"text": "Ａ٣٤٥٦'DZ'VE㋿mB/\r\n, \n👍🏽Z㍿ßfi'T३\n/DžunglaDž('M…\n/9…\r", "tokens": 56, "pieces": ["Ａ", "٣٤٥", "٦", "'D", "Z", "'VE", "㋿mB", "/\r\n", ",", " \n", "👍🏽", "Z", "㍿ßfi", "'T", "३", "\n", "/DžunglaDž", "('", "M", "…\n", "/", "9", "…\r"]} +{"text": "'Se½fi٣٤٥٦३İ😀🏽 ,-́fia\n!<|fim_prefix|>fiAb!/Z.écamelCaseB½'D㋿HTTPServera'VEå漢", "tokens": 59, "pieces": ["'S", "e", "½", "fi", "٣٤٥", "٦३", "İ", "😀🏽", " ,-́", "fia", "\n", "!<|", "fim", "_prefix", "|>", "fiAb", "!/", "Z", ".écamelCaseB", "½", "'D", "㋿HTTPServera", "'VE", "a", "̊漢"]} +{"text": "e👍🏽'­
\rsḍ̇ \r\n\r\n/\r\n/DžAb<|endoftext|>\r\nعⅣ
iOSfi\r\n👍🏽d३ Dž\r9tEOT½e#$%a/bDžungla", "tokens": 67, "pieces": ["e", "👍🏽'­", "
\r", "sḋ", "̣", " \r\n\r\n", "/\r\n", "/DžAb", "<|", "endoftext", "|>\r\n", "ع", "Ⅳ", "
iOSfi", "\r\n", "👍🏽", "d", "३", " Dž", "\r", "9", "tEOT", "½", "e", "#$%", "a", "/bDžungla"]} +{"text": "́​'Re'D", "tokens": 5, "pieces": ["́​'", "Re", "'D"]} +{"text": " \n …d㍿
<ꟲé.ع're're", "tokens": 20, "pieces": [" \n", " ", "…d", "㍿", "
", "<ꟲe", "́.", "ع", "'re", "'re"]} +{"text": "å'DABC's'D𐞁<|endoftext|>́😀🏽,​\r\n\r\n<|endoftext|>ḍ̇e​­👍🏽aBéعaB'Reꟲ㍿", "tokens": 69, "pieces": ["a", "̊'", "DABC", "'s", "'D", "𐞁", "<|", "endoftext", "|>́😀🏽,​\r\n\r\n", "<|", "endoftext", "|>", "ḋ", "̣e", "​­<", "META", "_START", ">👍🏽", "aBéعaB", "'Re", "ꟲ", "㍿"]} +{"text": "9 ३́/\r\n\t", "tokens": 7, "pieces": ["9", " ", "३", "́/\r\n", "\t"]} +{"text": "'re'字!é'ſaB漢٣٤٥٦'Re٣٤٥٦é", "tokens": 31, "pieces": ["'re", "'字", "!e", "́'", "ſaB漢", "٣٤٥", "٦", "'Re", "٣٤٥", "٦", "é"]} +{"text": "́a/b\u000b\r\n\r\n!!é 12345678'll\t,Ab'Sſſm-.\n/a/bAbⅣ<|fim_prefix|>/\r\n(>mcamelCase#$%a/b\t'HTTPServer>漢\t", "tokens": 54, "pieces": ["́a", "/b", "\u000b\r\n\r\n", "!!", "e", "́", " ", "123", "456", "78", "'ll", "\t", ",Ab", "'S", "ſſm", "-.\n", "/a", "/bAb", "Ⅳ", "<|", "fim", "_prefix", "|>/\r\n", "(>", "mcamelCase", "#$%", "a", "/b", "\t", "'HTTPServer", ">漢", "\t"]} +{"text": "عdＡBs'dcamelCase/\"DžunglaaBt𐞁>'ſ12345678aBZ.#$%éꟲ½\u000bs<", "tokens": 50, "pieces": ["عdＡBs", "'d", "camelCase", "/\"", "DžunglaaBt𐞁", ">'", "ſ", "123", "456", "78", "aBZ", ".<", "META", "_START", ">#$%", "e", "́ꟲ", "½", "\u000bs", "<<", "EOT", ">"]} +{"text": "9t<|fim_prefix|>́\n/-12345678dꟲ<|endoftext|>ſa'M½٣٤٥٦ \n \n३d>d", "tokens": 44, "pieces": ["9", "t", "<|", "fim", "_prefix", "|>́\n", "/-", "123", "456", "78", "dꟲ", "<|", "endoftext", "|>", "ſa", "'M", "½٣٤", "٥٦", " \n \n", "३", "d", ">d"]} +{"text": "\ta/bİ٣٤٥٦<|fim_prefix|>é漢>'Sſ<\t'D\u000bAb<|endoftext|>tḍ̇'MéDž \nḍ̇漢ᵃDžunglá", "tokens": 64, "pieces": ["\ta", "/bİ", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "é漢", ">'", "Sſ", "<", "\t", "'D", "\u000bAb", "<|", "endoftext", "|>", "tḋ", "̣'", "MéDž", " \n", "ḋ", "̣漢ᵃDžungla", "́"]} +{"text": "ſ'T/!aBaEOT३aB'Re/\"a/bſ", "tokens": 19, "pieces": ["ſ", "'T", "/!", "aBaEOT", "३", "aB", "'Re", "/\"", "a", "/bſ"]} +{"text": "'T.!Z
t漢ᵃ\r\n\r\n9👍🏽/\r\na/bع\réſZZ'Re,", "tokens": 30, "pieces": ["'T", ".!", "Z", "
t漢ᵃ", "\r\n\r\n", "9", "👍🏽/\r\n", "a", "/bع", "\r", "éſZZ", "'Re", ","]} +{"text": "ABCEOT>/\r\n'D're", "tokens": 9, "pieces": ["ABC", "EOT", ">/\r\n", "'D", "'re"]} +{"text": "dſ9HTTPServerEOTꟲ/", "tokens": 14, "pieces": ["dſ", "9", "HTTPServerEOTꟲ", "/"]} +{"text": "00ådDžungla'👍🏽…Dž're<\u000bZiOS𐞁'ſ'VEé9s'DDž Ａ<|fim_prefix|>'T𐞁\n/ß s'M('ſ", "tokens": 66, "pieces": ["00", "a", "̊dDžungla", "'👍🏽", "…Dž", "'re", "<", "\u000bZiOS𐞁", "'ſ", "'VE", "é", "9", "s", "'", "DDž", " ", " Ａ", "<|", "fim", "_prefix", "|>'", "T𐞁", "\n", "/ß", " s", "'M", "('", "ſ"]} +{"text": "　ḍ̇", "tokens": 6, "pieces": ["　ḋ", "̣"]} +{"text": "'s㍿　0aB\tHTTPServerEOTfi🙂 dé́ 'D!!\r\n\r\nt'M\r\nåDž.\n/HTTPServer", "tokens": 37, "pieces": ["'s", "㍿", "　", "0", "aB", "\tHTTPServerEOTfi", "🙂", " ", " dé", "́", " ", " '", "D", "!!\r\n\r\n", "t", "'M", "\r\n", "a", "̊Dž", ".\n", "/HTTPServer"]} +{"text": ">'S\n/Ａ'StEOT㍿३(å👍🏽\r/\r\n'll'M'/\r\n\r\n\r\n\r\n(.'D​ 　😀🏽ᵃ🙂!!Ａſ \nB　", "tokens": 62, "pieces": [">'", "S", "\n", "/Ａ", "'S", "tEOT", "㍿", "३", "(a", "̊👍🏽\r", "/\r\n", "'ll", "'M", "'/\r\n\r\n\r\n\r\n", "(.'", "D", "​", " ", " <", "EOT", ">", "　", "😀🏽", "ᵃ", "🙂<", "META", "_START", ">!!", "Ａſ", " \n", "B", "　"]} +{"text": "­é", "tokens": 2, "pieces": ["­é"]} +{"text": "<|fim_prefix|>\t𐞁字fi
", "tokens": 17, "pieces": ["<|", "fim", "_prefix", "|>", "\t𐞁字fi", "
"]} +{"text": ",ᵃ字aBa/bZABC字'漢­👍🏽字0,字 <|fim_prefix|>eABC㍿aB
", "tokens": 42, "pieces": [",ᵃ字aBa", "/bZABC字", "'漢", "­👍🏽", "字", "0", ",字", " ", "<|", "fim", "_prefix", "|>", "eABC", "㍿aB", "
"]} +{"text": "㋿३'S \n'siOS\n/٣٤٥٦camelCase'ſ­Ab'SＡaBsé", "tokens": 32, "pieces": ["㋿", "३", "'S", " \n", "'s", "iOS", "\n", "/", "٣٤٥", "٦", "camelCase", "'ſ", "­Ab", "'S", "ＡaBsé"]} +{"text": "漢!>'scamelCasé", "tokens": 8, "pieces": ["漢", "!>'", "scamelCase", "́"]} +{"text": "-9Džunglaåm/e'M-Ab\u000b \nＡᵃ\u000b", "tokens": 40, "pieces": ["-", "9", "Džunglaa", "̊m", "/e", "'M", "-Ab", "", "\u000b \n", "Ａᵃ", "\u000b"]} +{"text": " (camelCase　-#$%'Ta99!é#$%\n/'Dé㋿Džungla
\r\n\r\nᵃ<|fim_prefix|>aB㍿-　　\r\n/​\n\r\n\r\neABCa/b
", "tokens": 58, "pieces": [" (", "camelCase", "　", "-#$%'", "Ta", "99", "!e", "́#$%\n", "/'", "De", "́㋿", "Džungla", "
\r\n\r\n", "ᵃ", "<|", "fim", "_prefix", "|>", "aB", "㍿-", "　　\r\n", "/<", "META", "_START", ">​\n\r\n\r\n", "eABCa", "/b", "
"]} +{"text": "​é😀🏽ḍ̇Dž.\r\n\r\n's½\u000b'T́", "tokens": 20, "pieces": ["​é", "😀🏽", "ḋ", "̣Dž", ".\r\n\r\n", "'s", "½", "\u000b", "'T", "́"]} +{"text": ">å(
<㋿'s٣٤٥٦-(a/b#$%İ'll#$%'T'S\"", "tokens": 32, "pieces": [">a", "̊(", "
", "<㋿'", "s", "٣٤٥", "٦", "-(", "a", "/b", "#$%", "İ", "'ll", "#$%'", "T", "'S", "\""]} +{"text": "\r\nḍ̇\r\n12345678 \n (Ⅳd<|endoftext|>ABC'll…éDžZ'll‍­'T\u000b0­ \n ٣٤٥٦(éＡEOTeå!́𐞁ḍ̇…m<|endoftext|>", "tokens": 79, "pieces": ["\r\n", "ḋ", "̣\r\n", "123", "456", "78", " \n", " (", "Ⅳ", "d", "<|", "endoftext", "|>", "ABC", "'ll", "…éDžZ", "'ll", "‍­'", "T", "\u000b", "0", "­", " \n", " ", "٣٤٥", "٦", "(éＡEOTea", "̊!́", "𐞁ḋ", "̣", "…m", "<|", "endoftext", "|>"]} +{"text": "é 'ſ㋿Z'VE<|fim_prefix|>\n/12345678'Re
s<㋿eᵃ\t漢\r\n\r\n漢ᵃ\r\n12345678ᵃ­åsZⅣ", "tokens": 59, "pieces": ["e", "́", " ", " '", "ſ", "㋿Z", "'VE", "<|", "fim", "_prefix", "|>\n", "/", "123", "456", "78", "'Re", "
s", "<㋿", "eᵃ", "\t漢", "\r\n\r\n", "漢ᵃ", "\r\n", "123", "456", "78", "ᵃ", "­a", "̊sZ", "Ⅳ"]} +{"text": "𐞁𐞁½'s", "tokens": 10, "pieces": ["𐞁𐞁", "½", "'s"]} +{"text": "/\r\n ", "tokens": 2, "pieces": ["/\r\n", " "]} +{"text": "漢>0're", "tokens": 5, "pieces": ["漢", ">", "0", "'re"]} +{"text": "'SEOT\n/ᵃßEOTa/bZ㍿ \nع,عHTTPServer‍  ‍m'DＡ'Reعع", "tokens": 36, "pieces": ["'S", "EOT", "\n", "/ᵃßEOTa", "/bZ", "㍿", " \n", "ع", ",عHTTPServer", "‍", " ", " ", "‍m", "'D", "Ａ", "'Re", "عع"]} +{"text": "!ßⅣḍ̇ꟲ'ſ㍿", "tokens": 18, "pieces": ["!ß", "Ⅳ", "ḋ", "̣ꟲ", "'ſ", "㍿"]} +{"text": "ꟲ12345678\rſ<|fim_prefix|>٣٤٥٦​Džunglaé𐞁Dž#$%012345678\r\n\r\n٣٤٥٦!Džungla😀🏽!é\n/½12345678'D ㍿\r\nd\r\n\r\nİ
Ab-'😀🏽fi", "tokens": 94, "pieces": ["ꟲ", "123", "456", "78", "\r", "ſ", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "​Džunglaé𐞁", "Dž", "#$%", "012", "345", "678", "\r\n\r\n", "٣٤٥", "٦", "!Džungla", "😀🏽!", "é", "\n", "/", "½12", "345", "678", "'D", " ", "㍿\r\n", "d", "\r\n\r\n", "İ", "", "
Ab", "-'😀🏽", "fi"]} +{"text": "𐞁'Re\n/Ab𐞁ABC३ß㍿ \n३ABC'ſ字字m're 'ReDžungla　s👍🏽ḍ̇DžunglasDž9<|endoftext|>\t\n/३\r\n(𐞁ḍ̇'M'VEABC", "tokens": 83, "pieces": ["𐞁", "'Re", "\n", "/Ab𐞁ABC", "३", "ß", "㍿", " \n", "३", "ABC", "'ſ", "字字m", "'re", " ", "'Re", "Džungla", "　s", "👍🏽", "ḋ", "̣DžunglasDž", "9", "<|", "endoftext", "|>", "\t\n", "/", "३", "\r\n", "(𐞁ḋ", "̣'", "M", "'VE", "ABC"]} +{"text": "iOS(ABCfiDžunglae🙂EOT­Džungla㍿#$%12345678camelCase'ſ#$%#$%'MaB٣٤٥٦'Reé'llſſ", "tokens": 53, "pieces": ["iOS", "(ABCfiDžunglae", "🙂EOT", "­Džungla", "㍿#$%", "123", "456", "78", "camelCase", "'ſ", "#$%#$%'", "MaB", "٣٤٥", "٦", "'Re", "é", "'ll", "ſſ"]} +{"text": "٣٤٥٦㋿a\r\naa('llcamelCaseDžunglaDž\n/a ZiOS🙂३ABCa/b> \n0'ſ\n/-#$%𐞁'll\r\n\u000b'D>㍿\r\n\r\n", "tokens": 65, "pieces": ["٣٤٥", "٦", "㋿a", "\r\n", "aa", "('", "llcamelCaseDžunglaDž", "\n", "/a", " ", " <", "META", "_START", ">ZiOS", "🙂", "३", "ABCa", "/b", ">", " \n", "0", "'ſ", "\n", "/-#$%", "𐞁", "'ll", "\r\n", "\u000b", "'D", ">㍿\r\n\r\n"]} +{"text": "😀🏽Džꟲ0Dž'rea👍🏽 \nꟲ'D́Ａ­Z 9Ａs,#$% DžHTTPServer.\"Ⅳ", "tokens": 48, "pieces": ["😀🏽", "Džꟲ", "0", "Dž", "'re", "a", "👍🏽", " \n", "ꟲ", "'D", "́Ａ", "­Z", " ", " ", "9", "Ａs", ",#$%", " DžHTTPServer", ".\"", "Ⅳ"]} +{"text": "३ſ<|endoftext|>ⅣBB! \n\t'Dm…🙂-/\r\nⅣaB 're👍🏽camelCase😀🏽ḍ̇aBéßEOT👍🏽😀🏽½#$%'sع'Då", "tokens": 79, "pieces": ["३", "ſ", "<|", "endoftext", "|>", "Ⅳ", "BB", "!", " \n", "\t", "'D", "m", "…", "🙂-/\r\n", "Ⅳ", "aB", " '", "re", "👍🏽", "camelCase", "😀🏽", "ḋ", "̣aBe", "́ßEOT", "👍🏽😀🏽<", "EOT", ">", "½", "#$%'", "sع", "'D", "a", "̊"]} +{"text": "9< \n!m12345678a\r\n٣٤٥٦iOS \n
عé'ſ­,ꟲcamelCaseＡ३.\"'Re!\u000b,ᵃ٣٤٥٦'ſ'M\t", "tokens": 64, "pieces": ["9", "<", " \n", "!m", "123", "456", "78", "a", "\r\n", "٣٤٥", "٦", "iOS", "", " \n", "
عe", "́'", "ſ", "­,", "ꟲcamelCaseＡ", "३", ".\"'", "Re", "!", "\u000b", ",ᵃ", "٣٤٥", "٦", "'ſ", "'M", "\t"]} +{"text": "m\n…½\r३　Džungla!/ ع,<> 'S<|fim_prefix|>aB'Ma/bsaBİ.iOS", "tokens": 37, "pieces": ["m", "\n", "…", "½", "\r", "३", "　Džungla", "!/", " ع", ",<>", " '", "S", "<|", "fim", "_prefix", "|>", "aB", "'M", "a", "/bsaBİ", ".iOS"]} +{"text": ">>å'sZ(", "tokens": 8, "pieces": [">>", "a", "̊'", "sZ", "("]} +{"text": "0㋿🙂!!- \naBa/b \n/,
9ꟲ-a<|fim_prefix|>Dž　9fiＡiOS…'VE'D<|fim_prefix|>'T12345678 \n \n é12345678t\r ", "tokens": 62, "pieces": ["0", "㋿🙂!!-", " \n", "aBa", "/b", " \n", "/,", "
", "9", "ꟲ", "-a", "<|", "fim", "_prefix", "|>", "Dž", "　", "9", "fiＡiOS", "…", "'VE", "'D", "<|", "fim", "_prefix", "|>'", "T", "123", "456", "78", " \n \n", " é", "123", "456", "78", "t", "\r "]} +{"text": "'S \n عd'VE३(0-㍿", "tokens": 14, "pieces": ["'S", " \n", " عd", "'VE", "३", "(", "0", "-㍿"]} +{"text": "('Re's👍🏽12345678'VE‍EOTع.12345678!!漢 \n'ſＡ漢,a/b\r\n\r\n 
…'S\"Ａ\r\n…ᵃsEOTＡ'", "tokens": 62, "pieces": ["('", "Re", "'s", "👍🏽", "123", "456", "78", "'VE", "‍EOTع", ".", "123", "456", "78", "!!", "漢", " \n", "'ſ", "Ａ漢", ",a", "/b", "\r\n\r\n", " ", "
", "", "…", "'S", "\"Ａ", "\r\n", "…ᵃsEOTＡ", "'"]} +{"text": "­ e­'s\r\n\r\n", "tokens": 7, "pieces": ["­", " ", " e", "­'", "s", "\r\n\r\n"]} +{"text": "!\r\n\r\nåßDžunglaEOT漢HTTPServer漢camelCase<|endoftext|>\t \n ABC\r\n🙂𐞁\u000b<|fim_prefix|>\"'T'Re'sta　'DaBBé(‍\t👍🏽", "tokens": 70, "pieces": ["!\r\n\r\n", "a", "̊ßDžunglaEOT漢HTTPServer漢camelCase", "<|", "endoftext", "|>", "\t \n", " ABC", "\r\n", "🙂𐞁", "\u000b", "<|", "fim", "_prefix", "|>\"'", "T", "'Re", "'s", "ta", "　", "'", "DaBB", "e", "́(‍", "\t", "👍🏽"]} +{"text": " \n \r\n\r\n", "tokens": 2, "pieces": [" \n \r\n\r\n"]} +{"text": "٣٤٥٦aB́B\n/'ſ'sa\n/👍🏽9é🙂e!!𐞁३HTTPServer,Ａ", "tokens": 48, "pieces": ["٣٤٥", "٦", "aB", "́B", "\n", "/'", "ſ", "'s", "a", "\n", "/👍🏽", "9", "e", "́🙂", "e", "!!", "𐞁", "३", "HTTPServer", ",Ａ"]} +{"text": "ع<|endoftext|>­iOS漢", "tokens": 12, "pieces": ["ع", "<|", "endoftext", "|>­", "iOS漢"]} +{"text": "'VE \n　'S
s\n/'T㍿.aB9camelCase12345678Ａ", "tokens": 25, "pieces": ["'VE", " \n", "　", "'S", "
s", "\n", "/'", "T", "㍿.", "aB", "9", "camelCase", "123", "456", "78", "Ａ"]} +{"text": "ᵃİ,㍿e9!!㍿\r!!", "tokens": 19, "pieces": ["ᵃİ", ",㍿", "e", "9", "!!㍿\r", "!!"]} +{"text": "e<|endoftext|>t12345678ꟲé㍿fi'MＡ\na/b 'ABC(́å", "tokens": 36, "pieces": ["e", "<|", "endoftext", "|>", "t", "123", "456", "78", "ꟲe", "́㍿", "fi", "'M", "Ａ", "\n", "a", "/b", " ", "'ABC", "(́", "a", "̊"]} +{"text": "عHTTPServer\n/", "tokens": 5, "pieces": ["عHTTPServer", "\n", "/"]} +{"text": "ⅣiOS㋿'sABC'M<|endoftext|>.iOS >sßa/bAb\rḍ̇३!!­٣٤٥٦'DZ<𐞁/\r\n(#$%,½<'re𐞁", "tokens": 63, "pieces": ["Ⅳ", "iOS", "㋿'", "sABC", "'M", "<|", "endoftext", "|>.", "iOS", " >", "sßa", "/bAb", "\r", "ḋ", "̣", "३", "!!­", "٣٤٥", "٦", "'D", "Z", "<𐞁", "/\r\n", "(#$%,", "½", "<'", "re𐞁"]} +{"text": "-字ḍ̇s-aABCDžaB12345678d<|endoftext|>HTTPServerＡḍ̇\r\nZ0\nB\"ᵃeé\"", "tokens": 46, "pieces": ["-字ḋ", "̣s", "-aABCDžaB", "123", "456", "78", "d", "<|", "endoftext", "|>", "HTTPServerＡḋ", "̣\r\n", "Z", "0", "\n", "B", "\"ᵃeé", "\""]} +{"text": "édḍ̇ABC\u000bḍ̇ᵃ9<|fim_prefix|>\r㍿'reHTTPServerét\"İå🙂d ßaB㋿‍٣٤٥٦.aB'D
\r\n\r\n", "tokens": 67, "pieces": ["e", "́dḋ", "̣ABC", "\u000bḋ", "̣ᵃ", "9", "<|", "fim", "_prefix", "|>\r", "㍿'", "reHTTPServerét", "\"İa", "̊🙂", "d", " ", " ßaB", "㋿‍", "٣٤٥", "٦", ".aB", "'D", "
\r\n\r\n"]} +{"text": "🙂㋿Zꟲ½,‍", "tokens": 13, "pieces": ["🙂㋿", "Zꟲ", "½", ",‍"]} +{"text": " \n
३'ſꟲ", "tokens": 11, "pieces": [" \n", "
", "३", "'ſ", "ꟲ"]} +{"text": " 12345678Z\tİ㍿\n/'VE#$%…!.'Džunglas\r\n\r\n-s'Re‍'MHTTPServerfi<0عZ३㋿EOTＡ𐞁", "tokens": 61, "pieces": [" ", "123", "456", "78", "Z", "\tİ", "㍿\n", "/'", "VE", "#$%", "…", "!.'", "Džunglas", "<", "META", "_START", ">\r\n\r\n", "-s", "'", "Re", "‍'", "MHTTPServerfi", "<", "0", "عZ", "३", "㋿EOTＡ𐞁"]} +{"text": "'reaBſ<|endoftext|>😀🏽0tiOSＡſHTTPServer", "tokens": 30, "pieces": ["'re", "aBſ", "<|", "endoftext", "|>😀🏽", "0", "tiOS", "ＡſHTTPServer"]} +{"text": "a/३camelCaseＡZ'VE", "tokens": 11, "pieces": ["a", "/", "३", "camelCaseＡZ", "'VE"]} +{"text": "🙂<|endoftext|>EOT", "tokens": 11, "pieces": ["🙂<|", "endoftext", "|>", "EOT"]} +{"text": "a're𐞁'llEOTＡd​Dž'T<|fim_prefix|>/\r\n … /\r\n'Abꟲ३e#$%'Rea/b-", "tokens": 47, "pieces": ["a", "'re", "𐞁", "'ll", "EOTＡd", "​Dž", "'T", "<|", "fim", "_prefix", "|>/\r\n", " ", "…", "", " ", "/\r\n", "'Abꟲ", "३", "e", "#$%'", "Rea", "/b", "-"]} +{"text": "İ\t🙂𐞁#$%dm‍m,Ab漢\r\n\t👍🏽 \n 'sm", "tokens": 29, "pieces": ["İ", "\t", "🙂𐞁", "#$%", "dm", "‍m", ",Ab漢", "\r\n", "\t", "👍🏽", " \n", " '", "sm"]} +{"text": "\" \n 're\t-'D\nd \n 'SᵃZ\r\n\r\nع'ſ \n😀🏽'Då\n/　'T''D", "tokens": 37, "pieces": ["\"", " \n", " '", "re", "\t", "-'", "D", "\n", "d", " \n", " '", "SᵃZ", "\r\n\r\n", "ع", "'ſ", " \n", "😀🏽'", "Da", "̊\n", "/", "　", "'T", "''", "D"]} +{"text": "!!'ſ…\n#$%…\tİ𐞁½( ⅣHTTPServer12345678iOSZ \r\n\r\nt!!Ⅳ
9\n're", "tokens": 40, "pieces": ["!!'", "ſ", "…\n", "#$%", "…", "\tİ𐞁", "½", "(", " ", "Ⅳ", "HTTPServer", "123", "456", "78", "iOSZ", " \r\n\r\n", "t", "!!", "Ⅳ", "
", "9", "\n", "'re"]} +{"text": "\tcamelCase'S/\r\nABC'ReiOS㋿'Res \r\n12345678…\r \n  -iOS漢🙂٣٤٥٦é…㍿　\r\n\r\nＡ's!!-camelCase\rABC<|fim_prefix|>>/ḍ̇!!", "tokens": 69, "pieces": ["\tcamelCase", "'S", "/\r\n", "ABC", "'Re", "iOS", "㋿'", "Res", " \r\n", "123", "456", "78", "…\r \n", " ", " ", "-iOS漢", "🙂", "٣٤٥", "٦", "é", "…", "㍿", "　\r\n\r\n", "Ａ", "'s", "!!-", "camelCase", "\r", "ABC", "<|", "fim", "_prefix", "|>>/", "ḋ", "̣!!"]} +{"text": "'T å'VE \n \n 'T\nꟲſ ßé'M‍\rAb😀🏽/\r\nＡ३/\r\nع !!é㍿", "tokens": 44, "pieces": ["'T", " a", "̊'", "VE", " \n \n", " '", "T", "\n", "ꟲſ", " ", " ßé", "'M", "‍\r", "Ab", "😀🏽/\r\n", "Ａ", "३", "/\r\n", "ع", " ", "!!", "e", "́㍿"]} +{"text": "s㍿>٣٤٥٦12345678\r\n\r\n ​\r\n's字s字😀🏽's𐞁Džungla㋿ABC'ſDžungla🙂t'VE\n/eDž \n'SeiOS<字'SAbع", "tokens": 73, "pieces": ["s", "㍿>", "٣٤٥", "٦12", "345", "678", "\r\n\r\n", " ​\r\n", "'s", "字s字", "😀🏽'", "s𐞁Džungla", "㋿ABC", "'ſ", "Džungla", "🙂t", "'VE", "\n", "/eDž", " \n", "'S", "eiOS", "<<", "META", "_START", ">字", "'S", "Abع"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "‍! \n're​.", "tokens": 7, "pieces": ["‍!", " \n", "'re", "​."]} +{"text": "㍿'TiOSiOS!'re'S >ſ　ꟲ'T٣٤٥٦'Reſ漢'ReᵃiOS/\r\n-\r\n‍ع٣٤٥٦́0😀🏽ABC \n/\r\n\r\n", "tokens": 60, "pieces": ["㍿'", "TiOSiOS", "!'", "re", "'S", " ", ">ſ", "　ꟲ", "'T", "٣٤٥", "٦", "'Re", "ſ漢", "'Re", "ᵃiOS", "/\r\n", "-\r\n", "‍ع", "٣٤٥", "٦", "́", "0", "😀🏽", "ABC", " \n", "/\r\n\r\n"]} +{"text": "\"!\"½字½'reZéfiDžungla", "tokens": 14, "pieces": ["\"!\"", "½", "字", "½", "'re", "ZéfiDžungla"]} +{"text": "afi\t#$%<|endoftext|>Ab/\r\n EOT \n ꟲ'll\t👍🏽…aBaB'Ret㍿#$%ḍ̇", "tokens": 46, "pieces": ["afi", "\t", "#$%<|", "endoftext", "|>", "Ab", "/\r\n", " EOT", " \n", " ꟲ", "'ll", "\t", "👍🏽", "…aBaB", "'Re", "t", "㍿#$%", "ḋ", "̣"]} +{"text": "é-\r\n\r\n\u000b字('re'D \n ٣٤٥٦३\r\n\r\n!عB!iOSé-edABC\n/ \ncamelCase.漢mcamelCasedé", "tokens": 47, "pieces": ["é", "-\r\n\r\n", "\u000b字", "('", "re", "'D", " \n", " ", "٣٤٥", "٦", "", "३", "\r\n\r\n", "!عB", "!iOSé", "-edABC", "\n", "/", " \n", "camelCase", ".漢mcamelCasede", "́"]} +{"text": "́sDžunglaat \n 漢DžHTTPServer㋿ \n ABCꟲm", "tokens": 24, "pieces": ["́sDžunglaat", " \n", " 漢DžHTTPServer", "㋿", " \n", " ABCꟲm"]} +{"text": "½'M!!!", "tokens": 3, "pieces": ["½", "'M", "!!!"]} +{"text": "👍🏽　'll<|endoftext|>'D 𐞁0Dž­ſßZiOS👍🏽\"३å.ḍ̇­ ­-\r \ns'M", "tokens": 59, "pieces": ["👍🏽", "　", "'ll", "<|", "endoftext", "|>'", "D", " <", "EOT", ">𐞁", "0", "Dž", "­ſßZiOS", "👍🏽\"", "३", "a", "̊.", "ḋ", "̣­", " ", " ­-\r", " \n", "s", "'M"]} +{"text": "<|fim_prefix|>>字'M9a/béiOS12345678B ㋿å'll'Mt!!ḍ̇ḍ̇́<|endoftext|>!", "tokens": 50, "pieces": ["<|", "fim", "_prefix", "|>>", "字", "'M", "9", "a", "/be", "́iOS", "123", "456", "78", "B", " ", " ㋿", "a", "̊'", "ll", "'M", "t", "!!", "ḋ", "̣ḋ", "̣́<|", "endoftext", "|>!"]} +{"text": "ع\u000b/ \n #$%", "tokens": 10, "pieces": ["ع", "\u000b", "/", " \n", " <", "META", "_START", ">#$%"]} +{"text": "'réécamelCase/\r\n\"/\r\na/b'reABCfi<ع#$%aB\t", "tokens": 21, "pieces": ["'re", "́écamelCase", "/\r\n", "\"/\r\n", "a", "/b", "'re", "ABCfi", "<ع", "#$%", "aB", "\t"]} +{"text": "٣٤٥٦<|fim_prefix|>٣٤٥٦iOSḍ̇Dž'T\n/…Džungla…é", "tokens": 45, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "iOSḋ", "̣Dž", "'T", "\n", "/", "…Džungla", "…é"]} +{"text": "ABCcamelCase'DcamelCase 😀🏽字12345678ꟲ🙂 漢", "tokens": 23, "pieces": ["ABCcamelCase", "'D", "camelCase", " ", " 😀🏽", "字", "123", "456", "78", "ꟲ", "🙂", " 漢"]} +{"text": "ſ漢/iOS٣٤٥٦>\r㋿Z12345678'MDžungla\t'TDžungla\"'VEⅣEOTZ\n/½ta/b\r\n a'D… <|endoftext|>é'St\u000b", "tokens": 69, "pieces": ["ſ漢", "/iOS", "٣٤٥", "٦", ">\r", "㋿Z", "123", "456", "78", "'M", "Džungla", "\t", "'T", "Džungla", "\"'", "VE", "Ⅳ", "EOTZ", "\n", "/", "½", "ta", "/b", "\r\n", " a", "'D", "…", " ", "<|", "endoftext", "|>", "e", "́'", "S", "t", "\u000b"]} +{"text": "\"-", "tokens": 1, "pieces": ["\"-"]} +{"text": "/", "tokens": 1, "pieces": ["/"]} +{"text": "e's 'll'ſZ\u000bDž\u000b<|endoftext|>'M \n 's/\r\nås'VEm,\r\n\r\nåḍ̇/😀🏽<|endoftext|>,s", "tokens": 58, "pieces": ["e", "'s", " ", "'ll", "'ſ", "Z", "\u000bDž", "\u000b", "<|", "endoftext", "|>'", "M", " \n", " '", "s", "/\r\n", "a", "̊s", "'VE", "m", ",\r\n\r\n", "a", "̊<", "EOT", ">ḋ", "̣/😀🏽<|", "endoftext", "|>,", "s"]} +{"text": "😀🏽", "tokens": 5, "pieces": ["😀🏽"]} +{"text": "/Ⅳḍ̇(ABC　\r.­½​iOS…\r\n\r\n\"\n <12345678ß.HTTPServerZ ᵃ-'S\ta#$%", "tokens": 43, "pieces": ["/", "Ⅳ", "ḋ", "̣(", "ABC", "　\r", ".­", "½", "​iOS", "…\r\n\r\n", "\"\n", " ", " <", "123", "456", "78", "ß", ".HTTPServerZ", " ᵃ", "-<", "META", "_START", ">'", "S", "\ta", "#$%"]} +{"text": "aⅣİAb", "tokens": 5, "pieces": ["a", "Ⅳ", "İAb"]} +{"text": " (t'M​ꟲ \n(́ǻEOT \n e'sEOT \n　å<|fim_prefix|>camelCase　㍿\t\r\n\r\n\r\n", "tokens": 41, "pieces": [" ", "(t", "'M", "​ꟲ", " \n", "(́", "a", "̊́", "EOT", " \n", " e", "'s", "EOT", " \n", "　a", "̊<|", "fim", "_prefix", "|>", "camelCase", "　", "㍿", "\t\r\n\r\n\r\n"]} +{"text": "ⅣEOTt\t/\r\n­'VE𐞁\t<३-,'s\n/㋿'VE'ſ\n<|fim_prefix|>9३'Dع'M 'S
Ⅳ0aB३ABC\r\n​s🙂", "tokens": 69, "pieces": ["Ⅳ", "EOTt", "\t", "/\r\n", "­'", "VE𐞁", "\t", "<", "३", "-,'", "s", "\n", "/㋿'", "VE", "'ſ", "\n", "<|", "fim", "_prefix", "|>", "9३", "'D", "ع", "'M", " ", "'", "S", "", "
", "Ⅳ0", "aB", "३", "ABC", "\r\n", "​s", "🙂"]} +{"text": "å­𐞁<|fim_prefix|>'M٣٤٥٦ddsdcamelCase,iOSꟲé…\n/㋿écamelCase'Re'T'", "tokens": 48, "pieces": ["a", "̊­", "𐞁", "<|", "fim", "_prefix", "|>'", "M", "٣٤٥", "٦", "ddsdcamelCase", ",iOSꟲe", "́", "…\n", "/㋿", "écamelCase", "'Re", "'T", "'"]} +{"text": "‍ſ'reé\u000bḍ̇e🙂…ſ're….㋿é'T𐞁‍#$%٣٤٥٦　
-é", "tokens": 55, "pieces": ["‍ſ", "'re", "e", "́", "\u000bḋ", "̣e", "🙂", "…ſ", "'re", "…", ".㋿", "é", "'T", "𐞁", "‍#$%", "٣٤٥", "٦", "　", "
", "-é"]} +{"text": "<|fim_prefix|>漢-\r\nZ㍿!!'ſDžungla\rDžungla'Mt…Ａḍ̇​iOS
é'T𐞁(DžéeABCé", "tokens": 56, "pieces": ["<|", "fim", "_prefix", "|>", "漢", "-\r\n", "Z", "㍿!!'", "ſDžungla", "\r", "Džungla", "'M", "t", "…Ａḋ", "̣​", "iOS", "
é", "'T", "𐞁", "(Dže", "́eABCé"]} +{"text": ".漢", "tokens": 3, "pieces": [".漢"]} +{"text": "ᵃm", "tokens": 4, "pieces": ["ᵃm"]} +{"text": "́😀🏽DžHTTPServer.\n/İ< /\r\nḍ̇/\r\n're🙂'T'ſⅣ字a/b\u000b", "tokens": 36, "pieces": ["́😀🏽", "DžHTTPServer", ".\n", "/İ", "<", " /\r\n", "ḋ", "̣/\r\n", "'re", "🙂'", "T", "'ſ", "Ⅳ", "字a", "/b", "\u000b"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿½٣٤٥٦a/bå'Reå'T", "tokens": 24, "pieces": ["㍿", "½٣٤", "٥٦", "a", "/ba", "̊'", "Rea", "̊'", "T"]} +{"text": "ꟲ,
 \nmß漢\u000b\na/bB́a/b/\r\n\r\nİfiİ", "tokens": 24, "pieces": ["ꟲ", ",", "
 \n", "mß漢", "\u000b\n", "a", "/bB", "́a", "/b", "/\r\n\r\n", "İfiİ"]} +{"text": " \n ३'ſ \n", "tokens": 8, "pieces": [" \n", " ", "३", "'ſ", " \n"]} +{"text": "9\n/DžunglaHTTPServer/漢ᵃ'TiOSHTTPServer߅'S", "tokens": 27, "pieces": ["", "9", "\n", "/DžunglaHTTPServer", "/漢ᵃ", "'T", "iOSHTTPServerß", "…", "'S"]} +{"text": "fi's", "tokens": 3, "pieces": ["fi", "'s"]} +{"text": "\r'Re ́/'MaBB😀🏽#$%<|endoftext|>\"d🙂aB", "tokens": 26, "pieces": ["\r", "'Re", " ", "́/'", "MaBB", "😀🏽#$%<|", "endoftext", "|>\"", "d", "🙂aB"]} +{"text": "\tDžungla🙂漢字½ \n/\"'Mß\r\n\r\n\r\n/\r\n㋿å\"é.", "tokens": 28, "pieces": ["\tDžungla", "🙂漢字", "½", " \n", "/\"'", "Mß", "\r\n\r\n\r\n", "/\r\n", "㋿a", "̊\"", "e", "́."]} +{"text": "iOS㍿a", "tokens": 5, "pieces": ["iOS", "㍿a"]} +{"text": "ᵃB\"HTTPServerᵃ, \r\n\r\né<|endoftext|>İ­EOT\r\nåḍ̇'DⅣ,'Re…٣٤٥٦\"a/b(.'re-'D
'Tåd　😀🏽\r'", "tokens": 72, "pieces": ["ᵃB", "\"HTTPServerᵃ", ",", " \r\n\r\n", "e", "́<|", "endoftext", "|>", "İ", "­EOT", "\r\n", "a", "̊ḋ", "̣'", "D", "Ⅳ", ",'", "Re", "…", "٣٤٥", "٦", "\"a", "/b", "(.'", "re", "-'", "D", "
", "'T", "a", "̊d", "　", "😀🏽\r", "'"]} +{"text": "'re!!EOT'llAbs'VEå🙂…<',12345678a/b ​㍿\n/
 HTTPServer(\n'M​", "tokens": 36, "pieces": ["'re", "!!", "EOT", "'ll", "Abs", "'VE", "a", "̊🙂", "…", "<',", "123", "456", "78", "a", "/b", " ​㍿\n", "/", "
", " HTTPServer", "(\n", "'M", "​"]} +{"text": "eEOTfi' \n\"　'M", "tokens": 10, "pieces": ["eEOTfi", "'", " \n", "\"", "　", "'M"]} +{"text": "‍'SAbt㋿ !<㍿HTTPServer　'T/éåＡ/\r\n㍿३/\r\nع>!!<|fim_prefix|>\rt'TaB.9", "tokens": 55, "pieces": ["‍'", "SAbt", "㋿", " <", "META", "_START", ">!<㍿", "HTTPServer", "", "　", "'T", "/e", "́a", "̊Ａ", "/\r\n", "㍿", "३", "/\r\n", "ع", ">!!<|", "fim", "_prefix", "|>\r", "t", "'T", "aB", ".", "9"]} +{"text": "ḍ̇३9HTTPServer ,camelCaseåḍ̇İ\"'ſDžungla12345678 \n <|endoftext|><|fim_prefix|>éſ,Ⅳ𐞁åHTTPServerꟲ½fi'Reع #$%å ", "tokens": 77, "pieces": ["ḋ", "̣", "३9", "HTTPServer", " ", ",camelCasea", "̊ḋ", "̣İ", "\"'", "ſDžungla", "123", "456", "78", " \n", " <|", "endoftext", "|><|", "fim", "_prefix", "|>", "e", "́ſ", ",", "Ⅳ", "𐞁a", "̊HTTPServerꟲ", "½", "fi", "'Re", "ع", " ", " #$%", "a", "̊", " "]} +{"text": "< \n㍿'M😀🏽<㍿'Re\n/ḍ̇(\r\n\r\n<|endoftext|>ſ \n,'ſ", "tokens": 43, "pieces": ["<", " \n", "㍿<", "EOT", ">'", "M", "😀🏽<㍿'", "Re", "\n", "/ḋ", "̣(\r\n\r\n", "<|", "endoftext", "|>", "ſ", " \n", ",'", "ſ"]} +{"text": "ᵃ/ \né'll­iOSḍ̇HTTPServera½字漢👍🏽iOSé's<|fim_prefix|>iOS(…éHTTPServer‍!!0,ſa/b/\r\n
", "tokens": 60, "pieces": ["ᵃ", "/", " \n", "é", "'ll", "­iOS", "ḋ", "̣HTTPServera", "½", "字漢", "👍🏽", "iOSé", "'s", "<|", "fim", "_prefix", "|>", "iOS", "(", "…éHTTPServer", "‍!!", "0", ",ſa", "/b", "/\r\n", "
"]} +{"text": "­Ab㍿'ll字㋿>'re\t'reZ'漢'T\r\n\r\nfi'S…\n/'VE<|fim_prefix|>9s'Sé٣٤٥٦", "tokens": 49, "pieces": ["­Ab", "㍿'", "ll字", "㋿>'", "re", "\t", "'re", "Z", "'漢", "'T", "\r\n\r\n", "fi", "'S", "…\n", "/'", "VE", "<|", "fim", "_prefix", "|>", "9", "s", "'S", "e", "́", "٣٤٥", "٦"]} +{"text": "é'ſZ -ß,\r\n\r\ncamelCase/\r\n\" !!𐞁\r\n\u000b\r\nZ \n<|endoftext|>#$%٣٤٥٦​𐞁 \n", "tokens": 47, "pieces": ["é", "'ſ", "Z", " ", "-ß", ",\r\n\r\n", "camelCase", "/\r\n", "\"", " ", " !!", "𐞁", "\r\n\u000b\r\n", "Z", " \n", "<|", "endoftext", "|>#$%", "٣٤٥", "٦", "​𐞁", " \n"]} +{"text": "👍🏽iOS'SABC", "tokens": 9, "pieces": ["👍🏽", "iOS", "'S", "ABC"]} +{"text": "<漢ḍ̇/\r\n㋿fi字\u000b-", "tokens": 17, "pieces": ["<漢ḋ", "̣/\r\n", "㋿fi字", "\u000b", "-"]} +{"text": " \nåḍ̇ſ\"", "tokens": 16, "pieces": [" \n", "/,'", "S", "!<", "EOT", ">ḋ", "̣ſ", "\""]} +{"text": "d\u000b字ABC٣٤٥٦👍🏽", "tokens": 18, "pieces": ["d", "\u000b字ABC", "٣٤٥", "٦", "👍🏽"]} +{"text": "㋿\t", "tokens": 4, "pieces": ["㋿", "\t"]} +{"text": "ᵃ", "tokens": 6, "pieces": ["ᵃ"]} +{"text": "字'll\r\n /d \nDž　Džungla.12345678​ !!Dž'VEḍ̇/\r\n­'ſＡåfi'VE", "tokens": 52, "pieces": ["字", "'ll", "\r\n", " /", "d", " \n", "Dž", "　Džungla", ".", "123", "456", "78", "​", " <", "EOT", ">!!", "Dž", "'", "VEḋ", "̣/\r\n", "­'", "ſＡa", "̊fi", "'VE", ""]} +{"text": "'Må‍३'VE<|fim_prefix|>
Z'é<\r\n\r\n'S!fié‍", "tokens": 30, "pieces": ["'M", "a", "̊‍", "३", "'VE", "<|", "fim", "_prefix", "|>", "
Z", "'é", "<\r\n\r\n", "'S", "!fié", "‍"]} +{"text": "å /\r\n/!Ⅳ \n !٣٤٥٦", "tokens": 19, "pieces": ["a", "̊", " /\r\n", "/!", "Ⅳ", " \n", " !", "٣٤٥", "٦"]} +{"text": "'T 漢 \n 'Dḍ̇mDž<|fim_prefix|> ​
…ſ<३　 \u000b>Z'D 👍🏽ea­iOS0漢Džungla 'ſⅣZ.", "tokens": 63, "pieces": ["'T", " ", " 漢", " \n", " '", "Dḋ", "̣mDž", "<|", "fim", "_prefix", "|>", " ", " ​", "
", "…ſ", "<", "३", "　 ", "\u000b", ">Z", "'D", " ", "👍🏽", "ea", "­iOS", "0", "漢Džungla", " '", "ſ", "Ⅳ", "Z", "."]} +{"text": "HTTPServerß>camelCase12345678e İDžungla'Re\t", "tokens": 20, "pieces": ["HTTPServerß", ">", "camelCase", "123", "456", "78", "e", " İDžungla", "'Re", "\t"]} +{"text": ",\r\n\r\n<|endoftext|>å ABC字'S \n'VE㍿Dž(HTTPServer \n'S🙂é​'S'ſeaé /\r\n \r\n'll字", "tokens": 52, "pieces": [",\r\n\r\n", "<|", "endoftext", "|>", "a", "̊", " ", " ABC字", "'", "S", " \n", "'VE", "㍿Dž", "(HTTPServer", " \n", "'S", "🙂é", "​'", "S", "'ſ", "eaé", " ", " /\r\n", " ", " <", "META", "_START", ">\r\n", "'ll", "字"]} +{"text": "㍿iOSEOT<|fim_prefix|>a/b'T/\r\nABC'll'll'VEt'Sſet'SAb0-Džunglaſ'ſ'S\n字字>\r\ndé🙂å㋿👍🏽𐞁0Dž", "tokens": 71, "pieces": ["㍿iOSEOT", "<|", "fim", "_prefix", "|>", "a", "/b", "'T", "/\r\n", "ABC", "'", "ll", "'ll", "'VE", "t", "'S", "ſet", "'S", "Ab", "0", "-Džunglaſ", "'ſ", "'S", "\n", "字字", ">\r\n", "dé", "🙂a", "̊㋿👍🏽", "𐞁", "0", "Dž"]} +{"text": "'M½'ſ'll
\u000b iOSDž!!\t/\r\n字Z㍿ḍ̇<|fim_prefix|>ع㍿!", "tokens": 37, "pieces": ["'M", "½", "'ſ", "'ll", "
\u000b", " iOSDž", "!!", "\t", "/\r\n", "字Z", "㍿ḋ", "̣<|", "fim", "_prefix", "|>", "ع", "㍿!"]} +{"text": ">😀🏽ᵃ<|fim_prefix|>ꟲABC/\r\n \n ½\r\n> \n\r🙂's३ \n 漢ꟲ\n/'ſ(\niOS\r\n\t, \n\r\n", "tokens": 53, "pieces": [">😀🏽", "ᵃ", "<|", "fim", "_prefix", "|>", "ꟲABC", "/\r\n", " \n", " ", " ", "½", "\r\n", ">", " \n\r", "🙂'", "s", "३", " \n", " 漢ꟲ", "\n", "/'", "ſ", "(\n", "iOS", "\r\n", "\t", ",", " \n\r\n"]} +{"text": "㍿'S\n/-㋿
fi‍e'ſ'se‍é", "tokens": 29, "pieces": ["㍿'", "S", "\n", "/-㋿", "
fi", "‍e", "'ſ", "'", "se", "‍e", "́"]} +{"text": "🙂Džungla \n éABCABC \n/\r\n漢'T㋿👍🏽Ab😀🏽३\r\n'll\"'ſBaB<㋿㋿å\nd-iOS#$%漢aBHTTPServer", "tokens": 61, "pieces": ["🙂Džungla", " \n", " éABCABC", " \n", "/\r\n", "漢", "'T", "㋿👍🏽", "Ab", "😀🏽", "३", "\r\n", "'ll", "\"'", "ſBaB", "<㋿㋿", "a", "̊\n", "d", "-iOS", "#$%", "漢aBHTTPServer"]} +{"text": "/\r\n \n'VE/ \ne!\n/½'ſ/\r\n \n fi٣٤٥٦12345678
9-\ncamelCase\n/'sＡ\n!", "tokens": 21, "pieces": ["e", "'M", "", "123", "456", "78", "
", "9", "-\n", "camelCase", "\n", "/'", "sＡ", "\n", "!"]} +{"text": "ꟲEOT👍🏽字>dABC\t字's<'ſfi'S­㋿'Mß'D \r", "tokens": 33, "pieces": ["ꟲEOT", "👍🏽", "字", ">dABC", "\t字", "'s", "<'", "ſfi", "'S", "­㋿'", "Mß", "'D", " \r"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " 'S'Re(", "tokens": 4, "pieces": [" ", "'S", "'Re", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "​ \n \n'VE Ⅳ(a/bEOT<|endoftext|>!!#$%𐞁🙂३\r字/\r\n\"ß🙂'sḍ̇́", "tokens": 56, "pieces": ["​", " \n \n", "'VE", " ", " ", "Ⅳ", "(a", "/bEOT", "<|", "endoftext", "|>!!<", "META", "_START", ">#$%", "𐞁", "🙂", "३", "\r", "字", "/\r\n", "\"ß", "🙂'", "sḋ", "̣́"]} +{"text": "㋿å's", "tokens": 8, "pieces": ["㋿a", "̊'", "s"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "'ś३'re\r\r\n\r\n㋿.ꟲé㍿'M \n ſ<|endoftext|>å9Z0Z㍿>ABCAbⅣ'SaB'll'reB", "tokens": 52, "pieces": ["'s", "́", "३", "'re", "\r\r\n\r\n", "㋿.", "ꟲé", "㍿'", "M", " \n", " ſ", "<|", "endoftext", "|>", "a", "̊", "9", "Z", "0", "Z", "㍿>", "ABCAb", "Ⅳ", "'S", "aB", "'ll", "'re", "B"]} +{"text": "ABCⅣ-ᵃ \n 'THTTPServeréHTTPServer٣٤٥٦ \n<字'ſfi m𐞁é'llå1234567812345678", "tokens": 49, "pieces": ["ABC", "Ⅳ", "-ᵃ", " \n", " '", "THTTPServeréHTTPServer", "٣٤٥", "٦", " \n", "<字", "'ſ", "fi", " ", " m𐞁e", "́'", "lla", "̊", "123", "456", "781", "234", "567", "8"]} +{"text": "#$%mİع㍿a/bfi𐞁fi३\r\n\r\n>'llHTTPServer'ſé<|fim_prefix|>…ſeZt0!! 漢\u000b\r\n½ \n.İ㋿fi漢 ", "tokens": 71, "pieces": ["#$%", "m", "İع", "㍿a", "/bfi𐞁fi", "३", "\r\n\r\n", ">'", "llHTTPServer", "'ſ", "e", "́<|", "fim", "_prefix", "|>", "…ſeZt", "0", "!!", " ", " 漢", "\u000b\r\n", "½", "", " \n", ".İ", "㋿fi漢", " "]} +{"text": "　(.", "tokens": 2, "pieces": ["　", "(."]} +{"text": "㋿'ll…३😀🏽ABCEOT½'Re,'M'VE'reZAb　'ſ'VE('ll.ꟲ\t/'Dé-é\"!", "tokens": 49, "pieces": ["㋿'", "ll", "…", "३", "😀🏽", "ABCEOT", "½", "'Re", ",'", "M", "'VE", "'re", "ZAb", "　", "'ſ", "'VE", "('", "ll", ".ꟲ", "\t", "/'", "De", "́<", "EOT", ">-", "é", "\"!"]} +{"text": "'VEDžunglaＡ'ſ'S'Meé/\r\n.<|endoftext|>Ⅳ>Ａs٣٤٥٦12345678½　 'ſå12345678' sſ", "tokens": 60, "pieces": ["'VE", "DžunglaＡ", "'ſ", "'S", "'M", "ee", "́/\r\n", ".<", "META", "_START", "><|", "endoftext", "|>", "Ⅳ", ">Ａs", "٣٤٥", "٦12", "345", "678", "½", "　", " ", "'ſ", "a", "̊", "123", "456", "78", "'", " ", " sſ"]} +{"text": "'s.'ll🙂ſ/!aB", "tokens": 11, "pieces": ["'s", ".'", "ll", "🙂ſ", "/!", "aB"]} +{"text": ">(𐞁EOT>aa ! !Ａ漢9å \n 'Dᵃå \n ", "tokens": 37, "pieces": [">(", "𐞁EOT", ">aa", " <", "META", "_START", ">!", " !", "Ａ漢", "9", "a", "̊", " \n", " '", "Dᵃa", "̊", " \n", " <", "META", "_START", ">"]} +{"text": "aiOS \n ‍‍DžacamelCaseecamelCase
camelCaseḍ̇㋿​a/b\rt \n  camelCase‍t<|fim_prefix|>", "tokens": 47, "pieces": ["aiOS", " \n", " ‍‍", "DžacamelCaseecamelCase", "
camelCaseḋ", "̣㋿​", "a", "/b", "\r", "t", " \n", " ", " camelCase", "‍t", "<|", "fim", "_prefix", "|>"]} +{"text": "́'sfiع<|fim_prefix|>(!!٣٤٥٦Ⅳ>ḍ̇Z", "tokens": 35, "pieces": ["́'", "sfiع", "<|", "fim", "_prefix", "|>(!!", "٣٤٥", "٦Ⅳ", ">ḋ", "̣Z"]} +{"text": "\r\n\r\néHTTPServer👍🏽\r\n\r\nacamelCase‍­,(0\rHTTPServer👍🏽\n/å0,ß \n Dž🙂s \n ㍿", "tokens": 48, "pieces": ["\r\n\r\n", "e", "́HTTPServer", "👍🏽\r\n\r\n", "acamelCase", "‍­,(", "0", "\r", "HTTPServer", "👍🏽\n", "/a", "̊", "0", ",ß", " \n", " Dž", "🙂s", " \n", " ㍿"]} +{"text": "­
m…'T٣٤٥٦Ⅳ!'S!", "tokens": 20, "pieces": ["­", "
m", "…", "'T", "٣٤٥", "٦Ⅳ", "!'", "S", "!"]} +{"text": "<|fim_prefix|>.\u000baB \n/Abß٣٤٥٦a/b
　　㍿å\tع>ß\r\niOS'll㍿EOT", "tokens": 46, "pieces": ["<|", "fim", "_prefix", "|>.", "\u000baB", " \n", "/Abß", "٣٤٥", "٦", "a", "/b", "
　", "　", "㍿a", "̊", "\tع", ">ß", "\r\n", "iOS", "'ll", "㍿EOT"]} +{"text": "9.Z<|fim_prefix|>😀🏽ᵃt'D", "tokens": 23, "pieces": ["9", ".Z", "<|", "fim", "_prefix", "|>😀🏽", "ᵃt", "'D", ""]} +{"text": "'D३(字'S9½/\r\n
\ré'T", "tokens": 14, "pieces": ["'D", "३", "(字", "'S", "9½", "/\r\n", "
\r", "é", "'T"]} +{"text": "'VE-fiAb𐞁,😀🏽 \nHTTPServerİᵃ/(B", "tokens": 25, "pieces": ["'VE", "-fiAb𐞁", ",😀🏽", " \n", "HTTPServerİᵃ", "/(", "B"]} +{"text": "'Reé½('ABCa'D/\r\t👍🏽 \nİfi<​\">ßEOTꟲ9ßsd🙂/'re\r\n\r\n<12345678", "tokens": 46, "pieces": ["'Re", "e", "́", "½", "('", "ABCa", "'D", "/\r", "\t", "👍🏽", " \n", "İfi", "<​\">", "ßEOTꟲ", "9", "ßsd", "🙂/'", "re", "\r\n\r\n", "<", "123", "456", "78", ""]} +{"text": "é\r\r\n\r0'ſ'#$%<|fim_prefix|>a", "tokens": 19, "pieces": ["e", "́\r\r\n\r", "0", "'ſ", "'#$%<|", "fim", "_prefix", "|>", "a"]} +{"text": "/\r\n\r\nDžungla'M'ReAb\n/'ll㋿
३٣٤٥٦ 'SaBZé\tEOT12345678ع\nEOTſ#$%éiOS-tsa/bDž(㍿!!0", "tokens": 62, "pieces": ["/\r\n\r\n", "Džungla", "'M", "'Re", "Ab", "\n", "/'", "ll", "㋿", "
", "३٣٤", "٥٦", " ", "'S", "aBZe", "́", "\tEOT", "", "123", "456", "78", "ع", "\n", "EOTſ", "#$%", "e", "́iOS", "-tsa", "/bDž", "(㍿!!", "0"]} +{"text": "fi𐞁é‍t.", "tokens": 11, "pieces": ["fi𐞁é", "‍t", "."]} +{"text": "-…#$%.'<Džungla'StEOTEOT9'T½(\u000b!!\n/e#$%iOS \n", "tokens": 26, "pieces": ["-", "…", "#$%.'<", "Džungla", "'S", "tEOTEOT", "9", "'T", "½", "(", "\u000b", "!!\n", "/e", "#$%", "iOS", " \n"]} +{"text": "\r\n!!HTTPServer\t\"­sd\"½'VE'D're<|fim_prefix|>!!-ع", "tokens": 27, "pieces": ["\r\n", "!!", "HTTPServer", "\t", "\"­", "sd", "\"", "½", "'VE", "'D", "'re", "<|", "fim", "_prefix", "|>!!-", "ع"]} +{"text": "😀🏽​­'ſ's.　\r\n\r\néßm 're", "tokens": 20, "pieces": ["😀🏽​­'", "ſ", "'s", ".", "　\r\n\r\n", "e", "́ßm", " ", "'re"]} +{"text": "Z0‍e", "tokens": 12, "pieces": ["Z", "", "0", "‍", "e"]} +{"text": "\n𐞁ꟲ'sa/bＡ >a/be12345678 \nＡ\r<|endoftext|>a\tDž.…d㋿", "tokens": 45, "pieces": ["\n", "𐞁ꟲ", "'s", "a", "/bＡ", " >", "a", "/be", "123", "456", "78", " \n", "Ａ", "\r", "<|", "endoftext", "|>", "a", "\tDž", ".", "…d", "㋿"]} +{"text": "🙂 \n iOS\"camelCaseＡ", "tokens": 9, "pieces": ["🙂", " \n", " iOS", "\"camelCaseＡ"]} +{"text": "camelCaseDž'Re \nAb字 a \n!!éZé!
 \n 'Ａ,/😀🏽!! 's٣٤٥٦ſ/ ٣٤٥٦", "tokens": 59, "pieces": ["camelCase", "Dž", "'Re", " \n", "Ab字", " a", " \n", "!!", "e", "́Zé", "!", "
 \n", " '", "Ａ", ",/😀🏽!!", " '", "s", "٣٤٥", "٦", "ſ", "/", " ", " ", "٣٤٥", "٦"]} +{"text": "/İ>‍漢 \nfi​ /\r\nß(‍👍🏽'DⅣعᵃ \n aDžungla　 \n s", "tokens": 40, "pieces": ["/İ", ">‍", "漢", " \n", "fi", "​", " ", "/\r\n", "ß", "(‍👍🏽'", "D", "Ⅳ", "عᵃ", " \n", " aDžungla", "　 \n", " s"]} +{"text": " /\r\nعHTTPServer३./\r\n9३'ſ 'VEAb<|fim_prefix|>Ａé '३\r\n३㋿\nB9\r\na \n​\" ,عfiEOT-​", "tokens": 57, "pieces": [" ", "/\r\n", "عHTTPServer", "३", "./\r\n", "9३", "'ſ", " ", "'VE", "Ab", "<|", "fim", "_prefix", "|>", "Ａe", "́", " ", " '", "३", "\r\n", "३", "㋿\n", "B", "9", "\r\n", "a", " \n", "​\"", " ", " ,", "عfiEOT", "-​"]} +{"text": "👍🏽\r\n٣٤٥٦'S<|endoftext|>fiABCᵃ𐞁,sa/b\u000b\r\n.Džéꟲᵃ🙂", "tokens": 53, "pieces": ["👍🏽\r\n", "٣٤٥", "٦", "'S", "<|", "endoftext", "|>", "fiABCᵃ𐞁", ",sa", "/b", "\u000b", "\r\n", ".Dže", "́ꟲᵃ", "🙂"]} +{"text": "ABC'ReaB \nfi !.'ll  -ꟲ\r\n\r\n
é\u000b//dZ👍🏽ḍ̇'Mßs　​t🙂 \na/b'SB-‍ ", "tokens": 54, "pieces": ["ABC", "'Re", "aB", " \n", "fi", " !.'", "ll", " ", " -", "ꟲ", "\r\n\r\n", "
é", "\u000b", "//", "dZ", "👍🏽", "ḋ", "̣'", "Mßs", "　", "​t", "🙂", " \n", "a", "/b", "'", "SB", "-‍", " "]} +{"text": "a/béZ\rDžunglaᵃ😀🏽 \n\u000b're\rABC", "tokens": 22, "pieces": ["a", "/be", "́Z", "\r", "Džunglaᵃ", "😀🏽", " \n", "\u000b", "'re", "\r", "ABC"]} +{"text": "\u000bé0㍿fiB'S\t­'ll'camelCase \n'M", "tokens": 19, "pieces": ["\u000bé", "0", "㍿fiB", "'S", "\t", "­'", "ll", "'camelCase", " \n", "'M"]} +{"text": "\n-", "tokens": 2, "pieces": ["\n", "-"]} +{"text": "字é😀🏽ABC'ſ \n \n…'T‍Dž‍a/b३m/!'T'T🙂HTTPServer½㋿", " \n", "…", "'T", "‍Dž", "‍a", "/b", "३", "m", "/!'", "T", "'T", "🙂HTTPServer", "½", "㋿<", "AbꟲiOS", "/\r\n", "e", "'re", " ", "㍿HTTPServers", "३", "/\r\n\r\n"]} +{"text": "<|endoftext|> 漢'T३/\r\nع('St字\r\n\r\nع㍿ꟲ٣٤٥٦é#$%漢'VEcamelCase", "tokens": 44, "pieces": ["<|", "endoftext", "|>", " 漢", "'T", "३", "/\r\n", "ع", "('", "St字", "\r\n\r\n", "ع", "㍿ꟲ", "٣٤٥", "٦", "e", "́#$%", "漢", "'VE", "camelCase"]} +{"text": " \n'Mfi字　㍿'a/ba/ḍ̇camelCasem\r \n!!'ſ!eå\r\n('ſå\t\r\n\r\n'D👍🏽\r\n\r\n\r\n\r\nAb", "tokens": 53, "pieces": [" \n", "'M", "fi字", "　", "㍿'", "a", "/ba", "/ḋ", "̣camelCasem", "\r \n", "!!'", "ſ", "!ea", "̊\r\n", "('", "ſa", "̊", "\t\r\n\r\n", "'D", "👍🏽\r\n\r\n\r\n\r\n", "Ab"]} +{"text": "\r\n\r\nİZ🙂'İİa/b", "tokens": 12, "pieces": ["\r\n\r\n", "İZ", "🙂'", "İİ", "a", "/b"]} +{"text": "ß9.½'Re字//\r\n🙂a/baBعfi½camelCase/\r\n", "tokens": 23, "pieces": ["ß", "9", ".", "½", "'Re", "字", "//\r\n", "🙂a", "/baBعfi", "½", "camelCase", "/\r\n"]} +{"text": "\u000bZİ", "tokens": 5, "pieces": ["\u000b", "Zİ"]} +{"text": ">HTTPServerDžungla-HTTPServer aBAbm!!", "tokens": 15, "pieces": [">HTTPServerDžungla", "-HTTPServer", " aBAbm", "!!"]} +{"text": "iOS\"Bḍ̇ttᵃᵃ12345678ḍ̇9İ३🙂İ‍/.ſ'VE'ſ'SiOS'T'ꟲ'‍mᵃ. s", "tokens": 60, "pieces": ["iOS", "\"Bḋ", "̣ttᵃᵃ", "123", "456", "78", "ḋ", "̣", "9", "İ", "३", "🙂İ", "‍/.", "ſ", "'VE", "'ſ", "'S", "iOS", "'T", "'ꟲ", "'‍", "mᵃ", ".", " s"]} +{"text": "字<|fim_prefix|>", "tokens": 8, "pieces": ["字", "<|", "fim", "_prefix", "|>"]} +{"text": "🙂 're İ'S­ſ\r\r\n\r\nś'ſ‍0 \nſ'Re ½(aB're'Reé-ꟲ'ReDžungla‍EOT… (", "tokens": 48, "pieces": ["🙂", " '", "re", " İ", "'S", "­ſ", "\r\r\n\r\n", "s", "́'", "ſ", "‍", "0", " \n", "ſ", "'Re", " ", "½", "(aB", "'re", "'Re", "e", "́-", "ꟲ", "'Re", "Džungla", "‍EOT", "…", " ", "("]} +{"text": "BcamelCase'HTTPServer", "tokens": 9, "pieces": ["BcamelCase", "'<", "META", "_START", ">HTTPServer"]} +{"text": "'ll'EOT\r٣٤٥٦ꟲa/b'VE\r\n<|endoftext|>'ſḍ̇<|fim_prefix|>'S''llAb　.'re٣٤٥٦ḍ̇", "tokens": 67, "pieces": ["'ll", "'EOT", "\r", "٣٤٥", "٦", "ꟲa", "/b", "'VE", "\r\n", "<|", "endoftext", "|>'", "ſḋ", "̣<|", "fim", "_prefix", "|><", "META", "_START", ">'", "S", "''", "llAb", "　", ".'", "re", "٣٤٥", "٦", "ḋ", "̣"]} +{"text": "#$% \n ३'reꟲ", "tokens": 10, "pieces": ["#$%", " \n", " ", "३", "'re", "ꟲ"]} +{"text": " \n's'Re\n/漢३​\t \n…", "tokens": 13, "pieces": [" \n", "'s", "'Re", "\n", "/漢", "३", "​", "\t \n…"]} +{"text": "Ⅳ㋿漢-İꟲᵃ\"aB
字#$%<|fim_prefix|>\r\n\r\n sİ/\r\n'll>'DEOT<­aDžunglaB,٣٤٥٦aAb(/å\t", "tokens": 61, "pieces": ["Ⅳ", "㋿漢", "-İꟲᵃ", "\"aB", "
字", "#$%<|", "fim", "_prefix", "|>\r\n\r\n", " sİ", "/\r\n", "'ll", ">'", "DEOT", "<­", "aDžunglaB", ",", "٣٤٥", "٦", "aAb", "(/", "a", "̊", "\t"]} +{"text": "EOT㍿\ré\"ſ9<|endoftext|>Džungla's'S'll٣٤٥٦fi<|fim_prefix|>Ⅳ\n­ 0camelCaseé12345678e ' \naB<|fim_prefix|> ＡåDžungla0㋿‍", "tokens": 83, "pieces": ["EOT", "㍿\r", "e", "́\"", "ſ", "9", "<|", "endoftext", "|>", "Džungla", "'s", "'S", "'ll", "٣٤٥", "٦", "fi", "<|", "fim", "_prefix", "|>", "Ⅳ", "\n", "­", " ", "0", "camelCaseé", "123", "456", "78", "e", " ", " '", " \n", "aB", "<|", "fim", "_prefix", "|>", " Ａa", "̊Džungla", "0", "㋿‍"]} +{"text": "👍🏽㍿\r\n\r\n\u000bé12345678㋿camelCaseⅣa/bZ<\r\n\r\n漢fi", "tokens": 63, "pieces": ["éDž", "\"", "123", "456", "78", "/\r\n", "　", "'M", "<|", "fim", "_prefix", "|>㋿", "camelCase", "Ⅳ", "a", "/bZ", "<\r\n\r\n", "漢fi", ""]} +{"text": "㋿字'VE!!Ⅳ'T­#$%㍿漢camelCase", "tokens": 17, "pieces": ["'re", "\r\n", " 𐞁t", "/\r\n", ">㍿", "漢camelCase"]} +{"text": " \n 0a<​EOT٣٤٥٦‍( \n'reiOS'Reꟲ(字'MAb\n/'re­,#$%tEOTé😀🏽 ㍿a/b", "tokens": 54, "pieces": [" \n", " ", " ", "0", "a", "<​", "EOT", "٣٤٥", "٦", "‍(", " \n", "'re", "iOS", "'Re", "ꟲ", "(字", "'M", "Ab", "\n", "/'", "re", "­,#$%", "tEOTe", "́😀🏽", " ", "㍿a", "/b"]} +{"text": "HTTPServerſ'T0", "tokens": 6, "pieces": ["HTTPServerſ", "'T", "0"]} +{"text": "'M😀🏽٣٤٥٦<́é'D 'ſ㋿ ½a/bᵃ'Re…Ⅳ-ZſaB,​‍'\r\naBEOT‍ᵃ0,'ſ", "tokens": 61, "pieces": ["'M", "😀🏽", "٣٤٥", "٦", "<́", "e", "́'", "D", " '", "ſ", "㋿", " ", " ", "½", "a", "/bᵃ", "'Re", "…", "Ⅳ", "-ZſaB", ",​‍'\r\n", "aBEOT", "‍ᵃ", "0", ",'", "ſ"]} +{"text": "B\n'Re🙂 ABC-B", "tokens": 8, "pieces": ["B", "\n", "'Re", "🙂", " ", " ABC", "-B"]} +{"text": "-camelCaseeⅣé.३字\"deع'll😀🏽漢३'aB'Da/b\"camelCase(عſm漢
", "tokens": 41, "pieces": ["-camelCasee", "Ⅳ", "é", ".", "३", "字", "\"deع", "'ll", "😀🏽", "漢", "३", "'aB", "'D", "a", "/b", "\"camelCase", "(عſm漢", "
"]} +{"text": "eAbé३'T12345678 \n", "tokens": 11, "pieces": ["eAbe", "́", "३", "'T", "123", "456", "78", " \n"]} +{"text": "Z12345678é\n/0 m İ​EOT>!\r\n\n/'ll‍-'s", "tokens": 24, "pieces": ["Z", "123", "456", "78", "é", "\n", "/", "0", " ", " m", " İ", "​EOT", ">!\r\n\n", "/'", "ll", "‍-'", "s"]} +{"text": "'å㋿/\r\nＡḍ̇dⅣ
d👍🏽!\tå<​३'İ ­,'re", "tokens": 46, "pieces": ["'a", "̊㋿/\r\n", "Ａḋ", "̣d", "Ⅳ", "
d", "👍🏽!", "\ta", "̊<<", "META", "_START", "><", "EOT", ">​", "३", "'İ", " ", " ­,'", "re"]} +{"text": "Ⅳ Džungla,'VEꟲ㍿", "tokens": 22, "pieces": ["Ⅳ", " Džungla", ",'", "VEꟲ", "㍿"]} +{"text": "'ſ३㍿\u000b'Re​'reꟲᵃ😀🏽0٣٤٥٦\r\nmm\nAb/\r\n'T½'Re<|fim_prefix|>عéعcamelCase.'ſ \r\n\r\n12345678ＡAb‍<|endoftext|>\nfi", "tokens": 81, "pieces": ["'", "ſ", "३", "㍿", "\u000b", "'Re", "​'", "reꟲᵃ", "😀🏽", "0٣٤", "٥٦", "\r\n", "mm", "\n", "Ab", "/\r\n", "'T", "½", "'Re", "<|", "fim", "_prefix", "|>", "عe", "́عcamelCase", ".<", "EOT", ">'", "ſ", " \r\n\r\n", "123", "456", "78", "ＡAb", "‍<|", "endoftext", "|>\n", "fi"]} +{"text": "\raBcamelCasetḍ̇aficamelCaseع'ſßfiع­٣٤٥٦t/　'Re
'VEaB'll'ſ \n EOT'aBdd \n Ａعa", "tokens": 58, "pieces": ["\r", "aBcamelCasetḋ", "̣aficamelCaseع", "'ſ", "ßfiع", "­", "٣٤٥", "٦", "t", "/", "　", "'Re", "
", "'VE", "aB", "'ll", "'ſ", " \n", " EOT", "'aBdd", " \n", " Ａعa"]} +{"text": "'s9\n/éſ🙂\n/漢عHTTPServer漢㍿iOSع<ß'ZdiOS.\r\n\r\n're　ḍ̇ḍ̇\tß\ns<|endoftext|>\n/'VEſ", "tokens": 57, "pieces": ["'s", "9", "\n", "/e", "́ſ", "🙂\n", "/漢عHTTPServer漢", "㍿iOSع", "<ß", "'ZdiOS", ".\r\n\r\n", "'re", "　ḋ", "̣ḋ", "̣", "\tß", "\n", "s", "<|", "endoftext", "|>\n", "/'", "VEſ"]} +{"text": "'sm'S'reع
\n\r­ᵃm٣٤٥٦३'ll>漢'T'llHTTPServeråſ!!　漢𐞁Ab's", "tokens": 47, "pieces": ["'s", "m", "'S", "'re", "ع", "
\n\r", "­ᵃm", "٣٤٥", "٦३", "'ll", ">漢", "'T", "'ll", "HTTPServera", "̊ſ", "!!", "　漢𐞁Ab", "'s"]} +{"text": "9a/b​½12345678 \n 123456780'M<|endoftext|>㋿'TDžungla\t🙂a/bDž \n\neDž<|endoftext|>ḍ̇!HTTPServers\r\nDžunglaع9
ꟲ/ \nDžungla'Mé", "tokens": 80, "pieces": ["9", "a", "/b", "​", "½12", "345", "678", " \n", " ", "123", "456", "780", "'M", "<|", "endoftext", "|>㋿'", "TDžungla", "\t", "🙂a", "/bDž", " \n\n", "e", "Dž", "<|", "endoftext", "|>", "ḋ", "̣!", "HTTPServers", "\r\n", "Džunglaع", "9", "
ꟲ", "/", " \n", "Džungla", "'M", "é"]} +{"text": "ع㍿́å'Ⅳ👍🏽aBms\r🙂Z​!<'VE9t㋿<", "tokens": 37, "pieces": ["ع", "㍿́<", "EOT", ">a", "̊'", "Ⅳ", "👍🏽", "aBms", "\r", "🙂Z", "​!<'", "VE", "9", "t", "㋿<"]} +{"text": "m/d <|endoftext|>'M<|endoftext|>", "tokens": 16, "pieces": ["m", "/d", " <|", "endoftext", "|>'", "M", "<|", "endoftext", "|>"]} +{"text": "'iOS\tDžungla!\r\n\"-!'VE\r\n\r\nEOT \n/\r\n\r\n>tع99漢\t'ſDžᵃ'aé's…'٣٤٥٦\r\n\r\n-fi0", "tokens": 51, "pieces": ["'iOS", "\tDžungla", "!\r\n", "\"-!'", "VE", "\r\n\r\n", "EOT", " \n", "/\r\n\r\n", ">tع", "99", "漢", "\t", "'ſ", "Džᵃ", "'ae", "́'", "s", "…", "'", "٣٤٥", "٦", "\r\n\r\n", "-fi", "0"]} +{"text": "m　'Re12345678a/bＡma0", "tokens": 12, "pieces": ["m", "　", "'Re", "123", "456", "78", "a", "/bＡma", "0"]} +{"text": " \nå<㋿aaB\n/ABCiOSaB-😀🏽'T\r<\n/fifi-३åḍ̇<|endoftext|>\r\nd 
'Re٣٤٥٦'re", "tokens": 70, "pieces": [" \n", "a", "̊<㋿<", "META", "_START", ">aaB", "\n", "/ABCiOSaB", "-😀🏽'", "T", "\r", "<\n", "/fifi", "-", "३", "a", "̊ḋ", "̣<|", "endoftext", "|>\r\n", "d", "", " ", "
", "'Re", "٣٤٥", "٦", "'re"]} +{"text": " \n ", "tokens": 2, "pieces": [" \n "]} +{"text": " \n ३a/b\r\nZiOSé字!!Dž", "tokens": 16, "pieces": [" \n", " ", "३", "a", "/b", "\r\n", "ZiOSe", "́字", "!!", "Dž"]} +{"text": "'M\"ＡſⅣİⅣ'ss12345678ſ'TaBaB0", "tokens": 23, "pieces": ["'M", "\"Ａſ", "Ⅳ", "İ", "Ⅳ", "'s", "s", "123", "456", "78", "ſ", "'T", "aBaB", "0"]} +{"text": "mع'll\nDž😀🏽 \n ..<|endoftext|>\t٣٤٥٦ \r\n\r\niOS's
!'TEOTEOTꟲ'> 'M😀🏽 /'Re​", "tokens": 64, "pieces": ["mع", "'", "ll", "\n", "Dž", "😀🏽", " \n", " <", "META", "_START", ">..<|", "endoftext", "|>", "\t", "٣٤٥", "٦", " ", " <", "META", "_START", ">\r\n\r\n", "iOS", "'s", "
", "!'", "TEOTEOTꟲ", "'>", " ", "'M", "😀🏽", " ", " /'", "Re", "​"]} +{"text": "漢.٣٤٥٦a/baBDž>३ß<|fim_prefix|>漢Džungla- ><|fim_prefix|>9", "३", "ß", "<|", "fim", "_prefix", "|>", "漢Džungla", "-", " ", " ><|", "fim", "_prefix", "|>", "9", "fiå' \n fi/\r\nDž siOSḍ̇'VEé½'ll'ſ \n é'T\n \n'🙂ßé字'T字HTTPServerᵃ", "tokens": 69, "pieces": ["!!'", "Tع", "\n", "/'", "s", "👍🏽\r\n", "<|", "endoftext", "|>", "fia", "̊'", " \n", " fi", "/\r\n", "Dž", " siOSḋ", "̣'", "VEe", "́", "½", "'ll", "'ſ", " \n", " e", "́'", "T", "\n \n", "'🙂", "ßé字", "'T", "字HTTPServerᵃ"]} +{"text": "'VEaB\n<​a/bHTTPServer!e\n/fiſ́🙂d", "tokens": 26, "pieces": ["'VE", "aB", "\n", "<​", "a", "/bHTTPServer", "!e", "\n", "/fiſ", "́🙂", "d"]} +{"text": "
AbHTTPServerAbＡ", "tokens": 8, "pieces": ["
AbHTTPServerAbＡ"]} +{"text": "‍٣٤٥٦‍<|fim_prefix|>漢'S.ſᵃ!Bt,B½
''VEB   'T0å\nAbcamelCaseꟲſ \n", "tokens": 55, "pieces": ["‍", "٣٤٥", "٦", "‍<|", "fim", "_prefix", "|>", "漢", "'S", ".ſᵃ", "!Bt", ",B", "½", "
", "''", "VEB", "  ", " ", "'T", "0", "a", "̊\n", "AbcamelCaseꟲſ", " \n"]} +{"text": "½fiB\n!!#$%㋿ssEOT/‍Dž", "tokens": 19, "pieces": ["½", "fiB", "\n", "!!#$%㋿", "ssEOT", "/‍", "Dž"]} +{"text": "aB (iOS\r\n㋿,\n<İ 👍🏽㋿Džع\r\ns㍿ع'ſ'reİßⅣ­", "tokens": 41, "pieces": ["aB", " (", "iOS", "\r\n", "㋿,\n", "<İ", " 👍🏽㋿", "Dž", "ع", "\r\n", "s", "㍿ع", "'ſ", "'re", "İß", "Ⅳ", "­"]} +{"text": "'\"éaaB٣٤٥٦fi-…>\u000b½/\r\n'S\r\n\r\n/\r\n!!Ⅳ'TmiOSＡ　'VE ꟲ\"ع", "tokens": 45, "pieces": ["'\"", "e", "́aaB", "٣٤٥", "٦", "fi", "-", "…", ">", "\u000b", "½", "/\r\n", "'S", "\r\n\r\n", "/\r\n", "!!", "Ⅳ", "'T", "miOSＡ", "　", "'VE", " ", " ꟲ", "\"ع"]} +{"text": "​.ⅣHTTPServer　fi…­\nḍ̇'M<|endoftext|>٣٤٥٦'ll'M//㍿㋿ḍ̇Ⅳa\n/  \rß٣٤٥٦B!!𐞁0fi​😀🏽३", "tokens": 82, "pieces": ["​.", "Ⅳ", "HTTPServer", "　fi", "…", "­\n", "ḋ", "̣'", "M", "<|", "endoftext", "|>", "٣٤٥", "٦", "'ll", "'M", "//㍿㋿", "ḋ", "̣", "Ⅳ", "a", "\n", "/", "  \r", "ß", "٣٤٥", "٦", "B", "!!", "𐞁", "0", "fi", "​😀🏽", "३"]} +{"text": " e<|endoftext|>>🙂\r's'llAb åDžungla0", "tokens": 24, "pieces": [" ", " e", "<|", "endoftext", "|>>🙂\r", "'s", "'ll", "Ab", " ", " a", "̊Džungla", "0"]} +{"text": "'ſ‍👍🏽\r\n\r\nmHTTPServer\r\nAb", "tokens": 17, "pieces": ["'ſ", "‍👍🏽\r\n\r\n", "mHTTPServer", "\r\n", "Ab"]} +{"text": "　‍\"d'D
'M　å𐞁's.ḍ̇/\r\n३'½/ꟲB­fi/'Re'Re-\u000b/\r\n", "tokens": 43, "pieces": ["　", "‍\"", "d", "'D", "
", "'M", "　a", "̊𐞁", "'s", ".ḋ", "̣/\r\n", "३", "'", "½", "/ꟲB", "­fi", "/'", "Re", "'Re", "-", "\u000b", "/\r\n"]} +{"text": "EOT​é ́\r \n Dž,🙂\néꟲ　iOS'Re", "tokens": 24, "pieces": ["EOT", "​e", "́", " ", " ́\r", " \n", " Dž", ",🙂\n", "éꟲ", "　iOS", "'Re"]} +{"text": "İＡ'٣٤٥٦字\taBḍ̇𐞁‍\u000b'M\n/\r\n'½(#$%ß", "tokens": 36, "pieces": ["İＡ", "'", "٣٤٥", "٦", "字", "\taBḋ", "̣𐞁", "‍", "\u000b", "'M", "\n", "/\r\n", "'", "½", "(#$%", "ß"]} +{"text": " \n B#$%́\r\n a/b­ \nm\u000b　fi", "tokens": 16, "pieces": [" \n", " B", "#$%́\r\n", " a", "/b", "­", " \n", "m", "\u000b", "　fi"]} +{"text": "Dž👍🏽
're'Rea/bBᵃ­Dž​m𐞁-0--aABC😀🏽mta/bfi \n ḍ̇ ½३éd ", "tokens": 59, "pieces": ["Dž", "👍🏽", "
", "'re", "'Re", "a", "/bBᵃ", "­Dž", "​m𐞁", "-", "0", "--", "aABC", "😀🏽", "mta", "/bfi", " \n", " ḋ", "̣", " ", "½", "", "३", "e", "́d", " "]} +{"text": "㍿s9iOSſ- B!s🙂e/0'\rع́éa/bEOT 字Džungla😀🏽'S", "tokens": 41, "pieces": ["㍿s", "9", "iOSſ", "-", " ", " B", "!s", "🙂e", "/", "0", "'\r", "ع", "́e", "́a", "/bEOT", " 字Džungla", "😀🏽'", "S"]} +{"text": "'iOS0 \r!!<|fim_prefix|>　12345678\r<|fim_prefix|>éEOT!!'M/\r\n'Dfi\r\n\r\nHTTPServer\n're", "tokens": 41, "pieces": ["'iOS", "0", " \r", "!!<|", "fim", "_prefix", "|>", "　", "123", "456", "78", "\r", "<|", "fim", "_prefix", "|>", "e", "́EOT", "!!'", "M", "/\r\n", "'D", "fi", "\r\n\r\n", "HTTPServer", "\n", "'re"]} +{"text": "😀🏽>aB>İ/'S12345678 <|endoftext|>
a/bＡa\t'VE<|fim_prefix|>'VE字fi३́'llé9ß<|endoftext|>Ⅳ<|endoftext|>\r\n\r\nß🙂'MEOT'M", "tokens": 73, "pieces": ["😀🏽>", "aB", ">İ", "/'", "S", "123", "456", "78", " <|", "endoftext", "|>", "
a", "/bＡa", "\t", "'VE", "<|", "fim", "_prefix", "|>'", "VE字fi", "३", "́'", "llé", "9", "ß", "<|", "endoftext", "|>", "Ⅳ", "<|", "endoftext", "|>\r\n\r\n", "ß", "🙂'", "MEOT", "'M"]} +{"text": "'smDž'VEİé", "tokens": 9, "pieces": ["'s", "mDž", "'VE", "İe", "́"]} +{"text": ">-aDžＡZ>漢\u000bع👍🏽'Mꟲ'Mté", "tokens": 26, "pieces": [">-", "aDžＡZ", ">漢", "\u000bع", "👍🏽'", "Mꟲ", "'M", "te", "́"]} +{"text": "㋿३éABC(!'T½\"👍🏽9/\r\n'VE's　#$%\ra/b
camelCase's​​ iOS🙂‍", "tokens": 46, "pieces": ["㋿", "३", "e", "́ABC", "(!'", "T", "½", "\"👍🏽", "9", "/\r\n", "'VE", "'s", "　", "#$%\r", "a", "/b", "
camelCase", "'s", "​​", " iOS", "🙂‍"]} +{"text": "ⅣAb ​'s٣٤٥٦ \u000b½½ꟲ/\r\n>", "tokens": 24, "pieces": ["Ⅳ", "Ab", " ", "​'", "s", "٣٤٥", "٦", " ", "\u000b", "½½", "ꟲ", "/\r\n", ">"]} +{"text": "🙂BABC㍿Ab'llZＡZ\t…>9字​", "tokens": 20, "pieces": ["🙂BABC", "㍿Ab", "'ll", "ZＡZ", "\t", "…", ">", "9", "字", "​"]} +{"text": " \n\r'Dſ'S", "tokens": 6, "pieces": [" \n\r", "'D", "ſ", "'S"]} +{"text": "a/b/\r\n🙂٣٤٥٦!.‍<|fim_prefix|>>  \nſDž\n/camelCaseå𐞁 ", "tokens": 45, "pieces": ["a", "/b", "/\r\n", "🙂<", "EOT", ">", "٣٤٥", "٦", "!.‍<|", "fim", "_prefix", "|>>", "  \n", "ſDž", "\n", "/camelCasea", "̊𐞁", " "]} +{"text": "9Džungla\u000b\u000b12345678İB'VE-'M漢'VE\r​😀🏽é'M٣٤٥٦d<|endoftext|>'S><|fim_prefix|>9 \n AbfiEOTDž
! \n 'M'T", "tokens": 73, "pieces": ["9", "Džungla", "\u000b", "\u000b", "123", "456", "78", "İB", "'VE", "-'", "M漢", "'VE", "\r", "​😀🏽", "é", "'M", "٣٤٥", "٦", "d", "<|", "endoftext", "|>'", "S", "><|", "fim", "_prefix", "|>", "9", " \n", " AbfiEOTDž", "
", "!", " \n", " '", "M", "'T", ""]} +{"text": "å­٣٤٥٦å.\"　ᵃa/b\"\n/½a/b😀🏽 \u000b \n's<|endoftext|>'s \n\u000b 9½‍­İ'㋿m
字🙂ts", "tokens": 67, "pieces": ["a", "̊­", "٣٤٥", "٦", "a", "̊.\"", "　ᵃa", "/b", "\"\n", "/", "½", "a", "/b", "😀🏽", " \u000b \n", "'s", "<|", "endoftext", "|>'", "s", " \n", "\u000b", " ", "9½", "‍­", "İ", "'㋿", "m", "", "
字", "🙂ts"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": ", ('M(\n/'re \n ٣٤٥٦\r\n<​字👍🏽\n \n 😀🏽a/bᵃ>'ſ<|fim_prefix|>漢́", "tokens": 55, "pieces": [",", " ", " ('", "M", "(\n", "/'", "re", " \n", " ", "٣٤٥", "٦", "\r\n", "<​", "字", "👍🏽\n", "", " \n", " 😀🏽", "a", "/bᵃ", ">'", "ſ", "<|", "fim", "_prefix", "|>", "漢", "́"]} +{"text": "<|endoftext|>'Sع½<३-Džungla'ScamelCase<|endoftext|>­\r<|fim_prefix|><|fim_prefix|>Ⅳ0-ABCHTTPServer('ll‍½-\n…\r\n\r\n字,\r\n\r\n<<|fim_prefix|>́0", "tokens": 76, "pieces": ["<|", "endoftext", "|>'", "S", "ع", "½", "<", "३", "-Džungla", "'S", "camelCase", "<|", "endoftext", "|>­\r", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "Ⅳ0", "-ABCHTTPServer", "('", "ll", "‍", "½", "-\n", "…\r\n\r\n", "字", ",\r\n\r\n", "<<|", "fim", "_prefix", "|>́", "0"]} +{"text": "<|endoftext|>…👍🏽\"'Re字Za/b ㍿　#$%<|fim_prefix|>ß👍🏽fi're.ſⅣ\n/३EOT,漢aع\r åſ‍👍🏽camelCase漢d\n!", "tokens": 82, "pieces": ["<|", "endoftext", "|>", "…", "👍🏽\"'", "Re字Za", "/b", " ", "㍿", "　", "#$%<|", "fim", "_prefix", "|>", "ß", "👍🏽", "fi", "'re", ".ſ", "Ⅳ", "\n", "/", "३", "EOT", ",漢aع", "\r", " a", "̊ſ", "‍👍🏽", "camelCase漢d", "\n", "!"]} +{"text": " <|endoftext|>\r\n'T12345678a/bß'Re\"½\u000b're/́dᵃ'S­😀🏽 漢'M\ncamelCase
ᵃ­ \n ٣٤٥٦ \n", "tokens": 56, "pieces": [" ", " <|", "endoftext", "|>\r\n", "'T", "123", "456", "78", "a", "/bß", "'Re", "\"", "½", "\u000b", "'re", "/́", "dᵃ", "'S", "­😀🏽", " 漢", "'M", "\n", "camelCase", "
ᵃ", "­", " \n", " ", "٣٤٥", "٦", " \n"]} +{"text": "Abå.!\n/Ab/\">\n/\r\nDžungla　!HTTPServerABC­!!<|fim_prefix|>é", "tokens": 34, "pieces": ["Aba", "̊.!\n", "/Ab", "/\">\n", "/\r\n", "Džungla", "　", "!HTTPServerABC", "­!!<|", "fim", "_prefix", "|>", "é"]} +{"text": "/\r\n'T' 12345678İDžunglaDž'ſBᵃ😀🏽!'TBBß\r\n\r\n#$%EOT<́漢字 ", "tokens": 41, "pieces": ["/\r\n", "'T", "'", " ", "123", "456", "78", "İDžunglaDž", "'ſ", "Bᵃ", "😀🏽!'", "TBBß", "\r\n\r\n", "#$%", "EOT", "<́", "漢字", " "]} +{"text": "-…12345678㋿'", "tokens": 10, "pieces": ["-", "…", "123", "456", "78", "㋿'"]} +{"text": "Džᵃ\".EOTABC'll'rea/b12345678\n<|fim_prefix|>'ll㍿Džungla字​-e…ᵃعiOSعḍ̇­'ll٣٤٥٦'s😀🏽Ab…", "tokens": 69, "pieces": ["Džᵃ", "\".", "EOTABC", "'ll", "'re", "a", "/b", "123", "456", "78", "\n", "<|", "fim", "_prefix", "|>'", "ll", "㍿Džungla字", "​-", "e", "…ᵃعiOSعḋ", "̣­'", "ll", "٣٤٥", "٦", "'s", "😀🏽", "Ab", "…"]} +{"text": " 9a‍ İ　👍🏽camelCaseİ'!ꟲᵃ-,tt'VEås\"'Re 'reᵃ…'s'S>", "tokens": 50, "pieces": [" ", " ", "9", "a", "‍", " İ", "　", "👍🏽", "camelCaseİ", "'!", "ꟲᵃ", "-,", "tt", "'VE", "a", "̊s", "\"<", "META", "_START", ">'", "Re", " ", " '", "reᵃ", "…", "'s", "'S", ">"]} +{"text": "Džunglaſ😀🏽>ᵃ'TcamelCaseⅣ \nⅣ", "tokens": 23, "pieces": ["Džunglaſ", "😀🏽>", "ᵃ", "'T", "camelCase", "Ⅳ", " \n", "Ⅳ"]} +{"text": "åt​ſⅣiOS　 \r\niOSEOTiOS'Re-\t'ſ<|endoftext|>\r(㍿ \n iOS'Re\n'ſ漢'T ᵃ \n ", "tokens": 51, "pieces": ["a", "̊t", "​ſ", "Ⅳ", "iOS", "　 \r\n", "iOSEOTiOS", "'Re", "-", "\t", "'ſ", "<|", "endoftext", "|>\r", "(㍿", " \n", " iOS", "'Re", "\n", "'ſ", "漢", "'T", " ᵃ", " \n "]} +{"text": "́३३Ab<🙂", "tokens": 12, "pieces": ["́", "३३", "Ab", "<🙂"]} +{"text": "'s'lls😀🏽/ \n 0å\u000b", "tokens": 16, "pieces": ["'s", "'ll", "s", "😀🏽/", " \n", " ", "0", "a", "̊", "\u000b"]} +{"text": " \u000bs<|endoftext|>'ll>́\nt,'M,\u000b​\r", "tokens": 21, "pieces": [" ", "\u000bs", "<|", "endoftext", "|>'", "ll", ">́\n", "t", ",'", "M", ",", "\u000b", "​\r"]} +{"text": " ́!!Z 'DHTTPServerm㍿9'D½/", "tokens": 17, "pieces": [" ", " ́!!", "Z", " ", "'D", "HTTPServerm", "㍿", "9", "'D", "½", "/"]} +{"text": "a/bᵃ½.'D/\r\n\n", "tokens": 10, "pieces": ["a", "/bᵃ", "½", ".'", "D", "/\r\n\n"]} +{"text": "'ſ½\"ḍ̇漢/.-t\t'DBe\"\u000bß \n ३ßᵃ\tſ9ꟲ́Ⅳ­(ß's\u000b𐞁́
#$%ᵃ字B", "tokens": 58, "pieces": ["'ſ", "½", "\"ḋ", "̣漢", "/.-", "t", "\t", "'D", "Be", "\"", "\u000bß", " \n", " ", "३", "ßᵃ", "\tſ", "9", "ꟲ", "́", "Ⅳ", "­(", "ß", "'s", "\u000b𐞁", "́", "
", "#$%", "ᵃ字B"]} +{"text": "'Ａß 'T…'Refit('llaB \naB́ \n /३\r
,\r\n\r\n're​camelCase", "tokens": 31, "pieces": ["'Ａß", " ", "'T", "…", "'Re", "fit", "('", "llaB", " \n", "aB", "́", " \n", " /", "३", "\r", "
", ",\r\n\r\n", "'re", "​camelCase"]} +{"text": "!!\n/漢İ३\"('VEfiꟲd㋿9iOS,'Ree>", "tokens": 25, "pieces": ["!!\n", "/漢İ", "३", "\"('", "VEfiꟲd", "㋿", "9", "iOS", ",'", "Ree", ">"]} +{"text": "\r\nZ's'D\rſ'VE<|fim_prefix|>'D \n >", "tokens": 19, "pieces": ["\r\n", "Z", "'s", "'D", "\r", "ſ", "'VE", "<|", "fim", "_prefix", "|>'", "D", " \n", " >"]} +{"text": "'re \n a'reİe's𐞁𐞁", "tokens": 21, "pieces": ["'re", " \n", " a", "'re", "İ", "e", "'s", "𐞁𐞁", ""]} +{"text": "'ſ mfié0
éᵃé\n/­12345678t‍mEOT#$%ꟲ\r३३३aaBZ-‍👍🏽ßB", "tokens": 58, "pieces": ["'ſ", " mfié", "0", "
éᵃe", "́\n", "/­", "123", "456", "78", "t", "‍mEOT", "#$%", "ꟲ", "\r", "३३३", "aaBZ", "-<", "META", "_START", ">‍👍🏽", "ßB"]} +{"text": "'ſ'Msß's🙂!/㍿'sEOTåfi'saBsaḍ̇ＡİꟲåDžungla0(‍HTTPServer", "tokens": 52, "pieces": ["'ſ", "'", "Msß", "'s", "🙂!/㍿'", "sEOTa", "̊fi", "'s", "aBsaḋ", "̣Ａİꟲa", "̊Džungla", "0", "(‍", "HTTPServer"]} +{"text": " \n !́ſAbå>-\rḍ̇👍🏽BHTTPServercamelCaseḍ̇'T,\tt-字", "tokens": 40, "pieces": [" \n", " !́", "ſAb", "a", "̊>-\r", "ḋ", "̣👍🏽", "BHTTPServercamelCaseḋ", "̣'", "T", ",", "\tt", "-字"]} +{"text": "EOTعiOŚDžungla㋿👍🏽字🙂½t0'ſßⅣB!!
<|endoftext|>Dž👍🏽🙂Džḍ̇ .", "tokens": 62, "pieces": ["EOTعiOS", "́Džungla", "㋿👍🏽", "字", "🙂", "½", "t", "0", "'", "ſß", "Ⅳ", "B", "!!", "
", "<|", "endoftext", "|>", "Dž", "👍🏽🙂", "Džḋ", "̣", " ", " ."]} +{"text": "Ａ<|fim_prefix|>B'ree㍿字ABC/\r\ncamelCase0a/\r\n漢\ta٣٤٥٦३", "tokens": 36, "pieces": ["Ａ", "<|", "fim", "_prefix", "|>", "B", "'re", "e", "㍿字ABC", "/\r\n", "camelCase", "0", "a", "/\r\n", "漢", "\ta", "٣٤٥", "٦३"]} +{"text": " aHTTPServer\r\n\n,ZaB'll ㋿\r㍿'VE Ａ's'VE12345678're😀🏽'S( \n ABC‍ſ\t", "tokens": 52, "pieces": [" aHTTPServer", "\r\n\n", ",ZaB", "'ll", "", " ", " ㋿\r", "㍿'", "VE", " <", "META", "_START", ">Ａ", "'s", "'VE", "123", "456", "78", "'re", "😀🏽'", "S", "(", " \n", " ABC", "‍ſ", "\t"]} +{"text": "-a/b/\r\n- \n !é!\n/­>.é​\" \n \n -漢'S\r'Re'D!Džungla.'M३", "tokens": 33, "pieces": ["-a", "/b", "/\r\n", "-", " \n", " !", "e", "́!\n", "/­>.", "e", "́​\"", " \n \n", " -", "漢", "'S", "\r", "'Re", "'D", "!Džungla", ".'", "M", "३"]} +{"text": "\r\n🙂-/İDžunglaꟲ​ \n're🙂\u000bＡ'M'éé३\naé0𐞁fi12345678#$%HTTPServer\n/'re½\"ß", "tokens": 54, "pieces": ["\r\n", "🙂<", "EOT", ">-/", "İDžunglaꟲ", "​", " \n", "'re", "🙂", "\u000bＡ", "'M", "'e", "́e", "́", "३", "\n", "aé", "0", "𐞁fi", "123", "456", "78", "#$%", "HTTPServer", "\n", "/'", "re", "½", "\"ß"]} +{"text": "e-,𐞁t'ſⅣ​a/b>'ll'ſ<|fim_prefix|>!e12345678\n٣٤٥٦\tⅣt字ABC'reⅣ'reée'\r\n'
EOTHTTPServer'iOS<|fim_prefix|><|endoftext|>B", "tokens": 76, "pieces": ["e", "-,", "𐞁t", "'ſ", "Ⅳ", "​a", "/b", ">'", "ll", "'ſ", "<|", "fim", "_prefix", "|>!", "e", "123", "456", "78", "\n", "٣٤٥", "٦", "\t", "Ⅳ", "t字ABC", "'re", "Ⅳ", "'re", "ée", "'\r\n", "'", "
EOTHTTPServer", "'iOS", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "B"]} +{"text": "-\n/ABC \n 👍🏽́́ \n \naB/\r\n👍🏽'Dſ'ſ३camelCase0٣٤٥٦‍siOSDž#$%'😀🏽<\rꟲ<|endoftext|>'VE \n'ſa'T0'Re", "tokens": 80, "pieces": ["-\n", "/ABC", " \n", " ", " 👍🏽́́", " \n \n", "aB", "/\r\n", "👍🏽'", "Dſ", "'ſ", "", "३", "camelCase", "0٣٤", "٥٦", "‍siOSDž", "#$%'😀🏽<\r", "ꟲ", "<|", "endoftext", "|>'", "VE", " \n", "'ſ", "a", "'T", "0", "'Re"]} +{"text": "'reeé٣٤٥٦>٣٤٥٦ABĆ'ſ--'VEt0<|fim_prefix|>\r\n\r\n​
\tDž<­fi字''s", "tokens": 50, "pieces": ["'re", "eé", "٣٤٥", "٦", ">", "٣٤٥", "٦", "ABC", "́'", "ſ", "--'", "VEt", "0", "<|", "fim", "_prefix", "|>\r\n\r\n", "​", "
", "\tDž", "<­", "fi字", "''", "s"]} +{"text": "Džungla'llABC㍿ 9. 𐞁ᵃmſ𐞁Z", "tokens": 32, "pieces": ["Džungla", "'", "llABC", "㍿", " ", "9", ".", " 𐞁ᵃmſ𐞁Z"]} +{"text": "Dž12345678\u000b\"/ß\r\n𐞁'reé½Ab\n/0're'ſ>tAbß­a/a/b/AbHTTPServer\u000b'S", "tokens": 41, "pieces": ["Dž", "", "123", "456", "78", "\u000b", "\"/", "ß", "\r\n", "𐞁", "'re", "é", "½", "Ab", "\n", "/", "0", "'re", "'ſ", ">tAbß", "­a", "/a", "/b", "/AbHTTPServer", "\u000b", "'S"]} +{"text": "Ⅳ \n#$%İＡ<|fim_prefix|>sDžt!​'re\nfi", "tokens": 26, "pieces": ["Ⅳ", " \n", "#$%", "İＡ", "<|", "fim", "_prefix", "|>", "sDžt", "!​'", "re", "\n", "fi"]} +{"text": "''Tfi㋿!!", "tokens": 27, "pieces": ["''", "Tfi", "㋿!!"]} +{"text": "9 \n𐞁 \u000b  é-ḍ̇ \n aB'S'lla'T\u000b👍🏽🙂ABC漢åİ<|fim_prefix|>.漢'Ms", "tokens": 51, "pieces": ["9", " \n", "𐞁", " \u000b  ", " é", "-ḋ", "̣", " \n", " aB", "'S", "'ll", "a", "'T", "\u000b", "👍🏽🙂", "ABC漢a", "̊İ", "<|", "fim", "_prefix", "|>.", "漢", "'M", "s"]} +{"text": "camelCase'M‍ݽ‍fi<|endoftext|>　㋿>aBs㍿\r\r\nع", "tokens": 30, "pieces": ["camelCase", "'M", "‍İ", "½", "‍fi", "<|", "endoftext", "|>", "　", "㋿>", "aBs", "㍿\r\r\n", "ع"]} +{"text": "ḍ̇a/b#$%👍🏽
ᵃteåＡ½émⅣ", "tokens": 56, "pieces": ["", "ḋ", "̣a", "/b", "#$%👍🏽", "
ᵃtea", "̊Ａ", "½", "ém", "Ⅳ"]} +{"text": "'ſ漢​\r\n!d", "tokens": 9, "pieces": ["'ſ", "漢", "​\r\n", "!d"]} +{"text": "\t…👍🏽ḍ̇!!", "tokens": 22, "pieces": ["\t", "", "…", "👍🏽", "ḋ", "̣<", "META", "_START", ">!!"]} +{"text": "\rDž \n s👍🏽's字-mع🙂👍🏽/‍!å12345678🙂\r\n\r\n…/'Reḍ̇ <|endoftext|>👍🏽<|fim_prefix|><|fim_prefix|>​,Bå", "tokens": 84, "pieces": ["\r", "Dž", " \n", " s", "👍🏽'", "s字", "-mع", "🙂👍🏽/‍!", "a", "̊", "123", "456", "78", "🙂\r\n\r\n", "…", "/'", "Reḋ", "̣", " ", " <|", "endoftext", "|>👍🏽<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>​,", "Ba", "̊"]} +{"text": "ſ<|endoftext|>a'S😀🏽'EOTe(😀🏽İEOT9éDž", "tokens": 33, "pieces": ["ſ", "<|", "endoftext", "|>", "a", "'S", "😀🏽'", "EOTe", "(😀🏽", "İEOT", "9", "éDž"]} +{"text": "\r\n\r\n\tḍ̇aB\r\u000b‍\u000b  ­a/b''S'Me12345678३ABC​…< te're\n/　", "tokens": 39, "pieces": ["\r\n\r\n", "\tḋ", "̣aB", "\r", "\u000b", "‍", "\u000b ", " ", "­a", "/b", "''", "S", "'M", "e", "123", "456", "78३", "ABC", "​", "…", "<", " ", " te", "'re", "\n", "/", "　"]} +{"text": "'re!!/ \n", "tokens": 4, "pieces": ["'re", "!!/", " \n"]} +{"text": "‍𐞁漢'ſ😀🏽\r\n\r\nḍ̇sß Džungla'M\n!/\r\n", "tokens": 33, "pieces": ["‍𐞁漢", "'ſ", "😀🏽\r\n\r\n", "ḋ", "̣sß", " Džungla", "'M", "\n", "!/\r\n"]} +{"text": "٣٤٥٦'D're'\t \n 'fi'TⅣ३EOT!#$%", "tokens": 25, "pieces": ["٣٤٥", "٦", "'D", "'re", "'", "\t \n", " '", "fi", "'T", "Ⅳ३", "EOT", "!#$%"]} +{"text": "‍e'T ३‍/\r\n", "tokens": 11, "pieces": ["‍e", "'T", " ", " ", "३", "‍/\r\n"]} +{"text": "🙂'㋿.Z'­>é𐞁​", "tokens": 18, "pieces": ["🙂'㋿.", "Z", "'­>", "e", "́𐞁", "​"]} +{"text": "iOS㍿", "tokens": 4, "pieces": ["iOS", "㍿"]} +{"text": "EOT‍Ⅳé'M0
عiOSé漢dcamelCase0", "tokens": 20, "pieces": ["EOT", "‍", "Ⅳ", "é", "'M", "0", "
عiOSé漢dcamelCase", "0"]} +{"text": "/🙂\t<|endoftext|><|endoftext|>‍Ab#$%​
ABC-ḍ̇𐞁HTTPServer
́\ré'Re\u000b½'ll#$%‍'VE　½!!<|fim_prefix|>!!é're<|endoftext|>漢'll", "tokens": 78, "pieces": ["/🙂", "\t", "<|", "endoftext", "|><|", "endoftext", "|>‍", "Ab", "#$%​", "
ABC", "-ḋ", "̣𐞁HTTPServer", "
", "́\r", "é", "'Re", "\u000b", "½", "'ll", "#$%‍'", "VE", "　", "½", "!!<|", "fim", "_prefix", "|>!!", "e", "́'", "re", "<|", "endoftext", "|>", "漢", "'ll"]} +{"text": "'VE ḍ̇é‍ḍ̇ /\r\n-\rDž字,ḍ̇'S٣٤٥٦\t<|endoftext|>­.d'RemHTTPServer \n ½ßfiåع　'Re<|fim_prefix|>ss'Sd𐞁ABC", "tokens": 82, "pieces": ["'VE", " ", " ḋ", "̣e", "́‍", "ḋ", "̣", " /\r\n", "-\r", "Dž字", ",ḋ", "̣'", "S", "٣٤٥", "٦", "\t", "<|", "endoftext", "|>­.", "d", "'Re", "mHTTPServer", " \n", " ", "½", "ßfia", "̊ع", "　", "'Re", "<|", "fim", "_prefix", "|>", "ss", "'S", "d𐞁ABC"]} +{"text": "😀🏽s'Re's'D>'reta/bꟲHTTPServerm,!​ \n", "tokens": 21, "pieces": ["😀🏽", "s", "'Re", "'s", "'D", ">'", "reta", "/bꟲHTTPServerm", ",!​", " \n"]} +{"text": "camelCase<|fim_prefix|><|endoftext|>'s٣٤٥٦EOT//HTTPServer'ReBDž ABCßİⅣDž", "tokens": 40, "pieces": ["camelCase", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "s", "٣٤٥", "٦", "EOT", "//", "HTTPServer", "'Re", "BDž", " ABCßİ", "Ⅳ", "Dž"]} +{"text": "éع‍e\r\ne٣٤٥٦'re३ßaéꟲ漢'D.‍Džſ𐞁­-12345678👍🏽ꟲḍ̇(Ⅳ/\r\n,/fi ß½
", "tokens": 69, "pieces": ["éع", "‍e", "\r\n", "e", "٣٤٥", "٦", "'re", "३", "ßaéꟲ漢", "'D", ".‍", "Džſ𐞁", "­-", "123", "456", "78", "👍🏽", "ꟲḋ", "̣(", "Ⅳ", "/\r\n", ",/", "fi", " ß", "½", "
"]} +{"text": ",\r\n\r\nDžAb­\r\n\u000b12345678漢 \n .㍿́Džéᵃ's👍🏽,dé<|endoftext|>/
ßḍ̇<|fim_prefix|><|endoftext|>\nm'ſ \n ", "tokens": 73, "pieces": [",\r\n\r\n", "DžAb", "­\r\n", "\u000b", "123", "456", "78", "漢", " \n", " .㍿́", "Dže", "́<", "META", "_START", ">ᵃ", "'s", "👍🏽,", "de", "́<|", "endoftext", "|>/", "
ßḋ", "̣<|", "fim", "_prefix", "|><|", "endoftext", "|>\n", "m", "'ſ", " \n "]} +{"text": "\n/漢-İ9ꟲ're'Re<|endoftext|>#$%-㍿<|endoftext|>ABCfi.'llABCDžungla/\r\n'ſ'VEꟲ,漢,漢½Dž\n𐞁m…　<|endoftext|>ꟲ", "tokens": 81, "pieces": ["\n", "/漢", "-İ", "9", "ꟲ", "'re", "'Re", "<|", "endoftext", "|>#$%-㍿<|", "endoftext", "|>", "ABCfi", ".'", "llABCDžungla", "/\r\n", "'ſ", "'VE", "ꟲ", ",", "漢", ",漢", "½", "Dž", "\n", "𐞁m", "…", "　", "<|", "endoftext", "|>", "ꟲ"]} +{"text": "🙂\"'ſßcamelCase
-'s!\"", "tokens": 16, "pieces": ["🙂\"'", "ſßcamelCase", "", "
", "-'", "s", "!\""]} +{"text": "İ́Ⅳ​iOS're'٣٤٥٦㍿ \u000b\r\n\r\n‍'VE ", "tokens": 37, "pieces": ["Ⅳ", "'D", "a字", "<|", "endoftext", "|>", "Ⅳ", "​iOS", "'re", "'", "٣٤٥", "٦", "㍿", " \u000b\r\n\r\n", "‍'", "VE", " "]} +{"text": "ꟲa/b<|fim_prefix|>B\r字0<'s/\r\n!!𐞁\u000b\n/", "tokens": 27, "pieces": ["ꟲa", "/b", "<|", "fim", "_prefix", "|>", "B", "\r", "字", "0", "<'", "s", "/\r\n", "!!", "𐞁", "\u000b\n", "/"]} +{"text": "Ⅳ́\u000bEOTe'T s‍ \n'VE𐞁<|fim_prefix|>٣٤٥٦ABC0 \t/\r\n!/\r\n‍/ ㍿'s'sDž !!😀🏽", "tokens": 69, "pieces": ["a", "̊'", "ſ", "!", "Ⅳ३", ">", "\u000bEOTe", "'T", " s", "‍", " \n", "'VE", "𐞁", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ABC", "0", " ", "\t", "/\r\n", "!/\r\n", "‍/", " ", "㍿'", "s", "'s", "Dž", " ", "!!😀🏽"]} +{"text": "B'T0fié \né<|fim_prefix|>/‍iOSDž
", "tokens": 27, "pieces": ["B", "'T", "0", "fie", "́", " \n", "e", "́<|", "fim", "_prefix", "|>/‍", "iOSDž", "
"]} +{"text": "9字'VEe", "tokens": 9, "pieces": ["9", "字", "'VE", "e", ""]} +{"text": "\ra/b/'S'sfiİ'D's<|endoftext|>0​Ź३B ('M漢㍿ABC𐞁'Mع
'DＡſ'>/å㋿ᵃ", "tokens": 57, "pieces": ["\r", "a", "/b", "/'", "S", "'s", "fiİ", "'D", "'s", "<|", "endoftext", "|>", "0", "​Z", "́", "३", "B", " ('", "M漢", "㍿ABC𐞁", "'M", "ع", "
", "'D", "Ａſ", "'>/", "a", "̊㋿", "ᵃ"]} +{"text": "'s'Td9 🙂åꟲéé३Ⅳ३12345678'Re\u000b\r\n \nZ\n/!!'llḍ̇camelCase", "tokens": 42, "pieces": ["'s", "'T", "d", "9", " ", " 🙂", "a", "̊ꟲe", "́e", "́", "३Ⅳ३", "123", "456", "78", "'Re", "\u000b\r\n \n", "Z", "\n", "/!!'", "llḋ", "̣camelCase"]} +{"text": "fiZDžungla­<'re, !aB(Ａ\r\n\t ٣٤٥٦😀🏽‍", "tokens": 38, "pieces": ["fiZDžungla", "­<'", "re", ",", " !", "aB", "(Ａ", "\r\n", "\t", "", " ", "٣٤٥", "٦", "😀🏽‍"]} +{"text": "(a/bmßa/baB'aBé<|fim_prefix|>Džungla / (.'sDž(>iOS…𐞁½ABC.\r\nfia/b!!iOS‍9 \r\n\r\nḍ̇　", "tokens": 61, "pieces": ["(a", "/bmßa", "/b", "aB", "'aBe", "́<|", "fim", "_prefix", "|>", "Džungla", " ", " /", " ", " (.'", "sDž", "(>", "iOS", "…𐞁", "½", "ABC", ".\r\n", "fia", "/b", "!!", "iOS", "‍", "9", " \r\n\r\n", "ḋ", "̣", "　"]} +{"text": "​𐞁\n/ \n's\r\n\r\nmḍ̇'s\tＡİß\"Ab#$%'ll👍🏽's㋿㋿\n!!　d/عfi\n\"'re\rḍ̇
0", "tokens": 72, "pieces": ["​𐞁", "\n", "/<", "EOT", ">", " \n", "'s", "\r\n\r\n", "mḋ", "̣'", "s", "\t", "Ａİß", "\"Ab", "#$%'", "ll", "👍🏽'", "s", "㋿㋿\n", "!!", "　d", "/عfi", "\n", "\"'", "re", "\r", "ḋ", "̣", "
", "0"]} +{"text": "ſſEOT漢३t٣٤٥٦Ab-ſ́<|fim_prefix|>'ſiOS\"'ll\n\r\n\r\nſعaB'DiOS㍿🙂", "tokens": 53, "pieces": ["ſſEOT漢", "३", "t", "٣٤٥", "٦", "Ab", "-ſ", "́<|", "fim", "_prefix", "|>'", "ſiOS", "\"'", "ll", "\n\r\n\r\n", "ſعaB", "'D", "iOS", "㍿🙂"]} +{"text": "३!!a!Džunglaꟲ\r\nDžungla'…e½é漢ſé'ſEOTéİ'VE>Abꟲ \n\r\n\r\ńＡiOS㋿👍🏽t'ſ", "tokens": 64, "pieces": ["३", "!!", "a", "!Džunglaꟲ", "\r\n", "Džungla", "'", "…e", "½", "é", "漢ſé", "'ſ", "EOTéİ", "'VE", ">Abꟲ", " \n\r\n\r\n", "́ＡiOS", "㋿👍🏽", "t", "'ſ"]} +{"text": "'re🙂\"\naB!!EOT\"åaBiOSḍ̇<-字ع/\r\nDž​é'ſꟲع\nꟲEOTBt ", "tokens": 44, "pieces": ["'re", "🙂\"\n", "aB", "!!", "EOT", "\"a", "̊aBiOSḋ", "̣<-", "字ع", "/\r\n", "Dž", "​é", "'ſ", "ꟲع", "\n", "ꟲEOTBt", " "]} +{"text": "​é 字३é9é \n é\n/-'D", "tokens": 22, "pieces": ["​é", " <", "META", "_START", ">", " ", " 字", "३", "e", "́", "9", "e", "́", " \n", " e", "́\n", "/-'", "D"]} +{"text": "\r\n٣٤٥٦>

\rEOTme Z<|endoftext|>/ḍ̇㍿字 \n'T", "tokens": 37, "pieces": ["\r\n", "٣٤٥", "٦", ">", "

\r", "EOTme", " Z", "<|", "endoftext", "|>/", "ḋ", "̣㍿", "字", " \n", "'T"]} +{"text": " İ\r\n\r\nHTTPServerAb\n/-/'ll㍿ß😀🏽EOT ́㋿'M㋿㋿m<|endoftext|>sé½\r\n…", "tokens": 49, "pieces": [" ", " İ", "\r\n\r\n", "HTTPServerAb", "\n", "/-/'", "ll", "㍿ß", "😀🏽", "EOT", " ", " ́㋿'", "M", "㋿㋿", "m", "<|", "endoftext", "|>", "sé", "½", "\r\n…"]} +{"text": "'M!Dž㋿\n\n/İ ‍iOS'MᵃDžungla!!tß\na\u000b\n\nß -'VE\n/fiaBé9å<|fim_prefix|>12345678'ſé", "tokens": 60, "pieces": ["'M", "!Dž", "㋿\n\n", "/İ", " ", "‍iOS", "'M", "ᵃDžungla", "!!", "tß", "\n", "a", "\u000b\n\n", "ß", " ", " -'", "VE", "\n", "/fiaBé", "9", "a", "̊<|", "fim", "_prefix", "|>", "123", "456", "78", "'ſ", "e", "́"]} +{"text": ",ᵃcamelCase", "tokens": 6, "pieces": [",ᵃcamelCase"]} +{"text": "Ab
-ꟲ'D'\u000b>­#$%t ㍿iOS'M iOS\r\n\"🙂iOS👍🏽-ßa/b0　𐞁#$%\t", "tokens": 52, "pieces": ["Ab", "
", "-ꟲ", "'D", "'", "\u000b", "><", "META", "_START", ">­#$%", "t", " ㍿", "iOS", "'M", " iOS", "\r\n", "\"🙂", "iOS", "👍🏽-", "ßa", "/b", "0", "　", "𐞁", "#$%", "\t"]} +{"text": "ᵃ\r\n­३ZHTTPServerABC\n/<|endoftext|>", "tokens": 19, "pieces": ["ᵃ", "\r\n", "­", "३", "ZHTTPServerABC", "\n", "/<|", "endoftext", "|>"]} +{"text": "d३>'Re-\u000b", "tokens": 7, "pieces": ["d", "३", ">'", "Re", "-", "\u000b"]} +{"text": "9　㋿", "tokens": 5, "pieces": ["9", "　", "㋿"]} +{"text": "\u000bB\t㋿/s", "tokens": 8, "pieces": ["\u000bB", "\t", "㋿/", "s"]} +{"text": "ꟲ😀🏽Dž/ABCaBＡßiOS'Rea", "tokens": 20, "pieces": ["ꟲ", "😀🏽", "Dž", "/ABCaBＡßiOS", "'Re", "a"]} +{"text": "éḍ̇<٣٤٥٦HTTPServer,, ", "tokens": 19, "pieces": ["éḋ", "̣<", "٣٤٥", "٦", "HTTPServer", ",,", " "]} +{"text": "'T
İ'VEⅣß", "tokens": 9, "pieces": ["'T", "
İ", "'VE", "Ⅳ", "ß"]} +{"text": "12345678DžAb\u000be(.\tABCiOSß​ꟲå漢'll­<|fim_prefix|>­é㋿ \n ", "tokens": 38, "pieces": ["123", "456", "78", "DžAb", "\u000be", "(.", "\tABCiOSß", "​ꟲa", "̊漢", "'ll", "­<|", "fim", "_prefix", "|>­", "é", "㋿", " \n "]} +{"text": "ꟲ ع\n<|endoftext|> \n 'S'VE#$%'T👍🏽/\r\né", "tokens": 35, "pieces": ["ꟲ", " ", " ع", "\n", "<|", "endoftext", "|>", " \n", " '", "S", "'VE", "#$%'", "T", "👍🏽/\r\n", "<", "EOT", ">e", "́"]} +{"text": "\n/Z \n ", "tokens": 4, "pieces": ["\n", "/Z", " \n "]} +{"text": "!\n/'Reå😀🏽\r\n\n/Ⅳ'Me字fi'VEſ漢 \n!a/bDž­🙂.𐞁-", "tokens": 41, "pieces": ["!\n", "/'", "Rea", "̊😀🏽\r\n\n", "/", "Ⅳ", "'M", "e字fi", "'VE", "ſ漢", " \n", "!a", "/bDž", "­🙂.", "𐞁", "-"]} +{"text": "\rEOT​Z<|endoftext|>́#$%/", "tokens": 15, "pieces": ["\r", "EOT", "​Z", "<|", "endoftext", "|>́#$%/"]} +{"text": "-字‍\r‍!­'re३ß½a/b'٣٤٥٦0iOSＡ🙂ḍ̇ꟲ㍿camelCase/🙂ḍ̇‍'ll\tſḍ̇>'re३-a　/\r\n", "tokens": 78, "pieces": ["-字", "‍\r", "‍!­'", "re", "३", "ß", "½", "a", "/b", "'", "٣٤٥", "٦0", "iOSＡ", "🙂ḋ", "̣ꟲ", "㍿camelCase", "/🙂", "ḋ", "̣‍'", "ll", "\tſḋ", "̣>'", "re", "३", "-<", "META", "_START", ">a", "　", "/\r\n"]} +{"text": "😀🏽㋿­.éHTTPServer/\r\n9eABC!iOSİ'S12345678é'S-
  \n 's/​/-", "tokens": 36, "pieces": ["😀🏽㋿­.", "éHTTPServer", "/\r\n", "9", "eABC", "!iOSİ", "'S", "123", "456", "78", "é", "'S", "-", "
  \n", " '", "s", "/​/-"]} +{"text": "éAb\r\n/>BEOT!!sDžå0're\n'S\r\n\r\n\n/😀🏽 \n <|endoftext|>\u000b<'T'Re'TaBß \n<|endoftext|>/ \n㍿\u000b", "tokens": 58, "pieces": ["e", "́Ab", "\r\n", "/>", "BEOT", "!!", "sDža", "̊", "0", "'re", "\n", "'S", "\r\n\r\n\n", "/😀🏽", " \n", " <|", "endoftext", "|>", "\u000b", "<'", "T", "'Re", "'T", "aBß", " \n", "<|", "endoftext", "|>/", " \n", "㍿", "\u000b"]} +{"text": " 0,é'sé\r\n\r\nABC's\r\n\rⅣte!Ａ ㋿", "tokens": 23, "pieces": [" ", " ", "0", ",e", "́'", "se", "́\r\n\r\n", "ABC", "'s", "\r\n\r", "Ⅳ", "te", "!Ａ", " ", "㋿"]} +{"text": "ma/b字Džungla😀🏽B/\r\n🙂　td𐞁 \"#$%ABC'ssB\"'T\r\n'M", "tokens": 33, "pieces": ["ma", "/b字Džungla", "😀🏽", "B", "/\r\n", "🙂", "　td𐞁", " \"#$%", "ABC", "'s", "sB", "\"'", "T", "\r\n", "'M"]} +{"text": "٣٤٥٦0㍿ \n", "tokens": 13, "pieces": ["٣٤٥", "٦0", "㍿", " \n"]} +{"text": "'T\"å \n aBDžZ ́İ
'Re\r\n'S' 𐞁½\nå 漢s\t0\r\n<‍'ll'll٣٤٥٦㍿tHTTPServerfi", "tokens": 62, "pieces": ["'T", "\"a", "̊", " \n", " aBDžZ", " ́", "İ", "
", "'Re", "\r\n", "'S", "'", " ", " 𐞁", "½", "\n", "a", "̊", " 漢s", "\t", "0", "\r\n", "<<", "EOT", ">‍'", "ll", "'ll", "٣٤٥", "٦", "㍿tHTTPServerfi"]} +{"text": "İå㋿", "tokens": 7, "pieces": ["İa", "̊㋿"]} +{"text": "'VE're'ſſEOT\r\n\r\n", "tokens": 14, "pieces": ["'VE", "'re", "'", "ſſEOT", "\r\n\r\n"]} +{"text": "Bs ع>ſ\r\nd字漢", "tokens": 11, "pieces": ["Bs", " ع", ">ſ", "\r\n", "d字漢"]} +{"text": "Abḍ̇½mfi'retDž\r\ńss\r\n🙂", "tokens": 20, "pieces": ["Abḋ", "̣", "½", "mfi", "'re", "tDž", "\r\n", "́ss", "\r\n", "🙂"]} +{"text": "𐞁 \n ㋿😀🏽 \n aB0 \n EOTEOTꟲ áſſ٣٤٥٦​́! 'M\u000b　㍿", "tokens": 51, "pieces": ["𐞁", " \n", " ㋿😀🏽", " \n", " aB", "0", " \n", " EOTEOTꟲ", " a", "́ſſ", "٣٤٥", "٦", "​́!", " ", " '", "M", "\u000b", "　", "㍿"]} +{"text": "㍿('T\r<|endoftext|>,é'T'Std<…", "tokens": 20, "pieces": ["㍿('", "T", "\r", "<|", "endoftext", "|>,", "é", "'T", "'S", "td", "<", "…"]} +{"text": "(.'re're٣٤٥٦ \n 'sEOTⅣ'll㍿aBⅣ!'Re\n\t .", "tokens": 32, "pieces": ["(.'", "re", "'re", "٣٤٥", "٦", " \n", " '", "sEOT", "Ⅳ", "'ll", "㍿aB", "Ⅳ", "!'", "Re", "\n", "\t", " ."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ZaB/\r\nZmm​' 's'M!́'s'ſ\r\n👍🏽İAbfiع👍🏽Z'ReéHTTPServer( Z字ᵃ9½!!aaBAb\n/'re", "tokens": 58, "pieces": ["ZaB", "/\r\n", "Zmm", "​'", " ", " '", "s", "'M", "!́'", "s", "'ſ", "\r\n", "👍🏽", "İAbfiع", "👍🏽", "Z", "'Re", "éHTTPServer", "(", " ", " Z字ᵃ", "9½", "!!", "aaBAb", "\n", "/'", "re"]} +{"text": ">ß'S12345678>👍🏽Abᵃ!fi👍🏽<|endoftext|>\r12345678­a/b🙂\"sHTTPServer\u000b#$%/　ß\t", "tokens": 52, "pieces": [">ß", "'S", "123", "456", "78", ">👍🏽", "Abᵃ", "!fi", "👍🏽<|", "endoftext", "|>\r", "123", "456", "78", "­a", "/b", "🙂\"", "sHTTPServer", "\u000b", "#$%/", "　ß", "\t"]} +{"text": "t'!!Džungladß12345678", "tokens": 11, "pieces": ["t", "'!!", "Džungladß", "123", "456", "78"]} +{"text": " 'Re👍🏽 \n 'Dḍ̇ 'ſ'll \r\n\r\n\"🙂́'!!!", "tokens": 29, "pieces": [" ", " '", "Re", "👍🏽", " \n", " '", "Dḋ", "̣", " ", "'ſ", "'ll", " \r\n\r\n", "\"🙂́'!!!"]} +{"text": "fi<|endoftext|>ſ'så\r\n<|endoftext|>𐞁𐞁m½e𐞁d́½Ab/<|fim_prefix|>'Td'MaB0​(㍿'ſ'D'S'Re'Re­ſ́", "tokens": 76, "pieces": ["fi", "<|", "endoftext", "|>", "ſ", "'s", "a", "̊\r\n", "<|", "endoftext", "|>", "𐞁𐞁m", "½", "e𐞁", "d", "́", "½", "Ab", "/<|", "fim", "_prefix", "|>'", "Td", "'M", "aB", "0", "​(㍿'", "ſ", "'D", "'S", "'Re", "'Re", "­ſ", "́"]} +{"text": "aB\r\n\r\n㍿İ\r\n\r\n‍EOT \n d　Z\r\n\r\n… \n \n's'Re0\tİ\n/Ab'll字ß\n/'D", "tokens": 34, "pieces": ["aB", "\r\n\r\n", "㍿İ", "\r\n\r\n", "‍EOT", " \n", " d", "　Z", "\r\n\r\n… \n \n", "'s", "'Re", "0", "\tİ", "\n", "/Ab", "'ll", "字ß", "\n", "/'", "D"]} +{"text": "'D!12345678𐞁\r\n\r\na'DＡ'VE😀🏽'S(­
🙂mß\"٣٤٥٦ Džungla\u000b\"", "tokens": 47, "pieces": ["'D", "!", "123", "456", "78", "𐞁", "\r\n\r\n", "a", "'D", "Ａ", "'VE", "😀🏽'", "S", "(­", "
", "🙂mß", "\"", "٣٤٥", "٦", " Džungla", "\u000b", "\""]} +{"text": "ea/b're👍🏽عåtaB<|endoftext|>Ａs½㋿-Ab🙂,Zfi\n\r\n\r\n‍㍿'re<|fim_prefix|>­0\n/#$%İ're", "Ａs", "½", "㋿-", "Ab", "🙂,", "Zfi", "\n\r\n\r\n", "‍㍿'", "re", "<|", "fim", "_prefix", "|>­", "0", "\n", "/#$%", "İ", "'re", ",e're're", "tokens": 19, "pieces": [" ", " Abe", "́ᵃꟲ", "<|", "fim", "_prefix", "|>,", "e", "'re", "'re"]} +{"text": "　'llＡ<|endoftext|>Ab'll'sDž 'M#$%'D9#$%mᵃ/‍ſ'Re", "tokens": 37, "pieces": ["　", "'ll", "Ａ", "<|", "endoftext", "|>", "Ab", "'ll", "'s", "Dž", "", " ", "'M", "#$%'", "D", "9", "#$%", "mᵃ", "/‍", "ſ", "'Re"]} +{"text": "ꟲ\n9\r\n\r\né!'DiOS👍🏽//s\r'reᵃḍ̇'Ree(aAb'DᵃB\r 𐞁🙂!!", "tokens": 47, "pieces": ["ꟲ", "\n", "9", "\r\n\r\n", "é", "!'", "DiOS", "👍🏽//", "s", "\r", "'re", "ᵃḋ", "̣'", "Ree", "(aAb", "'D", "ᵃB", "\r", " 𐞁", "🙂!!"]} +{"text": "‍AbſcamelCase𐞁0\r\n\r\n12345678ḍ̇e!ḍ̇Bᵃ'Rea/b", "tokens": 35, "pieces": ["‍AbſcamelCase𐞁", "0", "\r\n\r\n", "123", "456", "78", "ḋ", "̣e", "!ḋ", "̣Bᵃ", "'Re", "a", "/b"]} +{"text": "'re…㋿​", "tokens": 7, "pieces": ["'re", "…", "㋿​"]} +{"text": " EOT're's'D'VEİ
s\t\u000b.'llDž,😀🏽", "tokens": 23, "pieces": [" EOT", "'re", "'s", "'D", "'VE", "İ", "
s", "\t", "\u000b", ".'", "llDž", ",😀🏽"]} +{"text": "👍🏽ḍ̇'D", "tokens": 13, "pieces": ["👍🏽", "ḋ", "̣'", "D"]} +{"text": "/Ａa/b 'M🙂é字😀🏽'ſ\t漢 字­😀🏽m३'D\u000bm​ 字'M\"ꟲ'M<|endoftext|> 'D<|endoftext|>å", "tokens": 69, "pieces": ["/Ａa", "/b", " ", "'M", "🙂e", "́字", "😀🏽'", "ſ", "\t漢", " ", " 字", "­😀🏽", "m", "३", "'", "D", "\u000bm", "​", " 字", "'M", "\"ꟲ", "'M", "<|", "endoftext", "|>", " '", "D", "<|", "endoftext", "|>", "a", "̊"]} +{"text": "é½\r\n\r\n\r\n<|fim_prefix|>́Dž३'TcamelCase", "tokens": 19, "pieces": ["e", "́", "½", "\r\n\r\n\r\n", "<|", "fim", "_prefix", "|>́", "Dž", "३", "'T", "camelCase"]} +{"text": "d٣٤٥٦'ſ\u000b​>३
'M", "tokens": 23, "pieces": ["d", "٣٤٥", "٦", "'ſ", "\u000b", "​>", "३", "
", "'M", ""]} +{"text": "\n'reع字BſéficamelCase👍🏽'ABC,9aB \n𐞁
\n/fi're", "tokens": 43, "pieces": ["\n", "'re", "ع字Bſe", "́ficamelCase", "👍🏽<", "META", "_START", ">'", "ABC", ",", "9", "aB", " \n", "𐞁", "
\n", "/fi", "'re"]} +{"text": "/\r\n's<|endoftext|>'s½…fia/b!!t'ſ9'\r\n é'SaBm\u000bZ'0é
<|fim_prefix|>d\rＡ㋿", "tokens": 55, "pieces": ["/\r\n", "'s", "<|", "endoftext", "|>'", "s", "½", "…fia", "/b", "!!", "t", "'", "ſ", "9", "'\r\n", " ", " e", "́'", "SaBm", "\u000bZ", "'", "0", "é", "
", "<|", "fim", "_prefix", "|>", "d", "\r", "Ａ", "㋿"]} +{"text": "ḍ̇漢fiḍ̇0--ꟲ\r​12345678ßİ　å<|fim_prefix|>DžunglaHTTPServer'Re\u000bAb👍🏽('e'T'T\n!Ａ㍿㍿­", "tokens": 67, "pieces": ["ḋ", "̣漢fiḋ", "̣", "0", "--", "ꟲ", "\r", "​", "123", "456", "78", "ßİ", "　a", "̊<|", "fim", "_prefix", "|>", "DžunglaHTTPServer", "'Re", "\u000bAb", "👍🏽('", "e", "'T", "'T", "\n", "!Ａ", "㍿㍿­"]} +{"text": "a/b-　𐞁\n/ t.fi e", "tokens": 16, "pieces": ["a", "/b", "-", "　𐞁", "\n", "/", " t", ".fi", " e"]} +{"text": "ᵃ!!'Re--. Z", "tokens": 10, "pieces": ["ᵃ", "!!'", "Re", "--.", " Z"]} +{"text": "İZDžungla㋿(å㍿iOS🙂aß३", "tokens": 23, "pieces": ["İZDžungla", "㋿(", "a", "̊㍿", "iOS", "🙂aß", "३"]} +{"text": "३å'M<|fim_prefix|>EOT'S
!!👍🏽é!　", "tokens": 29, "pieces": ["३", "a", "̊'", "M", "<|", "fim", "_prefix", "|>", "EOT", "'S", "
", "!!👍🏽", "é", "!", "　"]} +{"text": "­‍\"\n/EOT'll'D'llⅣ", "tokens": 11, "pieces": ["­‍\"\n", "/EOT", "'ll", "'D", "'ll", "Ⅳ"]} +{"text": "'\n‍'ll\n/t👍🏽 \n HTTPServer٣٤٥٦'T字🙂9\n", "tokens": 38, "pieces": ["'\n", "‍'", "ll", "\n", "/t", "👍🏽<", "EOT", ">", " \n", " <", "META", "_START", ">HTTPServer", "٣٤٥", "٦", "'T", "字", "🙂", "9", "\n"]} +{"text": "a/bİ\tİta/b'DcamelCase'll'ſ \n \r\n\r\n", "tokens": 16, "pieces": ["a", "/bİ", "\tİta", "/b", "'D", "camelCase", "'ll", "'ſ", " \n \r\n\r\n"]} +{"text": "\"́/ᵃ㋿​ >٣٤٥٦", "tokens": 23, "pieces": ["\"́/", "ᵃ", "㋿<", "EOT", ">​", " >", "٣٤٥", "٦"]} +{"text": "'VEABC's(e#$%<|endoftext|>𐞁'Da", "tokens": 20, "pieces": ["'VE", "ABC", "'s", "(e", "#$%<|", "endoftext", "|>", "𐞁", "'D", "a"]} +{"text": "t\r!!'sEOT'llع'T🙂😀🏽字​, ‍\r\n\r\nḍ̇\r\n\r\nB/'re<'ſm", "tokens": 47, "pieces": ["t", "\r", "!!'", "s", "EOT", "'ll", "ع", "'", "T", "🙂😀🏽", "字", "​,", " ", "‍<", "EOT", ">\r\n\r\n", "ḋ", "̣\r\n\r\n", "B", "/'", "re", "<'", "ſm"]} +{"text": "ع٣٤٥٦\r漢Ab<|fim_prefix|><|fim_prefix|>><|endoftext|> \n 
👍🏽d½㍿<'sZ'M'll./\r\n're é", "tokens": 61, "pieces": ["ع", "٣٤٥", "٦", "\r", "漢Ab", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>><|", "endoftext", "|>", " \n", " ", "
", "👍🏽", "d", "", "½", "㍿<'", "sZ", "'M", "'ll", "./\r\n", "'re", " ", " é"]} +{"text": "Dž'MAb", "tokens": 4, "pieces": ["Dž", "'M", "Ab"]} +{"text": "<\rꟲ٣٤٥٦a/bå'refiDžunglaᵃ0camelCaseḍ̇''ll'SⅣ'sⅣé<|endoftext|>/\r\n", "tokens": 54, "pieces": ["<\r", "ꟲ", "٣٤٥", "٦", "a", "/ba", "̊'", "refiDžunglaᵃ", "0", "camelCaseḋ", "̣''", "ll", "'S", "Ⅳ", "'s", "Ⅳ", "é", "<|", "endoftext", "|>/\r\n"]} +{"text": "/\r\n  -\u000b(<|fim_prefix|>emHTTPServer#$%ḿ½​#$%", "tokens": 23, "pieces": ["/\r\n", " ", " ", "-", "\u000b", "(<|", "fim", "_prefix", "|>", "emHTTPServer", "#$%", "m", "́", "½", "​#$%"]} +{"text": "é/0/\r\n!!a'll \nꟲ(Ab/٣٤٥٦'reå🙂<|fim_prefix|>'Re EOT'M\r\n\r\n/\r\n­ABC#$%ßHTTPServer'Deſ", "tokens": 54, "pieces": ["e", "́/", "0", "/\r\n", "!!", "a", "'ll", " \n", "ꟲ", "(Ab", "/", "٣٤٥", "٦", "'re", "a", "̊🙂<|", "fim", "_prefix", "|>'", "Re", " EOT", "'M", "\r\n\r\n", "/\r\n", "­ABC", "#$%", "ßHTTPServer", "'D", "eſ"]} +{"text": "'reABC'reHTTPServerHTTPServerfi
'M 'Tİ\"
'M…Ａ'S\ré٣٤٥٦\r\n\r\nfi­\r0Zᵃ́å!!'ſcamelCase.camelCase#$%>'Så", "tokens": 63, "pieces": ["'re", "ABC", "'re", "HTTPServerHTTPServerfi", "
", "'M", " ", "'T", "İ", "\"", "
", "'M", "…Ａ", "'S", "\r", "e", "́", "٣٤٥", "٦", "\r\n\r\n", "fi", "­\r", "0", "Zᵃ", "́a", "̊!!'", "ſcamelCase", ".camelCase", "#$%>'", "Sa", "̊"]} +{"text": "'t \n 'ſع'D½ع漢12345678­عå😀🏽😀🏽'", "tokens": 40, "pieces": ["'t", " \n", " '", "ſع", "'D", "½", "ع漢", "123", "456", "78", "­ع", "a", "̊😀🏽😀🏽'"]} +{"text": ".٣٤٥٦DžABC <😀🏽‍,B9\"Ａe, \n ", "tokens": 31, "pieces": [".", "٣٤٥", "٦", "DžABC", " ", " <😀🏽‍,", "B", "9", "\"Ａe", ",", " \n "]} +{"text": "\r\n\rße\u000bⅣ字-\r\n\r\n'Mḍ̇#$%12345678.३',ᵃ👍🏽då'ᵃ<-\r\n\r\nꟲ-'ſ's👍🏽(", "tokens": 56, "pieces": ["\r\n\r", "ße", "\u000b", "Ⅳ", "字", "-\r\n\r\n", "'M", "ḋ", "̣#$%", "123", "456", "78", ".", "३", "',", "ᵃ", "👍🏽", "da", "̊'", "ᵃ", "<-\r\n\r\n", "ꟲ", "-'", "ſ", "'s", "👍🏽("]} +{"text": "Båß\r!!ꟲ  ᵃ😀🏽😀🏽\"0-　'M", "tokens": 28, "pieces": ["Ba", "̊ß", "\r", "!!", "ꟲ", " ", " ᵃ", "😀🏽😀🏽\"", "0", "-", "　", "'M"]} +{"text": "'ReABCå'DＡa İ12345678/\r\n9", "tokens": 16, "pieces": ["'Re", "ABCa", "̊'", "DＡa", " İ", "123", "456", "78", "/\r\n", "9"]} +{"text": "'VEİßꟲ((​😀🏽<|fim_prefix|>ＡZ'Dß<|fim_prefix|>'ſ,'re#$%", "tokens": 42, "pieces": ["'VE", "İßꟲ", "((​😀🏽<|", "fim", "_prefix", "|>", "ＡZ", "'D", "ß", "<|", "fim", "_prefix", "|>'", "ſ", ",'", "re", "#$%"]} +{"text": ",\n \ndſ'D😀🏽#$%/\r\nss/e>字fia/b/Džungla\n/　🙂", "tokens": 32, "pieces": [",\n", " \n", "dſ", "'D", "😀🏽#$%/\r\n", "ss", "/e", ">字fia", "/b", "/Džungla", "\n", "/", "　", "🙂"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r​!!'resＡ…İ٣٤٥٦Ⅳꟲ''s½\tZ\"9३३\t\"İ\r\n\r\n'll(\n", "tokens": 39, "pieces": ["\r", "​!!'", "resＡ", "…İ", "٣٤٥", "٦Ⅳ", "ꟲ", "''", "s", "½", "\tZ", "\"", "9३३", "\t", "\"İ", "\r\n\r\n", "'ll", "(\n"]} +{"text": "så  \n ſⅣ/\r\n\u000b\"🙂're ㋿ \n - \n ZＡ Ab
d,🙂🙂'Re'Ma/b字<|fim_prefix|>'VEⅣ😀🏽\r\n\r\n\r\n<|fim_prefix|>İ\u000b<12345678, st", "tokens": 62, "pieces": ["'𐞁", "Z", "Ａ", " ", " Ab", "
d", ",🙂🙂'", "Re", "'M", "a", "/b字", "<|", "fim", "_prefix", "|>'", "VE", "Ⅳ", "😀🏽\r\n\r\n\r\n", "<|", "fim", "_prefix", "|>", "İ", "\u000b", "<", "123", "456", "78", ",", " ", " st"]} +{"text": "camelCasedåABCDž𐞁'VÉ'Mع😀🏽", "tokens": 24, "pieces": ["camelCaseda", "̊ABCDž𐞁", "'VE", "́'", "Mع", "😀🏽"]} +{"text": "Z\r\n😀🏽ABC'S .'Sddḍ̇'Res½'SiOS\nABC<|endoftext|>a/b12345678-İ \r\n\r\n👍🏽　ꟲ'T\"EOTé", "tokens": 58, "pieces": ["Z", "\r\n", "😀🏽", "ABC", "'S", " ", " .'", "Sddḋ", "̣'", "Res", "", "½", "'S", "iOS", "\n", "ABC", "<|", "endoftext", "|>", "a", "/b", "123", "456", "78", "-İ", " \r\n\r\n", "👍🏽", "　ꟲ", "'T", "\"EOTé"]} +{"text": "\tİB'ſ😀🏽<|endoftext|>İHTTPServer\r\nt٣٤٥٦'VE's㍿
🙂\r\n​'Re\n", "İHTTPServer", "\r\n", "t", "٣٤٥", "٦", "'VE", "'s", "㍿", "
", "🙂\r\n", "​'", "Re", "\n", "३字Džungla'S㋿'Re/…㍿tABCå'S<|fim_prefix|>३,́.aB'VE \n‍e​　𐞁", "tokens": 58, "pieces": ["", "३", "字Džungla", "'S", "㋿'", "Re", "/", "…", "㍿tABC", "a", "̊'", "S", "<|", "fim", "_prefix", "|>", "३", ",́.", "aB", "'VE", " \n", "‍e", "​", "　𐞁"]} +{"text": " \tZaBcamelCaset'ſ,ſ(mßå/12345678㍿½
Z'red", "tokens": 34, "pieces": [" ", "\t", "ZaBcamelCaset", "'ſ", ",ſ", "(mßa", "̊/", "123", "456", "78", "㍿", "½", "
Z", "'re", "d"]} +{"text": "'🙂camelCase<|fim_prefix|>", "tokens": 12, "pieces": ["'🙂", "camelCase", "<|", "fim", "_prefix", "|>"]} +{"text": "camelCaseḍ̇-ſ/\r\nåaB'ſ\u000b㍿㋿camelCase
'‍'.Z/\r\n-iOS#$%\t'S're㍿ḍ̇ fi‍\"", "tokens": 67, "pieces": ["camelCaseḋ", "̣-", "ſ", "/\r\n", "a", "̊<", "META", "_START", ">aB", "'ſ", "\u000b", "㍿㋿", "camelCase", "
", "'‍'.", "Z", "/\r\n", "-<", "META", "_START", ">iOS", "#$%", "\t", "'S", "'re", "㍿ḋ", "̣", " ", " fi", "‍\""]} +{"text": "aB's''  Ａ'/\r\nＡ'.", "tokens": 13, "pieces": ["aB", "'s", "''", " ", " Ａ", "'/\r\n", "Ａ", "'."]} +{"text": "'ſ  'llå‍ſABC'S😀🏽12345678-㋿Bm /\r\n…AbB㋿\r\n\r\n​é　'Ś‍\rDžungla字å", "tokens": 57, "pieces": ["'ſ", " ", " ", "'ll", "a", "̊‍", "ſABC", "'S", "😀🏽", "123", "456", "78", "-㋿", "Bm", " ", " /\r\n", "…AbB", "㋿\r\n\r\n", "​e", "́", "　", "'S", "́‍\r", "Džungla字a", "̊"]} +{"text": "'S\r\n​‍ 'så<|endoftext|>٣٤٥٦å!!\t/\r\n
Ａ!ḍ̇scamelCase(éꟲé½/\r\nꟲß<|endoftext|>å🙂\n/#$%ß", "tokens": 73, "pieces": ["'S", "\r\n", "​‍", " ", " '", "sa", "̊<|", "endoftext", "|>", "٣٤٥", "٦", "a", "̊!!", "\t", "/\r\n", "
Ａ", "!ḋ", "̣scamelCase", "(éꟲé", "½", "/\r\n", "ꟲß", "<|", "endoftext", "|>", "a", "̊🙂\n", "/#$%", "ß"]} +{"text": " \u000b/\r\n'll‍ſsß,३eéaB'ré'Sm㋿ /\r\n\n/Z😀🏽ſ\"iOS'㋿ᵃcamelCase𐞁 \n Ａa/bEOTB!", "tokens": 63, "pieces": [" ", "\u000b", "/\r\n", "'ll", "‍ſsß", ",", "३", "ee", "́aB", "'re", "́'", "Sm", "㋿", " ", "/\r\n\n", "/Z", "😀🏽", "ſ", "\"iOS", "'㋿", "ᵃcamelCase𐞁", " \n", " Ａa", "/bEOTB", "!"]} +{"text": "\n\taB,s9d Ⅳ\nع\n🙂…ABC…é/\r\nm㍿tt'ſ9é­#$%a's<|fim_prefix|>åABCعꟲḍ̇𐞁", "tokens": 61, "pieces": ["\n", "\taB", ",s", "9", "d", " ", "Ⅳ", "\n", "ع", "\n", "🙂", "…ABC", "…é", "/\r\n", "m", "㍿tt", "'ſ", "9", "e", "́­#$%", "a", "'s", "<|", "fim", "_prefix", "|>", "a", "̊ABCعꟲḋ", "̣𐞁"]} +{"text": "(iOS\r\n fi/'T字", "tokens": 15, "pieces": ["(iOS", "\r\n", "", " fi", "/'", "T字"]} +{"text": "𐞁'D'ſ🙂/­", "tokens": 19, "pieces": ["𐞁", "'", "D", "'ſ", "🙂<", "EOT", ">/­"]} +{"text": " åEOT camelCase­Ａ\r\nm\r\nd", "tokens": 16, "pieces": [" ", " a", "̊EOT", " ", " camelCase", "­Ａ", "\r\n", "m", "\r\n", "d"]} +{"text": "12345678'T'S,", "tokens": 6, "pieces": ["123", "456", "78", "'T", "'S", ","]} +{"text": "camelCase‍!'Re<|endoftext|>0Be('", "Re", "<|", "endoftext", "|>", "0", "Be", "(<", "m"]} +{"text": "İ'T \n/ABCm٣٤٥٦🙂ꟲ", "tokens": 19, "pieces": ["İ", "'T", " \n", "/ABCm", "٣٤٥", "٦", "🙂ꟲ"]} +{"text": "!0DžcamelCase٣٤٥٦a/bm 字a/b
ſ\r\n.'Ⅳ\r\n\r\n \na/b!İ\nＡ's'll<|fim_prefix|><|fim_prefix|>'Re'sꟲ<ꟲABCꟲEOT'D", "tokens": 73, "pieces": ["!", "0", "DžcamelCase", "٣٤٥", "٦", "a", "/bm", " ", " 字a", "/b", "", "
ſ", "\r\n", ".'", "Ⅳ", "\r\n\r\n \n", "a", "/b", "!İ", "\n", "Ａ", "'s", "'ll", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "Re", "'s", "ꟲ", "<ꟲABCꟲEOT", "'D"]} +{"text": " \n🙂!!", "tokens": 4, "pieces": [" \n", "🙂!!"]} +{"text": "m'VE 'll
.'Re<'ll m", "tokens": 17, "pieces": ["m", "'VE", " ", " '", "ll", "
", ".'", "Re", "<'", "ll", "", " ", " m"]} +{"text": "9d#$%Ⅳ!\t12345678𐞁\n/\r😀🏽12345678ع\ta,a/b'D!!漢#$%\"ſ", "tokens": 38, "pieces": ["9", "d", "#$%", "Ⅳ", "!", "\t", "123", "456", "78", "𐞁", "\n", "/\r", "😀🏽", "123", "456", "78", "ع", "\ta", ",a", "/b", "'D", "!!", "漢", "#$%\"", "ſ"]} +{"text": "e'M! \n\r\nſḍ̇'DDž́ABCABC Ａ'Ret'Sa's<ꟲݽs", "tokens": 34, "pieces": ["e", "'M", "!", " \n\r\n", "ſḋ", "̣'", "DDž", "́ABCABC", " Ａ", "'Re", "t", "'S", "a", "'s", "<ꟲİ", "½", "s"]} +{"text": "<|endoftext|>　/-🙂<", "tokens": 12, "pieces": ["<|", "endoftext", "|>", "　", "/-🙂<"]} +{"text": "s\u000b", "tokens": 2, "pieces": ["s", "\u000b"]} +{"text": " \n 'D'reİ𐞁a/b <|fim_prefix|>½́dm<|endoftext|>字\n/é0fi字\u000bHTTPServer漢'T ­🙂/9å ᵃ\u000b", "tokens": 61, "pieces": [" \n", " '", "D", "'re", "İ𐞁a", "/b", " ", "<|", "fim", "_prefix", "|>", "½", "́dm", "<|", "endoftext", "|>", "字", "\n", "/é", "0", "fi字", "\u000bHTTPServer漢", "'T", " ", " <", "EOT", ">­🙂/", "9", "a", "̊", " ᵃ", "\u000b"]} +{"text": "‍­", "tokens": 3, "pieces": ["‍­"]} +{"text": "iOS٣٤٥٦\r\n\r\n\r,\r \u000b's٣٤٥٦ßß­ \nABC漢😀🏽🙂'VE字0३a'sḍ̇\u000bHTTPServera३字\r\n\r\n<|endoftext|>
😀🏽٣٤٥٦'D㍿", "tokens": 85, "pieces": ["iOS", "٣٤٥", "٦", "\r\n\r\n\r", ",\r", " ", "\u000b", "'s", "٣٤٥", "٦", "ßß", "­", " \n", "ABC漢", "😀🏽🙂'", "VE字", "0३", "a", "'s", "ḋ", "̣", "\u000bHTTPServera", "३", "字", "\r\n\r\n", "<|", "endoftext", "|>", "
", "😀🏽", "٣٤٥", "٦", "'D", "㍿"]} +{"text": "\n/<|fim_prefix|>EOT 'M'D٣٤٥٦EOT.Ａ'DⅣ\t!!< \n/d<0,👍🏽<|endoftext|>t-BaBZABC", "tokens": 56, "pieces": ["\n", "/<|", "fim", "_prefix", "|>", "EOT", " '", "M", "'D", "٣٤٥", "٦", "EOT", ".Ａ", "'D", "Ⅳ", "\t", "!!<", " \n", "/d", "<", "0", ",👍🏽<|", "endoftext", "|>", "t", "-BaBZABC"]} +{"text": "EOTdsa/b🙂\r \n\r\n\r\n0'M", "tokens": 11, "pieces": ["EOTdsa", "/b", "🙂\r", " \n\r\n\r\n", "0", "'M"]} +{"text": "é​𐞁/\r\n-,fiＡ٣٤٥٦<|endoftext|>m​é\r>-d'Re'ſ>aBDž'\nABC", "tokens": 44, "pieces": ["é", "​𐞁", "/\r\n", "-,", "fiＡ", "٣٤٥", "٦", "<|", "endoftext", "|>", "m", "​e", "́\r", ">-", "d", "'Re", "'ſ", ">aBDž", "'\n", "ABC"]} +{"text": "ſ>'sḿ'rea👍🏽\"iOS12345678", "tokens": 18, "pieces": ["ſ", ">'", "sm", "́'", "rea", "👍🏽\"", "iOS", "123", "456", "78"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ＡcamelCase", "tokens": 4, "pieces": ["ＡcamelCase"]} +{"text": "Zs😀🏽\r\nd-09 !\r0EOT'McamelCase\u000b­३\r\n'Re \n <|fim_prefix|>👍🏽é", "tokens": 43, "pieces": ["Zs", "😀🏽\r\n", "d", "-", "09", "", " ", " !\r", "0", "EOT", "'M", "camelCase", "\u000b", "­", "३", "\r\n", "'Re", " \n", " <|", "fim", "_prefix", "|>👍🏽", "é"]} +{"text": "Z9Ａfi٣٤٥٦\r…å३", "tokens": 22, "pieces": ["Z", "9", "Ａfi", "٣٤٥", "٦", "\r", "…a", "̊", "३"]} +{"text": "#$%　12345678'll ३'s<|fim_prefix|><|endoftext|>𐞁
'RefiHTTPServerſ \n Bİ​ ‍åſa'T12345678eå>'re
\u000b\n/ /\r\n", "tokens": 67, "pieces": ["#$%", "　", "123", "456", "78", "'ll", " ", "३", "'s", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "𐞁", "
", "'Re", "fiHTTPServerſ", " \n", " Bİ", "​", " ", " ‍", "a", "̊ſa", "'T", "123", "456", "78", "ea", "̊>'", "re", "
\u000b\n", "/", " ", " /\r\n"]} +{"text": "㍿ 'VE😀🏽½/>'ſ字m'T字Dž㋿ḍ̇'ſ!'ſ½İ　ḍ̇‍fi🙂tss\t👍🏽\"'!!", "tokens": 65, "pieces": ["㍿", " ", "'VE", "😀🏽", "½", "/>'", "ſ字m", "'T", "字", "Dž", "㋿ḋ", "̣'", "ſ", "!'", "ſ", "½", "İ", "　ḋ", "̣‍", "fi", "🙂tss", "\t", "👍🏽\"'!!"]} +{"text": "/a/b \"𐞁Bd''VE0Ⅳᵃᵃ'Mé'Sعm#$%'VE's👍🏽…m\n!ᵃ<|fim_prefix|>aB½/\r\n 'T,
", "tokens": 63, "pieces": ["/a", "/b", " \"", "𐞁Bd", "''", "VE", "0Ⅳ", "ᵃᵃ", "'M", "e", "́'", "Sعm", "#$%'", "VE", "'s", "👍🏽", "…m", "\n", "!ᵃ", "<|", "fim", "_prefix", "|><", "EOT", ">aB", "½", "/\r\n", " '", "T", ",", "
"]} +{"text": "<|fim_prefix|>aa/b<😀🏽'M½/\r\n­\r'ſABC​é‍!!­\r\nHTTPServeŕ.Z漢s/ \n ", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>", "aa", "/b", "<😀🏽'", "M", "½", "/\r\n", "­\r", "'ſ", "ABC", "​e", "́‍!!­\r\n", "HTTPServer", "́.", "Z漢s", "/", " \n "]} +{"text": "0­٣٤٥٦Džungla𐞁'ſＡeABC're'VEع.(t'rea/b'ſ(…", "tokens": 40, "pieces": ["0", "­", "٣٤٥", "٦", "Džungla𐞁", "'ſ", "ＡeABC", "'re", "'VE", "ع", ".(", "t", "'re", "a", "/b", "'ſ", "(", "…"]} +{"text": "m iOS'TＡ12345678\u000b
'Re\r\n\r\ncamelCaseé٣٤٥٦DžDž0,́!!👍🏽å'D ½s
9s㋿ABCDž'ſAb(­", "tokens": 73, "pieces": ["m", " iOS", "'T", "Ａ", "123", "456", "78", "\u000b", "
", "'Re", "\r\n\r\n", "camelCaseé", "٣٤٥", "٦", "DžDž", "0", ",́!!👍🏽", "a", "̊<", "EOT", "><", "META", "_START", ">'", "D", " ", " ", "½", "s", "
", "9", "s", "㋿", "ABCDž", "'ſ", "Ab", "(­"]} +{"text": "é'­
#$%é\n/٣٤٥٦a dDžungla", "tokens": 26, "pieces": ["e", "́'­", "
", "#$%", "é", "\n", "/", "٣٤٥", "٦", "a", " ", " dDžungla"]} +{"text": "‍#$%👍🏽 \nİ/ſ㋿<\r\n\r\n\r\nİ,éaBＡDžungla\r\n\r\nＡßABCaB\r\n('S'McamelCase…㋿😀🏽\t(9", "tokens": 57, "pieces": ["‍#$%👍🏽", " \n", "İ", "/ſ", "㋿<\r\n\r\n\r\n", "İ", ",éaBＡDžungla", "\r\n\r\n", "ＡßABCaB", "\r\n", "('", "S", "'M", "camelCase", "…", "㋿😀🏽", "\t", "(", "9"]} +{"text": "\ré\r\nＡ /\r'TEOT!0\"'VEꟲḍ̇㍿​‍㋿fi'T😀🏽.HTTPServer", "tokens": 45, "pieces": ["\r", "é", "\r\n", "Ａ", " /\r", "'T", "EOT", "!", "0", "\"'", "VEꟲḋ", "̣㍿​‍㋿", "fi", "'T", "😀🏽.", "HTTPServer"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": "DžunglafiABC漢　''M
\r\n\r\nfia/bᵃé\"'Re…٣٤٥٦/\r\nDž​'s!!m,B", "tokens": 44, "pieces": ["DžunglafiABC漢", "　", "''", "M", "
\r\n\r\n", "fia", "/bᵃé", "\"'", "Re", "…", "٣٤٥", "٦", "/\r\n", "Dž", "​'", "s", "!!", "m", ",B"]} +{"text": "漢éßtiOS½\n/t/Ａ<|endoftext|>​é.\t \n Dž㋿.>'S㋿
> Ａ\tåaB>'Dfi9", "tokens": 51, "pieces": ["漢e", "́ßtiOS", "½", "\n", "/t", "/Ａ", "<|", "endoftext", "|>​", "é", ".", "\t \n", " Dž", "㋿.>'", "S", "㋿", "
", ">", " Ａ", "\ta", "̊aB", ">'", "Dfi", "9"]} +{"text": "iOS're(Ａ<|endoftext|>sEOT　 \u000b😀🏽.'Reſ \n­㍿#$%Ⅳİꟲ!<|endoftext|>½ß漢HTTPServer!!<|endoftext|>12345678é\u000b/<|endoftext|>", "tokens": 76, "pieces": ["iOS", "'re", "(Ａ", "<|", "endoftext", "|>", "s", "EOT", "　 ", "\u000b", "😀🏽.'", "Reſ", " \n", "­㍿#$%", "Ⅳ", "İꟲ", "!<|", "endoftext", "|>", "½", "ß漢HTTPServer", "!!<|", "endoftext", "|>", "123", "456", "78", "é", "\u000b", "/<|", "endoftext", "|>"]} +{"text": "ABCDžungla३…\n", "tokens": 10, "pieces": ["ABCDžungla", "३", "…\n"]} +{"text": "t\u000b'll!camelCase…EOT,m'S/\r\nＡ \n ABC'TeⅣ/​漢s'Mİ-👍🏽‍'TcamelCase", "tokens": 48, "pieces": ["t", "\u000b", "'ll", "!camelCase", "…EOT", ",m", "'S", "/\r\n", "Ａ", " \n", " ABC", "'T", "e", "Ⅳ", "/​<", "META", "_START", "><", "META", "_START", ">漢s", "'M", "İ", "-👍🏽‍'", "TcamelCase"]} +{"text": "…㍿Ⅳ,Džİ \ncamelCase'!!0́½ß \né\r<　\n/", "tokens": 28, "pieces": ["…", "㍿", "Ⅳ", ",Džİ", " \n", "camelCase", "'!!", "0", "́", "½", "ß", " \n", "e", "́\r", "<", "　\n", "/"]} +{"text": "12345678ſ!ABC0aBsdEOTHTTPServerＡ　Ab-漢½/må", "tokens": 27, "pieces": ["123", "456", "78", "ſ", "!ABC", "0", "aBsdEOTHTTPServerＡ", "　Ab", "-漢", "½", "/ma", "̊"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "aB", "tokens": 2, "pieces": ["aB"]} +{"text": "漢 \n \"
Ａ½ḍ̇<|fim_prefix|><|endoftext|>Džع9", "tokens": 34, "pieces": ["漢", "", " \n", " \"", "
Ａ", "½", "ḋ", "̣<|", "fim", "_prefix", "|><|", "endoftext", "|>", "Džع", "9"]} +{"text": "ꟲ", "tokens": 3, "pieces": ["ꟲ"]} +{"text": "ḍ̇", "tokens": 5, "pieces": ["ḋ", "̣"]} +{"text": "e३'s'Re,é,at🙂/\r\n½iOS'VEfi-🙂aB\r\nſDžungla \n㋿((", "tokens": 38, "pieces": ["e", "३", "'s", "'Re", ",é", ",at", "🙂/\r\n", "½", "iOS", "'VE", "fi", "-🙂", "aB", "\r\n", "ſDžungla", " \n", "㋿<", "EOT", ">(("]} +{"text": "ع​/\r\n \n İ", "tokens": 5, "pieces": ["ع", "​/\r\n", " \n", " İ"]} +{"text": " 𐞁😀🏽​", "tokens": 12, "pieces": [" ", " 𐞁", "😀🏽​"]} +{"text": " ‍‍#$%\tétaB''M", "tokens": 13, "pieces": [" ", " ‍‍#$%", "\tétaB", "''", "M"]} +{"text": "9㍿es", "tokens": 5, "pieces": ["9", "㍿es"]} +{"text": "\u000b9'Re­\t''ſ'llḍ̇m\u000bDžunglaḍ̇ \n a", "tokens": 28, "pieces": ["\u000b", "9", "'Re", "­", "\t", "''", "ſ", "'ll", "ḋ", "̣m", "\u000bDžunglaḋ", "̣", " \n", " ", " a"]} +{"text": "camelCase9Dž12345678camelCaseAb'sAbABCcamelCase'/\r\nDžungla!!>Dž­e…Ａ", "tokens": 32, "pieces": ["camelCase", "9", "Dž", "123", "456", "78", "camelCaseAb", "'s", "AbABCcamelCase", "'/\r\n", "Džungla", "!!>", "Dž", "­e", "…Ａ"]} +{"text": "İ'VE,", "tokens": 4, "pieces": ["İ", "'VE", ","]} +{"text": "aB🙂…ABCع'ſ'DHTTPServer😀🏽as#$%\r'VE'a/b٣٤٥٦३d <|endoftext|>㍿#$%'Re's 😀🏽", "tokens": 56, "pieces": ["aB", "🙂", "…ABCع", "'ſ", "'D", "HTTPServer", "😀🏽", "as", "#$%\r", "'VE", "'a", "/b", "٣٤٥", "٦३", "d", " <|", "endoftext", "|>㍿#$%'", "Re", "'s", " ", " 😀🏽"]} +{"text": "aB ­ ㍿ét,#$%㍿s's\r\nḍ̇ß\r\n!9ᵃ", "tokens": 29, "pieces": ["aB", " ­", " ", "㍿ét", ",#$%㍿", "s", "'s", "\r\n", "ḋ", "̣ß", "\r\n", "!", "9", "ᵃ"]} +{"text": "\n/åéHTTPServer", "tokens": 7, "pieces": ["\n", "/a", "̊éHTTPServer"]} +{"text": "\ré字\n", "tokens": 4, "pieces": ["\r", "é字", "\n"]} +{"text": "EOTİ(́HTTPServer'Re>　a
(\r\nfi 
/ iOS'D", "tokens": 23, "pieces": ["EOTİ", "(́", "HTTPServer", "'Re", ">", "　a", "
", "(\r\n", "fi", " ", "
", "/", " iOS", "'D"]} +{"text": "EOT- \r\n\r\n\nt'MB​ſ'Reaé-
㍿Džungla. 0㍿é", "tokens": 33, "pieces": ["EOT", "-", " \r\n\r\n\n", "t", "'M", "B", "​ſ", "'Re", "ae", "́-", "
", "㍿Džungla", ".", " ", " ", "0", "㍿é"]} +{"text": "a12345678<|endoftext|>!!édDž\rAb \nB#$%　𐞁🙂𐞁#$%'ll0'ſ'D🙂'Re123456780👍🏽're \n", "tokens": 56, "pieces": ["a", "123", "456", "78", "<|", "endoftext", "|>!!", "édDž", "\r", "Ab", " \n", "B", "#$%", "　𐞁", "🙂𐞁", "#$%'", "ll", "0", "'ſ", "'D", "🙂'", "Re", "123", "456", "780", "👍🏽'", "re", " \n"]} +{"text": "\r>ABC٣٤٥٦\"٣٤٥٦½　'sⅣEOT0­­ \n a(/\r\n🙂漢.-　DžunglaB
 \n عß👍🏽ع/'s", "tokens": 65, "pieces": ["\r", "><", "META", "_START", ">ABC", "٣٤٥", "٦", "\"", "٣٤٥", "٦½", "　", "'s", "Ⅳ", "EOT", "0", "­­", " \n", " a", "(/\r\n", "🙂漢", ".-", "　DžunglaB", "
 \n", " عß", "👍🏽", "ع", "/'", "s"]} +{"text": " 👍🏽
‍'ll.Džungla'ReABC12345678𐞁­Z\n/‍​'Re𐞁'M,", "tokens": 44, "pieces": [" ", " 👍🏽", "
", "‍'", "ll", ".Džungla", "'Re", "ABC", "123", "456", "78", "𐞁", "­Z", "\n", "/‍​<", "META", "_START", ">'", "Re𐞁", "'M", ","]} +{"text": "éⅣ0<|fim_prefix|>'fitdＡtfi'reİ🙂a/bfiſ㋿\r\n\r\n😀🏽ß
字 ß<'M", "tokens": 49, "pieces": ["é", "Ⅳ0", "<|", "fim", "_prefix", "|>'", "fitdＡtfi", "'re", "İ", "🙂a", "/bfiſ", "㋿\r\n\r\n", "😀🏽", "ß", "
字", " <", "META", "_START", ">ß", "<'", "M"]} +{"text": "字　\ndſ,　­(½\n<|fim_prefix|>ABC\"\r 'VE३'re'TiOS/½\"'ll३,'(>", "tokens": 42, "pieces": ["字", "　\n", "dſ", ",", "　", "­(", "½", "\n", "<|", "fim", "_prefix", "|>", "ABC", "\"\r", " '", "VE", "३", "'re", "'T", "iOS", "/", "½", "\"'", "ll", "३", ",'(>"]} +{"text": "\naZᵃ12345678'D9< \n aBᵃⅣiOSḍ̇", "tokens": 26, "pieces": ["\n", "aZᵃ", "123", "456", "78", "'D", "9", "<", " \n", " aBᵃ", "Ⅳ", "iOSḋ", "̣"]} +{"text": "Ab \n ́‍/\reß/\r\ne 'Re/½\r'TⅣEOT", "tokens": 22, "pieces": ["Ab", " \n", " ́‍/\r", "eß", "/\r\n", "e", " '", "Re", "/", "½", "\r", "'T", "Ⅳ", "EOT"]} +{"text": "ſ/ HTTPServerꟲ", "tokens": 9, "pieces": ["ſ", "/", " HTTPServerꟲ"]} +{"text": "ABCB'Re\n(ᵃ㋿\r\n 🙂/\r\n.!!'s'ſ́éiOSZ\n0", "tokens": 28, "pieces": ["ABCB", "'Re", "\n", "(ᵃ", "㋿\r\n", " ", " 🙂/\r\n", ".!!'", "s", "'ſ", "́éiOSZ", "\n", "0"]} diff --git a/litellm-rust/crates/token-counter/tests/fixtures/generate.py b/litellm-rust/crates/token-counter/tests/fixtures/generate.py new file mode 100644 index 00000000000..1bfbdf00218 --- /dev/null +++ b/litellm-rust/crates/token-counter/tests/fixtures/generate.py @@ -0,0 +1,422 @@ +"""Pin tiktoken reference counts for the Rust parity tests of one encoding. + +Run from the repository root with the project environment, once per encoding: + + uv run --no-sync python litellm-rust/crates/token-counter/tests/fixtures/generate.py cl100k_base + uv run --no-sync python litellm-rust/crates/token-counter/tests/fixtures/generate.py o200k_base + +`/texts.jsonl` holds `{"text", "tokens", "pieces"}` lines: `tokens` +counted with `tiktoken.get_encoding(name).encode(text, disallowed_special=())`, +the same call `litellm.token_counter` makes, and `pieces` the installed +encoding's split pattern applied with the `regex` module tiktoken itself uses, +so a scanner that splits differently fails even where BPE would count the same. +`/requests.jsonl` holds `{"body", "input_tokens"}` lines, `body` being +the exact request bytes as a JSON string, counted with the proxy's admission +counter (`_count_input_tokens(body, model)`) for a model Python counts with that +encoding. Every message in the 50k-token body is shorter than the Python chunk +size so the chunked Python count equals the exact whole-text tiktoken count the +Rust counter produces. +""" + +import itertools +import json +import random +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import regex +import tiktoken + +from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding +from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens + +HERE: Final = Path(__file__).resolve().parent +MODELS: Final = {"cl100k_base": "gpt-4", "o200k_base": "gpt-4o"} +ENCODING_NAME: Final = sys.argv[1] +MODEL: Final = MODELS[ENCODING_NAME] +ENCODING: Final = tiktoken.get_encoding(ENCODING_NAME) +assert openai_tokenizer_encoding(MODEL).name == ENCODING_NAME +SPLIT_PATTERN: Final = regex.compile(ENCODING._pat_str) # pyright: ignore[reportPrivateUsage] # tiktoken has no public accessor +OUT: Final = HERE / ENCODING_NAME.removesuffix("_base") + +# Mirrors ALPHABET in src/byte_level.rs, plus the pieces the tiktoken patterns treat differently. +ALPHABET: Final = ( + "a", + "Z", + "e", + "s", + "t", + "d", + "m", + "'", + "'s", + "'re", + "'ll", + "'S", + "0", + "9", + " ", + " ", + "\t", + "\n", + "\r\n", + "\x0b", + ".", + ",", + "!", + "-", + "(", + '"', + "\xa0", + "\x85", + "\u2028", + "\u3000", + "\u200b", + "\u200d", + "é", + "e\u0301", + "ß", + "漢", + "字", + "ع", + "३", + "½", + "Ⅳ", + "🙂", + "👍🏽", + "Ａ", + "fi", + "㍿", + "㋿", + "ꟲ", + "𐞁", + "a\u030a", + "\u1e0b\u0323", + "<", + ">", + "EOT", + "", + "", + "'D", + "'M", + "'T", + "'VE", + "'Re", + "'ſ", + "ſ", + "12345678", + "٣٤٥٦", + "<|endoftext|>", + "<|fim_prefix|>", + "\r", + "\r\n\r\n", + " \n", + "!!", + "#$%", + "\u00ad", + "\u0301", + "\U0001f600\U0001f3fd", + "İ", + "Dž", +) + +# The pieces the o200k case-shaped letter branch and slash-absorbing symbol branch split differently. +CASE_ALPHABET: Final = ALPHABET + ( + "B", + "Ab", + "aB", + "ABC", + "ᵃ", + "camelCase", + "HTTPServer", + "iOS", + "Džungla", + "/", + "\n/", + "/\r\n", + " \n ", + "a/b", +) + +CORPUS: Final = ( + "", + "Hello, how are you today?", + "I'm sure they're right, we'll see. WE'LL SEE, I'M SURE THEY'RE RIGHT, IT'S HERS AND IT'D BE 'D", + "don't Don'T DON'T won'T i've I'VE i'Ve you'RE 'S 'T 'M 'D 'LL 'VE 'RE 'ſ 'x", + "1234567890 123 12 1 0000000 ٣٤٥٦٧٨ ३४५६ 1,234,567.89 2026-09-11T18:00:00Z", + "$abc %def &ghi @jkl _mno #pqr ~stu ^vwx |yz \\a /b :c ;d ?e !f (g )h [i ]j {k }l n =o +p *q", + "foo bar baz \t qux\t\tquux \n\nline\r\nline\r\n\r\n \n\t\r\n x ", + "trailing spaces ", + "trailing tabs\t\t", + "trailing newline\n", + "\n\n\n", + "\r\n\r\n\r\n", + " ", + "😀😃😄 👍🏽 🇺🇸 👨‍👩‍👧‍👦 ✈️ ❤️‍🔥 ٭ ※ ⌘ ⏎", + "漢字かな交じり文、東京都千代田区。日本語のテキストです。中文测试。한국어 텍스트", + "مرحبا بالعالم، هذا نص عربي مع أرقام ١٢٣٤٥٦٧ و علامات ترقيم!", + "Zürich, façade, naïve, Ærøskøbing, Ελληνικά, Русский текст, עברית, हिन्दी, ไทย", + "e\u0301 a\u030a \u1e0b\u0323 \u0301\u0301 combining\u0308 marks\u0301!", + "ΣΊΣΥΦΟΣ Džungla İstanbul file flow Ａｂｃ ㍿ ㋿ ꟲ 𐞁", + "<|endoftext|> <|fim_prefix|>code<|fim_middle|>more<|fim_suffix|> <|endofprompt|> <|im_start|>", + " [INST] [/INST] <>", + "def f(x):\n return {'a': x ** 2, \"b\": [1, 2, 3]} # comment\n\nprint(f(10))\n", + '{"model":"gpt-4","messages":[{"role":"user","content":"hi\\n"}],"temperature":0.7}', + "https://example.com/path?query=1&other=two#fragment user@example.com 192.168.0.1", + "a" * 3000, + " " * 3000, + "." * 3000, + "ab" * 1500, + "\n" * 3000, + "0" * 3000, + "!" * 3000, + "😀" * 1000, + "漢" * 1000, + "\u00a0abc\u00a0! \u2028x \u3000y \u200bz \u200d\u200d q", + "x\u0085y \x0b\x0c z", + "\x00\x01\x02 \x7f \ufffd", + "tab\tseparated\tvalues\n1\t2\t3\n", + "MiXeD cAsE wOrDs AND ACRONYMS like NASA, HTTP/2, gRPC, iOS, macOS", + "snake_case_identifier camelCaseIdentifier PascalCaseIdentifier SCREAMING_SNAKE_CASE kebab-case", + "x'sy x'ty x'rey x'vey x'my x'lly x'dy x'S x'T x'RE x'VE x'M x'LL x'D x'sS x'llL", + "IT'SOK it'Dbe x'Sy x'Ty x'My x'Dy x'LLy x'VEy x'REy x'Ly x'Vy x'Ry 'Sx'Tx'Mx'LLx'VEx'REx'Dx", + "'s't're've'm'll'd 'S'T'RE'VE'M'LL'D ''s '''s", + "9'9 9's a'9 '9 ' 's' ' 's", + "١٢٣٤ ½⅓¼ ⅣⅤ 𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡 ①②③", + "camelCase PascalCase ABCdef ABCdeF ABC aB Ab ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABC", + "日本ABC ABC日本 日本語abc abc日本語 漢字Kanji kanji漢字 KANJI漢字kanji مرحباABC ABCمرحبا abcمرحبا", + "\u0301ABC \u0301abc \u0301\u0301A A\u0301\u0301 E\u0301A aE\u0301 !!\u0301a \u00a0\u0301A x\u0308Y X\u0308y", + "ᵃbc ᵃBC Aᵃbc Aᵃ ᵃ' ᵃ's Džungla aDžB ADžB ADžb DžDž Ljx İi ΣΊΣΥΦΟΣσ ΣσΣ", + "don'tx ABC's abc'S abc'ſ ABC'ſx IT'SOK it'Dbe 'sabc x's 's 'Sx'Tx x’s X'LLx X'Ll", + "!ABC !AbC !!abc #camelCase (ABCdef) \u00a0ABC\u00a0abc\u00a0Abc \tABC\tabc", + "!!/\n/x a/b !!\n/x /x // path/to/file.rs http://x.y/z?a=b/c \\/\\/ //\r\n//\n", + "x \n x \r\n \r\n y x \n a b \n\n c x\t\ty x\t\t end \n \n", + "12345 6 1abc abc1 ABC123abc 123ABC ١٢٣٤٥abc", +) + +WORDS: Final = ( + "the", + "quick", + "brown", + "fox", + "jumps", + "over", + "lazy", + "dog", + "while", + "counting", + "tokens", + "for", + "budget", + "reservation", + "before", + "admission", + "on", + "the", + "gateway", + "and", + "every", + "request", + "body", + "is", + "scanned", + "exactly", + "once", + "with", + "a", + "hand", + "written", + "piece", + "scanner", + "that", + "mirrors", + "tiktoken's", + "regex", + "boundaries", + "It's", + "faster", + "because", + "there's", + "no", + "backtracking", + "engine", + "involved", + "so", + "we'll", + "keep", + "it", + "that", + "way", + "Zürich", + "café", + "naïve", + "東京", + "مرحبا", + "🙂", + "42", + "1999", + "3.14159", + "$1,234.56", + "100%", + "user@example.com", + "https://example.com/a/b?c=d", + "C++", + "F#", + "node.js", + "v1.2.3", + "(parens)", + "[brackets]", + "{braces}", + "", + '"quotes"', + "'single'", + "don't", + "WON'T", + "I'M", + "They'RE", +) + + +def random_text(rng: random.Random, alphabet: tuple[str, ...]) -> str: + return "".join(rng.choice(alphabet) for _ in range(rng.randrange(0, 40))) + + +def paragraph(rng: random.Random, words: int) -> str: + return " ".join(rng.choice(WORDS) for _ in range(words)) + + +def short_paragraphs(rng: random.Random) -> Iterator[str]: + while True: + content = paragraph(rng, rng.randrange(60, 140)) + if len(content) < TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS: + yield content + + +def chat_body(rng: random.Random, target_tokens: int) -> dict[str, object]: + candidates: Final = tuple(itertools.islice(short_paragraphs(rng), 2000)) + running: Final = tuple(itertools.accumulate(len(ENCODING.encode(content)) + 3 for content in candidates)) + turns: Final = next(index for index, total in enumerate(running) if total >= target_tokens) + 1 + contents: Final = candidates[: turns + (turns % 2)] + return { + "model": MODEL, + "messages": [ + {"role": "system", "content": "You are a helpful assistant. Answer precisely and cite sources."}, + *( + {"role": "user" if index % 2 == 0 else "assistant", "content": content} + for index, content in enumerate(contents) + ), + {"role": "user", "content": "Summarise the conversation so far in three sentences."}, + ], + } + + +TOOLS: Final = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + "days": {"type": "integer"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "opts": { + "type": "object", + "properties": {"verbose": {"type": "boolean"}, "level": {"type": "integer", "enum": [1, 2]}}, + "required": ["verbose"], + }, + "anything": {}, + }, + "required": ["location"], + }, + }, + }, + {"type": "function", "function": {"name": "noop"}}, +] + +SMALL_REQUESTS: Final = ( + {"model": MODEL, "messages": [{"role": "user", "content": "Hello, how are you today?"}]}, + { + "model": MODEL, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + { + "role": "user", + "name": "alice", + "content": [ + {"type": "text", "text": "Summarise this paragraph about ships and harbours."}, + "plain string item", + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "Sure."}]}, + ], + }, + { + "model": MODEL, + "messages": [{"role": "user", "content": "weather?"}], + "tools": TOOLS, + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + }, + { + "model": MODEL, + "messages": [{"role": "system", "content": "sys"}, {"role": "user", "content": "weather?"}], + "tools": TOOLS, + "tool_choice": "none", + }, + {"model": MODEL, "prompt": "Write a haiku about ships."}, + {"model": MODEL, "prompt": ["first prompt", "second prompt"]}, + { + "model": MODEL, + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": 'Summarise caf\u00e9 menus, na\u00efve \u2014 ok? "quoted"\n'} + ], + }, + {"role": "assistant", "content": "Sure."}, + ], + "instructions": "be terse", + }, + {"model": MODEL, "input": [[101, 2023, 5], [7]], "encoding_format": "float"}, + { + "model": MODEL, + "query": "best harbour", + "documents": [ + "doc one", + {"text": "doc two", "title": "T", "n": 3, "ok": True, "none": None, "tags": ["a", "b"]}, + ], + }, +) + + +def main() -> None: + rng: Final = random.Random(2026) + case_rng: Final = random.Random(200_000) + texts: Final = ( + tuple(CORPUS) + + tuple(random_text(rng, ALPHABET) for _ in range(3000)) + + tuple(random_text(case_rng, CASE_ALPHABET) for _ in range(1000)) + ) + OUT.mkdir(exist_ok=True) + with (OUT / "texts.jsonl").open("w", encoding="utf-8") as handle: + for text in texts: + tokens = len(ENCODING.encode(text, disallowed_special=())) + pieces = SPLIT_PATTERN.findall(text) + handle.write(json.dumps({"text": text, "tokens": tokens, "pieces": pieces}, ensure_ascii=False) + "\n") + bodies: Final = tuple(SMALL_REQUESTS) + (chat_body(rng, 50_000),) + with (OUT / "requests.jsonl").open("w", encoding="utf-8") as handle: + for body in bodies: + input_tokens = _count_input_tokens(dict(body), MODEL) + assert input_tokens is not None + handle.write(json.dumps({"body": json.dumps(body), "input_tokens": input_tokens}) + "\n") + + +if __name__ == "__main__": + main() diff --git a/litellm-rust/crates/token-counter/tests/fixtures/o200k/requests.jsonl b/litellm-rust/crates/token-counter/tests/fixtures/o200k/requests.jsonl new file mode 100644 index 00000000000..f50b85c3922 --- /dev/null +++ b/litellm-rust/crates/token-counter/tests/fixtures/o200k/requests.jsonl @@ -0,0 +1,10 @@ +{"body": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello, how are you today?\"}]}", "input_tokens": 14} +{"body": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"system\", \"content\": \"You are a terse assistant.\"}, {\"role\": \"user\", \"name\": \"alice\", \"content\": [{\"type\": \"text\", \"text\": \"Summarise this paragraph about ships and harbours.\"}, \"plain string item\"]}, {\"role\": \"assistant\", \"content\": [{\"type\": \"text\", \"text\": \"Sure.\"}]}]}", "input_tokens": 39} +{"body": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"user\", \"content\": \"weather?\"}], \"tools\": [{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get weather\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"City name\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}, \"days\": {\"type\": \"integer\"}, \"tags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, \"opts\": {\"type\": \"object\", \"properties\": {\"verbose\": {\"type\": \"boolean\"}, \"level\": {\"type\": \"integer\", \"enum\": [1, 2]}}, \"required\": [\"verbose\"]}, \"anything\": {}}, \"required\": [\"location\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"noop\"}}], \"tool_choice\": {\"type\": \"function\", \"function\": {\"name\": \"get_weather\"}}}", "input_tokens": 104} +{"body": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"system\", \"content\": \"sys\"}, {\"role\": \"user\", \"content\": \"weather?\"}], \"tools\": [{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get weather\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"City name\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}, \"days\": {\"type\": \"integer\"}, \"tags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, \"opts\": {\"type\": \"object\", \"properties\": {\"verbose\": {\"type\": \"boolean\"}, \"level\": {\"type\": \"integer\", \"enum\": [1, 2]}}, \"required\": [\"verbose\"]}, \"anything\": {}}, \"required\": [\"location\"]}}}, {\"type\": \"function\", \"function\": {\"name\": \"noop\"}}], \"tool_choice\": \"none\"}", "input_tokens": 97} +{"body": "{\"model\": \"gpt-4o\", \"prompt\": \"Write a haiku about ships.\"}", "input_tokens": 7} +{"body": "{\"model\": \"gpt-4o\", \"prompt\": [\"first prompt\", \"second prompt\"]}", "input_tokens": 4} +{"body": "{\"model\": \"gpt-4o\", \"input\": [{\"role\": \"user\", \"content\": [{\"type\": \"input_text\", \"text\": \"Summarise caf\\u00e9 menus, na\\u00efve \\u2014 ok? \\\"quoted\\\"\\n\"}]}, {\"role\": \"assistant\", \"content\": \"Sure.\"}], \"instructions\": \"be terse\"}", "input_tokens": 60} +{"body": "{\"model\": \"gpt-4o\", \"input\": [[101, 2023, 5], [7]], \"encoding_format\": \"float\"}", "input_tokens": 5} +{"body": "{\"model\": \"gpt-4o\", \"query\": \"best harbour\", \"documents\": [\"doc one\", {\"text\": \"doc two\", \"title\": \"T\", \"n\": 3, \"ok\": true, \"none\": null, \"tags\": [\"a\", \"b\"]}]}", "input_tokens": 43} +{"body": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant. Answer precisely and cite sources.\"}, {\"role\": \"user\", \"content\": \"\\ud83d\\ude42 a every WON'T They'RE involved counting caf\\u00e9 backtracking boundaries Z\\u00fcrich WON'T 100% \\\"quotes\\\" caf\\u00e9 tiktoken's budget They'RE request request regex v1.2.3 hand hand fox tiktoken's on 3.14159 mirrors that don't WON'T admission before budget the admission WON'T no 1999 100% no admission budget mirrors way caf\\u00e9 dog a quick mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 hand gateway regex on jumps because {braces} 'single' the the the {braces} piece hand scanned reservation [brackets] we'll 3.14159 request don't \\\"quotes\\\" na\\u00efve C++ caf\\u00e9 caf\\u00e9 for https://example.com/a/b?c=d we'll no over the node.js for while over way\"}, {\"role\": \"assistant\", \"content\": \"it tiktoken's node.js it scanned v1.2.3 boundaries (parens) and while reservation lazy the that https://example.com/a/b?c=d quick don't budget engine boundaries F# budget every we'll before jumps scanned jumps there's counting the don't jumps a once admission 100% 3.14159 brown brown that because jumps They'RE 100% caf\\u00e9 WON'T They'RE boundaries \\u6771\\u4eac exactly scanner caf\\u00e9 fox (parens) body dog a 1999 boundaries 'single' {braces} reservation mirrors while {braces} {braces} so admission the F# https://example.com/a/b?c=d brown a caf\\u00e9 on admission that on counting that we'll over lazy https://example.com/a/b?c=d way over engine reservation gateway body I'M for \\\"quotes\\\" 42 and F# there's 'single' quick \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T scanner written every (parens) so\"}, {\"role\": \"user\", \"content\": \"I'M over faster counting https://example.com/a/b?c=d way while it over way mirrors boundaries keep body hand na\\u00efve once because that node.js for every the 'single' every caf\\u00e9 caf\\u00e9 piece there's v1.2.3 v1.2.3 tiktoken's brown so counting there's for we'll boundaries 'single' no https://example.com/a/b?c=d node.js \\u6771\\u4eac written with budget (parens) because \\\"quotes\\\" \\u6771\\u4eac while \\u6771\\u4eac counting scanned 1999 \\u6771\\u4eac hand keep so that gateway caf\\u00e9 for scanned it request keep counting user@example.com admission request \\\"quotes\\\" v1.2.3 caf\\u00e9 over that F# we'll regex a faster with that They'RE piece because tiktoken's engine hand 100% WON'T reservation (parens) caf\\u00e9 on Z\\u00fcrich dog \\u6771\\u4eac that They'RE 'single' the 'single' na\\u00efve involved the {braces} there's scanned no engine dog backtracking there's\"}, {\"role\": \"assistant\", \"content\": \"every node.js C++ for 3.14159 \\u6771\\u4eac 'single' gateway once the user@example.com a 100% every every tokens written and every brown na\\u00efve the body the because quick brown engine fox 3.14159 (parens) F# while with a 100% C++ with the written WON'T on request Z\\u00fcrich hand on https://example.com/a/b?c=d don't way way 42 that we'll \\ud83d\\ude42 [brackets] admission tiktoken's request hand caf\\u00e9 every every (parens) They'RE that jumps the regex 100% \\\"quotes\\\" is budget\"}, {\"role\": \"user\", \"content\": \"engine with I'M backtracking while F# quick 'single' mirrors \\\"quotes\\\" 'single' regex caf\\u00e9 It's Z\\u00fcrich exactly a counting tiktoken's we'll once \\u0645\\u0631\\u062d\\u0628\\u0627 42 v1.2.3 scanner WON'T \\u6771\\u4eac there's node.js It's budget that budget {braces} because [brackets] It's a $1,234.56 that v1.2.3 42 gateway backtracking it C++ scanned keep \\ud83d\\ude42 I'M hand a counting scanner reservation \\u6771\\u4eac written 3.14159 there's once fox I'M C++ lazy the \\u0645\\u0631\\u062d\\u0628\\u0627 before don't we'll gateway exactly \\ud83d\\ude42 Z\\u00fcrich 3.14159 with WON'T there's node.js \\ud83d\\ude42 quick written faster counting 1999 backtracking 'single' counting we'll engine counting don't \\\"quotes\\\" engine \\\"quotes\\\" way I'M budget [brackets] backtracking \\\"quotes\\\" 3.14159 don't written written \\u0645\\u0631\\u062d\\u0628\\u0627 body I'M 'single' while on and reservation \\ud83d\\ude42 regex and no budget regex (parens) we'll They'RE na\\u00efve v1.2.3\"}, {\"role\": \"assistant\", \"content\": \"hand lazy dog budget lazy involved because scanned piece quick that involved 'single' dog quick na\\u00efve budget scanner exactly $1,234.56 fox quick I'M hand (parens) node.js while jumps boundaries \\ud83d\\ude42 They'RE way request engine 'single' fox backtracking regex \\u0645\\u0631\\u062d\\u0628\\u0627 because and over jumps 3.14159 WON'T counting for and mirrors admission 'single' caf\\u00e9 https://example.com/a/b?c=d that \\ud83d\\ude42 faster \\u0645\\u0631\\u062d\\u0628\\u0627 admission jumps quick for $1,234.56 exactly exactly $1,234.56 scanned keep 3.14159 backtracking piece tiktoken's is is hand reservation before regex budget 3.14159 tiktoken's I'M it no budget user@example.com budget written budget over that https://example.com/a/b?c=d no written so quick tokens C++\"}, {\"role\": \"user\", \"content\": \"once body involved every brown every it that hand engine with scanned reservation \\u6771\\u4eac backtracking and {braces} over [brackets] brown over for is \\ud83d\\ude42 100% before quick no counting no v1.2.3 counting \\\"quotes\\\" mirrors 3.14159 1999 there's way \\\"quotes\\\" piece on na\\u00efve They'RE no every exactly node.js 42 because over there's 1999 C++ we'll mirrors F# it scanner jumps It's scanner mirrors mirrors It's tokens backtracking body brown $1,234.56 keep admission 100% the exactly we'll caf\\u00e9 jumps over 3.14159 we'll so 42 \\u6771\\u4eac I'M WON'T lazy brown It's a na\\u00efve \\\"quotes\\\" https://example.com/a/b?c=d (parens) \\u6771\\u4eac tiktoken's so dog body 'single' scanned piece body 3.14159 scanned body the so \\\"quotes\\\" counting\"}, {\"role\": \"assistant\", \"content\": \"boundaries request the that 100% gateway the there's hand the They'RE caf\\u00e9 fox C++ scanner written \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE scanner WON'T keep before for it so counting that (parens) {braces} They'RE scanner every {braces} no node.js keep I'M jumps backtracking gateway https://example.com/a/b?c=d don't tiktoken's a is 1999 don't F# v1.2.3 involved hand scanner They'RE backtracking exactly and for exactly for $1,234.56 counting v1.2.3 request gateway mirrors that no \\ud83d\\ude42 every dog once They'RE 100% reservation It's a with and 100% because so It's admission there's gateway 42 on over gateway is every 3.14159 boundaries no for admission quick lazy lazy fox node.js because while $1,234.56 quick I'M lazy involved WON'T {braces} na\\u00efve {braces} there's with caf\\u00e9 https://example.com/a/b?c=d \\\"quotes\\\" [brackets] lazy lazy it 100% caf\\u00e9 way lazy tokens C++ that it \\u6771\\u4eac lazy\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors budget reservation 'single' it scanner F# is exactly They'RE \\ud83d\\ude42 I'M boundaries F# {braces} https://example.com/a/b?c=d over backtracking node.js is \\ud83d\\ude42 \\ud83d\\ude42 $1,234.56 so and counting It's Z\\u00fcrich once node.js once v1.2.3 on that we'll 'single' mirrors I'M \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking They'RE tokens hand counting 1999 user@example.com Z\\u00fcrich They'RE tokens reservation exactly for https://example.com/a/b?c=d mirrors and boundaries regex the brown while keep \\\"quotes\\\" we'll the involved 42 {braces} scanner reservation Z\\u00fcrich no we'll [brackets] caf\\u00e9 written that hand user@example.com $1,234.56 42 while budget every 3.14159 a exactly way body scanned admission C++ (parens) tiktoken's body for WON'T hand no dog 1999 (parens) on don't 1999 we'll I'M v1.2.3 that WON'T fox scanned 1999\"}, {\"role\": \"assistant\", \"content\": \"gateway $1,234.56 $1,234.56 \\\"quotes\\\" scanner there's scanner $1,234.56 100% it budget \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) admission admission with I'M admission 'single' hand jumps scanned It's \\\"quotes\\\" is F# before engine counting dog because admission tiktoken's backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 that 42 written it body quick Z\\u00fcrich I'M F# I'M that brown It's request caf\\u00e9 is boundaries jumps don't written written faster for admission Z\\u00fcrich engine reservation and Z\\u00fcrich scanned written keep scanned is keep jumps the so F# 'single' user@example.com keep because They'RE over because C++ written faster 42 mirrors na\\u00efve 42 tiktoken's [brackets] na\\u00efve hand on no written so $1,234.56 there's 100% 100% $1,234.56 is \\ud83d\\ude42 scanner dog lazy 100% https://example.com/a/b?c=d tiktoken's They'RE [brackets] user@example.com while hand \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d we'll 42 WON'T it a mirrors v1.2.3\"}, {\"role\": \"user\", \"content\": \"that involved dog C++ way that regex with https://example.com/a/b?c=d because 1999 jumps and and caf\\u00e9 F# backtracking that 3.14159 the quick v1.2.3 engine piece request brown (parens) because tokens na\\u00efve we'll and \\ud83d\\ude42 user@example.com that na\\u00efve \\u6771\\u4eac na\\u00efve tokens jumps 3.14159 tiktoken's na\\u00efve brown It's jumps before and a the user@example.com Z\\u00fcrich WON'T mirrors It's fox It's involved don't \\u6771\\u4eac 'single' written that written fox and the 3.14159 Z\\u00fcrich way on once na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 request fox so fox 'single' admission admission node.js mirrors backtracking it the\"}, {\"role\": \"assistant\", \"content\": \"engine so $1,234.56 don't caf\\u00e9 lazy I'M while 'single' budget scanned \\ud83d\\ude42 Z\\u00fcrich piece tokens exactly budget boundaries admission written tokens every \\u0645\\u0631\\u062d\\u0628\\u0627 before with backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 engine regex faster brown \\ud83d\\ude42 before we'll reservation na\\u00efve regex there's faster counting na\\u00efve reservation quick Z\\u00fcrich tiktoken's that gateway we'll C++ They'RE scanned for engine budget 100% na\\u00efve $1,234.56 written admission no we'll WON'T scanned body dog node.js while\"}, {\"role\": \"user\", \"content\": \"once exactly scanner boundaries scanned every there's mirrors $1,234.56 there's so dog dog They'RE lazy fox because 'single' so gateway Z\\u00fcrich faster v1.2.3 quick I'M \\u6771\\u4eac exactly written tokens node.js na\\u00efve brown [brackets] mirrors while on \\u6771\\u4eac $1,234.56 once hand over exactly quick backtracking over exactly {braces} it don't regex 3.14159 way the 100% $1,234.56 is I'M admission admission [brackets] scanned while boundaries piece counting that node.js reservation \\u6771\\u4eac body jumps while node.js I'M Z\\u00fcrich with fox C++ reservation F# \\\"quotes\\\" They'RE {braces} (parens) caf\\u00e9 1999 there's {braces} every WON'T no dog WON'T 1999 admission [brackets] that body 'single' gateway while mirrors with with scanner hand no over scanned hand na\\u00efve It's the while with once \\\"quotes\\\" boundaries \\\"quotes\\\" It's tokens and\"}, {\"role\": \"assistant\", \"content\": \"jumps 100% before body there's a C++ $1,234.56 on exactly exactly every hand lazy dog don't every that so F# Z\\u00fcrich jumps caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 written scanned tokens admission WON'T backtracking scanned 1999 engine brown exactly counting node.js 'single' 1999 counting that keep keep $1,234.56 Z\\u00fcrich They'RE before scanned They'RE \\u6771\\u4eac It's over tiktoken's once hand request tiktoken's while gateway node.js no \\ud83d\\ude42 it https://example.com/a/b?c=d It's 100% written there's hand {braces} there's admission $1,234.56 F# [brackets]\"}, {\"role\": \"user\", \"content\": \"hand mirrors exactly 42 They'RE [brackets] before \\u0645\\u0631\\u062d\\u0628\\u0627 that while written caf\\u00e9 body because fox tokens no WON'T dog faster and fox keep don't caf\\u00e9 'single' node.js (parens) reservation C++ I'M fox F# \\u6771\\u4eac mirrors over 1999 written keep na\\u00efve gateway a 3.14159 keep once body \\\"quotes\\\" F# we'll $1,234.56 https://example.com/a/b?c=d regex fox backtracking is admission request involved so over engine caf\\u00e9 way backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac They'RE It's a mirrors \\\"quotes\\\" jumps They'RE way way the engine C++ request because backtracking with with written the scanned boundaries https://example.com/a/b?c=d (parens) no for gateway dog \\ud83d\\ude42 boundaries node.js\"}, {\"role\": \"assistant\", \"content\": \"and tiktoken's They'RE caf\\u00e9 exactly the quick gateway caf\\u00e9 budget hand node.js the node.js we'll faster fox 'single' involved scanner na\\u00efve reservation https://example.com/a/b?c=d {braces} way we'll backtracking keep while and no with dog exactly the WON'T dog \\u0645\\u0631\\u062d\\u0628\\u0627 regex [brackets] written 1999 keep v1.2.3 I'M mirrors admission \\u6771\\u4eac before we'll \\\"quotes\\\" (parens) 100% over mirrors 42 a request F# WON'T a counting \\ud83d\\ude42 scanner no we'll fox gateway boundaries 1999 WON'T I'M Z\\u00fcrich faster there's caf\\u00e9 They'RE that the with for faster quick that body reservation 42 don't I'M I'M scanned faster hand for\"}, {\"role\": \"user\", \"content\": \"and the admission caf\\u00e9 admission once F# written reservation budget written https://example.com/a/b?c=d scanner a that Z\\u00fcrich once the Z\\u00fcrich faster C++ no I'M counting is hand caf\\u00e9 before engine on C++ 'single' \\\"quotes\\\" I'M Z\\u00fcrich quick that boundaries node.js lazy 3.14159 while na\\u00efve Z\\u00fcrich it reservation request that before F# boundaries once written the WON'T https://example.com/a/b?c=d the written piece mirrors 100% mirrors https://example.com/a/b?c=d over before regex scanner \\u0645\\u0631\\u062d\\u0628\\u0627 before node.js WON'T \\u0645\\u0631\\u062d\\u0628\\u0627 way jumps for scanned that involved for every (parens) lazy C++ boundaries backtracking it user@example.com no that C++ faster engine I'M piece with faster for that brown 3.14159 user@example.com it while so on involved that 42 once quick written we'll a budget\"}, {\"role\": \"assistant\", \"content\": \"admission \\ud83d\\ude42 100% tiktoken's on jumps we'll hand body is tiktoken's and 100% \\ud83d\\ude42 the it 3.14159 $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 scanner over 100% scanned \\ud83d\\ude42 so way engine that scanner 100% so so tokens boundaries node.js we'll jumps user@example.com that tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 once don't way is body so user@example.com on request is lazy and every engine tokens no lazy admission jumps reservation v1.2.3 scanned \\u0645\\u0631\\u062d\\u0628\\u0627 the \\ud83d\\ude42 node.js so \\\"quotes\\\" every it \\u6771\\u4eac F# regex don't faster every $1,234.56 on way engine regex every keep $1,234.56 no that engine written don't involved {braces} \\u6771\\u4eac v1.2.3 while request 42 (parens)\"}, {\"role\": \"user\", \"content\": \"keep fox we'll backtracking once it engine with lazy (parens) faster caf\\u00e9 F# with It's (parens) user@example.com for over counting we'll it https://example.com/a/b?c=d fox the (parens) way over because https://example.com/a/b?c=d before written fox involved written [brackets] before it tokens \\\"quotes\\\" there's once [brackets] 'single' tiktoken's C++ v1.2.3 quick 100% user@example.com dog \\u6771\\u4eac I'M 'single' keep a before node.js F# exactly request the Z\\u00fcrich exactly tokens so faster admission lazy and every counting $1,234.56 brown\"}, {\"role\": \"assistant\", \"content\": \"'single' every budget 42 It's quick way dog {braces} because don't \\u0645\\u0631\\u062d\\u0628\\u0627 way brown involved counting body don't (parens) body tiktoken's scanned \\u0645\\u0631\\u062d\\u0628\\u0627 dog it (parens) the for once once engine written there's that node.js every \\u0645\\u0631\\u062d\\u0628\\u0627 1999 $1,234.56 caf\\u00e9 written body dog gateway for It's WON'T 42 'single' 3.14159 that Z\\u00fcrich jumps C++ while WON'T admission so involved It's \\u0645\\u0631\\u062d\\u0628\\u0627 node.js on with It's it They'RE lazy is engine way scanner $1,234.56 and lazy boundaries 'single' before dog that faster 3.14159 tokens It's tokens we'll once and reservation over They'RE\"}, {\"role\": \"user\", \"content\": \"I'M node.js C++ na\\u00efve once engine with \\ud83d\\ude42 involved 100% brown scanned and is scanner scanned 'single' counting don't fox way way C++ before every faster piece hand They'RE so brown piece because while on a WON'T 1999 backtracking budget \\u6771\\u4eac piece and engine every we'll dog user@example.com na\\u00efve https://example.com/a/b?c=d brown Z\\u00fcrich it way and so hand on \\\"quotes\\\" (parens) before we'll\"}, {\"role\": \"assistant\", \"content\": \"They'RE there's node.js mirrors there's the there's reservation engine is boundaries fox a body fox 42 user@example.com (parens) 100% on (parens) written request https://example.com/a/b?c=d {braces} It's Z\\u00fcrich every counting $1,234.56 scanned once user@example.com C++ so 100% exactly exactly {braces} body jumps on no involved 100% and admission every scanned reservation tokens exactly keep there's Z\\u00fcrich v1.2.3 keep 42 the 42 the with boundaries request involved there's faster keep on https://example.com/a/b?c=d user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 on (parens) na\\u00efve (parens) WON'T \\u6771\\u4eac with engine na\\u00efve body 1999 the engine quick counting boundaries there's counting {braces} for 100% involved there's involved so WON'T lazy a over way keep They'RE tokens keep piece na\\u00efve node.js exactly reservation body every faster backtracking\"}, {\"role\": \"user\", \"content\": \"and scanned C++ jumps They'RE scanned boundaries C++ that hand budget tokens \\\"quotes\\\" scanned I'M 100% 'single' counting because (parens) regex (parens) na\\u00efve F# (parens) I'M and with so once once for regex piece dog quick They'RE while keep exactly before 1999 is 100% keep caf\\u00e9 before 42 hand that reservation 3.14159 because fox that regex body gateway don't once gateway and engine with once don't I'M piece way 3.14159 admission the don't it body piece \\\"quotes\\\" 42 mirrors on body don't 100% that quick backtracking {braces} is we'll and body we'll budget every every v1.2.3 exactly way I'M \\\"quotes\\\" involved gateway scanned once \\u0645\\u0631\\u062d\\u0628\\u0627 keep jumps \\u6771\\u4eac backtracking dog engine \\u6771\\u4eac tiktoken's that 42 user@example.com don't scanner (parens) I'M\"}, {\"role\": \"assistant\", \"content\": \"the piece 1999 over v1.2.3 C++ It's https://example.com/a/b?c=d because request that fox \\ud83d\\ude42 way \\u6771\\u4eac tokens tokens brown (parens) way v1.2.3 that na\\u00efve mirrors \\\"quotes\\\" admission dog 100% 100% regex backtracking reservation 1999 user@example.com \\\"quotes\\\" 3.14159 I'M budget mirrors 1999 lazy admission a on that user@example.com They'RE They'RE \\\"quotes\\\" user@example.com node.js 100% so na\\u00efve and It's \\\"quotes\\\" with the piece boundaries we'll 42 42 while https://example.com/a/b?c=d user@example.com budget faster na\\u00efve and 1999 a admission fox and 'single' for They'RE boundaries scanned\"}, {\"role\": \"user\", \"content\": \"quick gateway They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 fox the (parens) mirrors because while we'll we'll every mirrors counting Z\\u00fcrich 42 on gateway WON'T written backtracking no user@example.com caf\\u00e9 \\u6771\\u4eac that boundaries while so fox backtracking involved They'RE exactly 1999 {braces} [brackets] exactly regex mirrors \\u6771\\u4eac 1999 over reservation way involved \\u0645\\u0631\\u062d\\u0628\\u0627 42 3.14159 while quick so a (parens) hand there's F# a They'RE body engine F# v1.2.3 faster hand over dog backtracking while brown that before I'M 1999 way before piece counting request that budget boundaries v1.2.3 exactly that They'RE faster involved \\\"quotes\\\" scanner C++ It's 100% $1,234.56 written\"}, {\"role\": \"assistant\", \"content\": \"https://example.com/a/b?c=d way \\\"quotes\\\" fox fox C++ is admission It's \\ud83d\\ude42 tiktoken's WON'T tokens so caf\\u00e9 way admission a scanned $1,234.56 3.14159 that F# don't fox counting every engine scanner scanned don't 'single' tiktoken's https://example.com/a/b?c=d I'M keep there's piece \\u6771\\u4eac is while way hand over fox WON'T \\u6771\\u4eac scanner v1.2.3 \\u6771\\u4eac don't written \\\"quotes\\\" keep the body and brown that counting before budget request 3.14159 boundaries {braces}\"}, {\"role\": \"user\", \"content\": \"exactly It's 3.14159 hand body They'RE tokens don't v1.2.3 backtracking on quick (parens) 100% body quick so scanner so \\ud83d\\ude42 backtracking is once exactly regex https://example.com/a/b?c=d na\\u00efve before there's every C++ [brackets] tokens They'RE with the 1999 piece a reservation request caf\\u00e9 it 'single' while \\\"quotes\\\" boundaries because lazy \\u0645\\u0631\\u062d\\u0628\\u0627 for Z\\u00fcrich It's (parens) a 100% there's dog caf\\u00e9 \\ud83d\\ude42 (parens) because quick with node.js https://example.com/a/b?c=d engine piece \\ud83d\\ude42 counting brown way It's user@example.com user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 scanned reservation 'single' don't request admission\"}, {\"role\": \"assistant\", \"content\": \"reservation it so before no scanned backtracking a 1999 every exactly jumps 100% over It's 1999 we'll boundaries don't scanned once and before \\u6771\\u4eac faster backtracking regex backtracking every 'single' admission caf\\u00e9 brown 3.14159 \\ud83d\\ude42 there's \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T 42 $1,234.56 is before reservation it admission backtracking before over $1,234.56 {braces} mirrors It's {braces} before admission quick hand lazy scanned \\ud83d\\ude42 exactly faster while reservation C++ They'RE it exactly gateway it body for quick so quick 3.14159 1999 for user@example.com involved {braces} https://example.com/a/b?c=d quick while I'M lazy brown backtracking keep\"}, {\"role\": \"user\", \"content\": \"quick keep v1.2.3 request budget a and regex keep body gateway so It's before \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 I'M 3.14159 and written counting {braces} v1.2.3 for brown engine user@example.com mirrors mirrors over \\u6771\\u4eac (parens) regex jumps keep written and Z\\u00fcrich a no dog and jumps piece C++ the counting with quick on the user@example.com request is jumps boundaries quick 'single' 100% 'single' over 100% the over there's na\\u00efve 1999 engine quick https://example.com/a/b?c=d 3.14159 it jumps way no hand \\ud83d\\ude42 [brackets] {braces} admission F# scanner 3.14159 F# it 3.14159 boundaries budget keep don't that quick 'single' reservation C++ the gateway and dog exactly body so https://example.com/a/b?c=d \\\"quotes\\\" 100%\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 budget \\\"quotes\\\" quick once body I'M that way no once a user@example.com boundaries admission gateway engine caf\\u00e9 $1,234.56 They'RE They'RE reservation user@example.com They'RE They'RE budget for \\u0645\\u0631\\u062d\\u0628\\u0627 dog is WON'T faster involved lazy so while faster backtracking 1999 user@example.com we'll engine reservation v1.2.3 it on tokens counting \\ud83d\\ude42 every every over written it every tiktoken's no before (parens) body the request quick every admission the \\ud83d\\ude42 body don't admission regex there's \\u6771\\u4eac \\\"quotes\\\" dog don't no Z\\u00fcrich $1,234.56 and scanned scanned https://example.com/a/b?c=d request while way written engine reservation there's scanned the request that jumps it caf\\u00e9 and v1.2.3 scanned na\\u00efve involved brown no user@example.com tokens because counting there's \\\"quotes\\\" on node.js quick exactly every that written\"}, {\"role\": \"user\", \"content\": \"counting {braces} \\ud83d\\ude42 C++ admission boundaries C++ it 'single' that mirrors jumps the while that don't They'RE while 42 Z\\u00fcrich 1999 scanner so quick v1.2.3 there's don't keep Z\\u00fcrich no keep faster reservation request is (parens) body we'll 100% don't na\\u00efve (parens) for backtracking fox boundaries backtracking v1.2.3 $1,234.56 we'll 42 tokens faster counting once keep budget fox v1.2.3 'single' mirrors no engine exactly fox 42 way the C++ quick tiktoken's counting 1999 It's we'll counting because hand \\ud83d\\ude42 user@example.com is request quick 42 it keep don't They'RE WON'T once and fox v1.2.3 and with na\\u00efve way lazy user@example.com written is 'single' brown scanned request is caf\\u00e9 we'll node.js a so while we'll WON'T way admission They'RE fox C++ once node.js WON'T that v1.2.3 every\"}, {\"role\": \"assistant\", \"content\": \"C++ don't no boundaries scanner https://example.com/a/b?c=d node.js backtracking hand [brackets] keep F# counting caf\\u00e9 scanner v1.2.3 https://example.com/a/b?c=d counting no https://example.com/a/b?c=d the backtracking it I'M I'M F# involved WON'T with exactly (parens) brown body v1.2.3 body https://example.com/a/b?c=d keep it It's 1999 F# \\u6771\\u4eac so \\ud83d\\ude42 scanner for user@example.com v1.2.3 I'M don't fox written request engine once regex counting tokens the admission (parens) admission reservation faster C++ 3.14159 tokens hand \\\"quotes\\\" counting caf\\u00e9 [brackets] It's don't Z\\u00fcrich 3.14159 the tiktoken's I'M v1.2.3 admission C++ They'RE hand tokens dog no $1,234.56 engine while boundaries so caf\\u00e9 100% \\u0645\\u0631\\u062d\\u0628\\u0627 F# scanned jumps faster while na\\u00efve lazy\"}, {\"role\": \"user\", \"content\": \"WON'T hand keep brown \\u0645\\u0631\\u062d\\u0628\\u0627 because counting faster with that involved is because because 42 Z\\u00fcrich that engine with the the brown {braces} tiktoken's [brackets] I'M It's over (parens) C++ It's over faster \\u6771\\u4eac user@example.com user@example.com 100% on it on with mirrors 3.14159 42 budget It's WON'T node.js keep tiktoken's \\u6771\\u4eac https://example.com/a/b?c=d 1999 involved body scanned hand involved engine faster every is and that tokens every 42 on dog admission (parens) body for caf\\u00e9 once before a \\u6771\\u4eac before with don't tokens It's because \\\"quotes\\\" node.js na\\u00efve it engine is backtracking [brackets] regex brown They'RE so is is involved engine so scanner gateway scanned They'RE keep na\\u00efve body reservation 42 scanned WON'T faster there's backtracking it caf\\u00e9 v1.2.3 brown regex {braces} lazy node.js C++ don't written WON'T with user@example.com piece over we'll v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich jumps (parens)\"}, {\"role\": \"assistant\", \"content\": \"na\\u00efve scanned every reservation no we'll faster before it {braces} admission 1999 scanned budget written there's WON'T $1,234.56 faster brown reservation (parens) 'single' every (parens) $1,234.56 It's is counting way jumps no scanned I'M the node.js that na\\u00efve over faster [brackets] 1999 there's keep user@example.com 100% user@example.com before once reservation brown don't C++ dog it over backtracking every \\\"quotes\\\" quick a budget \\ud83d\\ude42 budget because v1.2.3 dog 100% once v1.2.3 and and every don't 3.14159 once once quick gateway before for 3.14159 v1.2.3 \\u6771\\u4eac tokens that on because it budget a involved scanned scanner lazy 'single' caf\\u00e9 They'RE engine request while Z\\u00fcrich engine WON'T\"}, {\"role\": \"user\", \"content\": \"100% over written reservation https://example.com/a/b?c=d WON'T scanner F# before https://example.com/a/b?c=d we'll that while counting is admission 42 caf\\u00e9 C++ body tiktoken's body 3.14159 the I'M with node.js It's \\u6771\\u4eac [brackets] don't https://example.com/a/b?c=d C++ before on [brackets] 'single' piece brown before 1999 {braces} written {braces} way reservation request F# so https://example.com/a/b?c=d 100% $1,234.56 piece piece keep https://example.com/a/b?c=d way budget 100% boundaries node.js lazy because F# tokens (parens) and C++ backtracking tiktoken's piece a 3.14159 WON'T scanned backtracking node.js because regex backtracking so keep 1999 the is because WON'T node.js we'll lazy quick with once and once \\\"quotes\\\" we'll keep It's the on It's so is faster\"}, {\"role\": \"assistant\", \"content\": \"hand every for written mirrors backtracking $1,234.56 It's WON'T because faster Z\\u00fcrich don't don't {braces} jumps 'single' regex gateway user@example.com once there's a on over \\\"quotes\\\" 100% node.js regex a body that F# 100% once on 3.14159 while backtracking v1.2.3 hand \\u6771\\u4eac tokens admission \\\"quotes\\\" 100% admission and It's Z\\u00fcrich that so while and $1,234.56 caf\\u00e9 every F# involved fox backtracking the \\u0645\\u0631\\u062d\\u0628\\u0627 and https://example.com/a/b?c=d exactly node.js before mirrors 'single' exactly tokens that scanned body https://example.com/a/b?c=d na\\u00efve once it the the faster v1.2.3 scanned fox that jumps v1.2.3 1999 gateway F# caf\\u00e9 'single' fox brown [brackets] 100% over user@example.com on na\\u00efve https://example.com/a/b?c=d node.js They'RE and scanner 'single' involved the because hand scanner fox user@example.com\"}, {\"role\": \"user\", \"content\": \"1999 $1,234.56 that 'single' no counting (parens) because Z\\u00fcrich on (parens) a fox user@example.com admission over is there's \\\"quotes\\\" {braces} exactly piece that with I'M F# over for counting and \\ud83d\\ude42 scanner over every \\ud83d\\ude42 100% admission before \\u0645\\u0631\\u062d\\u0628\\u0627 reservation WON'T brown scanner on faster 100% caf\\u00e9 piece [brackets] counting scanner It's written {braces} C++ that is boundaries exactly \\u0645\\u0631\\u062d\\u0628\\u0627 once 'single' quick F# hand 'single' user@example.com reservation jumps it F# reservation request that 'single' (parens) the way WON'T (parens) a backtracking backtracking that \\\"quotes\\\" C++ I'M C++ C++ gateway with body body\"}, {\"role\": \"assistant\", \"content\": \"budget that 42 It's body backtracking caf\\u00e9 1999 because hand hand F# piece is (parens) and Z\\u00fcrich jumps user@example.com don't I'M They'RE regex body reservation once (parens) with F# piece is every node.js no piece because {braces} with 42 Z\\u00fcrich tiktoken's keep I'M is scanner no body 'single' 1999 that request so \\u0645\\u0631\\u062d\\u0628\\u0627 3.14159 \\u6771\\u4eac and https://example.com/a/b?c=d engine tiktoken's on while $1,234.56 over budget 1999 1999 backtracking is \\ud83d\\ude42 the tiktoken's involved over don't Z\\u00fcrich I'M so gateway written dog before written v1.2.3 backtracking gateway keep dog [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9\"}, {\"role\": \"user\", \"content\": \"exactly involved user@example.com fox keep boundaries and 42 reservation {braces} mirrors the It's budget C++ tiktoken's budget mirrors faster [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 over scanner way quick while \\\"quotes\\\" exactly 'single' F# no faster [brackets] backtracking scanner WON'T WON'T tokens quick that the node.js it regex user@example.com involved while boundaries regex while hand mirrors way na\\u00efve a hand request that before v1.2.3 1999 3.14159 caf\\u00e9 because on over scanner so body tokens quick because counting exactly hand user@example.com that It's\"}, {\"role\": \"assistant\", \"content\": \"so quick dog (parens) Z\\u00fcrich backtracking for the \\ud83d\\ude42 because regex involved tokens dog we'll 'single' we'll Z\\u00fcrich once 100% regex we'll [brackets] 42 while with 1999 Z\\u00fcrich fox admission while written the C++ over every mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 on it https://example.com/a/b?c=d scanned gateway no 1999 dog https://example.com/a/b?c=d we'll 1999 before the every budget on 1999 with piece 1999 faster user@example.com I'M body keep while the once scanned I'M\"}, {\"role\": \"user\", \"content\": \"tiktoken's quick budget on budget the \\u0645\\u0631\\u062d\\u0628\\u0627 for Z\\u00fcrich I'M don't WON'T we'll user@example.com It's jumps \\u6771\\u4eac we'll involved gateway it that jumps for for counting that $1,234.56 so while 'single' WON'T quick the brown we'll once mirrors [brackets] \\u6771\\u4eac \\\"quotes\\\" boundaries way for because {braces} it user@example.com faster $1,234.56 no a 'single' body backtracking before the 3.14159 scanner {braces} backtracking hand because so body [brackets] because that involved scanned WON'T there's \\u0645\\u0631\\u062d\\u0628\\u0627 it hand we'll dog a before the and faster \\ud83d\\ude42 hand 100% on body [brackets] 42 the node.js it counting and jumps we'll a fox 3.14159 once (parens) \\ud83d\\ude42 hand every don't way jumps body over involved \\u6771\\u4eac is budget way scanned $1,234.56 because request gateway involved the that dog the reservation while \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"counting written brown on once it They'RE involved we'll every hand \\ud83d\\ude42 it They'RE dog for reservation on every no tokens regex piece on (parens) tiktoken's before gateway it every the while that we'll gateway a the a v1.2.3 the over lazy node.js gateway budget WON'T \\ud83d\\ude42 is exactly jumps over lazy piece backtracking written with 3.14159 jumps involved lazy scanner keep keep [brackets] boundaries that no that because quick F# v1.2.3 reservation involved\"}, {\"role\": \"user\", \"content\": \"caf\\u00e9 engine request na\\u00efve jumps involved lazy regex scanned {braces} 3.14159 engine mirrors \\\"quotes\\\" Z\\u00fcrich on there's (parens) They'RE regex for over regex 1999 tiktoken's scanner that once admission F# mirrors 42 the body WON'T the WON'T dog 100% 42 is lazy \\ud83d\\ude42 that request once 100% na\\u00efve dog tokens budget jumps \\ud83d\\ude42 $1,234.56 user@example.com $1,234.56 don't admission that brown (parens) it node.js tokens na\\u00efve we'll it v1.2.3 once so is written admission faster way we'll request jumps v1.2.3 fox the lazy gateway $1,234.56 42 counting Z\\u00fcrich the 'single' exactly once way I'M \\u6771\\u4eac They'RE before caf\\u00e9 boundaries over counting piece brown faster while so counting na\\u00efve hand \\ud83d\\ude42 quick the every Z\\u00fcrich {braces} scanned \\\"quotes\\\" with regex keep while once engine before fox\"}, {\"role\": \"assistant\", \"content\": \"mirrors {braces} because \\u6771\\u4eac mirrors scanned exactly budget way mirrors https://example.com/a/b?c=d boundaries involved 100% 3.14159 exactly engine brown They'RE is keep that hand lazy exactly tokens It's keep every \\ud83d\\ude42 the F# written {braces} user@example.com the counting $1,234.56 keep regex tiktoken's and mirrors fox the gateway \\ud83d\\ude42 keep They'RE written I'M there's we'll na\\u00efve WON'T is brown C++ involved brown and piece \\ud83d\\ude42 reservation [brackets] reservation I'M \\\"quotes\\\" while exactly way https://example.com/a/b?c=d don't \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac over keep scanner \\u6771\\u4eac scanned over tiktoken's boundaries mirrors once for quick faster a \\u6771\\u4eac lazy (parens) na\\u00efve gateway \\u6771\\u4eac \\u6771\\u4eac brown They'RE there's on keep once dog that 1999 [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 while it It's no\"}, {\"role\": \"user\", \"content\": \"that body over Z\\u00fcrich before that dog keep and \\\"quotes\\\" and regex tiktoken's way piece is scanner is quick C++ node.js written dog regex the It's dog https://example.com/a/b?c=d scanned engine we'll counting it 'single' counting a boundaries is \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 brown budget fox the Z\\u00fcrich jumps {braces} that boundaries piece because the 'single' v1.2.3 {braces} once that mirrors \\ud83d\\ude42 backtracking no no mirrors on on scanned F# They'RE regex scanned the every for faster v1.2.3 \\ud83d\\ude42 I'M we'll exactly that is once for because it caf\\u00e9 It's caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 keep budget admission\"}, {\"role\": \"assistant\", \"content\": \"and node.js is so admission body They'RE https://example.com/a/b?c=d the 1999 we'll It's keep mirrors request so piece gateway engine way hand exactly is C++ 100% exactly for body piece with jumps WON'T the no keep WON'T so F# don't $1,234.56 {braces} and \\ud83d\\ude42 request reservation written tiktoken's that written way written the tokens 3.14159 WON'T admission \\u0645\\u0631\\u062d\\u0628\\u0627 C++ budget gateway every before brown WON'T piece 1999 [brackets] a dog [brackets] while scanner \\ud83d\\ude42 we'll v1.2.3 scanner quick dog a while admission hand so there's every 42 node.js that engine faster na\\u00efve is \\\"quotes\\\"\"}, {\"role\": \"user\", \"content\": \"scanner hand boundaries admission it \\u6771\\u4eac WON'T na\\u00efve user@example.com Z\\u00fcrich F# F# They'RE a is caf\\u00e9 \\\"quotes\\\" before on request $1,234.56 boundaries (parens) don't brown the every 1999 (parens) [brackets] so 1999 boundaries 100% the hand 42 tokens every over that the the F# that quick They'RE before because tokens caf\\u00e9 we'll exactly boundaries because brown keep no 'single' there's brown v1.2.3 user@example.com 42 1999 \\u6771\\u4eac {braces} for mirrors on tiktoken's WON'T \\\"quotes\\\" na\\u00efve They'RE {braces} v1.2.3 brown written involved tokens jumps boundaries budget jumps boundaries way 42 body written na\\u00efve written is request every admission counting before backtracking with They'RE fox\"}, {\"role\": \"assistant\", \"content\": \"\\ud83d\\ude42 we'll backtracking involved counting request node.js caf\\u00e9 lazy $1,234.56 so I'M while is every it 42 exactly node.js that They'RE there's I'M I'M a and counting admission no 'single' admission v1.2.3 request boundaries 'single' tiktoken's budget gateway 100% caf\\u00e9 jumps hand no the \\u6771\\u4eac I'M we'll fox 1999 piece and body {braces} na\\u00efve hand Z\\u00fcrich v1.2.3 over quick for 100% while backtracking scanned scanner scanner request for hand fox the that hand scanned It's while (parens) written\"}, {\"role\": \"user\", \"content\": \"body there's scanned na\\u00efve quick It's a request lazy hand 1999 https://example.com/a/b?c=d jumps and caf\\u00e9 every It's before we'll on body no lazy there's 'single' mirrors we'll tokens over (parens) while \\u0645\\u0631\\u062d\\u0628\\u0627 C++ v1.2.3 keep gateway it because request mirrors for there's $1,234.56 keep over gateway https://example.com/a/b?c=d brown before They'RE 42 [brackets] tokens once once while scanner scanned and I'M Z\\u00fcrich it involved na\\u00efve I'M hand exactly before node.js 'single' because no lazy 3.14159 It's scanner F# 'single' while exactly It's (parens) \\ud83d\\ude42 counting while for tokens backtracking fox (parens) v1.2.3 once quick because is there's reservation because It's node.js that regex for don't\"}, {\"role\": \"assistant\", \"content\": \"with with so written budget is jumps admission before that I'M \\\"quotes\\\" because 3.14159 on WON'T \\\"quotes\\\" caf\\u00e9 is for scanned engine so engine on admission no body before once node.js node.js [brackets] They'RE Z\\u00fcrich written the \\ud83d\\ude42 reservation once so it a budget the request \\u6771\\u4eac https://example.com/a/b?c=d request \\u0645\\u0631\\u062d\\u0628\\u0627 a the 3.14159 with mirrors because body [brackets] exactly \\ud83d\\ude42 the a 1999 every lazy 'single' the request keep once dog user@example.com the scanned \\\"quotes\\\" is regex scanner every hand keep faster request quick WON'T while 3.14159 It's 3.14159 mirrors reservation written the a It's don't reservation C++ It's 42 v1.2.3 involved reservation lazy keep \\\"quotes\\\" the body \\ud83d\\ude42 while 3.14159 node.js dog no user@example.com budget [brackets] hand over\"}, {\"role\": \"user\", \"content\": \"before 1999 \\ud83d\\ude42 on that don't Z\\u00fcrich keep a way for request \\u6771\\u4eac every \\\"quotes\\\" https://example.com/a/b?c=d 3.14159 once for every hand that WON'T 'single' request written jumps regex F# once quick \\u6771\\u4eac dog na\\u00efve the node.js way [brackets] gateway (parens) piece exactly \\ud83d\\ude42 v1.2.3 jumps They'RE written fox the scanner exactly WON'T na\\u00efve so body faster brown the that is with exactly Z\\u00fcrich [brackets] Z\\u00fcrich budget it lazy and the engine budget 3.14159 counting \\ud83d\\ude42 https://example.com/a/b?c=d it WON'T $1,234.56 {braces} a because we'll exactly it with request the that don't scanner jumps involved and tiktoken's the quick so \\ud83d\\ude42 [brackets] regex engine involved is\"}, {\"role\": \"assistant\", \"content\": \"They'RE body because engine 100% WON'T {braces} the tiktoken's before node.js it every na\\u00efve lazy the don't caf\\u00e9 admission 1999 faster dog 3.14159 {braces} once tokens we'll before the admission WON'T the it and It's because engine with user@example.com it \\u6771\\u4eac C++ over is tokens piece exactly it piece backtracking gateway node.js no the 3.14159 exactly regex fox \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors a 1999 v1.2.3 request 100% scanned reservation node.js I'M the 100% so quick before piece brown caf\\u00e9 gateway 42 involved It's involved admission Z\\u00fcrich and {braces} It's quick \\ud83d\\ude42 body fox counting\"}, {\"role\": \"user\", \"content\": \"I'M admission regex that involved engine it \\ud83d\\ude42 counting every dog hand before scanned the 100% we'll exactly Z\\u00fcrich \\u6771\\u4eac v1.2.3 engine na\\u00efve node.js body don't lazy tiktoken's gateway don't node.js request F# scanner there's 100% scanner fox the a 42 quick na\\u00efve admission while 1999 way regex there's \\\"quotes\\\" scanned is so before tiktoken's involved [brackets] quick 1999 \\u6771\\u4eac 1999 for lazy v1.2.3 regex lazy body F# hand boundaries once mirrors so brown \\\"quotes\\\" dog $1,234.56 tokens that once request hand tokens lazy lazy [brackets] quick don't mirrors quick don't lazy It's body boundaries mirrors F# [brackets] there's while 100% body\"}, {\"role\": \"assistant\", \"content\": \"budget before over They'RE I'M request is that while written before $1,234.56 {braces} F# that admission and with for [brackets] fox \\ud83d\\ude42 42 body \\\"quotes\\\" the F# reservation dog 3.14159 WON'T It's tiktoken's exactly regex 3.14159 there's involved backtracking don't 1999 with faster dog body before tiktoken's with and over 42 written there's $1,234.56 hand 42 \\\"quotes\\\" lazy gateway backtracking 100% because \\\"quotes\\\" hand caf\\u00e9 faster request on over \\\"quotes\\\" once scanned over 100% engine mirrors \\u6771\\u4eac gateway user@example.com fox every before [brackets] body admission WON'T engine no dog for node.js gateway node.js na\\u00efve piece the way scanner a the 'single' gateway user@example.com They'RE is don't \\u6771\\u4eac before scanned\"}, {\"role\": \"user\", \"content\": \"It's way that \\u0645\\u0631\\u062d\\u0628\\u0627 written tiktoken's over faster It's with WON'T it because faster because with backtracking Z\\u00fcrich the while because scanned mirrors that scanned They'RE fox exactly 100% It's the tiktoken's hand so user@example.com request \\u6771\\u4eac faster v1.2.3 quick on once over dog piece counting node.js They'RE the because (parens) hand F# \\u6771\\u4eac with hand dog over budget written Z\\u00fcrich jumps there's budget C++ backtracking v1.2.3 'single' don't fox They'RE for \\ud83d\\ude42 admission involved v1.2.3 3.14159 the while 42 quick regex 100% on node.js tokens scanner written every with They'RE \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"100% because user@example.com caf\\u00e9 way 1999 Z\\u00fcrich over written 'single' 3.14159 $1,234.56 tokens I'M C++ [brackets] na\\u00efve $1,234.56 we'll $1,234.56 node.js we'll it (parens) {braces} v1.2.3 with written admission 100% exactly {braces} reservation there's {braces} fox with engine na\\u00efve for 1999 brown hand while dog backtracking backtracking na\\u00efve user@example.com tokens \\u0645\\u0631\\u062d\\u0628\\u0627 body brown don't C++ piece over scanner (parens) 100% no F# keep there's v1.2.3 [brackets] we'll don't hand dog every engine 'single' v1.2.3\"}, {\"role\": \"user\", \"content\": \"before and every $1,234.56 caf\\u00e9 the mirrors reservation https://example.com/a/b?c=d 3.14159 the quick gateway exactly it $1,234.56 1999 v1.2.3 $1,234.56 it on node.js backtracking (parens) there's \\ud83d\\ude42 no quick regex \\ud83d\\ude42 before once \\u0645\\u0631\\u062d\\u0628\\u0627 involved gateway caf\\u00e9 admission $1,234.56 caf\\u00e9 backtracking scanner na\\u00efve a engine [brackets] a way written is caf\\u00e9 1999 gateway 100% boundaries [brackets] backtracking there's tokens while 42 na\\u00efve counting it C++ v1.2.3 100% v1.2.3 scanned \\u0645\\u0631\\u062d\\u0628\\u0627 'single' \\u6771\\u4eac WON'T there's keep It's so WON'T\"}, {\"role\": \"assistant\", \"content\": \"keep 42 because quick and because faster is 'single' scanned fox regex {braces} C++ I'M it 100% written tiktoken's engine so scanned reservation \\ud83d\\ude42 3.14159 \\\"quotes\\\" before the that exactly for with there's 100% over with WON'T It's quick Z\\u00fcrich v1.2.3 'single' the tiktoken's (parens) while we'll written [brackets] reservation the on engine way na\\u00efve caf\\u00e9 exactly $1,234.56 https://example.com/a/b?c=d before written 'single' (parens) piece They'RE Z\\u00fcrich written counting the every that written na\\u00efve hand admission over piece Z\\u00fcrich we'll no quick https://example.com/a/b?c=d keep we'll while 'single' request quick on we'll tiktoken's on regex Z\\u00fcrich jumps written that a because mirrors\"}, {\"role\": \"user\", \"content\": \"\\\"quotes\\\" reservation na\\u00efve there's 100% dog reservation [brackets] 3.14159 F# node.js \\u0645\\u0631\\u062d\\u0628\\u0627 the fox boundaries C++ na\\u00efve before It's so written brown C++ lazy 3.14159 100% [brackets] tokens lazy tiktoken's {braces} with and and fox I'M for the exactly gateway keep backtracking that fox Z\\u00fcrich quick lazy every \\u0645\\u0631\\u062d\\u0628\\u0627 hand Z\\u00fcrich \\u6771\\u4eac caf\\u00e9 C++ there's keep the over v1.2.3 mirrors user@example.com It's They'RE tiktoken's 1999 because piece jumps it dog a budget boundaries brown exactly piece lazy 100% so scanned https://example.com/a/b?c=d there's that don't the engine fox tiktoken's 42 $1,234.56 \\u6771\\u4eac keep the dog admission boundaries piece It's mirrors fox user@example.com boundaries I'M faster reservation It's for no fox \\u0645\\u0631\\u062d\\u0628\\u0627 and because boundaries budget involved written They'RE v1.2.3 admission while brown F# so 'single' body that\"}, {\"role\": \"assistant\", \"content\": \"body tokens exactly over dog Z\\u00fcrich WON'T exactly F# no engine and \\ud83d\\ude42 scanner way {braces} scanner jumps tokens piece [brackets] hand that na\\u00efve \\\"quotes\\\" dog for node.js the v1.2.3 that They'RE once gateway so {braces} with keep that regex no [brackets] admission on caf\\u00e9 fox scanner tokens don't before quick They'RE budget dog C++ once jumps user@example.com a node.js (parens) because WON'T \\ud83d\\ude42 while request 3.14159 with because there's regex don't Z\\u00fcrich written user@example.com with while request keep They'RE mirrors 1999 over involved \\u6771\\u4eac $1,234.56 C++ counting involved that tokens way for \\ud83d\\ude42 regex jumps we'll because na\\u00efve lazy with I'M don't before regex reservation fox that so\"}, {\"role\": \"user\", \"content\": \"involved because dog 1999 exactly 1999 keep boundaries node.js tiktoken's once Z\\u00fcrich [brackets] way na\\u00efve v1.2.3 faster C++ scanner mirrors \\u6771\\u4eac \\u6771\\u4eac [brackets] hand \\ud83d\\ude42 node.js tiktoken's don't counting exactly tiktoken's keep WON'T while with 42 brown it way Z\\u00fcrich over because keep before way {braces} the before way boundaries that \\u6771\\u4eac 1999 request mirrors over It's gateway with scanner It's tiktoken's Z\\u00fcrich gateway faster we'll Z\\u00fcrich admission\"}, {\"role\": \"assistant\", \"content\": \"once They'RE node.js we'll \\u0645\\u0631\\u062d\\u0628\\u0627 don't Z\\u00fcrich admission {braces} It's \\ud83d\\ude42 \\u6771\\u4eac gateway brown is so scanner [brackets] {braces} there's tiktoken's a \\u0645\\u0631\\u062d\\u0628\\u0627 node.js and [brackets] don't counting \\\"quotes\\\" They'RE every mirrors reservation body on piece on regex node.js a 'single' \\u6771\\u4eac tiktoken's \\\"quotes\\\" for scanned there's faster gateway They'RE mirrors jumps https://example.com/a/b?c=d faster tokens every WON'T WON'T that request budget It's counting we'll scanner faster tokens that [brackets] tiktoken's there's F# lazy the reservation on the because with lazy user@example.com $1,234.56 jumps gateway C++ [brackets]\"}, {\"role\": \"user\", \"content\": \"over on regex lazy piece don't (parens) don't so https://example.com/a/b?c=d WON'T v1.2.3 brown node.js 100% \\u0645\\u0631\\u062d\\u0628\\u0627 over admission {braces} https://example.com/a/b?c=d scanned hand It's tiktoken's C++ quick don't They'RE {braces} WON'T scanner there's admission the body $1,234.56 dog \\u6771\\u4eac Z\\u00fcrich involved (parens) for so hand 3.14159 so scanned \\u0645\\u0631\\u062d\\u0628\\u0627 42 100% \\u6771\\u4eac counting tiktoken's 1999 fox keep reservation caf\\u00e9 there's tokens quick F# no is v1.2.3 body \\ud83d\\ude42 brown dog way boundaries scanned node.js quick over admission user@example.com boundaries faster regex on 42 admission na\\u00efve engine lazy WON'T that user@example.com 100% I'M with because faster exactly way for faster C++ because scanner written the \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"there's for I'M on reservation once once mirrors the body body 3.14159 fox dog \\\"quotes\\\" written we'll \\ud83d\\ude42 regex with user@example.com fox 1999 Z\\u00fcrich na\\u00efve is hand hand before on for over piece exactly that the exactly 100% caf\\u00e9 that brown counting way admission 100% 1999 v1.2.3 don't \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} mirrors it once piece Z\\u00fcrich gateway caf\\u00e9 and the budget that because hand scanner tokens request regex for exactly over piece F# we'll [brackets] I'M while and fox scanner \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} on fox quick mirrors It's scanner the scanner no it because node.js 'single' regex written caf\\u00e9 v1.2.3 https://example.com/a/b?c=d {braces} the don't jumps It's hand 'single' scanned 3.14159 involved every fox over we'll keep there's mirrors written on I'M WON'T F# that F# that 3.14159 1999 keep 'single'\"}, {\"role\": \"user\", \"content\": \"exactly involved caf\\u00e9 that 1999 over a there's it F# exactly \\ud83d\\ude42 tokens we'll don't regex fox because that while involved backtracking involved reservation there's over They'RE admission 1999 regex counting keep $1,234.56 for engine it 'single' regex the that body v1.2.3 WON'T once It's 42 tiktoken's written no 'single' piece fox and before it tiktoken's a caf\\u00e9 v1.2.3 user@example.com quick user@example.com scanned admission the scanner on is over reservation request \\u6771\\u4eac we'll 3.14159 with before \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking 3.14159 counting reservation don't way with WON'T involved before It's admission keep C++ written with counting 'single' brown the 100% They'RE 42 before while every and \\\"quotes\\\" for They'RE there's 42 tokens v1.2.3 https://example.com/a/b?c=d before involved there's body once\"}, {\"role\": \"assistant\", \"content\": \"Z\\u00fcrich backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 the 3.14159 involved \\\"quotes\\\" 42 node.js once scanner mirrors {braces} na\\u00efve I'M brown the admission over They'RE piece it faster so request don't over way na\\u00efve (parens) once mirrors 100% don't scanner Z\\u00fcrich hand budget fox there's there's boundaries Z\\u00fcrich don't \\u6771\\u4eac scanner 42 v1.2.3 \\\"quotes\\\" na\\u00efve 3.14159 100% every we'll 3.14159 quick 1999 backtracking {braces} don't lazy dog lazy a dog the tokens with body that quick brown tokens lazy \\u6771\\u4eac piece piece every dog every 1999 \\\"quotes\\\" mirrors faster fox the 3.14159 way piece boundaries user@example.com there's caf\\u00e9 faster on F# https://example.com/a/b?c=d we'll admission \\u0645\\u0631\\u062d\\u0628\\u0627 that written piece scanned counting and admission we'll Z\\u00fcrich dog 3.14159 node.js every engine Z\\u00fcrich body involved backtracking mirrors 100% 100%\"}, {\"role\": \"user\", \"content\": \"(parens) that on 3.14159 \\u6771\\u4eac over is 42 \\\"quotes\\\" 'single' body faster exactly mirrors F# way (parens) no exactly mirrors that \\ud83d\\ude42 $1,234.56 faster brown Z\\u00fcrich a brown tiktoken's \\ud83d\\ude42 tokens regex while They'RE I'M budget node.js https://example.com/a/b?c=d 42 tiktoken's https://example.com/a/b?c=d over over every 'single' 100% tiktoken's scanner gateway for don't tiktoken's a hand v1.2.3 {braces} fox that quick every that fox faster exactly 'single' (parens) v1.2.3 request dog over no is jumps for Z\\u00fcrich WON'T so body once scanner 3.14159 every that \\u6771\\u4eac so\"}, {\"role\": \"assistant\", \"content\": \"no while faster the because there's \\\"quotes\\\" piece tiktoken's a reservation Z\\u00fcrich because caf\\u00e9 gateway tokens gateway over Z\\u00fcrich They'RE that 3.14159 quick They'RE na\\u00efve mirrors faster a 100% reservation I'M on F# lazy 1999 and budget and $1,234.56 exactly budget written fox 100% body and there's don't once body the on while budget \\ud83d\\ude42 100% \\\"quotes\\\" request \\u0645\\u0631\\u062d\\u0628\\u0627 keep lazy 1999 before piece that before involved \\ud83d\\ude42 written don't gateway the once user@example.com (parens) that backtracking budget request involved the v1.2.3 42 hand tokens mirrors I'M written \\u6771\\u4eac \\\"quotes\\\" body [brackets] written 'single' brown so request 3.14159 node.js budget node.js dog (parens) scanner {braces} a WON'T v1.2.3 because there's that engine tiktoken's every no over before on mirrors mirrors don't \\u0645\\u0631\\u062d\\u0628\\u0627 budget for quick 42 quick over lazy over v1.2.3 gateway \\u0645\\u0631\\u062d\\u0628\\u0627 and\"}, {\"role\": \"user\", \"content\": \"that caf\\u00e9 user@example.com hand over we'll no that Z\\u00fcrich brown 42 so fox counting quick every regex https://example.com/a/b?c=d brown once body 'single' reservation v1.2.3 $1,234.56 I'M that on \\ud83d\\ude42 F# scanner faster over F# we'll dog because mirrors \\ud83d\\ude42 we'll scanned regex budget on and I'M admission is written It's fox na\\u00efve with WON'T involved 'single' user@example.com WON'T the [brackets] exactly 42 'single' {braces} involved user@example.com They'RE written before keep tokens dog It's over on \\ud83d\\ude42 a that 1999 reservation I'M for that fox it boundaries no hand for $1,234.56 They'RE jumps hand WON'T https://example.com/a/b?c=d faster over [brackets] 3.14159 Z\\u00fcrich\"}, {\"role\": \"assistant\", \"content\": \"100% the quick scanner is with node.js [brackets] 100% keep scanned because brown every that $1,234.56 \\\"quotes\\\" because 3.14159 it 42 tiktoken's because once 1999 user@example.com node.js the admission it quick scanner involved It's $1,234.56 user@example.com 1999 3.14159 exactly https://example.com/a/b?c=d na\\u00efve dog counting exactly quick v1.2.3 piece involved once v1.2.3 exactly 42 scanner gateway (parens) request \\u6771\\u4eac fox keep 42 node.js while keep with the gateway with F# over no involved caf\\u00e9 [brackets] request They'RE scanner on with v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 every the 42 It's every jumps a scanned involved the brown scanned quick the tokens I'M I'M over no over faster v1.2.3 caf\\u00e9 \\\"quotes\\\" body request that fox fox before node.js brown dog a dog body engine faster Z\\u00fcrich https://example.com/a/b?c=d before I'M we'll hand WON'T admission and counting while on quick and\"}, {\"role\": \"user\", \"content\": \"na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 mirrors mirrors while tiktoken's tiktoken's hand Z\\u00fcrich there's backtracking brown brown \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors is (parens) it that tokens v1.2.3 way admission that caf\\u00e9 so exactly engine with scanner budget WON'T \\u6771\\u4eac 'single' hand na\\u00efve there's 'single' \\u6771\\u4eac piece na\\u00efve $1,234.56 is before faster admission request \\ud83d\\ude42 \\u6771\\u4eac counting backtracking written \\\"quotes\\\" there's involved lazy quick faster that request scanned v1.2.3 $1,234.56 every dog with for regex once with [brackets] v1.2.3 so caf\\u00e9 It's written it Z\\u00fcrich lazy caf\\u00e9 scanner counting there's no caf\\u00e9 every F# 100% the C++ every faster we'll on tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 engine $1,234.56 faster faster 1999 that exactly mirrors for admission budget 'single' 3.14159 no engine\"}, {\"role\": \"assistant\", \"content\": \"https://example.com/a/b?c=d there's gateway mirrors body mirrors written for brown node.js for brown node.js 'single' scanner na\\u00efve request is every 100% user@example.com dog 3.14159 no we'll that Z\\u00fcrich don't involved with https://example.com/a/b?c=d fox dog F# reservation C++ a v1.2.3 v1.2.3 backtracking WON'T F# no piece C++ {braces} a mirrors way $1,234.56 that WON'T body They'RE no we'll jumps F# once caf\\u00e9 reservation WON'T that piece 1999 They'RE counting piece They'RE request tiktoken's a engine na\\u00efve scanned body lazy faster It's over It's that tokens Z\\u00fcrich C++ scanner with \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"user\", \"content\": \"scanner while tiktoken's fox on boundaries tiktoken's tokens before tiktoken's it body gateway and over (parens) the (parens) 'single' on C++ faster budget every request once on exactly every that it Z\\u00fcrich quick regex and user@example.com jumps it user@example.com quick 100% it exactly every 'single' mirrors body jumps counting caf\\u00e9 tokens admission that piece scanner {braces} tokens is \\ud83d\\ude42 request admission C++ is node.js body for body They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking WON'T\"}, {\"role\": \"assistant\", \"content\": \"dog C++ (parens) It's with 'single' we'll [brackets] request is there's 3.14159 scanned 100% for request on admission $1,234.56 over there's scanned engine scanner They'RE that lazy \\u6771\\u4eac before budget user@example.com once C++ $1,234.56 that scanner exactly tokens is engine \\ud83d\\ude42 dog user@example.com na\\u00efve They'RE we'll \\ud83d\\ude42 that 'single' jumps the Z\\u00fcrich on jumps involved hand involved tiktoken's It's C++ \\ud83d\\ude42 user@example.com that before user@example.com $1,234.56 don't once scanner and a user@example.com user@example.com exactly and user@example.com $1,234.56 admission backtracking budget written 100% we'll 1999 scanned mirrors every \\u0645\\u0631\\u062d\\u0628\\u0627 a brown dog mirrors reservation 1999 regex hand budget \\u6771\\u4eac user@example.com it before 'single' hand over with {braces} no the a keep quick tiktoken's 3.14159 caf\\u00e9 every Z\\u00fcrich F# \\u6771\\u4eac the I'M Z\\u00fcrich na\\u00efve no way gateway na\\u00efve https://example.com/a/b?c=d request {braces} backtracking\"}, {\"role\": \"user\", \"content\": \"\\\"quotes\\\" \\\"quotes\\\" scanned is lazy user@example.com v1.2.3 regex a It's tokens piece They'RE engine over Z\\u00fcrich 3.14159 {braces} 'single' (parens) a boundaries node.js request jumps body before for hand dog boundaries budget on the boundaries we'll every because {braces} Z\\u00fcrich way https://example.com/a/b?c=d 1999 body Z\\u00fcrich there's budget 1999 before so v1.2.3 WON'T user@example.com F# it written lazy with once dog faster faster once we'll \\ud83d\\ude42 exactly user@example.com faster 42 \\u0645\\u0631\\u062d\\u0628\\u0627 a admission over \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T lazy\"}, {\"role\": \"assistant\", \"content\": \"na\\u00efve the (parens) for the brown https://example.com/a/b?c=d and 42 Z\\u00fcrich on {braces} \\ud83d\\ude42 regex over regex na\\u00efve F# over C++ before with the for don't 'single' counting \\u6771\\u4eac tiktoken's before mirrors fox gateway faster so regex \\ud83d\\ude42 keep $1,234.56 reservation so \\ud83d\\ude42 42 https://example.com/a/b?c=d tokens user@example.com for the for exactly \\\"quotes\\\" it written for F# dog I'M engine faster \\u6771\\u4eac gateway dog na\\u00efve is {braces} tokens over request backtracking 1999 body request written 'single' They'RE tokens F# tokens brown C++ boundaries admission every budget we'll no budget it keep don't https://example.com/a/b?c=d every exactly caf\\u00e9 'single' there's C++ 'single' dog v1.2.3 backtracking They'RE engine (parens) 'single' gateway every it $1,234.56 1999 lazy once reservation over fox that {braces} 1999 budget node.js written C++ way [brackets] before \\ud83d\\ude42 gateway on tiktoken's and (parens) the written \\u6771\\u4eac\"}, {\"role\": \"user\", \"content\": \"user@example.com scanned 42 user@example.com piece backtracking every engine that \\u0645\\u0631\\u062d\\u0628\\u0627 request $1,234.56 lazy Z\\u00fcrich mirrors fox no that so It's {braces} It's piece mirrors v1.2.3 regex quick counting for They'RE F# 3.14159 once brown no that no \\u6771\\u4eac so 100% scanner 1999 counting fox there's once engine caf\\u00e9 brown [brackets] piece hand mirrors budget lazy over the counting fox counting is for I'M faster a with node.js we'll \\u6771\\u4eac it 1999 user@example.com that It's mirrors written lazy lazy the before faster piece na\\u00efve fox piece 3.14159 fox na\\u00efve dog regex [brackets] [brackets] \\u6771\\u4eac over body user@example.com 42 node.js while and https://example.com/a/b?c=d na\\u00efve piece faster brown (parens) \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 \\\"quotes\\\"\"}, {\"role\": \"assistant\", \"content\": \"faster written 100% $1,234.56 engine 'single' gateway faster the boundaries exactly on C++ so so mirrors once backtracking caf\\u00e9 quick 1999 while we'll WON'T a Z\\u00fcrich boundaries involved piece scanner {braces} written 1999 and 3.14159 there's \\ud83d\\ude42 that the because quick scanner 3.14159 that it way counting na\\u00efve reservation for before $1,234.56 over quick engine scanner no the we'll scanned backtracking that no don't before 1999 \\u6771\\u4eac we'll before 3.14159 $1,234.56 budget counting tokens jumps while before tokens node.js tokens gateway scanner that \\ud83d\\ude42 3.14159 'single' WON'T so way (parens) backtracking \\\"quotes\\\" the v1.2.3 {braces} body body C++ it 'single'\"}, {\"role\": \"user\", \"content\": \"because request that that dog counting 100% mirrors I'M fox there's the piece with budget regex while before tokens with over there's \\u0645\\u0631\\u062d\\u0628\\u0627 reservation body on reservation tiktoken's and \\u0645\\u0631\\u062d\\u0628\\u0627 while so quick that budget quick brown and user@example.com \\u6771\\u4eac 100% dog keep there's It's WON'T caf\\u00e9 quick there's $1,234.56 way user@example.com budget for v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 quick way boundaries is reservation 42 no that \\ud83d\\ude42 the hand \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) na\\u00efve $1,234.56 3.14159 quick 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors na\\u00efve hand once every F# the \\ud83d\\ude42 once piece I'M 42 with involved the {braces} [brackets] a 'single' caf\\u00e9 that once before once 'single' counting {braces} every reservation budget jumps WON'T\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 {braces} it They'RE scanned for \\u6771\\u4eac hand mirrors written 42 with node.js quick admission I'M counting \\u0645\\u0631\\u062d\\u0628\\u0627 brown \\u6771\\u4eac 100% over \\u6771\\u4eac that \\ud83d\\ude42 \\u6771\\u4eac It's \\ud83d\\ude42 scanner budget so dog that regex keep hand piece lazy no is user@example.com counting tiktoken's request admission we'll mirrors hand gateway boundaries no https://example.com/a/b?c=d on engine reservation no that {braces} mirrors brown Z\\u00fcrich admission piece user@example.com scanned and with 100% the tiktoken's fox WON'T written\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich is over 'single' C++ It's gateway [brackets] no over way quick admission no faster written node.js 42 v1.2.3 involved {braces} 3.14159 because [brackets] reservation body before the exactly every [brackets] tokens so fox piece boundaries boundaries v1.2.3 caf\\u00e9 I'M F# 1999 backtracking user@example.com with with don't body a C++ brown don't It's caf\\u00e9 regex They'RE way https://example.com/a/b?c=d scanned 'single' boundaries that reservation \\ud83d\\ude42 brown fox node.js backtracking gateway na\\u00efve (parens)\"}, {\"role\": \"assistant\", \"content\": \"piece WON'T body it $1,234.56 that 3.14159 1999 it reservation \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's boundaries 'single' It's reservation gateway C++ body budget mirrors is the backtracking and \\u0645\\u0631\\u062d\\u0628\\u0627 gateway a 'single' is and while 42 hand it They'RE caf\\u00e9 backtracking counting gateway exactly F# because na\\u00efve scanned that C++ is 100% involved admission piece 3.14159 quick it the lazy involved WON'T a before It's tokens They'RE it mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 reservation it \\ud83d\\ude42 na\\u00efve so is a and that request while that written body keep reservation the on every They'RE WON'T budget scanned reservation $1,234.56 fox is there's $1,234.56 brown involved admission\"}, {\"role\": \"user\", \"content\": \"way F# don't brown v1.2.3 the regex na\\u00efve involved brown budget because dog that WON'T and v1.2.3 for it counting because because because brown \\u6771\\u4eac backtracking (parens) I'M that dog reservation keep that scanner request WON'T They'RE backtracking admission every 42 because 100% [brackets] https://example.com/a/b?c=d admission way the so don't I'M \\ud83d\\ude42 3.14159 that C++ 3.14159 https://example.com/a/b?c=d na\\u00efve \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com node.js way I'M over https://example.com/a/b?c=d engine once mirrors Z\\u00fcrich F# fox on \\u0645\\u0631\\u062d\\u0628\\u0627 na\\u00efve mirrors piece with the \\ud83d\\ude42 WON'T counting involved don't written boundaries gateway keep 1999 boundaries so Z\\u00fcrich way piece every user@example.com exactly They'RE piece boundaries written brown I'M C++ gateway keep the tiktoken's reservation counting Z\\u00fcrich a involved\"}, {\"role\": \"assistant\", \"content\": \"way is a for scanned dog exactly don't body na\\u00efve [brackets] fox na\\u00efve {braces} that body for scanner reservation user@example.com quick fox hand faster reservation mirrors (parens) counting no we'll a exactly reservation user@example.com tiktoken's hand tokens don't C++ body na\\u00efve hand with backtracking for engine for They'RE admission {braces} fox boundaries keep quick it with once body the body the budget \\\"quotes\\\" caf\\u00e9 a admission $1,234.56 we'll way v1.2.3 3.14159 reservation fox WON'T 3.14159 once that written Z\\u00fcrich for na\\u00efve once piece tiktoken's over engine \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 F# 3.14159 lazy \\ud83d\\ude42 user@example.com na\\u00efve \\u6771\\u4eac don't because 'single' They'RE \\ud83d\\ude42 \\u6771\\u4eac body because lazy a for exactly request tiktoken's gateway no C++ before with every {braces} keep v1.2.3 faster mirrors (parens) (parens) and quick scanned I'M budget engine with\"}, {\"role\": \"user\", \"content\": \"regex with exactly \\u0645\\u0631\\u062d\\u0628\\u0627 dog budget that 42 {braces} tokens https://example.com/a/b?c=d tiktoken's that I'M we'll 100% engine counting F# no regex node.js no boundaries way tiktoken's because \\\"quotes\\\" exactly 3.14159 and [brackets] mirrors written keep that F# backtracking scanned They'RE it gateway over {braces} node.js 100% budget scanner budget \\\"quotes\\\" we'll v1.2.3 we'll caf\\u00e9 \\ud83d\\ude42 tokens don't once because mirrors backtracking na\\u00efve every faster once budget caf\\u00e9 it I'M caf\\u00e9 a backtracking regex fox while engine faster scanned I'M tiktoken's don't \\\"quotes\\\" 3.14159 scanner It's na\\u00efve request boundaries hand lazy no $1,234.56 every for jumps written 1999 mirrors {braces} WON'T lazy engine and\"}, {\"role\": \"assistant\", \"content\": \"and reservation caf\\u00e9 it (parens) before piece backtracking and WON'T brown counting that 1999 $1,234.56 tokens user@example.com user@example.com it faster It's the body and request brown caf\\u00e9 keep scanner \\u0645\\u0631\\u062d\\u0628\\u0627 for tokens caf\\u00e9 'single' request boundaries 'single' request They'RE user@example.com tokens quick there's keep exactly a scanned is regex it hand quick hand {braces} so 'single' gateway and\"}, {\"role\": \"user\", \"content\": \"na\\u00efve is request that It's the way caf\\u00e9 the 42 there's quick user@example.com the over that $1,234.56 with user@example.com quick They'RE budget the it F# gateway It's fox we'll involved tiktoken's na\\u00efve over while https://example.com/a/b?c=d that na\\u00efve tiktoken's 42 scanned we'll na\\u00efve and tiktoken's (parens) piece 'single' exactly Z\\u00fcrich budget WON'T (parens) It's engine we'll body involved and keep\"}, {\"role\": \"assistant\", \"content\": \"way v1.2.3 it so once before \\\"quotes\\\" WON'T WON'T written and over F# the with no boundaries v1.2.3 body that dog exactly Z\\u00fcrich backtracking there's on \\u6771\\u4eac mirrors request on They'RE hand regex engine reservation it scanned \\\"quotes\\\" hand boundaries for reservation before brown WON'T once hand \\u0645\\u0631\\u062d\\u0628\\u0627 body 42 \\u0645\\u0631\\u062d\\u0628\\u0627 once every admission body $1,234.56 before $1,234.56 WON'T {braces} na\\u00efve mirrors before the that 3.14159 way brown It's 1999 gateway reservation F# for user@example.com exactly jumps https://example.com/a/b?c=d is and node.js 100% boundaries v1.2.3 'single' we'll no gateway They'RE with F# [brackets] C++ mirrors for tokens written every\"}, {\"role\": \"user\", \"content\": \"WON'T na\\u00efve 'single' It's 3.14159 \\ud83d\\ude42 don't 'single' \\\"quotes\\\" involved C++ caf\\u00e9 with mirrors 100% It's that no user@example.com WON'T before is gateway once node.js gateway while on quick \\ud83d\\ude42 boundaries \\u6771\\u4eac budget 1999 admission They'RE mirrors engine I'M is \\u0645\\u0631\\u062d\\u0628\\u0627 I'M budget for 'single' on \\u6771\\u4eac with WON'T and because brown regex hand reservation scanner 100% 100% user@example.com there's it\"}, {\"role\": \"assistant\", \"content\": \"there's exactly [brackets] faster budget while $1,234.56 (parens) engine caf\\u00e9 They'RE no 100% for because $1,234.56 regex Z\\u00fcrich https://example.com/a/b?c=d tokens jumps because counting brown user@example.com request lazy \\u0645\\u0631\\u062d\\u0628\\u0627 request scanned WON'T $1,234.56 exactly brown every mirrors with 'single' engine 42 \\ud83d\\ude42 engine reservation hand lazy 'single' counting hand (parens) gateway that counting user@example.com 42 fox request \\\"quotes\\\" hand lazy 100% 42 I'M It's WON'T https://example.com/a/b?c=d engine while\"}, {\"role\": \"user\", \"content\": \"C++ \\\"quotes\\\" we'll gateway \\ud83d\\ude42 counting written piece na\\u00efve \\u6771\\u4eac WON'T over caf\\u00e9 backtracking engine https://example.com/a/b?c=d the v1.2.3 that so quick counting na\\u00efve \\u6771\\u4eac don't WON'T It's we'll that the 100% request I'M brown na\\u00efve tokens engine budget before the scanned mirrors dog engine {braces} {braces} 42 don't WON'T It's piece \\\"quotes\\\" that no quick 3.14159 F# once fox don't for scanner \\u6771\\u4eac it brown backtracking tokens $1,234.56 reservation written They'RE the \\\"quotes\\\" caf\\u00e9 backtracking C++ reservation\"}, {\"role\": \"assistant\", \"content\": \"regex (parens) Z\\u00fcrich engine https://example.com/a/b?c=d written tiktoken's C++ dog It's lazy dog exactly with it that admission way no scanner a 3.14159 on They'RE It's involved caf\\u00e9 brown on dog that https://example.com/a/b?c=d don't \\u6771\\u4eac jumps \\ud83d\\ude42 and \\u6771\\u4eac so dog so don't (parens) scanned engine budget the exactly F# tokens over \\u0645\\u0631\\u062d\\u0628\\u0627 jumps backtracking 3.14159 budget 100% boundaries gateway backtracking don't scanner dog with [brackets] gateway 3.14159 caf\\u00e9 faster budget na\\u00efve caf\\u00e9 way for every reservation backtracking request They'RE It's \\u6771\\u4eac written\"}, {\"role\": \"user\", \"content\": \"42 1999 so boundaries $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} budget backtracking [brackets] we'll admission piece node.js admission Z\\u00fcrich fox a involved request is it node.js node.js so \\u6771\\u4eac [brackets] no \\u6771\\u4eac brown node.js quick na\\u00efve scanner C++ user@example.com admission 'single' involved 1999 user@example.com I'M v1.2.3 It's 100% there's gateway F# node.js gateway scanned that caf\\u00e9 3.14159 Z\\u00fcrich no admission {braces} tiktoken's over It's I'M 1999 [brackets] body written scanned that scanner on the on with that scanned hand\"}, {\"role\": \"assistant\", \"content\": \"{braces} regex [brackets] piece \\ud83d\\ude42 mirrors there's tiktoken's It's lazy mirrors every v1.2.3 I'M that boundaries 'single' node.js on C++ user@example.com budget user@example.com no user@example.com every engine while a I'M over [brackets] WON'T piece over there's lazy I'M that $1,234.56 budget https://example.com/a/b?c=d it a request exactly 42 keep They'RE caf\\u00e9 tokens \\u0645\\u0631\\u062d\\u0628\\u0627 keep scanner while dog F# mirrors backtracking we'll gateway scanner so we'll request there's budget it lazy scanner for exactly faster we'll reservation scanner no WON'T request (parens) tiktoken's exactly before so while hand involved with $1,234.56 mirrors 3.14159 on the [brackets] boundaries 100% with request way engine keep the https://example.com/a/b?c=d \\u6771\\u4eac the WON'T that so\"}, {\"role\": \"user\", \"content\": \"body hand backtracking (parens) counting na\\u00efve They'RE \\ud83d\\ude42 WON'T boundaries v1.2.3 budget na\\u00efve admission once the that because tokens dog tiktoken's scanner scanner (parens) I'M \\\"quotes\\\" budget 100% there's once WON'T keep {braces} the before fox (parens) the counting because lazy 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 admission 1999 and there's while admission 3.14159 a counting C++ with exactly don't don't lazy \\ud83d\\ude42 42 \\\"quotes\\\" backtracking C++ there's keep user@example.com request is node.js piece before brown budget 42 100% because admission and with tiktoken's no the 100%\"}, {\"role\": \"assistant\", \"content\": \"with 1999 Z\\u00fcrich that is with mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 the brown $1,234.56 way scanner https://example.com/a/b?c=d lazy written scanner quick v1.2.3 \\\"quotes\\\" regex \\\"quotes\\\" They'RE 3.14159 \\ud83d\\ude42 [brackets] Z\\u00fcrich the counting for so $1,234.56 It's WON'T the quick hand that mirrors the 42 caf\\u00e9 exactly node.js we'll backtracking while I'M and fox is before written it 1999 no for involved \\u0645\\u0631\\u062d\\u0628\\u0627 the \\\"quotes\\\" piece the way involved that no 100% before there's 100% we'll is [brackets] C++ the we'll the node.js reservation is while [brackets] engine scanner we'll no C++ don't dog v1.2.3 that so tiktoken's before \\ud83d\\ude42 mirrors 'single' it once caf\\u00e9 there's backtracking gateway body over quick way and that once \\u6771\\u4eac 'single' we'll (parens) and before for quick They'RE 1999 regex there's \\ud83d\\ude42\"}, {\"role\": \"user\", \"content\": \"engine They'RE https://example.com/a/b?c=d we'll quick tokens written WON'T don't regex backtracking It's it lazy 1999 brown budget on [brackets] node.js {braces} They'RE involved a mirrors exactly tokens with $1,234.56 F# 42 backtracking involved engine It's and {braces} \\u6771\\u4eac Z\\u00fcrich it it [brackets] gateway keep na\\u00efve request scanned don't regex we'll C++ counting tokens so request while scanned tokens scanner for v1.2.3 with tokens \\ud83d\\ude42 They'RE na\\u00efve 'single' keep node.js v1.2.3 'single' it way gateway It's keep reservation backtracking body fox with that [brackets] the and 1999 request no engine counting is [brackets] lazy on request quick involved \\ud83d\\ude42 brown v1.2.3 involved fox exactly hand {braces} we'll every regex\"}, {\"role\": \"assistant\", \"content\": \"(parens) faster tiktoken's with the every mirrors boundaries that caf\\u00e9 backtracking involved once 42 \\u6771\\u4eac Z\\u00fcrich {braces} boundaries that [brackets] 1999 it admission before a jumps and 3.14159 way scanner don't {braces} budget for a 3.14159 mirrors boundaries while caf\\u00e9 counting \\u6771\\u4eac a written tiktoken's and we'll https://example.com/a/b?c=d jumps while so [brackets] F# 3.14159 mirrors that {braces} request request I'M keep the over hand engine keep node.js dog na\\u00efve [brackets] every body $1,234.56 \\\"quotes\\\" $1,234.56 we'll \\u0645\\u0631\\u062d\\u0628\\u0627 that 42 budget scanned it They'RE no with the {braces} we'll tiktoken's a \\ud83d\\ude42 tiktoken's https://example.com/a/b?c=d It's exactly It's for over that 100% a \\\"quotes\\\" caf\\u00e9 no 3.14159 hand gateway quick budget over user@example.com I'M\"}, {\"role\": \"user\", \"content\": \"mirrors engine so Z\\u00fcrich and reservation caf\\u00e9 1999 na\\u00efve exactly involved caf\\u00e9 and 1999 over tiktoken's user@example.com over once that It's jumps involved C++ and is node.js brown gateway 100% 100% keep hand backtracking 3.14159 involved hand 'single' once because budget They'RE brown 3.14159 every faster request it regex while \\ud83d\\ude42 1999 exactly way we'll 'single' request scanner boundaries on (parens) we'll\"}, {\"role\": \"assistant\", \"content\": \"once 1999 fox that Z\\u00fcrich \\u6771\\u4eac that we'll 42 'single' dog \\u6771\\u4eac mirrors (parens) because na\\u00efve every and piece C++ 3.14159 https://example.com/a/b?c=d 42 once so gateway is scanned no user@example.com node.js caf\\u00e9 faster node.js tiktoken's It's F# brown it admission body don't don't before scanner before F# 3.14159 na\\u00efve backtracking user@example.com F# that (parens) (parens) there's exactly a reservation it before backtracking It's piece tiktoken's lazy written is {braces} 1999 scanned keep body because mirrors no written that piece boundaries request once scanner \\\"quotes\\\" with engine hand v1.2.3 we'll v1.2.3 the we'll $1,234.56 the quick and tokens backtracking is quick 'single' 42 once reservation admission fox scanner a scanner engine 1999 quick [brackets] \\u6771\\u4eac budget for quick engine keep user@example.com 100% C++ [brackets] hand on Z\\u00fcrich hand Z\\u00fcrich regex 1999\"}, {\"role\": \"user\", \"content\": \"\\u0645\\u0631\\u062d\\u0628\\u0627 scanner They'RE budget C++ exactly {braces} is \\ud83d\\ude42 because way body brown counting with regex it \\\"quotes\\\" we'll body node.js over a node.js that while it v1.2.3 3.14159 quick request that dog node.js They'RE 1999 that gateway 3.14159 gateway there's backtracking lazy C++ admission node.js 42 request They'RE we'll gateway so once 1999 brown na\\u00efve 3.14159 budget faster \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" backtracking that 100% because a fox once involved on involved v1.2.3 WON'T mirrors 'single' a \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" is They'RE that involved reservation They'RE (parens) v1.2.3 Z\\u00fcrich WON'T Z\\u00fcrich fox that don't boundaries engine regex don't exactly na\\u00efve don't caf\\u00e9 request budget [brackets] F# tokens exactly It's request it\"}, {\"role\": \"assistant\", \"content\": \"scanned (parens) reservation {braces} engine tokens v1.2.3 node.js scanner {braces} It's with admission the 100% F# \\u0645\\u0631\\u062d\\u0628\\u0627 body na\\u00efve with 'single' boundaries regex (parens) because mirrors there's keep once lazy it brown involved boundaries request engine before on It's \\ud83d\\ude42 na\\u00efve fox WON'T we'll backtracking jumps exactly scanner it admission mirrors don't that \\u6771\\u4eac faster body mirrors reservation (parens) \\\"quotes\\\" regex mirrors keep exactly They'RE 3.14159 there's a \\u0645\\u0631\\u062d\\u0628\\u0627 and with admission written C++ https://example.com/a/b?c=d the piece jumps that is lazy tokens before 100% boundaries tokens dog tiktoken's it $1,234.56 I'M 100% once (parens) I'M way brown keep\"}, {\"role\": \"user\", \"content\": \"no no hand F# jumps so faster every don't so tiktoken's 'single' I'M [brackets] \\\"quotes\\\" v1.2.3 over admission exactly scanner node.js 100% quick gateway \\ud83d\\ude42 faster budget over faster over and engine request faster fox 'single' backtracking body brown no na\\u00efve is it \\\"quotes\\\" piece with \\ud83d\\ude42 for tokens I'M scanned while admission way faster na\\u00efve mirrors tokens boundaries \\u6771\\u4eac jumps tiktoken's it there's faster\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 no It's piece request \\u6771\\u4eac Z\\u00fcrich don't \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE 3.14159 quick Z\\u00fcrich once tiktoken's with engine jumps over I'M because caf\\u00e9 faster tiktoken's that 'single' we'll dog 42 \\u6771\\u4eac way once counting it C++ on that the that backtracking we'll scanned before 100% na\\u00efve request dog every request 1999 request quick over 1999 gateway https://example.com/a/b?c=d because 3.14159 that reservation mirrors that a over I'M with regex Z\\u00fcrich mirrors piece regex caf\\u00e9 engine Z\\u00fcrich the written v1.2.3 \"}, {\"role\": \"user\", \"content\": \"hand is It's scanned regex 1999 way on It's we'll tokens with reservation while way mirrors regex no over 42 is user@example.com scanner scanned keep budget {braces} brown budget while is involved [brackets] body jumps F# dog no 3.14159 exactly over 100% (parens) It's jumps so we'll \\u6771\\u4eac faster admission exactly admission that for tiktoken's 100% once engine and every a before budget regex They'RE counting\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 involved backtracking regex na\\u00efve way tokens we'll 100% gateway the gateway the body lazy I'M budget is regex body (parens) C++ that body no the jumps [brackets] exactly 1999 budget request brown \\ud83d\\ude42 jumps keep scanned user@example.com for exactly regex dog node.js scanner that node.js 42 engine fox \\ud83d\\ude42 tiktoken's WON'T mirrors written faster while 42 100% \\u6771\\u4eac user@example.com dog engine lazy [brackets] exactly quick node.js written written 100% every \\ud83d\\ude42 body 3.14159 hand because once gateway faster before with It's a 1999 42 boundaries jumps that {braces} 100% caf\\u00e9 a They'RE 3.14159 [brackets] reservation 1999 faster \\ud83d\\ude42 hand 1999 [brackets] scanner\"}, {\"role\": \"user\", \"content\": \"https://example.com/a/b?c=d dog for C++ v1.2.3 because every 100% \\u6771\\u4eac and brown is request piece boundaries on before because that body WON'T [brackets] that keep tokens lazy WON'T keep on for WON'T body quick {braces} gateway body written tokens quick the mirrors is is with I'M 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 don't \\u0645\\u0631\\u062d\\u0628\\u0627 piece involved user@example.com written tiktoken's fox written and no 1999 don't on because counting \\\"quotes\\\" They'RE 42 a gateway \\u6771\\u4eac scanner admission user@example.com 3.14159 there's on over 'single' written that a {braces} that because gateway the\"}, {\"role\": \"assistant\", \"content\": \"WON'T backtracking \\ud83d\\ude42 node.js 3.14159 keep it \\u0645\\u0631\\u062d\\u0628\\u0627 there's is so user@example.com on there's 3.14159 mirrors involved keep \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's {braces} no engine brown brown there's dog way regex that 'single' for and 3.14159 node.js every there's a every They'RE WON'T exactly scanner tiktoken's They'RE regex the dog counting over the for request scanner with and mirrors dog the counting tokens gateway https://example.com/a/b?c=d for https://example.com/a/b?c=d backtracking\"}, {\"role\": \"user\", \"content\": \"don't because that fox Z\\u00fcrich with Z\\u00fcrich way piece (parens) every (parens) user@example.com {braces} keep so a that boundaries 100% keep C++ node.js 1999 piece $1,234.56 \\ud83d\\ude42 backtracking {braces} that involved gateway boundaries no 42 hand 3.14159 \\ud83d\\ude42 na\\u00efve na\\u00efve that for Z\\u00fcrich $1,234.56 engine lazy fox scanned before once hand no https://example.com/a/b?c=d admission WON'T admission It's no a because \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac exactly written \\u0645\\u0631\\u062d\\u0628\\u0627 faster that is user@example.com v1.2.3 'single' boundaries gateway faster backtracking no engine fox $1,234.56 lazy the $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 that WON'T before I'M scanned\"}, {\"role\": \"assistant\", \"content\": \"it is admission admission brown the They'RE we'll we'll user@example.com [brackets] way tokens $1,234.56 on caf\\u00e9 I'M gateway 3.14159 written 'single' on involved mirrors node.js that fox 'single' quick once \\u6771\\u4eac gateway $1,234.56 no exactly body Z\\u00fcrich that 'single' engine and Z\\u00fcrich don't no so tiktoken's I'M that over hand dog we'll (parens) 'single' \\ud83d\\ude42 while 100% is node.js no quick over while on budget that \\\"quotes\\\" budget It's don't regex on C++ 1999 [brackets] scanned 1999 that there's $1,234.56 way\"}, {\"role\": \"user\", \"content\": \"budget while tiktoken's backtracking gateway and hand no while \\ud83d\\ude42 request node.js we'll and don't while na\\u00efve quick WON'T don't [brackets] quick there's fox that that I'M keep \\ud83d\\ude42 tiktoken's v1.2.3 the body the \\u0645\\u0631\\u062d\\u0628\\u0627 reservation written F# jumps admission boundaries user@example.com while I'M brown lazy It's regex fox user@example.com {braces} is on 'single' written request there's engine exactly tiktoken's WON'T caf\\u00e9 the tokens on and brown exactly scanner with involved regex on v1.2.3 jumps that written F# on exactly before over once body \\u6771\\u4eac \\ud83d\\ude42 lazy that \\ud83d\\ude42\"}, {\"role\": \"assistant\", \"content\": \"don't every there's scanned while They'RE a tokens is scanned mirrors so is request counting once way (parens) {braces} counting 100% \\\"quotes\\\" before boundaries brown tiktoken's that way engine quick tiktoken's over involved 100% before for the C++ It's that node.js [brackets] because https://example.com/a/b?c=d engine 100% budget we'll with tokens 42 budget it dog 1999 user@example.com 42 C++ keep the for a 42 keep no caf\\u00e9 node.js 'single' the for don't It's C++ admission a involved 3.14159 'single' don't They'RE WON'T quick so backtracking because once body \\\"quotes\\\" WON'T user@example.com caf\\u00e9 v1.2.3 engine 'single' scanner hand I'M 'single' $1,234.56 [brackets] {braces} counting it don't there's no that\"}, {\"role\": \"user\", \"content\": \"so jumps dog hand no 100% while request tiktoken's there's that so regex $1,234.56 don't 'single' boundaries \\\"quotes\\\" that 100% the boundaries keep backtracking (parens) no \\u6771\\u4eac fox F# mirrors 3.14159 we'll there's \\\"quotes\\\" scanner backtracking $1,234.56 100% request the WON'T boundaries caf\\u00e9 tiktoken's node.js backtracking counting user@example.com C++ exactly is reservation 100% we'll quick regex \\ud83d\\ude42 'single' I'M request written https://example.com/a/b?c=d boundaries They'RE there's It's piece F# v1.2.3 boundaries [brackets] a 100% that scanner v1.2.3 42 because https://example.com/a/b?c=d [brackets] that I'M once body faster because with engine 'single' scanned on engine the 1999 \\u6771\\u4eac quick written no piece involved boundaries [brackets] counting while I'M Z\\u00fcrich scanner for the written $1,234.56 (parens) that boundaries the because\"}, {\"role\": \"assistant\", \"content\": \"(parens) https://example.com/a/b?c=d 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich Z\\u00fcrich \\u6771\\u4eac involved don't reservation there's we'll \\\"quotes\\\" reservation They'RE tokens because lazy there's a exactly It's jumps every $1,234.56 v1.2.3 exactly 42 a quick https://example.com/a/b?c=d caf\\u00e9 tiktoken's piece reservation every C++ caf\\u00e9 v1.2.3 scanned backtracking na\\u00efve 1999 piece engine while that I'M involved on fox 42 https://example.com/a/b?c=d (parens) tiktoken's for dog once the involved a dog once there's https://example.com/a/b?c=d tokens na\\u00efve (parens) once 1999 over na\\u00efve caf\\u00e9 hand They'RE every tiktoken's involved we'll node.js\"}, {\"role\": \"user\", \"content\": \"that request while jumps faster exactly written 3.14159 scanner that \\\"quotes\\\" with exactly dog that keep with piece piece boundaries no [brackets] backtracking \\u0645\\u0631\\u062d\\u0628\\u0627 F# hand $1,234.56 engine written (parens) and It's involved node.js Z\\u00fcrich boundaries with backtracking na\\u00efve and body counting node.js keep Z\\u00fcrich that while \\u6771\\u4eac because there's \\u6771\\u4eac boundaries no They'RE counting jumps v1.2.3 with don't a \\u0645\\u0631\\u062d\\u0628\\u0627 fox user@example.com no that it F# once it so every 100% a caf\\u00e9 the \\\"quotes\\\" [brackets] that gateway while we'll scanned caf\\u00e9 keep involved \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"the while I'M gateway WON'T tiktoken's 'single' over 'single' request faster (parens) faster faster \\u0645\\u0631\\u062d\\u0628\\u0627 hand tiktoken's keep while v1.2.3 $1,234.56 that written body WON'T way \\u0645\\u0631\\u062d\\u0628\\u0627 brown a F# scanned \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE keep lazy caf\\u00e9 100% 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 {braces} we'll exactly the C++ body Z\\u00fcrich Z\\u00fcrich way with reservation mirrors C++ backtracking involved WON'T exactly with involved WON'T way a counting the admission 1999 hand because once They'RE the backtracking 1999 no a 42 brown\"}, {\"role\": \"user\", \"content\": \"node.js so jumps is piece a way on because the that backtracking request the reservation {braces} {braces} the https://example.com/a/b?c=d it tokens reservation the lazy They'RE reservation a 1999 I'M boundaries no with no C++ is way 3.14159 WON'T hand 'single' counting Z\\u00fcrich on F# with 1999 admission every (parens) we'll request mirrors that the 'single' it faster dog \\\"quotes\\\" jumps that faster WON'T faster every 100% involved a there's \\ud83d\\ude42 way way 'single' na\\u00efve no on it (parens) WON'T user@example.com na\\u00efve boundaries mirrors [brackets] it regex regex 42 engine engine and faster \\\"quotes\\\" gateway \\u0645\\u0631\\u062d\\u0628\\u0627 over request $1,234.56 They'RE user@example.com that node.js I'M https://example.com/a/b?c=d involved on (parens) it a that 3.14159 way over no\"}, {\"role\": \"assistant\", \"content\": \"boundaries so budget involved exactly written node.js [brackets] with jumps caf\\u00e9 node.js (parens) [brackets] 100% tiktoken's 42 it that piece (parens) engine user@example.com involved no gateway \\\"quotes\\\" exactly It's budget exactly once It's a involved (parens) budget we'll fox node.js tokens Z\\u00fcrich no body \\\"quotes\\\" na\\u00efve while it is I'M the hand mirrors WON'T the caf\\u00e9 \\u6771\\u4eac mirrors https://example.com/a/b?c=d counting before way because \\\"quotes\\\" a admission lazy dog keep the budget quick mirrors there's [brackets] hand Z\\u00fcrich https://example.com/a/b?c=d mirrors user@example.com 42 regex request scanned \\\"quotes\\\" engine because the It's Z\\u00fcrich F# 42 boundaries there's keep engine \\ud83d\\ude42 I'M They'RE because over $1,234.56 admission for fox because exactly because piece regex on \\u6771\\u4eac 'single' backtracking a\"}, {\"role\": \"user\", \"content\": \"They'RE backtracking \\u6771\\u4eac boundaries mirrors that 100% {braces} 42 gateway boundaries tokens so on counting gateway tiktoken's tiktoken's tokens tiktoken's there's while tokens It's C++ mirrors the tokens budget hand we'll over over [brackets] jumps the \\ud83d\\ude42 way and that boundaries 100% counting reservation there's [brackets] 3.14159 They'RE keep the regex It's while budget request Z\\u00fcrich it https://example.com/a/b?c=d C++ admission quick engine keep quick \\u6771\\u4eac $1,234.56 engine budget hand \\u6771\\u4eac body 100% don't scanned $1,234.56 dog the \\u6771\\u4eac piece over 1999 $1,234.56 $1,234.56 the dog we'll gateway is dog there's fox Z\\u00fcrich over They'RE so a engine 'single' regex and lazy na\\u00efve tokens tokens \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 \\ud83d\\ude42 (parens)\"}, {\"role\": \"assistant\", \"content\": \"\\\"quotes\\\" {braces} so with brown (parens) that the because keep boundaries before because fox exactly every because F# so mirrors way C++ request gateway there's that on request tokens it \\\"quotes\\\" regex written dog scanner budget while it that $1,234.56 scanned every no we'll is because while reservation They'RE don't dog scanner dog WON'T fox no scanned \\ud83d\\ude42 budget no dog request it scanned is hand over no 3.14159 so regex backtracking exactly backtracking the 42 100% faster quick scanner before 'single' reservation engine it we'll caf\\u00e9 every quick for no faster 100% $1,234.56 on Z\\u00fcrich written keep\"}, {\"role\": \"user\", \"content\": \"on tokens don't F# counting keep exactly every \\ud83d\\ude42 keep Z\\u00fcrich for fox counting no caf\\u00e9 reservation na\\u00efve mirrors They'RE It's F# the that piece \\\"quotes\\\" \\u6771\\u4eac \\u6771\\u4eac engine no $1,234.56 \\\"quotes\\\" because dog 100% na\\u00efve user@example.com tokens [brackets] no with that boundaries I'M Z\\u00fcrich before (parens) while and scanned gateway user@example.com faster request boundaries \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d written C++ I'M while quick [brackets] before 1999 and They'RE C++ v1.2.3 for no {braces} request C++ They'RE over WON'T on caf\\u00e9 {braces} 100% the body quick keep C++ \\ud83d\\ude42 tokens scanned admission keep hand a node.js WON'T keep 100% $1,234.56 hand $1,234.56 (parens) 1999 because the \\ud83d\\ude42 is 42 body Z\\u00fcrich way [brackets] it once that dog C++ jumps fox tiktoken's piece \\\"quotes\\\" v1.2.3 exactly for 42 \\ud83d\\ude42 'single' na\\u00efve admission \\u6771\\u4eac They'RE fox mirrors regex\"}, {\"role\": \"assistant\", \"content\": \"They'RE scanner tiktoken's quick (parens) https://example.com/a/b?c=d budget quick for engine (parens) hand tiktoken's piece written https://example.com/a/b?c=d counting counting over no once engine I'M because fox exactly regex no once keep and written regex faster 100% the admission counting a scanner C++ quick involved gateway brown piece Z\\u00fcrich regex written is faster \\u6771\\u4eac body brown no tokens scanned every request it I'M \\ud83d\\ude42 for (parens) Z\\u00fcrich written 3.14159 'single' I'M no na\\u00efve quick regex gateway it counting (parens) the over so hand gateway is body way dog v1.2.3 faster so request while before counting body F# faster [brackets] engine involved a $1,234.56 dog that that budget mirrors user@example.com the brown that no C++ it every na\\u00efve exactly that body Z\\u00fcrich \\u6771\\u4eac with don't so It's \\u0645\\u0631\\u062d\\u0628\\u0627 that gateway a the\"}, {\"role\": \"user\", \"content\": \"dog is once budget \\ud83d\\ude42 quick regex fox (parens) [brackets] gateway \\ud83d\\ude42 no regex because lazy request with that written we'll It's that request user@example.com is \\\"quotes\\\" \\\"quotes\\\" faster the don't quick node.js \\u0645\\u0631\\u062d\\u0628\\u0627 faster na\\u00efve They'RE the 100% written that the 100% backtracking \\ud83d\\ude42 written the fox scanner na\\u00efve it piece It's no so It's before na\\u00efve gateway there's because gateway reservation \\ud83d\\ude42 quick involved \\u0645\\u0631\\u062d\\u0628\\u0627 with WON'T request F# scanned exactly F# reservation scanned engine jumps v1.2.3 we'll 'single' it quick WON'T jumps na\\u00efve tokens brown node.js backtracking reservation 100% v1.2.3 once exactly\"}, {\"role\": \"assistant\", \"content\": \"fox user@example.com it (parens) we'll 100% involved dog and brown 'single' exactly mirrors 100% WON'T na\\u00efve 1999 F# faster 'single' tokens tiktoken's admission \\\"quotes\\\" the WON'T request that on we'll I'M piece there's body Z\\u00fcrich boundaries because jumps the They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 boundaries \\u6771\\u4eac the https://example.com/a/b?c=d caf\\u00e9 piece don't brown exactly node.js the budget every so admission that and 100% fox 'single' {braces} 100% $1,234.56 C++ counting WON'T brown don't https://example.com/a/b?c=d boundaries I'M 100% [brackets] lazy quick It's brown \\u6771\\u4eac for is \\u6771\\u4eac for is and request\"}, {\"role\": \"user\", \"content\": \"{braces} counting I'M request fox involved boundaries with WON'T WON'T $1,234.56 is faster don't request https://example.com/a/b?c=d while It's every body the a brown engine 42 for we'll request (parens) involved is scanned backtracking jumps C++ gateway keep scanner for counting 42 reservation hand \\\"quotes\\\" hand C++ every reservation caf\\u00e9 that don't with $1,234.56 there's WON'T {braces} that 3.14159 F# before over exactly caf\\u00e9 budget \\u6771\\u4eac the before and body gateway hand admission It's C++ that F# It's faster no that we'll written (parens) fox mirrors we'll user@example.com no https://example.com/a/b?c=d node.js that brown request\"}, {\"role\": \"assistant\", \"content\": \"I'M admission counting Z\\u00fcrich user@example.com faster a \\ud83d\\ude42 mirrors once lazy is the involved \\ud83d\\ude42 \\u6771\\u4eac every scanned every because budget hand and it jumps there's while there's [brackets] na\\u00efve on \\\"quotes\\\" \\\"quotes\\\" a \\ud83d\\ude42 hand \\u6771\\u4eac we'll and It's with the {braces} piece WON'T 'single' for \\u6771\\u4eac no backtracking lazy fox a [brackets] fox 42 \\ud83d\\ude42 (parens) 1999 https://example.com/a/b?c=d 1999 lazy and 1999 reservation way (parens) 100% it na\\u00efve 100%\"}, {\"role\": \"user\", \"content\": \"regex on scanner once $1,234.56 1999 (parens) gateway budget dog F# before user@example.com once brown jumps 100% $1,234.56 that budget Z\\u00fcrich the reservation because a on \\\"quotes\\\" backtracking hand dog because backtracking a is reservation F# F# brown gateway for the we'll the node.js piece I'M keep tokens once body WON'T there's tokens piece 'single' They'RE node.js so 100% body over a is user@example.com no \\\"quotes\\\" boundaries F# They'RE the C++ backtracking and \\ud83d\\ude42 over WON'T\"}, {\"role\": \"assistant\", \"content\": \"is 'single' scanned \\u6771\\u4eac 3.14159 mirrors {braces} there's scanner every reservation mirrors reservation backtracking involved that tiktoken's {braces} node.js we'll F# jumps v1.2.3 v1.2.3 exactly keep \\u0645\\u0631\\u062d\\u0628\\u0627 1999 scanner with tokens once counting so for https://example.com/a/b?c=d piece the while request Z\\u00fcrich it [brackets] dog involved the involved the admission and (parens) engine F# \\u0645\\u0631\\u062d\\u0628\\u0627 gateway quick \\ud83d\\ude42 v1.2.3 tiktoken's https://example.com/a/b?c=d faster \\ud83d\\ude42 that 42 (parens) jumps caf\\u00e9 so hand budget admission [brackets] gateway brown I'M quick admission C++ C++ caf\\u00e9 \\u6771\\u4eac 'single' user@example.com hand that don't C++ once\"}, {\"role\": \"user\", \"content\": \"there's na\\u00efve reservation na\\u00efve before gateway 100% 1999 request that jumps tiktoken's for admission on quick on scanned boundaries jumps \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors we'll dog because a budget \\u6771\\u4eac it na\\u00efve 1999 that the caf\\u00e9 involved counting jumps tokens with \\u0645\\u0631\\u062d\\u0628\\u0627 involved before the over admission I'M dog v1.2.3 engine tiktoken's we'll scanner every tiktoken's They'RE once It's exactly request jumps node.js WON'T 3.14159 over that exactly engine \\u0645\\u0631\\u062d\\u0628\\u0627 once budget reservation because it 42 keep 'single' boundaries so \\u0645\\u0631\\u062d\\u0628\\u0627 node.js scanner budget 3.14159 engine node.js C++ I'M budget [brackets] a caf\\u00e9 user@example.com jumps tokens every 1999 It's\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 scanner (parens) fox gateway mirrors 42 They'RE reservation we'll fox written brown \\\"quotes\\\" mirrors so 42 hand because budget 100% I'M counting request quick 42 caf\\u00e9 C++ C++ 3.14159 request https://example.com/a/b?c=d jumps jumps scanned brown keep brown and \\ud83d\\ude42 once tokens admission fox F# Z\\u00fcrich user@example.com the gateway tiktoken's https://example.com/a/b?c=d \\\"quotes\\\" tokens \\ud83d\\ude42 \\ud83d\\ude42 [brackets] while node.js [brackets] 1999 on \\\"quotes\\\" caf\\u00e9 1999 jumps 3.14159 request over body admission tiktoken's \\ud83d\\ude42 regex that the once scanner because They'RE regex 'single' admission 42\"}, {\"role\": \"user\", \"content\": \"gateway \\u0645\\u0631\\u062d\\u0628\\u0627 keep quick https://example.com/a/b?c=d F# involved hand It's written request {braces} over mirrors there's gateway every so it caf\\u00e9 involved {braces} quick that gateway over 42 fox lazy involved every keep don't (parens) It's request every for \\u6771\\u4eac admission piece on 3.14159 backtracking written quick gateway and jumps F# (parens) 100% the budget quick keep it 100% \\ud83d\\ude42 for (parens) is \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"way boundaries F# (parens) that It's \\\"quotes\\\" scanned \\ud83d\\ude42 \\u6771\\u4eac that brown regex is over na\\u00efve 3.14159 because Z\\u00fcrich 1999 (parens) \\u0645\\u0631\\u062d\\u0628\\u0627 C++ over way so that user@example.com and user@example.com tiktoken's scanned \\u6771\\u4eac [brackets] engine lazy 1999 involved it 100% a so \\u0645\\u0631\\u062d\\u0628\\u0627 on every body na\\u00efve faster regex engine fox that tokens caf\\u00e9 na\\u00efve 3.14159 and hand engine with\"}, {\"role\": \"user\", \"content\": \"keep there's that every [brackets] $1,234.56 \\\"quotes\\\" on body https://example.com/a/b?c=d involved https://example.com/a/b?c=d tokens scanner 1999 https://example.com/a/b?c=d https://example.com/a/b?c=d node.js 1999 backtracking user@example.com that [brackets] don't body lazy WON'T 42 written hand we'll WON'T no \\u0645\\u0631\\u062d\\u0628\\u0627 $1,234.56 that $1,234.56 way piece https://example.com/a/b?c=d admission admission C++ boundaries we'll reservation scanner the C++ \\u0645\\u0631\\u062d\\u0628\\u0627 hand engine na\\u00efve I'M backtracking while regex \\ud83d\\ude42 budget quick v1.2.3 dog v1.2.3 quick [brackets] fox for node.js [brackets] admission \\ud83d\\ude42 They'RE that on boundaries every that regex 42 a {braces} way every scanned backtracking boundaries budget we'll \\\"quotes\\\" it no caf\\u00e9 scanner jumps backtracking while involved and it 1999 with way {braces} \\\"quotes\\\" 'single'\"}, {\"role\": \"assistant\", \"content\": \"42 is \\\"quotes\\\" budget It's They'RE dog fox exactly fox written regex backtracking written with \\\"quotes\\\" written WON'T that backtracking for a every a engine lazy 42 the once budget on because dog we'll and node.js it $1,234.56 mirrors (parens) It's 42 na\\u00efve it scanned https://example.com/a/b?c=d v1.2.3 C++ 'single' boundaries caf\\u00e9 1999 before once 3.14159 is [brackets] don't tiktoken's scanner fox [brackets] \\u6771\\u4eac over once [brackets] They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 we'll with 42 counting \\u0645\\u0631\\u062d\\u0628\\u0627 C++ 3.14159 it the quick\"}, {\"role\": \"user\", \"content\": \" (parens) with before so is that node.js C++ we'll on admission the \\u6771\\u4eac faster the caf\\u00e9 jumps Z\\u00fcrich (parens) fox \\u6771\\u4eac {braces} every {braces} fox mirrors They'RE They'RE fox counting C++ keep \\\"quotes\\\" 1999 once 100% C++ don't once way dog They'RE once involved exactly \\ud83d\\ude42 we'll piece request reservation quick the don't (parens) scanner I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 Z\\u00fcrich once admission no regex tiktoken's exactly 1999 over engine regex there's involved 42 admission no 'single' over there's (parens) caf\\u00e9 because request keep \\u6771\\u4eac keep \\\"quotes\\\" budget piece piece faster $1,234.56 quick WON'T it piece hand\"}, {\"role\": \"assistant\", \"content\": \"involved scanner WON'T {braces} {braces} It's F# piece it \\\"quotes\\\" I'M because body before quick counting piece na\\u00efve reservation hand WON'T hand the request 1999 involved because {braces} so don't F# and [brackets] faster na\\u00efve way for brown brown hand it Z\\u00fcrich 3.14159 the user@example.com with 1999 while every scanner lazy (parens) tokens [brackets] for tokens body exactly I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 over caf\\u00e9 exactly counting a tiktoken's while no https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 lazy lazy (parens) backtracking that involved keep and a tiktoken's is body They'RE it it 100% scanned jumps jumps (parens) na\\u00efve so for v1.2.3 F# 'single' way no exactly 'single' tiktoken's there's dog caf\\u00e9 engine $1,234.56 is tokens\"}, {\"role\": \"user\", \"content\": \"a C++ 100% faster the node.js gateway every It's budget reservation there's scanner before Z\\u00fcrich \\ud83d\\ude42 written written tokens faster tokens $1,234.56 counting quick Z\\u00fcrich for lazy tokens node.js before before backtracking WON'T It's jumps before Z\\u00fcrich $1,234.56 with boundaries mirrors the request once node.js faster that jumps backtracking 1999 boundaries node.js fox admission tiktoken's mirrors way the we'll involved backtracking request 3.14159 the jumps na\\u00efve na\\u00efve \\\"quotes\\\" mirrors 3.14159 na\\u00efve scanner F# because node.js 3.14159 that caf\\u00e9 100% and and (parens) \\\"quotes\\\" WON'T $1,234.56 \\u6771\\u4eac quick $1,234.56 reservation while the na\\u00efve exactly once written backtracking written the na\\u00efve\"}, {\"role\": \"assistant\", \"content\": \"tokens \\u0645\\u0631\\u062d\\u0628\\u0627 scanned [brackets] every admission [brackets] scanned 'single' exactly scanner tokens that [brackets] tokens scanned the before before They'RE every because \\\"quotes\\\" C++ scanner with with brown exactly is for user@example.com while caf\\u00e9 boundaries it WON'T scanner that admission reservation 100% is admission before gateway reservation caf\\u00e9 100% request tiktoken's 3.14159 before backtracking while so https://example.com/a/b?c=d 42 faster and and faster It's we'll mirrors 42 tiktoken's regex node.js there's scanned\"}, {\"role\": \"user\", \"content\": \"written v1.2.3 exactly reservation piece every F# reservation node.js node.js \\u6771\\u4eac that scanner https://example.com/a/b?c=d 42 {braces} it and keep It's no regex jumps with user@example.com body over It's \\\"quotes\\\" and 'single' counting I'M that \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) engine the way $1,234.56 scanned They'RE for caf\\u00e9 \\\"quotes\\\" lazy don't backtracking tokens \\\"quotes\\\" quick {braces} \\\"quotes\\\" involved involved engine scanner because hand tiktoken's v1.2.3 F# regex $1,234.56 engine {braces} gateway counting that (parens) that the before admission that every backtracking \\\"quotes\\\" [brackets] way gateway engine lazy fox 42 don't [brackets] once na\\u00efve \\\"quotes\\\" They'RE They'RE is jumps every 100% written backtracking body caf\\u00e9 that backtracking v1.2.3 (parens) F# keep exactly piece once v1.2.3 F# They'RE backtracking 100% the fox the Z\\u00fcrich involved F#\"}, {\"role\": \"assistant\", \"content\": \"{braces} 42 engine over na\\u00efve hand involved They'RE request It's Z\\u00fcrich before {braces} 42 dog every node.js is over https://example.com/a/b?c=d na\\u00efve engine fox is [brackets] is on over \\ud83d\\ude42 (parens) \\ud83d\\ude42 regex They'RE They'RE \\ud83d\\ude42 (parens) C++ node.js 100% jumps once that scanner on mirrors regex that faster tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's while F# exactly 100% 3.14159 engine \\u6771\\u4eac scanner {braces} na\\u00efve I'M the over brown written don't tokens user@example.com is F# once faster on It's a there's C++ 1999 we'll [brackets] over the dog there's request dog C++ the brown regex way na\\u00efve written before Z\\u00fcrich engine caf\\u00e9 WON'T 1999 with engine and scanned $1,234.56 keep I'M every user@example.com 'single' a and 1999 don't I'M written while and that\"}, {\"role\": \"user\", \"content\": \"budget {braces} hand so and (parens) fox for that no way is counting tokens WON'T tokens tokens lazy tokens piece on that request dog mirrors I'M mirrors scanned regex on na\\u00efve C++ once user@example.com before lazy user@example.com \\\"quotes\\\" I'M once over involved that request written backtracking \\\"quotes\\\" is no brown there's {braces} because \\ud83d\\ude42 keep reservation written 100% caf\\u00e9 v1.2.3 tiktoken's body F# \\u6771\\u4eac counting quick every faster It's while \\u6771\\u4eac every backtracking lazy They'RE tokens fox {braces} tokens \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" 1999 piece involved [brackets] tokens They'RE the a don't hand caf\\u00e9 quick before 'single' so \\u6771\\u4eac that [brackets] quick \\\"quotes\\\" counting gateway it that is It's scanner the 'single' the \\u0645\\u0631\\u062d\\u0628\\u0627 keep 100% it fox way na\\u00efve exactly mirrors caf\\u00e9 mirrors there's because piece every a\"}, {\"role\": \"assistant\", \"content\": \"counting no tiktoken's hand dog {braces} They'RE $1,234.56 involved \\\"quotes\\\" on \\u0645\\u0631\\u062d\\u0628\\u0627 the because tiktoken's 3.14159 that because counting a 3.14159 so node.js counting backtracking https://example.com/a/b?c=d C++ quick involved (parens) request (parens) F# no gateway exactly every backtracking hand on user@example.com It's written scanned before reservation {braces} there's [brackets] $1,234.56 I'M engine involved brown reservation Z\\u00fcrich request F# brown a way I'M written exactly faster \\u6771\\u4eac Z\\u00fcrich mirrors regex 42 tiktoken's caf\\u00e9 42 100% F# way 1999 no node.js way involved while and {braces} that 1999 They'RE node.js admission I'M the is there's 'single' scanner na\\u00efve scanned tiktoken's \\u6771\\u4eac before backtracking 'single'\"}, {\"role\": \"user\", \"content\": \"na\\u00efve tiktoken's boundaries admission 42 100% {braces} 'single' (parens) and and way I'M 3.14159 budget jumps with gateway the fox engine because on scanner \\\"quotes\\\" 1999 user@example.com that written 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 quick [brackets] WON'T 42 \\u6771\\u4eac \\\"quotes\\\" jumps so budget scanned 'single' boundaries and F# quick every https://example.com/a/b?c=d regex tiktoken's lazy involved we'll \\ud83d\\ude42 it admission v1.2.3 reservation counting scanner that \\u0645\\u0631\\u062d\\u0628\\u0627 over there's counting because 100% keep F# lazy $1,234.56 so that is once request is \\ud83d\\ude42 before once na\\u00efve keep\"}, {\"role\": \"assistant\", \"content\": \"with Z\\u00fcrich a 3.14159 faster faster faster jumps 'single' before over dog It's $1,234.56 node.js 'single' na\\u00efve body way we'll It's once regex the counting 3.14159 [brackets] gateway C++ on piece jumps body is node.js lazy user@example.com don't with v1.2.3 exactly user@example.com [brackets] once It's it Z\\u00fcrich [brackets] {braces} faster \\u0645\\u0631\\u062d\\u0628\\u0627 1999 that C++ involved on quick C++ scanner reservation reservation regex node.js faster because jumps It's mirrors that F# brown the request with every reservation backtracking (parens) v1.2.3 I'M {braces} once tokens reservation 42 a 3.14159 F# we'll (parens) way caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 1999 F# piece 1999 quick for scanned\"}, {\"role\": \"user\", \"content\": \"scanned piece F# reservation because regex regex counting written na\\u00efve budget 42 the before keep keep way It's It's exactly node.js because 3.14159 tokens keep with \\u0645\\u0631\\u062d\\u0628\\u0627 body exactly that reservation v1.2.3 42 [brackets] keep scanner dog piece it F# [brackets] regex that tokens exactly body node.js that once faster hand with lazy exactly boundaries jumps that scanned that written\"}, {\"role\": \"assistant\", \"content\": \"brown They'RE gateway the Z\\u00fcrich gateway user@example.com over quick and \\\"quotes\\\" boundaries regex \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 the tokens 'single' is every tokens 42 reservation (parens) request I'M backtracking regex tokens boundaries on is hand scanner boundaries caf\\u00e9 engine and jumps It's the over user@example.com user@example.com I'M on lazy while 1999 WON'T brown that reservation It's v1.2.3 WON'T exactly gateway with quick F# F# the I'M gateway I'M don't way boundaries \\\"quotes\\\" reservation Z\\u00fcrich and 3.14159 on faster na\\u00efve 42 v1.2.3 v1.2.3 3.14159 jumps hand we'll once (parens) C++ the faster keep They'RE node.js fox scanned 42 jumps faster way gateway hand 1999 it that request keep \\u6771\\u4eac $1,234.56 tokens engine that we'll F# we'll written \\\"quotes\\\" a is no keep tiktoken's node.js node.js on counting user@example.com \\\"quotes\\\" {braces} 1999 user@example.com hand fox I'M node.js is\"}, {\"role\": \"user\", \"content\": \"'single' \\u6771\\u4eac and scanned v1.2.3 dog \\u6771\\u4eac with budget $1,234.56 a exactly and scanned body way na\\u00efve on brown mirrors a 3.14159 dog fox jumps {braces} (parens) jumps the way over and we'll involved faster user@example.com fox body there's gateway body brown jumps there's for 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking It's mirrors Z\\u00fcrich 'single' tokens 'single' for user@example.com 100% faster there's keep C++ and don't gateway budget {braces} faster hand don't is involved \\u6771\\u4eac regex regex reservation scanner\"}, {\"role\": \"assistant\", \"content\": \"F# body is regex jumps WON'T reservation Z\\u00fcrich tokens piece that and \\u0645\\u0631\\u062d\\u0628\\u0627 [brackets] for 'single' we'll engine F# there's \\\"quotes\\\" so counting \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d [brackets] piece [brackets] involved exactly fox {braces} \\ud83d\\ude42 3.14159 way F# 3.14159 scanned while keep every request \\u6771\\u4eac way C++ na\\u00efve engine engine user@example.com because admission I'M gateway v1.2.3 v1.2.3 It's it Z\\u00fcrich https://example.com/a/b?c=d 3.14159 Z\\u00fcrich hand [brackets] [brackets] dog that 'single' exactly \\u6771\\u4eac regex brown reservation involved and WON'T F# na\\u00efve before It's a tiktoken's I'M we'll with written budget engine caf\\u00e9 tokens 100% scanned It's brown every a for\"}, {\"role\": \"user\", \"content\": \"involved because caf\\u00e9 so there's I'M 42 no na\\u00efve na\\u00efve exactly tiktoken's once scanner tiktoken's counting exactly dog the a and faster WON'T so They'RE 'single' faster for written C++ 3.14159 tiktoken's $1,234.56 C++ way exactly https://example.com/a/b?c=d every way node.js budget 42 before before gateway it mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 way faster budget (parens) budget every that don't over so v1.2.3 request we'll because with \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 don't 'single' reservation fox tokens there's request it They'RE [brackets] it node.js gateway 'single' node.js body once way quick 3.14159\"}, {\"role\": \"assistant\", \"content\": \"3.14159 while reservation body before on once (parens) request because https://example.com/a/b?c=d involved 42 user@example.com and no request They'RE dog F# na\\u00efve They'RE exactly [brackets] regex engine mirrors scanner engine It's there's counting \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich way 3.14159 over counting dog gateway no we'll for \\\"quotes\\\" no lazy 42 involved counting involved written counting 1999 a $1,234.56 tokens regex quick and no once \\u6771\\u4eac every so brown is \\u0645\\u0631\\u062d\\u0628\\u0627 fox https://example.com/a/b?c=d caf\\u00e9 brown \\u6771\\u4eac 42 exactly a scanner over \\ud83d\\ude42 we'll budget dog {braces} for before faster\"}, {\"role\": \"user\", \"content\": \" faster the over keep while \\u0645\\u0631\\u062d\\u0628\\u0627 tiktoken's once the because involved scanner regex that \\u6771\\u4eac there's brown WON'T dog \\u0645\\u0631\\u062d\\u0628\\u0627 budget lazy v1.2.3 v1.2.3 caf\\u00e9 counting so don't the scanner I'M caf\\u00e9 1999 don't once engine lazy (parens) request keep user@example.com the keep fox the Z\\u00fcrich https://example.com/a/b?c=d 3.14159 caf\\u00e9 (parens) It's piece I'M written \\\"quotes\\\" 'single' It's that {braces} reservation piece\"}, {\"role\": \"assistant\", \"content\": \"fox quick C++ brown before no lazy faster the that the every 100% na\\u00efve that 100% {braces} scanner we'll involved na\\u00efve before that {braces} faster mirrors because 'single' it faster v1.2.3 involved is so with request piece piece while that engine admission tokens gateway scanned before F# every $1,234.56 hand 1999 request is over v1.2.3 over quick regex v1.2.3 v1.2.3 boundaries Z\\u00fcrich and\"}, {\"role\": \"user\", \"content\": \"node.js WON'T faster \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 dog tokens and the no because They'RE and WON'T WON'T caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 42 na\\u00efve written keep don't the WON'T is admission $1,234.56 and every the I'M is C++ budget node.js tokens (parens) node.js caf\\u00e9 'single' with engine https://example.com/a/b?c=d v1.2.3 written node.js {braces} tiktoken's boundaries tiktoken's lazy scanned user@example.com quick body regex backtracking involved jumps faster quick They'RE request na\\u00efve body on written \\ud83d\\ude42 regex a It's a a no \\\"quotes\\\" \\ud83d\\ude42 while involved the jumps and \\ud83d\\ude42 node.js https://example.com/a/b?c=d boundaries na\\u00efve node.js brown hand 42 na\\u00efve for before C++ while https://example.com/a/b?c=d 42 because that for reservation \\\"quotes\\\" gateway budget is a so It's Z\\u00fcrich budget regex once scanned piece regex budget before over na\\u00efve regex\"}, {\"role\": \"assistant\", \"content\": \"WON'T on scanned user@example.com They'RE exactly faster the for a boundaries faster faster $1,234.56 I'M with don't {braces} jumps \\u0645\\u0631\\u062d\\u0628\\u0627 jumps so is keep while Z\\u00fcrich 42 so tiktoken's [brackets] written {braces} request engine no faster node.js counting \\u6771\\u4eac lazy counting faster is mirrors it there's jumps tiktoken's piece [brackets] \\u6771\\u4eac so brown admission brown Z\\u00fcrich way mirrors reservation user@example.com v1.2.3 $1,234.56 budget because the \"}, {\"role\": \"user\", \"content\": \"every jumps 'single' engine so dog scanned F# there's \\ud83d\\ude42 https://example.com/a/b?c=d user@example.com exactly backtracking node.js admission is na\\u00efve v1.2.3 tokens the fox involved gateway that admission don't reservation hand engine C++ so for we'll over written dog https://example.com/a/b?c=d while way dog and written with brown F# budget backtracking so tiktoken's \\u6771\\u4eac exactly written involved They'RE because reservation a caf\\u00e9 over boundaries regex is 'single' admission keep admission I'M WON'T admission counting scanned dog {braces} admission admission every F# 1999 scanner They'RE and the na\\u00efve 42 there's with involved 42 exactly 100% it [brackets] way They'RE node.js \\ud83d\\ude42 keep we'll I'M so I'M tiktoken's reservation WON'T caf\\u00e9 boundaries C++\"}, {\"role\": \"assistant\", \"content\": \"node.js the It's so C++ scanned so 1999 the budget request F# fox body body keep is [brackets] 42 over scanner there's \\u0645\\u0631\\u062d\\u0628\\u0627 node.js \\u0645\\u0631\\u062d\\u0628\\u0627 don't involved F# hand na\\u00efve scanned with involved engine jumps WON'T {braces} mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 is counting C++ fox it (parens) 3.14159 regex engine {braces} it keep 'single' the \\u0645\\u0631\\u062d\\u0628\\u0627 gateway engine WON'T user@example.com keep [brackets] once before budget the reservation regex hand keep user@example.com It's written reservation no fox \\\"quotes\\\" on \\u6771\\u4eac (parens) Z\\u00fcrich backtracking \\ud83d\\ude42 [brackets] mirrors gateway scanner boundaries v1.2.3 brown admission 1999 because way tokens regex admission so \\u0645\\u0631\\u062d\\u0628\\u0627 with counting fox tiktoken's na\\u00efve tokens counting gateway\"}, {\"role\": \"user\", \"content\": \"v1.2.3 dog counting 42 1999 with (parens) 42 scanner involved the it 'single' jumps over and user@example.com once I'M I'M engine before mirrors a faster brown quick Z\\u00fcrich v1.2.3 \\u6771\\u4eac body it mirrors na\\u00efve fox budget 'single' with mirrors WON'T user@example.com faster so Z\\u00fcrich scanned It's that written reservation is that 42 v1.2.3 over 1999 on 1999 that over na\\u00efve brown there's the 100% \\u6771\\u4eac request \\\"quotes\\\" $1,234.56 budget regex piece na\\u00efve 3.14159 the scanner \\u6771\\u4eac scanner scanner It's v1.2.3 on tokens engine 1999 42\"}, {\"role\": \"assistant\", \"content\": \"faster $1,234.56 piece user@example.com for \\u0645\\u0631\\u062d\\u0628\\u0627 while so on Z\\u00fcrich backtracking no It's \\\"quotes\\\" hand node.js before engine it tokens jumps reservation dog jumps [brackets] boundaries scanner for (parens) {braces} and request before scanned because before the for admission mirrors every piece admission keep and gateway we'll brown 1999 is I'M They'RE over https://example.com/a/b?c=d 3.14159 while hand once exactly way counting caf\\u00e9 3.14159 hand na\\u00efve every scanned \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors because They'RE backtracking \\ud83d\\ude42 \\\"quotes\\\" over admission piece on don't scanned exactly reservation and (parens)\"}, {\"role\": \"user\", \"content\": \"every quick that that while na\\u00efve fox {braces} while brown that v1.2.3 dog a C++ F# over for regex while every jumps it is 1999 caf\\u00e9 it and there's C++ we'll over counting boundaries brown 42 I'M involved and reservation boundaries so before I'M with scanned every is involved https://example.com/a/b?c=d lazy regex piece It's a while v1.2.3 user@example.com the 'single' I'M a is is \\\"quotes\\\" user@example.com because node.js faster F# Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 that na\\u00efve because brown involved the so involved WON'T body backtracking quick engine backtracking gateway that a user@example.com body [brackets] tokens every boundaries while \\ud83d\\ude42 is on boundaries scanner node.js Z\\u00fcrich is there's for regex it 100% piece tiktoken's\"}, {\"role\": \"assistant\", \"content\": \"reservation hand before F# counting $1,234.56 caf\\u00e9 faster \\u6771\\u4eac [brackets] tokens Z\\u00fcrich tiktoken's on written {braces} It's keep \\\"quotes\\\" while tokens na\\u00efve the no because keep https://example.com/a/b?c=d because counting request \\\"quotes\\\" it a because hand involved reservation node.js and backtracking 42 on piece I'M gateway 'single' mirrors the Z\\u00fcrich quick lazy brown {braces} admission Z\\u00fcrich C++ before https://example.com/a/b?c=d It's engine backtracking don't \\u6771\\u4eac piece dog no keep Z\\u00fcrich piece and 42 regex C++ caf\\u00e9 with we'll 3.14159 'single' mirrors counting Z\\u00fcrich it dog scanner a node.js piece {braces} we'll we'll because They'RE user@example.com fox caf\\u00e9 1999 42 every counting tiktoken's It's before backtracking scanner 42 the \\u6771\\u4eac admission lazy F# gateway mirrors It's C++ tiktoken's is dog no written https://example.com/a/b?c=d \\u6771\\u4eac exactly \\\"quotes\\\" body exactly\"}, {\"role\": \"user\", \"content\": \"that https://example.com/a/b?c=d the over boundaries scanner don't [brackets] quick every because because I'M because so once dog mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" quick on 'single' (parens) 42 and that before while while engine while over tiktoken's counting exactly {braces} tiktoken's piece gateway user@example.com It's that admission scanner It's request \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com that 42 for so \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 user@example.com for F# exactly that 100% backtracking boundaries it fox tiktoken's piece counting once engine admission piece Z\\u00fcrich a budget keep and {braces} is reservation scanner with https://example.com/a/b?c=d gateway no user@example.com na\\u00efve so user@example.com written brown\"}, {\"role\": \"assistant\", \"content\": \"piece They'RE body v1.2.3 \\ud83d\\ude42 C++ na\\u00efve It's we'll backtracking on because the counting a involved counting that there's request WON'T jumps that node.js Z\\u00fcrich quick https://example.com/a/b?c=d They'RE Z\\u00fcrich 3.14159 engine budget \\ud83d\\ude42 \\\"quotes\\\" scanner don't {braces} F# request don't $1,234.56 \\\"quotes\\\" tokens over while admission every with quick I'M \\\"quotes\\\" {braces} fox [brackets] gateway dog keep Z\\u00fcrich every [brackets] every mirrors C++ \\ud83d\\ude42 They'RE is [brackets] 3.14159 scanner hand and request 3.14159 with scanner no the regex 1999 They'RE every dog there's the admission that tiktoken's 'single' engine\"}, {\"role\": \"user\", \"content\": \"boundaries It's way dog I'M body lazy https://example.com/a/b?c=d budget faster for I'M regex every body piece with \\ud83d\\ude42 regex before for quick 'single' exactly with over brown there's jumps the don't {braces} and [brackets] it jumps on while don't no exactly every tokens and the 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 no [brackets] keep \\ud83d\\ude42 with Z\\u00fcrich gateway while backtracking the [brackets] every is \\u6771\\u4eac while faster for while budget [brackets] \\ud83d\\ude42 don't once quick so hand 'single' It's on dog exactly with once engine 'single' no written involved over tokens way over [brackets] with so regex caf\\u00e9 3.14159 \\ud83d\\ude42 a node.js every because faster reservation budget v1.2.3 with lazy I'M engine because and for before before engine mirrors \\\"quotes\\\" that node.js node.js counting a admission counting on WON'T no piece I'M once boundaries\"}, {\"role\": \"assistant\", \"content\": \"piece counting quick \\u0645\\u0631\\u062d\\u0628\\u0627 brown the there's mirrors regex faster involved tokens backtracking F# backtracking \\\"quotes\\\" \\ud83d\\ude42 with \\u6771\\u4eac body with gateway counting lazy exactly request [brackets] 'single' because while keep is I'M jumps body {braces} mirrors 1999 $1,234.56 faster it so {braces} v1.2.3 keep 42 admission admission It's that involved https://example.com/a/b?c=d na\\u00efve \\u6771\\u4eac \\u6771\\u4eac 42 fox node.js the (parens) (parens) [brackets] on faster hand node.js engine before \\u0645\\u0631\\u062d\\u0628\\u0627 jumps body we'll engine regex Z\\u00fcrich body admission 3.14159 They'RE Z\\u00fcrich scanner before scanner reservation \\u6771\\u4eac admission involved don't lazy $1,234.56 $1,234.56 that lazy scanned F# for request https://example.com/a/b?c=d there's before scanner reservation They'RE on \\u0645\\u0631\\u062d\\u0628\\u0627 quick we'll 100% lazy while budget 'single' once quick brown admission node.js gateway 1999 v1.2.3 piece because https://example.com/a/b?c=d reservation $1,234.56 for na\\u00efve \\ud83d\\ude42 exactly gateway that exactly on node.js and\"}, {\"role\": \"user\", \"content\": \"hand counting request while reservation there's body regex tokens don't regex every no a engine it a 1999 F# reservation with on mirrors for before Z\\u00fcrich v1.2.3 on the \\u0645\\u0631\\u062d\\u0628\\u0627 quick involved budget every node.js the hand it with node.js 1999 quick faster backtracking quick 42 while counting mirrors over so reservation involved [brackets] no written and tokens fox tiktoken's It's written \\ud83d\\ude42 over admission with over don't \\u6771\\u4eac way counting \\ud83d\\ude42 $1,234.56 https://example.com/a/b?c=d body hand over request tiktoken's that budget https://example.com/a/b?c=d exactly scanned gateway regex the while It's there's written for lazy F# counting body caf\\u00e9 jumps that counting jumps\"}, {\"role\": \"assistant\", \"content\": \"faster Z\\u00fcrich boundaries faster don't counting before dog engine I'M Z\\u00fcrich 3.14159 C++ \\u0645\\u0631\\u062d\\u0628\\u0627 for scanned involved engine backtracking brown \\u6771\\u4eac mirrors is (parens) backtracking tokens admission that Z\\u00fcrich don't C++ tiktoken's user@example.com that mirrors lazy scanner (parens) piece the body every \\\"quotes\\\" the request tokens body C++ (parens) a don't They'RE exactly the 3.14159 is budget user@example.com it \\\"quotes\\\" on \\ud83d\\ude42 backtracking admission written \\ud83d\\ude42 C++ quick fox 1999 I'M for it don't 1999 because way there's body reservation 100% WON'T dog 1999 tokens once a over once so $1,234.56 jumps They'RE scanner 3.14159 regex request 'single' exactly $1,234.56 exactly\"}, {\"role\": \"user\", \"content\": \"we'll 1999 lazy over 42 user@example.com over scanned gateway lazy piece so lazy exactly 'single' caf\\u00e9 node.js there's and once v1.2.3 while faster gateway every way 1999 gateway while 1999 1999 on caf\\u00e9 so written backtracking \\\"quotes\\\" backtracking scanner https://example.com/a/b?c=d body admission dog piece way brown fox 3.14159 with WON'T \\u6771\\u4eac \\\"quotes\\\" and exactly lazy na\\u00efve 1999 (parens) and involved keep that 3.14159 F# keep lazy no na\\u00efve request https://example.com/a/b?c=d jumps 'single' fox 'single' [brackets] tiktoken's counting once there's na\\u00efve admission faster F# we'll \\\"quotes\\\" https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 exactly \\u6771\\u4eac hand over https://example.com/a/b?c=d F# fox scanned 1999\"}, {\"role\": \"assistant\", \"content\": \"1999 no there's is 'single' C++ na\\u00efve that over admission \\\"quotes\\\" F# scanner gateway piece 100% while for no {braces} with fox scanned counting \\\"quotes\\\" there's gateway user@example.com keep body involved 3.14159 Z\\u00fcrich Z\\u00fcrich \\\"quotes\\\" [brackets] quick admission request a is 42 scanned caf\\u00e9 \\\"quotes\\\" I'M piece C++ mirrors 'single' every $1,234.56 3.14159 regex quick dog 42 WON'T brown no {braces} piece body quick before Z\\u00fcrich for for request {braces} engine while \\u6771\\u4eac Z\\u00fcrich brown gateway counting backtracking before don't $1,234.56 the request so {braces} \\u6771\\u4eac [brackets] because a node.js a 3.14159 boundaries every quick we'll jumps na\\u00efve that on we'll scanner tokens \\u6771\\u4eac https://example.com/a/b?c=d tokens 'single' keep\"}, {\"role\": \"user\", \"content\": \"fox admission exactly dog \\u6771\\u4eac backtracking {braces} node.js \\\"quotes\\\" way gateway on 1999 engine scanner keep quick way written boundaries because $1,234.56 don't there's way so over tokens F# that we'll v1.2.3 scanned F# for \\u0645\\u0631\\u062d\\u0628\\u0627 involved there's while {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 admission admission [brackets] They'RE \\ud83d\\ude42 regex body quick that v1.2.3 dog quick user@example.com over before admission hand is budget brown admission we'll body node.js na\\u00efve boundaries hand (parens) there's we'll v1.2.3 100% caf\\u00e9 for [brackets] 100% dog \\\"quotes\\\" piece boundaries I'M admission backtracking gateway engine no C++ there's we'll tiktoken's regex \\u0645\\u0631\\u062d\\u0628\\u0627 way WON'T {braces} once counting 1999 it before quick request WON'T we'll I'M admission lazy tokens involved 42 gateway lazy faster written that reservation exactly \\u6771\\u4eac 100% piece 100% scanner admission\"}, {\"role\": \"assistant\", \"content\": \"exactly because body and we'll so backtracking over fox F# I'M that jumps don't don't lazy scanned I'M lazy way piece It's They'RE scanner gateway 1999 so fox no because gateway 1999 that boundaries engine They'RE on admission scanner over Z\\u00fcrich that Z\\u00fcrich written v1.2.3 WON'T [brackets] tiktoken's scanner 3.14159 don't \\u6771\\u4eac tiktoken's user@example.com [brackets] before \\\"quotes\\\" no C++ node.js scanned 42\"}, {\"role\": \"user\", \"content\": \"once so 3.14159 because 100% with $1,234.56 faster no jumps once written gateway written tokens [brackets] before brown a counting \\\"quotes\\\" tiktoken's They'RE the quick (parens) exactly request jumps before involved mirrors dog budget \\ud83d\\ude42 every scanned \\ud83d\\ude42 fox https://example.com/a/b?c=d gateway body node.js lazy na\\u00efve v1.2.3 exactly way [brackets] gateway scanned WON'T [brackets] https://example.com/a/b?c=d piece \\u6771\\u4eac C++ 3.14159 fox v1.2.3 tokens mirrors caf\\u00e9 engine brown Z\\u00fcrich They'RE hand quick $1,234.56 WON'T tokens lazy written na\\u00efve 'single' dog lazy it that way \\ud83d\\ude42\"}, {\"role\": \"assistant\", \"content\": \"dog a They'RE It's so They'RE counting user@example.com jumps F# request no node.js there's na\\u00efve mirrors \\u6771\\u4eac tiktoken's scanner backtracking hand boundaries 1999 brown written mirrors written faster that budget because hand with 3.14159 and over WON'T every that way regex boundaries C++ \\u0645\\u0631\\u062d\\u0628\\u0627 piece Z\\u00fcrich because it They'RE involved https://example.com/a/b?c=d body 42 a fox that every and request {braces} with node.js F# [brackets] $1,234.56 on no budget way on backtracking node.js so 100%\"}, {\"role\": \"user\", \"content\": \"They'RE scanned dog body on that don't 1999 \\ud83d\\ude42 scanner 1999 that exactly C++ They'RE every caf\\u00e9 brown so \\ud83d\\ude42 caf\\u00e9 mirrors and while request WON'T budget scanner na\\u00efve Z\\u00fcrich the fox 3.14159 100% 'single' before \\ud83d\\ude42 Z\\u00fcrich scanner before that 1999 v1.2.3 body every boundaries that and v1.2.3 1999 100% because user@example.com dog engine keep the we'll I'M scanned na\\u00efve \\\"quotes\\\" there's that lazy https://example.com/a/b?c=d {braces} Z\\u00fcrich exactly \\\"quotes\\\" budget reservation involved that mirrors boundaries na\\u00efve that backtracking \\ud83d\\ude42 once \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 WON'T WON'T I'M node.js involved don't {braces} a and lazy piece backtracking for hand regex\"}, {\"role\": \"assistant\", \"content\": \"WON'T admission admission F# because don't [brackets] tiktoken's before the so na\\u00efve don't hand no na\\u00efve request on mirrors dog \\u0645\\u0631\\u062d\\u0628\\u0627 and before while body $1,234.56 v1.2.3 mirrors piece reservation so backtracking user@example.com every 100% F# regex 1999 brown [brackets] gateway 42 and on with 42 regex (parens) {braces} WON'T v1.2.3 {braces} a there's every I'M the 'single' 42 F# involved \\ud83d\\ude42 piece keep scanner the before I'M reservation written 3.14159 before there's piece once It's \\\"quotes\\\" dog written brown (parens) because while fox quick exactly user@example.com $1,234.56 boundaries Z\\u00fcrich is admission WON'T scanner we'll $1,234.56 hand [brackets] written {braces} {braces} {braces} the\"}, {\"role\": \"user\", \"content\": \"counting once C++ it the and involved keep it so there's I'M C++ 1999 no keep piece It's no user@example.com 'single' I'M WON'T node.js there's 100% quick 100% dog \\ud83d\\ude42 faster Z\\u00fcrich is before there's on don't F# {braces} keep that it na\\u00efve faster counting faster is over hand hand before involved tokens v1.2.3 {braces} boundaries backtracking so on before regex hand backtracking for while keep so (parens) tokens is with exactly backtracking (parens) admission $1,234.56 once no 42 C++ brown reservation Z\\u00fcrich \\\"quotes\\\"\"}, {\"role\": \"assistant\", \"content\": \"so {braces} It's involved body $1,234.56 lazy scanner caf\\u00e9 na\\u00efve v1.2.3 mirrors 3.14159 na\\u00efve the fox It's before we'll over no involved counting C++ brown quick hand \\u6771\\u4eac every exactly 42 boundaries $1,234.56 dog WON'T quick scanned na\\u00efve dog na\\u00efve boundaries quick backtracking body [brackets] tokens piece 100% exactly F# on admission the WON'T budget scanner \\\"quotes\\\" and WON'T we'll boundaries on budget scanned with \\ud83d\\ude42 $1,234.56 before we'll tiktoken's https://example.com/a/b?c=d https://example.com/a/b?c=d over no the there's It's don't over way keep [brackets] with body They'RE the once jumps keep admission They'RE tokens tokens jumps no 1999 fox 3.14159 100% reservation (parens) lazy [brackets] because https://example.com/a/b?c=d \\u0645\\u0631\\u062d\\u0628\\u0627 reservation 3.14159 keep while They'RE I'M brown quick They'RE admission exactly F# regex gateway and v1.2.3 while (parens) because with \\\"quotes\\\"\"}, {\"role\": \"user\", \"content\": \"scanner while node.js scanner 42 hand tokens tiktoken's They'RE engine is 'single' 'single' piece and v1.2.3 while it faster don't brown don't body na\\u00efve (parens) \\ud83d\\ude42 node.js the \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d caf\\u00e9 that $1,234.56 mirrors admission \\ud83d\\ude42 scanner that \\\"quotes\\\" C++ (parens) WON'T Z\\u00fcrich They'RE It's \\ud83d\\ude42 caf\\u00e9 that with WON'T it hand that over \\\"quotes\\\" 'single' na\\u00efve 1999 piece keep {braces} quick the no regex \\u0645\\u0631\\u062d\\u0628\\u0627 \\ud83d\\ude42 piece na\\u00efve the \\u0645\\u0631\\u062d\\u0628\\u0627 'single' no that a body with\"}, {\"role\": \"assistant\", \"content\": \"that C++ over caf\\u00e9 tiktoken's scanned \\u0645\\u0631\\u062d\\u0628\\u0627 fox \\\"quotes\\\" for node.js once keep C++ counting once body so for jumps every before reservation user@example.com budget [brackets] boundaries 42 1999 written jumps $1,234.56 that the https://example.com/a/b?c=d faster Z\\u00fcrich request and once involved [brackets] so over over na\\u00efve caf\\u00e9 for for \\\"quotes\\\" while that scanned tokens scanner no no It's F# request request reservation Z\\u00fcrich jumps the na\\u00efve (parens) the backtracking jumps na\\u00efve scanner caf\\u00e9 keep I'M \\u6771\\u4eac quick scanner body hand scanner I'M way https://example.com/a/b?c=d it caf\\u00e9 mirrors (parens) F# keep {braces} brown WON'T regex engine user@example.com reservation user@example.com brown don't engine brown hand They'RE written https://example.com/a/b?c=d way (parens) dog\"}, {\"role\": \"user\", \"content\": \"\\u6771\\u4eac F# scanner that regex It's lazy na\\u00efve 100% involved keep the admission na\\u00efve user@example.com a for Z\\u00fcrich {braces} exactly lazy counting while tokens with hand written a is counting I'M [brackets] [brackets] 100% tokens body \\\"quotes\\\" \\\"quotes\\\" tiktoken's v1.2.3 https://example.com/a/b?c=d counting $1,234.56 $1,234.56 don't keep that before scanned lazy before gateway na\\u00efve scanner fox $1,234.56 tiktoken's faster so I'M involved over (parens) admission lazy dog caf\\u00e9 don't jumps 100% node.js request because caf\\u00e9 https://example.com/a/b?c=d 42 before 1999 [brackets] {braces} once [brackets] caf\\u00e9 scanner quick there's\"}, {\"role\": \"assistant\", \"content\": \"scanner caf\\u00e9 (parens) keep so {braces} on F# it because so over brown It's exactly I'M I'M F# faster $1,234.56 tokens exactly $1,234.56 caf\\u00e9 the na\\u00efve engine user@example.com \\\"quotes\\\" written C++ so regex don't backtracking engine the caf\\u00e9 C++ They'RE fox lazy written WON'T boundaries keep It's caf\\u00e9 while {braces} C++ keep jumps no node.js 3.14159 there's faster no 'single' so It's so a once scanner \\u0645\\u0631\\u062d\\u0628\\u0627 engine every once before backtracking request because F# scanned jumps I'M exactly we'll we'll and once [brackets] there's exactly 100%\"}, {\"role\": \"user\", \"content\": \"C++ tiktoken's $1,234.56 body scanner dog I'M regex that https://example.com/a/b?c=d mirrors every before user@example.com \\\"quotes\\\" with involved scanner budget I'M reservation hand that for 42 fox 42 tokens scanned They'RE They'RE \\\"quotes\\\" keep reservation piece Z\\u00fcrich the 3.14159 counting counting boundaries F# every while body boundaries 42 don't hand faster the reservation no \\u0645\\u0631\\u062d\\u0628\\u0627 that \\u6771\\u4eac while involved lazy involved caf\\u00e9 that on \\u6771\\u4eac dog 'single' on over once keep Z\\u00fcrich lazy https://example.com/a/b?c=d backtracking scanned 100% 1999 C++ body \\u0645\\u0631\\u062d\\u0628\\u0627 scanner https://example.com/a/b?c=d {braces} don't 3.14159 node.js regex fox faster I'M we'll lazy because backtracking \\ud83d\\ude42 fox F# \\\"quotes\\\" mirrors tokens It's and the for a WON'T regex reservation 42 dog They'RE request on node.js 'single' tokens node.js that admission 1999 on WON'T budget \\ud83d\\ude42 admission exactly 1999 that we'll \\u0645\\u0631\\u062d\\u0628\\u0627 Z\\u00fcrich\"}, {\"role\": \"assistant\", \"content\": \"it 1999 engine \\\"quotes\\\" v1.2.3 every body boundaries over fox before I'M faster that fox \\u0645\\u0631\\u062d\\u0628\\u0627 involved node.js keep They'RE fox the with WON'T jumps hand caf\\u00e9 (parens) exactly user@example.com engine Z\\u00fcrich for request that involved na\\u00efve boundaries faster before no way 'single' piece don't tokens \\ud83d\\ude42 regex $1,234.56 \\ud83d\\ude42 keep [brackets] over 100% once that exactly node.js (parens) Z\\u00fcrich v1.2.3 scanned the admission there's F# and is tiktoken's on \\u0645\\u0631\\u062d\\u0628\\u0627 no involved I'M C++ 42 there's while before v1.2.3 piece over brown we'll 1999 lazy that user@example.com budget C++ on quick engine 1999 {braces} faster and that node.js budget body\"}, {\"role\": \"user\", \"content\": \"user@example.com no tiktoken's involved jumps exactly \\ud83d\\ude42 They'RE \\ud83d\\ude42 once because faster node.js I'M engine hand lazy we'll [brackets] fox budget quick every the tokens C++ a quick They'RE {braces} faster caf\\u00e9 for mirrors caf\\u00e9 it user@example.com node.js written on that engine regex https://example.com/a/b?c=d there's tokens don't na\\u00efve counting \\ud83d\\ude42 user@example.com mirrors engine $1,234.56 3.14159 backtracking hand request brown keep tiktoken's \\u6771\\u4eac a there's it scanner keep engine because fox scanned no I'M no scanned faster 42 [brackets] hand \\u0645\\u0631\\u062d\\u0628\\u0627 that budget scanned that tokens admission C++ and [brackets] that 'single'\"}, {\"role\": \"assistant\", \"content\": \"node.js regex while and WON'T body 100% gateway there's https://example.com/a/b?c=d WON'T keep brown 100% budget with budget \\u0645\\u0631\\u062d\\u0628\\u0627 and so don't while don't exactly backtracking faster tokens gateway node.js lazy over written counting 100% node.js tiktoken's brown dog v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 we'll C++ the with don't we'll tokens 100% no no written scanned counting involved hand user@example.com \\\"quotes\\\" lazy \\u0645\\u0631\\u062d\\u0628\\u0627 body counting for [brackets] 'single' backtracking piece It's regex don't admission once boundaries over lazy F# \\u6771\\u4eac tokens it \\u6771\\u4eac user@example.com fox that while tiktoken's Z\\u00fcrich and (parens) every [brackets] keep counting 100% 42 3.14159 counting it {braces} piece 'single' involved https://example.com/a/b?c=d involved quick \\ud83d\\ude42 tokens written that request tiktoken's lazy admission 42 I'M tokens \\\"quotes\\\" Z\\u00fcrich engine reservation tiktoken's once so a scanned {braces} scanner dog before piece written keep https://example.com/a/b?c=d while It's {braces} Z\\u00fcrich it\"}, {\"role\": \"user\", \"content\": \"Z\\u00fcrich 3.14159 They'RE They'RE there's budget we'll gateway a brown [brackets] jumps once fox there's F# boundaries node.js for once 42 F# dog once so jumps 42 and 100% while it user@example.com exactly brown so [brackets] Z\\u00fcrich 'single' request with tiktoken's Z\\u00fcrich every reservation hand boundaries keep a \\ud83d\\ude42 it once admission piece and we'll budget dog na\\u00efve 3.14159 backtracking so gateway 42 Z\\u00fcrich faster (parens) node.js 100% lazy jumps fox scanner engine 'single' counting request \\u6771\\u4eac a regex brown that 1999 https://example.com/a/b?c=d user@example.com They'RE regex I'M \\ud83d\\ude42 3.14159 \\u0645\\u0631\\u062d\\u0628\\u0627 scanner budget tokens [brackets] WON'T that quick budget v1.2.3 way way user@example.com request the [brackets] $1,234.56 gateway They'RE quick scanned $1,234.56 \\ud83d\\ude42 na\\u00efve don't admission \\u0645\\u0631\\u062d\\u0628\\u0627 budget \\u0645\\u0631\\u062d\\u0628\\u0627\"}, {\"role\": \"assistant\", \"content\": \"WON'T admission piece \\u6771\\u4eac gateway no \\ud83d\\ude42 WON'T tokens 'single' piece admission 3.14159 It's while I'M Z\\u00fcrich the keep boundaries Z\\u00fcrich WON'T It's [brackets] brown tokens They'RE exactly boundaries WON'T admission piece (parens) so tiktoken's backtracking is \\u0645\\u0631\\u062d\\u0628\\u0627 before mirrors 3.14159 dog backtracking written caf\\u00e9 hand scanned way that \\ud83d\\ude42 tiktoken's quick way They'RE {braces} counting na\\u00efve once reservation fox admission $1,234.56 hand and fox we'll hand because quick the is \\u6771\\u4eac {braces} \\\"quotes\\\" exactly and dog so scanner involved \\u6771\\u4eac [brackets]\"}, {\"role\": \"user\", \"content\": \"a \\u0645\\u0631\\u062d\\u0628\\u0627 I'M we'll before backtracking [brackets] scanner don't tokens admission tokens https://example.com/a/b?c=d C++ $1,234.56 is \\ud83d\\ude42 exactly dog piece no tiktoken's admission 'single' the quick that every before before dog It's engine so tiktoken's the boundaries https://example.com/a/b?c=d 'single' 1999 written body {braces} there's scanned budget admission don't for we'll \\\"quotes\\\" body quick regex quick there's we'll it https://example.com/a/b?c=d user@example.com involved na\\u00efve 3.14159 on we'll the over na\\u00efve na\\u00efve \\u6771\\u4eac gateway I'M v1.2.3 it way the scanner jumps budget C++ C++ \\u6771\\u4eac it with quick F# F# with piece budget involved request before backtracking the tiktoken's scanner $1,234.56 backtracking body admission over They'RE on engine request node.js while that They'RE backtracking request v1.2.3 42 3.14159 and 100% scanned a caf\\u00e9 every 100% exactly request tokens is gateway before a exactly caf\\u00e9 we'll node.js because \\ud83d\\ude42 admission\"}, {\"role\": \"assistant\", \"content\": \"F# \\u0645\\u0631\\u062d\\u0628\\u0627 the we'll user@example.com mirrors F# \\u6771\\u4eac faster over with C++ no because piece scanned because 42 piece (parens) scanned admission 100% Z\\u00fcrich It's 42 for 1999 the regex the dog faster budget (parens) brown engine It's on (parens) so https://example.com/a/b?c=d piece over v1.2.3 scanned don't It's piece regex admission https://example.com/a/b?c=d every don't dog F# no regex hand is dog piece no counting v1.2.3 don't https://example.com/a/b?c=d the 3.14159 because once https://example.com/a/b?c=d \\u6771\\u4eac\"}, {\"role\": \"user\", \"content\": \"we'll body tiktoken's F# lazy is is request we'll because dog \\u6771\\u4eac boundaries lazy caf\\u00e9 exactly quick jumps They'RE 3.14159 brown \\u6771\\u4eac 3.14159 'single' on keep faster 'single' because I'M no 'single' tiktoken's exactly F# v1.2.3 tokens hand because faster that over and gateway don't 42 \\u0645\\u0631\\u062d\\u0628\\u0627 lazy before that once for keep body lazy Z\\u00fcrich before faster piece gateway caf\\u00e9 exactly involved scanner the \\ud83d\\ude42 node.js WON'T way before C++ written 1999 It's with that is every na\\u00efve 1999 keep \\u6771\\u4eac quick request regex budget It's 1999 before a hand lazy engine WON'T don't while brown admission \\u0645\\u0631\\u062d\\u0628\\u0627 1999 caf\\u00e9 na\\u00efve WON'T boundaries counting reservation \\\"quotes\\\" there's the once over counting request\"}, {\"role\": \"assistant\", \"content\": \"gateway reservation once F# C++ written 42 on piece https://example.com/a/b?c=d [brackets] that \\u0645\\u0631\\u062d\\u0628\\u0627 we'll reservation They'RE node.js the fox piece https://example.com/a/b?c=d user@example.com every F# $1,234.56 \\\"quotes\\\" admission body [brackets] 'single' that and because regex we'll \\u6771\\u4eac 42 keep na\\u00efve F# user@example.com keep admission we'll user@example.com 1999 {braces} 100% because no $1,234.56 counting the 42 there's boundaries F# no admission 42 so so v1.2.3 for mirrors while the regex piece budget 42 node.js {braces} na\\u00efve regex with jumps that counting a involved $1,234.56 that admission tiktoken's na\\u00efve involved before $1,234.56 budget 1999 because v1.2.3 that backtracking \\u6771\\u4eac mirrors [brackets] node.js hand {braces} don't brown counting Z\\u00fcrich scanned over \\ud83d\\ude42 with I'M lazy tokens\"}, {\"role\": \"user\", \"content\": \"quick keep it body so every C++ gateway It's tokens 'single' 100% we'll quick regex so scanner v1.2.3 the involved It's while {braces} $1,234.56 that written that C++ scanned dog 100% 100% boundaries (parens) gateway exactly scanner regex gateway \\u6771\\u4eac mirrors brown 1999 regex v1.2.3 $1,234.56 caf\\u00e9 scanned brown there's caf\\u00e9 1999 \\\"quotes\\\" no F# for that 100% backtracking exactly admission the quick (parens) 'single' [brackets] {braces} that written boundaries reservation https://example.com/a/b?c=d request on quick 1999 hand so brown regex while exactly piece hand piece reservation tiktoken's scanner \\\"quotes\\\" caf\\u00e9 way because na\\u00efve (parens) tiktoken's 3.14159 admission so don't on involved na\\u00efve C++ dog reservation It's boundaries admission it (parens) gateway\"}, {\"role\": \"assistant\", \"content\": \"keep there's with scanner that budget no on node.js budget scanner dog before exactly quick 1999 WON'T reservation node.js it F# $1,234.56 caf\\u00e9 1999 scanned engine and \\ud83d\\ude42 [brackets] with scanned and admission \\u6771\\u4eac \\ud83d\\ude42 faster {braces} WON'T on I'M F# scanned while dog no exactly written C++ because tiktoken's request reservation written don't 100% gateway budget \\u6771\\u4eac that jumps counting \\u6771\\u4eac every engine scanned They'RE \\ud83d\\ude42 for engine F# and {braces} exactly exactly \\\"quotes\\\" the hand before\"}, {\"role\": \"user\", \"content\": \"jumps $1,234.56 we'll we'll {braces} user@example.com way hand fox way It's brown 3.14159 https://example.com/a/b?c=d regex jumps don't once dog written quick lazy tiktoken's WON'T way it budget the that quick It's WON'T They'RE brown keep because so keep \\u6771\\u4eac don't written because way with is I'M https://example.com/a/b?c=d 3.14159 scanner counting v1.2.3 gateway the it 42 while WON'T tokens caf\\u00e9 jumps once regex budget v1.2.3 C++ backtracking brown exactly involved every regex boundaries the mirrors F# F# $1,234.56 [brackets] WON'T \\u6771\\u4eac $1,234.56 jumps every that It's \\u0645\\u0631\\u062d\\u0628\\u0627 the keep while keep hand regex once jumps v1.2.3 caf\\u00e9 don't brown (parens) 'single'\"}, {\"role\": \"assistant\", \"content\": \"quick because faster $1,234.56 reservation F# don't the boundaries v1.2.3 on user@example.com reservation WON'T body 1999 brown \\u6771\\u4eac jumps \\\"quotes\\\" quick WON'T that (parens) the keep $1,234.56 C++ written {braces} so tokens on {braces} dog https://example.com/a/b?c=d \\ud83d\\ude42 3.14159 written backtracking gateway the way 'single' is gateway that jumps it fox piece backtracking tokens that faster backtracking there's a request request [brackets] and tokens $1,234.56 for user@example.com \\u6771\\u4eac so 'single' 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 WON'T don't 100% [brackets] for lazy the node.js hand because It's faster\"}, {\"role\": \"user\", \"content\": \"is faster admission budget that 1999 counting quick engine no while exactly while the is written v1.2.3 faster v1.2.3 mirrors once They'RE [brackets] tiktoken's 3.14159 quick with because fox while so [brackets] a dog while involved keep 1999 that involved \\ud83d\\ude42 keep is v1.2.3 100% once C++ node.js piece backtracking I'M 3.14159 that (parens) They'RE admission budget fox lazy scanned boundaries on on once that keep https://example.com/a/b?c=d lazy exactly because brown with exactly written \\u6771\\u4eac\"}, {\"role\": \"assistant\", \"content\": \"that 1999 node.js C++ \\u0645\\u0631\\u062d\\u0628\\u0627 no tiktoken's boundaries request so engine before 42 user@example.com a faster piece involved regex body for before lazy hand faster 'single' and hand tiktoken's lazy that node.js jumps budget fox way [brackets] v1.2.3 [brackets] 'single' brown piece It's counting mirrors every 3.14159 \\\"quotes\\\" https://example.com/a/b?c=d na\\u00efve we'll F# 100% once budget because scanned tokens Z\\u00fcrich engine scanned\"}, {\"role\": \"user\", \"content\": \"'single' It's over lazy piece 100% gateway caf\\u00e9 caf\\u00e9 \\\"quotes\\\" a way the request written the budget reservation counting $1,234.56 with Z\\u00fcrich counting I'M keep tokens once we'll the while https://example.com/a/b?c=d a on the tokens involved caf\\u00e9 gateway backtracking before fox na\\u00efve with on 3.14159 written lazy {braces} \\\"quotes\\\" there's don't quick 'single' that for They'RE faster because I'M is scanner \\u6771\\u4eac counting the every Z\\u00fcrich body $1,234.56 quick keep backtracking scanner 'single' lazy and 42 counting written so tokens engine 'single' because scanned They'RE v1.2.3 tokens \\u6771\\u4eac exactly 3.14159 on jumps jumps reservation tokens \\u0645\\u0631\\u062d\\u0628\\u0627 budget They'RE na\\u00efve na\\u00efve request\"}, {\"role\": \"assistant\", \"content\": \"lazy is reservation budget it jumps quick {braces} body 'single' is no jumps \\u6771\\u4eac Z\\u00fcrich no admission 100% and WON'T admission 'single' piece is we'll v1.2.3 is the we'll admission engine request body that body WON'T user@example.com 42 and piece I'M backtracking {braces} WON'T boundaries [brackets] engine request so backtracking budget {braces} caf\\u00e9 3.14159 {braces} It's hand once $1,234.56 They'RE budget It's quick brown regex 42 (parens) and 1999 regex Z\\u00fcrich Z\\u00fcrich Z\\u00fcrich exactly https://example.com/a/b?c=d way tokens no tiktoken's counting that hand backtracking jumps way over WON'T na\\u00efve {braces} piece it that 100% (parens) exactly and is over the WON'T user@example.com 1999 request written counting brown while on because 42 tokens no because (parens) Z\\u00fcrich lazy reservation admission once counting 42 exactly They'RE is dog\"}, {\"role\": \"user\", \"content\": \"that They'RE the admission F# (parens) https://example.com/a/b?c=d tiktoken's 3.14159 scanned reservation 'single' boundaries written exactly faster F# na\\u00efve while and [brackets] It's brown \\ud83d\\ude42 caf\\u00e9 jumps with \\\"quotes\\\" backtracking on reservation C++ written over regex [brackets] counting [brackets] lazy scanned $1,234.56 faster 'single' {braces} (parens) They'RE \\\"quotes\\\" on request F# while the dog faster https://example.com/a/b?c=d brown tokens 100% because WON'T way counting don't scanner a scanner 'single' dog because hand admission tiktoken's {braces} hand caf\\u00e9 that a \\ud83d\\ude42 fox \\ud83d\\ude42 \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors $1,234.56 once tiktoken's node.js \\u6771\\u4eac exactly \\\"quotes\\\" Z\\u00fcrich WON'T (parens) the hand It's once every no They'RE F# C++ lazy every don't 1999 user@example.com piece C++ admission 1999 because I'M budget\"}, {\"role\": \"assistant\", \"content\": \"dog over caf\\u00e9 the $1,234.56 before is that 'single' piece It's jumps exactly na\\u00efve every budget hand tokens $1,234.56 \\ud83d\\ude42 we'll (parens) hand for (parens) exactly way na\\u00efve body gateway brown keep It's before every lazy and while quick budget 100% (parens) no 1999 the 3.14159 regex that fox user@example.com 1999 it quick $1,234.56 there's no that exactly written there's \\u0645\\u0631\\u062d\\u0628\\u0627 request mirrors on involved involved C++ no [brackets] exactly na\\u00efve https://example.com/a/b?c=d 3.14159 gateway the jumps once scanned with that admission budget They'RE 'single' caf\\u00e9 mirrors WON'T \\u0645\\u0631\\u062d\\u0628\\u0627 reservation request It's the body admission reservation scanned we'll node.js regex counting before with \\ud83d\\ude42 backtracking [brackets] https://example.com/a/b?c=d 3.14159 the once It's request a written WON'T mirrors 1999\"}, {\"role\": \"user\", \"content\": \"1999 WON'T so user@example.com jumps fox \\ud83d\\ude42 {braces} there's budget before written quick mirrors {braces} for with 42 and gateway counting we'll request it so a for engine {braces} WON'T Z\\u00fcrich user@example.com because https://example.com/a/b?c=d the node.js is fox I'M 3.14159 tiktoken's faster is once 42 involved no 'single' I'M caf\\u00e9 WON'T I'M They'RE [brackets] $1,234.56 we'll I'M so 42 It's every (parens) we'll once \\ud83d\\ude42 once mirrors jumps budget body {braces} \\ud83d\\ude42 F# fox {braces} exactly \\ud83d\\ude42 WON'T that on 100% budget engine reservation C++ fox F# 42 no (parens) counting mirrors v1.2.3 for fox [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 while way 1999 boundaries piece 100% involved every \\ud83d\\ude42 'single' brown once v1.2.3 before 'single' (parens) v1.2.3 Z\\u00fcrich we'll involved 100% {braces}\"}, {\"role\": \"assistant\", \"content\": \"engine backtracking budget on na\\u00efve mirrors dog reservation don't jumps budget gateway counting dog tokens brown for way Z\\u00fcrich dog piece 'single' that quick it there's quick budget 42 It's that dog They'RE F# written dog piece \\u0645\\u0631\\u062d\\u0628\\u0627 F# regex every no hand body once (parens) every lazy 100% over quick once mirrors don't and v1.2.3 They'RE faster brown 1999 a \\ud83d\\ude42 42 gateway https://example.com/a/b?c=d Z\\u00fcrich for over na\\u00efve it lazy admission admission brown hand \\\"quotes\\\" gateway reservation 1999 keep and {braces} keep F# request written tiktoken's every budget every regex there's exactly I'M \\\"quotes\\\" \\u0645\\u0631\\u062d\\u0628\\u0627 there's backtracking and don't boundaries F# caf\\u00e9 3.14159 (parens) 3.14159 user@example.com https://example.com/a/b?c=d tiktoken's keep every\"}, {\"role\": \"user\", \"content\": \"gateway 'single' v1.2.3 request that hand tiktoken's over v1.2.3 'single' so \\ud83d\\ude42 1999 once once na\\u00efve admission na\\u00efve piece that \\\"quotes\\\" 3.14159 way exactly engine F# the \\u0645\\u0631\\u062d\\u0628\\u0627 3.14159 user@example.com on https://example.com/a/b?c=d It's https://example.com/a/b?c=d F# there's written with {braces} admission {braces} a on user@example.com dog dog [brackets] that caf\\u00e9 over for admission v1.2.3 100% node.js boundaries scanner 42 piece because is (parens) (parens) body no is {braces} that user@example.com \\u0645\\u0631\\u062d\\u0628\\u0627 hand \\\"quotes\\\" user@example.com on that \\\"quotes\\\" backtracking \\\"quotes\\\" on no hand 3.14159 brown faster 100% (parens)\"}, {\"role\": \"assistant\", \"content\": \"there's user@example.com They'RE It's lazy user@example.com before before https://example.com/a/b?c=d request WON'T WON'T fox WON'T and faster for body involved 3.14159 keep written admission on gateway written I'M with quick v1.2.3 C++ there's jumps quick admission scanned jumps node.js before while 1999 over node.js with reservation every once node.js brown don't there's mirrors a [brackets] so faster before \\ud83d\\ude42 tiktoken's admission F# don't written na\\u00efve admission is fox we'll way dog tokens scanned the \\\"quotes\\\" hand no 'single' \\ud83d\\ude42 over we'll that regex so every for (parens) is with the \\\"quotes\\\" exactly piece regex exactly brown for for hand tokens C++ user@example.com tiktoken's engine I'M the It's faster regex is dog 'single' C++ user@example.com \\ud83d\\ude42 Z\\u00fcrich {braces} node.js for \\u6771\\u4eac brown regex the with tiktoken's 42 $1,234.56 (parens)\"}, {\"role\": \"user\", \"content\": \"request it reservation that https://example.com/a/b?c=d so engine user@example.com jumps that and the 3.14159 because 3.14159 brown tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 scanner \\u0645\\u0631\\u062d\\u0628\\u0627 fox counting there's dog lazy node.js tiktoken's I'M we'll 42 it tokens faster faster before \\u0645\\u0631\\u062d\\u0628\\u0627 reservation so with the regex it once backtracking for It's hand regex way there's 100% exactly dog no we'll 42 brown lazy scanned budget na\\u00efve WON'T because and we'll 42 way tiktoken's WON'T gateway we'll I'M dog https://example.com/a/b?c=d scanned and that mirrors \\ud83d\\ude42 C++ They'RE with counting on keep na\\u00efve 3.14159 once request keep admission \\\"quotes\\\" C++ \\ud83d\\ude42 once quick boundaries user@example.com tokens budget budget scanner caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 exactly node.js hand\"}, {\"role\": \"assistant\", \"content\": \"fox for user@example.com there's jumps quick fox I'M boundaries mirrors before involved over node.js admission that is lazy It's 1999 1999 fox involved user@example.com \\\"quotes\\\" boundaries 100% \\u0645\\u0631\\u062d\\u0628\\u0627 that \\u6771\\u4eac backtracking 100% there's jumps because 42 tiktoken's caf\\u00e9 that v1.2.3 tokens brown \\ud83d\\ude42 backtracking user@example.com F# 100% 3.14159 no boundaries It's quick https://example.com/a/b?c=d Z\\u00fcrich tokens 42 100% over WON'T They'RE for lazy body involved every on on user@example.com \\u6771\\u4eac (parens) quick once faster quick over backtracking don't once They'RE engine caf\\u00e9 tokens fox because there's every dog we'll \\u6771\\u4eac exactly 3.14159 \\ud83d\\ude42 boundaries no every the F# 100% faster faster once boundaries a I'M {braces} 1999 over \\\"quotes\\\" mirrors that fox 3.14159 lazy na\\u00efve admission that reservation no for written because scanned caf\\u00e9 body\"}, {\"role\": \"user\", \"content\": \"and involved body jumps \\\"quotes\\\" {braces} it we'll regex it na\\u00efve counting the reservation keep admission once before while admission v1.2.3 I'M reservation 1999 scanned tokens counting don't boundaries lazy exactly we'll $1,234.56 and \\u0645\\u0631\\u062d\\u0628\\u0627 \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 engine I'M 42 F# keep every (parens) budget 100% written tiktoken's (parens) don't na\\u00efve I'M body because the is (parens) admission brown hand piece 42 every request don't $1,234.56 once there's 1999 with we'll that backtracking F# with fox mirrors [brackets] for WON'T with jumps request fox scanned a request while no that faster way I'M \\u6771\\u4eac boundaries that brown dog so (parens) written\"}, {\"role\": \"assistant\", \"content\": \" 42 exactly and \\\"quotes\\\" mirrors admission F# faster F# a on faster with quick brown reservation piece (parens) fox \\u0645\\u0631\\u062d\\u0628\\u0627 budget C++ v1.2.3 so over node.js Z\\u00fcrich WON'T tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 F# 1999 42 the involved once 100% is quick (parens) before jumps dog don't over because $1,234.56 reservation 3.14159 that WON'T caf\\u00e9 backtracking request the \\u0645\\u0631\\u062d\\u0628\\u0627 piece every involved 100% budget there's boundaries counting once written we'll na\\u00efve 42 \\ud83d\\ude42 I'M 3.14159 regex scanner reservation tiktoken's on I'M \\u6771\\u4eac boundaries on brown there's that 'single' the exactly scanned no admission request we'll tiktoken's 'single' because \\\"quotes\\\" \\u6771\\u4eac no tiktoken's caf\\u00e9 jumps it tiktoken's lazy faster exactly body tiktoken's fox budget\"}, {\"role\": \"user\", \"content\": \"so fox a backtracking Z\\u00fcrich admission budget fox scanned 42 request don't because reservation way admission gateway v1.2.3 the so \\u6771\\u4eac involved 'single' with a 100% 'single' v1.2.3 user@example.com dog backtracking body C++ piece the so mirrors written once $1,234.56 while fox 42 [brackets] F# we'll Z\\u00fcrich piece budget while for we'll the we'll written exactly $1,234.56 mirrors counting \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors on 'single' na\\u00efve is node.js while is a on\"}, {\"role\": \"assistant\", \"content\": \"on involved 42 'single' They'RE piece while [brackets] dog that and there's before once jumps counting that so is \\u6771\\u4eac [brackets] over and exactly with involved it once on 1999 42 3.14159 is keep \\u6771\\u4eac jumps that there's They'RE piece lazy over with lazy request \\ud83d\\ude42 written node.js admission backtracking way the over backtracking node.js and while $1,234.56 [brackets] while 42 \\u0645\\u0631\\u062d\\u0628\\u0627 I'M 'single' while user@example.com WON'T \\\"quotes\\\" before admission the faster budget mirrors counting we'll once jumps involved WON'T 3.14159 v1.2.3 [brackets] keep over for 'single' \\u6771\\u4eac regex written na\\u00efve way (parens) user@example.com request there's every involved I'M for scanned It's WON'T https://example.com/a/b?c=d C++ caf\\u00e9 once user@example.com [brackets] a 1999 it tiktoken's 'single' a over mirrors jumps request there's dog reservation mirrors tokens jumps tiktoken's is user@example.com tokens\"}, {\"role\": \"user\", \"content\": \"backtracking once over user@example.com body scanner piece faster \\ud83d\\ude42 over for C++ exactly https://example.com/a/b?c=d no engine 'single' piece 42 over hand for for backtracking quick way 1999 regex scanned every reservation no before over \\\"quotes\\\" so $1,234.56 [brackets] tokens na\\u00efve request a budget fox budget user@example.com tokens brown I'M \\\"quotes\\\" hand I'M \\u0645\\u0631\\u062d\\u0628\\u0627 \\u6771\\u4eac engine once engine backtracking while there's no [brackets] jumps the for [brackets] there's \\u0645\\u0631\\u062d\\u0628\\u0627 dog 42 admission so na\\u00efve F# there's \\u6771\\u4eac reservation F# keep piece keep user@example.com caf\\u00e9 the that It's $1,234.56 v1.2.3 It's na\\u00efve engine lazy They'RE counting faster 'single' piece scanned $1,234.56 user@example.com boundaries [brackets] 'single' \\u6771\\u4eac brown so scanned 'single' scanner boundaries 'single' admission admission \\\"quotes\\\" caf\\u00e9 piece\"}, {\"role\": \"assistant\", \"content\": \"Z\\u00fcrich counting while the the no 100% \\u0645\\u0631\\u062d\\u0628\\u0627 body 3.14159 before the while 1999 once that on caf\\u00e9 1999 on it tiktoken's piece scanned 'single' Z\\u00fcrich and [brackets] regex with regex \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com counting admission way keep scanner scanner \\u0645\\u0631\\u062d\\u0628\\u0627 every so [brackets] tokens $1,234.56 every budget boundaries because reservation and keep 1999 \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d tiktoken's https://example.com/a/b?c=d scanner admission caf\\u00e9 $1,234.56 F# faster so engine so it regex 'single' before hand with that admission fox keep Z\\u00fcrich no scanned I'M tiktoken's the it admission a jumps brown that there's it that (parens) 100% (parens) gateway for\"}, {\"role\": \"user\", \"content\": \"so 100% once it there's (parens) that with 1999 lazy written written keep na\\u00efve brown caf\\u00e9 regex the tiktoken's that a while brown It's dog and It's 'single' $1,234.56 42 \\u6771\\u4eac I'M v1.2.3 once na\\u00efve I'M {braces} node.js scanned It's hand piece no caf\\u00e9 backtracking admission with we'll is on node.js fox on tokens brown brown regex scanner admission $1,234.56 lazy regex piece regex node.js (parens) once on while quick dog gateway body that admission for written it scanned I'M it mirrors because the v1.2.3 every 3.14159 boundaries request na\\u00efve It's because faster hand dog tokens v1.2.3 hand that tiktoken's keep caf\\u00e9 keep tokens we'll it jumps no \\ud83d\\ude42 node.js exactly na\\u00efve na\\u00efve every involved tiktoken's \\u6771\\u4eac counting dog over user@example.com https://example.com/a/b?c=d 1999 3.14159 node.js is hand F# written a\"}, {\"role\": \"assistant\", \"content\": \"we'll fox there's 3.14159 C++ so budget because we'll Z\\u00fcrich the on before we'll that Z\\u00fcrich fox keep scanned dog once that \\\"quotes\\\" while scanned brown engine for fox while there's (parens) 42 every we'll caf\\u00e9 https://example.com/a/b?c=d budget exactly gateway F# https://example.com/a/b?c=d I'M dog way before brown with it 3.14159 don't no engine the piece keep user@example.com Z\\u00fcrich It's once written and They'RE the admission scanner with written that fox engine body on scanner 1999 the admission WON'T quick user@example.com I'M admission tiktoken's https://example.com/a/b?c=d while https://example.com/a/b?c=d every with involved the admission gateway there's 'single' gateway a scanner and request that is engine counting dog user@example.com that hand counting lazy admission boundaries engine while way WON'T brown \\ud83d\\ude42 F# I'M F# scanner \\u6771\\u4eac 3.14159 a backtracking scanner I'M brown\"}, {\"role\": \"user\", \"content\": \"I'M lazy It's over on faster fox tiktoken's while that It's for that reservation https://example.com/a/b?c=d keep node.js the \\u6771\\u4eac the admission body 1999 exactly and way dog (parens) node.js \\u6771\\u4eac on They'RE quick that faster F# boundaries tokens boundaries reservation reservation backtracking tokens is brown 100% {braces} quick engine we'll node.js 100% scanned over F# hand na\\u00efve that request jumps quick because regex\"}, {\"role\": \"assistant\", \"content\": \"with 100% $1,234.56 boundaries we'll so no \\\"quotes\\\" $1,234.56 Z\\u00fcrich we'll brown on exactly 1999 WON'T caf\\u00e9 every dog we'll regex before tokens before [brackets] lazy over faster once Z\\u00fcrich request lazy They'RE tokens https://example.com/a/b?c=d [brackets] don't It's with keep hand hand (parens) I'M budget {braces} faster it way is there's every dog gateway \\\"quotes\\\" it Z\\u00fcrich way the over quick tokens C++ hand mirrors It's It's (parens) F# 100% boundaries caf\\u00e9 F# we'll scanner admission\"}, {\"role\": \"user\", \"content\": \"it node.js caf\\u00e9 'single' quick C++ $1,234.56 3.14159 hand 42 $1,234.56 I'M every exactly 100% lazy the \\u6771\\u4eac tiktoken's WON'T jumps with \\u0645\\u0631\\u062d\\u0628\\u0627 I'M counting written piece and keep it regex on F# keep 3.14159 'single' dog and boundaries tokens \\\"quotes\\\" engine with hand that 3.14159 brown WON'T don't scanner keep I'M WON'T fox F# exactly 'single' piece engine and so on request and It's reservation written that boundaries is caf\\u00e9 \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 on every counting a written so jumps with scanner with quick don't jumps way \\u6771\\u4eac keep mirrors regex (parens) we'll it scanned node.js $1,234.56 reservation They'RE caf\\u00e9 node.js for while scanner it faster [brackets] gateway it scanner for mirrors na\\u00efve admission dog on\"}, {\"role\": \"assistant\", \"content\": \"that v1.2.3 written node.js I'M keep user@example.com tiktoken's tiktoken's way backtracking They'RE quick while scanned way we'll user@example.com way 1999 way caf\\u00e9 https://example.com/a/b?c=d hand faster is involved 1999 (parens) quick is (parens) on brown scanner with dog don't admission involved exactly F# They'RE is with exactly way before counting \\\"quotes\\\" backtracking once gateway faster 3.14159 [brackets] way [brackets] a keep \\ud83d\\ude42 hand (parens) counting boundaries budget I'M mirrors v1.2.3 F# once way 1999 WON'T reservation [brackets] for it scanned because 100% reservation 100% tiktoken's They'RE engine a it that so mirrors written over mirrors node.js F# \\ud83d\\ude42 brown dog it v1.2.3 user@example.com the 1999 reservation counting [brackets] budget way mirrors C++ F# and so there's so They'RE I'M scanner request request WON'T and written budget Z\\u00fcrich faster written lazy C++ brown user@example.com \\ud83d\\ude42 lazy reservation caf\\u00e9 node.js caf\\u00e9 brown\"}, {\"role\": \"user\", \"content\": \"budget budget and \\u6771\\u4eac tiktoken's tiktoken's don't that boundaries that 1999 backtracking with (parens) tiktoken's there's fox counting because exactly the every [brackets] engine Z\\u00fcrich admission way scanned boundaries \\ud83d\\ude42 user@example.com counting na\\u00efve F# fox 100% so faster 1999 because so quick engine way fox that engine lazy quick 1999 way $1,234.56 quick while body that 'single' Z\\u00fcrich They'RE na\\u00efve keep 3.14159 https://example.com/a/b?c=d the involved I'M jumps\"}, {\"role\": \"assistant\", \"content\": \"every jumps Z\\u00fcrich piece 'single' F# we'll 'single' over exactly written C++ 3.14159 with scanner exactly while user@example.com a body engine there's engine way 100% request [brackets] exactly way jumps Z\\u00fcrich budget once gateway before I'M \\\"quotes\\\" https://example.com/a/b?c=d user@example.com written admission exactly I'M {braces} \\ud83d\\ude42 They'RE with WON'T on backtracking node.js a with a lazy request na\\u00efve that scanner that once scanned quick 'single'\"}, {\"role\": \"user\", \"content\": \"v1.2.3 budget brown every [brackets] that na\\u00efve there's the and (parens) is brown They'RE F# there's every there's because the so because \\u0645\\u0631\\u062d\\u0628\\u0627 and over 3.14159 brown piece F# involved (parens) so regex while Z\\u00fcrich (parens) counting user@example.com while on $1,234.56 it v1.2.3 {braces} engine v1.2.3 while 1999 keep regex while https://example.com/a/b?c=d once engine WON'T involved \\u6771\\u4eac I'M it with for on hand \\ud83d\\ude42 v1.2.3 on a na\\u00efve regex there's na\\u00efve before v1.2.3 dog (parens) for brown 3.14159 v1.2.3 [brackets] exactly exactly no user@example.com on mirrors It's with that {braces} faster\"}, {\"role\": \"assistant\", \"content\": \"for once admission backtracking engine because once for I'M brown fox the quick quick backtracking there's lazy engine mirrors every don't caf\\u00e9 before mirrors engine 3.14159 on user@example.com with don't keep it node.js before hand regex reservation while scanner na\\u00efve body body no with F# admission Z\\u00fcrich F# WON'T no I'M \\ud83d\\ude42 way scanned $1,234.56 with a no it caf\\u00e9 lazy the dog https://example.com/a/b?c=d involved caf\\u00e9 jumps we'll scanner caf\\u00e9 42 na\\u00efve is we'll\"}, {\"role\": \"user\", \"content\": \"'single' scanner 100% we'll it a {braces} while it v1.2.3 while mirrors there's exactly caf\\u00e9 It's gateway (parens) fox every dog (parens) a written while while before hand involved admission written counting It's mirrors C++ 1999 scanned fox don't written don't reservation exactly mirrors it https://example.com/a/b?c=d there's no scanned that I'M involved $1,234.56 lazy fox scanned written I'M 100% is we'll every don't F# brown every It's keep while scanned jumps tokens 'single' hand Z\\u00fcrich request admission It's 'single' quick backtracking user@example.com request tokens budget exactly gateway jumps {braces} 'single' once tokens a Z\\u00fcrich before admission admission way jumps fox piece hand They'RE we'll exactly reservation on it written mirrors every once $1,234.56 scanner scanner Z\\u00fcrich jumps is over (parens) written piece Z\\u00fcrich $1,234.56 before request 'single' is 'single' keep\"}, {\"role\": \"assistant\", \"content\": \"scanned $1,234.56 gateway over because It's because tokens (parens) C++ admission is hand C++ brown v1.2.3 user@example.com brown \\\"quotes\\\" once backtracking lazy [brackets] with scanner dog lazy 'single' budget 1999 we'll reservation exactly on exactly the counting the [brackets] engine They'RE scanner caf\\u00e9 (parens) involved na\\u00efve on dog (parens) before don't while fox F# request once the lazy \\ud83d\\ude42 gateway user@example.com once that for 42 involved body user@example.com on regex the for lazy backtracking node.js body over involved v1.2.3 \\u6771\\u4eac regex mirrors there's involved regex faster that before request 3.14159 3.14159 that body 1999 faster for 42 'single' user@example.com request user@example.com piece scanned 1999 keep 3.14159 every fox F# scanned before jumps that admission counting They'RE that tokens I'M\"}, {\"role\": \"user\", \"content\": \"exactly quick and https://example.com/a/b?c=d a brown is dog hand regex It's tiktoken's the every with on scanner every keep scanned backtracking because budget keep fox there's so with so that on exactly counting once it exactly (parens) 3.14159 and that while fox admission 42 reservation It's the na\\u00efve 42 regex we'll piece hand so written regex faster fox backtracking that They'RE while 1999 \\u6771\\u4eac \\\"quotes\\\" request \\u6771\\u4eac They'RE and 100% the written for over way 1999 tokens https://example.com/a/b?c=d brown I'M brown brown keep while with $1,234.56 hand user@example.com {braces} [brackets] that faster na\\u00efve scanner on it caf\\u00e9 counting They'RE \\\"quotes\\\" {braces} counting there's body don't jumps budget fox reservation faster \\u0645\\u0631\\u062d\\u0628\\u0627 faster we'll with gateway [brackets] over counting It's there's \\u6771\\u4eac tokens that backtracking scanner that it so\"}, {\"role\": \"assistant\", \"content\": \"boundaries body fox counting that F# on there's caf\\u00e9 scanned jumps 42 dog hand jumps lazy \\\"quotes\\\" for WON'T is 42 on while 1999 scanner over there's engine 100% \\u0645\\u0631\\u062d\\u0628\\u0627 \\\"quotes\\\" jumps before don't [brackets] is counting involved It's I'M boundaries the (parens) with I'M engine 100% for exactly 'single' piece C++ na\\u00efve {braces} we'll every a no a \\\"quotes\\\" user@example.com body the C++ written https://example.com/a/b?c=d counting I'M a v1.2.3 dog 'single' Z\\u00fcrich keep scanner written before over lazy scanned the with keep over 'single' lazy 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 (parens) dog scanned the the dog no $1,234.56 with a so because quick Z\\u00fcrich fox v1.2.3 once faster \\ud83d\\ude42 faster boundaries it 1999 mirrors way a 3.14159 so it written \\u0645\\u0631\\u062d\\u0628\\u0627 https://example.com/a/b?c=d because the before involved Z\\u00fcrich budget while v1.2.3 scanned boundaries regex we'll backtracking\"}, {\"role\": \"user\", \"content\": \"involved $1,234.56 while jumps \\ud83d\\ude42 \\\"quotes\\\" piece hand written fox jumps v1.2.3 hand piece a jumps [brackets] brown involved Z\\u00fcrich engine that faster 'single' way faster budget F# 1999 so that tiktoken's involved regex WON'T budget mirrors mirrors F# before gateway I'M They'RE https://example.com/a/b?c=d lazy counting gateway regex boundaries \\\"quotes\\\" \\\"quotes\\\" 100% {braces} keep written 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 boundaries before written na\\u00efve 3.14159 that v1.2.3 no before request brown dog for $1,234.56 we'll node.js written mirrors scanned WON'T because we'll we'll way F# piece They'RE and while\"}, {\"role\": \"assistant\", \"content\": \"and that scanner brown They'RE we'll Z\\u00fcrich C++ the it 1999 hand F# admission \\u6771\\u4eac is mirrors so once the it admission counting is boundaries gateway that tiktoken's gateway lazy \\u6771\\u4eac na\\u00efve the we'll fox we'll is 42 that hand 3.14159 while reservation piece there's 1999 (parens) so involved we'll because involved written it caf\\u00e9 before once because https://example.com/a/b?c=d hand fox $1,234.56 admission is hand $1,234.56 boundaries for backtracking WON'T 1999 gateway the They'RE [brackets] budget na\\u00efve because that 42 They'RE C++ is don't \\ud83d\\ude42 C++ I'M $1,234.56 once 3.14159 1999 Z\\u00fcrich boundaries reservation quick exactly gateway $1,234.56 {braces}\"}, {\"role\": \"user\", \"content\": \"42 written (parens) backtracking jumps 1999 a \\\"quotes\\\" involved scanned admission They'RE gateway so backtracking 42 100% keep written It's WON'T user@example.com scanner that we'll WON'T Z\\u00fcrich don't na\\u00efve \\ud83d\\ude42 lazy mirrors (parens) before dog 'single' backtracking tokens faster 100% the body with 42 while {braces} while lazy brown I'M way exactly involved $1,234.56 'single' tokens engine counting tiktoken's $1,234.56 it user@example.com so reservation a 3.14159 dog that brown body fox mirrors quick before don't lazy gateway 100% \\u6771\\u4eac tokens the 42 dog 100% the gateway lazy because request there's $1,234.56 It's and regex 100% It's 'single' {braces} that keep involved caf\\u00e9 with {braces} They'RE faster the the $1,234.56 I'M backtracking I'M reservation 42 1999 before lazy request keep mirrors [brackets] 3.14159 42 {braces} $1,234.56 body 42 It's request 3.14159 {braces}\"}, {\"role\": \"assistant\", \"content\": \"and dog body user@example.com fox quick faster scanned the it faster $1,234.56 $1,234.56 with user@example.com over gateway fox tokens piece quick Z\\u00fcrich and counting engine no dog dog reservation on it it 100% every regex {braces} node.js mirrors before the Z\\u00fcrich v1.2.3 mirrors \\ud83d\\ude42 scanned WON'T brown there's \\u6771\\u4eac that mirrors for I'M it mirrors it node.js is \\u6771\\u4eac body \\\"quotes\\\" dog WON'T tiktoken's regex the It's the It's keep 3.14159 quick regex budget WON'T don't and counting over piece scanned C++ scanned is no \\u0645\\u0631\\u062d\\u0628\\u0627 1999 we'll it https://example.com/a/b?c=d keep https://example.com/a/b?c=d user@example.com admission backtracking once user@example.com piece request that (parens) budget written faster we'll budget scanned the user@example.com regex for WON'T piece counting\"}, {\"role\": \"user\", \"content\": \"{braces} 100% for It's the 1999 \\\"quotes\\\" scanned the 'single' quick caf\\u00e9 \\ud83d\\ude42 once once there's C++ It's we'll I'M engine no It's it na\\u00efve involved fox we'll written jumps 3.14159 dog exactly there's It's gateway tiktoken's every It's C++ WON'T we'll na\\u00efve reservation user@example.com 3.14159 every that with backtracking hand every v1.2.3 before so tiktoken's brown It's 3.14159 v1.2.3 caf\\u00e9 because \\\"quotes\\\" faster user@example.com It's before It's (parens) for I'M request written written dog \\u6771\\u4eac counting faster boundaries because tiktoken's on and gateway fox C++ admission It's {braces} v1.2.3 [brackets] 3.14159 exactly that budget that C++ (parens) so 1999 regex involved budget because scanner {braces} dog because I'M reservation exactly [brackets] reservation don't\"}, {\"role\": \"assistant\", \"content\": \"scanner They'RE \\\"quotes\\\" gateway 100% 3.14159 $1,234.56 so request 1999 C++ na\\u00efve quick faster we'll WON'T dog before engine 'single' tiktoken's node.js \\u0645\\u0631\\u062d\\u0628\\u0627 scanner it and na\\u00efve \\ud83d\\ude42 regex so once there's (parens) before 3.14159 dog [brackets] hand 100% backtracking over don't is once jumps caf\\u00e9 https://example.com/a/b?c=d so lazy that 3.14159 reservation on admission \\ud83d\\ude42 (parens) exactly budget v1.2.3 backtracking with brown request It's it with tokens (parens) scanner 1999 backtracking a fox faster v1.2.3 $1,234.56 I'M node.js the reservation piece [brackets] so \\u6771\\u4eac lazy counting the $1,234.56 I'M backtracking over 3.14159 fox fox keep the backtracking with It's [brackets] 3.14159 (parens) that They'RE so [brackets] piece [brackets] {braces} counting written boundaries mirrors budget (parens) gateway \\ud83d\\ude42 exactly hand engine WON'T 3.14159 faster WON'T fox request\"}, {\"role\": \"user\", \"content\": \"while quick \\u0645\\u0631\\u062d\\u0628\\u0627 so every 42 \\ud83d\\ude42 boundaries counting gateway that \\u0645\\u0631\\u062d\\u0628\\u0627 'single' no 100% and regex because before request and 'single' the once F# na\\u00efve $1,234.56 quick $1,234.56 reservation don't node.js scanner 42 node.js 1999 $1,234.56 https://example.com/a/b?c=d jumps lazy 'single' WON'T They'RE WON'T brown {braces} that scanner because Z\\u00fcrich written dog 3.14159 budget \\\"quotes\\\" engine 3.14159 3.14159 for before exactly tiktoken's jumps user@example.com is (parens) scanner lazy piece request brown Z\\u00fcrich node.js admission It's\"}, {\"role\": \"assistant\", \"content\": \"$1,234.56 C++ node.js because it 42 dog boundaries mirrors piece don't C++ 3.14159 https://example.com/a/b?c=d scanner there's is keep \\ud83d\\ude42 once before because backtracking over $1,234.56 [brackets] quick we'll gateway because we'll exactly once every gateway with over because 3.14159 keep every [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 the because 42 1999 no 100% backtracking 1999 \\u6771\\u4eac faster (parens) faster 42 the F# budget dog 1999 regex before\"}, {\"role\": \"user\", \"content\": \"no it \\ud83d\\ude42 mirrors https://example.com/a/b?c=d reservation C++ on budget \\u6771\\u4eac the for {braces} [brackets] and tiktoken's written 42 so body \\\"quotes\\\" involved C++ budget is written WON'T jumps It's {braces} counting reservation tokens brown piece hand before no It's gateway gateway [brackets] piece mirrors there's \\ud83d\\ude42 while a caf\\u00e9 \\u6771\\u4eac caf\\u00e9 counting counting it 3.14159 don't caf\\u00e9 so written for tiktoken's boundaries tokens 1999 fox keep quick that boundaries that no scanner Z\\u00fcrich budget it written that backtracking involved quick mirrors backtracking brown jumps the lazy They'RE and 42 backtracking 100% brown tokens gateway boundaries caf\\u00e9 reservation way https://example.com/a/b?c=d $1,234.56 100% regex \\u0645\\u0631\\u062d\\u0628\\u0627 lazy we'll \\u0645\\u0631\\u062d\\u0628\\u0627 involved jumps \\u0645\\u0631\\u062d\\u0628\\u0627 while node.js Z\\u00fcrich Z\\u00fcrich counting 100% (parens) that 100% on {braces} counting once user@example.com faster gateway \\u0645\\u0631\\u062d\\u0628\\u0627 that with\"}, {\"role\": \"assistant\", \"content\": \"faster on that boundaries on They'RE F# https://example.com/a/b?c=d that node.js quick for 100% the regex v1.2.3 don't with 42 C++ gateway is before keep is I'M the (parens) They'RE It's \\\"quotes\\\" so mirrors every backtracking tiktoken's v1.2.3 the 3.14159 They'RE scanner over no so that and WON'T that every the don't \\\"quotes\\\" 42 3.14159 node.js budget every 'single' caf\\u00e9 node.js before counting engine\"}, {\"role\": \"user\", \"content\": \"3.14159 piece tiktoken's with request regex on body [brackets] Z\\u00fcrich every gateway the that tiktoken's https://example.com/a/b?c=d no so budget node.js 1999 Z\\u00fcrich we'll They'RE keep and because piece every \\ud83d\\ude42 reservation faster written while request 3.14159 because {braces} fox WON'T written \\\"quotes\\\" (parens) over jumps mirrors body that jumps hand keep 3.14159 a the the it because so we'll gateway tiktoken's regex the scanned 1999 Z\\u00fcrich and don't and\"}, {\"role\": \"assistant\", \"content\": \"quick v1.2.3 100% node.js the na\\u00efve na\\u00efve involved once hand that \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors that request I'M tokens C++ don't na\\u00efve it mirrors there's with tokens https://example.com/a/b?c=d a there's so I'M mirrors $1,234.56 hand mirrors $1,234.56 admission C++ \\u0645\\u0631\\u062d\\u0628\\u0627 scanned user@example.com piece fox admission node.js https://example.com/a/b?c=d caf\\u00e9 It's \\ud83d\\ude42 scanned before https://example.com/a/b?c=d (parens) way https://example.com/a/b?c=d because F# I'M is the It's is reservation exactly user@example.com It's https://example.com/a/b?c=d lazy while the \\ud83d\\ude42 42 because for tiktoken's\"}, {\"role\": \"user\", \"content\": \"fox over {braces} is because caf\\u00e9 hand {braces} backtracking scanner F# engine engine user@example.com It's 42 quick jumps C++ \\\"quotes\\\" Z\\u00fcrich involved C++ na\\u00efve don't we'll admission exactly I'M a {braces} regex $1,234.56 exactly for [brackets] piece F# tokens \\\"quotes\\\" na\\u00efve dog lazy na\\u00efve there's on that with request \\u0645\\u0631\\u062d\\u0628\\u0627 quick a the gateway \\\"quotes\\\" and user@example.com 1999 (parens) over dog involved on engine over so hand over gateway Z\\u00fcrich counting keep that and $1,234.56 tokens scanned [brackets] reservation Z\\u00fcrich it node.js\"}, {\"role\": \"assistant\", \"content\": \"while on because fox \\u6771\\u4eac mirrors quick lazy I'M F# while a the because {braces} \\\"quotes\\\" once mirrors 100% a that on before body over request tokens we'll jumps that for 1999 mirrors hand lazy we'll over dog \\\"quotes\\\" because over way scanned that brown gateway over brown Z\\u00fcrich don't \\ud83d\\ude42 involved mirrors that once v1.2.3 \\ud83d\\ude42 v1.2.3 counting (parens) quick 'single' $1,234.56 scanner https://example.com/a/b?c=d no It's boundaries tokens hand caf\\u00e9 request body scanned 42 a counting with engine there's v1.2.3 fox gateway hand so counting scanned Z\\u00fcrich written https://example.com/a/b?c=d that {braces} v1.2.3 involved there's hand it there's is the exactly \\u0645\\u0631\\u062d\\u0628\\u0627 that scanner\"}, {\"role\": \"user\", \"content\": \"\\u0645\\u0631\\u062d\\u0628\\u0627 piece way scanned we'll hand written jumps fox https://example.com/a/b?c=d lazy $1,234.56 Z\\u00fcrich na\\u00efve WON'T scanner brown \\\"quotes\\\" mirrors It's caf\\u00e9 I'M \\\"quotes\\\" while admission fox engine brown tokens hand over while no scanner 3.14159 reservation budget \\\"quotes\\\" 100% hand while the It's faster tokens hand admission no and F# because \\u6771\\u4eac It's 'single' body that tiktoken's reservation tiktoken's is They'RE every (parens) Z\\u00fcrich scanner request the that F# caf\\u00e9 it They'RE way because\"}, {\"role\": \"assistant\", \"content\": \"brown \\\"quotes\\\" and don't regex [brackets] before on a that is every don't mirrors (parens) over lazy with F# on mirrors boundaries body jumps dog the It's \\u0645\\u0631\\u062d\\u0628\\u0627 so way before jumps WON'T request lazy v1.2.3 caf\\u00e9 it https://example.com/a/b?c=d $1,234.56 tiktoken's and {braces} admission \\u0645\\u0631\\u062d\\u0628\\u0627 fox on jumps jumps faster node.js user@example.com user@example.com so https://example.com/a/b?c=d counting It's \\\"quotes\\\" admission scanner on that while I'M piece written node.js way dog dog 100% before with \\u6771\\u4eac node.js hand mirrors involved 1999 a mirrors\"}, {\"role\": \"user\", \"content\": \"hand tiktoken's [brackets] 42 before 'single' F# F# for 'single' with body way every gateway {braces} brown keep scanner I'M the tiktoken's https://example.com/a/b?c=d for dog 100% tokens body counting over v1.2.3 https://example.com/a/b?c=d no 1999 They'RE {braces} tokens exactly scanner a for \\u0645\\u0631\\u062d\\u0628\\u0627 node.js way $1,234.56 gateway backtracking the brown a tiktoken's dog na\\u00efve I'M tokens for node.js boundaries a It's it C++ that tokens way C++\"}, {\"role\": \"assistant\", \"content\": \"a a counting don't WON'T no tokens quick we'll engine that written \\u0645\\u0631\\u062d\\u0628\\u0627 so 'single' every I'M Z\\u00fcrich there's v1.2.3 lazy there's caf\\u00e9 that scanned keep is fox 1999 100% tokens (parens) jumps 1999 v1.2.3 3.14159 WON'T {braces} for dog [brackets] the involved brown fox [brackets] (parens) They'RE (parens) gateway 1999 before a is scanned keep piece don't I'M exactly na\\u00efve we'll don't that 42 every gateway (parens) the na\\u00efve with Z\\u00fcrich backtracking lazy 100% is node.js faster while quick 42 engine tokens with \\u0645\\u0631\\u062d\\u0628\\u0627 C++ scanner\"}, {\"role\": \"user\", \"content\": \"that hand every exactly \\u6771\\u4eac $1,234.56 tokens {braces} body we'll engine quick once \\u6771\\u4eac on F# v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 I'M over over scanner tiktoken's tokens na\\u00efve that piece tiktoken's once and scanned scanned request with it jumps counting {braces} scanner 100% WON'T scanned exactly is piece 'single' request \\u6771\\u4eac regex for quick written 'single' every F# we'll way na\\u00efve and we'll tokens v1.2.3 regex piece v1.2.3 scanned is with counting with dog gateway admission 100% lazy because that \\ud83d\\ude42 3.14159 F# way hand exactly $1,234.56 while user@example.com It's jumps faster reservation It's don't it mirrors scanned $1,234.56 it while keep $1,234.56 fox $1,234.56 backtracking involved admission 3.14159 mirrors while with on over mirrors C++ for there's fox budget a keep\"}, {\"role\": \"assistant\", \"content\": \"100% scanner user@example.com backtracking every na\\u00efve quick hand jumps body tokens for v1.2.3 exactly \\u0645\\u0631\\u062d\\u0628\\u0627 that \\\"quotes\\\" \\ud83d\\ude42 gateway no caf\\u00e9 that written https://example.com/a/b?c=d the engine piece They'RE budget https://example.com/a/b?c=d involved the https://example.com/a/b?c=d the for involved once exactly every written exactly while 3.14159 user@example.com backtracking request fox is engine way budget counting 1999 piece so 3.14159 tokens F# involved brown a \\u6771\\u4eac scanned keep v1.2.3 for tiktoken's 1999 They'RE and on backtracking counting boundaries na\\u00efve regex\"}, {\"role\": \"user\", \"content\": \"fox is regex https://example.com/a/b?c=d is node.js mirrors user@example.com 100% involved request piece with \\u6771\\u4eac admission the on quick way \\u6771\\u4eac backtracking keep tiktoken's because F# lazy 100% WON'T tiktoken's Z\\u00fcrich involved quick reservation [brackets] written don't {braces} na\\u00efve caf\\u00e9 a $1,234.56 {braces} regex hand 100% 100% so reservation is there's engine \\u0645\\u0631\\u062d\\u0628\\u0627 is 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 backtracking v1.2.3 we'll so quick is before\"}, {\"role\": \"assistant\", \"content\": \"[brackets] request lazy C++ 100% scanned F# a because budget They'RE 1999 that is so \\u0645\\u0631\\u062d\\u0628\\u0627 I'M hand involved 3.14159 admission node.js faster once admission tiktoken's jumps is hand quick tokens mirrors lazy It's 42 fox gateway because request once $1,234.56 mirrors there's so backtracking a over engine we'll \\ud83d\\ude42 body engine fox {braces} {braces} counting hand we'll (parens) that 'single' \\\"quotes\\\" lazy budget na\\u00efve lazy WON'T body v1.2.3 the (parens) \"}, {\"role\": \"user\", \"content\": \"for before quick v1.2.3 scanner tokens faster is hand gateway lazy backtracking C++ budget \\u6771\\u4eac brown (parens) They'RE \\u6771\\u4eac It's \\ud83d\\ude42 3.14159 tokens na\\u00efve backtracking (parens) don't WON'T request WON'T that \\u6771\\u4eac piece v1.2.3 mirrors engine the over mirrors na\\u00efve gateway Z\\u00fcrich C++ written 3.14159 dog involved written \\\"quotes\\\" keep \\\"quotes\\\" They'RE fox written 'single' for backtracking gateway written 1999 the a for F# we'll F# brown faster once engine https://example.com/a/b?c=d it quick piece {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 mirrors [brackets] boundaries $1,234.56 {braces} request keep piece involved body quick once for for They'RE that involved scanned on for 100% mirrors involved budget 3.14159 They'RE faster with \\u6771\\u4eac backtracking reservation the 1999 for \\u0645\\u0631\\u062d\\u0628\\u0627 \\u0645\\u0631\\u062d\\u0628\\u0627 42 \\ud83d\\ude42 keep no caf\\u00e9 engine\"}, {\"role\": \"assistant\", \"content\": \"that tokens dog exactly piece Z\\u00fcrich quick the request 100% every it user@example.com that https://example.com/a/b?c=d way the reservation 1999 F# while a that They'RE backtracking It's Z\\u00fcrich admission budget C++ They'RE we'll keep counting \\ud83d\\ude42 \\\"quotes\\\" that \\ud83d\\ude42 lazy the engine They'RE \\u6771\\u4eac quick involved while no jumps the https://example.com/a/b?c=d is that 3.14159 100% 42 the no while tiktoken's over while written I'M backtracking node.js tokens F# $1,234.56 caf\\u00e9 it while keep (parens) boundaries admission so over no that fox involved F# brown na\\u00efve $1,234.56 reservation lazy body mirrors every $1,234.56 v1.2.3 that scanned user@example.com before quick body with [brackets] brown tiktoken's v1.2.3 fox budget \\\"quotes\\\" 42 every tokens budget don't counting counting no tiktoken's request C++ It's I'M and \\\"quotes\\\" tiktoken's every (parens) user@example.com the the we'll no I'M boundaries F# over on over tokens\"}, {\"role\": \"user\", \"content\": \"tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 It's counting piece $1,234.56 42 once counting there's faster [brackets] 100% piece counting quick \\ud83d\\ude42 caf\\u00e9 exactly written node.js is way on lazy counting it 42 scanned lazy It's v1.2.3 dog dog and because tiktoken's lazy \\ud83d\\ude42 don't involved and once 3.14159 1999 reservation boundaries caf\\u00e9 written written reservation we'll regex {braces} it the https://example.com/a/b?c=d lazy a once body na\\u00efve 1999 tiktoken's caf\\u00e9 caf\\u00e9 for body WON'T \\ud83d\\ude42 no because regex lazy 3.14159 with dog request scanned and involved involved scanned keep that fox https://example.com/a/b?c=d on we'll we'll budget boundaries exactly exactly admission lazy backtracking that F# while fox 1999 boundaries request \\u6771\\u4eac mirrors brown brown 100% Z\\u00fcrich {braces} They'RE 3.14159 v1.2.3 there's v1.2.3 once 1999 faster and WON'T there's we'll on written C++ v1.2.3\"}, {\"role\": \"assistant\", \"content\": \"user@example.com \\u6771\\u4eac because before so keep written for WON'T the They'RE is hand \\ud83d\\ude42 tiktoken's on 42 F# we'll written body user@example.com \\u6771\\u4eac 1999 admission with reservation 42 and that before 3.14159 piece request exactly 1999 for node.js gateway 100% no hand hand it and admission body scanner https://example.com/a/b?c=d involved user@example.com dog boundaries on while 42 body written don't once keep that node.js on reservation \\u0645\\u0631\\u062d\\u0628\\u0627 lazy \\\"quotes\\\" They'RE that with regex while Z\\u00fcrich na\\u00efve that lazy caf\\u00e9 don't jumps before {braces} don't for https://example.com/a/b?c=d keep user@example.com once lazy because counting brown written that 'single' F# \\u6771\\u4eac regex tiktoken's admission I'M a that It's exactly while with so hand don't we'll on\"}, {\"role\": \"user\", \"content\": \"{braces} (parens) fox on over tiktoken's budget lazy for \\\"quotes\\\" 100% so because 100% boundaries Z\\u00fcrich gateway boundaries don't jumps faster don't F# the \\\"quotes\\\" {braces} tokens backtracking They'RE regex every tokens the \\\"quotes\\\" once user@example.com {braces} every \\\"quotes\\\" counting budget It's gateway there's exactly regex is node.js request for F# piece so don't quick 'single' is user@example.com once I'M because user@example.com before user@example.com it v1.2.3 piece a lazy scanned 100% lazy counting dog and we'll we'll for tiktoken's exactly no fox once 1999 \\u6771\\u4eac we'll every the because regex $1,234.56 100% scanner so \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 v1.2.3 on while that na\\u00efve body brown way 3.14159 scanner scanner on They'RE don't over because the 42 involved node.js that a They'RE caf\\u00e9 for \\ud83d\\ude42 user@example.com user@example.com I'M so faster\"}, {\"role\": \"assistant\", \"content\": \"a don't It's gateway fox for \\u6771\\u4eac exactly that exactly I'M piece https://example.com/a/b?c=d \\ud83d\\ude42 faster so once jumps reservation piece boundaries v1.2.3 scanned a while that user@example.com [brackets] keep no there's brown the 1999 backtracking They'RE brown node.js v1.2.3 tiktoken's with is v1.2.3 that hand so I'M and engine boundaries C++ user@example.com tokens keep na\\u00efve {braces} gateway because backtracking [brackets] gateway don't we'll for we'll 100% boundaries counting 3.14159 is the F# it fox node.js for is tiktoken's no backtracking engine and 'single' and involved before fox tokens It's https://example.com/a/b?c=d mirrors written boundaries (parens) reservation the tiktoken's (parens) fox hand body (parens) (parens) it is It's and way because {braces} dog tiktoken's the counting while \\ud83d\\ude42 42 scanned over the\"}, {\"role\": \"user\", \"content\": \"it brown brown tokens backtracking jumps over boundaries with quick counting faster gateway {braces} \\u6771\\u4eac 42 mirrors \\ud83d\\ude42 engine hand \\u6771\\u4eac reservation v1.2.3 quick no no that involved over the scanner faster every mirrors for admission I'M we'll and scanned node.js tokens brown budget 42 written gateway lazy 3.14159 engine it we'll it 3.14159 $1,234.56 the that it (parens) over \\u0645\\u0631\\u062d\\u0628\\u0627 node.js for that {braces} \\\"quotes\\\" way before that no quick tokens and scanner the WON'T boundaries dog C++ boundaries while F# once backtracking 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 a while counting node.js Z\\u00fcrich engine fox faster it F# 1999 gateway written It's 'single' na\\u00efve there's 3.14159 na\\u00efve Z\\u00fcrich brown 3.14159 \\u6771\\u4eac user@example.com WON'T exactly Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 node.js 100% gateway regex while user@example.com and no quick lazy with\"}, {\"role\": \"assistant\", \"content\": \"1999 100% dog so for mirrors tiktoken's backtracking on that fox I'M every for involved involved \\\"quotes\\\" 100% so while 42 and it for F# node.js that gateway backtracking $1,234.56 fox boundaries on budget backtracking mirrors body Z\\u00fcrich for on 42 hand 1999 counting way body brown with scanned 42 [brackets] a jumps I'M written there's no They'RE faster backtracking I'M the don't once admission scanner 'single' \\ud83d\\ude42 way on\"}, {\"role\": \"user\", \"content\": \"every regex \\ud83d\\ude42 3.14159 we'll fox it tiktoken's Z\\u00fcrich \\\"quotes\\\" user@example.com na\\u00efve quick \\u0645\\u0631\\u062d\\u0628\\u0627 fox fox it C++ fox \\u0645\\u0631\\u062d\\u0628\\u0627 body gateway 3.14159 100% fox there's keep They'RE so reservation it lazy with lazy gateway request keep scanned hand every fox budget for quick $1,234.56 piece piece faster hand we'll 42 user@example.com written 1999 \\u6771\\u4eac every na\\u00efve request It's involved caf\\u00e9 [brackets] faster scanned keep 42 It's \\u6771\\u4eac every way is 3.14159 written mirrors brown WON'T once WON'T exactly the They'RE there's scanner because [brackets] gateway v1.2.3 fox tiktoken's 'single' Z\\u00fcrich {braces} jumps caf\\u00e9 I'M https://example.com/a/b?c=d C++ boundaries \\\"quotes\\\" with written we'll quick a with the every way involved scanned so scanner request scanner $1,234.56 exactly the caf\\u00e9 so there's is lazy that jumps user@example.com so with don't faster 1999 request admission They'RE while boundaries reservation\"}, {\"role\": \"assistant\", \"content\": \"jumps no that dog \\u0645\\u0631\\u062d\\u0628\\u0627 the lazy request user@example.com dog with lazy Z\\u00fcrich and tokens \\u6771\\u4eac boundaries engine keep boundaries 'single' so C++ \\ud83d\\ude42 gateway \\ud83d\\ude42 mirrors F# on {braces} once the before body user@example.com boundaries na\\u00efve and keep counting \\\"quotes\\\" while hand request body tiktoken's request we'll so I'M \\\"quotes\\\" on {braces} https://example.com/a/b?c=d They'RE is regex don't dog They'RE (parens) once caf\\u00e9 involved written faster keep before scanner while They'RE boundaries admission once gateway reservation every caf\\u00e9 dog tokens regex 'single' 3.14159 there's counting that tokens dog https://example.com/a/b?c=d piece F# before engine scanner exactly \\\"quotes\\\" don't while\"}, {\"role\": \"user\", \"content\": \"WON'T node.js v1.2.3 involved mirrors on brown keep every tiktoken's F# 100% 'single' They'RE \\u0645\\u0631\\u062d\\u0628\\u0627 na\\u00efve don't 3.14159 \\\"quotes\\\" there's once I'M $1,234.56 dog tokens that the Z\\u00fcrich \\\"quotes\\\" exactly C++ 3.14159 (parens) v1.2.3 no It's no keep user@example.com 42 Z\\u00fcrich a over written tiktoken's keep tiktoken's written with no WON'T $1,234.56 scanner keep keep that fox regex admission every [brackets] WON'T \\u6771\\u4eac 42 engine (parens) C++ we'll tokens a it while [brackets] counting involved the \\u0645\\u0631\\u062d\\u0628\\u0627 lazy 'single' WON'T 3.14159 1999 $1,234.56 regex that 3.14159 because caf\\u00e9 mirrors \\\"quotes\\\" counting 3.14159 with exactly admission every They'RE because 100% body piece na\\u00efve regex counting I'M that admission lazy \\ud83d\\ude42 scanner quick fox we'll brown admission C++\"}, {\"role\": \"assistant\", \"content\": \"\\ud83d\\ude42 the that \\u6771\\u4eac exactly while engine counting \\ud83d\\ude42 42 admission admission once caf\\u00e9 for tokens v1.2.3 jumps piece piece scanner caf\\u00e9 is \\\"quotes\\\" boundaries so jumps hand so while exactly involved written lazy (parens) quick F# with 1999 C++ jumps Z\\u00fcrich faster because quick the scanned C++ because because F# for no there's keep caf\\u00e9 jumps while is tokens quick lazy C++ boundaries 'single' exactly once tiktoken's node.js Z\\u00fcrich https://example.com/a/b?c=d before exactly {braces} boundaries written no caf\\u00e9 100% dog regex WON'T counting dog jumps F# for tokens no budget scanner that {braces} user@example.com C++ scanned \\u6771\\u4eac \\u0645\\u0631\\u062d\\u0628\\u0627 for every way and 'single' mirrors\"}, {\"role\": \"user\", \"content\": \"gateway because tokens jumps \\u6771\\u4eac exactly 3.14159 I'M 'single' It's WON'T body 42 and way is tiktoken's dog that admission on (parens) \\u6771\\u4eac C++ node.js over boundaries piece piece backtracking node.js dog hand on there's C++ 'single' so mirrors (parens) that (parens) the 1999 scanner hand node.js keep (parens) {braces} every [brackets] user@example.com that They'RE over tokens is with every so is quick request piece so reservation 42 brown on \\u6771\\u4eac fox They'RE $1,234.56 They'RE na\\u00efve boundaries jumps 42 quick exactly \\u0645\\u0631\\u062d\\u0628\\u0627 the 3.14159 over na\\u00efve is reservation mirrors it tokens no \\u6771\\u4eac on \\u6771\\u4eac mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 dog for engine before before node.js $1,234.56 100% every no 100% a budget backtracking jumps there's faster there's boundaries tokens every tiktoken's way F# $1,234.56 that every exactly piece\"}, {\"role\": \"assistant\", \"content\": \"over 3.14159 body 100% admission is F# tiktoken's user@example.com once with \\u0645\\u0631\\u062d\\u0628\\u0627 no with quick \\\"quotes\\\" the and $1,234.56 with faster exactly is involved keep exactly before 'single' there's body regex for involved counting https://example.com/a/b?c=d every over user@example.com C++ there's there's They'RE I'M scanned https://example.com/a/b?c=d scanned hand \\ud83d\\ude42 every counting don't fox 'single' WON'T na\\u00efve 100% gateway 'single' I'M no once involved node.js 1999 every admission caf\\u00e9 regex with regex written mirrors $1,234.56 tokens while backtracking we'll on it dog na\\u00efve that I'M every [brackets] a tokens I'M engine \\u0645\\u0631\\u062d\\u0628\\u0627 F# body don't request admission mirrors WON'T keep Z\\u00fcrich engine $1,234.56 request $1,234.56\"}, {\"role\": \"user\", \"content\": \"It's way body mirrors involved written 100% WON'T [brackets] no boundaries piece brown and while over before is there's we'll jumps scanned so \\ud83d\\ude42 backtracking keep don't dog and 42 that \\u0645\\u0631\\u062d\\u0628\\u0627 over and before and \\u6771\\u4eac on for (parens) https://example.com/a/b?c=d the request user@example.com I'M \\u0645\\u0631\\u062d\\u0628\\u0627 [brackets] 3.14159 so Z\\u00fcrich \\u0645\\u0631\\u062d\\u0628\\u0627 involved the hand once there's caf\\u00e9 jumps is body gateway v1.2.3 budget They'RE that it I'M 'single' gateway scanner that brown brown They'RE {braces} caf\\u00e9 hand admission budget scanner $1,234.56 mirrors hand 1999 scanned 3.14159 a It's\"}, {\"role\": \"assistant\", \"content\": \"regex Z\\u00fcrich https://example.com/a/b?c=d lazy engine keep regex admission with before body \\\"quotes\\\" boundaries once tiktoken's gateway 'single' and the way I'M $1,234.56 backtracking the engine dog Z\\u00fcrich 3.14159 Z\\u00fcrich user@example.com fox that keep while over once {braces} \\ud83d\\ude42 admission tokens regex tokens budget no I'M that on caf\\u00e9 faster I'M dog 'single' dog we'll with quick no because tiktoken's that counting tiktoken's we'll involved node.js involved with C++ body (parens) dog over before v1.2.3 100% na\\u00efve while request F# every jumps gateway over a that for it body exactly Z\\u00fcrich \\u6771\\u4eac a mirrors caf\\u00e9 I'M that Z\\u00fcrich brown that https://example.com/a/b?c=d no 'single' boundaries \\u0645\\u0631\\u062d\\u0628\\u0627 we'll They'RE exactly request once lazy It's 3.14159 before \\\"quotes\\\" \\u6771\\u4eac scanned scanned It's jumps C++ They'RE C++ and\"}, {\"role\": \"user\", \"content\": \"the fox They'RE budget faster with brown 3.14159 because backtracking \\ud83d\\ude42 that because reservation the so {braces} is WON'T request 3.14159 100% over a \\u0645\\u0631\\u062d\\u0628\\u0627 brown while admission because there's and once tokens [brackets] is no with a over there's over written scanned don't exactly I'M gateway is boundaries it tokens faster exactly tokens 42 lazy node.js https://example.com/a/b?c=d is engine with\"}, {\"role\": \"assistant\", \"content\": \"3.14159 I'M gateway scanner tokens C++ node.js 1999 engine for jumps user@example.com It's request (parens) so [brackets] \\ud83d\\ude42 that it admission 3.14159 budget WON'T caf\\u00e9 that involved budget v1.2.3 [brackets] node.js They'RE mirrors \\u0645\\u0631\\u062d\\u0628\\u0627 request with v1.2.3 WON'T we'll admission brown don't 1999 \\u6771\\u4eac \\u6771\\u4eac F# for the scanner is 'single' is it piece tokens body body Z\\u00fcrich tiktoken's we'll involved \\u6771\\u4eac on no tokens on {braces} It's mirrors I'M [brackets] \\u0645\\u0631\\u062d\\u0628\\u0627 lazy Z\\u00fcrich a while caf\\u00e9 involved request $1,234.56 tiktoken's once lazy\"}, {\"role\": \"user\", \"content\": \"3.14159 lazy piece fox on lazy v1.2.3 dog while piece 'single' involved written 100% jumps hand scanner v1.2.3 don't 'single' scanned with piece don't They'RE piece (parens) for with F# scanned don't no backtracking written 100% 100% lazy backtracking involved user@example.com scanner way we'll 42 'single' reservation 'single' lazy the written don't 'single' every and before reservation so it Z\\u00fcrich before a node.js before F# involved jumps counting involved that scanned regex so that user@example.com it with \\u0645\\u0631\\u062d\\u0628\\u0627 involved admission caf\\u00e9 the reservation [brackets] over mirrors that hand {braces} \\u0645\\u0631\\u062d\\u0628\\u0627 don't that we'll [brackets] hand faster I'M regex boundaries piece keep reservation WON'T hand backtracking counting piece $1,234.56 \\\"quotes\\\" Z\\u00fcrich tokens jumps it body before don't scanner {braces} body It's 42 caf\\u00e9 $1,234.56 scanner tokens\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 engine involved faster involved keep with $1,234.56 while piece F# Z\\u00fcrich 100% tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 is budget that tiktoken's admission scanned node.js 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 that on on body dog with regex admission 3.14159 piece na\\u00efve so 'single' mirrors boundaries boundaries keep on no while \\u0645\\u0631\\u062d\\u0628\\u0627 reservation the node.js F# 42 piece reservation It's tiktoken's reservation $1,234.56 hand tiktoken's that I'M it \\ud83d\\ude42 (parens) counting Z\\u00fcrich is [brackets] https://example.com/a/b?c=d reservation boundaries lazy (parens) that with over Z\\u00fcrich don't counting\"}, {\"role\": \"user\", \"content\": \"body dog backtracking that we'll don't \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com keep quick before scanned caf\\u00e9 It's body quick na\\u00efve 100% before quick jumps \\\"quotes\\\" once regex 42 I'M \\ud83d\\ude42 and keep a faster brown caf\\u00e9 regex \\\"quotes\\\" we'll engine exactly so 42 written They'RE every caf\\u00e9 the the once na\\u00efve piece no hand that boundaries scanned before there's there's reservation https://example.com/a/b?c=d admission I'M v1.2.3 dog They'RE dog scanned engine It's user@example.com no reservation \\u0645\\u0631\\u062d\\u0628\\u0627 regex scanned Z\\u00fcrich is it a and \\\"quotes\\\" involved exactly exactly way hand jumps \\ud83d\\ude42 reservation quick tokens mirrors over user@example.com boundaries 3.14159 I'M node.js piece F# https://example.com/a/b?c=d before tiktoken's I'M piece brown is na\\u00efve and {braces} WON'T It's don't it is {braces} on so so \\u6771\\u4eac it it user@example.com brown \\\"quotes\\\" I'M 100% scanned C++ body tokens \\u6771\\u4eac [brackets]\"}, {\"role\": \"assistant\", \"content\": \"mirrors F# $1,234.56 for dog (parens) keep that [brackets] (parens) 3.14159 v1.2.3 WON'T is don't the a It's https://example.com/a/b?c=d faster 'single' it user@example.com is dog dog counting so $1,234.56 v1.2.3 boundaries every on regex backtracking before quick brown quick C++ quick v1.2.3 scanner written \\ud83d\\ude42 user@example.com before is 42 3.14159 on engine dog dog once request https://example.com/a/b?c=d lazy on caf\\u00e9 faster don't before is exactly written brown C++ piece a 3.14159 no WON'T faster it https://example.com/a/b?c=d and boundaries \\ud83d\\ude42 reservation that engine we'll fox while the dog v1.2.3 because boundaries scanned It's \\u6771\\u4eac \\\"quotes\\\" scanned that so is tiktoken's scanned {braces} gateway exactly regex quick over is fox It's we'll {braces} faster\"}, {\"role\": \"user\", \"content\": \"$1,234.56 scanned node.js budget with na\\u00efve \\\"quotes\\\" hand because admission before F# [brackets] C++ v1.2.3 backtracking with 'single' hand on the for backtracking (parens) lazy is 1999 faster is [brackets] over v1.2.3 don't there's C++ na\\u00efve quick scanned lazy engine I'M scanned gateway They'RE gateway before there's scanned is v1.2.3 reservation once gateway (parens) It's so 'single' regex it It's brown counting 3.14159 lazy tokens engine backtracking with \\\"quotes\\\" there's [brackets] caf\\u00e9 reservation for scanned there's lazy mirrors WON'T (parens) so user@example.com It's It's {braces} {braces} 3.14159 42 that budget boundaries once no faster F# it WON'T admission because it that involved and every way C++ on jumps user@example.com request for engine keep engine regex It's node.js\"}, {\"role\": \"assistant\", \"content\": \"v1.2.3 brown piece counting backtracking no for budget C++ boundaries I'M tiktoken's tiktoken's {braces} jumps dog \\u6771\\u4eac while brown it with {braces} that counting tokens na\\u00efve because 100% na\\u00efve that I'M $1,234.56 with backtracking with keep reservation brown F# with v1.2.3 exactly jumps that exactly {braces} 1999 tokens $1,234.56 backtracking tokens once exactly \\u6771\\u4eac It's jumps user@example.com boundaries 'single' backtracking scanner scanned node.js 42 mirrors no backtracking caf\\u00e9 3.14159 {braces} It's WON'T dog I'M way that\"}, {\"role\": \"user\", \"content\": \"dog mirrors budget Z\\u00fcrich don't while request is a 1999 that jumps 3.14159 body caf\\u00e9 don't mirrors \\ud83d\\ude42 because Z\\u00fcrich way na\\u00efve tokens dog \\u0645\\u0631\\u062d\\u0628\\u0627 with reservation 100% with there's reservation request with 1999 scanned is faster that keep $1,234.56 I'M budget request because is counting the that and 3.14159 user@example.com scanner request WON'T [brackets] lazy \\ud83d\\ude42 because {braces} with on scanned the there's Z\\u00fcrich mirrors while jumps 'single' \\u6771\\u4eac keep so https://example.com/a/b?c=d mirrors don't $1,234.56 no there's engine (parens) mirrors regex scanned quick \\u6771\\u4eac They'RE WON'T (parens) $1,234.56 node.js \\ud83d\\ude42 don't counting They'RE (parens) because WON'T over request hand It's before a body quick brown caf\\u00e9 caf\\u00e9 $1,234.56 there's brown involved I'M we'll 42 lazy (parens) \\u0645\\u0631\\u062d\\u0628\\u0627 the involved quick over engine caf\\u00e9 v1.2.3 [brackets] reservation a F# counting we'll involved tiktoken's brown\"}, {\"role\": \"assistant\", \"content\": \"I'M backtracking no faster exactly 'single' the before on boundaries because with \\u0645\\u0631\\u062d\\u0628\\u0627 because that the every the involved na\\u00efve It's a jumps it it fox written WON'T and we'll reservation piece tiktoken's fox $1,234.56 v1.2.3 counting hand tokens budget jumps counting v1.2.3 budget budget that and \\u0645\\u0631\\u062d\\u0628\\u0627 dog every 100% that dog brown \\ud83d\\ude42 admission tiktoken's I'M is user@example.com admission while gateway because v1.2.3 for we'll and {braces} request reservation fox exactly no tokens jumps body because before faster once {braces} tokens hand https://example.com/a/b?c=d once jumps 3.14159 budget so C++ exactly lazy once tiktoken's 100% because on 1999 1999 tokens every \\ud83d\\ude42 body\"}, {\"role\": \"user\", \"content\": \"v1.2.3 \\u0645\\u0631\\u062d\\u0628\\u0627 They'RE we'll way fox scanner jumps node.js the user@example.com C++ regex scanned lazy exactly while v1.2.3 hand scanner piece there's involved 42 \\u6771\\u4eac 3.14159 C++ fox the hand reservation before 3.14159 gateway body the jumps na\\u00efve with regex the \\\"quotes\\\" the \\\"quotes\\\" scanned so dog no lazy \\\"quotes\\\" It's scanned \\ud83d\\ude42 mirrors regex body fox faster because engine that dog tiktoken's gateway 42 tiktoken's on body keep \\\"quotes\\\" They'RE is and written $1,234.56 every engine request caf\\u00e9 hand Z\\u00fcrich v1.2.3 mirrors the way piece the 1999 (parens) 42 $1,234.56 engine regex https://example.com/a/b?c=d F# 42 body 100% is caf\\u00e9 Z\\u00fcrich user@example.com with [brackets] mirrors keep over exactly C++ reservation it budget Z\\u00fcrich caf\\u00e9 fox way budget caf\\u00e9 budget boundaries (parens) fox 100% boundaries mirrors\"}, {\"role\": \"assistant\", \"content\": \"[brackets] over because a every boundaries keep tiktoken's the hand 42 backtracking written jumps hand before there's a with don't body [brackets] https://example.com/a/b?c=d request request reservation and 42 admission scanned node.js engine piece 100% reservation I'M https://example.com/a/b?c=d admission no involved F# fox [brackets] regex so They'RE 1999 budget admission no I'M node.js exactly scanner so tiktoken's node.js written \\\"quotes\\\" every brown body regex na\\u00efve that backtracking for 1999 brown brown while v1.2.3 $1,234.56 written hand node.js every scanner Z\\u00fcrich way [brackets] no is It's {braces} \\u6771\\u4eac regex dog user@example.com request for $1,234.56 quick node.js $1,234.56 the engine on admission that written https://example.com/a/b?c=d boundaries a It's once body budget exactly every the before 'single' F# because {braces} once fox no because involved once quick that 42 no reservation there's there's\"}, {\"role\": \"user\", \"content\": \"They'RE 3.14159 42 scanned mirrors jumps reservation we'll caf\\u00e9 counting mirrors {braces} quick a fox written no scanner body https://example.com/a/b?c=d 3.14159 \\\"quotes\\\" scanner mirrors no on with no budget WON'T \\ud83d\\ude42 over brown piece jumps It's counting hand https://example.com/a/b?c=d lazy involved 'single' (parens) with scanned \\u6771\\u4eac engine no request {braces} jumps with {braces} admission for It's keep \\u6771\\u4eac don't gateway because admission don't there's https://example.com/a/b?c=d for v1.2.3 Z\\u00fcrich [brackets] piece because no the engine mirrors 3.14159 42 1999 no scanner the don't mirrors node.js\"}, {\"role\": \"assistant\", \"content\": \"before while [brackets] involved involved https://example.com/a/b?c=d It's hand on keep way we'll quick every exactly They'RE mirrors way before the reservation 1999 body 'single' and quick WON'T C++ {braces} over caf\\u00e9 and brown 42 counting hand user@example.com that so (parens) hand hand hand a hand engine every that counting tiktoken's admission WON'T counting dog on https://example.com/a/b?c=d tiktoken's $1,234.56 https://example.com/a/b?c=d scanner gateway while lazy C++ na\\u00efve 3.14159 don't {braces} faster over body because is gateway backtracking\"}, {\"role\": \"user\", \"content\": \"we'll that once tokens before lazy Z\\u00fcrich no Z\\u00fcrich the scanned \\\"quotes\\\" way is we'll jumps [brackets] involved \\ud83d\\ude42 v1.2.3 while WON'T written F# there's (parens) admission way I'M node.js \\ud83d\\ude42 written budget (parens) {braces} 1999 mirrors so engine \\u0645\\u0631\\u062d\\u0628\\u0627 because 'single' WON'T {braces} body reservation caf\\u00e9 so mirrors once over fox on regex quick dog exactly WON'T every a I'M regex \\u0645\\u0631\\u062d\\u0628\\u0627 don't na\\u00efve way backtracking admission over budget (parens) exactly 3.14159 we'll jumps there's v1.2.3 for counting reservation budget exactly counting while Z\\u00fcrich is $1,234.56 don't mirrors 3.14159 we'll user@example.com {braces} so so [brackets] tokens the scanned user@example.com na\\u00efve that involved don't the there's fox 42 there's budget before scanned It's boundaries tokens piece It's \\u0645\\u0631\\u062d\\u0628\\u0627 for counting written boundaries {braces} jumps $1,234.56 over lazy F# written on\"}, {\"role\": \"assistant\", \"content\": \"on F# mirrors hand \\ud83d\\ude42 exactly Z\\u00fcrich every admission with hand $1,234.56 scanned hand \\u0645\\u0631\\u062d\\u0628\\u0627 node.js that counting piece is there's don't fox node.js for so admission quick faster every that that jumps because tiktoken's over caf\\u00e9 F# once [brackets] F# lazy I'M \\\"quotes\\\" caf\\u00e9 over a no that before v1.2.3 piece {braces} because quick \\u6771\\u4eac once tokens while https://example.com/a/b?c=d na\\u00efve every lazy WON'T don't F#\"}, {\"role\": \"user\", \"content\": \"a a 'single' \\u6771\\u4eac (parens) counting F# with quick the (parens) tiktoken's \\u0645\\u0631\\u062d\\u0628\\u0627 I'M quick because counting backtracking https://example.com/a/b?c=d written 'single' \\u0645\\u0631\\u062d\\u0628\\u0627 caf\\u00e9 caf\\u00e9 dog counting It's and keep caf\\u00e9 tiktoken's budget budget piece piece over piece tiktoken's scanned before over request brown that involved quick reservation the that scanner tiktoken's hand 42 'single' keep exactly it 100% on engine engine budget (parens) the regex written node.js \\u0645\\u0631\\u062d\\u0628\\u0627 user@example.com (parens) faster \\ud83d\\ude42 jumps 100% WON'T https://example.com/a/b?c=d a reservation there's so Z\\u00fcrich https://example.com/a/b?c=d every with It's Z\\u00fcrich tiktoken's\"}, {\"role\": \"assistant\", \"content\": \"don't a a faster 42 that 'single' backtracking before that over engine it hand don't it jumps budget 'single' $1,234.56 piece there's the with \\ud83d\\ude42 42 boundaries It's no that reservation backtracking dog https://example.com/a/b?c=d tokens don't piece on body once the once jumps while \\u6771\\u4eac fox $1,234.56 \\u0645\\u0631\\u062d\\u0628\\u0627 3.14159 we'll every hand v1.2.3 They'RE that a user@example.com https://example.com/a/b?c=d for budget quick hand 100% They'RE caf\\u00e9 while budget is request way They'RE tiktoken's once piece with They'RE gateway scanner hand engine C++ piece a exactly [brackets] we'll before for 42 \\u6771\\u4eac and reservation dog request \\u6771\\u4eac once Z\\u00fcrich v1.2.3 3.14159 \\u6771\\u4eac faster lazy regex user@example.com gateway tokens involved fox They'RE before that (parens) and C++ mirrors node.js once user@example.com over before 'single' there's 100% that faster the don't written scanner admission dog user@example.com\"}, {\"role\": \"user\", \"content\": \"na\\u00efve budget keep request C++ \\ud83d\\ude42 \\\"quotes\\\" involved that Z\\u00fcrich that https://example.com/a/b?c=d with before we'll user@example.com backtracking before while 'single' budget 100% fox They'RE keep {braces} mirrors quick Z\\u00fcrich before caf\\u00e9 way jumps na\\u00efve that hand [brackets] for and It's na\\u00efve Z\\u00fcrich it faster 'single' boundaries with scanned fox faster I'M engine na\\u00efve written node.js na\\u00efve a It's 42 $1,234.56 backtracking body scanner 1999 3.14159 quick scanned scanned tokens 'single' (parens) every 42 quick engine that tokens because admission node.js request backtracking https://example.com/a/b?c=d They'RE counting for we'll body is piece C++ \\ud83d\\ude42 we'll na\\u00efve 3.14159 3.14159 the \\\"quotes\\\" v1.2.3 that the [brackets] $1,234.56 scanner \\ud83d\\ude42 user@example.com v1.2.3 'single' v1.2.3 lazy don't request \\u6771\\u4eac scanned dog They'RE admission [brackets] there's piece It's involved so F#\"}, {\"role\": \"assistant\", \"content\": \"body once mirrors dog there's because that is quick reservation fox because with C++ every tokens node.js 100% 3.14159 no WON'T 1999 counting a no 100% every don't the boundaries engine 42 WON'T budget lazy 42 caf\\u00e9 budget scanner keep hand tiktoken's WON'T because the body regex because there's node.js written involved lazy counting exactly so with once 'single' dog because it before tokens once we'll 42 keep WON'T dog quick na\\u00efve it keep 42 quick\"}, {\"role\": \"user\", \"content\": \"node.js the backtracking scanner while \\u0645\\u0631\\u062d\\u0628\\u0627 while we'll Z\\u00fcrich It's C++ piece \\u0645\\u0631\\u062d\\u0628\\u0627 na\\u00efve counting over {braces} over admission it because and there's a mirrors tokens while keep body exactly WON'T written F# we'll I'M don't for request before with mirrors scanner boundaries (parens) counting that hand so on piece (parens) that before gateway boundaries request 'single' boundaries mirrors written a exactly C++ quick engine involved [brackets] for jumps gateway tiktoken's written piece so 100% engine v1.2.3 3.14159 It's 'single' \\u6771\\u4eac 1999 na\\u00efve is the\"}, {\"role\": \"assistant\", \"content\": \"so mirrors over the lazy and 3.14159 for brown dog engine \\\"quotes\\\" dog engine C++ It's F# user@example.com reservation piece that for caf\\u00e9 body gateway involved is body involved dog tokens involved reservation user@example.com (parens) [brackets] v1.2.3 caf\\u00e9 https://example.com/a/b?c=d quick written hand because lazy 42 42 3.14159 gateway keep I'M lazy counting It's caf\\u00e9 regex a scanned there's there's\"}, {\"role\": \"user\", \"content\": \"node.js lazy faster gateway budget we'll once \\u0645\\u0631\\u062d\\u0628\\u0627 admission tiktoken's that na\\u00efve boundaries budget user@example.com that Z\\u00fcrich gateway user@example.com way it 3.14159 {braces} for before that They'RE brown so for written counting body that 'single' na\\u00efve lazy there's body tokens it engine 3.14159 so mirrors written 3.14159 dog on counting request They'RE we'll admission na\\u00efve reservation jumps $1,234.56 a for every tiktoken's before caf\\u00e9\"}, {\"role\": \"assistant\", \"content\": \"piece the quick 1999 reservation way na\\u00efve F# every v1.2.3 (parens) Z\\u00fcrich counting once na\\u00efve Z\\u00fcrich admission \\u6771\\u4eac exactly [brackets] there's we'll budget we'll 1999 budget lazy \\\"quotes\\\" {braces} counting tokens boundaries keep tiktoken's dog piece mirrors keep exactly over caf\\u00e9 admission C++ quick F# lazy \\ud83d\\ude42 before keep jumps It's the involved so admission the C++ node.js tokens mirrors WON'T reservation once no\"}, {\"role\": \"user\", \"content\": \"Summarise the conversation so far in three sentences.\"}]}", "input_tokens": 50422} diff --git a/litellm-rust/crates/token-counter/tests/fixtures/o200k/texts.jsonl b/litellm-rust/crates/token-counter/tests/fixtures/o200k/texts.jsonl new file mode 100644 index 00000000000..9d78131a456 --- /dev/null +++ b/litellm-rust/crates/token-counter/tests/fixtures/o200k/texts.jsonl @@ -0,0 +1,4053 @@ +{"text": "", "tokens": 0, "pieces": []} +{"text": "Hello, how are you today?", "tokens": 7, "pieces": ["Hello", ",", " how", " are", " you", " today", "?"]} +{"text": "I'm sure they're right, we'll see. WE'LL SEE, I'M SURE THEY'RE RIGHT, IT'S HERS AND IT'D BE 'D", "tokens": 32, "pieces": ["I'm", " sure", " they're", " right", ",", " we'll", " see", ".", " WE'LL", " SEE", ",", " I'M", " SURE", " THEY'RE", " RIGHT", ",", " IT'S", " HERS", " AND", " IT'D", " BE", " '", "D"]} +{"text": "don't Don'T DON'T won'T i've I'VE i'Ve you'RE 'S 'T 'M 'D 'LL 'VE 'RE 'ſ 'x", "tokens": 33, "pieces": ["don't", " Don'T", " DON'T", " won'T", " i've", " I'VE", " i'Ve", " you'RE", " '", "S", " '", "T", " '", "M", " '", "D", " '", "LL", " '", "VE", " '", "RE", " '", "ſ", " '", "x"]} +{"text": "1234567890 123 12 1 0000000 ٣٤٥٦٧٨ ३४५६ 1,234,567.89 2026-09-11T18:00:00Z", "tokens": 48, "pieces": ["123", "456", "789", "0", " ", "123", " ", "12", " ", "1", " ", "000", "000", "0", " ", "٣٤٥", "٦٧٨", " ", "३४५", "६", " ", "1", ",", "234", ",", "567", ".", "89", " ", "202", "6", "-", "09", "-", "11", "T", "18", ":", "00", ":", "00", "Z"]} +{"text": "$abc %def &ghi @jkl _mno #pqr ~stu ^vwx |yz \\a /b :c ;d ?e !f (g )h [i ]j {k }l n =o +p *q", "tokens": 56, "pieces": ["$abc", " %", "def", " &", "ghi", " @", "jkl", " _", "mno", " #", "pqr", " ~", "stu", " ^", "vwx", " |", "yz", " \\", "a", " /", "b", " :", "c", " ;", "d", " ?", "e", " !", "f", " (", "g", " )", "h", " [", "i", " ]", "j", " {", "k", " }", "l", " <", "m", " >", "n", " =", "o", " +", "p", " *", "q"]} +{"text": "foo bar baz \t qux\t\tquux \n\nline\r\nline\r\n\r\n \n\t\r\n x ", "tokens": 22, "pieces": ["foo", " ", " bar", " ", " baz", " \t", " qux", "\t", "\tquux", " \n\n", "line", "\r\n", "line", "\r\n\r\n \n\t\r\n", " ", " x", " "]} +{"text": "trailing spaces ", "tokens": 4, "pieces": ["trailing", " spaces", " "]} +{"text": "trailing tabs\t\t", "tokens": 4, "pieces": ["trailing", " tabs", "\t\t"]} +{"text": "trailing newline\n", "tokens": 4, "pieces": ["trailing", " newline", "\n"]} +{"text": "\n\n\n", "tokens": 1, "pieces": ["\n\n\n"]} +{"text": "\r\n\r\n\r\n", "tokens": 1, "pieces": ["\r\n\r\n\r\n"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "😀😃😄 👍🏽 🇺🇸 👨‍👩‍👧‍👦 ✈️ ❤️‍🔥 ٭ ※ ⌘ ⏎", "tokens": 38, "pieces": ["😀😃😄", " 👍🏽", " 🇺🇸", " 👨‍👩‍👧‍👦", " ✈️", " ❤️‍🔥", " ٭", " ※", " ⌘", " ⏎"]} +{"text": "漢字かな交じり文、東京都千代田区。日本語のテキストです。中文测试。한국어 텍스트", "tokens": 30, "pieces": ["漢字かな交じり文", "、東京都千代田区", "。日本語のテキストです", "。中文测试", "。한국어", " 텍스트"]} +{"text": "مرحبا بالعالم، هذا نص عربي مع أرقام ١٢٣٤٥٦٧ و علامات ترقيم!", "tokens": 24, "pieces": ["مرحبا", " بالعالم", "،", " هذا", " نص", " عربي", " مع", " أرقام", " ", "١٢٣", "٤٥٦", "٧", " و", " علامات", " ترقيم", "!"]} +{"text": "Zürich, façade, naïve, Ærøskøbing, Ελληνικά, Русский текст, עברית, हिन्दी, ไทย", "tokens": 29, "pieces": ["Zürich", ",", " façade", ",", " naïve", ",", " Ærøskøbing", ",", " Ελληνικά", ",", " Русский", " текст", ",", " עברית", ",", " हिन्दी", ",", " ไทย"]} +{"text": "é å ḍ̇ ́́ combining̈ markś!", "tokens": 16, "pieces": ["é", " å", " ḍ̇", " ́́", " combining̈", " markś", "!"]} +{"text": "ΣΊΣΥΦΟΣ Džungla İstanbul file flow Ａｂｃ ㍿ ㋿ ꟲ 𐞁", "tokens": 40, "pieces": ["ΣΊΣΥΦΟΣ", " Džungla", " İstanbul", " file", " flow", " Ａｂｃ", " ㍿", " ㋿", " ꟲ", " 𐞁"]} +{"text": "<|endoftext|> <|fim_prefix|>code<|fim_middle|>more<|fim_suffix|> <|endofprompt|> <|im_start|>", "tokens": 40, "pieces": ["<|", "endoftext", "|>", " <|", "fim", "_prefix", "|>", "code", "<|", "fim", "_middle", "|>", "more", "<|", "fim", "_suffix", "|>", " <|", "endofprompt", "|>", " <|", "im", "_start", "|>"]} +{"text": " [INST] [/INST] <>", "tokens": 22, "pieces": ["", " <", "META", "_START", ">", " <", "s", ">", " ", " [", "INST", "]", " [/", "INST", "]", " <<", "SYS", ">>"]} +{"text": "def f(x):\n return {'a': x ** 2, \"b\": [1, 2, 3]} # comment\n\nprint(f(10))\n", "tokens": 35, "pieces": ["def", " f", "(x", "):\n", " ", " return", " {'", "a", "':", " x", " **", " ", "2", ",", " \"", "b", "\":", " [", "1", ",", " ", "2", ",", " ", "3", "]}", " ", " #", " comment", "\n\n", "print", "(f", "(", "10", "))\n"]} +{"text": "{\"model\":\"gpt-4\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\\n\"}],\"temperature\":0.7}", "tokens": 27, "pieces": ["{\"", "model", "\":\"", "gpt", "-", "4", "\",\"", "messages", "\":[{\"", "role", "\":\"", "user", "\",\"", "content", "\":\"", "hi", "\\n", "\"}],\"", "temperature", "\":", "0", ".", "7", "}"]} +{"text": "https://example.com/path?query=1&other=two#fragment user@example.com 192.168.0.1", "tokens": 26, "pieces": ["https", "://", "example", ".com", "/path", "?query", "=", "1", "&other", "=two", "#fragment", " user", "@example", ".com", " ", "192", ".", "168", ".", "0", ".", "1"]} +{"text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "tokens": 375, "pieces": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]} +{"text": " ", "tokens": 24, "pieces": [" "]} +{"text": "........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................", "tokens": 48, "pieces": ["........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................"]} +{"text": "abababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab", "tokens": 750, "pieces": ["abababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab"]} +{"text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", "tokens": 188, "pieces": ["\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"]} +{"text": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "tokens": 1000, "pieces": ["000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000", "000"]} +{"text": "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", "tokens": 188, "pieces": ["!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"]} +{"text": "😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀", "tokens": 1000, "pieces": ["😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀😀"]} +{"text": "漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢", "tokens": 1000, "pieces": ["漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢漢"]} +{"text": " abc ! 
x 　y ​z ‍‍ q", "tokens": 15, "pieces": [" abc", " ", "!", " ", "
x", " ", "　y", " ​", "z", " ‍‍", " q"]} +{"text": "x…y \u000b\f z", "tokens": 8, "pieces": ["x", "…y", " \u000b\f", " z"]} +{"text": "\u0000\u0001\u0002  �", "tokens": 6, "pieces": ["\u0000\u0001\u0002", " ", " �"]} +{"text": "tab\tseparated\tvalues\n1\t2\t3\n", "tokens": 11, "pieces": ["tab", "\tseparated", "\tvalues", "\n", "1", "\t", "2", "\t", "3", "\n"]} +{"text": "MiXeD cAsE wOrDs AND ACRONYMS like NASA, HTTP/2, gRPC, iOS, macOS", "tokens": 29, "pieces": ["Mi", "Xe", "D", " c", "As", "E", " w", "Or", "Ds", " AND", " ACRONYMS", " like", " NASA", ",", " HTTP", "/", "2", ",", " g", "RPC", ",", " i", "OS", ",", " mac", "OS"]} +{"text": "snake_case_identifier camelCaseIdentifier PascalCaseIdentifier SCREAMING_SNAKE_CASE kebab-case", "tokens": 19, "pieces": ["snake", "_case", "_identifier", " camel", "Case", "Identifier", " Pascal", "Case", "Identifier", " SCREAMING", "_SNAKE", "_CASE", " kebab", "-case"]} +{"text": "x'sy x'ty x'rey x'vey x'my x'lly x'dy x'S x'T x'RE x'VE x'M x'LL x'D x'sS x'llL", "tokens": 44, "pieces": ["x's", "y", " x't", "y", " x're", "y", " x've", "y", " x'm", "y", " x'll", "y", " x'd", "y", " x'S", " x'T", " x'RE", " x'VE", " x'M", " x'LL", " x'D", " x's", "S", " x'll", "L"]} +{"text": "IT'SOK it'Dbe x'Sy x'Ty x'My x'Dy x'LLy x'VEy x'REy x'Ly x'Vy x'Ry 'Sx'Tx'Mx'LLx'VEx'REx'Dx", "tokens": 57, "pieces": ["IT'S", "OK", " it'D", "be", " x'S", "y", " x'T", "y", " x'M", "y", " x'D", "y", " x'LL", "y", " x'VE", "y", " x'RE", "y", " x", "'Ly", " x", "'Vy", " x", "'Ry", " '", "Sx'T", "x'M", "x'LL", "x'VE", "x'RE", "x'D", "x"]} +{"text": "'s't're've'm'll'd 'S'T'RE'VE'M'LL'D ''s '''s", "tokens": 22, "pieces": ["'s't", "'re've", "'m'll", "'d", " '", "S'T", "'RE'VE", "'M'LL", "'D", " ''", "s", " '''", "s"]} +{"text": "9'9 9's a'9 '9 ' 's' ' 's", "tokens": 18, "pieces": ["9", "'", "9", " ", "9", "'s", " a", "'", "9", " '", "9", " '", " '", "s", "'", " '", " '", "s"]} +{"text": "١٢٣٤ ½⅓¼ ⅣⅤ 𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡 ①②③", "tokens": 48, "pieces": ["١٢٣", "٤", " ", "½⅓¼", " ", "ⅣⅤ", " ", "𝟘𝟙𝟚", "𝟛𝟜𝟝", "𝟞𝟟𝟠", "𝟡", " ", "①②③"]} +{"text": "camelCase PascalCase ABCdef ABCdeF ABC aB Ab ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABC", "tokens": 21, "pieces": ["camel", "Case", " Pascal", "Case", " ABCdef", " ABCde", "F", " ABC", " a", "B", " Ab", " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "ABC"]} +{"text": "日本ABC ABC日本 日本語abc abc日本語 漢字Kanji kanji漢字 KANJI漢字kanji مرحباABC ABCمرحبا abcمرحبا", "tokens": 35, "pieces": ["日本", "ABC", " ABC日本", " 日本語abc", " abc日本語", " 漢字Kanji", " kanji漢字", " KANJI漢字kanji", " مرحبا", "ABC", " ABCمرحبا", " abcمرحبا"]} +{"text": "́ABC ́abc ́́A Á́ ÉA aÉ !!́a  ́A ẍY Ẍy", "tokens": 31, "pieces": ["́", "ABC", " ́abc", " ́́", "A", " Á́", " É", "A", " a", "É", " !!́", "a", " ", " ́", "A", " ẍ", "Y", " Ẍy"]} +{"text": "ᵃbc ᵃBC Aᵃbc Aᵃ ᵃ' ᵃ's Džungla aDžB ADžB ADžb DžDž Ljx İi ΣΊΣΥΦΟΣσ ΣσΣ", "tokens": 67, "pieces": ["ᵃbc", " ᵃ", "BC", " Aᵃbc", " Aᵃ", " ᵃ", "'", " ᵃ's", " Džungla", " a", "DžB", " ADžB", " ADžb", " DžDž", " Ljx", " İi", " ΣΊΣΥΦΟΣσ", " Σσ", "Σ"]} +{"text": "don'tx ABC's abc'S abc'ſ ABC'ſx IT'SOK it'Dbe 'sabc x's 's 'Sx'Tx x’s X'LLx X'Ll", "tokens": 40, "pieces": ["don't", "x", " ABC's", " abc'S", " abc'ſ", " ABC'ſ", "x", " IT'S", "OK", " it'D", "be", " '", "sabc", " x's", " '", "s", " '", "Sx'T", "x", " x", "’s", " X'LL", "x", " X'Ll"]} +{"text": "!ABC !AbC !!abc #camelCase (ABCdef)  ABC abc Abc \tABC\tabc", "tokens": 27, "pieces": ["!ABC", " !", "Ab", "C", " !!", "abc", " #", "camel", "Case", " (", "ABCdef", ")", " ", " ABC", " abc", " Abc", " ", "\tABC", "\tabc"]} +{"text": "!!/\n/x a/b !!\n/x /x // path/to/file.rs http://x.y/z?a=b/c \\/\\/ //\r\n//\n", "tokens": 29, "pieces": ["!!/\n/", "x", " a", "/b", " !!\n/", "x", " ", " /", "x", " ", " //", " path", "/to", "/file", ".rs", " http", "://", "x", ".y", "/z", "?a", "=b", "/c", " \\/\\/", " //\r\n//\n"]} +{"text": "x \n x \r\n \r\n y x \n a b \n\n c x\t\ty x\t\t end \n \n", "tokens": 23, "pieces": ["x", " \n", " x", " \r\n \r\n", " y", " x", " \n", " ", " a", " ", " b", " \n\n", " ", " c", " x", "\t", "\ty", " x", "\t\t", " end", " \n \n"]} +{"text": "12345 6 1abc abc1 ABC123abc 123ABC ١٢٣٤٥abc", "tokens": 22, "pieces": ["123", "45", " ", "6", " ", "1", "abc", " abc", "1", " ABC", "123", "abc", " ", "123", "ABC", " ", "١٢٣", "٤٥", "abc"]} +{"text": "Ⅳ٣٤٥٦<|endoftext|>9
Dž#$%", "tokens": 19, "pieces": ["Ⅳ٣٤", "٥٦", "<|", "endoftext", "|>", "9", "
Dž", "#$%"]} +{"text": "́!!ſİ'D​a'll 字0'MZſⅣ ḍ̇éfi㍿𐞁<|endoftext|>'reＡ'S#$%", "tokens": 40, "pieces": ["́", "!!", "ſ", "İ'D", "​a'll", " 字", "0", "'MZſ", "Ⅳ", " ḍ̇éfi", "㍿𐞁", "<|", "endoftext", "|>'", "re", "Ａ'S", "#$%"]} +{"text": "ع字'T\r\ń½sꟲ㋿'VE'S<😀🏽!!12345678 ٣٤٥٦Džſḍ̇\réEOT­'ſ<|endoftext|><|fim_prefix|>ś
\tm", "tokens": 65, "pieces": ["ع字'T", "\r\n", "́", "½", "sꟲ", "㋿'", "VE", "'", "S", "<😀🏽!!", "123", "456", "78", " <", "EOT", ">", "٣٤٥", "٦", "Džſḍ̇", "\r", "é", "EOT", "­'", "ſ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "ś", "
", "\tm"]} +{"text": "9'Re\r\n'ſ　 'T'Re \né#$%<½!!٣٤٥٦'ſ<\"عßZt\u000bDž'T<|fim_prefix|>ſ㋿\"éſ", "tokens": 51, "pieces": ["9", "'", "Re", "\r\n", "'ſ", "　 ", " '", "T'Re", " \n", "é", "#$%<", "½", "!!", "٣٤٥", "٦", "'ſ", "<<", "META", "_START", ">\"", "عß", "Zt", "\u000bDž'T", "<|", "fim", "_prefix", "|>", "ſ", "㋿\"", "éſ"]} +{"text": "<|endoftext|>12345678#$%tعⅣ'T0'D<|endoftext|>é'M-'ſ'sß12345678ꟲ0(>\r\n'MZ'Sa'M", "tokens": 55, "pieces": ["<|", "endoftext", "|>", "123", "456", "78", "#$%", "tع", "Ⅳ", "'T", "0", "'D", "<|", "endoftext", "|>", "é'M", "-<", "EOT", ">'", "ſ's", "ß", "123", "456", "78", "ꟲ", "0", "(>\r\n", "'MZ'S", "a'M"]} +{"text": " \n'll\"‍ d \nß㍿\u000baⅣ😀🏽́ſ\n \n
'Reİ\tDž٣٤٥٦ع'llEOT.\nݽ٣٤٥٦>å\u000b<|fim_prefix|>\"𐞁", "tokens": 55, "pieces": [" \n", "'ll", "\"‍", " ", " d", " \n", "ß", "㍿", "\u000ba", "Ⅳ", "😀🏽́", "ſ", "\n \n", "
", "'Re", "İ", "\tDž", "٣٤٥", "٦", "ع'll", "EOT", ".\n", "İ", "½٣٤", "٥٦", ">å", "\u000b", "<|", "fim", "_prefix", "|>\"", "𐞁"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "å'remfi㍿s", "tokens": 9, "pieces": ["å're", "mfi", "㍿s"]} +{"text": "<|endoftext|>", "tokens": 7, "pieces": ["<|", "endoftext", "|>"]} +{"text": "🙂३'M🙂 12345678'VÉ,\u000bİ<|fim_prefix|> Ａ'TDž　's!!", " ", "Ａ'T", "Dž", "　", "'s", "!!<", "t", "\"!", "d", " \n", "m'M", "('", "ſ"]} +{"text": "\t", "tokens": 1, "pieces": ["\t"]} +{"text": "…<|fim_prefix|>\n.-åß\r\n\r\nEOT'llå½-fi!é'VE12345678EOT字'VE!🙂<|fim_prefix|>'ſ'D \r\r漢0…", "tokens": 52, "pieces": ["…", "<|", "fim", "_prefix", "|>\n", ".-", "åß", "\r\n\r\n", "EOT'll", "å", "½", "-fi", "!é'VE", "123", "456", "78", "EOT字'VE", "!🙂<|", "fim", "_prefix", "|>'", "ſ'D", " \r\r", "漢", "0", "…"]} +{"text": "a'S'T👍🏽> \n-s㍿.㍿#$%́\r\n\r\n", "tokens": 20, "pieces": ["a'S", "'T", "👍🏽>", " \n", "-s", "㍿.㍿#$%́\r\n\r\n"]} +{"text": " 漢#$%­…­ع­ß#$%\t0é", "tokens": 17, "pieces": [" 漢", "#$%­", "…", "­ع", "­ß", "#$%", "\t", "0", "é"]} +{"text": ",9'㍿-", "tokens": 10, "pieces": [",", "9", "'㍿-"]} +{"text": "٣٤٥٦Ⅳ漢İ'sع", "tokens": 10, "pieces": ["٣٤٥", "٦Ⅳ", "漢", "İ's", "ع"]} +{"text": ".éé'sZ>éEOTZ'0​e½ 0#$%́fiⅣ", "tokens": 24, "pieces": [".éé's", "Z", ">é", "EOTZ", "'", "0", "​e", "½", " ", " ", "0", "#$%́", "fi", "Ⅳ"]} +{"text": "t<Ⅳ…Dž'VE ꟲ٣٤٥٦éåع㍿'s字👍🏽ع ३EOTⅣ😀🏽e \nＡ\"", "tokens": 51, "pieces": ["t", "<", "Ⅳ", "…Dž'VE", "", " ", " ꟲ", "٣٤٥", "٦", "éåع", "㍿'<", "META", "_START", ">s字", "👍🏽", "ع", " ", " ", "३", "EOT", "Ⅳ", "😀🏽", "e", " \n", "Ａ", "\""]} +{"text": "' . t\t<|fim_prefix|>㍿Dž३s😀🏽\t㍿EOTå𐞁​EOT
\t- \ns \n#$%ḍ̇é\r\n　ee漢", "tokens": 55, "pieces": ["'", " .", " ", " t", "\t", "<|", "fim", "_prefix", "|>㍿", "Dž", "३", "s", "😀🏽", "\t", "㍿EOTå𐞁", "​EOT", "
", "\t", "-", " \n", "s", " \n", "#$%", "ḍ̇é", "\r\n", "　ee漢"]} +{"text": "'ſEOTéⅣ\nİ'S!!!t<|endoftext|>éＡ<|fim_prefix|>Ⅳ'Z‍'Re'
<|endoftext|>\u000b<㋿　#$%漢Ａ\"ꟲ㍿'T'T", "tokens": 65, "pieces": ["'ſ", "EOTé", "Ⅳ", "\n", "İ'S", "!!!", "t", "<|", "endoftext", "|>", "é", "Ａ", "<|", "fim", "_prefix", "|>", "Ⅳ", "'Z", "‍'", "Re", "'", "
", "<|", "endoftext", "|>", "\u000b", "<㋿", "　", "#$%", "漢", "Ａ", "\"ꟲ", "㍿'", "T'T"]} +{"text": "'Re😀🏽½(.>\u000bſa㍿<|fim_prefix|>>t!'ll३ꟲ \né\n0e\r\n\r\n…😀🏽½́dḍ̇𐞁\r\n\r\n<|fim_prefix|>.<|endoftext|>9", "tokens": 63, "pieces": ["'Re", "😀🏽", "½", "(.>", "\u000bſa", "㍿<|", "fim", "_prefix", "|>>", "t", "!'", "ll", "३", "ꟲ", " \n", "é", "\n", "0", "e", "\r\n\r\n", "…", "😀🏽", "½", "́dḍ̇𐞁", "\r\n\r\n", "<|", "fim", "_prefix", "|>.<|", "endoftext", "|>", "9"]} +{"text": "\r\nå👍🏽!!é", "tokens": 9, "pieces": ["\r\n", "å", "👍🏽!!", "é"]} +{"text": "dİ(.عع \n字㍿\nå(Z'ſ㍿\r\n\r\n,<|fim_prefix|>", "tokens": 29, "pieces": ["d", "İ", "(.", "عع", " \n", "字", "㍿\n", "å", "(Z'ſ", "㍿\r\n\r\n", ",<|", "fim", "_prefix", "|>"]} +{"text": "!", "tokens": 1, "pieces": ["!"]} +{"text": " \n\r\n.s <|endoftext|>aꟲ'sꟲ३…\r\n\r\n\u000bDž‍\t9🙂ſ", "tokens": 30, "pieces": [" \n\r\n", ".s", " <|", "endoftext", "|>", "aꟲ's", "ꟲ", "३", "…\r\n\r\n", "\u000bDž", "‍", "\t", "9", "🙂ſ"]} +{"text": "'re𐞁é३ fi", "tokens": 10, "pieces": ["'re𐞁é", "३", " ", " fi"]} +{"text": "12345678 #$%<|fim_prefix|>‍㍿'T😀🏽fi'll's'S12345678½é
,🙂٣٤٥٦#$%👍🏽🙂12345678<Ⅳ!\"'VE
", "tokens": 59, "pieces": ["123", "456", "78", " ", " #$%<|", "fim", "_prefix", "|>‍㍿'", "T", "😀🏽", "fi'll", "'s'S", "123", "456", "78½", "é", "
", ",🙂<", "META", "_START", ">", "٣٤٥", "٦", "#$%👍🏽🙂", "123", "456", "78", "<", "Ⅳ", "!\"'", "VE", "
"]} +{"text": "fi>İ𐞁…\u000b'D­\tß👍🏽 Ⅳß'Dé\r\n \nß　👍🏽", "tokens": 36, "pieces": ["fi", ">İ𐞁", "…", "\u000b", "'D", "­", "\t", "ß", "👍🏽", " ", " ", "Ⅳ", "ß'D", "é", "\r\n \n", "ß", "　", "👍🏽"]} +{"text": "#$% ßع́'T<Ａ012345678 \n<|fim_prefix|> ㍿👍🏽'", "tokens": 62, "pieces": ["👍🏽>", "Ａ", "012", "345", "678", " \n", "<|", "fim", "_prefix", "|>", " ", "㍿👍🏽'"]} +{"text": "ꟲ'll#$%(\r\n\r\nZ0\u000b👍🏽
'VE ,½'ll­EOT", "tokens": 23, "pieces": ["ꟲ'll", "#$%(\r\n\r\n", "Z", "0", "\u000b", "👍🏽", "
", "'VE", " ", ",", "½", "'ll", "­EOT"]} +{"text": "İ#$%\n9sßEOTd-!!<|endoftext|>　'ReＡＡfiⅣ'ſéſ🙂étſ\ne'ſ㋿é'VE\"ꟲ漢", "tokens": 47, "pieces": ["İ", "#$%\n", "9", "sß", "EOTd", "-!!<|", "endoftext", "|>", "　", "'Re", "ＡＡfi", "Ⅳ", "'ſéſ", "🙂étſ", "\n", "e'ſ", "㋿é'VE", "\"ꟲ漢"]} +{"text": "<३…😀🏽m>'s<|endoftext|>\r\n\r\n३'ſ'S<|endoftext|><|fim_prefix|>Dž🙂ſⅣＡ㋿-'re#$%!é\r<|fim_prefix|>å9…s'VE", "tokens": 63, "pieces": ["<", "३", "…", "😀🏽", "m", ">'", "s", "<|", "endoftext", "|>\r\n\r\n", "३", "'ſ'S", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "Dž", "🙂ſ", "Ⅳ", "Ａ", "㋿-'", "re", "#$%!", "é", "\r", "<|", "fim", "_prefix", "|>", "å", "9", "…s'VE"]} +{"text": " <|endoftext|>\r\t<|endoftext|>…ßſ\n#$%🙂㋿ḍ̇\r\n\r\n", "tokens": 34, "pieces": [" ", "<|", "endoftext", "|>\r", "\t", "<|", "endoftext", "|>", "…ßſ", "\n", "#$%🙂㋿", "ḍ̇", "\r\n\r\n"]} +{"text": " \n(\u000b!!\u000b\r\n\t́'t漢३!\nß \n㍿\t'T,-m-\u000b>ſ \ns३
<|endoftext|>㋿ \n'S'VE>\u000b🙂\r\n", "tokens": 47, "pieces": [" \n", "(", "\u000b", "!!", "\u000b\r\n", "\t́'t", "漢", "३", "!\n", "ß", " \n", "㍿", "\t", "'T", ",-", "m", "-", "\u000b", ">ſ", " \n", "s", "३", "
", "<|", "endoftext", "|>㋿", " \n", "'S'VE", ">", "\u000b", "🙂\r\n"]} +{"text": "d'Rea'0㍿İé👍🏽s.٣٤٥٦('S\",​
12345678…ꟲ.\r\n\r\n'T㋿ ½'D", "tokens": 51, "pieces": ["d'Re", "a", "'", "0", "㍿İé", "👍🏽", "s", ".", "٣٤٥", "٦", "('", "S", "<", "META", "_START", ">\",​", "
", "123", "456", "78", "…ꟲ", ".\r\n\r\n", "<", "EOT", ">'", "T", "㋿", " ", "½", "'D"]} +{"text": "'ſ'S…\" 'll㍿. ! (s!!\r\nß字㋿ḍ̇Z'ſ<|fim_prefix|>'Tß㍿ſ\n9\r\n#$%
㍿'T", "tokens": 58, "pieces": ["'ſ'S", "…", "\"", " '", "ll", "㍿.", " !", " ", "(<", "EOT", ">s", "!!\r\n", "ß字", "㋿ḍ̇", "Z'ſ", "<|", "fim", "_prefix", "|>'", "Tß", "㍿ſ", "\n", "9", "\r\n", "#$%", "
", "㍿'", "T"]} +{"text": "İm#$%🙂é'll'VE'VEfi \n\r\r0漢عt'llİd's\r'M­9", "tokens": 26, "pieces": ["İm", "#$%🙂", "é'll", "'VE'VE", "fi", " \n\r\r", "0", "漢عt'll", "İd's", "\r", "'M", "­", "9"]} +{"text": "́'T'ſ \ne\u000bⅣ\ns9ḍ̇'S,\r\n\r\néed're<|fim_prefix|> 👍🏽\u000b½'Re", "tokens": 33, "pieces": ["́'T", "'ſ", " \n", "e", "\u000b", "Ⅳ", "\n", "s", "9", "ḍ̇'S", ",\r\n\r\n", "éed're", "<|", "fim", "_prefix", "|>", " ", " 👍🏽", "\u000b", "½", "'Re"]} +{"text": "🙂'VE‍<​<|endoftext|>ⅣEOTdſ‍Dž-'D(", "tokens": 25, "pieces": ["🙂'", "VE", "‍<​<|", "endoftext", "|>", "Ⅳ", "EOTdſ", "‍Dž", "-'", "D", "("]} +{"text": "İ'Reİ𐞁'ſ\re", "tokens": 11, "pieces": ["İ'Re", "İ𐞁'ſ", "\r", "e"]} +{"text": "…åt\r'VE\nع​<|fim_prefix|>Ⅳß😀🏽s㍿<|fim_prefix|>,\r ­ꟲ…İ're!!,'T< 9EOT", "tokens": 55, "pieces": ["…åt", "\r", "'VE", "\n", "ع", "​<|", "fim", "_prefix", "|>", "Ⅳ", "ß", "😀🏽", "s", "㍿<|", "fim", "_prefix", "|>,\r", " ", " ­", "ꟲ", "…İ're", "!!,'", "T", "<", " ", "9", "EOT"]} +{"text": "a‍\"mé,\rßꟲ'llé,t…#$% 'M!!t㍿'VE<|endoftext|>t('ſ", "tokens": 37, "pieces": ["a", "‍\"", "mé", ",\r", "ßꟲ'll", "é", ",t", "…", "#$%", " '", "M", "!!", "t", "㍿'", "VE", "<|", "endoftext", "|>", "t", "('", "ſ"]} +{"text": "ع'S,", "tokens": 3, "pieces": ["ع'S", ","]} +{"text": "漢 a字", "tokens": 4, "pieces": ["漢", " a字"]} +{"text": "d'Reé're \n,!!<|fim_prefix|>😀🏽 \n​12345678m \n㍿　\r\n\r\nḍ̇'VE'S‍tZ>å#$%'S'D!,​,#$%\"٣٤٥٦Ａ<漢,", "tokens": 57, "pieces": ["d'Re", "é're", " \n", ",!!<|", "fim", "_prefix", "|>😀🏽", " \n", "​", "123", "456", "78", "m", " \n", "㍿", "　\r\n\r\n", "ḍ̇'VE", "'S", "‍t", "Z", ">å", "#$%'", "S'D", "!,​,#$%\"", "٣٤٥", "٦", "Ａ", "<漢", ","]} +{"text": "'Sefi're\t­<|fim_prefix|>‍'Re\u000b🙂!12345678!! \na𐞁'S12345678EOT­Ａ<|endoftext|>㍿'llİeé", "tokens": 51, "pieces": ["'Sefi're", "\t", "­<|", "fim", "_prefix", "|>‍'", "Re", "\u000b", "🙂!", "123", "456", "78", "!!", " \n", "a𐞁'S", "123", "456", "78", "EOT", "­Ａ", "<|", "endoftext", "|>㍿'", "ll", "İeé"]} +{"text": " 𐞁‍३Dž́­!½\r\n\r\nZsＡ!'T", "tokens": 19, "pieces": [" 𐞁", "‍", "३", "Dž́", "­!", "½", "\r\n\r\n", "Zs", "Ａ", "!'", "T"]} +{"text": "0‍'Re.٣٤٥٦'ſ 's\ta\r½\r\n>ée'Dع\u000b𐞁a'Dİ 0　🙂'D'så漢'D'D३é'M>", "tokens": 46, "pieces": ["0", "‍'", "Re", ".", "٣٤٥", "٦", "'ſ", " '", "s", "\ta", "\r", "½", "\r\n", ">ée'D", "ع", "\u000b𐞁a'D", "İ", " ", " ", "0", "　", "🙂'", "D's", "å漢'D", "'D", "३", "é'M", ">"]} +{"text": "عⅣ,9!!s …ع<
🙂,0　å\tDž👍🏽\r\n\r\nḍ̇ !!३ \n\r\n\r\n𐞁éfi'M", "tokens": 42, "pieces": ["ع", "Ⅳ", ",", "9", "!!", "s", " ", "…ع", "<", "
", "🙂,", "0", "　å", "\tDž", "👍🏽\r\n\r\n", "ḍ̇", " ", "!!", "३", " \n\r\n\r\n", "𐞁éfi'M"]} +{"text": "!<|endoftext|>३t\"३,😀🏽\t'D𐞁12345678'½", "tokens": 30, "pieces": ["!<|", "endoftext", "|>", "३", "t", "\"", "३", ",<", "META", "_START", ">😀🏽", "\t", "'D𐞁", "123", "456", "78", "'", "½"]} +{"text": "Dž<|endoftext|>", "tokens": 9, "pieces": ["Dž", "<|", "endoftext", "|>"]} +{"text": "#$%३>
\r\n\r\n<|endoftext|>字٣٤٥٦fifiå\r
ZEOT\rå㋿‍#$%", "tokens": 37, "pieces": ["#$%", "३", ">", "
\r\n\r\n", "<|", "endoftext", "|>", "字", "٣٤٥", "٦", "fifiå", "\r", "
ZEOT", "\r", "å", "㋿‍#$%"]} +{"text": "𐞁㍿s9​…!ſḍ̇'Re.<|endoftext|>(ꟲs \n'll0 …ḍ̇ 'TDžfi<|fim_prefix|>0EOT​🙂½a0'sＡ\u000b", "tokens": 64, "pieces": ["𐞁", "㍿s", "9", "​", "…", "!ſḍ̇'Re", ".<", "META", "_START", "><|", "endoftext", "|>(", "ꟲs", " \n", "'ll", "0", " ", "…ḍ̇", " ", " '", "TDžfi", "<|", "fim", "_prefix", "|>", "0", "EOT", "​🙂", "½", "a", "0", "'s", "Ａ", "\u000b"]} +{"text": ">09(!!ſＡDž- 'Sfi​\u000b'D'VE0!!\t'Se'VE's'D12345678''M", "tokens": 39, "pieces": [">", "09", "(!!", "ſ", "ＡDž", "-", " ", " '", "Sfi", "​", "\u000b", "'D'VE", "0", "!!<", "EOT", ">", "\t", "'Se'VE", "'", "s'D", "123", "456", "78", "''", "M"]} +{"text": "fi0m>-'sé \n\r‍9fi,Z\r\n½é9
㋿'re>'lĺéⅣ", "tokens": 44, "pieces": ["fi", "0", "m", ">-<", "EOT", ">'", "sé", " \n\r", "‍<", "EOT", ">", "9", "fi", ",Z", "\r\n", "½", "é", "", "9", "
", "㋿'", "re", ">'", "lĺé", "Ⅳ", ""]} +{"text": "😀🏽½　\r-\rEOTét#$%é\r'Tع>é İ'D.㍿<|fim_prefix|>½é½ ‍a\"DžDžＡⅣ Ａ<'VE𐞁", "tokens": 59, "pieces": ["😀🏽", "½", "　\r", "-\r", "EOTét", "#$%", "é", "\r", "'Tع", ">é", " İ'D", ".㍿<|", "fim", "_prefix", "|>", "½", "é", "½", " ‍", "a", "\"DžDžＡ", "Ⅳ", " ", " Ａ", "<<", "EOT", ">'", "VE𐞁"]} +{"text": "ꟲ'M😀🏽🙂­", "tokens": 13, "pieces": ["ꟲ", "'", "M", "😀🏽🙂­"]} +{"text": "㍿Z㍿åfié'ſZ㋿>'VEdeع \n \nm 👍🏽éå३é", "tokens": 37, "pieces": ["㍿Z", "㍿åfié'ſ", "Z", "㋿>'", "VEdeع", " \n \n", "m", " 👍🏽<", "META", "_START", ">éå", "३", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "…''re　!m㋿ \n'T9\r\n\r\n<|fim_prefix|>m", "tokens": 21, "pieces": ["…", "''", "re", "　", "!m", "㋿", " \n", "'T", "9", "\r\n\r\n", "<|", "fim", "_prefix", "|>", "m"]} +{"text": "\t9EOT　 's9'reİåt\n'D#$%s字>ꟲ", "tokens": 23, "pieces": ["\t", "9", "EOT", "　", " '", "s", "9", "'re", "İåt", "\n", "'D", "#$%", "s字", ">ꟲ"]} +{"text": "'D(mEOT½\u000bß\r\n\r\n,ḍ̇m's'T́>'ſ'D\r\na字0\t(ß'VE\r\u000b 🙂.عs9(", "tokens": 40, "pieces": ["'D", "(m", "EOT", "½", "\u000bß", "\r\n\r\n", ",ḍ̇m's", "'T́", ">'", "ſ'D", "\r\n", "a字", "0", "\t", "(", "ß'VE", "\r", "\u000b", " ", "🙂.", "عs", "9", "("]} +{"text": "å're㋿ḍ̇'reZ㋿́'S漢!!
(Ａé'S३\r\n\r\n#$% \n're
'VE#$%fi\n,\u000b", "tokens": 38, "pieces": ["å're", "㋿ḍ̇'re", "Z", "㋿́'S", "漢", "!!", "
", "(Ａé'S", "३", "\r\n\r\n", "#$%", " \n", "'re", "
", "'VE", "#$%", "fi", "\n", ",", "\u000b"]} +{"text": "!!é åéİ'Re㋿((!㋿", "tokens": 16, "pieces": ["!!", "é", " ", " åé", "İ'Re", "㋿((!㋿"]} +{"text": "'sDž\n字\r\n\r\nm#$%fi
漢'Ret½\u000bß'Tḍ̇9 ½ éEOT're'ſⅣ字३\tm", "tokens": 37, "pieces": ["'s", "Dž", "\n", "字", "\r\n\r\n", "m", "#$%", "fi", "
漢'Re", "t", "½", "\u000bß'T", "ḍ̇", "9", " ", "½", " ", " é", "EOT're", "'ſ", "Ⅳ", "字", "३", "\tm"]} +{"text": "ع'S\"", "tokens": 7, "pieces": ["ع", "'", "S", "\""]} +{"text": " \nZsⅣ\"sⅣ0é12345678<|fim_prefix|>>", "tokens": 19, "pieces": [" \n", "Zs", "Ⅳ", "\"s", "Ⅳ0", "é", "123", "456", "78", "<|", "fim", "_prefix", "|>>"]} +{"text": " \n
d㍿́12345678ſ'Ａ㋿\" \né#$%\rfi<\r\n\r\n'lle", "tokens": 29, "pieces": [" \n", "
d", "㍿́", "123", "456", "78", "ſ", "'", "Ａ", "㋿\"", " \n", "é", "#$%\r", "fi", "<\r\n\r\n", "'lle"]} +{"text": "'VEmDžd'Re\r\n'Re< ㍿ é ‍
 漢…'TZ t\r'Refi!", "tokens": 37, "pieces": ["'VEm", "Džd'Re", "\r\n", "'Re", "<", " ", " ㍿", " ", " é", " ", " ‍<", "EOT", ">", "
", " 漢", "…", "'TZ", " ", " t", "\r", "'Refi", "!"]} +{"text": "\t\"३!½字#$%\"'Sḍ̇𐞁ꟲ… \nDž́ſéⅣ​👍🏽", "tokens": 49, "pieces": ["\t", "\"", "३", "!", "½", "字", "<", "META", "_START", ">#$%<", "META", "_START", ">\"'", "Sḍ̇𐞁ꟲ", "… \n", "Dž́ſé", "Ⅳ", "​👍🏽"]} +{"text": "'S(ß-'ll'T!

½>'VE'Td漢ع'Sd漢'VEＡ'så\r<|endoftext|>ⅣEOT'S 漢 '<|fim_prefix|>\t", "tokens": 52, "pieces": ["'S", "(ß", "-'", "ll'T", "!", "
", "
", "½", ">'", "VE'T", "d漢ع'S", "d漢'VE", "Ａ's", "å", "\r", "<|", "endoftext", "|><", "META", "_START", ">", "Ⅳ", "EOT'S", " ", " 漢", " ", " '<|", "fim", "_prefix", "|>", "\t"]} +{"text": "🙂́Ⅳ
éßZ字,­漢'ſ漢'T<|fim_prefix|>", "tokens": 23, "pieces": ["🙂́", "Ⅳ", "
éß", "Z字", ",­", "漢'ſ", "漢'T", "<|", "fim", "_prefix", "|>"]} +{"text": "!!\tßß", "tokens": 4, "pieces": ["!!", "\tßß"]} +{"text": "Z!ḍ̇'Sꟲ<#$%a#$%a's'edⅣ\teeſmeİ9'Dm", "tokens": 28, "pieces": ["Z", "!ḍ̇'S", "ꟲ", "<#$%", "a", "#$%", "a's", "'ed", "Ⅳ", "\teeſme", "İ", "9", "'Dm"]} +{"text": "Dž\u000b12345678\"
tꟲ'Mt\"t½", "tokens": 17, "pieces": ["Dž", "\u000b", "123", "456", "78", "\"", "
tꟲ'M", "t", "\"t", "½"]} +{"text": "٣٤٥٦'ſ'M\r0EOT<'re9½<​㍿'ll'lla-s‍<|endoftext|>​tm½a aa½,å\"'D", "tokens": 44, "pieces": ["٣٤٥", "٦", "'ſ'M", "\r", "0", "EOT", "<'", "re", "9½", "<​㍿'", "ll'll", "a", "-s", "‍<|", "endoftext", "|>​", "tm", "½", "a", " aa", "½", ",å", "\"'", "D"]} +{"text": "123456780Ａfi", "tokens": 9, "pieces": ["", "123", "456", "780", "Ａfi"]} +{"text": "İ‍d'll\nEOT'Dd<|endoftext|>'S\u000b", "tokens": 18, "pieces": ["İ", "‍d'll", "\n", "EOT'D", "d", "<|", "endoftext", "|>'", "S", "\u000b"]} +{"text": "'M'ReeＡ12345678!😀🏽, Dž<|endoftext|>'s(\"ſ'\tſ0<|endoftext|>EOT ꟲ#$% \n😀🏽DžDž9عſe<'ReＡⅣé", "tokens": 63, "pieces": ["'M'Re", "e", "Ａ", "123", "456", "78", "!😀🏽,", " Dž", "<|", "endoftext", "|>'", "s", "(\"", "ſ", "'", "\tſ", "0", "<|", "endoftext", "|>", "EOT", " ", " ꟲ", "#$%", " \n", "😀🏽", "DžDž", "9", "عſe", "<'", "Re", "Ａ", "Ⅳ", "é"]} +{"text": "Ⅳ'ḍ̇　👍🏽0ḍ̇12345678EOT<|fim_prefix|>\r\nⅣ😀🏽'VE\"é!!'S\nß 'T.𐞁é
>ḍ̇\u000b<|fim_prefix|>,́s", "tokens": 66, "pieces": ["Ⅳ", "'<", "EOT", ">ḍ̇", "　", "👍🏽", "0", "ḍ̇", "123", "456", "78", "EOT", "<|", "fim", "_prefix", "|>\r\n", "Ⅳ", "😀🏽'", "VE", "\"é", "!!'", "S", "\n", "ß", " ", " '", "T", ".𐞁é", "
", ">ḍ̇", "\u000b", "<|", "fim", "_prefix", "|>,́", "s"]} +{"text": "!Z\r\n½\u000b\u000bsع\n<.<|fim_prefix|>
'VEsDž㋿d𐞁' \n<|fim_prefix|>İ#$%'MZ३ Ⅳ 'VE\r\n\r\nm9　　\r\n", "tokens": 56, "pieces": ["!Z", "\r\n", "½", "\u000b", "\u000bsع", "\n", "<.<|", "fim", "_prefix", "|>", "
", "'VEs", "Dž", "㋿d𐞁", "'", " \n", "<|", "fim", "_prefix", "|>", "İ", "#$%'", "MZ", "३", " ", " ", "Ⅳ", " ", " '", "VE", "\r\n\r\n", "m", "9", "　　\r\n"]} +{"text": "!åİ㋿('Re\r\n'VEḍ̇t<|endoftext|>\n0İ<9!0(", "tokens": 33, "pieces": ["!å", "İ", "㋿<", "META", "_START", ">('", "Re", "\r\n", "'VEḍ̇t", "<|", "endoftext", "|>\n", "0", "İ", "<", "9", "!", "0", "("]} +{"text": "fi<''Tde,\t… \n<😀🏽ḍ̇Dž👍🏽 é
\tſعḍ̇Ⅳ🙂t's\"㋿ mm(", "tokens": 47, "pieces": ["fi", "<''", "Tde", ",", "\t… \n", "<😀🏽", "ḍ̇", "Dž", "👍🏽", " é", "
", "\tſعḍ̇", "Ⅳ", "🙂t's", "\"㋿", " ", " mm", "("]} +{"text": "'re'Re're ,ß
​㋿'reEOTＡ!!㋿tDž#$%> \ne9🙂é…9å'red(s\r\n", "tokens": 40, "pieces": ["'re'Re", "'re", " ", " ,", "ß", "
", "​㋿'", "re", "EOTＡ", "!!㋿", "t", "Dž", "#$%>", " \n", "e", "9", "🙂é", "…", "9", "å're", "d", "(s", "\r\n"]} +{"text": "fia.9㍿.aꟲ 'llß", "tokens": 16, "pieces": ["fia", ".", "9", "㍿.", "aꟲ", " ", " '", "llß"]} +{"text": "\nDžé <|endoftext|>…字 ­e\r\n…<|endoftext|><'s　ꟲ\t \t-!…<|fim_prefix|>𐞁 ſ\r\n\r\n'VEſḍ̇dع", "tokens": 59, "pieces": ["\n", "Džé", " <|", "endoftext", "|>", "…字", " ­", "e", "\r\n", "…", "<|", "endoftext", "|><'", "s", "　ꟲ", "\t ", "\t", "-!", "…", "<|", "fim", "_prefix", "|>", "𐞁", " ſ", "\r\n\r\n", "'VEſḍ̇dع"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "½,字å<|fim_prefix|>🙂字\u000bé\n'Mß're.ſ½é'D<🙂", "字", "\u000bé", "\n", "'M", "ß're", ".ſ", "½", "é'D", "<<", "d漢"]} +{"text": "'S
　ḍ̇m㋿Ａe字́ḍ̇㋿'TŹ>mＡé
　 <|endoftext|>'VE,å \n'ſ", "tokens": 47, "pieces": ["'S", "
", "　ḍ̇m", "㋿Ａe字́ḍ̇", "㋿'", "TŹ", ">m", "Ａ", "é", "
　", " ", "<|", "endoftext", "|>'", "VE", ",å", " \n", "'ſ"]} +{"text": "9ZDž<字é字<Dž½d‍'T!sⅣEOT \n‍\tEOTåa'ſＡå٣٤٥٦İ𐞁", "tokens": 43, "pieces": ["9", "ZDž", "<字é字", "<Dž", "½", "d", "‍'", "T", "!s", "Ⅳ", "EOT", " \n", "‍", "\tEOTåa'ſ", "Ａå", "٣٤٥", "٦", "İ𐞁"]} +{"text": "'ll>३ \n'12345678#$%\r!d‍́,-t😀🏽\r\n\r\n'D>́👍🏽", "tokens": 31, "pieces": ["'ll", ">", "३", " \n", "'", "123", "456", "78", "#$%\r", "!d", "‍́", ",-", "t", "😀🏽\r\n\r\n", "'D", ">́", "👍🏽"]} +{"text": "12345678'S as­<|fim_prefix|>Dž12345678>es!<|endoftext|>\t", "tokens": 28, "pieces": ["123", "456", "78", "'S", " as", "­<|", "fim", "_prefix", "|>", "Dž", "123", "456", "78", ">es", "!<|", "endoftext", "|>", "\t"]} +{"text": "Z12345678e'S\r\n<|fim_prefix|>é½-", "tokens": 19, "pieces": ["Z", "123", "456", "78", "e'S", "\r\n", "<|", "fim", "_prefix", "|>", "é", "½", "-"]} +{"text": "<|fim_prefix|>\r\nm𐞁\u000b漢߅'s'VE'Dßåß ,  'Re'D'MDž.'ll字'Sꟲ…ع", "tokens": 48, "pieces": ["<|", "fim", "_prefix", "|>\r\n", "m𐞁", "\u000b漢ß", "…", "'s'VE", "'Dß", "åß", " ", ",", " ", " <", "META", "_START", ">'", "Re'D", "'MDž", ".'", "ll字'S", "ꟲ", "…ع"]} +{"text": "'Dḍ̇'D\nfi!'s\r\n\r\nİ́t …,'re\tḍ̇<|endoftext|>", "tokens": 33, "pieces": ["'Dḍ̇'D", "\n", "fi", "!'", "s", "\r\n\r\n", "İ́t", " ", "…", ",'", "re", "\t", "ḍ̇", "<|", "endoftext", "|>"]} +{"text": "İ.,'VE'S
\r( !­ßſ'Sꟲs\tꟲ\u000b-'ſع,Ａå", "tokens": 30, "pieces": ["İ", ".,'", "VE'S", "
\r", "(", " ", " !­", "ßſ'S", "ꟲs", "\tꟲ", "\u000b", "-'", "ſع", ",Ａå"]} +{"text": "12345678­٣٤٥٦('Re0ſ<…ع🙂\u000btéⅣa're're", "tokens": 33, "pieces": ["123", "456", "78", "­", "٣٤٥", "٦", "('", "Re", "0", "ſ", "<", "…ع", "🙂", "\u000b", "t", "é", "Ⅳ", "a're", "'re"]} +{"text": "ꟲ字>t́㋿\r\n\r\n字👍🏽ds'Tß<|fim_prefix|>🙂 字", "tokens": 30, "pieces": ["ꟲ字", ">t́", "㋿\r\n\r\n", "字", "👍🏽", "ds'T", "ß", "<|", "fim", "_prefix", "|>🙂", " 字"]} +{"text": " a  ​'sꟲdés!!\"'VE<|endoftext|>'re👍🏽EOT'sZ're' ́Ａ字é½𐞁's㋿9㍿ſ", "tokens": 54, "pieces": [" a", " ", " ", "​'", "sꟲ", "dés", "!!\"'", "VE", "<|", "endoftext", "|>'", "re", "👍🏽", "EOT's", "Z're", "'", " ́Ａ字é", "½", "𐞁's", "㋿", "9", "㍿ſ"]} +{"text": "‍\n'sm,fiع́t,'s!!٣٤٥٦'字😀🏽👍🏽‍'re३!
", "tokens": 30, "pieces": ["‍\n", "'sm", ",fiع́t", ",'", "s", "!!", "٣٤٥", "٦", "'字", "😀🏽👍🏽‍'", "re", "३", "!", "
"]} +{"text": "　…𐞁­'D­Ⅳ0EOT\r🙂字\r\nß'VE३é<|endoftext|>Z👍🏽😀🏽ß٣٤٥٦d𐞁ꟲ㋿<|fim_prefix|>㍿\n", "tokens": 69, "pieces": ["　", "…𐞁", "­'", "D", "­", "Ⅳ0", "EOT", "\r", "🙂字", "\r\n", "ß'VE", "३", "é", "<|", "endoftext", "|>", "Z", "👍🏽😀🏽", "ß", "٣٤٥", "٦", "d𐞁ꟲ", "㋿<|", "fim", "_prefix", "|>㍿<", "META", "_START", ">\n"]} +{"text": "-\r\n\r\nß🙂're<|fim_prefix|>EOT…ésfi<'ll…ꟲ12345678㋿e>", "tokens": 37, "pieces": ["-\r\n\r\n", "ß", "🙂<", "META", "_START", ">'", "re", "<|", "fim", "_prefix", "|>", "EOT", "…ésfi", "<'", "ll", "…ꟲ", "123", "456", "78", "㋿e", ">"]} +{"text": "字.EOTå \n ½ ‍'re'VE'llß
sſ漢'T'M'Då(#$%<|fim_prefix|><­'re\r\n\r\nZ½ß're<|fim_prefix|>'s　é", "tokens": 55, "pieces": ["字", ".EOTå", " \n", " ", " ", "½", " ", "‍'", "re'VE", "'llß", "
sſ漢'T", "'M'D", "å", "(#$%<|", "fim", "_prefix", "|><­'", "re", "\r\n\r\n", "Z", "½", "ß're", "<|", "fim", "_prefix", "|><", "META", "_START", ">'", "s", "　é"]} +{"text": "Ⅳ", "tokens": 5, "pieces": ["", "Ⅳ"]} +{"text": "　9ß(😀🏽㋿漢< <|endoftext|>''Mfi𐞁'D\r\n\r\n 12345678\r\n漢å<ع're,\r\nZ", "tokens": 41, "pieces": ["　", "9", "ß", "(😀🏽㋿", "漢", "<", " <|", "endoftext", "|>''", "Mfi𐞁'D", "\r\n\r\n", " ", "123", "456", "78", "\r\n", "漢å", "<ع're", ",\r\n", "Z"]} +{"text": "٣٤٥٦ß!!Ａ\t👍🏽!!e.ꟲ'VE", "tokens": 19, "pieces": ["٣٤٥", "٦", "ß", "!!", "Ａ", "\t", "👍🏽!!", "e", ".ꟲ'VE"]} +{"text": "'D\r\n\r\n'sſ𐞁<|fim_prefix|><|endoftext|>12345678<|endoftext|>é́́İ< ſDž'T \nfiⅣſＡ>Zꟲ㋿ع\r(>ſ' \n \n12345678٣٤٥٦Ａ", "tokens": 73, "pieces": ["'D", "\r\n\r\n", "'sſ𐞁", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "123", "456", "78", "<|", "endoftext", "|>", "é́́", "İ", "<", " ", " ſ", "Dž'T", " \n", "fi", "Ⅳ", "ſ", "Ａ", "><", "EOT", ">Zꟲ", "㋿ع", "\r", "(>", "ſ", "'", " \n \n", "123", "456", "78٣", "٤٥٦", "Ａ"]} +{"text": "'sḍ̇-é३<|fim_prefix|>'T'sſ'Mé<|fim_prefix|>", "tokens": 25, "pieces": ["'sḍ̇", "-é", "३", "<|", "fim", "_prefix", "|>'", "T's", "ſ'M", "é", "<|", "fim", "_prefix", "|>"]} +{"text": ">漢d\"\nZ're­'S'D,\n", "tokens": 11, "pieces": [">漢d", "\"\n", "Z're", "­'", "S'D", ",\n"]} +{"text": "m 🙂'TＡtⅣ\rd'Reſ'VE٣٤٥٦Ａ३㍿'Reİ'Ss å😀🏽́!…\r(\n'M'S'ſ漢Ⅳ 漢…Ⅳ ", "tokens": 55, "pieces": ["m", " ", "🙂'", "TＡt", "Ⅳ", "\r", "d'Re", "ſ'VE", "٣٤٥", "٦", "Ａ", "३", "㍿'", "Re", "İ'S", "s", " å", "😀🏽́!", "…\r", "(\n", "'M'S", "'ſ漢", "Ⅳ", " ", " 漢", "…", "Ⅳ", " "]} +{"text": "🙂́ 12345678<|endoftext|>'M(", "tokens": 15, "pieces": ["🙂́", " ", "123", "456", "78", "<|", "endoftext", "|>'", "M", "("]} +{"text": "<'ſ½'ſ're½'VE\"\n
'S,ßꟲ\n!!ſ'ſ 
!'MßéZ", "tokens": 29, "pieces": ["<'", "ſ", "½", "'ſ're", "½", "'VE", "\"\n", "
", "'S", ",ßꟲ", "\n", "!!", "ſ'ſ", " ", "
", "!'", "Mßé", "Z"]} +{"text": "
d́EOT漢!<|fim_prefix|>.½  ३…\"Ａ́'S<|endoftext|>", "tokens": 31, "pieces": ["
d́", "EOT漢", "!<|", "fim", "_prefix", "|>.", "½", "  ", " ", "३", "…", "\"Ａ́'S", "<|", "endoftext", "|>"]} +{"text": "<|endoftext|>0ع👍🏽'Re字ع're🙂\r\n\r\n𐞁ḍ̇,", "tokens": 27, "pieces": ["<|", "endoftext", "|>", "0", "ع", "👍🏽'", "Re字ع're", "🙂\r\n\r\n", "𐞁ḍ̇", ","]} +{"text": "'re\n㋿\u000bs३\r\nḍ̇ꟲ's'llå-ḍ̇Ａ \n​㍿'!'Re'rem㋿\r\n \nḍ̇㍿😀🏽½", "tokens": 49, "pieces": ["'re", "\n", "㋿", "\u000bs", "३", "\r\n", "ḍ̇ꟲ's", "'llå", "-ḍ̇", "Ａ", " \n", "​㍿'!'", "Re're", "m", "㋿\r\n", " \n", "ḍ̇", "㍿😀🏽", "½"]} +{"text": "e🙂fi'S<|endoftext|>
met'M's½å🙂㋿👍🏽'M!, ß<|fim_prefix|>\r\n\r\n<|fim_prefix|>　m,", "tokens": 45, "pieces": ["e", "🙂fi'S", "<|", "endoftext", "|>", "
met'M", "'s", "½", "å", "🙂㋿👍🏽'", "M", "!,", " ß", "<|", "fim", "_prefix", "|>\r\n\r\n", "<|", "fim", "_prefix", "|>", "　m", ","]} +{"text": "<Ａét<|fim_prefix|>İ<|fim_prefix|>,
", "tokens": 17, "pieces": ["<Ａét", "<|", "fim", "_prefix", "|>", "İ", "<|", "fim", "_prefix", "|>,", "
"]} +{"text": "٣٤٥٦ß aＡİa \t\r\n'Dꟲ🙂　", "tokens": 21, "pieces": ["٣٤٥", "٦", "ß", " a", "Ａİa", "", " \t\r\n", "'Dꟲ", "🙂", "　"]} +{"text": "½#$%​é字12345678ß½fi'llDž­Ⅳ㋿(,Dž \n'S'VEd", "tokens": 31, "pieces": ["½", "#$%​", "é字", "123", "456", "78", "ß", "½", "fi'll", "Dž", "­", "Ⅳ", "㋿(,", "Dž", " \n", "'S'VE", "d"]} +{"text": ">'S(#$%'D…Z‍'M㋿\r\n>", "tokens": 17, "pieces": [">'", "S", "(#$%'", "D", "…Z", "‍'", "M", "㋿\r\n", ">"]} +{"text": "aⅣ12345678<|fim_prefix|>>m🙂 !!…'Re< \r\nＡ​😀🏽#$%-Ａ'Re🙂​'Tt'Re(s٣٤٥٦Z­m", "tokens": 43, "pieces": ["a", "Ⅳ12", "345", "678", "<|", "fim", "_prefix", "|>>", "m", "🙂", " !!", "…", "'Re", "<", " \r\n", "Ａ", "​😀🏽#$%-", "Ａ'Re", "🙂​'", "Tt'Re", "(s", "٣٤٥", "٦", "Z", "­m"]} +{"text": "å.'M'VE
\tEOT漢漢🙂.12345678ع
EOT!!Zß  dع9ſ.<'S漢\n,👍🏽9🙂\u000b", "tokens": 41, "pieces": ["å", ".'", "M'VE", "
", "\tEOT漢漢", "🙂.", "123", "456", "78", "ع", "
EOT", "!!", "Zß", "  ", " dع", "9", "ſ", ".<'", "S漢", "\n", ",👍🏽", "9", "🙂", "\u000b"]} +{"text": "ḍ̇Ａ'ſ<", "tokens": 7, "pieces": ["ḍ̇", "Ａ'ſ", "<"]} +{"text": "㍿\u000b>e㋿#$%'S\r<|endoftext|>fi \ń(<|fim_prefix|>
.İDžs'M'Ret0
'S'MＡ12345678", "tokens": 46, "pieces": ["㍿", "\u000b", ">e", "㋿#$%'", "S", "\r", "<|", "endoftext", "|>", "fi", " \n", "́", "(<|", "fim", "_prefix", "|>", "
", ".İDžs'M", "'Ret", "0", "
", "'S'M", "Ａ", "123", "456", "78"]} +{"text": "å", "tokens": 2, "pieces": ["å"]} +{"text": " ḍ̇Dž", "tokens": 7, "pieces": [" ", " ḍ̇", "Dž"]} +{"text": "'ſå३ß'MDž'M<|endoftext|>​12345678٣٤٥٦😀🏽 'VE'T'(ع\t(åⅣ", "tokens": 47, "pieces": ["'ſå", "३", "ß'M", "Dž'M", "<|", "endoftext", "|>​", "123", "456", "78", "", "٣٤٥", "٦", "😀🏽", " ", " '", "VE'T", "'(", "ع", "\t", "(å", "Ⅳ"]} +{"text": "EOT𐞁d-!!‍'D👍🏽ß'T漢0👍🏽字12345678'Re(…('T字　å(Z½éḍ̇0#$%'S ٣٤٥٦​", "tokens": 59, "pieces": ["EOT𐞁d", "-<", "META", "_START", ">!!‍'", "D", "👍🏽", "ß'T", "漢", "0", "👍🏽", "字", "123", "456", "78", "'Re", "(", "…", "('", "T字", "　", "å", "(Z", "½", "éḍ̇", "0", "#$%'", "S", " ", " ", "٣٤٥", "٦", "​"]} +{"text": "0'Re\r\n\r\n-mⅣ½a½́ ꟲ…", "tokens": 15, "pieces": ["0", "'Re", "\r\n\r\n", "-m", "Ⅳ½", "a", "½", "́", " ꟲ", "…"]} +{"text": "9EOT٣٤٥٦'ſe'VE٣٤٥٦字é\"'ll", "tokens": 21, "pieces": ["9", "EOT", "٣٤٥", "٦", "'ſe'VE", "٣٤٥", "٦", "字é", "\"'", "ll"]} +{"text": "　EOT'ſfi'S-12345678.ꟲfi🙂🙂 'D½12345678\tEOTdm'll\n'!åå-'D", "tokens": 41, "pieces": ["　EOT'ſ", "fi'S", "-", "123", "456", "78", ".ꟲfi", "🙂<", "EOT", ">🙂", " '", "D", "½12", "345", "678", "\tEOTdm'll", "\n", "'!", "åå", "-'", "D"]} +{"text": "'VE EOT'Mḍ̇'ſ're𐞁İ'ſ\"'reß \n­0­<|endoftext|>漢sDž 'Re­'VEع𐞁.\n३㍿Zdé'M", "tokens": 63, "pieces": ["'VE", " EOT'M", "ḍ̇", "'", "ſ're", "𐞁", "İ'ſ", "\"'", "re", "ß", " \n", "­", "0", "­<|", "endoftext", "|>", "漢s", "Dž", " ", "'Re", "­'", "VEع𐞁", ".\n", "३", "㍿Zdé'M"]} +{"text": "!!½!!s12345678!!.", "tokens": 8, "pieces": ["!!", "½", "!!", "s", "123", "456", "78", "!!."]} +{"text": "fi㍿t३…Dž12345678é३Z\u000bfi,0­d'ſ‍\r\n\r\nⅣ…fi ", "tokens": 42, "pieces": ["fi", "㍿t", "३", "…Dž", "123", "456", "78", "é", "३", "Z", "\u000bfi", ",", "0", "­<", "EOT", ">d'ſ", "‍\r\n\r\n", "", "Ⅳ", "…fi", " "]} +{"text": "dع'Rem\r'M>'S​'ſＡdéDž́ſİ9ḍ̇,\nå-字­- 'Ms'M!!٣٤٥٦12345678३Ａé́\t", "tokens": 46, "pieces": ["dع'Re", "m", "\r", "'M", ">'", "S", "​'", "ſ", "Ａdé", "Dž́ſ", "İ", "9", "ḍ̇", ",\n", "å", "-字", "­-", " '", "Ms'M", "!!", "٣٤٥", "٦12", "345", "678", "३", "Ａé́", "\t"]} +{"text": "é🙂\t\n,Ａsé　🙂'lĺ'Ddß\"字 !!'S'D\u000b'll< \n(DžＡå<|endoftext|>0!'re", "tokens": 48, "pieces": ["é", "🙂", "\t\n", ",Ａsé", "　", "🙂'", "lĺ'D", "dß", "\"字", " ", "!!'", "S'D", "\u000b", "'ll", "<", " \n", "(DžＡå", "<|", "endoftext", "|>", "0", "!'", "re"]} +{"text": "é", "tokens": 2, "pieces": ["é"]} +{"text": " '​🙂-字'ſ<|endoftext|>'M㋿\"tt㍿d-ſ'Tꟲ'M½٣٤٥٦\r\n\r\n'ſ \n'llt'reZ٣٤٥٦0d", "tokens": 50, "pieces": [" '​🙂-", "字'ſ", "<|", "endoftext", "|>'", "M", "㋿\"", "tt", "㍿d", "-ſ'T", "ꟲ'M", "½٣٤", "٥٦", "\r\n\r\n", "'ſ", " \n", "'llt're", "Z", "٣٤٥", "٦0", "d"]} +{"text": "\"\nt 'ꟲß", "tokens": 11, "pieces": ["\"\n", "t", " ", "'<", "META", "_START", ">ꟲß"]} +{"text": "12345678'll\t​12345678.\r\n\r\nſ!tßfi‍\"'s< \r!!0é½\"('Dḍ̇et'S.", "tokens": 38, "pieces": ["123", "456", "78", "'ll", "\t", "​", "123", "456", "78", ".\r\n\r\n", "ſ", "!tßfi", "‍\"'", "s", "<", " \r", "!!", "0", "é", "½", "\"('", "Dḍ̇et'S", "."]} +{"text": "-m𐞁å", "tokens": 10, "pieces": ["-m𐞁å", ""]} +{"text": "\r\nZ🙂👍🏽­　\u000b", "tokens": 9, "pieces": ["\r\n", "Z", "🙂👍🏽­", "　\u000b"]} +{"text": "\t👍🏽e12345678m'Sé>㋿'MEOT's\t㍿\né́'S\"!!0 ßå!d ,Ａ٣٤٥٦ \n‍\n'Re", "tokens": 51, "pieces": ["\t", "👍🏽", "e", "123", "456", "78", "m'S", "é", ">㋿'", "MEOT's", "\t", "㍿\n", "é́'S", "\"!!", "0", " ", " ßå", "!d", " ", ",Ａ", "٣٤٥", "٦", " \n", "‍\n", "'Re"]} +{"text": "𐞁( 😀🏽\n\r\n\r\n", "tokens": 11, "pieces": ["𐞁", "(", " ", "😀🏽\n\r\n\r\n"]} +{"text": "9tm,­ع-字३. 漢‍ 　Ⅳ'Re'Re\u000b9Z", "tokens": 22, "pieces": ["9", "tm", ",­", "ع", "-字", "३", ".", " ", " 漢", "‍", " ", "　", "Ⅳ", "'Re'Re", "\u000b", "9", "Z"]} +{"text": "‍👍🏽'M­́İḍ̇ 🙂(mDžＡ㋿\r\n 9'll ><|endoftext|>ſ\n!!🙂ét9🙂>'VE-9 ", "tokens": 51, "pieces": ["‍👍🏽'", "M", "­́İḍ̇", " ", " 🙂(", "m", "DžＡ", "㋿\r\n", " ", " ", "9", "'ll", " ><|", "endoftext", "|>", "ſ", "\n", "!!🙂", "ét", "9", "🙂>'", "VE", "-", "9", "", " "]} +{"text": "-0 ḍ̇", "tokens": 6, "pieces": ["-", "0", " ḍ̇"]} +{"text": "é字fifi!!é's👍🏽ſm<|fim_prefix|>(漢s\r\n\r\n
İ३
ݽ's9İaſs'sé'ſ \néİ😀🏽\t", "tokens": 47, "pieces": ["é字fifi", "!!", "é's", "👍🏽", "ſm", "<|", "fim", "_prefix", "|>(", "漢s", "\r\n\r\n", "
İ", "३", "
İ", "½", "'s", "9", "İaſs's", "é'ſ", " \n", "é", "İ", "😀🏽", "\t"]} +{"text": "عs", "tokens": 2, "pieces": ["عs"]} +{"text": "İ's\refi　Ａt's­9😀🏽åⅣ\r\n'd‍", "tokens": 21, "pieces": ["İ's", "\r", "efi", "　Ａt's", "­", "9", "😀🏽", "å", "Ⅳ", "\r\n", "'d", "‍"]} +{"text": "<\r漢\u000ba\"½t'T𐞁字0㋿\t३😀🏽'Re'T", "tokens": 26, "pieces": ["<\r", "漢", "\u000ba", "\"", "½", "t'T", "𐞁字", "0", "㋿", "\t", "३", "😀🏽'", "Re'T"]} +{"text": "👍🏽", "tokens": 3, "pieces": ["👍🏽"]} +{"text": "　å👍🏽𐞁\r\n'' 0e!!'D🙂
", "tokens": 20, "pieces": ["　å", "👍🏽", "𐞁", "\r\n", "''", " ", "0", "e", "!!'", "D", "🙂", "
"]} +{"text": "12345678Z", "tokens": 4, "pieces": ["123", "456", "78", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi…123456780Z0'M", "tokens": 9, "pieces": ["fi", "…", "123", "456", "780", "Z", "0", "'M"]} +{"text": "'ſ 'VE,\t३Ⅳ😀🏽", "tokens": 13, "pieces": ["'ſ", " ", " '", "VE", ",", "\t", "३Ⅳ", "😀🏽"]} +{"text": "字­'漢ꟲ\r\n­‍'S('S\"́sa\u000bßßt#$% <|fim_prefix|>👍🏽 ḍ̇t𐞁ḍ̇ …!ß𐞁㍿", "tokens": 61, "pieces": ["字", "­'", "漢ꟲ", "\r\n", "­‍'", "S", "('", "S", "\"́sa", "\u000b", "ßßt", "#$%", " <|", "fim", "_prefix", "|><", "EOT", ">👍🏽", " ḍ̇t𐞁ḍ̇", " ", "…", "!ß𐞁", "㍿"]} +{"text": "ſ́😀🏽㋿0'T­漢ع‍\t d12345678'VE\tⅣſ🙂㋿dDž't!!", "tokens": 37, "pieces": ["ſ́", "😀🏽㋿", "0", "'T", "­漢", "ع", "‍", "\t", " d", "123", "456", "78", "'VE", "\t", "Ⅳ", "ſ", "🙂㋿", "d", "Dž't", "!!"]} +{"text": "t…عꟲß́
👍🏽\u000b'MＡdſ'VE'D", "tokens": 21, "pieces": ["t", "…عꟲß́", "
", "👍🏽", "\u000b", "'MＡdſ'VE", "'D"]} +{"text": "İß漢e㋿ ३…<|endoftext|>9Ａ́", "tokens": 22, "pieces": ["İß漢e", "㋿", " ", " ", "३", "…", "<|", "endoftext", "|>", "9", "Ａ́"]} +{"text": "ß'ret(ßt'Re👍🏽0,\u000b!!s\rＡ'​𐞁​字ꟲع\u000b", "tokens": 36, "pieces": ["ß're", "t", "(ßt'Re", "👍🏽", "0", ",", "\u000b", "!!", "s", "\r", "Ａ", "'​", "𐞁", "​字ꟲع", "", "\u000b"]} +{"text": "٣٤٥٦Ａ'T\t  mꟲ9 >'T𐞁ＡaＡ\r\n>👍🏽'S'Dm
​", "tokens": 41, "pieces": ["٣٤٥", "٦", "Ａ'T", "\t ", " mꟲ", "9", "", " >'", "T𐞁Ａa", "Ａ", "\r\n", ">👍🏽'", "S'D", "m", "
", "​<", "EOT", ">"]} +{"text": "Džḍ̇㍿(<|fim_prefix|>ḍ̇½!!\n('Re<|fim_prefix|>👍🏽 \n…字\n12345678Ａ㋿ḍ̇d éİEOTt.…ſ\u000bå're​9", "tokens": 64, "pieces": ["Džḍ̇", "㍿(<|", "fim", "_prefix", "|>", "ḍ̇", "½", "!!\n", "('", "Re", "<|", "fim", "_prefix", "|>👍🏽", " \n", "…字", "\n", "123", "456", "78", "Ａ", "㋿ḍ̇d", " é", "İEOTt", ".", "…ſ", "\u000bå're", "​", "9"]} +{"text": "\r\n\r\nⅣſ'redé,é \n<|endoftext|>ßſ\rꟲＡ,\n\r👍🏽(👍🏽😀🏽<|endoftext|>\"-\n𐞁…9", "tokens": 51, "pieces": ["\r\n\r\n", "Ⅳ", "ſ're", "dé", ",é", " \n", "<|", "endoftext", "|>", "ßſ", "\r", "ꟲ", "Ａ", ",\n\r", "👍🏽(👍🏽😀🏽<|", "endoftext", "|>\"-\n", "𐞁", "…", "9"]} +{"text": "Ａ0's", "tokens": 3, "pieces": ["Ａ", "0", "'s"]} +{"text": "😀🏽Z\n'Tſfi\r\n'll>.'SßéEOT0　漢…😀🏽́Ⅳ́𐞁ＡéEOT\u000bt", "tokens": 39, "pieces": ["😀🏽", "Z", "\n", "'Tſfi", "\r\n", "'ll", ">.'", "Sßé", "EOT", "0", "　漢", "…", "😀🏽́", "Ⅳ", "́𐞁Ａé", "EOT", "\u000bt"]} +{"text": "🙂漢!!İ🙂٣٤٥٦EOTß\r\n\t
#$%å0>-👍🏽\ŕ𐞁𐞁३İ", "tokens": 36, "pieces": ["🙂漢", "!!", "İ", "🙂", "٣٤٥", "٦", "EOTß", "\r\n", "\t", "
", "#$%", "å", "0", ">-👍🏽\r", "́𐞁𐞁", "३", "İ"]} +{"text": "Z'T're ㋿​½­‍å \n<　<'T \nmm½", "tokens": 24, "pieces": ["Z'T", "'re", " ", "㋿​", "½", "­‍", "å", " \n", "<", "　", "<'", "T", " \n", "mm", "", "½"]} +{"text": "'VE​㍿­½<|endoftext|>", "tokens": 19, "pieces": ["'VE", "​㍿­<", "META", "_START", ">", "½", "<|", "endoftext", "|>"]} +{"text": "\r\nå\r ٣٤٥٦!!'ll\nsß're'ſ'S9sZ🙂\r\nß­s", "tokens": 30, "pieces": ["\r\n", "å", "\r", " ", " ", "٣٤٥", "٦", "!!'", "ll", "\n", "s", "ß're", "'ſ'S", "9", "s", "Z", "🙂\r\n", "ß", "­s"]} +{"text": "'VE🙂", "tokens": 3, "pieces": ["'VE", "🙂"]} +{"text": "é\u000bDž‍s­!", "tokens": 9, "pieces": ["é", "\u000bDž", "‍s", "­!"]} +{"text": "'SZ#$%Z½m🙂ſ字ع字\råeſ'll😀🏽å", "tokens": 23, "pieces": ["'SZ", "#$%", "Z", "½", "m", "🙂ſ字ع字", "\r", "åeſ'll", "😀🏽", "å"]} +{"text": "ḍ̇'T<|endoftext|>
9<|fim_prefix|>漢㋿‍0ß🙂　a\u000bＡ३", "tokens": 32, "pieces": ["ḍ̇'T", "<|", "endoftext", "|>", "
", "9", "<|", "fim", "_prefix", "|>", "漢", "㋿‍", "0", "ß", "🙂", "　a", "\u000bＡ", "३"]} +{"text": "'ll-'ll0'Re字ZꟲⅣ漢👍🏽\r\n\r\n😀🏽\u000bZ字,dß", "tokens": 25, "pieces": ["'ll", "-'", "ll", "0", "'Re字", "Zꟲ", "Ⅳ", "漢", "👍🏽\r\n\r\n", "😀🏽", "\u000bZ字", ",dß"]} +{"text": "\r\n\r\nſDžZ're \nⅣ9Ａ'ſ\r\n\r\n'VÉé٣٤٥٦𐞁ém!<'Re #$%EOT!!", "tokens": 39, "pieces": ["\r\n\r\n", "ſ", "DžZ're", " \n", "Ⅳ9", "Ａ'ſ", "\r\n\r\n", "'VÉé", "٣٤٥", "٦", "𐞁ém", "!<'", "Re", " #$%", "EOT", "!!"]} +{"text": "'S<|fim_prefix|>ꟲ😀🏽s　d'VE漢<|endoftext|>'llDžſ😀🏽", "tokens": 38, "pieces": ["'S", "<|", "fim", "_prefix", "|>", "ꟲ", "😀🏽", "s", "　d'VE", "漢", "<|", "endoftext", "|>'", "ll", "Džſ", "😀🏽"]} +{"text": "'Sde㍿漢>🙂Dž!!<👍🏽 \n½ḍ̇'ll­fi12345678\"é<|fim_prefix|>", "tokens": 34, "pieces": ["'Sde", "㍿漢", ">🙂", "Dž", "!!<👍🏽", " \n", "½", "ḍ̇'ll", "­fi", "123", "456", "78", "\"é", "<|", "fim", "_prefix", "|>"]} +{"text": "\"\r\n'VE<|fim_prefix|>🙂\"", "tokens": 11, "pieces": ["\"\r\n", "'VE", "<|", "fim", "_prefix", "|>🙂\""]} +{"text": " ,<​ß'T<\r\n\r\nåeß' é<|endoftext|>>'ll-ß's㋿'VEß‍<|fim_prefix|>İt…-12345678'>٣٤٥٦　ß", "tokens": 54, "pieces": [" ,<​", "ß'T", "<\r\n\r\n", "åeß", "'", " é", "<|", "endoftext", "|>>'", "ll", "-ß's", "㋿'", "VEß", "‍<|", "fim", "_prefix", "|>", "İt", "…", "-", "123", "456", "78", "'>", "٣٤٥", "٦", "　ß"]} +{"text": "t​!12345678'll😀🏽\r\n", "tokens": 11, "pieces": ["t", "​!", "123", "456", "78", "'ll", "😀🏽\r\n"]} +{"text": ".é　,<|fim_prefix|>ḍ̇\rEOT'Ret​åé\t'ſt㋿ſⅣ🙂漢e\r\n\r\n'SＡa12345678　!­", "tokens": 45, "pieces": [".é", "　", ",<|", "fim", "_prefix", "|>", "ḍ̇", "\r", "EOT'Re", "t", "​åé", "\t", "'ſt", "㋿ſ", "Ⅳ", "🙂漢e", "\r\n\r\n", "'SＡa", "123", "456", "78", "　", "!­"]} +{"text": "　́  're12345678dm", "tokens": 9, "pieces": ["　́", " ", " ", "'re", "123", "456", "78", "dm"]} +{"text": "ſéſ9́EOTEOT!!\" ßİ!!fia é!…s字e", "tokens": 26, "pieces": ["ſéſ", "9", "́", "EOTEOT", "!!\"", " ", " ß", "İ", "!!", "fia", " é", "!", "…s字e"]} +{"text": "३'s\r'M
㍿,\r\n\r\nعEOT 'll,'re\r\n'M'ſİ!", "tokens": 26, "pieces": ["३", "'s", "\r", "'M", "
", "㍿,\r\n\r\n", "ع", "EOT", " ", "'ll", ",'", "re", "\r\n", "'M'ſ", "İ", "!<", "EOT", ">"]} +{"text": "'ſm'S'llſ'Re漢éꟲ9", "tokens": 16, "pieces": ["'ſm'S", "'llſ'Re", "漢é", "ꟲ", "9"]} +{"text": "-.ꟲſ'\r'll'VE
å", "tokens": 13, "pieces": ["-.", "ꟲſ", "'\r", "'ll'VE", "
å"]} +{"text": "'S٣٤٥٦é0Ａ🙂é<|endoftext|>s\r\n\r\n", "tokens": 23, "pieces": ["'S", "٣٤٥", "٦", "é", "0", "Ａ", "🙂é", "<|", "endoftext", "|><", "EOT", ">s", "\r\n\r\n"]} +{"text": "\u000b", "tokens": 1, "pieces": ["\u000b"]} +{"text": ".\"-ßⅣEOT🙂aⅣé
", "tokens": 14, "pieces": [".\"-", "ß", "Ⅳ", "EOT", "🙂a", "Ⅳ", "é", "
"]} +{"text": "½\u000b🙂‍0!a ३9\r\n😀🏽߅.'S", "tokens": 19, "pieces": ["½", "\u000b", "🙂‍", "0", "!a", " ", "३9", "\r\n", "😀🏽", "ß", "…", ".'", "S"]} +{"text": "漢Ⅳḍ̇Ａ((\r\r\n .<|endoftext|>
ſ'Re३٣٤٥٦ꟲm", "tokens": 29, "pieces": ["漢", "Ⅳ", "ḍ̇", "Ａ", "((\r\r\n", " ", ".<|", "endoftext", "|>", "
ſ'Re", "३٣٤", "٥٦", "ꟲm"]} +{"text": "!!٣٤٥٦\r\n\n", "tokens": 6, "pieces": ["!!", "٣٤٥", "٦", "\r\n\n"]} +{"text": "٣٤٥٦字İ!!m'T…eé", "tokens": 17, "pieces": ["٣٤٥", "٦", "字", "İ", "!!", "m'T", "…eé", ""]} +{"text": ".　😀🏽d(Ａ́Z \nＡ🙂ß'Dİd \n½㍿é‍", "tokens": 24, "pieces": [".", "　", "😀🏽", "d", "(Ａ́", "Z", " \n", "Ａ", "🙂ß'D", "İd", " \n", "½", "㍿é", "‍"]} +{"text": "\r\n\r\n​Dž😀🏽s<|endoftext|>½0½½'re…", "tokens": 26, "pieces": ["\r\n\r\n", "​Dž", "😀🏽", "s", "<|", "endoftext", "|>", "½0", "", "½½", "'re", "…"]} +{"text": "Z'Dꟲ‍åéḍ̇mع\r", "tokens": 18, "pieces": ["Z'D", "ꟲ", "‍åéḍ̇mع", "\r"]} +{"text": "­>'S'ſ!!étß'll字𐞁m>ꟲ漢\nEOT٣٤٥٦'ll12345678(fi​9å0'D🙂́ Ａm", "tokens": 47, "pieces": ["­>'", "S'ſ", "!!", "étß'll", "字𐞁m", ">ꟲ漢", "\n", "EOT", "٣٤٥", "٦", "'ll", "123", "456", "78", "(fi", "​", "9", "å", "0", "'D", "🙂́", " ", " Ａm"]} +{"text": "s,'ll
…0's9é㍿é३🙂́Džſ0\r\n\r\n'sع'M 
Z!!ḍ̇", "tokens": 38, "pieces": ["s", ",'", "ll", "
", "…", "0", "'s", "9", "é", "㍿é", "३", "🙂́Džſ", "0", "\r\n\r\n", "'s", "ع'M", " ", "
Z", "!!", "ḍ̇"]} +{"text": "é ,İ<|endoftext|>عéé𐞁👍🏽​ſa👍🏽ZZ𐞁\u000ba0 ­9½0\t​\u000bfi''re'Re<|fim_prefix|>​\u000bå‍(<", "tokens": 60, "pieces": ["é", " ", ",İ", "<|", "endoftext", "|>", "عéé𐞁", "👍🏽​", "ſa", "👍🏽", "ZZ𐞁", "\u000ba", "0", " ", "­", "9½0", "\t", "​", "\u000bfi", "''", "re'Re", "<|", "fim", "_prefix", "|>​", "\u000bå", "‍(<"]} +{"text": "'Tḍ̇½ \n''Re<'re‍½.e12345678'D́é 𐞁fi🙂12345678\t٣٤٥٦fi漢'ſḍ̇12345678'D𐞁　漢é٣٤٥٦", "tokens": 63, "pieces": ["'Tḍ̇", "½", " \n", "''", "Re", "<'", "re", "‍", "½", ".e", "123", "456", "78", "'D́é", " ", " 𐞁fi", "🙂", "123", "456", "78", "\t", "٣٤٥", "٦", "fi漢'ſ", "ḍ̇", "", "123", "456", "78", "'D𐞁", "　漢é", "٣٤٥", "٦"]} +{"text": " s, ­
fi're \nEOT　ꟲ३字é漢\"\"!😀🏽ع
s‍méݽ 12345678‍㍿ſ.", "tokens": 46, "pieces": [" ", " s", ",", " ", "­", "
fi're", " \n", "EOT", "　ꟲ", "३", "字", "é漢", "\"\"!😀🏽", "ع", "
s", "‍mé", "İ", "½", " ", " ", "123", "456", "78", "‍㍿", "ſ", "."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " s's…å,ſa\"t½𐞁're\rEOT½'llⅣfi\">\"Ａ½İ-३t", "tokens": 40, "pieces": [" ", " s's", "…å", ",ſa", "\"t", "½", "𐞁're", "\r", "EOT", "½", "'ll", "Ⅳ", "fi", "\">\"", "Ａ", "½", "İ", "-<", "META", "_START", ">", "३", "t"]} +{"text": "\r\n\r\n<,9Ⅳ३'Re'Re0\n12345678३ḍ̇\u000b\r\n t.\t<|endoftext|>Ａ\r\n\r\n \n\ŕfi'Ss<|fim_prefix|>\r\n\r\n\n!!‍", "tokens": 47, "pieces": ["\r\n\r\n", "<,", "9Ⅳ३", "'Re'Re", "0", "\n", "123", "456", "78३", "ḍ̇", "\u000b\r\n", " t", ".", "\t", "<|", "endoftext", "|>", "Ａ", "\r\n\r\n \n\r", "́fi'S", "s", "<|", "fim", "_prefix", "|>\r\n\r\n\n", "!!‍"]} +{"text": "Ⅳ́é \n​Ⅳع́ ", "tokens": 11, "pieces": ["Ⅳ", "́é", " \n", "​", "Ⅳ", "ع́", " "]} +{"text": ">ꟲ\r\n👍🏽٣٤٥٦\"!é​
's#$%½'Re𐞁\r\n\r\né,", "tokens": 32, "pieces": [">ꟲ", "\r\n", "👍🏽", "٣٤٥", "٦", "\"!", "é", "​<", "META", "_START", ">", "
", "'s", "#$%", "½", "'Re𐞁", "\r\n\r\n", "é", ","]} +{"text": "İ's👍🏽<|fim_prefix|><|fim_prefix|>'ll#$%'saḍ̇
ſ12345678 'S'VE\n", "tokens": 33, "pieces": ["İ's", "👍🏽<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "ll", "#$%'", "saḍ̇", "
ſ", "123", "456", "78", " '", "S'VE", "\n"]} +{"text": "३\r\nꟲåéDž
(㋿éfiꟲEOT-a'VE \"'DéⅣ\n'\n
.Ａ!!\" \n👍🏽 🙂…é\r\n\r\n\n<|endoftext|>", "tokens": 57, "pieces": ["३", "\r\n", "ꟲåé", "Dž", "
", "(㋿", "éfiꟲ", "EOT", "-a'VE", " ", "\"'", "Dé", "Ⅳ", "\n", "'\n", "
", ".Ａ", "!!\"", " \n", "👍🏽", " 🙂", "…é", "\r\n\r\n\n", "<|", "endoftext", "|>"]} +{"text": "ß𐞁½,!!­½-😀🏽\r\n\r\n'ſ　‍#$%(,Z,\r\n,tt\u000b'ſ#$%", "tokens": 37, "pieces": ["ß𐞁", "½", ",!!­", "½", "-😀🏽\r\n\r\n", "'ſ", "　", "‍#$%(<", "META", "_START", "><", "EOT", ">,", "Z", ",\r\n", ",tt", "\u000b", "'ſ", "#$%"]} +{"text": "🙂३'ſ\rꟲ", "tokens": 8, "pieces": ["🙂", "३", "'ſ", "\r", "ꟲ"]} +{"text": "ع,'S", "tokens": 3, "pieces": ["ع", ",'", "S"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'VE'VE٣٤٥٦㋿İ'Re‍İ\r\n'VE'ss", "tokens": 20, "pieces": ["'VE'VE", "٣٤٥", "٦", "㋿İ'Re", "‍İ", "\r\n", "'VE's", "s"]} +{"text": " 'M", "tokens": 2, "pieces": [" '", "M"]} +{"text": "👍🏽 \nع><|fim_prefix|>(㍿", "tokens": 14, "pieces": ["👍🏽", " \n", "ع", "><|", "fim", "_prefix", "|>(㍿"]} +{"text": "½Z", "tokens": 2, "pieces": ["½", "Z"]} +{"text": "'s\r'T\r\nfi\tésEOT \nEOT​Z　12345678ع \né👍🏽…!!'llZ㋿́٣٤٥٦", "tokens": 39, "pieces": ["'s", "\r", "'T", "\r\n", "fi", "\tés", "EOT", " \n", "EOT", "​Z", "　", "123", "456", "78", "ع", " \n", "é", "👍🏽", "…", "!!'", "ll", "Z", "㋿́", "٣٤٥", "٦"]} +{"text": "(字'reſd", "tokens": 5, "pieces": ["(字're", "ſd"]} +{"text": "'s…EOT\r\nع　\nſås(m#$%㍿'D", "tokens": 20, "pieces": ["'s", "…EOT", "\r\n", "ع", "　\n", "ſås", "(m", "#$%㍿'", "D"]} +{"text": "
('VEꟲ㋿12345678 \n😀🏽'Rea\u000b \nⅣ\u000b漢'S \u000bß٣٤٥٦…३m'VE<|fim_prefix|>", "tokens": 45, "pieces": ["
", "('", "VEꟲ", "㋿", "123", "456", "78", " \n", "😀🏽'", "Rea", "\u000b \n", "Ⅳ", "\u000b漢'S", " ", "\u000bß", "٣٤٥", "٦", "…", "३", "m'VE", "<|", "fim", "_prefix", "|>"]} +{"text": "ع\t½0fiß<|endoftext|>-'D٣٤٥٦!", "tokens": 23, "pieces": ["ع", "\t", "½0", "fiß", "<|", "endoftext", "|>-'", "D", "٣٤٥", "٦", "!"]} +{"text": "㍿字tꟲ　'D 'ſéea\n३tDžع字'12345678漢0'S\"", "tokens": 31, "pieces": ["㍿字tꟲ", "　", "'D", " ", "'ſéea", "\n", "३", "t", "Džع字", "'", "123", "456", "78", "漢", "0", "'S", "\""]} +{"text": "ſ字'sⅣsⅣZ 12345678're½0'T>👍🏽­t'Séé́३,Ａ㍿\r\n>é EOT \n…'VE'\r\n\r\n‍😀🏽", "tokens": 53, "pieces": ["ſ字's", "Ⅳ", "s", "Ⅳ", "Z", " ", " ", "123", "456", "78", "'re", "½0", "'T", ">👍🏽­", "t'S", "éé́", "३", ",Ａ", "㍿\r\n", ">é", " ", " EOT", " \n", "…", "'VE", "'\r\n\r\n", "‍😀🏽"]} +{"text": "३!! \n\r\n \n🙂'D-😀🏽fi\rع12345678'.👍🏽३‍\té👍🏽0ꟲ㋿ſå", "tokens": 43, "pieces": ["३", "!!", " \n\r\n \n", "🙂'", "D", "-😀🏽", "fi", "\r", "ع", "123", "456", "78", "'.<", "META", "_START", ">👍🏽", "३", "‍", "\té", "👍🏽", "0", "ꟲ", "㋿ſå"]} +{"text": "漢𐞁é", "tokens": 6, "pieces": ["漢𐞁é"]} +{"text": "ſ'Mfi fi'ré'Reß'Re!", "tokens": 11, "pieces": ["ſ'M", "fi", " ", " fi're", "́'Re", "ß'Re", "!"]} +{"text": "'T\rfi­
\r\n\u000bEOTDžfi \n　fis\r½'Re,
٣٤٥٦'T​(ꟲfi　½… ", "tokens": 37, "pieces": ["'T", "\r", "fi", "­", "
\r\n", "\u000bEOTDžfi", " \n", "　fis", "\r", "½", "'Re", ",", "
", "٣٤٥", "٦", "'T", "​(", "ꟲfi", "　", "½", "… "]} +{"text": "㍿👍🏽d!\t\t́12345678", "tokens": 14, "pieces": ["㍿👍🏽", "d", "!", "\t", "\t́", "123", "456", "78"]} +{"text": "'D😀🏽'M𐞁Ａ<ꟲꟲع'T,.㍿'T\n'll<|fim_prefix|>'D<|fim_prefix|>12345678e's\tꟲ'\u000b😀🏽İ(\"", "tokens": 60, "pieces": ["'D", "😀🏽'", "M𐞁", "Ａ", "<ꟲꟲع'T", ",.㍿<", "META", "_START", ">'", "T", "\n", "'ll", "<|", "fim", "_prefix", "|>'", "D", "<|", "fim", "_prefix", "|>", "123", "456", "78", "e's", "\tꟲ", "'", "\u000b", "😀🏽", "İ", "(\""]} +{"text": "9'D'ſ𐞁\r\n\r\n'M字㋿å👍🏽sm'll\rfi(😀🏽'ſ𐞁#$%", "tokens": 35, "pieces": ["9", "'D'ſ", "𐞁", "\r\n\r\n", "'M字", "㋿å", "👍🏽", "sm'll", "\r", "fi", "(😀🏽'", "ſ𐞁", "#$%"]} +{"text": " ­'ll's!", "tokens": 5, "pieces": [" ­'", "ll's", "!"]} +{"text": "Džİ 'SＡ​0 …dꟲ'VE9'Re\r\n\r\nEOTꟲḍ̇ mmå-\"‍字ⅣＡ", "tokens": 39, "pieces": ["Džİ", " ", "'SＡ", "​", "0", " ", "…dꟲ'VE", "9", "'Re", "\r\n\r\n", "EOTꟲḍ̇", " mmå", "-\"‍", "字", "Ⅳ", "Ａ"]} +{"text": "-'M.d(", "tokens": 4, "pieces": ["-'", "M", ".d", "("]} +{"text": "\t", "tokens": 1, "pieces": ["\t"]} +{"text": "\n👍🏽'Rem<|fim_prefix|>'M字e'Reée'D!!'Re-", "tokens": 22, "pieces": ["\n", "👍🏽'", "Rem", "<|", "fim", "_prefix", "|>'", "M字e'Re", "ée'D", "!!'", "Re", "-"]} +{"text": "\t#$%Ⅳ'M\r\n İſ ​'ll\u000b'reDžḍ̇e're(ßZ字㍿ſ<㋿<|fim_prefix|><|endoftext|>㋿ß12345678'", "tokens": 58, "pieces": ["\t", "#$%", "Ⅳ", "'M", "\r\n", " İſ", " ", "​'", "ll", "\u000b", "'re", "Džḍ̇e're", "(ß", "Z字", "㍿ſ", "<㋿<", "META", "_START", "><|", "fim", "_prefix", "|><|", "endoftext", "|>㋿", "ß", "123", "456", "78", "'"]} +{"text": "'ſ'D<|endoftext|>sfi'T<é \r\n\r\ndm\"½Dž́'T\t㋿EOTDž fi", "tokens": 36, "pieces": ["'ſ'D", "<|", "endoftext", "|>", "sfi'T", "<é", " \r\n\r\n", "dm", "\"", "½", "Dž́'T", "", "\t", "㋿EOTDž", " fi"]} +{"text": "'ReⅣs'VE🙂'D", "tokens": 9, "pieces": ["'Re", "Ⅳ", "s'VE", "🙂'", "D"]} +{"text": " å\u000bع\"><'VEé👍🏽 ", "tokens": 13, "pieces": [" ", " å", "\u000bع", "\"><'", "VEé", "👍🏽", " "]} +{"text": "t!!'ſ!!!\td½ſ,<|fim_prefix|>'s'D12345678>Z…́Džſ\u000bé𐞁\"'T'Re​'D", "tokens": 45, "pieces": ["t", "!!<", "EOT", ">'", "ſ", "!!!", "\td", "½", "ſ", ",<|", "fim", "_prefix", "|>'", "s'D", "123", "456", "78", ">Z", "…́Džſ", "\u000bé𐞁", "\"'", "T", "'", "Re", "​'", "D"]} +{"text": "Ⅳ…eé,'ſ'Re\r\nd😀🏽-\nß0Z(👍🏽'S12345678m½\"㍿㍿å٣٤٥٦​<|endoftext|>\te \nZ", "tokens": 56, "pieces": ["Ⅳ", "…eé", ",'", "ſ'Re", "\r\n", "d", "😀🏽-\n", "ß", "0", "Z", "(👍🏽'", "S", "123", "456", "78", "m", "½", "\"㍿㍿", "å", "٣٤٥", "٦", "​<|", "endoftext", "|>", "\te", " \n", "Z"]} +{"text": "'ll\r\nꟲd", "tokens": 6, "pieces": ["'ll", "\r\n", "ꟲd"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿‍s#$%\t\r\n\r\na<|endoftext|>٣٤٥٦åع\n 0㋿\r\n<​Ⅳ,​9!!'<😀🏽 \n", "tokens": 44, "pieces": ["㍿‍", "s", "#$%", "\t\r\n\r\n", "a", "<|", "endoftext", "|>", "٣٤٥", "٦", "åع", "\n", " ", " ", "0", "㋿\r\n", "<​", "Ⅳ", ",​", "9", "!!'<😀🏽", " \n"]} +{"text": "''Re　12345678漢\ntḍ̇0'DEOTEOT'VEꟲ-ꟲ­", "tokens": 27, "pieces": ["''", "Re", "　", "123", "456", "78", "漢", "\n", "tḍ̇", "0", "'DEOTEOT'VE", "ꟲ", "-ꟲ", "­"]} +{"text": "0", "tokens": 4, "pieces": ["", "0"]} +{"text": "<ß're'S🙂'Reſ𐞁　0 ㋿㍿0\r\n𐞁\u000bé>👍🏽　
 \nſfi\t\t́Z'Re'D", "tokens": 45, "pieces": ["<ß're", "'S", "🙂'", "Reſ𐞁", "　", "0", " ㋿㍿", "0", "\r\n", "𐞁", "\u000bé", ">👍🏽", "　
 \n", "ſfi", "\t", "\t́", "Z'Re", "'D"]} +{"text": "'Re\t#$%<|endoftext|>é\u000b'ḍ̇,tＡ‍éＡß'T9're 're,㍿'D​𐞁a'D\"fi३", "tokens": 45, "pieces": ["'Re", "\t", "#$%<|", "endoftext", "|>", "é", "\u000b", "'ḍ̇", ",t", "Ａ", "‍é", "Ａß'T", "9", "'re", " ", " '", "re", ",㍿'", "D", "​𐞁a'D", "\"fi", "३"]} +{"text": "'S\r\nZ\rd!!  ​\t　", "tokens": 11, "pieces": ["'S", "\r\n", "Z", "\r", "d", "!!", " ", " ", "​", "\t　"]} +{"text": "𐞁'ſt'Re", "tokens": 8, "pieces": ["𐞁'ſ", "t'Re"]} +{"text": "éd0 \n'ſ \nⅣİ12345678e,Dž漢me字😀🏽", "tokens": 24, "pieces": ["éd", "0", " \n", "'ſ", " \n", "Ⅳ", "İ", "123", "456", "78", "e", ",Dž漢me字", "😀🏽"]} +{"text": "　!!<|endoftext|>'VE'T　字'", "tokens": 25, "pieces": ["　", "!!<|", "endoftext", "|>'", "VE", "'", "T", "", "　字", "'<", "META", "_START", ">"]} +{"text": "'s", "tokens": 1, "pieces": ["'s"]} +{"text": "漢!!'VE\t<|fim_prefix|>'Reem㍿\r\n🙂Ａ'Me٣٤٥٦ꟲ\u000bİ👍🏽३ 👍🏽 'T", "tokens": 44, "pieces": ["漢", "!!'", "VE", "\t", "<|", "fim", "_prefix", "|>'", "Reem", "㍿\r\n", "🙂<", "META", "_START", ">Ａ'M", "e", "٣٤٥", "٦", "ꟲ", "\u000bİ", "👍🏽", "३", " ", " 👍🏽", " ", "'T"]} +{"text": "'s!!‍'Re'llt-‍​-s12345678 9ḍ̇", "tokens": 25, "pieces": ["'s", "!!‍'", "Re'll", "t", "-‍​-", "s", "123", "456", "78", " ", " ", "9", "ḍ̇"]} +{"text": "é'VE!s\"Dž字s𐞁e\r\n\r\nZé's", "tokens": 26, "pieces": ["é'VE", "!<", "EOT", "><", "META", "_START", ">s", "\"Dž字s𐞁e", "\r\n\r\n", "Zé's"]} +{"text": "　\tⅣ<|fim_prefix|>\n12345678🙂<|fim_prefix|>'Sé'Re<|fim_prefix|>　ع\r\nfi𐞁'T'D𐞁́ꟲ…Ａd fi's<9s<|endoftext|>😀🏽Ⅳ٣٤٥٦12345678", "tokens": 76, "pieces": ["　", "\t", "Ⅳ", "<|", "fim", "_prefix", "|>\n", "123", "456", "78", "🙂<|", "fim", "_prefix", "|>'", "Sé'Re", "<|", "fim", "_prefix", "|>", "　ع", "\r\n", "fi𐞁'T", "'D𐞁́ꟲ", "…Ａd", " fi's", "<", "9", "s", "<|", "endoftext", "|>😀🏽", "Ⅳ٣٤", "٥٦1", "234", "567", "8"]} +{"text": " 9👍🏽\r㋿>d㍿!\u000b㋿\n'VEdét\u000b>éé👍🏽३ ३😀🏽㍿aＡéİ…m‍", "tokens": 50, "pieces": [" ", "9", "👍🏽\r", "㋿>", "d", "㍿!", "\u000b", "㋿\n", "'VEdét", "\u000b", ">éé", "👍🏽", "३", " ", "३", "😀🏽㍿", "a", "Ａé", "İ", "…m", "‍"]} +{"text": "s(.<|endoftext|> !ḍ̇'Mꟲ'VEDž", "tokens": 26, "pieces": ["s", "(.<|", "endoftext", "|>", " ", "!<", "META", "_START", ">ḍ̇'M", "ꟲ'VE", "Dž"]} +{"text": "​'TDž \n…㍿\u000b ſt㍿åé½😀🏽é­9👍🏽<|endoftext|>ḍ̇'reⅣ#$%…ſＡ", "tokens": 59, "pieces": ["​'", "TDž", " \n", "…", "㍿", "\u000b ", " ſt", "㍿", "åé", "½", "😀🏽", "é", "­", "9", "👍🏽<|", "endoftext", "|>", "ḍ̇'re", "Ⅳ", "#$%", "…ſ", "Ａ", ""]} +{"text": "!!!\tß'ſ漢s'Re9<|endoftext|>d'S -'!!​ 字t½\n‍ ḍ̇½३", "tokens": 39, "pieces": ["!!!", "\tß'ſ", "漢s'Re", "9", "<|", "endoftext", "|>", "d'S", " -'!!​", " ", " 字t", "½", "\n", "‍<", "EOT", ">", " ", " ḍ̇", "½३"]} +{"text": "'llＡ -​\u000b'VÉ字'ſß३Ⅳſ٣٤٥٦'😀🏽é<.Dž'VE.0<|fim_prefix|>ع…<|endoftext|>", "tokens": 52, "pieces": ["'ll", "Ａ", " ", "-​", "\u000b", "'VÉ字'ſ", "ß", "३Ⅳ", "ſ", "٣٤٥", "٦", "'😀🏽", "é", "<.", "Dž'VE", ".", "0", "<|", "fim", "_prefix", "|>", "ع", "…", "<|", "endoftext", "|>"]} +{"text": "'reꟲd٣٤٥٦EOT9'D\n 字😀🏽㋿字m'VE😀🏽-0 -㍿😀🏽's　!!", "tokens": 46, "pieces": ["'reꟲd", "٣٤٥", "٦", "EOT", "9", "'D", "\n", " ", " 字", "😀🏽<", "EOT", ">㋿", "字m'VE", "😀🏽-", "0", " -㍿😀🏽'", "s", "　", "!!"]} +{"text": "mt'S.Ａ \n‍३!!<|fim_prefix|>🙂'sZ<|endoftext|>Ａḍ̇.Dž㍿12345678s- 'ś12345678ß　Ⅳ㍿ꟲåＡ#$%字å½m", "tokens": 69, "pieces": ["mt'S", ".Ａ", " \n", "‍", "३", "!!<|", "fim", "_prefix", "|>🙂'", "s", "Z", "<|", "endoftext", "|>", "Ａḍ̇", ".Dž", "㍿", "123", "456", "78", "s", "-", " '", "ś", "123", "456", "78", "ß", "", "　", "Ⅳ", "㍿ꟲå", "Ａ", "#$%", "字å", "½", "m"]} +{"text": "👍🏽Ⅳ漢<|endoftext|>EOT́\t㋿\r\nع㍿", "tokens": 25, "pieces": ["👍🏽", "Ⅳ", "漢", "<|", "endoftext", "|>", "EOT́", "\t", "㋿\r\n", "ع", "㍿"]} +{"text": "　, é12345678Ａfi​0
t\r\n\r\nİDž😀🏽<|endoftext|>'ll\u000b'S'll  ㋿­EOT­­\r", "tokens": 48, "pieces": ["　", ",", " é", "123", "456", "78", "Ａfi", "​", "0", "
t", "\r\n\r\n", "İDž", "😀🏽<|", "endoftext", "|><", "META", "_START", ">'", "ll", "\u000b", "'S'll", " ", " <", "EOT", ">", " ", " ㋿­", "EOT", "­­\r"]} +{"text": "\"\r\n\r\n>́ß ", "tokens": 5, "pieces": ["\"\r\n\r\n", ">́ß", " "]} +{"text": "字t'SEOTعZ㋿''ll'sEOTfi", "tokens": 16, "pieces": ["字t'S", "EOTع", "Z", "㋿''", "ll's", "EOTfi"]} +{"text": "!!​👍🏽漢aꟲa\r\n🙂́\tⅣḍ̇Ⅳßd🙂Ⅳé'VEEOT…​㍿㍿\r\n\r\n!!…🙂 tfiⅣ🙂'Reé", "tokens": 56, "pieces": ["!!​👍🏽", "漢aꟲa", "\r\n", "🙂́", "\t", "Ⅳ", "ḍ̇", "Ⅳ", "ßd", "🙂", "Ⅳ", "é'VE", "EOT", "…", "​㍿㍿\r\n\r\n", "!!", "…", "🙂", " ", " tfi", "Ⅳ", "🙂'", "Reé"]} +{"text": "
…\tfiḍ̇'Re0'é\u000b", "tokens": 13, "pieces": ["
…", "\tfiḍ̇'Re", "0", "'é", "\u000b"]} +{"text": "Džs㍿-ſ", "tokens": 8, "pieces": ["Džs", "㍿-", "ſ"]} +{"text": "'VEé <|fim_prefix|>é\r'M", "tokens": 15, "pieces": ["'VEé", " ", "<|", "fim", "_prefix", "|>", "é", "\r", "'M"]} +{"text": "ßfi'ſ'ſ𐞁d", "tokens": 11, "pieces": ["ßfi'ſ", "'ſ𐞁d"]} +{"text": "'ſ　t\t fi\tEOT<|fim_prefix|>åafi\n!!m'\r\nع…'ſ ३ع'll…é👍🏽漢t'VEé \u000b­…漢<|endoftext|>", "tokens": 61, "pieces": ["'ſ", "　t", "\t ", " <", "EOT", ">fi", "\tEOT", "<|", "fim", "_prefix", "|>", "åafi", "\n", "!!", "m", "'\r\n", "ع", "…", "'ſ", " ", " ", "३", "ع'll", "…é", "👍🏽", "漢t'VE", "é", " ", "\u000b", "­", "…漢", "<|", "endoftext", "|>"]} +{"text": "😀🏽\r\n12345678 \nⅣ's,\t.9'é­'M😀🏽\rs㍿'M‍å㍿'Msa…ß'ſ9t'Re\tꟲZ 'S're", "tokens": 55, "pieces": ["😀🏽\r\n", "123", "456", "78", " \n", "Ⅳ", "'s", ",", "\t", ".", "9", "'é", "­'", "M", "😀🏽\r", "s", "㍿'", "M", "‍å", "㍿'", "Msa", "…ß'ſ", "9", "t'Re", "\tꟲ", "Z", " '", "S're"]} +{"text": "EOT", "tokens": 2, "pieces": ["EOT"]} +{"text": "'S'D　!9 ­३é,　'ReＡßs", "tokens": 16, "pieces": ["'S'D", "　", "!", "9", " ", "­", "३", "é", ",", "　", "'Re", "Ａßs"]} +{"text": " 'ſ're#$%0'De'عDž'Dعfi😀🏽Ⅳmå'sd\r\n\r\n'll\r'S\r\n\r\n", "tokens": 34, "pieces": [" ", " '", "ſ're", "#$%<", "EOT", ">", "0", "'De", "'ع", "Dž'D", "عfi", "😀🏽", "Ⅳ", "må's", "d", "\r\n\r\n", "'ll", "\r", "'S", "\r\n\r\n"]} +{"text": "\n\r\n\r\n<㍿½\"é's​­…㍿ꟲ३(𐞁㋿\r\n\r\n é
字å9\"aå\"", "tokens": 50, "pieces": ["\n\r\n\r\n", "<㍿", "½", "\"é's", "​­", "…", "㍿", "ꟲ", "", "३", "(𐞁", "㋿\r\n\r\n", " ", " é", "
字å", "9", "\"aå", "\""]} +{"text": ".'ſ'ſZt­ 'M-İḍ̇'S12345678#$%0'D㋿İfi\rEOTm'DZßع", "tokens": 36, "pieces": [".'", "ſ'ſ", "Zt", "­", " ", " '", "M", "-İḍ̇'S", "123", "456", "78", "#$%", "0", "'D", "㋿İfi", "\r", "EOTm'D", "Zßع"]} +{"text": "9½Dž\r\n\r\nZ!!‍Ａ!\t㋿-åꟲ…fiém 'M'sa­㍿,😀🏽🙂 é\r\n👍🏽's'", "tokens": 50, "pieces": ["9½", "Dž", "\r\n\r\n", "Z", "!!‍", "Ａ", "!", "\t", "㋿-", "åꟲ", "…", "fiém", " ", " '", "M's", "a", "­㍿,😀🏽🙂", " é", "\r\n", "👍🏽'", "s", "'"]} +{"text": "­'Tİ \n'ß\"…'VE'Mé​-<|endoftext|>\r\n\r\n​३'ſ.-a\"", "tokens": 31, "pieces": ["­'", "Tİ", " \n", "'ß", "\"", "…", "'VE'M", "é", "​-<|", "endoftext", "|>\r\n\r\n", "​", "३", "'ſ", ".-", "a", "\""]} +{"text": "\r\n12345678ßm'D
٣٤٥٦ḍ̇​ß,عs9<|fim_prefix|><|fim_prefix|><12345678'S­Ze9", "tokens": 44, "pieces": ["\r\n", "123", "456", "78", "ßm'D", "
", "٣٤٥", "٦", "ḍ̇", "​ß", ",", "عs", "9", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><", "123", "456", "78", "'S", "­Ze", "9"]} +{"text": "'T\r\n'S\r\n\r\nꟲ,- 　å12345678EOT㋿ \r\n\r\nⅣ<|endoftext|>t#$%👍🏽ßaé \n㋿", "tokens": 44, "pieces": ["'T", "\r\n", "'S", "\r\n\r\n", "ꟲ", ",-", " ", "　å", "123", "456", "78", "EOT", "㋿", " \r\n\r\n", "Ⅳ", "<|", "endoftext", "|>", "t", "#$%👍🏽", "ßaé", " \n", "㋿"]} +{"text": "'s", "tokens": 1, "pieces": ["'s"]} +{"text": "ß字ع#$%e.ee\r\u000bm\n​字\u000bß\r\n", "tokens": 19, "pieces": ["ß字ع", "#$%", "e", ".ee", "\r", "", "\u000bm", "\n", "​字", "\u000bß", "\r\n"]} +{"text": "­
m'lle-Ａ'MＡ😀🏽!!٣٤٥٦🙂-<|endoftext|>'Re'ſ'S12345678'D\t'll'safi ", "tokens": 40, "pieces": ["­", "
m'll", "e", "-Ａ'M", "Ａ", "😀🏽!!", "٣٤٥", "٦", "🙂-<|", "endoftext", "|>'", "Re'ſ", "'S", "123", "456", "78", "'D", "\t", "'ll's", "afi", " "]} +{"text": "aDž　e'Sİ
d½s'VE　…m'S…\nꟲ
ꟲ'Re'T…‍#$%å𐞁 'Re<|endoftext|>'VEEOT'Tꟲeé", "tokens": 63, "pieces": ["a", "Dž", "　e'S", "İ", "
d", "½", "s'VE", "　", "…", "m'S", "…\n", "ꟲ", "
ꟲ'Re", "'T", "…", "‍#$%", "å𐞁", " '", "Re", "<|", "endoftext", "|>'", "VEEOT'T", "ꟲeé"]} +{"text": "å🙂\t'S\t𐞁ⅣdaEOT\t<|fim_prefix|>Ⅳ½(12345678fi're漢­㍿#$%\n ,>\n<|endoftext|>å(ꟲ're \nḍ̇", "tokens": 59, "pieces": ["å", "🙂", "\t", "'S", "\t𐞁", "Ⅳ", "da", "EOT", "\t", "<|", "fim", "_prefix", "|>", "Ⅳ½", "(", "123", "456", "78", "fi're", "漢", "­㍿#$%\n", " ", " ,>\n", "<|", "endoftext", "|>", "å", "(ꟲ're", " \n", "ḍ̇"]} +{"text": "\r\n", "tokens": 1, "pieces": ["\r\n"]} +{"text": "😀🏽
　.\n👍🏽'‍\n12345678t \n'VE\"", "tokens": 23, "pieces": ["😀🏽", "
", "　", ".\n", "👍🏽'‍\n", "123", "456", "78", "t", " \n", "'VE", "\"<", "META", "_START", ">"]} +{"text": "​👍🏽😀🏽.<|endoftext|>", "tokens": 14, "pieces": ["​👍🏽😀🏽.<|", "endoftext", "|>"]} +{"text": "9'D…é.㍿!dm 'Rem's漢Džd​9s\r　\r\n字Ａ'DⅣ㋿𐞁㋿<|endoftext|>'M漢a12345678<", "tokens": 59, "pieces": ["9", "'D", "…é", ".㍿!", "dm", " ", "'Rem's", "漢Džd", "​", "9", "s", "\r　\r\n", "字", "Ａ'D", "Ⅳ", "㋿𐞁", "㋿<|", "endoftext", "|>'", "M漢a", "123", "456", "78", "<"]} +{"text": "½'m", "tokens": 2, "pieces": ["½", "'m"]} +{"text": "'Re　
<'D
 t<ß \n'VEé',e<|fim_prefix|>३‍<|fim_prefix|>fi\t\r\n🙂9\"<|fim_prefix|>\r\n s…ع", "tokens": 48, "pieces": ["'Re", "　", "
", "<'", "D", "
 ", " t", "<ß", " \n", "'VEé", "',", "e", "<|", "fim", "_prefix", "|>", "३", "‍<|", "fim", "_prefix", "|>", "fi", "\t\r\n", "🙂", "9", "\"<|", "fim", "_prefix", "|>\r\n", "", " ", " s", "…ع"]} +{"text": "'s\r\n\t!!s'VEs'Dßå\nd😀🏽EOT \n　 \n漢eé​🙂\">Z\t >", "tokens": 38, "pieces": ["'s", "\r\n", "\t", "!!", "s'VE", "s'D", "ßå", "\n", "d", "😀🏽", "EOT", " \n　 \n", "漢eé", "​🙂\">", "Z", "\t ", " >"]} +{"text": "‍sİé́<|fim_prefix|>🙂'ſfi'Mعt٣٤٥٦ ſDž🙂👍🏽‍-mⅣ
́ 'S'漢", "tokens": 45, "pieces": ["‍s", "İé́", "<|", "fim", "_prefix", "|>🙂'", "ſfi'M", "عt", "", "٣٤٥", "٦", " ſ", "Dž", "🙂👍🏽‍-", "m", "Ⅳ", "
́", " '", "S", "'漢"]} +{"text": "'MⅣfiḍ̇eſ㍿漢0٣٤٥٦-m's\r\n'Dع　ꟲ字字", "tokens": 29, "pieces": ["'M", "Ⅳ", "fiḍ̇eſ", "㍿漢", "0٣٤", "٥٦", "-m's", "\r\n", "'Dع", "　ꟲ字字"]} +{"text": "ḍ̇ḍ̇‍٣٤٥٦e's!", "tokens": 14, "pieces": ["ḍ̇ḍ̇", "‍", "٣٤٥", "٦", "e's", "!"]} +{"text": " 're'ſ.‍ꟲ'llå'ſ'ſİDž>­　(Ａt<", "tokens": 29, "pieces": [" '", "re'ſ", ".‍", "ꟲ'll", "å'ſ", "'ſ", "İDž", ">­", "　", "(<", "META", "_START", ">Ａt", "<"]} +{"text": "　-''D字9", "tokens": 10, "pieces": ["　", "-''", "D", "字", "9"]} +{"text": "(\r 's👍🏽é>\t…'Tعḍ̇'T<ꟲ'reDž½- 'ſ(t㍿\r \n-́", "tokens": 39, "pieces": ["(\r", " ", "'s", "👍🏽", "é", ">", "\t", "…", "'Tعḍ̇'T", "<ꟲ're", "Dž", "½", "-", " ", "'ſ", "(t", "㍿\r", " \n", "-́"]} +{"text": "­åEOT<　ßå😀🏽t'ſt३\r\n\r\n'Reع \r
ꟲs<|fim_prefix|> ­", "tokens": 38, "pieces": ["­å", "EOT", "<", "　", "ßå", "😀🏽", "t'ſ", "t", "३", "\r\n\r\n", "'Reع", " \r", "
ꟲs", "<|", "fim", "_prefix", "|>", " ­"]} +{"text": "m\nEOT\u000b'VE,'D३㋿t३<|endoftext|>fi㍿ꟲmEOT漢㍿Z\"\rfi\r\nＡ\r漢İ👍🏽३,", "tokens": 50, "pieces": ["m", "\n", "EOT", "\u000b", "'VE", ",'", "D", "३", "㋿t", "३", "<|", "endoftext", "|>", "fi", "㍿ꟲm", "EOT漢", "㍿Z", "\"\r", "fi", "\r\n", "Ａ", "\r", "漢", "İ", "👍🏽", "३", ","]} +{"text": " 👍🏽\u000b-\r\"\r\nsＡ Dž'VE", "tokens": 20, "pieces": [" ", " 👍🏽", "\u000b", "-<", "EOT", ">\r", "\"\r\n", "s", "Ａ", " ", " Dž'VE"]} +{"text": "ꟲ--,eع㍿…ḍ̇\nZ \n\"sع \r\nſ'D
Z'M(", "tokens": 27, "pieces": ["ꟲ", "--,", "eع", "㍿", "…ḍ̇", "\n", "Z", " \n", "\"sع", " \r\n", "ſ'D", "
Z'M", "("]} +{"text": "'M'Re Ａ𐞁ḍ̇,é\r𐞁…　Z.\na٣٤٥٦12345678é'T", "tokens": 35, "pieces": ["'M'Re", " ", " Ａ𐞁ḍ̇", ",é", "\r", "𐞁", "…", "　Z", ".\n", "a", "٣٤٥", "٦12", "345", "678", "é'T"]} +{"text": "'S'T́", "tokens": 3, "pieces": ["'S'T", "́"]} +{"text": "0😀🏽­𐞁sſé 'MⅣEOT\u000b", "tokens": 20, "pieces": ["0", "😀🏽­", "𐞁sſé", " ", " '", "M", "Ⅳ", "EOT", "\u000b"]} +{"text": "Z<fi're9…fi\n!!\r!å9\r\n\r\nZEOTß\"12345678…㍿", "tokens": 28, "pieces": ["Z", "<fi're", "9", "…fi", "\n", "!!\r", "!å", "9", "\r\n\r\n", "ZEOTß", "\"", "123", "456", "78", "…", "㍿"]} +{"text": "\t\r\n\r\n　Ⅳ \u000bꟲt'Sſꟲ'ſ\r\n
 ㋿'M\u000bfi㍿\né0!!㍿", "tokens": 42, "pieces": ["\t\r\n\r\n", "　", "Ⅳ", " ", "\u000bꟲt'S", "ſ", "ꟲ'ſ", "\r\n", "
", " ㋿'", "M", "\u000bfi", "㍿\n", "é", "0", "!!㍿"]} +{"text": "́Ａ👍🏽 \n'Tع!　're'reİ'Dß,ß\u000b🙂㍿!!㍿
'T½t'M'll<|fim_prefix|>s'!t㋿🙂‍é\r\n\r\n㍿", "tokens": 55, "pieces": ["́", "Ａ", "👍🏽", " \n", "'Tع", "!", "　", "'re're", "İ'D", "ß", ",ß", "\u000b", "🙂㍿!!㍿", "
", "'T", "½", "t'M", "'ll", "<|", "fim", "_prefix", "|>", "s", "'<", "META", "_START", ">!", "t", "㋿🙂‍", "é", "\r\n\r\n", "㍿"]} +{"text": "#$%ßEOT…½Dž　٣٤٥٦ \n'M,𐞁㋿s　", "tokens": 27, "pieces": ["#$%", "ß", "EOT", "…", "½", "Dž", "　", "٣٤٥", "٦", " \n", "'M", ",𐞁", "㋿s", "　"]} +{"text": "0' \nḍ̇ßꟲ<\"
<|endoftext|>漢́amEOT‍Dž 🙂👍🏽 a", "tokens": 37, "pieces": ["0", "'", " \n", "ḍ̇ßꟲ", "<\"", "
", "<|", "endoftext", "|><", "META", "_START", ">漢́am", "EOT", "‍Dž", " ", " 🙂👍🏽", " a"]} +{"text": "½'ſå\t­'Daꟲ👍🏽fi㋿!!EOTé𐞁m👍🏽
 's'Mß're's \n 's\"\u000b'T\r\n", "tokens": 47, "pieces": ["½", "'ſå", "\t", "­'", "Daꟲ", "👍🏽", "fi", "㋿!!", "EOTé𐞁m", "👍🏽", "
", " ", "'s'M", "ß're", "'s", " \n", " ", " '", "s", "\"", "\u000b", "'T", "\r\n"]} +{"text": "­ع́ ٣٤٥٦'Sſ 9٣٤٥٦'T 漢٣٤٥٦-'T!!EOT​<|endoftext|>Ⅳ0\" ꟲ㋿
'ſ<'Mع", "tokens": 56, "pieces": ["­ع́", " ", "٣٤٥", "٦", "'Sſ", " ", "9٣٤", "٥٦", "'T", " 漢", "٣٤٥", "٦", "-'", "T", "!!", "EOT", "​<|", "endoftext", "|>", "Ⅳ0", "\"", " ꟲ", "㋿", "
", "'ſ", "<'", "Mع"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": "-!,'ll
́　　.,.İ'\t#$%s're'VE(­\u000b!​<|endoftext|>'VE'D\r\n\u000bＡ'SDž", "tokens": 47, "pieces": ["-!,'", "ll", "
́", "　", "　", ".,.", "İ", "'", "\t", "#$%", "s", "'", "re'VE", "(­", "\u000b", "!<", "EOT", ">​<|", "endoftext", "|>'", "VE'D", "\r\n", "\u000bＡ'S", "Dž"]} +{"text": "
\u000b\tß(<|fim_prefix|>-　 ㍿😀🏽'll'ſ", "tokens": 22, "pieces": ["
\u000b", "\tß", "(<|", "fim", "_prefix", "|>-", "　", " ㍿😀🏽'", "ll'ſ"]} +{"text": "½'३😀🏽((…'Ts>å", "tokens": 13, "pieces": ["½", "'", "३", "😀🏽((", "…", "'Ts", ">å"]} +{"text": "e½Z👍🏽ḍ̇12345678<|fim_prefix|>३\"s>\t!!'re'llDž<|endoftext|>ſ́'ll㋿0d😀🏽Dž", "tokens": 51, "pieces": ["e", "½", "Z", "👍🏽", "ḍ̇", "", "123", "456", "78", "<|", "fim", "_prefix", "|>", "३", "\"s", ">", "\t", "!!'", "re'll", "Dž", "<|", "endoftext", "|>", "ſ́'ll", "㋿", "0", "d", "😀🏽", "Dž"]} +{"text": "ḍ̇字'ſ<|fim_prefix|>字t'S#$%ßa'M　'T\u000b'M­ＡⅣ'< eé!!fi'reéé !ꟲ!!", "tokens": 48, "pieces": ["ḍ̇字'ſ", "<|", "fim", "_prefix", "|>", "字t'S", "#$%", "ßa'M", "　", "'T", "\u000b", "'M", "­Ａ", "Ⅳ", "'<", " ", " eé", "!!", "fi're", "éé", " ", "!<", "EOT", ">ꟲ", "!!"]} +{"text": "İ㋿🙂\r٣٤٥٦عfi३Ⅳ'Re\u000b'VE‍🙂'D ३\t'ſ½🙂9'll٣٤٥٦ſ­å'S字.eéEOT\n'T !é字", "tokens": 52, "pieces": ["İ", "㋿🙂\r", "٣٤٥", "٦", "عfi", "३Ⅳ", "'Re", "\u000b", "'VE", "‍🙂'", "D", " ", "३", "\t", "'ſ", "½", "🙂", "9", "'ll", "٣٤٥", "٦", "ſ", "­å'S", "字", ".eé", "EOT", "\n", "'T", " ", " !", "é字"]} +{"text": "́fi.'S\t'VEDž㍿ \"<ꟲt-𐞁", "tokens": 23, "pieces": ["́fi", ".'", "S", "\t", "'VEDž", "㍿", " ", " \"<", "ꟲt", "-𐞁"]} +{"text": "9Ａ\u000bZＡéſſ\r\n\r\nꟲe !!\r\n\r\nİ", "tokens": 23, "pieces": ["9", "Ａ", "\u000bZＡé", "ſſ", "\r\n\r\n", "ꟲe", " ", "!!\r\n\r\n", "İ"]} +{"text": "fi½", "tokens": 2, "pieces": ["fi", "½"]} +{"text": "'s 　…㋿'D'Re　'S\r\n's 9's­!İsEOTs12345678İ's'S😀🏽　fié", "tokens": 48, "pieces": ["'s", " 　", "…", "㋿<", "é", "​,>'", "D'Re", "", "　", "'S", "\r\n", "'s", " ", " ", "9", "'s", "­!", "İs", "EOTs", "123", "456", "78", "İ's", "'S", "😀🏽", "　fié"]} +{"text": "!!\r\n\r\n'　ß
>…<|endoftext|>'D<|endoftext|>字!!.😀🏽'sß­'Re'VE's\t 字'ſ.(", "tokens": 43, "pieces": ["!!\r\n\r\n", "'", "　ß", "
", ">", "…", "<|", "endoftext", "|>'", "D", "<|", "endoftext", "|>", "字", "!!.😀🏽'", "sß", "­'", "Re'VE", "'s", "\t", " 字'ſ", ".("]} +{"text": " ع(", "tokens": 3, "pieces": [" ", " ع", "("]} +{"text": "字EOT.\r\nfi!ꟲ>.\r", "tokens": 11, "pieces": ["字", "EOT", ".\r\n", "fi", "!ꟲ", ">.\r"]} +{"text": " <'M0'VE ㍿…👍🏽é>9 漢12345678900EOT\r", "tokens": 31, "pieces": [" ", "<'", "M", "0", "'VE", " ", "㍿", "…", "👍🏽", "é", ">", "9", " ", " 漢", "123", "456", "789", "00", "EOT", "\r"]} +{"text": "'S,'ſ.<|fim_prefix|><|endoftext|>'ſ'sß0Ａ\r9", "tokens": 22, "pieces": ["'S", ",'", "ſ", ".<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "ſ's", "ß", "0", "Ａ", "\r", "9"]} +{"text": "عDžEOT!(👍🏽", "tokens": 9, "pieces": ["ع", "DžEOT", "!(👍🏽"]} +{"text": "<😀🏽'Ａfi \r\n0Dž-'MDž‍0ſ'll''re\n­ \nع٣٤٥٦👍🏽s
ßfi0　३12345678​𐞁å\r\n\r\n", "tokens": 62, "pieces": ["<😀🏽'", "Ａfi", " \r\n", "0", "Dž", "-'", "MDž", "‍<", "META", "_START", ">", "0", "ſ", "'", "ll", "''", "re", "\n", "­", " \n", "ع", "٣٤٥", "٦", "👍🏽", "s", "
ßfi", "0", "　", "३12", "345", "678", "​", "𐞁å", "\r\n\r\n"]} +{"text": "'S-a​'reéß𐞁½ 'SZ漢fi　", "tokens": 17, "pieces": ["'S", "-a", "​'", "reéß𐞁", "½", " '", "SZ漢fi", "　"]} +{"text": "\t㋿'VE<|endoftext|>", "tokens": 13, "pieces": ["\t", "㋿'", "VE", "<|", "endoftext", "|>"]} +{"text": "<#$%#$% a9EOT𐞁a \r\n\r\nt<'s　 \n-ḍ̇fi٣٤٥٦åaådfi 0…-", "tokens": 44, "pieces": ["<#$%#$%", " a", "9", "EOT𐞁a", " \r\n\r\n", "t", "<'", "s", "　 \n", "-ḍ̇fi", "٣٤٥", "٦", "å", "aådfi", " ", " ", "0", "…", "-"]} +{"text": "­😀🏽\"'ع👍🏽12345678's 'll🙂12345678<|fim_prefix|>😀🏽<.'M", "tokens": 36, "pieces": ["­😀🏽\"'<", "EOT", ">ع", "👍🏽", "123", "456", "78", "'s", " ", " '", "ll", "🙂", "123", "456", "78", "<|", "fim", "_prefix", "|>😀🏽<.'", "M"]} +{"text": ". ½.\u000b㍿d𐞁　12345678‍\n'DＡ", "tokens": 21, "pieces": [".", " ", "½", ".", "\u000b", "㍿d𐞁", "　", "123", "456", "78", "‍\n", "'DＡ"]} +{"text": "½a…😀🏽EOT'll-", "tokens": 11, "pieces": ["½", "a", "…", "😀🏽", "EOT'll", "-"]} +{"text": "\tＡ‍(😀🏽 \nعꟲ㍿(a!!㍿", "tokens": 25, "pieces": ["\t", "Ａ", "‍(😀🏽", " \n", "عꟲ", "㍿(", "a", "!!㍿"]} +{"text": "́🙂'T\u000b!!a𐞁\te　\",!!", "tokens": 15, "pieces": ["́", "🙂'", "T", "\u000b", "!!", "a𐞁", "\te", "　", "\",!!"]} +{"text": "\r\n\r\n'll👍🏽ts㋿m#$%.tＡ😀🏽㍿ 'ſعſ12345678‍ſfi>३é's'Ms\t\r\n\r\nDžع", "tokens": 49, "pieces": ["\r\n\r\n", "'ll", "👍🏽", "ts", "㋿m", "#$%.", "t", "Ａ", "😀🏽㍿", " ", " '", "ſعſ", "123", "456", "78", "‍<", "EOT", ">ſfi", ">", "३", "é's", "'Ms", "\t\r\n\r\n", "Džع"]} +{"text": "!<|fim_prefix|>㍿", "tokens": 10, "pieces": ["!<|", "fim", "_prefix", "|>㍿"]} +{"text": "-漢é tm
,åß,'S", "tokens": 11, "pieces": ["-漢é", " tm", "
", ",åß", ",'", "S"]} +{"text": "\r\n\r\n३㍿!!", "tokens": 6, "pieces": ["\r\n\r\n", "३", "㍿!!"]} +{"text": "'D'smⅣé𐞁ꟲ'12345678EOTå'ſ12345678 \n'S㋿ \néⅣع👍🏽́fi
å é'Dſ-Z'", "sm", "Ⅳ", "é𐞁ꟲ", "'", "123", "456", "78", "EOTå'ſ", "123", "456", "78", " \n", "'S", "㋿", " \n", "é", "Ⅳ", "ع", "👍🏽́", "fi", "
å", " é'D", "ſ", "-Z", "<|endoftext|>s½,ꟲ\u000b9", "tokens": 28, "pieces": ["etſ", "9", "'re", "㋿\r\n", "½0३", "<|", "endoftext", "|>", "s", "½", ",ꟲ", "\u000b", "9"]} +{"text": "å\r\nsſ字Ⅳ​Ａ­٣٤٥٦𐞁!EOTⅣfi\nßt's­'ll…d", "tokens": 34, "pieces": ["å", "\r\n", "sſ字", "Ⅳ", "​Ａ", "­", "٣٤٥", "٦", "𐞁", "!EOT", "Ⅳ", "fi", "\n", "ßt's", "­'", "ll", "…d"]} +{"text": " ḍ̇\r\n>\r\n'll\r\n½Ⅳ㋿,Dž'MDž㋿Ａ'VE\"\t'så…㋿é", "tokens": 38, "pieces": [" ", " ḍ̇", "\r\n", ">\r\n", "'ll", "\r\n", "½Ⅳ", "㋿,", "Dž'M", "Dž", "㋿Ａ'VE", "\"", "\t", "'så", "…", "㋿é"]} +{"text": "
‍ésⅣ👍🏽s\r\n\r\n'VE\nḍ̇EOTd.  !!'T", "tokens": 26, "pieces": ["
", "‍és", "Ⅳ", "👍🏽", "s", "\r\n\r\n", "'VE", "\n", "ḍ̇", "EOTd", ".", "  ", " !!'", "T"]} +{"text": "'Sḍ̇.\r\n\r\n­३　­!㍿", "tokens": 13, "pieces": ["'Sḍ̇", ".\r\n\r\n", "­", "३", "　", "­!㍿"]} +{"text": "'VE‍㍿\naå'S.", "tokens": 11, "pieces": ["'VE", "‍㍿\n", "aå'S", "."]} +{"text": "åDžſ'T😀🏽\r\n\r\n\n'SEOT\r\n\r\nḍ̇İ9 ㋿'Re㋿Dž''re'Ma㋿'T!a,\"\r\n\r\n\na'VE0'Reé", "tokens": 51, "pieces": ["å", "Džſ'T", "😀🏽\r\n\r\n\n", "'SEOT", "\r\n\r\n", "ḍ̇", "İ", "9", " ", "㋿'", "Re", "㋿Dž", "''", "re'M", "a", "㋿'", "T", "!a", ",\"\r\n\r\n\n", "a'VE", "0", "'Reé"]} +{"text": "tt\r\nDžm'a,<🙂\u000b­.‍9EOT<\rع\r
9! \tİع٣٤٥٦👍🏽Z'ſꟲåİ", "tokens": 48, "pieces": ["tt", "\r\n", "Džm", "'<", "EOT", ">a", ",<🙂", "\u000b", "­.‍", "9", "EOT", "<\r", "ع", "\r", "
", "9", "!", " ", "\tİع", "", "٣٤٥", "٦", "👍🏽", "Z'ſ", "ꟲå", "İ"]} +{"text": "\n\u000bfi<|fim_prefix|>, t'M-a\r\n\r\néⅣ'reＡfi　\r\n", "tokens": 25, "pieces": ["\n", "\u000bfi", "<|", "fim", "_prefix", "|>,", " ", " t'M", "-a", "\r\n\r\n", "é", "Ⅳ", "'re", "Ａfi", "　\r\n"]} +{"text": "'sع!! \nعm👍🏽fiß ́Ａ(<㋿'ſ'D!\t漢's­🙂,", "tokens": 31, "pieces": ["'sع", "!!", " \n", "عm", "👍🏽", "fiß", " ́", "Ａ", "(<㋿'", "ſ'D", "!", "\t漢's", "­🙂,"]} +{"text": "́<|fim_prefix|>å12345678>½'S‍ع😀🏽'T‍12345678\nfi", "tokens": 32, "pieces": ["́", "<|", "fim", "_prefix", "|>", "å", "123", "456", "78", ">", "½", "'S", "‍ع", "😀🏽'", "T", "‍<", "EOT", ">", "123", "456", "78", "\n", "fi"]} +{"text": "\r\n\r\n DžZfi'ſ<|endoftext|> \n漢09\r'VE😀🏽Z.ß'ſ, \n𐞁\u000b'Re<|endoftext|>éß", "tokens": 51, "pieces": ["\r\n\r\n", "", " DžZfi'ſ", "<|", "endoftext", "|>", " \n", "漢", "09", "\r", "'VE", "😀🏽", "Z", ".ß'ſ", ",", " \n", "𐞁", "\u000b", "'Re", "<|", "endoftext", "|>", "éß"]} +{"text": "'D㍿'re'ſ's\naé​d>\ré३Ａt​ \n-½ \nfiİfi字㍿'VE'D!Z\r\n\u000b'ſ", "tokens": 46, "pieces": ["'D", "㍿'", "re'ſ", "'s", "\n", "aé", "​", "d", ">\r", "é", "३", "Ａt", "​", " \n", "-", "½", " \n", "fi", "İfi字", "㍿'", "VE'D", "!Z", "\r\n", "\u000b", "'ſ"]} +{"text": "🙂'VEſ👍🏽İ're­字( \n", "tokens": 13, "pieces": ["🙂'", "VEſ", "👍🏽", "İ're", "­字", "(", " \n"]} +{"text": "a!عDža​ 😀🏽…åع'M­d‍\"", "tokens": 19, "pieces": ["a", "!عDža", "​", " 😀🏽", "…åع'M", "­d", "‍\""]} +{"text": "9\r\n\"<> \n…(Ⅳ'M'S
'VE㋿>­٣٤٥٦'M0…字…", "tokens": 33, "pieces": ["9", "\r\n", "\"<<", "EOT", ">>", " \n", "…", "(", "Ⅳ", "'M'S", "
", "'VE", "㋿>­", "٣٤٥", "٦", "'M", "0", "…字", "…"]} +{"text": "'Re!­'३　d​-\u000b", "tokens": 10, "pieces": ["'Re", "!­'", "३", "　d", "​-", "\u000b"]} +{"text": "½'re'VE…Z9ꟲ", "tokens": 11, "pieces": ["½", "'re'VE", "…Z", "9", "ꟲ"]} +{"text": "åⅣd👍🏽Ⅳ㍿dd,३́'Re<|fim_prefix|>\u000b \n>.", "tokens": 30, "pieces": ["å", "Ⅳ", "d", "👍🏽", "Ⅳ", "㍿dd", ",", "३", "́'Re", "<|", "fim", "_prefix", "|>", "\u000b", "", " \n", ">."]} +{"text": "𐞁İ㋿\u000b'Ḿ \nZ‍é't0éZＡ9३字fi", "tokens": 26, "pieces": ["𐞁", "İ", "㋿", "\u000b", "'Ḿ", " \n", "Z", "‍é't", "0", "é", "ZＡ", "9३", "字fi"]} +{"text": "'ll'Sİ(́s­字\r\n\r\nEOT'👍🏽'İ…>é'll 𐞁ݽ字'T'D🙂…sⅣé٣٤٥٦\" e\n", "tokens": 51, "pieces": ["'ll'S", "İ", "(́s", "­字", "\r\n\r\n", "EOT", "'👍🏽'", "İ", "…", ">é'll", " ", " 𐞁", "İ", "½", "字'T", "'D", "🙂", "…s", "Ⅳ", "é", "٣٤٥", "٦", "\"", " e", "\n"]} +{"text": " 字a(\r\nعⅣfi'M!d😀🏽12345678", "tokens": 17, "pieces": [" ", " 字a", "(\r\n", "ع", "Ⅳ", "fi'M", "!d", "😀🏽", "123", "456", "78"]} +{"text": "㍿<ß'VE>d's३'llm…​(ꟲ!'reé漢12345678're\u000b're'M'D", "tokens": 31, "pieces": ["㍿<", "ß'VE", ">d's", "३", "'llm", "…", "​(", "ꟲ", "!'", "reé漢", "123", "456", "78", "'re", "\u000b", "'re'M", "'D"]} +{"text": "e​ae!<ع", "tokens": 14, "pieces": ["e", "​", "a", "e", "!<", "ع"]} +{"text": "​字e𐞁EOTaſḍ̇#$%　\"0\u000bſßm字… \n<|fim_prefix|>åEOT", "tokens": 41, "pieces": ["​字e𐞁", "EOTaſḍ̇", "#$%", "　", "\"", "0", "\u000bſßm", "字", "… \n", "<|", "fim", "_prefix", "|>", "å", "EOT"]} +{"text": "#$%!!Dž½३'VE.!e's Dž#$%Z'D​-🙂३'VEs'S,", "tokens": 32, "pieces": ["#$%!!", "Dž", "½३", "'VE", ".!", "e's", " ", "Dž", "#$%", "Z'D", "​-🙂", "३", "'VEs'S", ","]} +{"text": "-'VE<|fim_prefix|>'<ꟲfi'T٣٤٥٦عſ字字fi\t́字", "tokens": 26, "pieces": ["-'", "VE", "<|", "fim", "_prefix", "|>'<", "ꟲfi'T", "٣٤٥", "٦", "عſ字字fi", "\t́字"]} +{"text": "<'M's<|endoftext|>d(👍🏽٣٤٥٦字𐞁.t<漢'ſ\n​é<|endoftext|>'re9𐞁", "tokens": 46, "pieces": ["<'", "M's", "<|", "endoftext", "|>", "d", "(👍🏽", "٣٤٥", "٦", "字𐞁", ".t", "<漢'ſ", "\n", "​é", "<|", "endoftext", "|>'", "re", "9", "𐞁"]} +{"text": "\"ع३EOT", "tokens": 5, "pieces": ["\"ع", "३", "EOT"]} +{"text": "'ś\r!.,'M", "tokens": 7, "pieces": ["'ś", "\r", "!.,'", "M"]} +{"text": ">,<|endoftext|>9!!㋿½Ⅳ\r\n\r\nd'T9𐞁İİ\n­㍿​EOT‍\"'M><|endoftext|> \n\r", "tokens": 46, "pieces": [">,<|", "endoftext", "|>", "9", "!!㋿", "½Ⅳ", "\r\n\r\n", "d'T", "9", "𐞁", "İİ", "\n", "­㍿​", "EOT", "‍\"'", "M", "><|", "endoftext", "|>", " \n\r"]} +{"text": "s漢漢　٣٤٥٦", "tokens": 8, "pieces": ["s漢漢", "　", "٣٤٥", "٦"]} +{"text": "'ſ \n字‍\rḍ̇Dž\"EOT'ſⅣ\"'M'S\u000b!!‍ßm's.fi­'ReⅣ", "tokens": 36, "pieces": ["'ſ", " \n", "字", "‍\r", "ḍ̇", "Dž", "\"EOT'ſ", "Ⅳ", "\"<", "EOT", ">'", "M'S", "\u000b", "!!‍", "ßm's", ".fi", "­'", "Re", "Ⅳ"]} +{"text": "t#$%#$%''s\r\n\r\nſZ\"<|endoftext|><٣٤٥٦<|fim_prefix|>!!-'Sé😀🏽٣٤٥٦d字#$%0s'Re漢é'Reeḍ̇s㋿ع", "tokens": 61, "pieces": ["t", "#$%#$%''", "s", "\r\n\r\n", "ſ", "Z", "\"<|", "endoftext", "|><", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>!!-'", "Sé", "😀🏽", "٣٤٥", "٦", "d字", "#$%", "0", "s'Re", "漢é", "'", "Reeḍ̇s", "㋿ع"]} +{"text": "\ta #$%,a\u000bt,Dža'll'Re-efi𐞁ſ٣٤٥٦
,éa㍿12345678👍🏽\r\n'D12345678d㋿12345678åEOT", "tokens": 56, "pieces": ["\ta", " #$%,", "a", "\u000bt", ",Dža'll", "'Re", "-efi𐞁ſ", "٣٤٥", "٦", "
", ",éa", "㍿", "123", "456", "78", "👍🏽\r\n", "'D", "123", "456", "78", "d", "㋿", "123", "456", "78", "å", "EOT"]} +{"text": "éḍ̇'T👍🏽　‍ Ⅳ'ReZ…\"Ⅳİ!12345678ſd, \n 漢s\r\n\r\nDžå<|endoftext|>t(😀🏽 \n'Re!eé'red", "tokens": 57, "pieces": ["éḍ̇'T", "👍🏽", "　", "‍", " ", " ", "Ⅳ", "'Re", "Z", "…", "\"", "Ⅳ", "İ", "!", "123", "456", "78", "ſd", ",", " \n", " 漢s", "\r\n\r\n", "Džå", "<|", "endoftext", "|>", "t", "(😀🏽", " \n", "'Re", "!eé're", "d"]} +{"text": "İ
‍ḍ̇ꟲfi…", "tokens": 12, "pieces": ["İ", "
", "‍ḍ̇ꟲfi", "…"]} +{"text": "!!㍿́'ll𐞁㋿\r", "tokens": 15, "pieces": ["!!㍿́'", "ll𐞁", "㋿\r"]} +{"text": " 0<|fim_prefix|>'ll㋿'så­t😀🏽३'Taå\r\nZ㍿12345678𐞁​
ꟲİ ꟲ\r\n", "tokens": 46, "pieces": [" ", " ", "0", "<|", "fim", "_prefix", "|>'", "ll", "㋿'", "så", "­t", "😀🏽", "३", "'Taå", "\r\n", "Z", "㍿", "123", "456", "78", "𐞁", "​", "
ꟲ", "İ", " ꟲ", "\r\n"]} +{"text": "e𐞁㍿é 12345678'sß　12345678meZ㋿字0'Re!'MZ-ſ'T𐞁", "tokens": 41, "pieces": ["e𐞁", "㍿<", "META", "_START", ">é", " ", "123", "456", "78", "'sß", "　", "123", "456", "78", "me", "Z", "㋿字", "0", "'Re", "!'", "MZ", "-ſ'T", "𐞁"]} +{"text": " 0𐞁Z…#$%<|endoftext|>'D …👍🏽12345678á<|fim_prefix|>'re!!'Sfi \n😀🏽's٣٤٥٦'s'T<(", "tokens": 54, "pieces": [" ", "0", "𐞁", "Z", "…", "#$%<|", "endoftext", "|>'", "D", " ", "…", "👍🏽", "123", "456", "78", "á", "<|", "fim", "_prefix", "|>'", "re", "!!'", "Sfi", " \n", "😀🏽'", "s", "٣٤٥", "٦", "'s'T", "<("]} +{"text": "👍🏽Ａ'reé'reİ\n३字字,åZt३ꟲ<|endoftext|> 'Mꟲ", "tokens": 36, "pieces": ["👍🏽", "Ａ're", "é're", "İ", "\n", "३", "字", "字", ",å", "Zt", "३", "ꟲ", "<|", "endoftext", "|>", " ", "'Mꟲ"]} +{"text": " 　‍…\r漢ع're­ع \né'S👍🏽", "tokens": 17, "pieces": [" ", "　", "‍", "…\r", "漢ع're", "­ع", " \n", "é'S", "👍🏽"]} +{"text": "\r#$%<|endoftext|>\u000b𐞁'T𐞁#$%e's \u000b…!'T́漢㍿>\r>👍🏽\r
å9,fi'VE‍\r㍿ < 𐞁<|fim_prefix|>漢ſ", "tokens": 71, "pieces": ["\r", "#$%<|", "endoftext", "|>", "\u000b𐞁'T", "𐞁", "#$%", "e's", " \u000b", "…", "!'", "T́漢", "㍿>\r", ">👍🏽\r", "
å", "9", ",fi'VE", "‍\r", "㍿", " ", "<", " ", " 𐞁", "<|", "fim", "_prefix", "|>", "漢ſ"]} +{"text": "ſ9ع'Reé३'reſ'll\t'ſ㋿Ⅳ'Retfi­'re'VE​", "tokens": 26, "pieces": ["ſ", "9", "ع'Re", "é", "३", "'reſ'll", "\t", "'ſ", "㋿", "Ⅳ", "'Retfi", "­'", "re'VE", "​"]} +{"text": "EOTd90'llＡ.'ll\" ", "tokens": 10, "pieces": ["EOTd", "90", "'ll", "Ａ", ".'", "ll", "\"", " "]} +{"text": "dＡ٣٤٥٦½td<­३å'Té12345678㍿字>\t'D <|fim_prefix|>ꟲ\n'MＡ'reå　'Dß🙂å 9", "tokens": 49, "pieces": ["d", "Ａ", "٣٤٥", "٦½", "td", "<­", "३", "å'T", "é", "123", "456", "78", "㍿字", ">", "\t", "'D", " ", "<|", "fim", "_prefix", "|>", "ꟲ", "\n", "'MＡ're", "å", "　", "'Dß", "🙂å", " ", "9"]} +{"text": "'İDž\t½(漢", "tokens": 8, "pieces": ["'İDž", "\t", "½", "(漢"]} +{"text": ".ß𐞁‍İ \r\n", "tokens": 16, "pieces": [".", "ß𐞁", "‍", "İ", " \r\n"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'ſ ㋿ée👍🏽9\rEOT'\n<|endoftext|> ꟲ'S", " ꟲ'S", " \n​😀🏽 \nſDžſ'VE٣٤٥٦t\t<>", "tokens": 49, "pieces": ["'re", "🙂ḍ̇", "9", "​\n", "㋿'", "S", "­", "9", "éd漢", "
e", "İ", "", " \n", "​😀🏽", " \n", "ſ", "Džſ'VE", "٣٤٥", "٦", "<", "META", "_START", ">t", "\t", "<>"]} +{"text": " #$%'D🙂9,…'re", "tokens": 10, "pieces": [" ", "#$%'", "D", "🙂", "9", ",", "…", "'re"]} +{"text": "ſß!<", "tokens": 4, "pieces": ["ſß", "!<"]} +{"text": "ßd'Re", "tokens": 3, "pieces": ["ßd'Re"]} +{"text": "fi( 'M\t''ReEOTéa\r\n㍿𐞁½ ३ é😀🏽\u000b>😀🏽'Dع'Dé\"İ ", "tokens": 42, "pieces": ["fi", "(", " ", "'M", "\t", "''", "Re", "EOTéa", "\r\n", "㍿𐞁", "½", " ", "३", " é", "😀🏽", "\u000b", ">😀🏽'", "Dع'D", "é", "\"İ", " "]} +{"text": "㋿́!!-😀🏽'0ع\t'ſ'D!!#$%<é漢fi<­", "tokens": 40, "pieces": ["㋿́", "!!-😀🏽'", "0", "ع", "", "\t", "'ſ'D", "!!<", "åå", " ", " '", "D", "#$%<", "é漢fi", "<­"]} +{"text": ".'Re­>", "tokens": 4, "pieces": [".'", "Re", "­>"]} +{"text": " <|endoftext|>​ \n-", "tokens": 10, "pieces": [" <|", "endoftext", "|>​", " \n", "-"]} +{"text": "'ſ!m漢😀🏽m㋿ſ9( \n𐞁Zꟲ'llé'M‍'s#$%(åḍ̇㍿٣٤٥٦'Re \n\r\nZ", "tokens": 49, "pieces": ["'ſ", "!m漢", "😀🏽", "m", "㋿ſ", "9", "(", " \n", "𐞁Zꟲ'll", "é'M", "‍'", "s", "#$%(", "åḍ̇", "㍿", "٣٤٥", "٦", "'Re", " \n\r\n", "Z"]} +{"text": "t'ReEOTdＡ'M\r\n'D'VE\r\n'ſſ𐞁㋿'M'ſt>字", "tokens": 32, "pieces": ["t'Re", "EOTd", "Ａ", "'", "M", "\r\n", "'D'VE", "\r\n", "'ſſ𐞁", "㋿'", "M'ſ", "t", ">字"]} +{"text": "'Mſ'T३d​\r\n<|fim_prefix|>'ſḍ̇'VE-Z-'Sİ㋿\r­ⅣEOT㋿\r\n\né😀🏽㍿'D>'M", "tokens": 47, "pieces": ["'Mſ'T", "३", "d", "​\r\n", "<|", "fim", "_prefix", "|>'", "ſḍ̇'VE", "-Z", "-'", "Sİ", "㋿\r", "­", "Ⅳ", "EOT", "㋿\r\n\n", "é", "😀🏽㍿'", "D", ">'", "M"]} +{"text": "漢　\r
  
'll½字Ⅳm\r\n\r\n,eémDž<|fim_prefix|>! 0​'S're-!!'D'D
ꟲ\"𐞁ḍ̇\u000b‍", "tokens": 59, "pieces": ["漢", "　\r", "
  ", "
", "'ll", "½", "字", "Ⅳ", "m", "\r\n\r\n", ",<", "EOT", ">eém", "Dž", "<|", "fim", "_prefix", "|>!", " ", " ", "0", "​'", "S're", "-!!'", "D", "'", "D", "
ꟲ", "\"𐞁ḍ̇", "\u000b", "‍"]} +{"text": "\t12345678ßd\r\nm'Re​", "tokens": 10, "pieces": ["\t", "123", "456", "78", "ßd", "\r\n", "m'Re", "​"]} +{"text": ".!!㋿ḍ̇éⅣ'red३'Dåt", "tokens": 22, "pieces": [".!!㋿", "ḍ̇é", "Ⅳ", "'red", "", "३", "'Dåt"]} +{"text": "-‍\",
", "tokens": 4, "pieces": ["-‍\",", "
"]} +{"text": "'s'T DžEOT12345678'S", "tokens": 11, "pieces": ["'s'T", " DžEOT", "123", "456", "78", "'S"]} +{"text": "­㋿\r字\r\n\r\n㍿ (​é\r\n", "tokens": 20, "pieces": ["­㋿<", "META", "_START", ">\r", "字", "\r\n\r\n", "㍿", " ", "(​", "é", "\r\n"]} +{"text": "字!9", "tokens": 3, "pieces": ["字", "!", "9"]} +{"text": "e\nḍ̇字Ａ-٣٤٥٦\"​
\"'DZⅣ𐞁'S<|endoftext|>> (İ \nm,><漢", "tokens": 48, "pieces": ["e", "\n", "ḍ̇字", "Ａ", "-", "٣٤٥", "٦", "\"<", "EOT", ">​", "
", "\"'", "DZ", "", "Ⅳ", "𐞁'S", "<|", "endoftext", "|>>", " ", " (", "İ", " \n", "m", ",<", "META", "_START", ">><", "漢"]} +{"text": "é<‍㍿漢…\u000b​9'M字漢 \r\n㍿ꟲ<'så\rm漢fi‍!!㋿\"٣٤٥٦ſ", "tokens": 28, "pieces": ["é", "Ⅳ", "…", "!!", "m", "(d'VE", "漢", ">å", "\r", "m漢fi", "‍!!㋿\"", "٣٤٥", "٦", "ſ"]} +{"text": "… \nßⅣ12345678
ḍ̇mعßＡ​İ-é12345678🙂'M‍ 'M9EOT\r\n'll\r\n\r\n٣٤٥٦ſ　", "tokens": 43, "pieces": ["… \n", "ß", "Ⅳ12", "345", "678", "
ḍ̇mعß", "Ａ", "​İ", "-é", "123", "456", "78", "🙂'", "M", "‍", " ", " '", "M", "9", "EOT", "\r\n", "'ll", "\r\n\r\n", "٣٤٥", "٦", "ſ", "　"]} +{"text": "'ll🙂\rꟲ #$%, 'Re½​㋿…'D漢ꟲ'SＡＡ e\t\tḍ̇'reİ👍🏽e😀🏽fi🙂s, 'VEt!", "tokens": 55, "pieces": ["'ll", "🙂\r", "ꟲ", " ", "#$%,", " '", "Re", "½", "​㋿", "…", "'D漢ꟲ'S", "ＡＡ", " ", " e", "\t", "\tḍ̇'re", "İ", "👍🏽", "e", "😀🏽<", "EOT", ">fi", "🙂s", ",", " ", " '", "VEt", "!"]} +{"text": "漢ß\t​", "tokens": 4, "pieces": ["漢ß", "\t", "​"]} +{"text": "　​må12345678#$%<|fim_prefix|>½Ⅳ㋿字'D字­!é\rꟲ'T", "tokens": 43, "pieces": ["", "　", "​må", "123", "456", "78", "#$%<|", "fim", "_prefix", "|>", "½", "<", "META", "_START", ">", "Ⅳ", "㋿字'D", "字", "­!", "é", "\r", "ꟲ'T"]} +{"text": "㍿
'D9𐞁½dß\r\n\r\n\u000bEOTd'sDžså'M<|endoftext|>ZDž 
.é'Re'TZ \n𐞁Ⅳ\t", "tokens": 57, "pieces": ["㍿", "
", "'D", "9", "𐞁", "½", "dß", "\r\n\r\n", "\u000bEOTd", "'", "s", "Džså'M", "<|", "endoftext", "|>", "ZDž", " ", "
", ".é'Re", "'TZ", " \n", "𐞁", "", "Ⅳ", "\t"]} +{"text": "EOTm'll're'reé ­　m字,<|fim_prefix|>'Re'tfi‍'VE🙂0é'M'VEt' ㋿-\n
'ſ'ſ‍9EOT𐞁'S", "tokens": 50, "pieces": ["EOTm'll", "'re're", "é", " ­", "　m字", ",<|", "fim", "_prefix", "|>'", "Re't", "fi", "‍'", "VE", "🙂", "0", "é'M", "'VEt", "'", " ㋿-\n", "
", "'ſ'ſ", "‍", "9", "EOT𐞁'S"]} +{"text": "Ⅳ#$%ååß 'VE0fi㋿½'T'Re0-.12345678'>'s.🙂>ꟲ…", "tokens": 36, "pieces": ["Ⅳ", "#$%", "ååß", " ", "'VE", "0", "fi", "㋿", "½", "'T'Re", "0", "-.", "123", "456", "78", "'>'", "s", ".🙂>", "ꟲ", "…"]} +{"text": "<|endoftext|>fi-㍿'T​", "tokens": 15, "pieces": ["<|", "endoftext", "|>", "fi", "-㍿'", "T", "​"]} +{"text": "İſ٣٤٥٦09字'­#$%\t'VE's'­e\"३\r \n<|fim_prefix|>s㋿Dž'T ", "tokens": 37, "pieces": ["İſ", "٣٤٥", "٦09", "字", "'­#$%", "\t", "'VE's", "'­", "e", "\"", "३", "\r \n", "<|", "fim", "_prefix", "|>", "s", "㋿Dž'T", " "]} +{"text": "Ａ​Ａſſé-12345678.\u000b-　३½s'Tḍ̇#$%(ع 
ß\nDž'D'MDž 'seß", "tokens": 41, "pieces": ["Ａ", "​Ａ", "ſſé", "-", "123", "456", "78", ".", "\u000b", "-", "　", "३½", "s'T", "ḍ̇", "#$%(", "ع", " ", "
ß", "\n", "Dž'D", "'MDž", " ", "'seß"]} +{"text": "- \né'Sa'VEſ漢عſ𐞁..EOT<字,'s'ſ<\r\n\r\nt٣٤٥٦ꟲ\r\n\r\n👍🏽é­字9", "tokens": 47, "pieces": ["-", " \n", "é'S", "a'VE", "ſ漢عſ𐞁", "..", "EOT", "<字", ",'", "s'ſ", "<\r\n\r\n", "t", "٣٤٥", "٦", "ꟲ", "\r\n\r\n", "👍🏽", "é", "­字", "9", ""]} +{"text": "9\n \n　'D,Dž'ع<<|fim_prefix|><|fim_prefix|>​EOT>İ'!ع字éée字('ll‍👍🏽\t'VEḍ̇字🙂a字 ,", "tokens": 54, "pieces": ["9", "\n \n", "　", "'D", ",Dž", "'ع", "<<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><", "EOT", ">​", "EOT", ">İ", "'!", "ع字éée字", "('", "ll", "‍👍🏽", "\t", "'VEḍ̇字", "🙂a字", " ", ","]} +{"text": "å<'re<|endoftext|>½ \n<|endoftext|>\r\n\r\n٣٤٥٦fi ​\r0'reZ \nEOTDž. ㍿­\r\n#$%İ, 9 \n\"'M'TＡ", "tokens": 55, "pieces": ["å", "<'", "re", "<|", "endoftext", "|>", "½", " \n", "<|", "endoftext", "|>\r\n\r\n", "٣٤٥", "٦", "fi", " ", " ​\r", "0", "'re", "Z", " \n", "EOTDž", ".", " ", "㍿­\r\n", "#$%", "İ", ",", " ", " ", "9", " \n", "\"'", "M'T", "Ａ"]} +{"text": "\u000b<|endoftext|>\u000b㋿'s३ſ‍.'VE
\u000ba㍿Ⅳt​'Re\r ꟲ", "tokens": 43, "pieces": ["\u000b", "<|", "endoftext", "|>", "\u000b", "㋿'", "s", "३", "ſ", "‍.'", "VE", "
", "\u000ba", "㍿", "Ⅳ", "t", "​'", "Re", "\r", " ꟲ", ""]} +{"text": "EOT#$%😀🏽9㋿'T'D
<|endoftext|>'M,", "tokens": 24, "pieces": ["EOT", "#$%😀🏽", "9", "㋿'", "T'D", "
", "<|", "endoftext", "|>'", "M", ","]} +{"text": "'ſ ", "tokens": 6, "pieces": ["'ſ", "", " "]} +{"text": "éå'M字
>­​dé\r\n\r\n½\r\n, \u000bmfi.'llſ👍🏽 \n ع​t …é'T12345678fi('t\n½", "tokens": 42, "pieces": ["éå'M", "字", "
", ">­​", "dé", "\r\n\r\n", "½", "\r\n", ",", " ", "\u000bmfi", ".'", "llſ", "👍🏽", " \n", " ع", "​t", " ", "…é'T", "123", "456", "78", "fi", "('", "t", "\n", "½"]} +{"text": "٣٤٥٦å👍🏽t'M'!'ſm'Ś12345678…(ſ\u000b👍🏽漢's'Re'T!", "tokens": 36, "pieces": ["٣٤٥", "٦", "å", "👍🏽", "t'M", "'!'", "ſm'S", "́", "123", "456", "78", "…", "(ſ", "", "\u000b", "👍🏽", "漢's", "'Re'T", "!"]} +{"text": "'s­'S\r", "tokens": 5, "pieces": ["'s", "­'", "S", "\r"]} +{"text": "́㋿ 
Ⅳ字! ‍Dž'T\rß(Ａß漢字🙂.'M'VEDže'VE'ét٣٤٥٦…(9\r\n\r\n'VE\r\n\r\n
Ａ", "tokens": 49, "pieces": ["́", "㋿", " ", "
", "Ⅳ", "字", "!", " ", " ‍", "Dž'T", "\r", "ß", "(Ａß漢字", "🙂.'", "M'VE", "Dže'VE", "'ét", "٣٤٥", "٦", "…", "(", "9", "\r\n\r\n", "'VE", "\r\n\r\n", "
Ａ"]} +{"text": "DžéaＡ㋿ .,é
٣٤٥٦ßⅣ12345678", "tokens": 22, "pieces": ["Džéa", "Ａ", "㋿", " .,", "é", "
", "٣٤٥", "٦", "ß", "Ⅳ12", "345", "678"]} +{"text": " 👍🏽!!'M\t!!>½s३<|fim_prefix|>åＡ३'D\tZ\r\n\r\n'Re>#$%ḍ̇Ⅳd\r\n\r\n\tdt!!< t \"ع", "tokens": 57, "pieces": [" ", " 👍🏽!!'", "M", "\t", "!!>", "½", "s", "३", "<|", "fim", "_prefix", "|>", "å", "Ａ", "३", "'D", "\tZ", "\r\n\r\n", "'Re", ">#$%", "ḍ̇", "Ⅳ", "d", "\r\n\r\n", "\t", "dt", "!!<", " t", " ", " \"", "ع"]} +{"text": "㍿३عＡſ,>#$%!!'ll12345678'", "tokens": 21, "pieces": ["㍿", "३", "ع", "Ａ", "ſ", ",>#$%!!'", "ll", "123", "456", "78", "'"]} +{"text": "'S-　!!\"😀🏽 aåé　'S
", "tokens": 21, "pieces": ["<", "EOT", ">'", "S", "-", "　", "!!\"😀🏽", " aåé", "　", "'S", "
"]} +{"text": "Dž( \n12345678sé#$%(12345678e😀🏽", "tokens": 17, "pieces": ["Dž", "(", " \n", "123", "456", "78", "sé", "#$%(", "123", "456", "78", "e", "😀🏽"]} +{"text": "'ReéDžé 😀🏽­㍿　'lle", "tokens": 23, "pieces": ["'Reé", "Dž", "é", " 😀🏽­㍿", "　", "'lle", ""]} +{"text": "३!㋿m\t'D㍿\r\n\r\n.'llDž　'lls'é<", "tokens": 25, "pieces": ["३", "!㋿", "m", "", "\t", "'D", "㍿\r\n\r\n", ".'", "ll", "Dž", "　", "'lls", "'é", "<"]} +{"text": " \n'VE'Sd­　ع'Må\r\n\r\n'ree 'ReeZ's‍　('re🙂 (\r\u000bꟲ", "tokens": 36, "pieces": [" \n", "'VE'S", "d", "­", "　ع'M", "å", "\r\n\r\n", "'ree", " ", " '", "Ree", "Z's", "‍", "　", "('", "re", "🙂<", "META", "_START", ">", " ", "(\r", "\u000bꟲ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'sZ", "tokens": 2, "pieces": ["'s", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n'S!!‍字\r\n\r\ns漢Ａ‍漢0!!‍dém0漢-́Dž \r\n\r\n<漢're‍'M12345678ḍ̇", "tokens": 40, "pieces": ["\r\n", "'S", "!!‍", "字", "\r\n\r\n", "s漢", "Ａ", "‍漢", "0", "!!‍", "dém", "0", "漢", "-́", "Dž", " \r\n\r\n", "<漢're", "‍'", "M", "123", "456", "78", "ḍ̇"]} +{"text": "<|endoftext|>𐞁३'S,ſ́!!e\t­\r\n'Reḍ̇'s'Ss३ ㍿#$%Z'ſa½<(!9", "tokens": 46, "pieces": ["<|", "endoftext", "|>", "𐞁", "३", "'S", ",ſ́", "!!", "e", "\t", "­<", "META", "_START", ">\r\n", "'Reḍ̇'s", "'Ss", "३", " ", "㍿#$%", "Z'ſ", "a", "½", "<(!", "9"]} +{"text": "méꟲé\n漢's<|fim_prefix|>
9३ éa Ａ'VE३!!漢 \na'VE>éEOT<|endoftext|>ḍ̇ Ａ,字'll#$%ſ", "tokens": 58, "pieces": ["méꟲé", "\n", "漢's", "<|", "fim", "_prefix", "|>", "
", "9", "", "३", " éa", " ", " Ａ'VE", "३", "!!", "漢", " \n", "a'VE", ">é", "EOT", "<|", "endoftext", "|>", "ḍ̇", " Ａ", ",字'll", "#$%", "ſ"]} +{"text": "EOT३('ll㍿e😀🏽Ａ'ſ­é<12345678
漢\u000b ́12345678", "tokens": 29, "pieces": ["EOT", "३", "('", "ll", "㍿e", "😀🏽", "Ａ'ſ", "­é", "<", "123", "456", "78", "
漢", "\u000b", " ́", "123", "456", "78"]} +{"text": "!👍🏽'S'ſ \r\n'VE‍'!!'ḍ̇>Dž\r\n'ſ٣٤٥٦md é字ع!!<|endoftext|>0!!!'S
𐞁d", "tokens": 51, "pieces": ["!👍🏽'", "S'ſ", " \r\n", "'VE", "‍'!!'", "ḍ̇", ">Dž", "\r\n", "'ſ", "٣٤٥", "٦", "md", " é字ع", "!!<|", "endoftext", "|>", "0", "!!!'", "S", "
𐞁d"]} +{"text": "é'ſtＡ'ſ\r\n<|fim_prefix|>\t'ſ३漢\r\n\u000b\nét!'s\r\n\r\n-Dž<|fim_prefix|>ع>9'VEİé㍿", "tokens": 47, "pieces": ["é'ſ", "t", "Ａ'ſ", "\r\n", "<|", "fim", "_prefix", "|>", "\t", "'ſ", "३", "漢", "\r\n\u000b\n", "ét", "!'", "s", "\r\n\r\n", "-Dž", "<|", "fim", "_prefix", "|>", "ع", ">", "9", "'VEİé", "㍿"]} +{"text": "<12345678👍🏽\r­‍", "tokens": 10, "pieces": ["<", "123", "456", "78", "👍🏽\r", "­‍"]} +{"text": "'s <'s!!ع\n-​\u000b Dž👍🏽Ⅳ's're<‍<|endoftext|>😀🏽\"'M 'll>٣٤٥٦ \tEOTm ", "tokens": 45, "pieces": ["'s", " ", "<'", "s", "!!", "ع", "\n", "-​", "\u000b", " Dž", "👍🏽", "Ⅳ", "'s're", "<‍<|", "endoftext", "|>😀🏽\"'", "M", " ", "'ll", ">", "٣٤٥", "٦", " ", "\tEOTm", " "]} +{"text": "\r\n\"🙂fi é‍٣٤٥٦ḍ̇å<|endoftext|>
!å​　'Mé'T9!!!३㍿.d12345678'VE'D<(", "tokens": 48, "pieces": ["\r\n", "\"🙂", "fi", " ", " é", "‍", "٣٤٥", "٦", "ḍ̇å", "<|", "endoftext", "|>", "
", "!å", "​", "　", "'Mé'T", "9", "!!!", "३", "㍿.", "d", "123", "456", "78", "'VE'D", "<("]} +{"text": "́ ḍ̇\"㍿'VEsⅣſ(#$%ß 　
\rＡعt\t𐞁!!👍🏽­sfi!'Reå-fiEOT", "tokens": 45, "pieces": ["́", " ḍ̇", "\"㍿'", "VEs", "Ⅳ", "ſ", "(#$%", "ß", " 　
\r", "Ａعt", "\t𐞁", "!!👍🏽­", "sfi", "!'", "Reå", "-fi", "EOT"]} +{"text": "'T‍३Ⅳ", "tokens": 5, "pieces": ["'T", "‍", "३Ⅳ"]} +{"text": "!!tＡ'reꟲ½ع­३a!s'll\r\n<<|fim_prefix|>'S're<|endoftext|>👍🏽 Ａ \n,fi\r\n,İ'ſ👍🏽d#$%t-", "tokens": 52, "pieces": ["!!", "t", "Ａ're", "ꟲ", "½", "ع", "­", "३", "a", "!s'll", "\r\n", "<<|", "fim", "_prefix", "|>'", "S're", "<|", "endoftext", "|>👍🏽", " Ａ", " \n", ",fi", "\r\n", ",İ'ſ", "👍🏽", "d", "#$%", "t", "-"]} +{"text": ">'s'll‍ꟲ\u000b​'s#$%'Td👍🏽", "tokens": 17, "pieces": [">'", "s'll", "‍ꟲ", "\u000b", "​'", "s", "#$%'", "Td", "👍🏽"]} +{"text": "\r\n\"​'ſ漢\u000bⅣ-٣٤٥٦t.", "tokens": 18, "pieces": ["\r\n", "\"​'", "ſ漢", "\u000b", "Ⅳ", "-", "٣٤٥", "٦", "t", "."]} +{"text": "9​å\r\n\r\n''VE३d,​́és're'T'ſmḍ̇, \n é
\"mſİ", "tokens": 48, "pieces": ["9", "​å", "\r\n\r\n", "''", "VE", "३", "d", ",​́", "és're", "'T'ſ", "mḍ̇", ",", " \n", " é", "
", "\"mſ", "İ"]} +{"text": "‍\r\n㍿Zİ9é.>fi'VEEOT ٣٤٥٦å😀🏽'D  \n#$%‍'re߅\r\n\r\n\u000b\"a　12345678", "tokens": 48, "pieces": ["‍<", "EOT", ">\r\n", "㍿Zİ", "9", "é", ".>", "fi'VE", "EOT", " ", "٣٤٥", "٦", "å", "😀🏽'", "D", "  \n", "#$%‍'", "reß", "…\r\n\r\n", "\u000b", "\"a", "　", "123", "456", "78"]} +{"text": "Ａ.३-\u000b9字ꟲZ'll㋿(a!𐞁sEOT'll.'å!! ḍ̇", "tokens": 38, "pieces": ["Ａ", ".", "३", "-", "\u000b", "9", "字ꟲ", "Z'll", "㋿(", "a", "!𐞁s", "EOT'll", ".'", "å", "!!", " ḍ̇"]} +{"text": "!!s'fifi👍🏽m…\u000ba\"Ⅳ're½.9İEOTꟲſe字ꟲ'ſ", "tokens": 34, "pieces": ["!!", "s", "'fifi", "👍🏽", "m", "…", "\u000ba", "\"", "Ⅳ", "'re", "½", ".", "9", "İEOTꟲſe字ꟲ'ſ"]} +{"text": "𐞁'Sß​<|fim_prefix|>ſ \n>㋿9ꟲＡ9 a", "tokens": 27, "pieces": ["𐞁'S", "ß", "​<|", "fim", "_prefix", "|>", "ſ", " \n", ">㋿", "9", "ꟲ", "Ａ", "9", " ", " a"]} +{"text": "fi३\u000b're'VEع", "tokens": 7, "pieces": ["fi", "३", "\u000b", "'re'VE", "ع"]} +{"text": "👍🏽'séd<­Ａ㋿
'Mm'M½
", "tokens": 19, "pieces": ["👍🏽'", "séd", "<­", "Ａ", "㋿", "
", "'Mm'M", "½", "
"]} +{"text": "'VEs-漢>Z'", "tokens": 7, "pieces": ["'VEs", "-漢", ">Z", "'"]} +{"text": "İḍ̇t're½e0😀🏽'\"d", "tokens": 14, "pieces": ["İḍ̇t're", "½", "e", "0", "😀🏽'\"", "d"]} +{"text": "efi're🙂\r㋿(字
…", "tokens": 13, "pieces": ["efi're", "🙂\r", "㋿(", "字", "
…"]} +{"text": "9​é", "tokens": 9, "pieces": ["", "9", "​", "é"]} +{"text": "ß😀🏽å's", "tokens": 7, "pieces": ["ß", "😀🏽", "å's"]} +{"text": "‍Z<|fim_prefix|>३!!\n­", "tokens": 11, "pieces": ["‍Z", "<|", "fim", "_prefix", "|>", "३", "!!\n", "­"]} +{"text": "漢é.…", "tokens": 9, "pieces": ["漢é", ".", "…", ""]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "9.9 ㋿é(\r\nİ,s
", "tokens": 17, "pieces": ["9", ".", "9", " ", " ㋿<", "EOT", ">é", "(\r\n", "İ", ",s", "
"]} +{"text": "\u000b　#$%Ａ \n'VE'ſ🙂́'Tsé'ſt ", "tokens": 23, "pieces": ["\u000b", "　", "#$%", "Ａ", " \n", "'VE'ſ", "🙂<", "META", "_START", ">́'T", "sé'ſ", "t", " "]} +{"text": "𐞁३ \n٣٤٥٦'ll­‍'ReꟲEOT'D👍🏽'M'll<'ll.d\"'Md'T漢(", "tokens": 38, "pieces": ["𐞁", "३", " \n", "٣٤٥", "٦", "'ll", "­‍'", "Reꟲ", "EOT'D", "👍🏽'", "M'll", "<'", "ll", ".d", "\"'", "Md'T", "漢", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "0fiß.'re\"\u000b 'ſfiß!!'ll\r\n'S​'D㋿\r\n\r\n…", "tokens": 30, "pieces": ["0", "fiß", ".'", "re", "\"", "\u000b", "", " ", " '", "ſfiß", "!!'", "ll", "\r\n", "'S", "​'", "D", "㋿\r\n\r\n", "…"]} +{"text": "ß३'VEع<|endoftext|>Ⅳ👍🏽ß,", "tokens": 19, "pieces": ["ß", "३", "'VEع", "<|", "endoftext", "|>", "Ⅳ", "👍🏽", "ß", ","]} +{"text": "t\r'T9­Ａ'S🙂　#$%!'T\tt½'VE㋿
-m👍🏽\n
 \n", "tokens": 31, "pieces": ["t", "\r", "'T", "9", "­Ａ'S", "🙂", "　", "#$%!'", "T", "", "\tt", "½", "'VE", "㋿", "
", "-m", "👍🏽\n", "
 \n"]} +{"text": "0ḍ̇Zdé.\r\n\r\nZ­<|endoftext|>as…'ll'M12345678<|fim_prefix|><\n‍ 'S👍🏽9!e'Re'Re", "tokens": 47, "pieces": ["0", "ḍ̇", "Zdé", ".\r\n\r\n", "Z", "­<|", "endoftext", "|>", "as", "…", "'ll'M", "123", "456", "78", "<|", "fim", "_prefix", "|><\n", "‍", " ", " '", "S", "👍🏽", "9", "!e'Re", "'Re"]} +{"text": "é漢ḍ̇", "tokens": 5, "pieces": ["é漢ḍ̇"]} +{"text": "ß٣٤٥٦'TZt३<|endoftext|>å.漢a\tm'TＡ Ａ½e‍​ ㋿㋿'M
 \n<'Re \n٣٤٥٦́", "tokens": 53, "pieces": ["ß", "٣٤٥", "٦", "'TZt", "३", "<|", "endoftext", "|>", "å", ".漢a", "\tm'T", "Ａ", " Ａ", "½", "e", "‍​", " ", "㋿㋿'", "M", "
 \n", "<'", "Re", " \n", "٣٤٥", "٦", "́"]} +{"text": " ㍿ 'll Ａ字<|fim_prefix|>'Re　<|fim_prefix|>٣٤٥٦'M'Mꟲ\r\nå>Ⅳ.𐞁EOT‍'ll\n\rt!\nm<|fim_prefix|>", "tokens": 62, "pieces": [" ", " ㍿", " ", "'ll", " Ａ字", "<|", "fim", "_prefix", "|>'", "Re", "　", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "'M'M", "ꟲ", "\r\n", "å", ">", "Ⅳ", ".𐞁", "EOT", "‍'", "ll", "\n\r", "t", "!\n", "m", "<|", "fim", "_prefix", "|>"]} +{"text": "d\nḍ̇fiß\r'M😀🏽'S'T㋿ \r\n\r\n㍿<­漢٣٤٥٦'ReDž\n<|endoftext|>a--!!漢fi…\u000bé\r३­", "tokens": 52, "pieces": ["d", "\n", "ḍ̇fiß", "\r", "'M", "😀🏽'", "S'T", "㋿", " \r\n\r\n", "㍿<­", "漢", "٣٤٥", "٦", "'Re", "Dž", "\n", "<|", "endoftext", "|>", "a", "--!!", "漢fi", "…", "\u000bé", "\r", "३", "­"]} +{"text": "́a!!", "tokens": 3, "pieces": ["́a", "!!"]} +{"text": "a<|endoftext|>𐞁\nꟲ'S\r\nfiEOT12345678🙂", "tokens": 25, "pieces": ["a", "<|", "endoftext", "|>", "𐞁", "\n", "ꟲ'S", "\r\n", "fi", "EOT", "123", "456", "78", "🙂"]} +{"text": "👍🏽'reé㍿ 🙂0\t㋿\r#$%　
m!!é#$%🙂's㍿s'VEeſ'M#$%'ll漢Z,a( ", "tokens": 45, "pieces": ["👍🏽'", "reé", "㍿", " 🙂", "0", "\t", "㋿\r", "#$%", "　", "
m", "!!", "é", "#$%🙂'", "s", "㍿s'VE", "eſ'M", "#$%'", "ll漢", "Z", ",a", "(", " "]} +{"text": "👍🏽…㋿İd ", "tokens": 11, "pieces": ["👍🏽", "…", "㋿İd", " "]} +{"text": " 字　\u000bmEOT​ḍ̇ İ٣٤٥٦👍🏽'ſDža'T\r>'TDžDž!!٣٤٥٦\"'VE", "tokens": 43, "pieces": [" 字", "　", "\u000bm", "EOT", "​ḍ̇", " İ", "٣٤٥", "٦", "👍🏽'", "ſ", "Dža'T", "\r", ">'", "TDžDž", "!!", "٣٤٥", "٦", "\"<", "EOT", ">'", "VE"]} +{"text": "ḍ̇  \n é<|endoftext|>'ll<|endoftext|>é…\t \né\r\nm.'DéＡ#$%\u000bⅣ­", "tokens": 42, "pieces": ["ḍ̇", "  \n", " é", "<|", "endoftext", "|>'", "ll", "<|", "endoftext", "|>", "é", "…\t \n", "é", "\r\n", "m", ".'", "Dé", "Ａ", "#$%", "\u000b", "Ⅳ", "­"]} +{"text": "'­ßع<|endoftext|>'llss\"٣٤٥٦é", "tokens": 19, "pieces": ["'­", "ßع", "<|", "endoftext", "|>'", "llss", "\"", "٣٤٥", "٦", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Ⅳ!!éé<|endoftext|>m
Dž'D ḍ̇\"३<|endoftext|>'👍🏽́'́'VEa!\n'll㋿́EOT<|endoftext|>!!‍9ḍ̇", "tokens": 60, "pieces": ["Ⅳ", "!!", "éé", "<|", "endoftext", "|>", "m", "
Dž'D", " ḍ̇", "\"", "३", "<|", "endoftext", "|>'👍🏽́'́'", "VEa", "!\n", "'ll", "㋿́", "EOT", "<|", "endoftext", "|>!!‍", "9", "ḍ̇"]} +{"text": " ٣٤٥٦ſ!!ع𐞁-漢å漢३İ‍'M\"ع!Ａ#$%EOTe9\u000b<|endoftext|>‍'VE ٣٤٥٦12345678 ée9>\r\n\r\n\r\n\r\n'S", "tokens": 65, "pieces": [" ", " ", "٣٤٥", "٦", "ſ", "!!", "ع𐞁", "-漢å漢", "३", "İ", "‍'", "M", "\"ع", "!Ａ", "#$%", "EOTe", "9", "\u000b", "<|", "endoftext", "|>‍'", "VE", " ", "٣٤٥", "٦12", "345", "678", " ée", "9", ">\r\n\r\n\r\n\r\n", "<", "EOT", ">'", "S"]} +{"text": "'D½!é0- \r\n\r\nd😀🏽 9­#$%Z…éꟲ　e'Reé>漢'S9", "tokens": 34, "pieces": ["'D", "½", "!é", "0", "-", " \r\n\r\n", "d", "😀🏽", " ", " ", "9", "­#$%", "Z", "…éꟲ", "　e'Re", "é", ">漢'S", "9"]} +{"text": "　\r\né t! ' #$%12345678!e12345678! \r", "tokens": 24, "pieces": ["　\r\n", "é", " t", "!", " ", "'", " ", " #$%", "123", "456", "78", "!e", "123", "456", "78", "!", " \r"]} +{"text": "EOT!'ſ­\r\nع", "tokens": 7, "pieces": ["EOT", "!'", "ſ", "­\r\n", "ع"]} +{"text": "㍿9#$%(d!!Ａ́
ع٣٤٥٦\"\"eß\u000b \n!!𐞁𐞁٣٤٥٦​👍🏽'll　
", "tokens": 50, "pieces": ["㍿", "9", "#$%(", "d", "!!", "Ａ́", "
ع", "", "٣٤٥", "٦", "\"\"", "eß", "\u000b \n", "!!", "𐞁", "𐞁", "٣٤٥", "٦", "​👍🏽'", "ll", "　
"]} +{"text": ", EOT\n<|endoftext|>'ReEOT
ſ'ſ㍿İſ\r\n\r\n㍿½🙂<|endoftext|>'\r\n\r\nⅣꟲm", "tokens": 47, "pieces": [",", " EOT", "\n", "<|", "endoftext", "|>'", "Re", "EOT", "
ſ'ſ", "㍿İſ", "\r\n\r\n", "㍿", "½", "🙂<|", "endoftext", "|>'\r\n\r\n", "Ⅳ", "ꟲ", "m"]} +{"text": "ß \n<|endoftext|>Ａ'llEOT \r åé9é👍🏽㍿㍿'Ḿm\"ꟲḍ̇漢'ſ12345678'T३ \n're'('D\u000b", "tokens": 57, "pieces": ["ß", " \n", "<|", "endoftext", "|>", "Ａ'll", "EOT", " \r", " åé", "9", "é", "👍🏽㍿㍿'", "Ḿm", "\"ꟲḍ̇漢'ſ", "123", "456", "78", "'T", "३", " \n", "'re", "'('", "D", "\u000b"]} +{"text": "\n're😀🏽‍éḍ̇9㋿", "tokens": 15, "pieces": ["\n", "'re", "😀🏽‍", "éḍ̇", "9", "㋿"]} +{"text": "漢\u000b\r\n\r\n
 漢'redİ\r\n", "tokens": 10, "pieces": ["漢", "\u000b\r\n\r\n", "
", " 漢're", "d", "İ", "\r\n"]} +{"text": "½!e", "tokens": 3, "pieces": ["½", "!e"]} +{"text": "　'VE​t३", "tokens": 6, "pieces": ["　", "'VE", "​t", "३"]} +{"text": "'M,d㋿\r>as>!'re\tſ́<|fim_prefix|>#$%-fi𐞁½㋿́\ré", "tokens": 34, "pieces": ["'M", ",d", "㋿\r", ">as", ">!'", "re", "\tſ́", "<|", "fim", "_prefix", "|>#$%-", "fi𐞁", "½", "㋿́", "\r", "é"]} +{"text": "e,😀🏽9\r\n#$%'Dt👍🏽s'ReⅣ", "tokens": 21, "pieces": ["e", ",😀🏽", "9", "\r\n", "#$%'", "Dt", "👍🏽", "s'Re", "Ⅳ", ""]} +{"text": "…\"\"½#$%😀🏽's!!\"ea< \n", "tokens": 16, "pieces": ["…", "\"\"", "½", "#$%😀🏽'", "s", "!!\"", "ea", "<", " \n"]} +{"text": "'TⅣ're㋿\r\n\r\n🙂'Re<|endoftext|>　'reꟲ ḍ̇m😀🏽字'S'ReåＡ", "tokens": 41, "pieces": ["'T", "Ⅳ", "'re", "㋿\r\n\r\n", "🙂<", "META", "_START", ">'", "Re", "<|", "endoftext", "|>", "　", "'reꟲ", " ", " ḍ̇m", "😀🏽", "字'S", "'Reå", "Ａ"]} +{"text": "'<漢…🙂<|endoftext|> ßé\r\n\r\nḍ̇Dž", "tokens": 24, "pieces": ["'<", "漢", "…", "🙂<", "META", "_START", "><|", "endoftext", "|>", " ßé", "\r\n\r\n", "ḍ̇", "Dž"]} +{"text": "a'fi.EOT( 'T-\r\n\r\n12345678….", "tokens": 16, "pieces": ["a", "'fi", ".EOT", "(", " '", "T", "-\r\n\r\n", "123", "456", "78", "…", "."]} +{"text": "­'Re<㋿", "tokens": 7, "pieces": ["­'", "Re", "<㋿"]} +{"text": " \n>😀🏽 -ع'S३
🙂ß,!'s<-'re'Mmm'TdEOT-­\r\n\r\n", "tokens": 33, "pieces": [" \n", "><", "META", "_START", ">😀🏽", " -", "ع'S", "३", "
", "🙂ß", ",!'", "s", "<-'", "re'M", "mm'T", "d", "EOT", "-­\r\n\r\n"]} +{"text": "å'Refie　<|fim_prefix|>'TEOT a३d'sſ' \n!'reDž(߅(Ⅳeع", "tokens": 35, "pieces": ["å'Re", "fie", "　", "<|", "fim", "_prefix", "|>'", "TEOT", " a", "३", "d's", "ſ", "'", " \n", "!'", "re", "Dž", "(ß", "…", "(", "Ⅳ", "eع"]} +{"text": "ع‍'ſDž12345678😀🏽 \n #$%(s<|endoftext|>'VE", "tokens": 26, "pieces": ["ع", "‍'", "ſ", "Dž", "123", "456", "78", "😀🏽", " \n", " ", " #$%(", "s", "<|", "endoftext", "|>'", "VE"]} +{"text": "‍👍🏽", "tokens": 4, "pieces": ["‍👍🏽"]} +{"text": "å　字­👍🏽ésm٣٤٥٦Dže12345678\r\n,9İ'Re
Ａ", "tokens": 27, "pieces": ["å", "　字", "­👍🏽", "ésm", "٣٤٥", "٦", "Dže", "123", "456", "78", "\r\n", ",", "9", "İ'Re", "
Ａ"]} +{"text": "#$%Zd (\r\n,'ſ漢\r\nDž#$%‍Ａ𐞁", "tokens": 19, "pieces": ["#$%", "Zd", " ", "(\r\n", ",'", "ſ漢", "\r\n", "Dž", "#$%‍", "Ａ𐞁"]} +{"text": "(a'D's<|fim_prefix|>(Z åt😀🏽EOTe!\r\na'TEOT…!", "tokens": 28, "pieces": ["(a'D", "'s", "<|", "fim", "_prefix", "|>(", "Z", " ", " åt", "😀🏽", "EOTe", "!\r\n", "a'T", "EOT", "…", "!"]} +{"text": "\u000b'De\ns", "tokens": 5, "pieces": ["\u000b", "'De", "\n", "s"]} +{"text": "s", "tokens": 1, "pieces": ["s"]} +{"text": "'Refi EOTſ 𐞁\u000b字!'ReⅣ'ſ", "tokens": 20, "pieces": ["'Refi", " ", " EOTſ", " ", " 𐞁", "\u000b字", "!'", "Re", "Ⅳ", "'ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "- 'llＡſ İ.\r\n\r\n𐞁#$%½9\u000b㋿\u000b　​m'St‍å0ſ… ", "tokens": 41, "pieces": ["-", " '", "ll", "Ａſ", " <", "EOT", ">İ", ".\r\n\r\n", "𐞁", "#$%", "½9", "\u000b", "㋿", "\u000b", "　", "​m'S", "t", "‍å", "0", "ſ", "… "]} +{"text": "\r\n'Rete\u000bfi'M\rꟲ9'll㍿ḍ̇ és'9Dž‍'T!​", "tokens": 30, "pieces": ["\r\n", "'Rete", "\u000bfi'M", "\r", "ꟲ", "9", "'ll", "㍿ḍ̇", " ", " és", "'", "9", "Dž", "‍'", "T", "!​"]} +{"text": "‍Ａ'\r\n…\u000b#$%.9'T12345678
\u000b mſ\r\n\r\n", "tokens": 23, "pieces": ["‍Ａ", "'\r\n", "…", "\u000b", "#$%.", "9", "'T", "123", "456", "78", "
\u000b", " m", "ſ", "\r\n\r\n"]} +{"text": "🙂Ａ, 12345678…㋿­३٣٤٥٦12345678Ａ३\r#$%<​­å", "tokens": 36, "pieces": ["🙂Ａ", ",", " ", "123", "456", "78", "…", "㋿­", "३٣٤", "٥٦1", "234", "567", "8", "Ａ", "३", "\r", "#$%<​­", "å"]} +{"text": "㍿é'M\t'S!!s\n,aİZé12345678‍­('M\"å😀🏽'M<|endoftext|>fiḍ̇d \n𐞁", "tokens": 46, "pieces": ["㍿é'M", "\t", "'S", "!!", "s", "\n", ",a", "İZé", "123", "456", "78", "‍­('", "M", "\"å", "😀🏽'", "M", "<|", "endoftext", "|>", "fiḍ̇d", " \n", "𐞁"]} +{"text": " '😀🏽ḍ̇<|endoftext|><|endoftext|>Ⅳ\" 'ḍ̇🙂'S'S'll m'VE Ⅳ \n'🙂字'ſ'VE\rEOT\r‍‍!!\r9", "tokens": 58, "pieces": [" ", " '😀🏽", "ḍ̇", "<|", "endoftext", "|><|", "endoftext", "|>", "Ⅳ", "\"", " ", " '", "ḍ̇", "🙂'", "S'S", "'ll", " m'VE", " ", "Ⅳ", " \n", "'🙂", "字'ſ", "'VE", "\r", "EOT", "\r", "‍‍!!\r", "9"]} +{"text": "0<|fim_prefix|>(Dž \n㍿\u000bDž<…​漢Ａ(,.12345678٣٤٥٦\"字​'Re㍿\r\n\r\n‍㍿", "tokens": 44, "pieces": ["0", "<|", "fim", "_prefix", "|>(", "Dž", " \n", "㍿", "\u000bDž", "<", "…", "​漢", "Ａ", "(,.", "123", "456", "78٣", "٤٥٦", "\"字", "​'", "Re", "㍿\r\n\r\n", "‍㍿"]} +{"text": "漢३DžⅣ\n'ſ३fi \n 'Re'MＡ<|fim_prefix|>\r\n\r\n\r\n-ꟲ½!!<|endoftext|>½ 's½\r\n३ḍ̇\r\n\r\nſ
𐞁Ⅳa　‍٣٤٥٦", "tokens": 64, "pieces": ["漢", "३", "Dž", "Ⅳ", "\n", "'", "ſ", "३", "fi", " \n", " ", " '", "Re'M", "Ａ", "<|", "fim", "_prefix", "|>\r\n\r\n\r\n", "-ꟲ", "½", "!!<|", "endoftext", "|>", "½", " ", " '", "s", "½", "\r\n", "३", "ḍ̇", "\r\n\r\n", "ſ", "
𐞁", "Ⅳ", "a", "　", "‍", "٣٤٥", "٦"]} +{"text": "\t\r\n<am
>('S're\n㋿'Re\u000b‍\"́ \n<ß0
t 're''½>­\"'S'reDž-", "tokens": 41, "pieces": ["\t", "\r\n", "<<", "META", "_START", ">am", "
", ">('", "S're", "\n", "㋿'", "Re", "\u000b", "‍\"́", " \n", "<ß", "0", "
t", " ", "'re", "''", "½", ">­\"'", "S're", "Dž", "-"]} +{"text": "\r\n<𐞁9'M​ß🙂'D", "tokens": 13, "pieces": ["\r\n", "<𐞁", "9", "'M", "​ß", "🙂'", "D"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "…ꟲ 𐞁'Té​Z(é\u000bＡ\n\u000b!\"!!#$%  ſ\r0're 'T(٣٤٥٦<|fim_prefix|>\r're<|endoftext|>12345678Ⅳ㍿\r 😀🏽", "tokens": 69, "pieces": ["…ꟲ", " 𐞁'T", "é", "​Z", "(é", "\u000bＡ", "\n", "\u000b", "!\"!!#$%", " ", " ſ", "\r", "0", "'re", " ", "'T", "(", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\r", "'", "re", "<|", "endoftext", "|>", "123", "456", "78Ⅳ", "㍿\r", " 😀🏽"]} +{"text": "#$%eİ\nḍ̇", "tokens": 8, "pieces": ["#$%", "e", "İ", "\n", "ḍ̇"]} +{"text": "
-­😀🏽", "tokens": 6, "pieces": ["
", "-­😀🏽"]} +{"text": "(Ⅳß　'ſ-<|endoftext|>", "tokens": 15, "pieces": ["(", "Ⅳ", "ß", "　", "'ſ", "-<|", "endoftext", "|>"]} +{"text": "mt0#$%sＡ\nع 'T\r\n\r\nfiå(Džḍ̇İ \n\r", "tokens": 31, "pieces": ["mt", "0", "#$%", "s", "Ａ", "\n", "ع", " ", "'T", "\r\n\r\n", "fiå", "(<", "EOT", ">Džḍ̇", "İ", " \n\r"]} +{"text": "İ㋿#$%EOT𐞁३­#$%­-ꟲåḍ̇ⅣⅣḍ̇t's漢漢ꟲs'M'VEs\r\n½ İEOT٣٤٥٦Ａ'll'M12345678", "tokens": 61, "pieces": ["İ", "㋿#$%", "EOT𐞁", "३", "­#$%­-", "ꟲåḍ̇", "ⅣⅣ", "ḍ̇t's", "漢漢ꟲs'M", "'VEs", "\r\n", "½", " İEOT", "٣٤٥", "٦", "Ａ'll", "'M", "123", "456", "78"]} +{"text": "ſd\r\n\r\ne!!12345678'a'Dſ漢  EOT<|endoftext|>t!! -',\n🙂<|endoftext|> <|endoftext|>", "tokens": 44, "pieces": ["ſd", "\r\n\r\n", "e", "!!", "123", "456", "78", "'a'D", "ſ漢", " ", " EOT", "<|", "endoftext", "|>", "t", "!!", " ", " -',\n", "🙂<|", "endoftext", "|>", " ", "<|", "endoftext", "|>"]} +{"text": "!.Z'Dfié", "tokens": 6, "pieces": ["!.", "Z'D", "fié"]} +{"text": "​Zé<#$%12345678d
EOT३́\r\n\r\né­!EOTå'T'T३
\r\n12345678
", "tokens": 33, "pieces": ["​Zé", "<#$%", "123", "456", "78", "d", "
EOT", "३", "́", "\r\n\r\n", "é", "­!", "EOTå'T", "'T", "३", "
\r\n", "123", "456", "78", "
"]} +{"text": "ſ🙂'lléع́>😀🏽'VEe#$%😀🏽\tfiDžé'Re‍🙂ß'<Ⅳ9​'M㋿\r\nd'S　\t>\n", "tokens": 55, "pieces": ["ſ", "🙂'", "lléع́", ">😀🏽'", "VEe", "#$%😀🏽", "\tfi", "Džé'Re", "‍🙂", "ß", "'<", "Ⅳ9", "​'", "M", "㋿\r\n", "d'S", "　", "", "\t", ">\n"]} +{"text": "İas👍🏽'ſé'llع\tDžé12345678!é!<İ\r\nfi　'VE", "tokens": 37, "pieces": ["İ", "as", "👍🏽'", "ſé'll", "ع", "\tDžé", "123", "456", "78", "!é", "!<", "İ", "\r\n", "fi", "　", "'VE"]} +{"text": "Zdꟲ(EOT'T\n", "tokens": 8, "pieces": ["Zdꟲ", "(EOT'T", "\n"]} +{"text": "字'D­'M

'll>İ(½'s३!㍿‍İ's#$%ééaß're.'T \n\"\r\nß­٣٤٥٦ꟲfi😀🏽", "tokens": 46, "pieces": ["字'D", "­'", "M", "
", "
", "'ll", ">İ", "(", "½", "'s", "३", "!㍿‍", "İ's", "#$%", "ééaß're", ".'", "T", " \n", "\"\r\n", "ß", "­", "٣٤٥", "٦", "ꟲfi", "😀🏽"]} +{"text": "9('VEß 'T<|fim_prefix|><|endoftext|>🙂\t३>!!å'D㋿(", "tokens": 42, "pieces": ["9", "('", "VEß", "", " ", "'", "T", "<|", "fim", "_prefix", "|><|", "endoftext", "|>🙂", "\t", "३", ">!!", "å'D", "㋿("]} +{"text": "½'D́e'sßEOT >'ReDžt", "tokens": 17, "pieces": ["½", "'", "D́e's", "ß", "EOT", " ", ">'", "Re", "Džt"]} +{"text": "‍­", "tokens": 2, "pieces": ["‍­"]} +{"text": "<|endoftext|> 'M㋿ß\r\nⅣ㋿​'Re\ne\n 12345678　'Reḍ̇'VE\"ḍ̇", "tokens": 45, "pieces": ["<|", "endoftext", "|>", " '", "M", "㋿ß", "\r\n", "", "Ⅳ", "㋿​'", "Re", "\n", "e", "\n", " ", " ", "123", "456", "78", "　", "'Reḍ̇'VE", "\"ḍ̇"]} +{"text": "é'SZß\r\n\r\n\u000b<|endoftext|>!! 　\u000b >(ḍ̇Džds😀🏽'll'", "SZß", "\r\n\r\n", "\u000b", "<|", "endoftext", "|>!!", " 　\u000b ", " >(", "ḍ̇", "Džds", "😀🏽'", "ll", "३#$%a'ſ'S­
's字m", "tokens": 20, "pieces": ["३", "\"<|", "endoftext", "|>", "३", "#$%", "a'ſ", "'S", "­", "
", "'s字m"]} +{"text": "'re٣٤٥٦‍ḍ̇'S'ſ漢m…३ſꟲ‍½  'ſ<|fim_prefix|>́'Re'Reع\té,ḍ̇.ß \n字'ſ", "tokens": 53, "pieces": ["'re", "٣٤٥", "٦", "‍ḍ̇'S", "'ſ漢m", "…", "३", "ſꟲ", "‍", "½", " ", " ", "'ſ", "<|", "fim", "_prefix", "|>́'", "Re'Re", "ع", "\té", ",ḍ̇", ".ß", " \n", "字'ſ", ""]} +{"text": "-d(\u000b\"३fi३ \n🙂\r'T'M", "tokens": 12, "pieces": ["-d", "(", "\u000b", "\"", "३", "fi", "३", " \n", "🙂\r", "'T'M"]} +{"text": "\r\n字!\n0٣٤٥٦👍🏽㍿ⅣZ𐞁ß㍿\" é,Z 
३'M", "tokens": 35, "pieces": ["\r\n", "字", "!\n", "0٣٤", "٥٦", "👍🏽㍿", "Ⅳ", "Z𐞁ß", "㍿\"", " é", ",Z", " ", "
", "३", "'M"]} +{"text": ">'d12345678\"…ßⅣ\n12345678Džع𐞁<|endoftext|><|endoftext|>\r'llEOTd½ 'T😀🏽0'​EOT\"٣٤٥٦㋿'ll12345678'll'll,'D", "tokens": 72, "pieces": ["><", "EOT", ">'", "d", "123", "456", "78", "\"", "…ß", "Ⅳ", "\n", "123", "456", "78", "Džع𐞁", "<|", "endoftext", "|><|", "endoftext", "|>\r", "'ll", "EOTd", "½", " ", " '", "T", "😀🏽", "0", "'​", "EOT", "\"", "٣٤٥", "٦", "㋿'", "ll", "123", "456", "78", "'ll'll", ",'", "D"]} +{"text": "­", "tokens": 1, "pieces": ["­"]} +{"text": "Džå\u000b'Re🙂é>'re", "tokens": 11, "pieces": ["Džå", "\u000b", "'Re", "🙂é", ">'", "re"]} +{"text": "٣٤٥٦d,\n'll'sعfi \r\n!!é!\t'Sfi😀🏽(­", "tokens": 23, "pieces": ["٣٤٥", "٦", "d", ",\n", "'ll's", "عfi", " \r\n", "!!", "é", "!", "\t", "'Sfi", "😀🏽(­"]} +{"text": "🙂'sＡå \ts漢Z'll-éع", "tokens": 14, "pieces": ["🙂'", "s", "Ａå", " ", "\ts漢", "Z'll", "-éع"]} +{"text": "'Re½\r\nm", "tokens": 4, "pieces": ["'Re", "½", "\r\n", "m"]} +{"text": ">'s!!　漢३<|endoftext|>Ａعa>!!<|endoftext|>\r!s\r\n‍ Dž \nⅣ's#$% \r\né٣٤٥٦<|endoftext|>'D0Z\t", "tokens": 63, "pieces": [">'", "s", "!!<", "META", "_START", ">", "　漢", "३", "<|", "endoftext", "|>", "Ａعa", ">!!<|", "endoftext", "|>\r", "!s", "\r\n", "‍", " ", " Dž", " \n", "Ⅳ", "'s", "#$%", " \r\n", "é", "٣٤٥", "٦", "<|", "endoftext", "|>'", "D", "0", "Z", "\t"]} +{"text": "'s,ßDž'Re'!!!e9'Då\r\nİ \n-㍿d\u000b'T", "tokens": 23, "pieces": ["'s", ",ß", "Dž'Re", "'!!!", "e", "9", "'Då", "\r\n", "İ", " \n", "-㍿", "d", "\u000b", "'T"]} +{"text": "\u000b'T9　😀🏽'Té\n're\t́ß!'M　<…
­😀🏽Ⅳa", "tokens": 29, "pieces": ["\u000b", "'T", "9", "　", "😀🏽'", "Té", "\n", "'re", "\t́ß", "!'", "M", "　", "<", "…", "
", "­😀🏽", "Ⅳ", "a"]} +{"text": "ḍ̇ꟲ‍m ­　'Re㋿,'M ㍿-EOT‍́ \nع\n'ḾDž \n\"é'Red½éEOT'S", "tokens": 44, "pieces": ["ḍ̇ꟲ", "‍m", " ", " ­", "　", "'Re", "㋿,'", "M", " ", " ㍿-", "EOT", "‍́", " \n", "ع", "\n", "'Ḿ", "Dž", " \n", "\"é'Re", "d", "½", "é", "EOT'S"]} +{"text": "'Re \n's­ع㋿字é \nEOT'Re\" 'M😀🏽EOTḍ̇0<|fim_prefix|>漢-!३½Ⅳ𐞁ꟲ's", "tokens": 52, "pieces": ["'Re", " \n", "'s", "­ع", "㋿字é", " \n", "EOT'Re", "\"", " ", "'M", "😀🏽", "EOTḍ̇", "0", "<|", "fim", "_prefix", "|>", "漢", "-!", "३½Ⅳ", "𐞁ꟲ's"]} +{"text": "fi!fi<‍Ａ‍́😀🏽\r\n\r\n'll\"Ａ漢'Re<|fim_prefix|><|fim_prefix|><‍㍿", "tokens": 32, "pieces": ["fi", "!fi", "<‍", "Ａ", "‍́", "😀🏽\r\n\r\n", "'ll", "\"Ａ漢'Re", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|><‍㍿"]} +{"text": "👍🏽'VE'Reé👍🏽d\" \n-\"𐞁d<|endoftext|>\r\n­EOT(12345678३'reＡ㋿", "tokens": 49, "pieces": ["👍🏽'", "VE'Re", "é", "👍🏽", "d", "\"", " \n", "-\"", "𐞁", "d", "<|", "endoftext", "|>\r\n", "­", "EOT", "(", "123", "456", "78३", "'re", "Ａ", "㋿"]} +{"text": "Ａå🙂.Ａ㋿½EOTꟲ12345678İ'M's", "tokens": 25, "pieces": ["Ａå", "🙂.", "Ａ", "㋿<", "META", "_START", ">", "½", "EOTꟲ", "123", "456", "78", "İ'M", "'s"]} +{"text": "-Džt,#$%EOT's0ꟲ
👍🏽mé३ꟲ'S. ع字ß\"''re
å'S　\u000bé ", "tokens": 44, "pieces": ["-Džt", ",#$%", "EOT's", "0", "ꟲ", "
", "👍🏽", "mé", "३", "ꟲ'S", ".<", "META", "_START", ">", " ", " ع字ß", "\"''", "re", "
å'S", "　", "\u000bé", " "]} +{"text": "३ dDž", "tokens": 11, "pieces": ["", "३", " ", " <", "META", "_START", ">d", "Dž"]} +{"text": "‍'ReDžfiİé'ſ\r'ſꟲＡ٣٤٥٦👍🏽e½\r\né.'M'", "tokens": 32, "pieces": ["‍'", "Re", "Džfi", "İé'ſ", "\r", "'ſꟲ", "Ａ", "٣٤٥", "٦", "👍🏽", "e", "½", "\r\n", "é", ".'", "M", "'"]} +{"text": "Ⅳ　́0 \n>a

a½'<|fim_prefix|>𐞁ع(t\"<|fim_prefix|>'ſDžt", "tokens": 37, "pieces": ["", "Ⅳ", "　́", "0", " \n", ">a", "
", "
a", "½", "'<|", "fim", "_prefix", "|>", "𐞁ع", "(t", "\"<|", "fim", "_prefix", "|>'", "ſ", "Džt"]} +{"text": "\r\n\r\n​३<|endoftext|>­'ll\u000b　s㋿a
", "tokens": 21, "pieces": ["\r\n\r\n", "​", "३", "<|", "endoftext", "|>­'", "ll", "\u000b", "　s", "㋿a", "
"]} +{"text": "\u000b<|endoftext|>0\n9𐞁fi'Så\u000b‍>㍿é𐞁m　İtéé (12345678", "tokens": 44, "pieces": ["\u000b", "<|", "endoftext", "|>", "0", "\n", "9", "𐞁fi'S", "å", "\u000b", "‍>㍿", "é𐞁m", "　İ", "téé", " ", "(", "123", "456", "78"]} +{"text": "½'ſfi👍🏽‍\u000b\r\nEOT \r\n\r\nfi'M12345678\u000bdＡ9 \"٣٤٥٦<<|fim_prefix|> ", "tokens": 39, "pieces": ["", "½", "'ſfi", "👍🏽‍", "\u000b\r\n", "EOT", " \r\n\r\n", "fi'M", "123", "456", "78", "\u000bd", "Ａ", "9", " \"", "٣٤٥", "٦", "<<|", "fim", "_prefix", "|>", " "]} +{"text": "(\n漢EOT😀🏽", "tokens": 7, "pieces": ["(\n", "漢", "EOT", "😀🏽"]} +{"text": "ſ㍿aZ ß'Re'é…​ＡعⅣéfi12345678½", "tokens": 24, "pieces": ["ſ", "㍿a", "Z", " ß'Re", "'é", "…", "​Ａع", "Ⅳ", "éfi", "123", "456", "78½"]} +{"text": "İ'D㍿ \n字é‍ßd😀🏽ḍ̇a>'ſ漢 >12345678ß<>'ll fi'😀🏽<|fim_prefix|>", "tokens": 42, "pieces": ["İ'D", "㍿", " \n", "字é", "‍ßd", "😀🏽", "ḍ̇a", ">'", "ſ漢", " >", "123", "456", "78", "ß", "<>'", "ll", " fi", "'😀🏽<|", "fim", "_prefix", "|>"]} +{"text": "\"'M\t\r\ns12345678.éé\"\tß‍\"!!é-　
a#$%ß　 \n 'VE", "tokens": 31, "pieces": ["'Re", "½", "👍🏽", "½", "'ſſ", "!<|", "endoftext", "|>\"!!", "é", "-", "　", "
a", "#$%", "ß", "　 \n", " ", " '", "VE"]} +{"text": "㍿!Ａ字Dž", "tokens": 8, "pieces": ["㍿!", "Ａ字", "Dž"]} +{"text": "٣٤٥٦'Sꟲ\r\n\r\n😀🏽é(!>'re字é㋿e
-́​😀🏽́\r\néDž<|fim_prefix|>𐞁​​'llå\"😀🏽", "tokens": 57, "pieces": ["٣٤٥", "٦", "'Sꟲ", "\r\n\r\n", "😀🏽", "é", "(!>'", "re字é", "㋿e", "
", "-́", "​😀🏽́\r\n", "é", "Dž", "<|", "fim", "_prefix", "|>", "𐞁", "​​'", "llå", "\"😀🏽"]} +{"text": "…

…‍Ⅳ'D'ſ'll.½fi\r\n\r\n-9<|endoftext|> 漢!!", "tokens": 30, "pieces": ["…

", "…", "‍", "Ⅳ", "'D'ſ", "'ll", ".", "½", "fi", "\r\n\r\n", "-", "9", "<|", "endoftext", "|>", " ", " 漢", "!!"]} +{"text": "'D́
's​EOT٣٤٥٦İꟲ\r\n\r\nＡ 9dß'Sm<|endoftext|>😀🏽
's", "tokens": 49, "pieces": ["'D́", "
", "'s", "​EOT", "٣٤٥", "٦", "İꟲ", "\r\n\r\n", "Ａ", " ", " ", "9", "d", "", "ß'S", "m", "<|", "endoftext", "|>😀🏽", "
", "'s"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ع\n'D(ß㍿‍-'D😀🏽ß½Z'll\u000b'reİ", "tokens": 24, "pieces": ["ع", "\n", "'D", "(ß", "㍿‍-'", "D", "😀🏽", "ß", "½", "Z", "'", "ll", "\u000b", "'re", "İ"]} +{"text": "Ⅳ!!👍🏽Ⅳ🙂ſ ㋿!>½\rß½ ''D<fi'ſ", "tokens": 33, "pieces": ["Ⅳ", "!!👍🏽", "Ⅳ", "🙂ſ", " ", "㋿!>", "½", "\r", "ß", "", "½", " ", " ''", "D", "<fi'ſ"]} +{"text": "́EOT'D<‍!'S9字🙂\nå<|fim_prefix|>.a\t\r\né!!0<|endoftext|>Ⅳİ'll!\" 0", "tokens": 40, "pieces": ["́", "EOT'D", "<‍!'", "S", "9", "字", "🙂\n", "å", "<|", "fim", "_prefix", "|>.", "a", "\t\r\n", "é", "!!", "0", "<|", "endoftext", "|>", "Ⅳ", "İ'll", "!\"", " ", "0"]} +{"text": "<9\u000b'ſmⅣ'MßtＡ😀🏽ꟲ'reع ع'M<,ſİé'ſ-\r\n9Ａ­漢　9👍🏽.३mt", "tokens": 46, "pieces": ["<", "9", "\u000b", "'ſm", "Ⅳ", "'Mßt", "Ａ", "😀🏽", "ꟲ're", "ع", " ع'M", "<,", "ſ", "İé'ſ", "-\r\n", "9", "Ａ", "­漢", "　", "9", "👍🏽<", "EOT", ">.", "३", "mt"]} +{"text": "0'M", "tokens": 2, "pieces": ["0", "'M"]} +{"text": "åe𐞁
\r,ß'VEa12345678ḍ̇a'Ree.> 
́😀🏽'T㋿Z,'ſ\r\n 𐞁åé\t", "tokens": 54, "pieces": ["åe𐞁", "
\r", ",ß'VE", "a", "123", "456", "78", "ḍ̇a'Re", "e", ".>", " ", "
́", "😀🏽'", "T", "㋿Z", ",'", "ſ", "\r\n", " ", " 𐞁åé", "\t"]} +{"text": "'ſ́é \n½\n's𐞁>a​ꟲfi-Z'ſ\r\n㋿ é \n
Dž…́'M…­éⅣ😀🏽字ſ're'Re­d'VE", "tokens": 52, "pieces": ["'ſ́é", " \n", "½", "\n", "'s𐞁", ">a", "​ꟲfi", "-Z'ſ", "\r\n", "㋿", " é", " \n", "
Dž", "…́'M", "…", "­é", "Ⅳ", "😀🏽", "字ſ're", "'Re", "­d'VE"]} +{"text": "\r\n\r\nt 😀🏽ßfi'reé'ſe\u000b漢!ß　'Reſ'Mع<|endoftext|>ſ Ａfi-s字㋿", "tokens": 40, "pieces": ["\r\n\r\n", "t", " ", "😀🏽", "ßfi're", "é'ſ", "e", "\u000b漢", "!ß", "　", "'Reſ'M", "ع", "<|", "endoftext", "|>", "ſ", " ", " Ａfi", "-s字", "㋿"]} +{"text": "\n👍🏽!!9'Re😀🏽", "tokens": 10, "pieces": ["\n", "👍🏽!!", "9", "'Re", "😀🏽"]} +{"text": "<|fim_prefix|>½'½　'D", "tokens": 14, "pieces": ["<|", "fim", "_prefix", "|>", "½", "'", "½", "　", "'", "D"]} +{"text": "٣٤٥٦Ⅳ", "tokens": 6, "pieces": ["٣٤٥", "٦Ⅳ"]} +{"text": "#$% \nꟲ\"Ａ.😀🏽 9're( ٣٤٥٦ ㍿😀🏽", "tokens": 29, "pieces": ["#$%", " \n", "ꟲ", "\"Ａ", ".😀🏽", " ", " ", "9", "'re", "(", " ", "٣٤٥", "٦", " ", "㍿😀🏽"]} +{"text": "'re'T 😀🏽½EOTſ\r\n\r\n㋿912345678<|endoftext|>'ReeＡ \n12345678Z,\n'reḍ̇ḍ̇\n", "tokens": 41, "pieces": ["'re'T", " ", "😀🏽", "½", "EOTſ", "\r\n\r\n", "㋿", "912", "345", "678", "<|", "endoftext", "|>'", "Ree", "Ａ", " \n", "123", "456", "78", "Z", ",\n", "'reḍ̇ḍ̇", "\n"]} +{"text": "​.ſ", "tokens": 2, "pieces": ["​.", "ſ"]} +{"text": "㍿Ⅳ\téfia\t́,''M> İ३", "tokens": 17, "pieces": ["㍿", "Ⅳ", "\téfia", "\t́", ",''", "M", ">", " ", " İ", "३"]} +{"text": ">👍🏽e㋿\né​ ३('D", "tokens": 19, "pieces": [">👍🏽", "e", "㋿\n", "é", "​", " ", "३", "('", "D"]} +{"text": "\n'>'Ddé🙂12345678å'Mع0 9 ३㍿!!", "tokens": 25, "pieces": ["\n", "'>'", "Ddé", "🙂", "123", "456", "78", "å'M", "ع", "0", " ", " ", "9", " ", " ", "३", "㍿!!"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "İåfiå \n­\r\n\r\nt​.ꟲ­-'llꟲ\r\né!!DžＡ!d🙂\"　a'D \né‍ßde(fi　\n🙂", "tokens": 44, "pieces": ["İåfiå", " \n", "­\r\n\r\n", "t", "​.", "ꟲ", "­-'", "llꟲ", "\r\n", "é", "!!", "DžＡ", "!d", "🙂\"", "　a'D", " \n", "é", "‍ßde", "(fi", "　\n", "🙂"]} +{"text": "\n­\nꟲ½e\n😀🏽…عİ \n㍿!\"'Re", "tokens": 22, "pieces": ["\n", "­\n", "ꟲ", "½", "e", "\n", "😀🏽", "…ع", "İ", " \n", "㍿!\"'", "Re"]} +{"text": "e><<|fim_prefix|>å'Dع9<'VEé'D…½३!　'S😀🏽's👍🏽", "tokens": 33, "pieces": ["e", "><<|", "fim", "_prefix", "|>", "å'D", "ع", "9", "<'", "VEé'D", "…", "½३", "!", "　", "'S", "😀🏽'", "s", "👍🏽"]} +{"text": "　", "tokens": 1, "pieces": ["　"]} +{"text": " \u000b'M\nEOT　 \n<|fim_prefix|>'SEOTſ'll'VE‍Dž'll字'\r9.'reع　", "tokens": 34, "pieces": [" ", "\u000b", "'M", "\n", "EOT", "　 \n", "<|", "fim", "_prefix", "|>'", "SEOTſ'll", "'VE", "‍Dž'll", "字", "'\r", "9", ".'", "reع", "　"]} +{"text": "t (12345678😀🏽字½ع'DⅣ👍🏽㍿\"👍🏽<|endoftext|>İfiꟲⅣ(", "tokens": 43, "pieces": ["t", " ", "(", "123", "456", "78", "😀🏽", "字", "½", "ع'D", "Ⅳ", "👍🏽㍿\"👍🏽<", "EOT", "><|", "endoftext", "|>", "İfiꟲ", "Ⅳ", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\n \n12345678👍🏽İ'llZ!!ſ\u000b\"<㋿Džtİ'ReİＡa👍🏽'Så\t<½,. d'T", "tokens": 49, "pieces": ["\n \n", "123", "456", "78", "👍🏽", "İ'll", "Z", "!!", "ſ", "\u000b", "\"<㋿", "Dž", "t", "İ'Re", "İＡa", "👍🏽'", "Så", "\t", "<", "½", ",.", " d", "'", "T"]} +{"text": "ḍ̇㍿ḍ̇", "tokens": 9, "pieces": ["ḍ̇", "㍿ḍ̇"]} +{"text": "'VE㍿EOT\nİ", "tokens": 9, "pieces": ["'VE", "㍿EOT", "\n", "İ"]} +{"text": "Ⅳ́ 's \n", "tokens": 6, "pieces": ["Ⅳ", "́", " '", "s", " \n"]} +{"text": "(", "tokens": 9, "pieces": ["å", "<|", "endoftext", "|>("]} +{"text": "!‍ \n'Tſ9<𐞁Ⅳ\rt'sfi!!", "tokens": 18, "pieces": ["!‍", " \n", "'Tſ", "9", "<𐞁", "Ⅳ", "\r", "t's", "fi", "!!"]} +{"text": "ḍ̇‍Ａ'Mſİع́İ0fi", "tokens": 16, "pieces": ["ḍ̇", "‍Ａ'M", "ſ", "İع́", "İ", "0", "fi"]} +{"text": "ß👍🏽 ꟲ.'D­字\"😀🏽'll>ع< ſ \r\n\r\n
𐞁😀🏽Z‍٣٤٥٦e", "tokens": 42, "pieces": ["ß", "👍🏽", " ꟲ", ".'", "D", "­字", "\"😀🏽'", "ll", ">", "ع", "<", " ſ", " \r\n\r\n", "
𐞁", "😀🏽", "Z", "‍", "٣٤٥", "٦", "e"]} +{"text": "㋿\"'sꟲ. åéé(😀🏽", "tokens": 17, "pieces": ["㋿\"'", "sꟲ", ".", " åéé", "(😀🏽"]} +{"text": "!", "tokens": 1, "pieces": ["!"]} +{"text": "<ḍ̇\"　'Re!!#$%​!
😀🏽'T", "tokens": 22, "pieces": ["<ḍ̇", "\"", "　", "'Re", "!!#$%​!", "
", "😀🏽'", "T"]} +{"text": "'VE'T­​Ａ #$%t​　\r\n\r\nꟲm0å'३!\tfit<|fim_prefix|>\r12345678'D.'VE<|fim_prefix|>'re", "tokens": 46, "pieces": ["'VE'T", "­​", "Ａ", " ", "#$%", "t", "​", "　\r\n\r\n", "ꟲm", "0", "å", "'", "३", "!", "\tfit", "<|", "fim", "_prefix", "|>\r", "123", "456", "78", "'D", ".'", "VE", "<|", "fim", "_prefix", "|>'", "re"]} +{"text": "ZZß👍🏽٣٤٥٦d'D漢𐞁\r\n'll'ſ…😀🏽!!\"㍿", "tokens": 33, "pieces": ["ZZ", "ß", "👍🏽", "٣٤٥", "٦", "d'D", "漢𐞁", "\r\n", "'ll'ſ", "…", "😀🏽!!\"㍿"]} +{"text": "½'Mdꟲ　\"9Ａfi's'D'ſåé
\r\n\n!ḍ̇\"\r\n'T0ḍ̇👍🏽漢 漢12345678👍🏽ع!\"", "tokens": 47, "pieces": ["½", "'Mdꟲ", "　", "\"", "9", "Ａfi", "'", "s'D", "'ſåé", "
\r\n\n", "!ḍ̇", "\"\r\n", "'T", "0", "ḍ̇", "👍🏽", "漢", " 漢", "123", "456", "78", "👍🏽", "ع", "!\""]} +{"text": "EOT\r\n\r\n'MZ'D<|endoftext|>ḍ̇'T\"'ſ<|endoftext|>", "tokens": 29, "pieces": ["EOT", "\r\n\r\n", "'", "MZ'D", "<|", "endoftext", "|>", "ḍ̇'T", "\"'", "ſ", "<|", "endoftext", "|>"]} +{"text": "İ\rß>t٣٤٥٦éfi", "tokens": 12, "pieces": ["İ", "\r", "ß", ">t", "٣٤٥", "٦", "éfi"]} +{"text": " ſ!!!\r\n\r\n㍿'TDž३é!'Ś'll9३ḍ̇", "tokens": 27, "pieces": [" ſ", "!!!\r\n\r\n", "㍿'", "TDž", "३", "é", "!'", "S", "́'ll", "9३", "ḍ̇"]} +{"text": "\"#$%é🙂fi", "tokens": 6, "pieces": ["\"#$%", "é", "🙂fi"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "(٣٤٥٦३𐞁\r-'D́'漢'ſعEOT.ſt\u000b'…-<|fim_prefix|>😀🏽٣٤٥٦\"12345678ꟲs𐞁\u000b<|fim_prefix|>ḍ̇EOT-,", "tokens": 70, "pieces": ["(", "٣٤٥", "٦३", "𐞁", "\r", "-'", "D́", "'漢'ſ", "ع", "EOT", ".ſt", "\u000b", "'", "…", "-<|", "fim", "_prefix", "|>😀🏽", "٣٤٥", "٦", "\"", "123", "456", "78", "ꟲs𐞁", "", "\u000b", "<|", "fim", "_prefix", "|>", "ḍ̇", "EOT", "-,"]} +{"text": "-­(Ⅳe>Zع,", "tokens": 13, "pieces": ["-­(", "Ⅳ", "e", ">Z", "ع", ","]} +{"text": "㋿\u000b㍿\r\nİ'\r😀🏽'ſ😀🏽a!!'ſ\t>.漢٣٤٥٦ 's㋿9.ås \n\r\n\r\n0 \n'll\"㋿<|endoftext|>m", "tokens": 60, "pieces": ["㋿", "\u000b", "㍿\r\n", "İ", "'\r", "😀🏽'", "ſ", "😀🏽", "a", "!!'", "ſ", "\t", ">.", "漢", "٣٤٥", "٦", " ", " <", "META", "_START", ">'", "s", "㋿", "9", ".ås", " \n\r\n\r\n", "0", " \n", "'ll", "\"㋿<|", "endoftext", "|>", "m"]} +{"text": "ſ　eⅣt're漢<ḍ̇'Dd'DEOT're​", "tokens": 25, "pieces": ["ſ", "　e", "Ⅳ", "t're", "漢", "<ḍ̇'D", "d", "'", "D", "EOT're", "​"]} +{"text": "\u000bEOT00<|endoftext|>‍Z,ḍ̇12345678<|fim_prefix|>sEOTs­am <|fim_prefix|>å👍🏽sＡ漢", "tokens": 50, "pieces": ["\u000bEOT", "00", "<|", "endoftext", "|>‍", "Z", ",<", "EOT", ">ḍ̇", "123", "456", "78", "<|", "fim", "_prefix", "|>", "s", "EOTs", "­am", " ", " <|", "fim", "_prefix", "|>", "å", "👍🏽", "s", "Ａ漢"]} +{"text": "<|endoftext|>9<ꟲEOTⅣfi!tⅣ ß\r\n\r\n​-३'Re", "tokens": 29, "pieces": ["<|", "endoftext", "|>", "9", "<ꟲ", "EOT", "Ⅳ", "fi", "!t", "Ⅳ", " ", " ß", "\r\n\r\n", "​-", "३", "'Re"]} +{"text": "'D😀🏽 'D0٣٤٥٦\r9m🙂\r \n‍", "tokens": 18, "pieces": ["'D", "😀🏽", " '", "D", "0٣٤", "٥٦", "\r", "9", "m", "🙂\r", " \n", "‍"]} +{"text": "é'M🙂Dž
,<㍿'ll\"\t字aḍ̇字<|fim_prefix|>'ſ!!d🙂<|fim_prefix|>İ'VE .‍'s㍿'VE\t12345678İ\u000bⅣé", "tokens": 67, "pieces": ["é'M", "🙂<", "META", "_START", ">Dž", "
", ",<㍿'", "ll", "\"", "\t字aḍ̇字", "<|", "fim", "_prefix", "|>'", "ſ", "!!", "d", "🙂<|", "fim", "_prefix", "|>", "İ'VE", " ", " .‍'", "s", "㍿'", "VE", "\t", "123", "456", "78", "İ", "\u000b", "Ⅳ", "é"]} +{"text": "0,　å漢 ſ", "tokens": 9, "pieces": ["0", ",", "　å漢", " ", " ſ"]} +{"text": "'s<‍\n're‍🙂 \n#$%　'Re'Ms😀🏽s\r'llt…fiḍ̇'ll​ßİ0…\"<ß‍", "tokens": 43, "pieces": ["'s", "<‍\n", "'re", "‍🙂", " \n", "#$%", "　", "'Re'M", "s", "😀🏽", "s", "\r", "'ll", "t", "…fiḍ̇'ll", "​ß", "İ", "0", "…", "\"<", "ß", "‍"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "9'SDž<|fim_prefix|> 𐞁<|endoftext|>'ll !>㍿\u000b", "tokens": 30, "pieces": ["9", "'SDž", "<|", "fim", "_prefix", "|>", " 𐞁", "<|", "endoftext", "|>'", "ll", " ", " !>㍿", "\u000b"]} +{"text": "㍿EOT'ſ㋿\u000b\r\n\r\n　's \nd \nع🙂0!!ſDž'T>éd३é,Ⅳt<​‍İ12345678\r\n\r\n'S\r", "tokens": 51, "pieces": ["㍿EOT'ſ", "㋿", "\u000b\r\n\r\n", "　", "'s", " \n", "d", " \n", "ع", "🙂", "0", "!!", "ſ", "Dž'T", ">éd", "३", "é", ",", "Ⅳ", "t", "<​<", "EOT", ">‍", "İ", "123", "456", "78", "\r\n\r\n", "'S", "\r", ""]} +{"text": "㍿-㋿٣٤٥٦d🙂>t 's\r\n\r\nZ.'VEé\r\nDž å👍🏽‍ ", "tokens": 33, "pieces": ["㍿-㋿", "٣٤٥", "٦", "d", "🙂>", "t", " '", "s", "\r\n\r\n", "Z", ".'", "VEé", "\r\n", "Dž", " ", " å", "👍🏽‍", " "]} +{"text": "́('D…😀🏽ß\r\n 'll'Re'sⅣ\r\n", "tokens": 17, "pieces": ["́", "('", "D", "…", "😀🏽", "ß", "\r\n", " '", "ll'Re", "'s", "Ⅳ", "\r\n"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "½‍😀🏽'll'Mfiå<　12345678­👍🏽å
ḍ̇\r\ń㍿'Reå​ḍ̇", "tokens": 42, "pieces": ["", "½", "‍😀🏽'", "ll'M", "fiå", "<", "　", "123", "456", "78", "­👍🏽", "å", "
ḍ̇", "\r\n", "́", "㍿'", "Reå", "​ḍ̇"]} +{"text": ",ß\r\n'S<ſée‍tt", "tokens": 9, "pieces": [",ß", "\r\n", "'S", "<ſée", "‍tt"]} +{"text": "ſEOT<|fim_prefix|>\u000b0 ꟲ<|fim_prefix|>İ\t <|endoftext|>​<|endoftext|>t,9", "tokens": 41, "pieces": ["ſ", "EOT", "<|", "fim", "_prefix", "|>", "\u000b", "0", " ", " ꟲ", "<|", "fim", "_prefix", "|>", "İ", "\t ", " <|", "endoftext", "|>​<|", "endoftext", "|>", "t", ",", "9"]} +{"text": "\r\n\r\nع३", "tokens": 3, "pieces": ["\r\n\r\n", "ع", "३"]} +{"text": "#$%\r‍­Džḍ̇'T\nḍ̇🙂>!!é'VEé.-𐞁­­Ａ🙂ſ'Sſ\t'ſع字>", "tokens": 47, "pieces": ["#$%\r", "‍­", "Džḍ̇'T", "\n", "ḍ̇", "🙂>!!", "é'VE", "é", ".-", "𐞁", "­­", "Ａ", "🙂ſ'S", "ſ", "", "\t", "'", "ſع字", ">"]} +{"text": "𐞁\r\n½d \n'Reséé'ſſ\"­'llⅣİ😀🏽<|endoftext|>fi", "tokens": 34, "pieces": ["𐞁", "\r\n", "½", "d", " \n", "'Reséé'ſ", "ſ", "\"­'", "ll", "Ⅳ", "İ", "😀🏽<|", "endoftext", "|>", "fi"]} +{"text": "å<|fim_prefix|>'T<\n\na👍🏽\rſ#$%'ll\r<|endoftext|>-'T(\u000b'm0‍ß'ReDž", "tokens": 43, "pieces": ["å", "<|", "fim", "_prefix", "|>'", "T", "<\n\n", "a", "👍🏽\r", "ſ", "#$%'", "ll", "\r", "<|", "endoftext", "|>-'", "T", "(", "\u000b", "'m", "0", "‍ß", "'", "Re", "Dž"]} +{"text": "ßع👍🏽m-'ſ'('M", "tokens": 11, "pieces": ["ßع", "👍🏽", "m", "-'", "ſ", "'('", "M"]} +{"text": "‍EOTꟲ٣٤٥٦", "tokens": 10, "pieces": ["‍EOTꟲ", "٣٤٥", "٦"]} +{"text": "'M½!!\n​ \nⅣꟲ́ ㋿३Ⅳꟲ<|fim_prefix|>'　 stfi<|fim_prefix|>'VEm då\n​", "tokens": 43, "pieces": ["'M", "½", "!!\n", "​", " \n", "Ⅳ", "ꟲ́", " ", " ㋿", "३Ⅳ", "ꟲ", "<|", "fim", "_prefix", "|>'", "　 ", " stfi", "<|", "fim", "_prefix", "|>'", "VEm", " då", "\n", "​"]} +{"text": "👍🏽‍½t", "tokens": 10, "pieces": ["👍🏽‍<", "EOT", ">", "½", "t"]} +{"text": "\r\n,s!!(　'M ꟲ\"‍😀🏽　​😀🏽EOT'reé", "tokens": 24, "pieces": ["\r\n", ",s", "!!(", "　", "'M", " ꟲ", "\"‍😀🏽", "　", "​😀🏽", "EOT're", "é"]} +{"text": "'M- é \n字Dž'll9…EOTḍ̇eİaⅣ\nfi'llＡ'S漢.٣٤٥٦½​9.…\n", "tokens": 44, "pieces": ["'M", "-", " é", " \n", "字", "Dž'll", "9", "…EOTḍ̇e", "İa", "Ⅳ", "\n", "fi'll", "Ａ'S", "漢", ".", "٣٤٥", "٦½", "​<", "EOT", ">", "9", ".", "…\n"]} +{"text": "e!🙂'Reſ<ſ́👍🏽㍿'sm👍🏽　0eＡm", "tokens": 25, "pieces": ["e", "!🙂'", "Reſ", "<ſ́", "👍🏽㍿'", "sm", "👍🏽", "　", "0", "e", "Ａm"]} +{"text": "\n0'<|endoftext|>'ll\u000b­(Ａd́", "tokens": 16, "pieces": ["\n", "0", "'<|", "endoftext", "|>'", "ll", "\u000b", "­(", "Ａd́"]} +{"text": "ſ\t", "tokens": 2, "pieces": ["ſ", "\t"]} +{"text": "‍½m ́​Dž", "tokens": 8, "pieces": ["‍", "½", "m", " ́", "​Dž"]} +{"text": "ḍ̇　fiEOT", "tokens": 10, "pieces": ["ḍ̇", "　fi", "EOT"]} +{"text": ".a(İ
d'D…ع're字㍿😀🏽…'ll字😀🏽>#$%\r\n\r\nm<|fim_prefix|><|endoftext|>!'Mfi,-.é>­'S9dİ", "tokens": 55, "pieces": [".a", "(İ", "
d'D", "…ع're", "字", "㍿😀🏽", "…", "'ll字", "😀🏽>#$%\r\n\r\n", "m", "<|", "fim", "_prefix", "|><|", "endoftext", "|>!'", "Mfi", ",-.", "é", ">­'", "S", "9", "d", "İ"]} +{"text": "ß🙂ع('s'Da👍🏽å'T'M ́عfiZİ'M", "tokens": 21, "pieces": ["ß", "🙂ع", "('", "s'D", "a", "👍🏽", "å'T", "'M", " ́عfi", "Zİ'M"]} +{"text": "ع'VE'sZ'\tİ​　😀🏽", "tokens": 17, "pieces": ["ع'VE", "'s", "Z", "'", "\t", "İ", "​", "　", "😀🏽"]} +{"text": "\r३'s'VEfiꟲ>", "tokens": 10, "pieces": ["\r", "३", "'s'VE", "fiꟲ", ">"]} +{"text": "👍🏽\tEOT漢'Re .　tEOT\u000bß​\n\t!Ⅳ👍🏽\r\nع.a𐞁'VEZ
漢字é'ſ", "tokens": 48, "pieces": ["👍🏽", "\tEOT漢'Re", " ", " <", "META", "_START", ">", " .", "　t", "EOT", "\u000bß", "​\n", "\t", "!", "Ⅳ", "👍🏽\r\n", "ع", ".a", "𐞁'VE", "Z", "
漢字é'ſ"]} +{"text": "d<|fim_prefix|>\r\n\r\n漢'Re\"t　عéå-0　字''ſ'D́9
　 EOT", "tokens": 29, "pieces": ["d", "<|", "fim", "_prefix", "|>\r\n\r\n", "漢'Re", "\"t", "　عéå", "-", "0", "　字", "''", "ſ'D", "́", "9", "
　", " EOT"]} +{"text": " t !", "tokens": 4, "pieces": [" ", " t", " ", " !"]} +{"text": " 👍🏽 Ⅳ\n", "tokens": 8, "pieces": [" ", " 👍🏽", " ", "Ⅳ", "\n"]} +{"text": "<|fim_prefix|>漢s'Me'T'll<|endoftext|><|endoftext|>e!!9㋿,'reⅣ'VEee‍'T>dfiḍ̇İ'lld३", "tokens": 57, "pieces": ["<|", "fim", "_prefix", "|>", "漢s'M", "e'T", "'ll", "<|", "endoftext", "|><|", "endoftext", "|>", "e", "!!", "9", "㋿,'", "re", "Ⅳ", "'VEee", "‍'", "T", ">dfi", "ḍ̇", "İ", "'", "lld", "३"]} +{"text": "½a\"\t😀🏽ſé٣٤٥٦漢\u000b,\t9Ⅳ'T\r\n", "tokens": 26, "pieces": ["½", "a", "\"", "\t", "😀🏽", "ſé", "٣٤٥", "٦", "漢", "\u000b", ",", "\t", "9", "", "Ⅳ", "'T", "\r\n"]} +{"text": "t! 😀🏽ſⅣ \n ꟲ　 m'll漢㍿'漢!!12345678ſd́\t🙂ꟲ\"'Re<|endoftext|>", "tokens": 47, "pieces": ["t", "!", " 😀🏽", "ſ", "Ⅳ", " \n", " ", " ꟲ", "　 ", " m'll", "漢", "㍿'", "漢", "!!", "123", "456", "78", "ſd́", "\t", "🙂ꟲ", "\"'", "Re", "<|", "endoftext", "|>"]} +{"text": "e\né'T<😀🏽ꟲ.\u000ba \nＡ'llfi'll\r\n\r\n'll,\rDžeع<|fim_prefix|>👍🏽ét'(t'Re'ſ#$%Z👍🏽​'reZ", "tokens": 54, "pieces": ["e", "\n", "é'T", "<😀🏽", "ꟲ", ".", "\u000ba", " \n", "Ａ'll", "fi'll", "\r\n\r\n", "'ll", ",\r", "Džeع", "<|", "fim", "_prefix", "|>👍🏽", "ét", "'(", "t'Re", "'", "ſ", "#$%", "Z", "👍🏽​'", "re", "Z"]} +{"text": "́!!ⅣmEOT", "tokens": 7, "pieces": ["́", "!!", "Ⅳ", "m", "EOT"]} +{"text": "!👍🏽 İ\nß>", "tokens": 9, "pieces": ["!👍🏽", " İ", "\n", "ß", ">"]} +{"text": "ſ\r\n\r\n'll(𐞁're㋿'M", "tokens": 14, "pieces": ["ſ", "\r\n\r\n", "'ll", "(𐞁're", "㋿'", "M"]} +{"text": "-(½'Rea!\r\n('Re", "tokens": 7, "pieces": ["-(", "½", "'Rea", "!\r\n", "('", "Re"]} +{"text": "!Zع\r\n­'Da漢ZİDž'　#$%\r\n", "tokens": 16, "pieces": ["!Zع", "\r\n", "­'", "Da漢", "ZİDž", "'", "　", "#$%\r\n"]} +{"text": "ſع \n'ſ#$%!é­s\u000b​ß<字m字𐞁éⅣ'T.<<|endoftext|>'ſ́½9­-(🙂'reſ\n12345678🙂", "tokens": 48, "pieces": ["ſع", " \n", "'ſ", "#$%!", "é", "­s", "\u000b", "​ß", "<字m字𐞁é", "Ⅳ", "'T", ".<<|", "endoftext", "|>'", "ſ́", "½9", "­-(🙂'", "reſ", "\n", "123", "456", "78", "🙂"]} +{"text": "åEOT \nＡ", "tokens": 6, "pieces": ["å", "EOT", " \n", "Ａ"]} +{"text": "'sd'T　 \n漢<>'llas(ḍ̇漢​㍿३字­<<|endoftext|>,…ß㍿㍿", "tokens": 41, "pieces": ["'sd'T", "　 \n", "漢", "<>'", "llas", "(ḍ̇漢", "​㍿", "३", "字", "­<<|", "endoftext", "|>,", "…ß", "㍿㍿<", "META", "_START", ">"]} +{"text": "#$%\t½㍿tßEOT\u000b\r\nZ.'llé३>", "tokens": 19, "pieces": ["#$%", "\t", "½", "㍿tß", "EOT", "\u000b\r\n", "Z", ".'", "llé", "३", ">"]} +{"text": "!!👍🏽👍🏽eİ
­é'll𐞁'll!!<|fim_prefix|>0'Mé \r\n\r\ń㍿<|fim_prefix|>́'ſ'SⅣ-ZＡt", "tokens": 50, "pieces": ["!!👍🏽👍🏽", "e", "İ", "
", "­é'll", "𐞁'll", "!!<|", "fim", "_prefix", "|>", "0", "'Mé", " \r\n\r\n", "́", "㍿<|", "fim", "_prefix", "|>́'", "ſ'S", "Ⅳ", "-ZＡt"]} +{"text": " Dž𐞁 \n字EOT​ḍ̇é𐞁t'Re\u000bm \r\n\r\n'M12345678(<|endoftext|>fiZ.'VE,ꟲ㍿\r\n\r\n🙂'll", "tokens": 57, "pieces": [" ", " Dž𐞁", " \n", "字", "EOT", "​ḍ̇é𐞁t'Re", "\u000bm", "", " \r\n\r\n", "'M", "123", "456", "78", "(<|", "endoftext", "|>", "fi", "Z", ".'", "VE", ",ꟲ", "㍿\r\n\r\n", "🙂'", "ll"]} +{"text": "12345678
عEOTİ,'VE👍🏽Džfi,'VE'👍🏽Ⅳ", "tokens": 27, "pieces": ["", "123", "456", "78", "
ع", "EOTİ", ",'", "VE", "👍🏽", "Džfi", ",'", "VE", "'👍🏽", "Ⅳ"]} +{"text": "< 0s", "tokens": 4, "pieces": ["<", " ", "0", "s"]} +{"text": "ꟲ", "tokens": 3, "pieces": ["ꟲ"]} +{"text": "é'S漢.d'Z!!\ta\u000bＡ<Ⅳ­😀🏽 é'T\t㋿\"0'ſDž< ", "tokens": 40, "pieces": ["é'S", "漢", ".d", "'Z", "!!", "\ta", "\u000bＡ", "<", "Ⅳ", "­<", "META", "_START", ">😀🏽", " é'T", "", "\t", "㋿\"", "0", "'ſ", "Dž", "<", " "]} +{"text": "'re𐞁åm'D \n12345678漢ع<😀🏽\r\n\r\n٣٤٥٦'Seꟲ\r\n\r\n!!<|endoftext|>\t‍'VE<漢٣٤٥٦­ꟲ e'M", "tokens": 55, "pieces": ["'re𐞁åm'D", " \n", "123", "456", "78", "漢ع", "<😀🏽\r\n\r\n", "٣٤٥", "٦", "'Seꟲ", "\r\n\r\n", "!!<|", "endoftext", "|>", "\t", "‍'", "VE", "<漢", "٣٤٥", "٦", "­ꟲ", " e'M"]} +{"text": ",🙂\r\n\r\nعİ ßḍ̇😀🏽12345678 'ſ'", "tokens": 22, "pieces": [",🙂\r\n\r\n", "ع", "İ", " ßḍ̇", "😀🏽", "123", "456", "78", " '", "ſ", "'"]} +{"text": "s\n ­👍🏽90'Re𐞁Ａ>", "tokens": 14, "pieces": ["s", "\n", " ­👍🏽", "90", "'Re𐞁", "Ａ", ">"]} +{"text": "ꟲ(<|endoftext|>", "tokens": 10, "pieces": ["ꟲ", "(<|", "endoftext", "|>"]} +{"text": "‍eEOT12345678'Dꟲ👍🏽ꟲ漢e　İ!'Ⅳ­.e٣٤٥٦字\u000b🙂0'Sع'ſ e'Dꟲte", "tokens": 46, "pieces": ["‍e", "EOT", "123", "456", "78", "'Dꟲ", "👍🏽", "ꟲ漢e", "　İ", "!'", "Ⅳ", "­.", "e", "٣٤٥", "٦", "字", "\u000b", "🙂", "0", "'Sع'ſ", " ", " e'D", "ꟲte"]} +{"text": "Ａ<|endoftext|>\rEOTé!!é>ḍ̇åé
\nmfi \r\n", "tokens": 27, "pieces": ["Ａ", "<|", "endoftext", "|>\r", "EOTé", "!!", "é", ">ḍ̇åé", "
\n", "mfi", " \r\n"]} +{"text": "'sḍ̇<|fim_prefix|>'٣٤٥٦'re
…ꟲe　٣٤٥٦'VE's\u000b '", "٣٤٥", "٦", "'re", "
", "…ꟲe", "　", "٣٤٥", "٦", "'VE's", "\u000b", " <", "Zİ", "½", "\t", "(㋿", "Ａ", "!ß", "\"", "३", "EOT", "!!🙂\r\n", "'re", "İ", " \n", " ", " !'", "T"]} +{"text": "!! 😀🏽- ​s​Ａ\r\nع‍ß\t字'Re'　ſ<'Re\"👍🏽\t>\n<|endoftext|>😀🏽ß\u000b", "tokens": 40, "pieces": ["!!", " ", "😀🏽-", " ​", "s", "​Ａ", "\r\n", "ع", "‍ß", "\t字'Re", "'", "　ſ", "<'", "Re", "\"👍🏽", "\t", ">\n", "<|", "endoftext", "|>😀🏽", "ß", "\u000b"]} +{"text": "🙂-<|endoftext|>‍𐞁(", "tokens": 15, "pieces": ["🙂-<|", "endoftext", "|>‍", "𐞁", "("]} +{"text": "0\n字\ré!👍🏽'ſ'llEOT0<|endoftext|>.😀🏽\rꟲⅣⅣⅣ㍿'T9🙂٣٤٥٦'VEꟲé­​Ⅳ
", "tokens": 58, "pieces": ["0", "\n", "字", "\r", "é", "!👍🏽'", "ſ'll", "EOT", "0", "<|", "endoftext", "|>.😀🏽\r", "ꟲ", "ⅣⅣⅣ", "㍿'", "T", "9", "🙂", "٣٤٥", "٦", "'VEꟲé", "­​", "Ⅳ", "
"]} +{"text": "Ａ…fi٣٤٥٦ \r\n\r\n🙂!'s…'Red\r\nZ字#$%🙂12345678ꟲm!!漢'T½𐞁#$%", "tokens": 44, "pieces": ["Ａ", "…fi", "٣٤٥", "٦", " \r\n\r\n", "🙂!'", "s", "…", "'Red", "\r\n", "Z字", "#$%🙂", "123", "456", "78", "ꟲm", "!!", "漢'T", "½", "𐞁", "#$%<", "EOT", ">"]} +{"text": "0!a.字9'M'llḍ̇𐞁9漢'ſ>fi\t'T", "tokens": 27, "pieces": ["0", "!a", ".字", "9", "'M", "'", "llḍ̇𐞁", "9", "漢'ſ", ">fi", "\t", "'T"]} +{"text": "👍🏽Ⅳ\"!İ0,aİ'D字Ⅳ<ꟲ\t½\u000b 😀🏽", "tokens": 32, "pieces": ["👍🏽", "Ⅳ", "\"!", "İ", "0", ",a", "İ'D", "字", "Ⅳ", "<ꟲ", "\t", "½", "<", "EOT", ">", "\u000b", " ", "😀🏽"]} +{"text": "s\"mＡ12345678å ́🙂'Tع👍🏽'M…(漢ꟲ- 'M\n9", "tokens": 36, "pieces": ["s", "\"m", "Ａ", "123", "456", "78", "å", " ́", "🙂'", "T", "ع", "👍🏽'", "M", "…", "(漢ꟲ", "-", " ", "'M", "\n", "9"]} +{"text": ">ꟲ٣٤٥٦\n漢.ß漢Ⅳ0عå 'ſ'ReEOT'T
​-(12345678, 's🙂stꟲ", "tokens": 41, "pieces": [">ꟲ", "٣٤٥", "٦", "\n", "漢", ".ß漢", "Ⅳ0", "عå", " ", " '", "ſ'Re", "EOT'T", "
", "​-(", "123", "456", "78", ",", " ", " '", "s", "🙂stꟲ"]} +{"text": "ß>12345678🙂<|fim_prefix|>\"<|endoftext|>ع <|fim_prefix|>>0'SEOT  <|endoftext|><|fim_prefix|>漢३,é字é‍\r\n're漢e(d(#$%ßå\u000b", "tokens": 68, "pieces": ["ß", ">", "123", "456", "78", "🙂<|", "fim", "_prefix", "|>\"<|", "endoftext", "|>", "ع", " ", " <|", "fim", "_prefix", "|>>", "0", "'SEOT", " ", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "漢", "३", ",é字é", "‍\r\n", "'re漢e", "(d", "(#$%", "ßå", "\u000b", ""]} +{"text": "fi İDž\r\n
0.'T'VE'Dḍ̇\r\n\r\nt a­𐞁#$%🙂.'", "T'VE", "'Dḍ̇", "\r\n\r\n", "t", " a", "­𐞁", "#$%🙂<", "dé", "(𐞁", "½", "s", "\r\n\r\n", "s"]} +{"text": "!!<'ſå \r\n𐞁fiZ 12345678'reİ( \nḍ̇३#$%'S'ſ\r,m­㍿.\n\r\n​\n'VE \n'ſ", "tokens": 43, "pieces": ["!!<'", "ſå", " \r\n", "𐞁fi", "Z", " ", "123", "456", "78", "'re", "İ", "(", " \n", "ḍ̇", "३", "#$%'", "S'ſ", "\r", ",m", "­㍿.\n\r\n", "​\n", "'VE", " \n", "'ſ"]} +{"text": "å\r'VE\u000b\u000b0>m'sa字a0٣٤٥٦.'S0\t", "tokens": 23, "pieces": ["å", "\r", "'VE", "\u000b", "\u000b", "0", ">m's", "a字a", "0٣٤", "٥٦", ".'", "S", "0", "\t"]} +{"text": " Z!!'D<|fim_prefix|>­ å('VÉ", "tokens": 21, "pieces": [" Z", "!!'", "D", "<|", "fim", "_prefix", "|>­", " å", "('", "VÉ"]} +{"text": "… 'T'VE👍🏽!‍", "tokens": 12, "pieces": ["… ", " '", "T'VE", "👍🏽!‍"]} +{"text": "\r\n\r\n(𐞁ꟲ \u000b'VE㋿👍🏽", "tokens": 19, "pieces": ["\r\n\r\n", "(𐞁ꟲ", " ", "\u000b", "'VE", "㋿👍🏽"]} +{"text": "​s'llé", "tokens": 9, "pieces": ["​s'll", "é", ""]} +{"text": " \u000b 'VE‍m!!!éå. 're'M", "tokens": 19, "pieces": [" \u000b", " '", "VE", "‍<", "EOT", ">m", "!!!", "éå", ".", " '", "re'M"]} +{"text": "३Ⅳ0\n\r\n\r\n,å", "tokens": 11, "pieces": ["३Ⅳ0", "\n\r\n\r\n", ",å", ""]} +{"text": "ms's\r\nd\r\n\r\n12345678'S''Té(Ａ字'VE\tet(
 a
é12345678'D𐞁\r\r\né", "tokens": 37, "pieces": ["ms's", "\r\n", "d", "\r\n\r\n", "123", "456", "78", "'S", "''", "Té", "(Ａ字'VE", "\tet", "(", "
", " a", "
é", "123", "456", "78", "'D𐞁", "\r\r\n", "é"]} +{"text": " fi👍🏽 'll
<　'
'ſ'S \ne👍🏽𐞁‍'sꟲḍ̇", "tokens": 36, "pieces": ["", " ", " fi", "👍🏽", " ", "'ll", "
", "<", "　", "'", "
", "'ſ'S", " \n", "e", "👍🏽", "𐞁", "‍'", "sꟲḍ̇"]} +{"text": "'re-㍿\nå-㍿\r\"𐞁!!½🙂
Dž𐞁#$%ſꟲ,\n!!😀🏽'll'Re𐞁\r\n…e👍🏽", "tokens": 53, "pieces": ["'re", "-㍿\n", "å", "-㍿\r", "\"𐞁", "!!", "½", "🙂", "
Dž𐞁", "#$%", "ſꟲ", ",\n", "!!😀🏽'", "ll'Re", "𐞁", "\r\n", "…e", "👍🏽"]} +{"text": "½Ａ><|endoftext|>½'M>Ⅳع\t>字é🙂 \n!　d\u000b ,​t㋿漢>\r\n", "tokens": 38, "pieces": ["½", "Ａ", "><|", "endoftext", "|>", "½", "'M", ">", "Ⅳ", "ع", "\t", ">字é", "🙂", " \n", "!", "　d", "\u000b ", " ,​", "t", "㋿漢", ">\r\n"]} +{"text": "'٣٤٥٦ßꟲ漢 'Re​ \n … ​'VE\"0İ12345678　…0'T𐞁", "tokens": 35, "pieces": ["'", "٣٤٥", "٦", "ßꟲ漢", " '", "Re", "​", " \n", " …", " ​'", "VE", "\"", "0", "İ", "123", "456", "78", "　", "…", "0", "'T𐞁"]} +{"text": "e漢'S ", "tokens": 4, "pieces": ["e漢'S", " "]} +{"text": "ḍ̇ß 
\n", "tokens": 11, "pieces": ["ḍ̇ß", " 
\n"]} +{"text": "ſＡ", "tokens": 2, "pieces": ["ſ", "Ａ"]} +{"text": "\n­Dž٣٤٥٦,.
'D'ſ s'EOT'字e😀🏽0aEOT'M㍿'S\"㋿'VEع", "tokens": 42, "pieces": ["\n", "­Dž", "٣٤٥", "٦", ",.", "
", "'D'ſ", " s", "'EOT", "'字e", "😀🏽", "0", "a", "EOT'M", "㍿'", "S", "\"㋿'", "VE", "ع"]} +{"text": "字😀🏽9ḍ̇EOTZ'll\t३३'T
s­İ­12345678'VE漢(\r\n\r\n漢İ're㋿ع'VEİ漢👍🏽Ａa", "tokens": 53, "pieces": ["字", "😀🏽", "9", "ḍ̇", "EOTZ'll", "\t", "३३", "'T", "
s", "­İ", "­", "123", "456", "78", "'VE漢", "(\r\n\r\n", "漢", "İ", "'", "re", "㋿ع'VE", "İ漢", "👍🏽<", "META", "_START", ">Ａa"]} +{"text": "­\"…‍.
d#$%'ſ'S'll'VE.ſ12345678𐞁's<|fim_prefix|> 'ſ12345678d<|endoftext|>\"!!​ å𐞁DžⅣ\r", "tokens": 65, "pieces": ["­\"", "…", "‍.", "
d", "#$%'", "ſ'S", "'ll'VE", ".ſ", "123", "456", "78", "𐞁's", "<|", "fim", "_prefix", "|>", " ", " '", "ſ", "123", "456", "78", "d", "<|", "endoftext", "|>\"<", "EOT", "><", "META", "_START", ">!!​", " ", " å𐞁", "Dž", "Ⅳ", "\r"]} +{"text": "å'SéꟲDžⅣ३㍿m\r\n😀🏽Ⅳ'Reḍ̇㍿ ٣٤٥٦", "tokens": 37, "pieces": ["å'S", "éꟲ", "Dž", "Ⅳ३", "㍿m", "\r\n", "😀🏽", "Ⅳ", "'Reḍ̇", "㍿", " ", "٣٤٥", "٦"]} +{"text": "\r\n>ée'S<|endoftext|>ḍ̇Ａ字<|fim_prefix|>fi", "tokens": 25, "pieces": ["\r\n", ">ée'S", "<|", "endoftext", "|>", "ḍ̇", "Ａ字", "<|", "fim", "_prefix", "|>", "fi"]} +{"text": "\n'VE…\"ſß's'ſ<|fim_prefix|>\r", "tokens": 18, "pieces": ["\n", "'VE", "…", "\"ſß's", "'ſ", "<|", "fim", "_prefix", "|>\r"]} +{"text": "'Dm​'VEåع12345678!!　👍🏽\n \né", "tokens": 19, "pieces": ["'Dm", "​'", "VEåع", "123", "456", "78", "!!", "　", "👍🏽\n", " \n", "é"]} +{"text": "字-0'VE,­ع#$%'D'T\r\n-fi", "tokens": 19, "pieces": ["字", "-", "0", "'VE", ",­", "ع", "#$%'", "D'T", "\r\n", "-fi"]} +{"text": "'ll>🙂漢eḍ̇'M\n\r\n‍ 👍🏽", "tokens": 15, "pieces": ["'ll", ">🙂", "漢eḍ̇'M", "\n\r\n", "‍", " ", "👍🏽"]} +{"text": "12345678éſ漢!\n9­'ſ>ḍ̇'́ ", "tokens": 19, "pieces": ["123", "456", "78", "éſ漢", "!\n", "9", "­'", "ſ", ">ḍ̇", "'́", " "]} +{"text": "\t,漢>'　dDž>\rDžZꟲ­'ll \n Ⅳ d's0'Re'D​", "tokens": 30, "pieces": ["\t", ",漢", ">'", "　d", "Dž", ">\r", "DžZꟲ", "­'", "ll", " \n", " ", "Ⅳ", " ", " d's", "0", "'Re'D", "​"]} +{"text": "\r\n\r\n-'S !!-\u000b‍", "tokens": 8, "pieces": ["\r\n\r\n", "-'", "S", " ", " !!-", "\u000b", "‍"]} +{"text": "é​ꟲ'll‍'re…\rع😀🏽tꟲ\t\"a🙂m,\r\n\r\n
३½ \nt,'ſDžع👍🏽𐞁EOTfiß#$%Dž'll'M ", "tokens": 53, "pieces": ["é", "​ꟲ'll", "‍'", "re", "…\r", "ع", "😀🏽", "tꟲ", "\t", "\"a", "🙂m", ",\r\n\r\n", "
", "३½", " \n", "t", ",'", "ſ", "Džع", "👍🏽", "𐞁EOTfiß", "#$%", "Dž'll", "'M", " "]} +{"text": "́㍿'llt'll\"'S　字", "tokens": 11, "pieces": ["́", "㍿'", "llt'll", "\"'", "S", "　字"]} +{"text": "'M\t　é>'D㋿İ", "tokens": 10, "pieces": ["'M", "\t", "　é", ">'", "D", "㋿İ"]} +{"text": "\"é>t'D<|endoftext|>", "tokens": 12, "pieces": ["\"é", ">t'D", "<|", "endoftext", "|>"]} +{"text": "Ａ😀🏽'ſ#$%'re'll#$%​ ㍿　 ", "tokens": 22, "pieces": ["Ａ", "😀🏽'", "ſ", "#$%'", "re", "'", "ll", "#$%​", " ㍿", "　 "]} +{"text": "ꟲꟲfi< m漢9EOTs𐞁
'll㋿𐞁.a.Ⅳ… \n<|fim_prefix|>٣٤٥٦🙂½é'Reſ", "tokens": 57, "pieces": ["ꟲꟲfi", "<<", "EOT", ">", " m漢", "9", "EOTs𐞁", "
", "'ll", "㋿𐞁", ".a", ".", "Ⅳ", "… \n", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "🙂", "½", "é", "'", "Reſ"]} +{"text": "'S'D'se'ſ9Ⅳſ🙂㍿Z\r\n­\u000b", "tokens": 18, "pieces": ["'S'D", "'se'ſ", "9Ⅳ", "ſ", "🙂㍿", "Z", "\r\n", "­", "\u000b"]} +{"text": "!\n,(a…aEOT𐞁'VE0'", "VE", "0", "#$%\"", "tokens": 16, "pieces": ["'llſ", "\r", "123", "456", "78", "d", "<|", "endoftext", "|>#$%\""]} +{"text": "'", "tokens": 1, "pieces": ["'"]} +{"text": "㋿Ａ漢s'Tع
'Mꟲḍ̇é(-　åaſ​Džt\tＡ'S!漢 -漢 ‍<|endoftext|>😀🏽'D", "tokens": 50, "pieces": ["㋿Ａ漢s'T", "ع", "
", "'Mꟲḍ̇é", "(-", "　åaſ", "​Džt", "\tＡ'S", "!漢", " ", " -", "漢", " ", " ‍<|", "endoftext", "|>😀🏽'", "D"]} +{"text": "'ll'VE३ß 0字­ 'ReİeEOTⅣ\nZ​", "tokens": 21, "pieces": ["'ll'VE", "३", "ß", " ", "0", "字", "­", " ", " '", "Re", "İe", "EOT", "Ⅳ", "\n", "Z", "​"]} +{"text": "İＡſ　#$%㋿'M", "tokens": 11, "pieces": ["İＡſ", "　", "#$%㋿'", "M"]} +{"text": "\"", "tokens": 1, "pieces": ["\""]} +{"text": "m#$%>'Mm.ZZ👍🏽㋿'T
㍿e\"㋿٣٤٥٦9\tḍ̇0½", "tokens": 39, "pieces": ["m", "#$%>'", "Mm", ".ZZ", "👍🏽<", "META", "_START", ">㋿'", "T", "
", "㍿e", "\"㋿", "٣٤٥", "٦9", "\tḍ̇", "0½"]} +{"text": " \n\t(½EOTßt漢\t㋿EOTß\r\n\r\n㍿'𐞁\u000ba½🙂", "tokens": 28, "pieces": [" \n", "\t", "(", "½", "EOTßt漢", "\t", "㋿EOTß", "\r\n\r\n", "㍿'", "𐞁", "\u000ba", "½", "🙂"]} +{"text": "9漢'SDž𐞁㍿ع'ſ३Ⅳ(é½👍🏽\t\r\n,", "tokens": 26, "pieces": ["9", "漢'S", "Dž𐞁", "㍿ع'ſ", "३Ⅳ", "(é", "½", "👍🏽", "\t\r\n", ","]} +{"text": "'ſ😀🏽#$%\r\n­<|fim_prefix|><> EOT>​éⅣ'D\r'D\tt­'s 'Re>\"‍EOT#$%\r\nḍ̇fi d\r\n\r\n😀🏽", "tokens": 54, "pieces": ["'ſ", "😀🏽#$%\r\n", "­<|", "fim", "_prefix", "|><>", " ", " EOT", ">​", "é", "Ⅳ", "'", "D", "\r", "'D", "\tt", "­'", "s", " ", " '", "Re", ">\"‍", "EOT", "#$%\r\n", "ḍ̇fi", " ", " d", "\r\n\r\n", "😀🏽"]} +{"text": "'Ree'sé(#$%Dž🙂́'ſ‍", "tokens": 14, "pieces": ["'Ree's", "é", "(#$%", "Dž", "🙂́'ſ", "‍"]} +{"text": "ع𐞁eＡ\r\n\r\nfi‍\"　'ſ'VE'sſ'Re.Dž𐞁å'll<|endoftext|>,😀🏽㋿👍🏽ḍ̇<|endoftext|>t .㍿<<🙂
", "tokens": 69, "pieces": ["ع𐞁e", "Ａ", "\r\n\r\n", "fi", "‍\"", "　", "'ſ", "'", "VE's", "ſ'Re", ".Dž𐞁å'll", "<|", "endoftext", "|>,😀🏽㋿👍🏽", "ḍ̇", "<|", "endoftext", "|>", "t", " ", " <", "META", "_START", ">.㍿<<🙂", "
"]} +{"text": "'ll>'s
\r,𐞁<|fim_prefix|>Z afi9字're字 ḍ̇<|endoftext|>字\r\n\r\n'sＡ0 d\n#$%12345678‍s'VEm字…", "tokens": 60, "pieces": ["'ll", ">'", "s", "", "
\r", ",𐞁", "<|", "fim", "_prefix", "|>", "Z", " ", " afi", "9", "字're", "字", " ḍ̇", "<|", "endoftext", "|>", "字", "\r\n\r\n", "'s", "Ａ", "0", " ", " d", "\n", "#$%", "123", "456", "78", "‍s'VE", "m字", "…"]} +{"text": "Ａ\n-漢Dž👍🏽٣٤٥٦'ſ'D३ 漢　é", "tokens": 22, "pieces": ["Ａ", "\n", "-漢", "Dž", "👍🏽", "٣٤٥", "٦", "'ſ'D", "३", " 漢", "　é"]} +{"text": "'S#$%'ß(😀🏽,é½EOTꟲ'VE\r'SⅣ12345678<(ßt'D‍　'M'Ret'VE", "tokens": 36, "pieces": ["'S", "#$%'", "ß", "(😀🏽,", "é", "½", "EOTꟲ'VE", "\r", "'S", "Ⅳ12", "345", "678", "<(", "ßt'D", "‍", "　", "'M'Re", "t'VE"]} +{"text": "'re字'M(\u000b're𐞁'Ś9Dž\r\n\r\nDž \n('reꟲḍ̇
\rs \n", "tokens": 34, "pieces": ["'re字'M", "(", "\u000b", "'re", "𐞁'S", "́", "9", "Dž", "\r\n\r\n", "Dž", " \n", "('", "reꟲḍ̇", "
\r", "s", " \n"]} +{"text": "​!!İＡ'M!!'sfimⅣ　ꟲ👍🏽t👍🏽…é0é's>å　'ſ<|endoftext|>٣٤٥٦éＡ　Dž(  Dž㍿́ḍ̇😀🏽", "tokens": 66, "pieces": ["​!!", "İＡ'M", "!!'", "sfim", "Ⅳ", "　ꟲ", "👍🏽", "t", "👍🏽", "…é", "0", "é's", ">å", "　", "'ſ", "<|", "endoftext", "|>", "٣٤٥", "٦", "é", "Ａ", "　Dž", "(", " ", " Dž", "㍿́ḍ̇", "😀🏽"]} +{"text": "🙂\t😀🏽dＡ-😀🏽éⅣ́\r\n\r\n👍🏽 ſ́ \n- m\" 'Sd", "tokens": 31, "pieces": ["🙂", "\t", "😀🏽", "d", "Ａ", "-😀🏽", "é", "Ⅳ", "́", "\r\n\r\n", "👍🏽", " ſ́", " \n", "-", " ", " m", "\"", " ", "'Sd"]} +{"text": "😀🏽!!éꟲEOT́ꟲ😀🏽'T𐞁½…👍🏽'EOT.ßa𐞁a", "tokens": 40, "pieces": ["😀🏽!!", "éꟲ", "EOT́ꟲ", "😀🏽'", "T𐞁", "½", "…", "👍🏽'", "EOT", ".ßa𐞁a"]} +{"text": "!!t.99٣٤٥٦Ⅳع,漢字\u000bİ9\u000b", "tokens": 18, "pieces": ["!!", "t", ".", "99٣", "٤٥٦", "Ⅳ", "ع", ",漢字", "\u000bİ", "9", "\u000b"]} +{"text": ".\"Ａ\r>\u000bte<|endoftext|>#$%<|fim_prefix|>,eeå'Re👍🏽 s'Re 'Reſ\n>12345678#$%漢… \n<|fim_prefix|>\t", "tokens": 52, "pieces": [".\"", "Ａ", "\r", ">", "\u000bte", "<|", "endoftext", "|>#$%<|", "fim", "_prefix", "|>,", "eeå'Re", "👍🏽", " s'Re", " ", " '", "Reſ", "\n", ">", "123", "456", "78", "#$%", "漢", "… \n", "<|", "fim", "_prefix", "|>", "\t"]} +{"text": "'VE're​​\u000b\n\n'> 'll\u000b𐞁 ß' t㍿m٣٤٥٦‍t0­ İ👍🏽Dž‍t\r", "tokens": 44, "pieces": ["'VE're", "​​", "\u000b\n\n", "'>", " '", "ll", "\u000b𐞁", " ß", "'", " ", " t", "㍿", "m", "٣٤٥", "٦", "‍t", "0", "­", " ", " İ", "👍🏽", "Dž", "‍t", "\r"]} +{"text": " ㍿㍿\nſ‍!'0👍🏽", "tokens": 15, "pieces": [" ", "㍿㍿\n", "ſ", "‍!'", "0", "👍🏽"]} +{"text": "ḍ̇ḍ̇'M字sḍ̇\t#$%t,'VEéß३0'Sé'll‍👍🏽,s", "tokens": 33, "pieces": ["ḍ̇ḍ̇'M", "字sḍ̇", "\t", "#$%", "t", ",'", "VEéß", "३0", "'Sé'll", "‍👍🏽,", "s"]} +{"text": "㍿sfi३Z
…😀🏽!!\"漢12345678'sfiZd字Dž𐞁Zå0.d9aß\"<|fim_prefix|>'12345678㋿字Z's're", "tokens": 55, "pieces": ["㍿sfi", "३", "Z", "", "
", "…", "😀🏽!!\"", "漢", "123", "456", "78", "'sfi", "Zd字", "Dž𐞁Zå", "0", ".d", "9", "aß", "\"<|", "fim", "_prefix", "|>'", "123", "456", "78", "㋿字", "Z's", "'re"]} +{"text": " \"'VE're 9ḍ̇éeⅣ'VE!!㍿\t.", "tokens": 22, "pieces": [" ", " \"'", "VE're", " ", "9", "ḍ̇ée", "Ⅳ", "'VE", "!!㍿", "\t", "."]} +{"text": "½å㋿🙂𐞁\r\n12345678's…", "tokens": 18, "pieces": ["½", "å", "㋿🙂", "𐞁", "\r\n", "123", "456", "78", "'s", "…"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "
ع…
séİ\r\n\r\n,dſéZ \n 'DEOT'Re", "tokens": 18, "pieces": ["
ع", "…", "
sé", "İ", "\r\n\r\n", ",dſé", "Z", " \n", " '", "DEOT'Re"]} +{"text": "9dZ\rZＡ'll \t!!(\t​\u000bfi0　sİe­#$%'ſm\u000bm", "tokens": 37, "pieces": ["9", "d", "Z", "\r", "Z", "Ａ", "'", "ll", " ", "\t", "!!(", "\t", "​", "\u000bfi", "0", "　s", "İe", "­#$%'", "ſm", "\u000bm"]} +{"text": "<0‍ZEOTm'D
'll12345678.Z'llZé<ſfiſ0Ⅳ>", "tokens": 24, "pieces": ["<", "0", "‍ZEOTm'D", "
", "'ll", "123", "456", "78", ".Z'll", "Zé", "<ſfiſ", "0Ⅳ", ">"]} +{"text": " å👍🏽12345678٣٤٥٦㍿ ३𐞁s \t\"३'lle'T'‍''VE 漢Ⅳ<|endoftext|> \n.👍🏽漢fí٣٤٥٦ſ12345678'VE", "tokens": 66, "pieces": [" ", " å", "👍🏽", "123", "456", "78٣", "٤٥٦", "㍿", " ", "३", "𐞁s", " ", "\t", "\"", "३", "'lle'T", "'‍''", "VE", " 漢", "Ⅳ", "<|", "endoftext", "|>", " \n", ".👍🏽", "漢fí", "٣٤٥", "٦", "ſ", "123", "456", "78", "'", "VE"]} +{"text": "Ⅳ \nḍ̇t<|endoftext|>Ａt　!,㍿!!٣٤٥٦🙂ⅣEOT#$%Ａ\r\n\r\n", "Ａt", "　", "!,㍿!!", "٣٤٥", "٦", "🙂", "Ⅳ", "EOT", "#$%", "Ａ", "\r\n\r\n", ",½\"\té'reİ\t(漢#$%\r\n\r\nå'!!٣٤٥٦\r'M
Zé٣٤٥٦́İ ́­'३😀🏽å'<|fim_prefix|>", "tokens": 57, "pieces": ["<|", "fim", "_prefix", "|>,", "½", "\"", "\té're", "İ", "\t", "(漢", "#$%\r\n\r\n", "å", "'!!", "٣٤٥", "٦", "\r", "'M", "
Zé", "٣٤٥", "٦", "́", "İ", " ", " ́", "­'", "३", "😀🏽", "å", "'<|", "fim", "_prefix", "|>"]} +{"text": "\r\n…\u000b𐞁é-mſ…\r\n\r\nß<|endoftext|>d0 (字३½s字́'T\r\t'ſ'sⅣ́\tDž\r\n", "tokens": 44, "pieces": ["\r\n", "…", "\u000b𐞁é", "-mſ", "…\r\n\r\n", "ß", "<|", "endoftext", "|>", "d", "0", " (", "字", "३½", "s字́'T", "\r", "\t", "'ſ's", "Ⅳ", "́", "\tDž", "\r\n"]} +{"text": "#$%!EOTdé'S\u000bß \n३<|endoftext|>ß­s‍'ll0‍", "tokens": 33, "pieces": ["#$%!", "EOTdé'S", "\u000b", "ß", " \n", "३", "<|", "endoftext", "|>", "ß", "­", "s", "‍'", "ll", "0", "‍"]} +{"text": "İꟲ12345678", "tokens": 7, "pieces": ["İꟲ", "123", "456", "78"]} +{"text": "Dž(!!Zİ<­ 'Ree,
", "tokens": 14, "pieces": ["Dž", "(!!", "Zİ", "<­", " ", " '", "Ree", ",", "
"]} +{"text": "字 🙂s…Dž\r\n\r\n'ſ\tsİ𐞁\r'T\rm'VE-a9,e0!漢eİ'VE's s'Re字#$%‍<|endoftext|>", "tokens": 47, "pieces": ["(s", "!!", "ع", "​d", "'", "ſ", "\ts", "İ𐞁", "\r", "'T", "\r", "m'VE", "-a", "9", ",e", "0", "!漢e", "İ'VE", "'s", " s'Re", "字", "#$%‍<|", "endoftext", "|>"]} +{"text": "٣٤٥٦٣٤٥٦EOTé\u000b<|fim_prefix|>𐞁!9漢(字é…😀🏽eİ\"e \t!!Ａ'reḍ̇'saZ\r\ntḍ̇­㍿\r\n\r\n𐞁", "tokens": 63, "pieces": ["٣٤٥", "٦٣٤", "٥٦", "EOTé", "\u000b", "<|", "fim", "_prefix", "|>", "𐞁", "!", "9", "漢", "(字é", "…", "😀🏽", "e", "İ", "\"e", " ", "\t", "!!", "Ａ're", "ḍ̇'s", "a", "Z", "\r\n", "tḍ̇", "­㍿\r\n\r\n", "𐞁"]} +{"text": "éع(Ａ>३#$%!
\n\t'llع\r\n\r\nⅣd'VEs(\n'Re", "tokens": 26, "pieces": ["éع", "(Ａ", ">", "३", "#$%!", "
\n", "\t", "'llع", "\r\n\r\n", "Ⅳ", "d'VE", "s", "(\n", "'Re"]} +{"text": "عＡZ٣٤٥٦#$%", "tokens": 13, "pieces": ["ع", "ＡZ", "٣٤٥", "٦", "#$%"]} +{"text": "\u000bd😀🏽DžＡ३٣٤٥٦\u000bs#$%\t㍿'S,عé‍m\u000b-s!!­Z'#$%", "tokens": 37, "pieces": ["\u000bd", "😀🏽", "DžＡ", "३٣٤", "٥٦", "\u000bs", "#$%", "\t", "㍿'", "S", ",عé", "‍m", "\u000b", "-s", "!!­", "Z", "'#$%"]} +{"text": "eé
<|fim_prefix|>'re'D'", "re'D", "'T㋿! \n…!\"'s٣٤٥٦٣٤٥٦'re　", "tokens": 32, "pieces": ["Ⅳ", "'VE字'VE", "'", "T", "㋿!", " \n", "…", "!\"'", "s", "٣٤٥", "٦٣٤", "٥٦", "'re", "　"]} +{"text": "-🙂d𐞁🙂 <\r\n㍿t­#$%'T å<|endoftext|>🙂'lls", "tokens": 36, "pieces": ["𐞁", "🙂", " ", "<\r\n", "㍿t", "­#$%'", "T", " å", "<|", "endoftext", "|>🙂'", "lls"]} +{"text": "'ſ !㍿,\r\n\r\n<|fim_prefix|>ßefi,㍿'Dİ🙂\t\r\n<|endoftext|>.ß9🙂😀🏽", "tokens": 37, "pieces": ["'ſ", " !㍿,\r\n\r\n", "<|", "fim", "_prefix", "|>", "ßefi", ",㍿'", "Dİ", "🙂", "\t\r\n", "<|", "endoftext", "|>.", "ß", "9", "🙂😀🏽"]} +{"text": "漢漢eعés\rDž'VE　Z", "tokens": 12, "pieces": ["漢漢eعés", "\r", "Dž'VE", "　Z"]} +{"text": "!fit#$%‍Džfi İ🙂🙂 ㍿‍́'ſ<|fim_prefix|>", "tokens": 27, "pieces": ["!fit", "#$%‍", "Džfi", " İ", "🙂🙂", " ", "㍿‍́'", "ſ", "<|", "fim", "_prefix", "|>"]} +{"text": "Z👍🏽s#$%<|fim_prefix|>ḍ̇", "tokens": 16, "pieces": ["Z", "👍🏽", "s", "#$%<|", "fim", "_prefix", "|>", "ḍ̇"]} +{"text": "\n'VEZé'll,…😀🏽d<|endoftext|>㋿ß!<|endoftext|>½é\r'lléＡt👍🏽.'VE'!!Ⅳ", "tokens": 49, "pieces": ["\n", "'VEZé'll", ",", "…", "😀🏽", "d", "<|", "endoftext", "|>㋿", "ß", "!<|", "endoftext", "|>", "½", "é", "\r", "'llé", "Ａt", "👍🏽.'", "VE", "'!!", "Ⅳ"]} +{"text": "ſ½​­😀🏽½\n'VE('عⅣ…'reé'reee12345678😀🏽- <'ꟲ'ReDž12345678< 　'VEt!! é\r漢's", "tokens": 53, "pieces": ["ſ", "½", "​­😀🏽", "½", "\n", "'VE", "('", "ع", "Ⅳ", "…", "'reé're", "ee", "123", "456", "78", "😀🏽-", " <'", "ꟲ'Re", "Dž", "123", "456", "78", "<", " ", "　", "'VEt", "!!", " é", "\r", "漢's"]} +{"text": "Ａ­ꟲ<'ll
", "tokens": 8, "pieces": ["Ａ", "­ꟲ", "<'", "ll", "
"]} +{"text": "Dž　#$%㋿字㍿٣٤٥٦d\t… \n🙂's,-", "tokens": 25, "pieces": ["Dž", "　", "#$%㋿", "字", "㍿", "٣٤٥", "٦", "d", "\t… \n", "🙂'", "s", ",-"]} +{"text": "🙂'VE\r­'ſⅣꟲfié<|fim_prefix|>,(", "tokens": 24, "pieces": ["🙂'", "VE", "\r", "­<", "EOT", ">'", "ſ", "Ⅳ", "ꟲfié", "<|", "fim", "_prefix", "|>,("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "12345678Z<|endoftext|>\".\t> ½\u000b'M 12345678#$%㋿\u000bå漢at\u000bd<|endoftext|>!t👍🏽 > …s\tſ", "tokens": 56, "pieces": ["123", "456", "78", "Z", "<|", "endoftext", "|><", "EOT", ">\".", "\t", ">", " ", "½", "\u000b", "'M", " ", "123", "456", "78", "#$%㋿", "\u000bå漢at", "\u000bd", "<|", "endoftext", "|>!", "t", "👍🏽", " ", " >", " ", "…s", "\tſ"]} +{"text": "٣٤٥٦ Z漢́ 'VE‍'VEعع🙂३té\t\r\n\r\n>🙂'Sé 'S\r½　", "tokens": 37, "pieces": ["٣٤٥", "٦", "", " Z漢́", " ", "'VE", "‍'", "VEعع", "🙂", "३", "té", "\t\r\n\r\n", "><", "EOT", ">🙂'", "Sé", " '", "S", "\r", "½", "　"]} +{"text": " <|endoftext|><|endoftext|>..t<|fim_prefix|>'T#$%Dž<'D👍🏽ås'll'fi​'llꟲ'Sd", "tokens": 46, "pieces": [" ", "<|", "endoftext", "|><|", "endoftext", "|>..", "t", "<|", "fim", "_prefix", "|>'", "T", "#$%", "Dž", "<'", "D", "👍🏽", "ås'll", "'fi", "​'", "llꟲ'S", "d"]} +{"text": "(.Ⅳ
Dž9", "tokens": 7, "pieces": ["(.", "Ⅳ", "
Dž", "9"]} +{"text": "… -,-Ａtfi.عſ'VE'VE㋿㋿'\r> \n'字 ㋿<|fim_prefix|>'ll
'S㋿ 漢9e漢Z", "tokens": 50, "pieces": ["… ", " -,-", "Ａtfi", ".عſ'VE", "'VE", "㋿㋿'\r", ">", " \n", "'字", " ", " ㋿<|", "fim", "_prefix", "|>'", "ll", "
", "'S", "㋿", " 漢", "9", "e漢", "Z"]} +{"text": "́㋿<|fim_prefix|>🙂\r\n\r\nＡ
e
\"d12345678fiDž　's'Re <|endoftext|><|fim_prefix|>fi𐞁å㋿ع…🙂­३'(ß🙂\r\n\r\n", "Ａ", "
e", "
", "\"d", "123", "456", "78", "fi", "Dž", "　", "'s'Re", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "fi𐞁å", "㋿ع", "…", "🙂­", "३", "'(", "ß", "é>#$% '", "tokens": 40, "pieces": ["!'", "T", "٣٤٥", "٦", "m", "123", "456", "78", "é", "　", "👍🏽", "é", " \n", "…é", ",'", "S", "(­", "İ'S", "\n", "<|", "endoftext", "|>", "é", ">#$%", " '"]} +{"text": "!👍🏽
é12345678'll'Re🙂…Ａ#$%!🙂're\rİ>", "tokens": 31, "pieces": ["!👍🏽", "
é", "123", "456", "78", "'ll'Re", "🙂", "…Ａ", "#$%!🙂'", "re", "\r", "İ", "><", "EOT", ">"]} +{"text": "DžⅣ \nſ 're…'-tfi'reⅣ \ń.dma", "tokens": 28, "pieces": ["Dž", "Ⅳ", " \n", "ſ", " ", " '", "re", "…", "'-", "t", "fi're", "Ⅳ", " \n", "́", ".dma", ""]} +{"text": "'(👍🏽e#$%'S😀🏽\tß-​Ａa㍿\r\n\r\n eå<|fim_prefix|>\u000b!\n\t😀🏽🙂\tt'llZ㍿　", "tokens": 47, "pieces": ["'(👍🏽", "e", "#$%'", "S", "😀🏽", "\tß", "-​", "Ａa", "㍿\r\n\r\n", " ", " <", "META", "_START", ">eå", "<|", "fim", "_prefix", "|>", "\u000b", "!\n", "\t", "😀🏽🙂", "\tt'll", "Z", "㍿", "　"]} +{"text": "字ꟲ fi​t12345678<'llEOT ­👍🏽\r\n
<|fim_prefix|>a㍿́-<|fim_prefix|>\t <́#$%Z,\nİ \n\r\n", "tokens": 48, "pieces": ["字ꟲ", " fi", "​t", "123", "456", "78", "<'", "ll", "EOT", " ­👍🏽\r\n", "
", "<|", "fim", "_prefix", "|>", "a", "㍿́", "-<|", "fim", "_prefix", "|>", "\t", " <́#$%", "Z", ",\n", "İ", " \n\r\n"]} +{"text": "㋿", "tokens": 3, "pieces": ["㋿"]} +{"text": "<|fim_prefix|>d字\r\n\r\n-('Reé'VEع　👍🏽٣٤٥٦\r\n\r\nſ𐞁\n٣٤٥٦İ!!>\r​😀🏽9", "tokens": 45, "pieces": ["<|", "fim", "_prefix", "|>", "d字", "\r\n\r\n", "-('", "Reé'VE", "ع", "　", "👍🏽", "٣٤٥", "٦", "\r\n\r\n", "ſ𐞁", "\n", "٣٤٥", "٦", "İ", "!!>\r", "​😀🏽", "9"]} +{"text": "
a'Dß,!'S<|endoftext|>\tⅣ'D'S½å😀🏽ع　­ßtع!\u000b!! \n­漢9㋿‍!", "tokens": 47, "pieces": ["
a'D", "ß", ",!'", "S", "<|", "endoftext", "|>", "\t", "Ⅳ", "'D'S", "½", "å", "😀🏽", "ع", "　", "­ß", "tع", "!", "\u000b", "!!", " \n", "­漢", "9", "㋿‍!"]} +{"text": "३ ß😀🏽<|endoftext|>½​́'S\"!!Ａ𐞁漢😀🏽\r٣٤٥٦'VE　😀🏽\u000b​\n
Z\r\n\r\n!!㍿EOT<|fim_prefix|>Ａ‍Z‍ (", "tokens": 62, "pieces": ["३", " ", " ß", "😀🏽<|", "endoftext", "|>", "½", "​́'S", "\"!!", "Ａ𐞁漢", "😀🏽\r", "٣٤٥", "٦", "'VE", "　", "😀🏽", "\u000b", "​\n", "
Z", "\r\n\r\n", "!!㍿", "EOT", "<|", "fim", "_prefix", "|>", "Ａ", "‍Z", "‍", " ("]} +{"text": ".a12345678å 'll 's!!𐞁 \n.t'Re३٣٤٥٦ⅣEOTEOT𐞁", "tokens": 34, "pieces": [".a", "123", "456", "78", "å", " ", " '", "ll", " ", " '", "s", "!!", "𐞁", " \n", ".t'Re", "३٣٤", "٥٦Ⅳ", "EOTEOT𐞁"]} +{"text": " \n٣٤٥٦", "tokens": 5, "pieces": [" \n", "٣٤٥", "٦"]} +{"text": ">ſ9!!İ,عß½te\"​é'Té漢m\u000b \t­ #$%9<İ\r's09a\r👍🏽", "tokens": 38, "pieces": [">ſ", "9", "!!", "İ", ",عß", "½", "te", "\"​", "é'T", "é漢m", "\u000b ", "\t", "­", " ", " #$%", "9", "<İ", "\r", "'s", "09", "a", "\r", "👍🏽"]} +{"text": "½​- ३𐞁𐞁éع字's9t'D🙂😀🏽#$%ß字٣٤٥٦s'M", "tokens": 40, "pieces": ["½", "​-<", "EOT", ">", " ", " ", "३", "𐞁𐞁éع字's", "9", "t'D", "🙂😀🏽#$%", "ß字", "٣٤٥", "٦", "s'M"]} +{"text": "३é.'Re-٣٤٥٦Dž#$%𐞁​́é<|fim_prefix|>ꟲ\".😀🏽<𐞁t12345678dd\t\t(ḍ̇-　", "tokens": 52, "pieces": ["३", "é", ".'", "Re", "-", "٣٤٥", "٦", "Dž", "#$%", "𐞁", "​́é", "<|", "fim", "_prefix", "|>", "ꟲ", "\".😀🏽<", "𐞁t", "123", "456", "78", "dd", "\t", "\t", "(ḍ̇", "-", "　"]} +{"text": "9…३Ⅳ漢EOTe👍🏽𐞁s漢EOTå9fi'll", "tokens": 26, "pieces": ["9", "…", "३Ⅳ", "漢EOTe", "👍🏽", "𐞁s漢", "EOTå", "9", "fi'll"]} +{"text": "㋿\r…'VEEOTß12345678å'lléfi​
å'T㍿'s'M　\r'TåDžd>漢 d　㍿", "tokens": 50, "pieces": ["㋿\r", "…", "'VEEOTß", "123", "456", "78", "å'll", "éfi", "​<", "EOT", ">", "
å'T", "㍿'", "s'M", "　\r", "'Tå", "Džd", ">漢", " ", " d", "　", "㍿"]} +{"text": "ꟲ's'ſ…Zt\r\n\r\n<🙂ꟲ\teß'T'Dꟲ'll.𐞁dZdtDžEOT", "tokens": 43, "pieces": ["ꟲ", "'", "s'ſ", "…Zt", "\r\n\r\n", "<🙂", "ꟲ", "\teß'T", "'Dꟲ'll", ".𐞁d", "Zdt", "DžEOT", ""]} +{"text": "s<|fim_prefix|>​ſ\u000b-d<漢're\"👍🏽(…\r'ſ#$%\"'S<|fim_prefix|>a
'D\n\té-åİ'Ⅳd<|endoftext|>字\r<|fim_prefix|>", "tokens": 65, "pieces": ["s", "<|", "fim", "_prefix", "|>​", "ſ", "\u000b", "-d", "<漢're", "\"👍🏽(", "…\r", "'ſ", "#$%\"'", "S", "<|", "fim", "_prefix", "|>", "a", "
", "'D", "\n", "\té", "-å", "İ", "'<", "EOT", ">", "Ⅳ", "d", "<|", "endoftext", "|>", "字", "\r", "<|", "fim", "_prefix", "|>"]} +{"text": "́'re9\r\n\r\n>٣٤٥٦asß>​ع'ſ'VEe\t'S9, Z'Re­𐞁'Re", "tokens": 36, "pieces": ["́'re", "9", "\r\n\r\n", ">", "٣٤٥", "٦", "asß", ">​", "ع'ſ", "'VEe", "\t", "'S", "9", ",", " Z'Re", "­<", "META", "_START", ">𐞁'Re"]} +{"text": "😀🏽👍🏽\t", "tokens": 7, "pieces": ["😀🏽👍🏽", "\t"]} +{"text": "a㋿'ll!!\r\n\r\né\"! ' -9'reＡ३ß <#$%'VE\"0!#$%Dž漢'T\r\n", "tokens": 33, "pieces": ["a", "㋿'", "ll", "!!\r\n\r\n", "é", "\"!", " '", " ", "-", "9", "'re", "Ａ", "३", "ß", " ", "<#$%'", "VE", "\"", "0", "!#$%", "Dž漢'T", "\r\n"]} +{"text": " \nßDž😀🏽's,m-'ſEOT𐞁𐞁", "tokens": 25, "pieces": [" \n", "ß", "Dž", "😀🏽<", "EOT", ">'", "s", ",m", "-'", "ſ", "EOT𐞁𐞁"]} +{"text": "
\r㍿<|endoftext|>'ll(<|fim_prefix|>　fi​eⅣ٣٤٥٦", "tokens": 29, "pieces": ["
\r", "㍿<|", "endoftext", "|>'", "ll", "(<|", "fim", "_prefix", "|>", "　fi", "​e", "Ⅳ٣٤", "٥٦"]} +{"text": "İZ \n12345678fi'Re'sſ
字,　<|fim_prefix|>'D,12345678\"a ,㋿", "tokens": 30, "pieces": ["İZ", " \n", "123", "456", "78", "fi'Re", "'sſ", "
字", ",", "　", "<|", "fim", "_prefix", "|>'", "D", ",", "123", "456", "78", "\"a", " ,㋿"]} +{"text": "EOT漢!fi,İ'se'Re㍿३'VE#$%t0Z\u000bⅣZ", "tokens": 31, "pieces": ["EOT漢", "!fi", ",İ's", "e'Re", "㍿", "३", "'VE", "#$%<", "EOT", ">t", "0", "Z", "\u000b", "Ⅳ", "Z"]} +{"text": " \u000b🙂<|fim_prefix|>　
12345678'TtⅣꟲ>fi३…㍿sⅣEOT'VE½étİé", "tokens": 42, "pieces": [" ", "\u000b", "🙂<|", "fim", "_prefix", "|>", "　", "
", "123", "456", "78", "'Tt", "Ⅳ", "ꟲ", ">fi", "३", "…", "㍿s", "Ⅳ", "EOT'VE", "½", "ét", "İé"]} +{"text": "٣٤٥٦0\r\n12345678<|endoftext|>m'D \n٣٤٥٦'S३ſ\t漢'Re३'M12345678Ⅳ'VE're.字　.<|fim_prefix|>\u000b<|fim_prefix|>m", "tokens": 59, "pieces": ["٣٤٥", "٦0", "\r\n", "123", "456", "78", "<|", "endoftext", "|>", "m'D", " \n", "٣٤٥", "٦", "'S", "", "३", "ſ", "\t漢'Re", "३", "'M", "123", "456", "78Ⅳ", "'VE're", ".字", "　", ".<|", "fim", "_prefix", "|>", "\u000b", "<|", "fim", "_prefix", "|>", "m"]} +{"text": "㋿ß(12345678\tmع'S", "tokens": 11, "pieces": ["㋿ß", "(", "123", "456", "78", "\tmع'S"]} +{"text": "ꟲ​½s\u000b㋿#$%Ａ\"​ Dž!!ḍ̇Z字>fi'S\r\n\rå!e漢.ßa.👍🏽>'", "tokens": 42, "pieces": ["ꟲ", "​", "½", "s", "\u000b", "㋿#$%", "Ａ", "\"​", " Dž", "!!", "ḍ̇", "Z字", ">fi'S", "\r\n\r", "å", "!e漢", ".ßa", ".👍🏽>'"]} +{"text": "‍\t\t'DéＡ \n <|fim_prefix|>", "tokens": 15, "pieces": ["‍", "\t", "\t", "'Dé", "Ａ", " \n", " ", " <|", "fim", "_prefix", "|>"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " Dž#$%'T'T'M漢🙂<|fim_prefix|><'Té'D㋿👍🏽…fi🙂0Dž9👍🏽're😀🏽
éع\téa-.", "tokens": 59, "pieces": [" ", " Dž", "#$%'", "T'T", "'M漢", "🙂<|", "fim", "_prefix", "|><'", "Té'D", "㋿<", "EOT", ">👍🏽", "…fi", "🙂", "0", "Dž", "9", "👍🏽'", "re", "😀🏽", "
éع", "\téa", "-."]} +{"text": "​\r!,ad字t…𐞁>🙂'VE\r\n\r\nſ Dž, 'ſ'Re \n're're㍿'S0<漢'D,", "tokens": 39, "pieces": ["​\r", "!,", "ad字t", "…𐞁", ">🙂'", "VE", "\r\n\r\n", "ſ", " Dž", ",", " ", " '", "ſ'Re", " \n", "'re're", "㍿'", "S", "0", "<漢'D", ","]} +{"text": "Z12345678.\"३\u000bİ字fi!!​\r\n\r\n🙂!!'ſ​㍿
'D!!t😀🏽½ſ٣٤٥٦
fi", "tokens": 43, "pieces": ["Z", "123", "456", "78", ".\"", "३", "\u000bİ字fi", "!!​\r\n\r\n", "🙂!!<", "EOT", ">'", "ſ", "​<", "EOT", ">㍿", "
", "'D", "!!", "t", "😀🏽", "½", "ſ", "٣٤٥", "٦", "
fi"]} +{"text": "\r\n\r\ns\t>\"étfiém'VEa're字\"Z\u000b m#$%İ\r<|endoftext|>s🙂ꟲ\te>", "tokens": 37, "pieces": ["\r\n\r\n", "s", "\t", ">\"", "étfiém'VE", "a're", "字", "\"Z", "\u000b", " m", "#$%", "İ", "\r", "<|", "endoftext", "|>", "s", "🙂ꟲ", "\te", ">"]} +{"text": "'VE#$%ꟲ'll٣٤٥٦ſꟲ åⅣ.…<|fim_prefix|>٣٤٥٦ea'M!!३Ⅳå'D\r\n​\r\n'S㋿½'ll½ع'MZEOT'", "tokens": 61, "pieces": ["'VE", "#$%", "ꟲ'll", "٣٤٥", "٦", "ſꟲ", " å", "Ⅳ", ".", "…", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ea'M", "!!", "३Ⅳ", "å'D", "\r\n", "​\r\n", "'S", "㋿", "½", "'ll", "½", "ع'M", "ZEOT", "'"]} +{"text": "​Ⅳ EOT<|endoftext|>as!㋿éſd'DDž é,㍿(d​
\r\né a🙂ſ", "tokens": 46, "pieces": ["​", "Ⅳ", " EOT", "<|", "endoftext", "|>", "as", "!㋿", "éſd'D", "Dž", " ", " é", ",㍿(", "d", "​", "
\r\n", "é", " ", " a", "🙂ſ"]} +{"text": "㍿<|endoftext|>fitZ 𐞁'VE'll'D…\t🙂<Dž>Ａ!!'VEEOT'sع字's \"(…😀🏽'T!!'ſ​Z", "tokens": 59, "pieces": ["㍿<|", "endoftext", "|>", "fit", "Z", " ", " 𐞁'VE", "'ll'D", "…", "\t", "🙂<", "Dž", ">Ａ", "!!'", "VEEOT's", "ع字's", " ", "\"(", "…", "😀🏽'", "T", "!!'", "ſ", "​Z"]} +{"text": "ḍ̇½'Sa'́9Ⅳfi
 \n\u000bfi e,d…", "tokens": 25, "pieces": ["ḍ̇", "½", "'Sa", "'́", "9", "", "Ⅳ", "fi", "
 \n", "\u000bfi", " ", " e", ",d", "…"]} +{"text": "漢 éå🙂é \n(ſs#$%'s e( ,㋿🙂ꟲ\n㍿
", "tokens": 35, "pieces": ["漢", " éå", "🙂é", " \n", "(ſs", "#$%'", "s", " ", " e", "(", " ", ",㋿🙂", "ꟲ", "\n", "㍿<", "EOT", ">", "
"]} +{"text": "s'VEß𐞁ſ9 mfi!!㍿<|fim_prefix|>s9 　", "tokens": 39, "pieces": ["s'VE", "ß𐞁ſ", "9", " ", " mfi", "!!㍿<|", "fim", "_prefix", "|>", "s", "", "9", " 　"]} +{"text": ">-ſEOT'T<<|endoftext|>\r　fi漢👍🏽 a\"å㍿ ​EOT\n're-Z \n<|endoftext|>sm٣٤٥٦字", "tokens": 51, "pieces": [">-", "ſ", "EOT'T", "<<|", "endoftext", "|>\r", "", "　fi漢", "👍🏽", " ", " a", "\"å", "㍿", " ", " ​", "EOT", "\n", "'re", "-Z", " \n", "<|", "endoftext", "|>", "sm", "٣٤٥", "٦", "字"]} +{"text": "\"𐞁ſ-٣٤٥٦ßémİ!३٣٤٥٦á>‍\r\n\r\n<|fim_prefix|>#$%ß're
🙂a<|fim_prefix|><|endoftext|>12345678d\",", "tokens": 55, "pieces": ["\"𐞁ſ", "-", "٣٤٥", "٦", "ßém", "İ", "!", "३٣٤", "٥٦", "á", ">‍\r\n\r\n", "<|", "fim", "_prefix", "|>#$%", "ß're", "
", "🙂a", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "123", "456", "78", "d", "\","]} +{"text": "😀🏽…,‍('T𐞁#$%
Ⅳع\t'字's're\r\n\r\n'll👍🏽ع .½< 😀🏽é३'VE漢漢'S\nⅣع…Ａ#$%", "tokens": 54, "pieces": ["😀🏽", "…", ",‍('", "T𐞁", "#$%", "
", "Ⅳ", "ع", "\t", "'字's", "'re", "\r\n\r\n", "'ll", "👍🏽", "ع", " ", " .", "½", "<", " ", " 😀🏽", "é", "३", "'VE漢漢'S", "\n", "Ⅳ", "ع", "…Ａ", "#$%"]} +{"text": "Z \n>३d'S<|endoftext|>12345678👍🏽😀🏽'lĺ(­…㍿\t㋿㋿字fie0'Ａ'll漢s's🙂👍🏽😀🏽'VE<🙂fi", "tokens": 61, "pieces": ["Z", " \n", ">", "३", "d'S", "<|", "endoftext", "|>", "123", "456", "78", "👍🏽😀🏽'", "lĺ", "(­", "…", "㍿", "\t", "㋿㋿", "字fie", "0", "'Ａ'll", "漢s's", "🙂👍🏽😀🏽'", "VE", "<🙂", "fi"]} +{"text": "é>EOT'VE'VE　'VEⅣ
 'll३0d12345678👍🏽Ⅳ字å字Ａḍ̇½<|fim_prefix|>Ⅳ<　<|endoftext|>åEOT'VE", "'VE", "　", "'VE", "Ⅳ", "
", " '", "ll", "३0", "d", "123", "456", "78", "👍🏽", "Ⅳ", "字å字", "Ａḍ̇", "½", "<|", "fim", "_prefix", "|>", "Ⅳ", "<", "　", "<|", "endoftext", "|>", "å", "!字<|fim_prefix|>#$%عté12345678😀🏽İ३d­>ꟲ-'sa३½­…é", "tokens": 53, "pieces": ["٣٤٥", "٦", " fi", "!<", "META", "_START", ">字", "<|", "fim", "_prefix", "|>#$%", "عté", "123", "456", "78", "😀🏽", "İ", "३", "d", "­>", "ꟲ", "-'", "sa", "३", "", "½", "­", "…é"]} +{"text": "éEOTe.(　٣٤٥٦<|fim_prefix|>e😀🏽🙂 😀🏽><|endoftext|>é٣٤٥٦Z \n>　'llfié<|fim_prefix|>👍🏽漢ꟲ .'M", "tokens": 66, "pieces": ["é", "EOT", "e", ".(", "　", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "e", "😀🏽🙂", " 😀🏽><|", "endoftext", "|>", "é", "٣٤٥", "٦", "Z", " \n", "><", "META", "_START", ">", "　", "'llfié", "<|", "fim", "_prefix", "|>👍🏽", "漢ꟲ", " ", " .'", "M"]} +{"text": "ßꟲs𐞁🙂<|fim_prefix|>e\"\r\n\r\n \n", "tokens": 22, "pieces": ["ßꟲs𐞁", "🙂<", "META", "_START", "><|", "fim", "_prefix", "|>", "e", "\"\r\n\r\n", " \n"]} +{"text": "<|fim_prefix|>'T　#$%‍'ll<|fim_prefix|>'T \t0字ꟲé😀🏽é\n'🙂( m½é'M\u000b<|fim_prefix|>㋿>", "tokens": 52, "pieces": ["<|", "fim", "_prefix", "|>'", "T", "　", "#$%‍'", "ll", "<|", "fim", "_prefix", "|>'", "T", " ", "\t", "0", "字ꟲé", "😀🏽", "é", "\n", "'🙂(", " m", "½", "é'M", "\u000b", "<|", "fim", "_prefix", "|>㋿>"]} +{"text": "́​ꟲ", "tokens": 5, "pieces": ["́", "​ꟲ"]} +{"text": "'lla३́ꟲ　 🙂
", "tokens": 13, "pieces": ["'lla", "३", "́ꟲ", "", "　", " 🙂", "
"]} +{"text": "0٣٤٥٦İ's(12345678\u000b0'M ", "tokens": 52, "pieces": ["0", "", "٣٤٥", "٦", "İ's", "(", "123", "456", "78", "\u000b", "0", "'M", " "]} +{"text": "\u000bſ9́ \n'Re𐞁\r漢<|fim_prefix|>", "tokens": 18, "pieces": ["\u000bſ", "9", "́", " \n", "'Re𐞁", "\r", "漢", "<|", "fim", "_prefix", "|>"]} +{"text": "\r\n'D\r\nİ ('M٣٤٥٦ ع", "tokens": 12, "pieces": ["\r\n", "'D", "\r\n", "İ", " ('", "M", "٣٤٥", "٦", " ع"]} +{"text": "Ⅳ\"ꟲ\nß😀🏽9👍🏽é㍿ع㍿Džع!!𐞁　'S9😀🏽9'Ssꟲ'S‍<|endoftext|>\r\n३-
<|fim_prefix|>'EOTḍ̇३a", "tokens": 73, "pieces": ["Ⅳ", "\"ꟲ", "\n", "ß", "😀🏽", "9", "👍🏽", "é", "㍿ع", "㍿Džع", "!!", "𐞁", "　", "'S", "9", "😀🏽", "9", "'Ssꟲ'S", "‍<|", "endoftext", "|>\r\n", "३", "-", "
", "<|", "fim", "_prefix", "|>'", "EOTḍ̇", "३", "a"]} +{"text": "🙂́\u000bß!!Ⅳ,\r(漢'sꟲ½字\r\nＡ,\t'Tå𐞁३!!👍🏽Ⅳ\nt'Se", "tokens": 47, "pieces": ["🙂́", "", "\u000bß", "!!", "Ⅳ", ",\r", "(漢's", "ꟲ", "½", "字", "\r\n", "Ａ", ",", "\t", "'Tå𐞁", "३", "!!👍🏽", "Ⅳ", "\n", "t'S", "e"]} +{"text": "'D\"se🙂ع12345678e", "tokens": 9, "pieces": ["'D", "\"se", "🙂ع", "123", "456", "78", "e"]} +{"text": "<( \n (", "tokens": 12, "pieces": ["<(<", "META", "_START", ">", " \n", "", " ", "("]} +{"text": "İ'MEOT'ſ\u000b३\r\n\r\nع'M\r\n>\t ", "tokens": 18, "pieces": ["İ'M", "EOT'ſ", "", "\u000b", "३", "\r\n\r\n", "ع'M", "\r\n", ">", "\t "]} +{"text": " -'S\r\né漢0ſ​'s'T'ſ '\u000b'D,㋿'M​td㍿<|endoftext|>", "tokens": 42, "pieces": [" -<", "META", "_START", ">'", "S", "\r\n", "é漢", "0", "ſ", "​<", "EOT", ">'", "s'T", "'ſ", " ", " '", "\u000b", "'D", ",㋿'", "M", "​td", "㍿<|", "endoftext", "|>"]} +{"text": "\nꟲ㍿<|endoftext|> 'M'sm½EOT 　sꟲ ع🙂!\n!!Ⅳ-<|fim_prefix|>.m'M\r㍿'Rem", "tokens": 52, "pieces": ["\n", "ꟲ", "㍿<|", "endoftext", "|>", " ", " '", "M's", "m", "½", "EOT", " ", "　sꟲ", " ع", "🙂!\n", "!!", "Ⅳ", "-<|", "fim", "_prefix", "|>.", "m'M", "\r", "㍿'", "Rem"]} +{"text": "ع 'Res", "tokens": 8, "pieces": ["ع", "", " ", "'Res"]} +{"text": "9! \ń'SDž३'Ss'D0(Z🙂ſDž!!ꟲß0…fi", "tokens": 26, "pieces": ["9", "!", " \n", "́'S", "Dž", "३", "'Ss'D", "0", "(Z", "🙂ſ", "Dž", "!!", "ꟲß", "0", "…fi"]} +{"text": "\"́½ \nⅣＡéꟲ👍🏽<Ａ( ́d漢👍🏽'ſ'S12345678\t'D,Dž12345678३", "tokens": 40, "pieces": ["\"́", "½", " \n", "Ⅳ", "Ａéꟲ", "👍🏽<", "Ａ", "(", " ", " ́d漢", "👍🏽'", "ſ'S", "123", "456", "78", "\t", "'D", ",Dž", "123", "456", "78३"]} +{"text": "\r\nå, !!fi ́字½12345678\n're𐞁'Re　<|endoftext|>👍🏽tEOT👍🏽 emⅣe", "tokens": 43, "pieces": ["\r\n", "å", ",", " ", " !!", "fi", " ́字", "½12", "345", "678", "\n", "'re𐞁'Re", "　", "<|", "endoftext", "|>👍🏽", "t", "EOT", "👍🏽", " ", " em", "Ⅳ", "e"]} +{"text": "\réße字0's
३ḍ̇0EOT", "tokens": 20, "pieces": ["\r", "éß", "e字", "0", "'s", "
", "३", "ḍ̇", "0", "EOT"]} +{"text": "Ａ'ſ🙂EOT\"ḍ̇Ⅳ㍿́٣٤٥٦", "tokens": 23, "pieces": ["Ａ'ſ", "🙂EOT", "\"ḍ̇", "Ⅳ", "㍿́", "٣٤٥", "٦"]} +{"text": "३.'M\t", "tokens": 4, "pieces": ["३", ".'", "M", "\t"]} +{"text": "'Re'llİ< \r\n\r\n\nEOT.d'll!'S,", "tokens": 19, "pieces": ["'Re'll", "İ", "<", " \r\n\r\n\n", "EOT", ".d", "'", "ll", "!'", "S", ","]} +{"text": "1234567812345678'D'M\"㍿🙂'S'S<|endoftext|> >\n.'S.é𐞁​🙂 ſ \n<\r\nḍ̇0👍🏽!३,", "tokens": 52, "pieces": ["123", "456", "781", "234", "567", "8", "'D'M", "\"㍿🙂'", "S'S", "<|", "endoftext", "|>", " >\n", ".'", "S", ".é", "𐞁", "​🙂", " ſ", " \n", "<\r\n", "ḍ̇", "0", "👍🏽!", "३", ","]} +{"text": "'Sß'll
‍ß-é'D.Ａ३!!字're'll(0a'\"ⅣEOT‍🙂\r\n㍿😀🏽Ａ#$%EOT \n!!fíḍ̇", "tokens": 48, "pieces": ["'Sß'll", "
", "‍ß", "-é'D", ".Ａ", "३", "!!", "字're", "'ll", "(", "0", "a", "'\"", "Ⅳ", "EOT", "‍🙂\r\n", "㍿😀🏽", "Ａ", "#$%", "EOT", " \n", "!!", "fíḍ̇"]} +{"text": "👍🏽Z'ſ \r\n-\r\n!字fi \n,s \n㍿,<|fim_prefix|>12345678🙂漢عemsḍ̇'e\"<|endoftext|>\tⅣ", "tokens": 45, "pieces": ["👍🏽", "Z'ſ", " \r\n", "-\r\n", "!字fi", " \n", ",s", " \n", "㍿,<|", "fim", "_prefix", "|>", "123", "456", "78", "🙂漢عemsḍ̇", "'e", "\"<|", "endoftext", "|>", "\t", "Ⅳ"]} +{"text": "!!\t\r\n\r\n
😀🏽\r0\u000b😀🏽ع <|endoftext|>0 ٣٤٥٦\u000b​s३<|endoftext|> ́'VEEOT　mmå-३字३  漢", "tokens": 60, "pieces": ["!!", "\t\r\n\r\n", "
", "😀🏽\r", "0", "\u000b", "😀🏽", "ع", " <|", "endoftext", "|>", "0", " ", "٣٤٥", "٦", "\u000b", "​s", "३", "<|", "endoftext", "|>", " ́'VE", "EOT", "　mmå", "-", "३", "字", "", "३", " ", " 漢"]} +{"text": "ع½śdDž!!,Z'llt३́Ａ", "tokens": 15, "pieces": ["ع", "½", "śd", "Dž", "!!,", "Z'll", "t", "३", "́", "Ａ"]} +{"text": "<|endoftext|>'ll12345678 \n漢're३㍿'S-\"'re…<|endoftext|>'Śḍ̇!!", "tokens": 38, "pieces": ["<|", "endoftext", "|>'", "ll", "123", "456", "78", " \n", "漢're", "३", "㍿'", "S", "-\"'", "re", "…", "<|", "endoftext", "|>'", "Śḍ̇", "!!"]} +{"text": "ḍ̇0!\t​👍🏽9ſ(dⅣt'VE­0'D<漢\r\n\r\nＡ<|fim_prefix|>EOTfi", "tokens": 34, "pieces": ["ḍ̇", "0", "!", "\t", "​👍🏽", "9", "ſ", "(d", "Ⅳ", "t'VE", "­", "0", "'D", "<漢", "\r\n\r\n", "Ａ", "<|", "fim", "_prefix", "|>", "EOTfi"]} +{"text": "‍ \"\rsß½ꟲß😀🏽…漢'M㍿<́-t‍漢Zعd<|endoftext|>", "tokens": 37, "pieces": ["‍", " ", " \"\r", "sß", "½", "ꟲß", "😀🏽", "…漢'M", "㍿<́-", "t", "‍漢Zعd", "<|", "endoftext", "|>"]} +{"text": " 're!ém<|fim_prefix|>​\r\n<|endoftext|>\rع​#$%'llḍ̇㋿𐞁Ａ!>\rå­12345678ſéꟲ٣٤٥٦Ａm\r\n\r\nع'ſ'", "tokens": 64, "pieces": [" '", "re", "!ém", "<|", "fim", "_prefix", "|>​\r\n", "<|", "endoftext", "|>\r", "ع", "​#$%'", "llḍ̇", "㋿𐞁", "Ａ", "!>\r", "å", "­", "123", "456", "78", "ſéꟲ", "٣٤٥", "٦", "Ａm", "\r\n\r\n", "ع'ſ", "'"]} +{"text": "'sEOTefi'\r\n\r\ń­ß<|endoftext|>t'reⅣ🙂", "tokens": 21, "pieces": ["'s", "EOTefi", "'\r\n\r\n", "́", "­ß", "<|", "endoftext", "|>", "t're", "Ⅳ", "🙂"]} +{"text": ">(😀🏽'M\téⅣ\u000b'VE!! \n(\tm\t12345678<.t<|fim_prefix|>'re​, ́ꟲ!!​\r\nDž'sḍ̇", "tokens": 51, "pieces": [">(😀🏽'", "M", "\té", "Ⅳ", "\u000b", "'VE", "!!", " \n", "(", "\tm", "\t", "123", "456", "78", "<.", "t", "<|", "fim", "_prefix", "|>'", "re", "​,", " ", " ́ꟲ", "!!​<", "EOT", ">\r\n", "Dž's", "ḍ̇"]} +{"text": "🙂'ſⅣ'Re漢́㍿'Mmd-t!!🙂-!!'Sİ'ſ३㋿\r𐞁'S'T<|endoftext|>m'VEſ…dع'rem٣٤٥٦", "tokens": 59, "pieces": ["🙂'", "ſ", "Ⅳ", "'Re漢́", "㍿<", "META", "_START", ">'", "Mmd", "-t", "!!🙂-!!'", "Sİ'ſ", "३", "㋿\r", "𐞁'S", "'T", "<|", "endoftext", "|>", "m'VE", "ſ", "…dع're", "m", "٣٤٥", "٦"]} +{"text": "-👍🏽­'ll12345678!३㋿ß‍ 'så\r\nⅣ\r\n\r\n
\r\n ㍿३ m \n\t", "tokens": 34, "pieces": ["-👍🏽­'", "ll", "123", "456", "78", "!", "३", "㋿ß", "‍", " '", "så", "\r\n", "Ⅳ", "\r\n\r\n
\r\n", " ", "㍿", "३", " m", " \n", "\t"]} +{"text": "ḍ̇!m㍿s\r　,𐞁ع fí", "tokens": 20, "pieces": ["ḍ̇", "!m", "㍿s", "\r", "　", ",𐞁ع", " ", " fí"]} +{"text": ",İ\r\n\u000b", "tokens": 4, "pieces": [",İ", "\r\n", "\u000b"]} +{"text": "‍#$%<'S'ReⅣee𐞁'M \r\n9-٣٤٥٦…,'re٣٤٥٦'M'VE 'T٣٤٥٦Ａ-\"-'VEſſ-", "tokens": 55, "pieces": ["‍#$%<'", "S'Re", "Ⅳ", "ee𐞁'M", " \r\n", "9", "-", "٣٤٥", "٦", "…", ",'", "re", "٣٤٥", "٦", "'M'VE", "", " ", " '", "T", "٣٤٥", "٦", "Ａ", "-\"-'", "VEſſ", "-"]} +{"text": "<|endoftext|>३ <‍İ'så३'Mta's'M'ſ're \r\n\r\nꟲ\n😀🏽'Reꟲå\ndꟲ9Z\r\u000b \n\r\nté­İ", "tokens": 55, "pieces": ["<|", "endoftext", "|>", "३", " <‍<", "EOT", ">İ's", "å", "३", "'Mta's", "'M'ſ", "'re", " \r\n\r\n", "ꟲ", "\n", "😀🏽'", "Reꟲå", "\n", "dꟲ", "9", "Z", "\r\u000b \n\r\n", "té", "­İ"]} +{"text": "㋿<|endoftext|>é‍å \n३!😀🏽 \n'VE\t\r\n\r\n\néßå🙂\r\n\r\nfia", "tokens": 43, "pieces": ["㋿<", "META", "_START", "><|", "endoftext", "|>", "é", "‍", "å", " \n", "३", "!😀🏽", " \n", "'VE", "\t\r\n\r\n\n", "éßå", "🙂\r\n\r\n", "fia"]} +{"text": "字s12345678", "tokens": 5, "pieces": ["字s", "123", "456", "78"]} +{"text": "ḍ̇m㋿\n👍🏽,😀🏽 漢 \n­'T漢\t́'MåEOT're9t­\rß!　Ⅳ‍'Re\r", "tokens": 43, "pieces": ["\u000b́'M", "İＡ", "123", "456", "78", "EOT", "<|", "fim", "_prefix", "|>'", "T漢", "\t́'M", "å", "EOT're", "", "9", "t", "­\r", "ß", "!", "　", "Ⅳ", "‍'", "Re", "\r"]} +{"text": "漢٣٤٥٦d<|fim_prefix|>'sEOT #$%​<|endoftext|> 'Sfiع!\"ſ漢", "tokens": 33, "pieces": ["漢", "٣٤٥", "٦", "d", "<|", "fim", "_prefix", "|>'", "s", "EOT", " ", "#$%​<|", "endoftext", "|>", " ", "'Sfiع", "!\"", "ſ漢"]} +{"text": "'🙂ſ३'VE99
t…🙂字EOT0. \n​عEOTtꟲDžfí'llİ‍\r\né", "tokens": 36, "pieces": ["'🙂", "ſ", "३", "'VE", "99", "
t", "…", "🙂字", "EOT", "0", ".", " \n", "​عEOTtꟲ", "Džfí'll", "İ", "‍\r\n", "é"]} +{"text": "'S​'ll‍
İ", "tokens": 7, "pieces": ["'S", "​'", "ll", "‍", "
İ"]} +{"text": "å'Så ३é👍🏽'VE><\r\n\r\n😀🏽𐞁'aEOT\r\n\r\nß'll0Dže​Ａꟲ", "tokens": 44, "pieces": ["å'S", "å", " ", " ", "३", "é", "👍🏽'", "VE", "><\r\n\r\n", "😀🏽", "𐞁", "'<", "META", "_START", ">a", "EOT", "\r\n\r\n", "ß'll", "0", "Dže", "​Ａꟲ"]} +{"text": "''Mع>🙂ꟲ\r\n 9e\"字
", "tokens": 18, "pieces": ["''", "Mع", ">🙂", "ꟲ", "\r\n", " ", "9", "e", "\"字", "
"]} +{"text": ",
!<|endoftext|>‍㍿…'M‍㍿ \r\n'Mtİḍ̇fi👍🏽.\r\n\r\né>'re\"\r\n", "tokens": 37, "pieces": [",", "
", "!<|", "endoftext", "|>‍㍿", "…", "'M", "‍㍿", " \r\n", "'Mt", "İḍ̇fi", "👍🏽.\r\n\r\n", "é", ">'", "re", "\"\r\n"]} +{"text": "'M'D‍012345678'll12345678<|fim_prefix|>\u000bＡ　ḍ̇!Ⅳ,'ll‍Ａ>İé \n👍🏽're9''ll漢-.a\n㋿é", "tokens": 54, "pieces": ["'M'D", "‍", "012", "345", "678", "'ll", "123", "456", "78", "<|", "fim", "_prefix", "|>", "\u000bＡ", "　ḍ̇", "!", "Ⅳ", ",'", "ll", "‍Ａ", ">İé", " \n", "👍🏽'", "re", "9", "''", "ll", "漢", "-.", "a", "\n", "㋿é"]} +{"text": "'VE-'llḍ̇fi'DⅣ.​,́", "tokens": 14, "pieces": ["'VE", "-'", "llḍ̇fi'D", "Ⅳ", ".​,́"]} +{"text": "<|endoftext|>tm😀🏽!!'T,'sfi,漢,'ſfiZé<|fim_prefix|>,,!!
\t", "tokens": 34, "pieces": ["<|", "endoftext", "|>", "tm", "😀🏽!!'", "T", ",'", "sfi", ",漢", ",'", "ſfi", "Zé", "<|", "fim", "_prefix", "|>,,!!", "
\t"]} +{"text": "́'Reꟲ \n'M>.　9 EOT#$%👍🏽t字 \nt​  ٣٤٥٦(<|fim_prefix|>'re\t", "tokens": 36, "pieces": ["́'Re", "ꟲ", " \n", "'M", ">.", "　", "9", " EOT", "#$%👍🏽", "t字", " \n", "t", "​", " ", " ", "٣٤٥", "٦", "(<|", "fim", "_prefix", "|>'", "re", "\t"]} +{"text": "\t're…'VEſ字ea🙂\re٣٤٥٦'T字'reé'llꟲ'ſ½ \nⅣa", "tokens": 32, "pieces": ["\t", "'re", "…", "'VEſ字ea", "🙂\r", "e", "٣٤٥", "٦", "'T字're", "é'll", "ꟲ'ſ", "½", " \n", "Ⅳ", "a"]} +{"text": "\tⅣe　0漢\u000b'D,　,漢Ⅳع😀🏽é́
'D \n\t.‍dꟲİ‍#$%Ａ'T", "tokens": 44, "pieces": ["\t", "Ⅳ", "e", "　", "0", "漢", "\u000b", "'", "D", ",", "　", ",漢", "Ⅳ", "ع", "😀🏽", "é́", "
", "'D", "", " \n", "\t", ".‍", "dꟲ", "İ", "‍#$%", "Ａ'T"]} +{"text": "字!!'s 're'D\r\n\r\n(​👍🏽\r\n\r\n𐞁'll fiDž123456780‍'Tå漢٣٤٥٦", "tokens": 36, "pieces": ["字", "!!'", "s", " ", " '", "re'D", "\r\n\r\n", "(​👍🏽\r\n\r\n", "𐞁'll", " fi", "Dž", "123", "456", "780", "‍'", "Tå漢", "٣٤٥", "٦"]} +{"text": "Ⅳ. ⅣEOT \raſ漢<👍🏽-!e.٣٤٥٦.m", "tokens": 26, "pieces": ["Ⅳ", ".", " ", "Ⅳ", "EOT", " \r", "aſ漢", "<👍🏽-!", "e", ".", "٣٤٥", "٦", ".m"]} +{"text": "-#$%'T…Z<|fim_prefix|>\"9 \n½<|fim_prefix|>'M", "tokens": 23, "pieces": ["-#$%'", "T", "…Z", "<|", "fim", "_prefix", "|>\"", "9", " \n", "½", "<|", "fim", "_prefix", "|>'", "M"]} +{"text": "ꟲ३'T .
<''T'ReＡe'S<|endoftext|>​a<|endoftext|>𐞁\r\n　>EOT're😀🏽Z'T\"#$%ßZ'll\r\n\r\n㍿", "tokens": 58, "pieces": ["ꟲ", "३", "'T", " ", " .", "
", "<''", "T'Re", "Ａe'S", "<|", "endoftext", "|>​", "a", "<|", "endoftext", "|>", "𐞁", "\r\n", "　", ">EOT're", "😀🏽", "Z'T", "\"#$%", "ß", "Z'll", "\r\n\r\n", "㍿"]} +{"text": ">m­​ \n'Ds\u000bé㋿́", "tokens": 13, "pieces": [">m", "­​", " \n", "'Ds", "\u000bé", "㋿́"]} +{"text": "<‍\n'MZfiß\tfi!'S\r…'s!!tǻ漢عꟲعå'ſEOT", "tokens": 31, "pieces": ["<‍\n", "'MZfiß", "\tfi", "!'", "S", "\r", "…", "'s", "!!", "tǻ漢عꟲعå'ſ", "EOT"]} +{"text": "字😀🏽👍🏽‍ع\u000b'S㍿<|endoftext|>a!'ll", "tokens": 24, "pieces": ["字", "😀🏽👍🏽‍", "ع", "\u000b", "'S", "㍿<|", "endoftext", "|>", "a", "!'", "ll"]} +{"text": "ꟲ<|endoftext|>‍😀🏽​('ſ<|endoftext|>\t ​\u000b'D!!㍿'s\r\n\r\n>\"字 \n‍", "tokens": 43, "pieces": ["ꟲ", "<|", "endoftext", "|>‍😀🏽<", "META", "_START", ">​('", "ſ", "<|", "endoftext", "|>", "\t ", " ​", "\u000b", "'D", "!!㍿'", "s", "\r\n\r\n", ">\"", "字", " \n", "‍"]} +{"text": "m \n漢'S \"İ漢𐞁", "tokens": 15, "pieces": ["m", " \n", "漢", "'", "S", " ", " \"", "İ漢𐞁"]} +{"text": " EOTé٣٤٥٦a#$%عEOT#$%ع\n🙂 aZ​EOT\r\nع字m'Re𐞁Ⅳ'SＡ'S  \u000b
!EOT'll字", "tokens": 51, "pieces": [" EOTé", "٣٤٥", "٦", "a", "#$%", "ع", "EOT", "#$%", "ع", "\n", "🙂", " ", " a", "Z", "​EOT", "\r\n", "ع字m'Re", "𐞁", "Ⅳ", "'SＡ'S", "  \u000b", "
", "!EOT'll", "字"]} +{"text": "́عé३'re!!.'­a !0s…\"åé's😀🏽9", "tokens": 24, "pieces": ["́عé", "३", "'re", "!!.'­", "a", " !", "0", "s", "…", "\"åé's", "😀🏽", "9"]} +{"text": "ḍ̇é\t>\r.d‍'Re字 \tß!\t \n漢'fi t", "tokens": 25, "pieces": ["ḍ̇é", "\t", ">\r", ".d", "‍'", "Re字", " ", "\tß", "!<", "EOT", ">", "\t \n", "漢", "'fi", " t"]} +{"text": "Z'S0Ａ\r\n\r\n\r\n\r\nß,㋿", "tokens": 10, "pieces": ["Z'S", "0", "Ａ", "\r\n\r\n\r\n\r\n", "ß", ",㋿"]} +{"text": "½é½😀🏽#$%­😀🏽0 9ſ(\r\r>𐞁'Sß-<|fim_prefix|>👍🏽\n👍🏽İ\"عZ'Z,
", "tokens": 48, "pieces": ["½", "é", "½", "😀🏽#$%­😀🏽", "0", " ", "9", "ſ", "(\r\r", ">𐞁'S", "ß", "-<|", "fim", "_prefix", "|>👍🏽\n", "👍🏽", "İ", "\"ع", "Z", "'Z", ",", "
"]} +{"text": "!DžEOT's𐞁t ́\t🙂a٣٤٥٦ḍ̇ꟲＡZ .m0 !!İ'Dt\r'll<|endoftext|>Z\nZ#$%'s­㋿‍#$%", "tokens": 62, "pieces": ["!DžEOT's", "𐞁t", " ́", "\t", "🙂a", "٣٤٥", "٦", "ḍ̇ꟲ", "ＡZ", " ", " <", "EOT", ">.", "m", "0", " ", " !!", "İ'D", "t", "\r", "'ll", "<|", "endoftext", "|>", "Z", "\n", "Z", "#$%'", "s", "­㋿‍#$%"]} +{"text": "<(d İs 9!!\r\n\r\n<|endoftext|>s…\r\n'ſ'D'ſ", "tokens": 24, "pieces": ["<(", "d", " ", " İs", " ", "9", "!!\r\n\r\n", "<|", "endoftext", "|>", "s", "…\r\n", "'ſ'D", "'ſ"]} +{"text": "́½0", "tokens": 3, "pieces": ["́", "½0"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㋿é­", "tokens": 5, "pieces": ["㋿é", "­"]} +{"text": "😀🏽aع'VE\"9㍿#$%😀🏽
eEOT!!'sſ<|fim_prefix|>'Re\u000bZ漢'T>­𐞁३　e'ſ‍", "tokens": 51, "pieces": ["😀🏽<", "EOT", ">aع'VE", "\"", "9", "㍿#$%😀🏽", "
e", "EOT", "!!'", "sſ", "<|", "fim", "_prefix", "|>'", "Re", "\u000bZ漢'T", ">­", "𐞁", "३", "　e'ſ", "‍"]} +{"text": "#$% #$%Zꟲ漢　EOT'ſ'reＡ­'VE漢😀🏽३😀🏽s9( >\r \n\u000b
m<|fim_prefix|>< \n٣٤٥٦​e\r\n٣٤٥٦
", "tokens": 56, "pieces": ["#$%", " ", "#$%", "Zꟲ漢", "　EOT'ſ", "'re", "Ａ", "­'", "VE漢", "😀🏽", "३", "😀🏽", "s", "9", "(", " >\r", " \n", "\u000b", "
m", "<|", "fim", "_prefix", "|><", " \n", "٣٤٥", "٦", "​e", "\r\n", "٣٤٥", "٦", "
"]} +{"text": "'s 
\nİ,'VE　-a \n\r12345678#$%'S", "tokens": 21, "pieces": ["'s", "", " 
\n", "İ", ",'", "VE", "　", "-a", " \n\r", "123", "456", "78", "#$%'", "S"]} +{"text": "'Mdꟲ字", "tokens": 6, "pieces": ["'Mdꟲ字"]} +{"text": "…ſ　12345678…㍿  😀🏽m!!ḍ̇", "tokens": 22, "pieces": ["…ſ", "　", "123", "456", "78", "…", "㍿", " ", " ", "😀🏽", "m", "!!", "ḍ̇"]} +{"text": "12345678e.fi're#$%­ \n,漢𐞁👍🏽", "tokens": 20, "pieces": ["123", "456", "78", "e", ".fi're", "#$%­", " \n", ",漢𐞁", "👍🏽"]} +{"text": ". 漢sss\u000b\r\n\r\nſs\n😀🏽\t'ret EOT!!‍㍿,\r're\u000b'Re\"Ⅳ'M ", "tokens": 33, "pieces": [".", " 漢sss", "\u000b\r\n\r\n", "ſs", "\n", "😀🏽", "\t", "'ret", " EOT", "!!‍㍿,\r", "'re", "\u000b", "'Re", "\"", "Ⅳ", "'M", " "]} +{"text": "!!(!!🙂Ａ㋿㍿0㍿", "tokens": 15, "pieces": ["!!(!!🙂", "Ａ", "㋿㍿", "0", "㍿"]} +{"text": "𐞁Ź\r\n\r\n're'Tſ'D\td'T're9­​e🙂Zع\n'D­\r\n\u000b😀🏽½.ع字😀🏽İEOT½", "tokens": 44, "pieces": ["𐞁Ź", "\r\n\r\n", "'re'T", "ſ'D", "\td'T", "'re", "9", "­<", "EOT", ">​", "e", "🙂Zع", "\n", "'D", "­\r\n", "\u000b", "😀🏽", "½", ".ع字", "😀🏽", "İEOT", "½"]} +{"text": "İ>", "tokens": 2, "pieces": ["İ", ">"]} +{"text": "é𐞁漢", "tokens": 6, "pieces": ["é𐞁漢"]} +{"text": "!! \n½👍🏽😀🏽#$%<|endoftext|><|fim_prefix|>.㋿<|endoftext|>\u000b", "tokens": 34, "pieces": ["!!", " \n", "½", "👍🏽😀🏽#$%<|", "endoftext", "|><|", "fim", "_prefix", "|>.㋿<|", "endoftext", "|>", "\u000b"]} +{"text": "'ll\r\n<|endoftext|>İ­\r\n\r\n's t's'reét\r٣٤٥٦dd(>m<|endoftext|>'D㍿é", "tokens": 43, "pieces": ["'ll", "\r\n", "<|", "endoftext", "|>", "İ", "­\r\n\r\n", "'s", " t's", "'reét", "\r", "٣٤٥", "٦", "dd", "(><", "EOT", ">m", "<|", "endoftext", "|>'", "D", "㍿é"]} +{"text": "𐞁𐞁'T", "tokens": 9, "pieces": ["𐞁𐞁'T"]} +{"text": "12345678'T'ſ#$%d​EOTⅣ-…<|fim_prefix|>ꟲ's12345678Džé'llm'VE(Ａåİ́", "tokens": 43, "pieces": ["123", "456", "78", "'T'ſ", "#$%", "d", "​EOT", "Ⅳ", "-", "…", "<|", "fim", "_prefix", "|>", "ꟲ's", "123", "456", "78", "Džé'll", "m'VE", "(Ａå", "İ́"]} +{"text": "👍🏽12345678­  \n0å'S​́ſ0Dž9s12345678é\"ſEOT'llḍ̇-éDžfi're'S", "tokens": 44, "pieces": ["👍🏽", "123", "456", "78", "­", " ", "", " \n", "0", "å'S", "​́ſ", "0", "Dž", "9", "s", "123", "456", "78", "é", "\"ſ", "EOT'll", "ḍ̇", "-é", "Džfi're", "'S"]} +{"text": "'llåfi😀🏽e\r\n\t　d<|fim_prefix|>", "tokens": 18, "pieces": ["'llåfi", "😀🏽", "e", "\r\n", "\t", "　d", "<|", "fim", "_prefix", "|>"]} +{"text": "'Re\r\n\r\n<|endoftext|>#$%( \n", "tokens": 15, "pieces": ["'Re", "\r\n\r\n", "<|", "endoftext", "|>#$%(", " \n"]} +{"text": ",­s.\t\u000b㋿
!!😀🏽'M<|endoftext|>#$%㋿漢e's🙂\t३e'T'm!!.,EOT
­mꟲZ", "tokens": 55, "pieces": [",<", "META", "_START", ">­", "s", ".", "\t", "\u000b", "㋿", "
", "!!😀🏽'", "M", "<|", "endoftext", "|>#$%㋿", "漢e's", "🙂", "\t", "३", "e'T", "'m", "!!.,", "EOT", "
", "­", "mꟲ", "Z"]} +{"text": "​>ꟲꟲ㋿\r\nſ👍🏽'Re'T ½'M(", "tokens": 23, "pieces": ["​>", "ꟲꟲ", "㋿\r\n", "ſ", "👍🏽'", "Re'T", " ", "½", "'M", "("]} +{"text": "Ａꟲſ
!dfiEOT12345678", "tokens": 14, "pieces": ["Ａꟲſ", "
", "!dfi", "EOT", "123", "456", "78"]} +{"text": "字'S<|endoftext|>9('llꟲ,d-\r\n\r\n12345678té\r\n\r\n٣٤٥٦'S9🙂\"-ꟲＡ9 \n>'llſ­EOT", "tokens": 43, "pieces": ["字'S", "<|", "endoftext", "|>", "9", "('", "llꟲ", ",d", "-\r\n\r\n", "123", "456", "78", "té", "\r\n\r\n", "٣٤٥", "٦", "'S", "9", "🙂\"-", "ꟲ", "Ａ", "9", " \n", ">'", "llſ", "­EOT"]} +{"text": " ३fi㍿(\re'reſ\"'VE0😀🏽!!𐞁12345678<|endoftext|>ع ZZ'Reé­a9🙂́'ſ­s​", "tokens": 45, "pieces": [" ", " ", "३", "fi", "㍿(\r", "e're", "ſ", "\"'", "VE", "0", "😀🏽!!", "𐞁", "123", "456", "78", "<|", "endoftext", "|>", "ع", " ZZ'Re", "é", "­a", "9", "🙂́'ſ", "­s", "​"]} +{"text": "‍'re!!'Re\rfiDž½🙂'Dḍ̇d'ſ…'s\r\n\r\n\r\n('", "tokens": 25, "pieces": ["‍'", "re", "!!'", "Re", "\r", "fi", "Dž", "½", "🙂'", "Dḍ̇d'ſ", "…", "'s", "\r\n\r\n\r\n", "('"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "!
\t​İ'M ㋿'Dm>ꟲ
'llEOTꟲ ꟲꟲ\"fi\r\nm!ß(…\u000b…𐞁", "tokens": 45, "pieces": ["!", "
", "\t", "​İ'M", " ㋿'", "Dm", ">ꟲ", "
", "'ll", "EOTꟲ", " ꟲꟲ", "\"fi", "\r\n", "m", "!ß", "(", "…\u000b", "…𐞁"]} +{"text": "٣٤٥٦'D \n#$%\t0-aå😀🏽0‍!e㍿🙂#$%fi-…٣٤٥٦é", "tokens": 34, "pieces": ["٣٤٥", "٦", "'D", " \n", "#$%", "\t", "0", "-aå", "😀🏽", "0", "‍!", "e", "㍿🙂#$%", "fi", "-", "…", "٣٤٥", "٦", "é"]} +{"text": "'re\r\n​!! \n​漢Ａ\r\n㍿!!\r\nع.😀🏽'ſ<|fim_prefix|>\rZ<\r\n", "tokens": 33, "pieces": ["'re", "\r\n", "​!!", " \n", "​漢", "Ａ", "\r\n", "㍿!!\r\n", "ع", ".😀🏽'", "ſ", "<|", "fim", "_prefix", "|>\r", "Z", "<\r\n"]} +{"text": "­ꟲ9'M.Z　😀🏽'VE's😀🏽>\r\n\r\nß .
 \ń'sḍ̇'VEm", "tokens": 36, "pieces": ["­ꟲ", "9", "'M", ".Z", "　", "😀🏽'", "VE's", "😀🏽>\r\n\r\n", "ß", " .", "
 \n", "́", "'", "sḍ̇", "'", "VEm"]} +{"text": "Dž \r\n\r\nß㍿'re<|fim_prefix|>👍🏽'Re ​0", "tokens": 23, "pieces": ["Dž", " \r\n\r\n", "ß", "㍿'", "re", "<|", "fim", "_prefix", "|>👍🏽'", "Re", " ​", "0"]} +{"text": "İe'll", "tokens": 3, "pieces": ["İe'll"]} +{"text": "ſ<|endoftext|>
é0'Re#$%daꟲ 'S٣٤٥٦'Re½漢\r\n ٣٤٥٦́　㍿Ａ'll㋿(", "tokens": 51, "pieces": ["ſ", "<|", "endoftext", "|>", "
é", "0", "'Re", "#$%<", "EOT", ">daꟲ", " ", " '", "S", "٣٤٥", "٦", "'Re", "½", "漢", "\r\n", " ", " ", "٣٤٥", "٦", "́", "　", "㍿Ａ'll", "㋿("]} +{"text": "٣٤٥٦-tꟲ9EOTſ\u000b́Zm0-.<|endoftext|>\r\n ́ <'s<|endoftext|>
", "tokens": 45, "pieces": ["٣٤٥", "٦", "-tꟲ", "9", "EOT", "ſ", "\u000b́Zm", "0", "-.<|", "endoftext", "|>\r\n", " ", " ́", " ", " <'", "s", "<|", "endoftext", "|>", "
"]} +{"text": "a'd12345678𐞁", "tokens": 9, "pieces": ["a'd", "123", "456", "78", "𐞁"]} +{"text": "ß\r'ſ\u000b٣٤٥٦>Ⅳ㍿Ⅳséåſ𐞁\u000bİ'VE字EOT'Re😀🏽 0😀🏽eaⅣ", "tokens": 60, "pieces": ["ß", "\r", "'ſ", "\u000b", "٣٤٥", "٦", ">", "Ⅳ", "㍿", "Ⅳ", "séå", "ſ𐞁", "\u000bİ'VE", "字", "EOT'Re", "😀🏽", " ", "0", "😀🏽", "ea", "Ⅳ"]} +{"text": "\r\n\r\n<|endoftext|>!!\u000b'Dḍ̇å\r …a<|fim_prefix|>!!#$%👍🏽ꟲⅣ-\tt.\r\n
\"​𐞁\"\r\n\r\n'Z'ſ'S#$%", "tokens": 63, "pieces": ["\r\n\r\n", "<|", "endoftext", "|>!!", "\u000b", "'Dḍ̇å", "\r", " ", "…a", "<|", "fim", "_prefix", "|>!!#$%👍🏽", "ꟲ", "Ⅳ", "-", "\tt", ".<", "EOT", ">\r\n", "
", "\"​", "𐞁", "\"\r\n\r\n", "'Z'ſ", "'S", "#$%<", "META", "_START", ">"]} +{"text": "\r'T½\n٣٤٥٦'\"m\"𐞁", "tokens": 22, "pieces": ["\r", "'T", "½", "\n", "٣٤٥", "٦", "'\"", "m", "\"𐞁", ""]} +{"text": "\r\nⅣ\r'Reع㋿​d🙂m<|endoftext|>12345678dḍ̇\"'sſ'Re½t#$%🙂e#$%　'll", "tokens": 45, "pieces": ["\r\n", "Ⅳ", "\r", "'Reع", "㋿​", "d", "🙂m", "<|", "endoftext", "|>", "123", "456", "78", "dḍ̇", "\"'", "sſ", "'", "Re", "½", "t", "#$%🙂", "e", "#$%", "　", "'ll"]} +{"text": "🙂!!\n<|fim_prefix|>​'M0<|endoftext|>…‍fieſaet😀🏽​aZ 　'll👍🏽Ａ", "tokens": 40, "pieces": ["🙂!!\n", "<|", "fim", "_prefix", "|>​'", "M", "0", "<|", "endoftext", "|>", "…", "‍fieſaet", "😀🏽​", "a", "Z", " ", "　", "'ll", "👍🏽", "Ａ"]} +{"text": "#$%>. 😀🏽e\u000b'll👍🏽EOT\"٣٤٥٦eꟲ!!>٣٤٥٦12345678<\"ſa'Re'S\tDž12345678'D", "tokens": 45, "pieces": ["#$%>.", " 😀🏽", "e", "\u000b", "'ll", "👍🏽", "EOT", "\"", "٣٤٥", "٦", "eꟲ", "!!>", "٣٤٥", "٦12", "345", "678", "<\"", "ſa'Re", "'S", "\tDž", "123", "456", "78", "'D"]} +{"text": "㍿\nع\r'ſ912345678Ⅳå,'M'Ré", "tokens": 19, "pieces": ["㍿\n", "ع", "\r", "'ſ", "912", "345", "678", "Ⅳ", "å", ",'", "M'Re", "́"]} +{"text": "t'Dm🙂<\n0\r\n\r\n(d-🙂😀🏽-'SＡ😀🏽", "tokens": 19, "pieces": ["t'D", "m", "🙂<\n", "0", "\r\n\r\n", "(d", "-🙂😀🏽-'", "SＡ", "😀🏽"]} +{"text": "Ⅳ9små're३ >​\"fi's<|fim_prefix|>!٣٤٥٦d‍İ#$%<|fim_prefix|>", "tokens": 35, "pieces": ["Ⅳ9", "små're", "३", " >​\"", "fi's", "<|", "fim", "_prefix", "|>!", "٣٤٥", "٦", "d", "‍İ", "#$%<|", "fim", "_prefix", "|>"]} +{"text": "㋿ Z", "tokens": 5, "pieces": ["㋿", " Z"]} +{"text": "\"'\r\nİ<|fim_prefix|>. \nå.(\u000bfi🙂ḍ̇　 ٣٤٥٦İ𐞁åſ👍🏽", "tokens": 38, "pieces": ["\"'\r\n", "İ", "<|", "fim", "_prefix", "|>.", " \n", "å", ".(", "\u000bfi", "🙂ḍ̇", "　", " ", "٣٤٥", "٦", "İ𐞁åſ", "👍🏽"]} +{"text": "é \r𐞁½🙂३", "tokens": 14, "pieces": ["é", " \r", "𐞁", "½", "🙂<", "META", "_START", ">", "३"]} +{"text": "å👍🏽'Dع　　́𐞁㍿'re👍🏽㍿½漢<|fim_prefix|>　😀🏽'ſå e-㍿३", "tokens": 52, "pieces": ["å", "👍🏽'", "D", "ع", "　", "　́𐞁", "㍿'", "re", "👍🏽㍿", "½", "漢", "<|", "fim", "_prefix", "|>", "　", "😀🏽'", "ſå", " ", " e", "-㍿", "३"]} +{"text": "漢٣٤٥٦\r<|endoftext|>👍🏽٣٤٥٦½t…'VE'VE", "tokens": 28, "pieces": ["漢", "٣٤٥", "٦", "\r", "<|", "endoftext", "|>👍🏽", "٣٤٥", "٦½", "t", "…", "'VE'VE"]} +{"text": " \nİ-<漢\r\"\u000b
'S\r<|fim_prefix|>👍🏽sععt́㍿sꟲ𐞁\t'9\t😀🏽", "tokens": 52, "pieces": [" \n", "İ", "-<", "漢", "\r", "\"", "\u000b", "
", "'S", "\r", "<|", "fim", "_prefix", "|><", "META", "_START", ">👍🏽", "sععt́", "㍿s", "ꟲ𐞁", "\t", "'", "9", "\t", "😀🏽"]} +{"text": " 👍🏽 #$%-​<ꟲİ'ſ㍿EOT><,!me👍🏽Ａ\n.d😀🏽Z'VE'VE>m'sZ", "tokens": 44, "pieces": [" ", " 👍🏽", " #$%-​<", "META", "_START", "><", "ꟲ", "İ'ſ", "㍿EOT", "><,!", "me", "👍🏽", "Ａ", "\n", ".d", "😀🏽", "Z'VE", "'VE", ">m's", "Z"]} +{"text": "ſ 'Re-e\nDž\n", "tokens": 11, "pieces": ["ſ", " ", "'Re", "-e", "\n", "Dž", "\n", ""]} +{"text": "ḍ̇ſésＡ'é'lls'ſ­'M'll'sa😀🏽s", "tokens": 26, "pieces": ["ḍ̇ſés", "Ａ", "'é'll", "s'ſ", "­'", "M'll", "'sa", "😀🏽", "s"]} +{"text": "'​\u000b'Mꟲs字👍🏽t\n𐞁\"İZ'ſ½\r ſ'T🙂㋿😀🏽٣٤٥٦<|fim_prefix|>é½ Ａ​漢'VE9🙂é", "tokens": 64, "pieces": ["'​", "\u000b", "'Mꟲs字", "👍🏽", "t", "\n", "𐞁", "\"İZ'ſ", "½", "\r", " ſ'T", "🙂㋿😀🏽", "٣٤٥", "٦", "<|", "fim", "_prefix", "|><", "META", "_START", ">é", "½", " Ａ", "​<", "META", "_START", ">漢'VE", "9", "🙂é"]} +{"text": "
sé'M'VEsDžé.(‍\"'ſ", "tokens": 13, "pieces": ["
sé'M", "'VEs", "Džé", ".(‍\"'", "ſ"]} +{"text": "'T12345678-\nd!!mé,9'll 'D\" ḍ̇m'ſ'T…<|endoftext|>12345678Ａ字#$%(fi12345678漢", "tokens": 43, "pieces": ["'T", "123", "456", "78", "-\n", "d", "!!", "mé", ",", "9", "'ll", " '", "D", "\"", " ḍ̇m'ſ", "'T", "…", "<|", "endoftext", "|>", "123", "456", "78", "Ａ字", "#$%(", "fi", "123", "456", "78", "漢"]} +{"text": "<|endoftext|>Dž", "tokens": 9, "pieces": ["<|", "endoftext", "|>", "Dž"]} +{"text": "‍<|endoftext|>><|fim_prefix|>'ſ \nd's'llmⅣ'ſ'VE٣٤٥٦'VE\u000b'VE#$%dⅣꟲ字‍\r", "tokens": 49, "pieces": ["‍<|", "endoftext", "|>><", "EOT", "><|", "fim", "_prefix", "|>'", "ſ", " \n", "d's", "'llm", "Ⅳ", "'ſ'VE", "٣٤٥", "٦", "'VE", "\u000b", "'VE", "#$%", "d", "Ⅳ", "ꟲ字", "‍\r"]} +{"text": "'VEå9fi 'll-🙂!!
‍ééＡ\t\rfié'D", "tokens": 28, "pieces": ["'VEå", "9", "fi", " ", " '", "ll", "-🙂!!", "
", "‍éé", "Ａ", "\t\r", "fié", "'", "D"]} +{"text": "\rDž#$%0\r\n㍿́12345678å😀🏽'ſ👍🏽", "tokens": 24, "pieces": ["\r", "Dž", "#$%", "0", "\r\n", "㍿́", "123", "456", "78", "å", "😀🏽'", "ſ", "👍🏽"]} +{"text": " ́<'Re912345678!'re<|fim_prefix|>0\"EOT​t#$%9Z12345678­'ll\"'re🙂😀🏽　　0<'ſ", "tokens": 45, "pieces": [" ", " ́", "<'", "Re", "912", "345", "678", "!'", "re", "<|", "fim", "_prefix", "|><", "META", "_START", ">", "0", "\"EOT", "​t", "#$%", "9", "Z", "123", "456", "78", "­'", "ll", "\"'", "re", "🙂😀🏽", "　", "　", "0", "<'", "ſ"]} +{"text": "\r\n12345678#$%​<|endoftext|>>…EOTⅣ'Séå,́\tＡ\nßZ 😀🏽٣٤٥٦'M!#$%", "tokens": 43, "pieces": ["\r\n", "123", "456", "78", "#$%​<|", "endoftext", "|>>", "…EOT", "Ⅳ", "'Séå", ",́", "\tＡ", "\n", "ß", "Z", " ", "😀🏽", "٣٤٥", "٦", "'M", "!#$%"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'!e'ſ!漢'M're0\"İ\r\n\r\n\"😀🏽ꟲeåé>s", "tokens": 25, "pieces": ["'!", "e'ſ", "!漢'M", "'re", "0", "\"İ", "\r\n\r\n", "\"😀🏽", "ꟲeåé", ">s"]} +{"text": "Dž<|endoftext|>!!ſ …t0　å'ſ", "tokens": 21, "pieces": ["Dž", "<|", "endoftext", "|>!!", "ſ", " ", "…t", "0", "　å'ſ"]} +{"text": "😀🏽\r's12345678Dž'漢​…(s'0\r\n\r\n…\r٣٤٥٦İ漢<|endoftext|>'Re\u000b ", "tokens": 38, "pieces": ["😀🏽\r", "'s", "123", "456", "78", "Dž", "'漢", "​", "…", "(s", "'", "0", "\r\n\r\n…\r", "٣٤٥", "٦", "İ漢", "<|", "endoftext", "|>'", "Re", "\u000b "]} +{"text": "'D!!m'Sé​ß \"‍Z'T'ssİ‍Dž<\t>Dž", "tokens": 23, "pieces": ["'D", "!!", "m'S", "é", "​ß", " ", " \"‍", "Z'T", "'ss", "İ", "‍Dž", "<", "\t", ">Dž"]} +{"text": "ꟲ(9 \n
m👍🏽0Ⅳ\r'T 𐞁!\r\n\r\n>­ſ　\u000b>\t'd'S🙂Ⅳé<\"", "tokens": 39, "pieces": ["ꟲ", "(", "9", " \n", "
m", "👍🏽", "0Ⅳ", "\r", "'T", " ", " 𐞁", "!\r\n\r\n", ">­", "ſ", "　", "\u000b", ">", "\t", "'d'S", "🙂", "Ⅳ", "é", "<\""]} +{"text": "𐞁'S#$%字ſ\u000b", "tokens": 14, "pieces": ["𐞁", "'", "S", "#$%", "字ſ", "\u000b"]} +{"text": ".🙂ꟲ<|endoftext|>s𐞁३12345678!ß(½ét!é  t\r\n३.", "tokens": 36, "pieces": [".🙂", "ꟲ", "<|", "endoftext", "|>", "s𐞁", "३12", "345", "678", "!ß", "(", "½", "ét", "!é", " ", " t", "\r\n", "३", "."]} +{"text": " ́s'VE.'S٣٤٥٦३٣٤٥٦-sſ'Ms…9're😀🏽t́Ａfi㋿‍'Re<½'M字0-'re!!EOT'Ⅳå", "tokens": 54, "pieces": [" ́s'VE", ".'", "S", "٣٤٥", "٦३٣", "٤٥٦", "-sſ'M", "s", "…", "9", "'re", "😀🏽", "t́", "Ａfi", "㋿‍'", "Re", "<", "½", "'M字", "0", "-'", "re", "!!", "EOT", "'", "Ⅳ", "å"]} +{"text": "é('D 9ḿ'ſ\r'S", "tokens": 11, "pieces": ["é", "('", "D", " ", "9", "ḿ'ſ", "\r", "'S"]} +{"text": " Zſ'\"", "tokens": 3, "pieces": [" Zſ", "'\""]} +{"text": "漢​'Re<|endoftext|>Ⅳfi!!", "tokens": 18, "pieces": ["漢", "​'", "Re", "<|", "endoftext", "|>", "Ⅳ", "fi", "!!"]} +{"text": "'M字㋿éİß!!㋿ḍ̇Ａsꟲfi'T's漢!(ḍ̇Ⅳ😀🏽0åå9é", "tokens": 44, "pieces": ["'M字", "㋿", "é", "İß", "!!㋿", "ḍ̇", "Ａsꟲfi'T", "'s漢", "!(", "ḍ̇", "Ⅳ", "😀🏽", "0", "åå", "9", "é"]} +{"text": "\te字ꟲ….ſ", "tokens": 9, "pieces": ["\te字ꟲ", "…", ".ſ"]} +{"text": "éd…­ ㋿٣٤٥٦​'t㋿İ.'Ts0𐞁
🙂'lltع !!fi👍🏽!!́ ", "tokens": 42, "pieces": ["éd", "…", "­", " ㋿", "٣٤٥", "٦", "​'", "t", "㋿İ", ".'", "Ts", "0", "𐞁", "
", "🙂'", "lltع", " ", " !!", "fi", "👍🏽!!́", " "]} +{"text": "🙂'Re🙂'll​漢'D(\r\n\r\nİ!👍🏽's'M\n-😀🏽Z\r\r\n<<12345678İa\r\n\r\n", "tokens": 33, "pieces": ["🙂'", "Re", "🙂'", "ll", "​漢'D", "(\r\n\r\n", "İ", "!👍🏽'", "s'M", "\n", "-😀🏽", "Z", "\r\r\n", "<<", "123", "456", "78", "İa", "\r\n\r\n"]} +{"text": "'ll字é'M\r\n\r\n\n<|endoftext|>'D'🙂Džéİ<'reå'Reé<|endoftext|>　'T\r'D…
s12345678>< sḍ̇𐞁٣٤٥٦'ll<fi\r\n\r\n", "tokens": 66, "pieces": ["'ll字é'M", "\r\n\r\n\n", "<|", "endoftext", "|>'", "D", "'🙂", "Džé", "İ", "<'", "reå'Re", "é", "<|", "endoftext", "|>", "　", "'T", "\r", "'", "D", "…", "
s", "123", "456", "78", "><", " ", " sḍ̇𐞁", "٣٤٥", "٦", "'ll", "<fi", "\r\n\r\n"]} +{"text": "
㍿'DＡ\r\n\r\n'!!>.'ſꟲ\neDž'Re\u000bع-🙂éé,m-\raİ ", "tokens": 35, "pieces": ["
", "㍿'", "DＡ", "\r\n\r\n", "'!!>.'", "ſꟲ", "\n", "e", "Dž'Re", "\u000bع", "-🙂", "éé", ",m", "-\r", "a", "İ", " "]} +{"text": "é0'Re٣٤٥٦", "tokens": 8, "pieces": ["é", "0", "'Re", "٣٤٥", "٦"]} +{"text": "d👍🏽 '\u000b…字'll٣٤٥٦ ſ'D", "tokens": 22, "pieces": ["d", "👍🏽", " ", " '", "\u000b", "…字'll", "٣٤٥", "٦", "", " ſ'D"]} +{"text": "🙂­㋿́s​å​'redfiİ'D
\r\n\r\ne0'T,e…'re漢é!!", "tokens": 29, "pieces": ["🙂­㋿́", "s", "​å", "​'", "redfi", "İ'D", "
\r\n\r\n", "e", "0", "'T", ",e", "…", "'re漢é", "!!"]} +{"text": "'0! \nⅣ9å㍿𐞁Z12345678é's…㍿!!('T
", "tokens": 32, "pieces": ["'", "0", "!", " \n", "Ⅳ9", "å", "㍿𐞁", "Z", "123", "456", "78", "é's", "…", "㍿!!('", "T", "
"]} +{"text": "m🙂'Ré", "tokens": 5, "pieces": ["m", "🙂'", "Ré"]} +{"text": "٣٤٥٦ع́ḍ̇,👍🏽(́\r\nm's­'VE
👍🏽\"\r\nſ\u000b sZ٣٤٥٦.'Te𐞁\r'Da,ḍ̇\u000bꟲ!!", "tokens": 52, "pieces": ["٣٤٥", "٦", "ع́ḍ̇", ",👍🏽(́\r\n", "m's", "­'", "VE", "
", "👍🏽\"\r\n", "ſ", "\u000b", " s", "Z", "٣٤٥", "٦", ".'", "Te𐞁", "\r", "'Da", ",ḍ̇", "\u000bꟲ", "!!"]} +{"text": "12345678'MsＡ𐞁e٣٤٥٦🙂　", "tokens": 21, "pieces": ["123", "456", "78", "'Ms", "Ａ𐞁e", "٣٤٥", "٦", "🙂<", "EOT", ">", "　"]} +{"text": "fi­Ａ字mⅣ'D𐞁'T👍🏽😀🏽're!#$%٣٤٥٦\nd", "tokens": 30, "pieces": ["fi", "­Ａ字m", "Ⅳ", "'D𐞁'T", "👍🏽😀🏽'", "re", "!#$%", "٣٤٥", "٦", "\n", "d"]} +{"text": "<'ReＡ\r\ns's'ret㍿́𐞁.12345678-'S🙂>字\r\n\r\ns㍿\r\ne\r\na12345678👍🏽\r\n", "tokens": 57, "pieces": ["<'", "Re", "Ａ", "\r\n", "s", "'", "s're", "t", "㍿́𐞁", ".", "123", "456", "78", "-'", "S", "🙂>", "字", "\r\n\r\n", "s", "㍿\r\n", "e", "\r\n", "a", "123", "456", "78", "👍🏽\r\n"]} +{"text": "m'ReⅣ​㋿…'VEعßd'S'Re　漢12345678Dž \té \n's.\rfi \n.", "tokens": 37, "pieces": ["m'Re", "Ⅳ", "​㋿", "…", "'VEعßd'S", "'Re", "　漢", "123", "456", "78", "Dž", " ", "\té", " \n", "'s", ".\r", "fi", " \n", ".<", "EOT", ">"]} +{"text": "​>", "tokens": 2, "pieces": ["​>"]} +{"text": "!!t३#$% \n\r\n\r㍿#$%9ß'ḍ̇­mdé
́.", "tokens": 29, "pieces": ["!!", "t", "३", "#$%", " \n\r\n\r", "㍿#$%", "9", "ß", "'ḍ̇", "­mdé", "
́", "."]} +{"text": "m t　'reḍ̇​<|endoftext|>0<|endoftext|>👍🏽", "tokens": 31, "pieces": ["m", " ", " t", "　", "'reḍ̇", "​<|", "endoftext", "|>", "0", "<|", "endoftext", "|>👍🏽<", "META", "_START", ">"]} +{"text": " s ,İꟲع''s'S字t漢-😀🏽12345678\r<|fim_prefix|>ꟲ's\ré's'Dfi<|endoftext|>ḍ̇'ſDž'll,字", "tokens": 62, "pieces": [" s", " ,", "İꟲع", "'<", "EOT", ">'", "s'S", "字t漢", "-<", "EOT", ">😀🏽", "123", "456", "78", "\r", "<|", "fim", "_prefix", "|>", "ꟲ's", "\r", "é's", "'Dfi", "<|", "endoftext", "|>", "ḍ̇'ſ", "Dž'll", ",字"]} +{"text": "
m<|endoftext|><|endoftext|>㋿<|endoftext|>­\"(½\u000b\r'S\"(a'Sḍ̇é½'ſå😀🏽a­🙂­ \nſ'Re👍🏽,\u000bd 👍🏽", "tokens": 68, "pieces": ["
m", "<|", "endoftext", "|><|", "endoftext", "|>㋿<|", "endoftext", "|>­<", "EOT", ">\"(", "½", "\u000b\r", "'S", "\"(", "a'S", "ḍ̇é", "½", "'ſå", "😀🏽", "a", "­🙂­", " \n", "ſ'Re", "👍🏽,", "\u000bd", " ", "👍🏽"]} +{"text": "d‍३", "tokens": 3, "pieces": ["d", "‍", "३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b,\r٣٤٥٦'ll ‍<-!­.\r", "tokens": 15, "pieces": ["\u000b", ",\r", "٣٤٥", "٦", "'ll", " ", "‍<-!­.\r"]} +{"text": "😀🏽'D​​字", "tokens": 7, "pieces": ["😀🏽'", "D", "​​", "字"]} +{"text": "'S­ \nⅣ>", "tokens": 6, "pieces": ["'S", "­", " \n", "Ⅳ", ">"]} +{"text": "́İ㋿ 字é!'Re😀🏽#$%字 ‍é''VE\tⅣd\rＡ ", "tokens": 28, "pieces": ["­m", "\r", "\"<('", "Re", " s", ">😀🏽#$%", "字", " ", " ‍", "é", "''", "VE", "\t", "Ⅳ", "d", "\r", "Ａ", " "]} +{"text": "'VE<|endoftext|>\r\n\r\ntaEOT<\u000b🙂​Ａ-́𐞁-'ſ-👍🏽'́d​ſ​e'M<9\r\n-'s'Déet́9", "tokens": 51, "pieces": ["'VE", "<|", "endoftext", "|>\r\n\r\n", "ta", "EOT", "<", "\u000b", "🙂​", "Ａ", "-́", "𐞁", "-'", "ſ", "-👍🏽'́", "d", "​ſ", "​e'M", "<", "9", "\r\n", "-'", "s'D", "éet́", "9"]} +{"text": "३\t 'VE12345678'S \nſDž\r\néd'M'S…s\n　>t<|fim_prefix|>👍🏽<|fim_prefix|>́'VE1234567812345678<|endoftext|>ꟲ३", "tokens": 64, "pieces": ["३", "\t", " ", "'VE", "123", "456", "78", "'S", " \n", "ſ", "Dž", "\r\n", "éd'M", "'S", "…s", "\n", "　", ">", "t", "<|", "fim", "_prefix", "|>👍🏽<|", "fim", "_prefix", "|>́'", "VE", "123", "456", "781", "234", "567", "8", "<|", "endoftext", "|>", "ꟲ", "३"]} +{"text": "0'M　ḍ̇🙂　", "tokens": 8, "pieces": ["0", "'M", "　ḍ̇", "🙂", "　"]} +{"text": "-<|fim_prefix|>Dž́㋿-'TⅣ🙂३ \n漢're-e'VE'M(\"𐞁字", "tokens": 35, "pieces": ["-<|", "fim", "_prefix", "|>", "Dž́", "㋿-'", "T", "Ⅳ", "🙂", "३", " \n", "漢're", "-e'VE", "'M", "(\"", "𐞁字"]} +{"text": ".㋿㍿", "tokens": 7, "pieces": [".㋿㍿"]} +{"text": ",0'll#$%ꟲ\t-…　३ Ａ　ſİ'S'Smfi\n
", "tokens": 26, "pieces": [",", "0", "'ll", "#$%", "ꟲ", "\t", "-", "…", "　", "३", " ", " Ａ", "　ſ", "İ'S", "'Smfi", "\n", "
"]} +{"text": "ع́​ 　<字 \n'VÉ#$%\u000b('D å३½'TDžＡ-EOT ꟲßEOTＡ'S", "tokens": 36, "pieces": ["ع́", "​", " ", "　", "<字", " \n", "'VÉ", "#$%", "\u000b", "('", "D", " å", "३½", "'TDžＡ", "-EOT", " ꟲß", "EOTＡ'S"]} +{"text": "ع‍'s'S٣٤٥٦Z字'VEİ<'reéé!!EOT12345678…½<'VE\tåaⅣ…éZ'S\n
Ａ-​\r\n\r\n", "tokens": 48, "pieces": ["ع", "‍'", "s'S", "٣٤٥", "٦", "Z字'VE", "İ", "<'", "re", "éé", "!!", "EOT", "123", "456", "78", "…", "½", "<'", "VE", "\tåa", "Ⅳ", "…é", "Z'S", "\n", "
Ａ", "-​\r\n\r\n"]} +{"text": "­　½d\u000bedZ're-m字#$%ḍ̇'re
ḍ̇́漢(>(İ👍🏽<|endoftext|>𐞁<|fim_prefix|>㋿'T字'Re\r\nİ0字", "tokens": 56, "pieces": ["­", "　", "½", "d", "\u000bed", "Z're", "-m字", "#$%", "ḍ̇'re", "
ḍ̇́漢", "(>(", "İ", "👍🏽<|", "endoftext", "|>", "𐞁", "<|", "fim", "_prefix", "|>㋿'", "T字'Re", "\r\n", "İ", "0", "字"]} +{"text": "0‍é٣٤٥٦<|endoftext|><'Re३ -\r\n٣٤٥٦́\r\te ,''T9㍿👍🏽字​<|fim_prefix|>", "tokens": 49, "pieces": ["0", "‍é", "٣٤٥", "٦", "<|", "endoftext", "|><", "META", "_START", "><'", "Re", "३", " ", " -\r\n", "٣٤٥", "٦", "́", "\r", "\te", " ", " ,''", "T", "9", "㍿👍🏽", "字", "​<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|><|fim_prefix|>'S🙂a!३\"'T字\ré0,123456780　\t­''VE㋿<|endoftext|> é", "tokens": 45, "pieces": ["<|", "endoftext", "|><|", "fim", "_prefix", "|>'", "S", "🙂a", "!", "३", "\"'", "T字", "\r", "é", "0", ",", "123", "456", "780", "　", "\t", "­''", "VE", "㋿<|", "endoftext", "|>", " é"]} +{"text": "!'M\u000b<|endoftext|>å\u000b", "tokens": 13, "pieces": ["!'", "M", "\u000b", "<|", "endoftext", "|>", "å", "\u000b"]} +{"text": ",'ll\nd漢ſ", "tokens": 6, "pieces": [",'", "ll", "\n", "d漢ſ"]} +{"text": "eEOT'Tå\r'sḍ̇'ß\"EOT\n㋿", "tokens": 22, "pieces": ["e", "EOT'T", "å", "\r", "'sḍ̇", "'ß", "\"<", "META", "_START", ">EOT", "\n", "㋿"]} +{"text": "'sfißḍ̇Ａé\n\u000bＡé'Mm<|fim_prefix|>å<|fim_prefix|>٣٤٥٦", "tokens": 33, "pieces": ["'sfißḍ̇", "Ａé", "\n", "\u000bＡé'M", "m", "<|", "fim", "_prefix", "|>", "å", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦"]} +{"text": "'M漢#$%ß..'\r\n0\u000b(́\"\r\nééfiⅣDž　\n\u000bＡḍ̇Ａs", "tokens": 30, "pieces": ["'M漢", "#$%", "ß", "..'\r\n", "0", "\u000b", "(́", "\"\r\n", "ééfi", "Ⅳ", "Dž", "", "　\n", "\u000bＡḍ̇", "Ａs"]} +{"text": "é \nſ", "tokens": 4, "pieces": ["é", " \n", "ſ"]} +{"text": "t　Z㋿,12345678.　EOT'VE
३­ Ⅳ㋿<|endoftext|>ßEOT!EOT'S", "tokens": 40, "pieces": ["t", "　Z", "㋿,", "123", "456", "78", ".", "　EOT'VE", "
", "३", "­", " ", " ", "Ⅳ", "㋿<|", "endoftext", "|>", "ß", "EOT", "!EOT'S"]} +{"text": "٣٤٥٦<|endoftext|>😀🏽३😀🏽'DＡ'reß 'Re. \n　9éå're'ſ<'reع!! <‍me👍🏽'llm", "tokens": 52, "pieces": ["٣٤٥", "٦", "<|", "endoftext", "|>😀🏽", "३", "😀🏽'", "DＡ're", "ß", " ", "'Re", ".<", "EOT", ">", " \n", "　", "9", "éå're", "'ſ", "<'", "reع", "!!", " ", "<‍", "me", "👍🏽'", "llm"]} +{"text": "'M Zſtaé9́ Ⅳ'S٣٤٥٦\r\n\r\n'ſ're'Re…", "tokens": 25, "pieces": ["'M", " Zſtaé", "9", "́", " ", "Ⅳ", "'S", "٣٤٥", "٦", "\r\n\r\n", "'ſ're", "'Re", "…"]} +{"text": "!!漢EOT字<漢", "tokens": 9, "pieces": ["!!", "漢", "EOT字", "<漢"]} +{"text": "\n \n🙂​>‍'s字㍿s
're½EOT'Re'reé. \n", "tokens": 23, "pieces": ["\n \n", "🙂​>‍'", "s字", "㍿s", "
", "'re", "½", "EOT'Re", "'reé", ".", " \n"]} +{"text": "\r\n\r\n'D́é-'lld!𐞁\u000bm㋿ع'D", "tokens": 22, "pieces": ["\r\n\r\n", "'D́é", "-'", "lld", "!<", "META", "_START", ">𐞁", "\u000bm", "㋿ع'D"]} +{"text": "​…३(​9t ", "tokens": 9, "pieces": ["​", "…", "३", "(​", "9", "t", " "]} +{"text": "🙂\r\nd0'VE🙂'M字
㋿!!'!!'D🙂", "tokens": 23, "pieces": ["🙂\r\n", "d", "0", "'VE", "🙂'", "M字", "
", "㋿!!'<", "EOT", ">!!'", "D", "🙂"]} +{"text": "-.m'　\r\nt<|fim_prefix|>👍🏽👍🏽a😀🏽½!!'VEDž\r ", "tokens": 30, "pieces": ["-.", "m", "'", "　\r\n", "t", "<|", "fim", "_prefix", "|>👍🏽👍🏽", "a", "😀🏽", "½", "!!'", "VEDž", "\r", " "]} +{"text": " \n<'M#$%\n\r\nİéＡ🙂 漢å\"　ع", "tokens": 21, "pieces": [" \n", "<'", "M", "#$%\n\r\n", "İé", "Ａ", "🙂", " 漢å", "\"", "　ع"]} +{"text": "9'Dİ👍🏽㍿🙂m 
e fi<|fim_prefix|>㋿t'M'll \n!Ⅳ'VEİ-'reé½ !fi9\t\n \n<|endoftext|>", "tokens": 52, "pieces": ["9", "'Dİ", "👍🏽㍿🙂", "m", " ", "
e", " ", " fi", "<|", "fim", "_prefix", "|>㋿", "t'M", "'ll", " \n", "!", "Ⅳ", "'VEİ", "-'", "reé", "½", " ", " !", "fi", "9", "\t\n \n", "<|", "endoftext", "|>"]} +{"text": "́…́\"Dž ", "tokens": 11, "pieces": ["́", "…́", "\"Dž", " "]} +{"text": "\r\n\r\nds,㋿'VE", "tokens": 8, "pieces": ["\r\n\r\n", "ds", ",㋿'", "VE"]} +{"text": "-a<|endoftext|>'s12345678!!s　.　", "tokens": 21, "pieces": ["-a", "<|", "endoftext", "|>'", "s", "123", "456", "78", "!!", "s", "", "　", ".", "　"]} +{"text": "12345678‍'s(t's 𐞁éعꟲ٣٤٥٦>0\r\nİ12345678!!३\",,fie'rea12345678Džꟲé' .EOT", "tokens": 60, "pieces": ["123", "456", "78", "‍'", "s", "(t's", " 𐞁éع", "ꟲ", "٣٤٥", "٦", ">", "0", "\r\n", "İ", "123", "456", "78", "!!", "३", "\",<", "META", "_START", ">,", "fie", "'", "rea", "123", "456", "78", "Džꟲé", "'", " .", "EOT"]} +{"text": "a'llEOT-ع𐞁're٣٤٥٦㋿dé<|fim_prefix|>at,é字#$%-0👍🏽😀🏽.½ḍ̇<|fim_prefix|>ß<𐞁
㍿ (", "tokens": 61, "pieces": ["a'll", "EOT", "-ع𐞁're", "٣٤٥", "٦", "㋿dé", "<|", "fim", "_prefix", "|>", "at", ",é字", "#$%-", "0", "👍🏽😀🏽.", "½", "ḍ̇", "<|", "fim", "_prefix", "|>", "ß", "<𐞁", "
", "㍿", " ", "("]} +{"text": "a-ßt𐞁e<|endoftext|>éfi字're
\tß　'VE\r\n\r\n \n㋿'TfiEOTdm𐞁\rß́'M", "tokens": 50, "pieces": ["a", "-ßt𐞁e", "<|", "endoftext", "|>", "éfi字", "'", "re", "
", "\tß", "　", "'VE", "\r\n\r\n \n", "㋿'", "Tfi", "EOTdm𐞁", "\r", "ß́'M"]} +{"text": "ꟲ!!٣٤٥٦\r\ne's't\n字　EOTmⅣ ß漢0dZ.ßsꟲ \" 'refié", "tokens": 42, "pieces": ["ꟲ", "!!", "٣٤٥", "٦", "\r\n", "e's", "'t", "\n", "字", "　EOTm", "Ⅳ", " ß漢", "0", "d", "Z", ".ßsꟲ", " ", "\"", " ", " '", "re", "fié"]} +{"text": "-😀🏽12345678å.0fiEOT'Tع!㋿　Z>字 'Tꟲ#$%\r\nt'VEع
\u000bßḍ̇'S12345678'S \n\t\u000b٣٤٥٦", "tokens": 58, "pieces": ["-😀🏽", "123", "456", "78", "å", ".", "0", "fi", "EOT'T", "ع", "!㋿", "　Z", ">字", " ", " '", "Tꟲ", "#$%\r\n", "t'VE", "ع", "
", "\u000bßḍ̇", "'", "S", "123", "456", "78", "'S", " \n", "\t", "\u000b", "٣٤٥", "٦"]} +{"text": "\nİ\"\n'T.e9'reé'ſZé👍🏽,12345678  👍🏽'll漢　Ⅳ'VEta's'VE \n's'Tعꟲ", "tokens": 44, "pieces": ["\n", "İ", "\"\n", "'T", ".e", "9", "'reé'ſ", "Zé", "👍🏽,", "123", "456", "78", " ", " ", "👍🏽'", "ll漢", "　", "Ⅳ", "'VEta's", "'VE", " \n", "'s'T", "عꟲ"]} +{"text": "'VE'sDž½fi㍿漢ſ<\rDž\r\n<\u000b\"\u000b漢Ａḍ̇­ad🙂", "tokens": 29, "pieces": ["'VE's", "Dž", "½", "fi", "㍿漢ſ", "<\r", "Dž", "\r\n", "<", "\u000b", "\"", "\u000b漢Ａḍ̇", "­ad", "🙂"]} +{"text": "\u000b!\ra漢 EOT.‍!\u000b​​a👍🏽 \r\n'́", "tokens": 23, "pieces": ["\u000b", "!\r", "a漢", " EOT", ".‍!", "\u000b", "​​", "a", "👍🏽", " \r\n", "'́"]} +{"text": " Z!!½-,#$%fi'MDž 漢ꟲ>'reꟲ0t'reéḍ̇12345678(12345678ſ३#$%'ſdＡet12345678t३", "tokens": 55, "pieces": [" Z", "!!", "½", "-,#$%", "fi'M", "Dž", " ", " 漢", "ꟲ", ">'", "reꟲ", "0", "t're", "éḍ̇", "123", "456", "78", "(", "123", "456", "78", "ſ", "३", "#$%'", "ſd", "Ａ", "et", "123", "456", "78", "t", "३"]} +{"text": "㋿\t\r\n12345678é\u000b!!'ſ'MＡ,\r\n\r\n\"'ll'EOT٣٤٥٦'MZ", "tokens": 28, "pieces": ["㋿", "\t\r\n", "123", "456", "78", "é", "\u000b", "!!'", "ſ'M", "Ａ", ",\r\n\r\n", "\"<", "META", "_START", ">'", "ll", "'EOT", "٣٤٥", "٦", "'MZ"]} +{"text": "'Re
‍\t\r\n\r\nİ ㍿d \nmİ#$%ḍ̇字#$%eZ(12345678\r\n\r\n t \n'ReＡ‍\r\n's#$%ⅣＡ<|endoftext|>'VE'VE", "tokens": 55, "pieces": ["'Re", "
", "‍", "\t\r\n\r\n", "İ", " ", " ㍿", "d", " \n", "m", "İ", "#$%", "ḍ̇字", "#$%", "e", "Z", "(", "123", "456", "78", "\r\n\r\n", " ", " t", " \n", "'Re", "Ａ", "‍\r\n", "'s", "#$%", "Ⅳ", "Ａ", "<|", "endoftext", "|>'", "VE'VE"]} +{"text": "'lld㍿
'red٣٤٥٦👍🏽aDž'reꟲ<<|fim_prefix|>\r\n'Re'VEa\"<|fim_prefix|>é''🙂EOT'0're'ſ'T", "tokens": 52, "pieces": ["'lld", "㍿", "
", "'red", "٣٤٥", "٦", "👍🏽", "a", "Dž're", "ꟲ", "<<", "EOT", "><|", "fim", "_prefix", "|>\r\n", "'Re'VE", "a", "\"<|", "fim", "_prefix", "|>", "é", "''🙂", "EOT", "'", "0", "'re'ſ", "'T"]} +{"text": " EOT　漢\"<\"٣٤٥٦ꟲ12345678!\u000b👍🏽'M!Ａ'VE", "tokens": 28, "pieces": [" ", " EOT", "　漢", "\"<\"", "٣٤٥", "٦", "ꟲ", "123", "456", "78", "!", "\u000b", "👍🏽'", "M", "!Ａ'VE"]} +{"text": "<|fim_prefix|>　'reſZ‍🙂\r\n\r\nİfi\r\n'S👍🏽ſİ…'S ", "tokens": 29, "pieces": ["<|", "fim", "_prefix", "|>", "　", "'reſ", "Z", "‍🙂\r\n\r\n", "İfi", "\r\n", "'S", "👍🏽", "ſ", "İ", "…", "'", "S", " "]} +{"text": "de३　", "tokens": 7, "pieces": ["de", "", "३", "　"]} +{"text": "\n𐞁㍿Dž ㋿a'll!!ſ字…Ⅳ'sé12345678", "tokens": 33, "pieces": ["\n", "𐞁", "㍿Dž", " ", " ㋿", "a'll", "!!", "ſ字", "…", "Ⅳ", "'sé", "", "123", "456", "78"]} +{"text": "<|fim_prefix|>e😀🏽tZ​\t#$%عå३　㍿ß😀🏽İ9", "tokens": 34, "pieces": ["<|", "fim", "_prefix", "|>", "e", "😀🏽", "t", "Z", "​", "\t", "#$%", "عå", "३", "　", "㍿<", "META", "_START", ">ß", "😀🏽", "İ", "9"]} +{"text": "#$%!!½ ٣٤٥٦'D'VEa
\t ٣٤٥٦('VE!　\"​.", "tokens": 27, "pieces": ["#$%!!", "½", " ", " ", "٣٤٥", "٦", "'D'VE", "a", "
\t", " ", "٣٤٥", "٦", "('", "VE", "!", "　", "\"​."]} +{"text": "9<'re'M'T😀🏽('ſ'M.'VE\r\n\r\n👍🏽éa\u000bⅣعé<|fim_prefix|>>'D\tZDž　 ", "tokens": 42, "pieces": ["9", "<'", "re'M", "'T", "😀🏽('", "ſ'M", ".'", "VE", "\r\n\r\n", "👍🏽", "éa", "", "\u000b", "Ⅳ", "عé", "<|", "fim", "_prefix", "|>>'", "D", "\tZDž", "　 "]} +{"text": "'‍ſ#$%…३\r\n\r\ne𐞁'T!٣٤٥٦-sé é​'s𐞁d\r\n\r\nſå<|endoftext|>字 \n👍🏽DžⅣ½\",Z👍🏽", "tokens": 63, "pieces": ["'‍", "ſ", "#$%", "…", "३", "\r\n\r\n", "e𐞁'T", "!", "٣٤٥", "٦", "-sé", " é", "​'", "s𐞁d", "\r\n\r\n", "ſå", "<|", "endoftext", "|>", "字", " \n", "👍🏽", "Dž", "Ⅳ½", "\",", "Z", "👍🏽"]} +{"text": "9Z…🙂३ß \n(<|endoftext|>\t9'D٣٤٥٦EOT ꟲ.é
å'll漢İＡ", "tokens": 40, "pieces": ["9", "Z", "…", "🙂", "३", "ß", " \n", "(<|", "endoftext", "|>", "\t", "", "9", "'D", "٣٤٥", "٦", "EOT", " ", " ꟲ", ".é", "
å'll", "漢", "İＡ"]} +{"text": "
-\u000b'Tß'M　𐞁😀🏽\r'll३'Ⅳ…9Zm9Ａe'Tİ漢<éßå", "tokens": 43, "pieces": ["
", "-<", "EOT", ">", "\u000b", "'Tß'M", "　𐞁", "😀🏽\r", "'ll", "३", "'", "Ⅳ", "…", "9", "Zm", "9", "Ａe'T", "İ漢", "<éßå"]} +{"text": "\r\n'é\r\nع㋿'reEOT a'll \r0'D\u000bİ#$%🙂\r\n>㍿>å\r's😀🏽́ 'S Ａ ", "tokens": 47, "pieces": ["\r\n", "'é", "\r\n", "ع", "㋿'", "re", "EOT", " a'll", " \r", "0", "'D", "\u000bİ", "#$%🙂\r\n", ">㍿>", "å", "\r", "'s", "😀🏽<", "META", "_START", ">́", " ", "'S", " Ａ", " "]} +{"text": ".Ａ<|fim_prefix|>'ſfiDž­'eée٣٤٥٦'VE<|endoftext|>👍🏽mİ!!
", "tokens": 39, "pieces": [".Ａ", "<|", "fim", "_prefix", "|>'", "ſfi", "Dž", "­'", "e", "ée", "٣٤٥", "٦", "'VE", "<|", "endoftext", "|>👍🏽", "m", "İ", "!!", "
"]} +{"text": "!漢‍<|fim_prefix|>Ａ'D'ſ 𐞁- .  a", "tokens": 24, "pieces": ["!漢", "‍<|", "fim", "_prefix", "|>", "Ａ'D", "'ſ", " ", " 𐞁", "-", " .", " ", " a"]} +{"text": "İé<|fim_prefix|>'re'T½ꟲ!!t Z　​ꟲEOT\r\n(fi's#$%sDž́ ㋿…'T!m\u000b", "tokens": 48, "pieces": ["İé", "<|", "fim", "_prefix", "|>'", "re'T", "½", "ꟲ", "!!", "t", " Z", "　", "​ꟲ", "EOT", "\r\n", "(fi's", "#$%", "s", "Dž́", " ", "㋿", "…", "'T", "!m", "\u000b"]} +{"text": "'re漢<|endoftext|>½ (😀🏽a#$%0<|fim_prefix|>å𐞁<|endoftext|>'Tſ\r\n\r\n‍Ⅳ\né<٣٤٥٦('ſ'T 9​漢३é,\u000b12345678İ\u000b", "tokens": 71, "pieces": ["'re漢", "<|", "endoftext", "|>", "½", " ", "(😀🏽", "a", "#$%", "0", "<|", "fim", "_prefix", "|>", "å𐞁", "<|", "endoftext", "|>'", "Tſ", "\r\n\r\n", "‍", "Ⅳ", "\n", "é", "<", "٣٤٥", "٦", "('", "ſ'T", " ", " ", "9", "​漢", "३", "é", ",", "\u000b", "123", "456", "78", "İ", "", "\u000b"]} +{"text": "#$%
(㍿Z!!s👍🏽\"EOT<\t𐞁,'T!'DⅣ'Re👍🏽३ḍ̇", "tokens": 36, "pieces": ["#$%", "
", "(㍿", "Z", "!!", "s", "👍🏽\"", "EOT", "<", "\t𐞁", ",'", "T", "!'", "D", "Ⅳ", "'Re", "👍🏽", "३", "ḍ̇"]} +{"text": "mm 're𐞁då'ReⅣ🙂 　>\r\n'ſ", "tokens": 19, "pieces": ["mm", " ", " '", "re𐞁då'Re", "Ⅳ", "🙂", " ", "　", ">\r\n", "'ſ"]} +{"text": "٣٤٥٦'ll\r\nZ字9½́#$%!t", "tokens": 14, "pieces": ["٣٤٥", "٦", "'ll", "\r\n", "Z字", "9½", "́", "#$%!", "t"]} +{"text": "fi'Dt.'llḍ̇'så.​\u000b'D㍿ḍ̇'ll'T　>Ⅳ \n#$%'D!'ll\r", "tokens": 36, "pieces": ["fi'D", "t", ".<", "META", "_START", ">'", "llḍ̇'s", "å", ".​", "\u000b", "'D", "㍿ḍ̇'ll", "'T", "　", ">", "Ⅳ", " \n", "#$%'", "D", "!'", "ll", "\r"]} +{"text": "'Reİfi\t'VE12345678字's'Mꟲſé(>'S ٣٤٥٦,😀🏽ſa\r\nmDža…ⅣＡ'\r\n", "tokens": 43, "pieces": ["'Re", "İfi", "\t", "'VE", "123", "456", "78", "字's", "'Mꟲſé", "(>'", "S", " ", "٣٤٥", "٦", ",😀🏽", "ſa", "\r\n", "m", "Dža", "…", "Ⅳ", "Ａ", "'\r\n"]} +{"text": "!!EOT½​\r\n\r\n\t½'ſ'll's½
\"d½'\r\n\r\n٣٤٥٦t½ ḍ̇(<|fim_prefix|>.ꟲm'D​sm!😀🏽9 ", "tokens": 46, "pieces": ["!!", "EOT", "½", "​\r\n\r\n", "\t", "½", "'ſ'll", "'s", "½", "
", "\"d", "½", "'\r\n\r\n", "٣٤٥", "٦", "t", "½", " ḍ̇", "(<|", "fim", "_prefix", "|>.", "ꟲm'D", "​sm", "!😀🏽", "9", " "]} +{"text": "🙂fiع#$%\ŕ", "tokens": 7, "pieces": ["🙂fiع", "#$%\r", "́"]} +{"text": "EOTİ\n#$%٣٤٥٦fi9́'S'VEZa12345678字s0>'re'Re<<|fim_prefix|>́", "tokens": 39, "pieces": ["EOTİ", "\n", "#$%<", "EOT", ">", "٣٤٥", "٦", "fi", "9", "́'S", "'VEZa", "123", "456", "78", "字s", "0", ">'", "re'Re", "<<|", "fim", "_prefix", "|>́"]} +{"text": "…漢'ſeZꟲ'VE­'Sa　0\u000b\r", "tokens": 19, "pieces": ["…漢'ſ", "e", "Zꟲ'VE", "­'", "Sa", "　", "0", "\u000b\r"]} +{"text": "­ \nm!", "tokens": 4, "pieces": ["­", " \n", "m", "!"]} +{"text": "'DＡm½é,'ſ,-e'D😀🏽'S,'T0é<
　㍿e㍿'D'Re٣٤٥٦#$%\"𐞁'DＡa", "tokens": 50, "pieces": ["'DＡm", "½", "é", ",'", "ſ", ",-", "e'D", "😀🏽'", "S", ",'", "T", "0", "é", "<", "
", "　", "㍿e", "㍿'", "D'Re", "٣٤٥", "٦", "#$%\"", "𐞁'D", "Ａa"]} +{"text": "\r𐞁<|endoftext|>9dd!'T\r\n\r\n'sDž#$%9ß́\u000b<|fim_prefix|>\u000bḍ̇‍ ", "tokens": 38, "pieces": ["\r", "𐞁", "<|", "endoftext", "|>", "9", "dd", "!'", "T", "\r\n\r\n", "'s", "Dž", "#$%", "9", "ß́", "\u000b", "<|", "fim", "_prefix", "|>", "\u000bḍ̇", "‍", " "]} +{"text": "-𐞁!!ḍ̇‍.　0ꟲ '字a‍fiEOT\"ſ \ns'DZ're ٣٤٥٦'s", "tokens": 36, "pieces": ["-𐞁", "!!", "ḍ̇", "‍.", "　", "0", "ꟲ", " ", "'字a", "‍fi", "EOT", "\"ſ", " \n", "s'D", "Z're", " ", "٣٤٥", "٦", "'s"]} +{"text": "'VE!!!😀🏽\tEOT'M-t>
\re0\"t\r\n\r\n'SEOT́
Dž字‍(", "tokens": 31, "pieces": ["'VE", "!!!😀🏽", "\tEOT'M", "-t", ">", "
\r", "e", "0", "\"t", "\r\n\r\n", "'SEOT́", "
Dž字", "‍<", "EOT", ">("]} +{"text": "EOT'ſ​'M", "tokens": 7, "pieces": ["EOT'ſ", "​'", "M"]} +{"text": "𐞁👍🏽👍🏽Ａß0½'MDže!!
EOT'llع<|fim_prefix|>é", "tokens": 32, "pieces": ["𐞁", "👍🏽👍🏽", "Ａß", "0½", "'MDže", "!!", "
EOT'll", "ع", "<|", "fim", "_prefix", "|>", "é"]} +{"text": "#$%ع'TEOTs'S#$%ßd­é​ 'Re-#$%́", "tokens": 30, "pieces": ["#$%", "ع'T", "EOTs'S", "#$%", "ßd", "­é", "​", " ", " <", "META", "_START", ">'", "Re", "-#$%́<", "EOT", ">"]} +{"text": "ꟲ字​\r'll!!", "tokens": 8, "pieces": ["ꟲ字", "​\r", "'ll", "!!"]} +{"text": "……a㍿m
𐞁𐞁,0fiſ‍d .>👍🏽 -é,\u000b́Dž字👍🏽㍿\n\u000b", "tokens": 50, "pieces": ["…", "…a", "㍿m", "
𐞁𐞁", ",", "0", "fiſ", "‍d", " ", " .>👍🏽", " ", "-", "é", ",", "\u000b́Dž字", "👍🏽㍿\n", "\u000b"]} +{"text": "d½Džtaع​s#$%t'reع'VE👍🏽're", "tokens": 23, "pieces": ["d", "½", "Džt", "aع", "​s", "#$%", "t're", "ع'VE", "👍🏽'", "re"]} +{"text": "ꟲaſ \n'D-", "tokens": 8, "pieces": ["ꟲaſ", " \n", "'D", "-"]} +{"text": "😀🏽(३'ſZ\r\nſ  t \n<|endoftext|> Ⅳꟲḍ̇İ12345678e<|fim_prefix|>٣٤٥٦\n'S t's　İd👍🏽d", "tokens": 60, "pieces": ["😀🏽(", "३", "'ſ", "Z", "\r\n", "ſ", " ", " t", " \n", "<|", "endoftext", "|>", " ", " ", "Ⅳ", "ꟲḍ̇", "İ", "123", "456", "78", "e", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "\n", "'S", " t's", "　İd", "👍🏽", "d"]} +{"text": ",…'llḍ̇s\"<\"३a(#$%\"s👍🏽字é'D!!
", "tokens": 26, "pieces": [",", "…", "'llḍ̇s", "\"<<", "META", "_START", ">\"", "३", "a", "(#$%\"", "s", "👍🏽", "字é'D", "!!", "
"]} +{"text": "👍🏽 ㍿½'VE#$%­'Re३　ſ'D­t'M३👍🏽Ａ<,'lla<|fim_prefix|>ＡDž🙂sd", "tokens": 40, "pieces": ["👍🏽", " ", "㍿", "½", "'VE", "#$%­'", "Re", "३", "　ſ'D", "­t'M", "३", "👍🏽", "Ａ", "<,'", "lla", "<|", "fim", "_prefix", "|>", "ＡDž", "🙂sd"]} +{"text": "Ａ𐞁\r\n\r\n#$%0㋿EOT-عſ'MmⅣd#$%'D12345678d… ", "tokens": 32, "pieces": ["Ａ𐞁", "\r\n\r\n", "#$%", "0", "㋿EOT", "-عſ'M", "m", "Ⅳ", "d", "#$%'", "D", "123", "456", "78", "d", "… "]} +{"text": "<|endoftext|>…'㋿<|endoftext|>t> fim's㋿(…e👍🏽<|fim_prefix|>t ,ꟲ", "tokens": 49, "pieces": ["<|", "endoftext", "|>", "…", "'㋿<|", "endoftext", "|>", "t", ">", " ", " fim's", "㋿(", "…e", "👍🏽<|", "fim", "_prefix", "|>", "t", " ,", "ꟲ"]} +{"text": "
字́ſ'M'D<|endoftext|> (́
­,9#$%\"'s'Tſḍ̇>9‍d \n\"३fi!!å'S!td<|fim_prefix|>", "tokens": 48, "pieces": ["
字́ſ'M", "'D", "<|", "endoftext", "|>", " (́", "
", "­,", "9", "#$%\"'", "s'T", "ſḍ̇", ">", "9", "‍d", " \n", "\"", "३", "fi", "!!", "å'S", "!td", "<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|>12345678sfi(a　d", "tokens": 15, "pieces": ["<|", "endoftext", "|>", "123", "456", "78", "sfi", "(a", "　d"]} +{"text": "Džß'll😀🏽EOT٣٤٥٦ſZ0<|fim_prefix|>-", "tokens": 26, "pieces": ["Džß'll", "😀🏽", "EOT", "٣٤٥", "٦", "ſ", "Z", "0", "<|", "fim", "_prefix", "|>-"]} +{"text": "字'VE'T'M'll(\t<'ſſ ‍!!!!d#$%!!", "tokens": 21, "pieces": ["字'VE", "'T'M", "'ll", "(", "\t", "<'", "ſſ", " ", " ‍!!!!", "d", "#$%!!"]} +{"text": "<|fim_prefix|>!!…́'㍿a­\"㋿\r\n -<|endoftext|>>eḍ̇🙂字'Re->
.s", "tokens": 39, "pieces": ["<|", "fim", "_prefix", "|>!!", "…́", "'㍿", "a", "­\"㋿\r\n", " -<|", "endoftext", "|>>", "eḍ̇", "🙂字'Re", "->", "
", ".s"]} +{"text": "(9漢'S're\r\ń's'ſ\r\n\r\n
('Dd<​12345678'D٣٤٥٦½#$%t­
ſ'S\r\nꟲ", "tokens": 40, "pieces": ["(", "9", "漢'S", "'re", "\r\n", "́'s", "'ſ", "\r\n\r\n", "
", "('", "Dd", "<​", "123", "456", "78", "'D", "٣٤٥", "٦½", "#$%", "t", "­", "
ſ'S", "\r\n", "ꟲ"]} +{"text": "🙂m(('s.½  ſ.'Tfiåa‍-9\t\"#$%#$%​ꟲ-'Re \n'Tꟲ\n㍿字-‍#$%", "tokens": 44, "pieces": ["🙂m", "(('", "s", ".", "½", " ", " ſ", ".'", "Tfiåa", "‍-", "9", "\t", "\"#$%#$%​", "ꟲ", "-'", "Re", " \n", "'Tꟲ", "\n", "㍿字", "-‍#$%"]} +{"text": "(\r<|fim_prefix|>½'re'll​aß", "tokens": 14, "pieces": ["(\r", "<|", "fim", "_prefix", "|>", "½", "'re'll", "​aß"]} +{"text": "\r\n٣٤٥٦d,\r\n\r\n", "tokens": 7, "pieces": ["\r\n", "٣٤٥", "٦", "d", ",\r\n\r\n"]} +{"text": "é\t­ \n0(12345678👍🏽٣٤٥٦　ḍ̇", "tokens": 20, "pieces": ["é", "\t", "­", " \n", "0", "(", "123", "456", "78", "👍🏽", "٣٤٥", "٦", "　ḍ̇"]} +{"text": "㍿ßém'Tå't🙂<|endoftext|>!　'M३.09Dž'ReZ!!👍🏽\u000bfid(e​漢 
'ſ12345678éＡ'T", "tokens": 47, "pieces": ["㍿ßém'T", "å't", "🙂<|", "endoftext", "|>!", "　", "'M", "३", ".", "09", "Dž'Re", "Z", "!!👍🏽", "\u000bfid", "(e", "​漢", " ", "
", "'ſ", "123", "456", "78", "é", "Ａ'T"]} +{"text": "(½İ<|fim_prefix|>!!a'　", "tokens": 13, "pieces": ["(", "½", "İ", "<|", "fim", "_prefix", "|>!!", "a", "'", "　"]} +{"text": "\r\n½ſ㋿ ㋿👍🏽dſ½㋿", "tokens": 19, "pieces": ["\r\n", "½", "ſ", "㋿", " ", "㋿👍🏽", "dſ", "½", "㋿"]} +{"text": "å½EOT's漢EOT‍ Dž' \n9é \n😀🏽é\rع>ḍ̇'VEß'Dmsİḍ̇", "tokens": 42, "pieces": ["å", "½", "EOT's", "漢", "EOT", "‍", " Dž", "'", " \n", "9", "é", " \n", "😀🏽", "é", "\r", "ع", ">ḍ̇'VE", "ß", "'", "Dms", "İḍ̇"]} +{"text": "Ⅳ12345678ḍ̇字,㍿0عs\u000b0fi'Ré.eßＡDž​漢ſ", "tokens": 29, "pieces": ["Ⅳ12", "345", "678", "ḍ̇字", ",㍿", "0", "عs", "\u000b", "0", "fi'Re", "́", ".eß", "ＡDž", "​漢ſ"]} +{"text": "é'Maa'lls😀🏽
,Dž \nع,", "tokens": 16, "pieces": ["é'M", "aa'll", "s", "😀🏽", "
", ",Dž", " \n", "ع", ","]} +{"text": "'sＡé… \ne'T½12345678s", "tokens": 14, "pieces": ["'s", "Ａé", "… \n", "e'T", "½12", "345", "678", "s"]} +{"text": "12345678<|fim_prefix|>😀🏽t字é'ع㋿👍🏽é\t fid'VE\r\" s\r\nß", "tokens": 38, "pieces": ["123", "456", "78", "<|", "fim", "_prefix", "|>😀🏽", "t字é", "'ع", "㋿👍🏽", "é", "\t", " fid'VE", "\r", "\"", " ", " s", "\r\n", "ß"]} +{"text": "ꟲDž,'VE", "tokens": 7, "pieces": ["ꟲ", "Dž", ",'", "VE"]} +{"text": "Dž<|endoftext|>-'𐞁'll
𐞁å\u000b<😀🏽#$%\t Ａ\u000bé'S'Ree12345678𐞁‍éＡ #$%ß'VE", "tokens": 58, "pieces": ["Dž", "<|", "endoftext", "|>-'", "𐞁'll", "
𐞁å", "\u000b", "<😀🏽#$%", "\t", " Ａ", "\u000bé'S", "'Ree", "123", "456", "78", "𐞁", "‍é", "Ａ", " <", "EOT", ">#$%", "ß'VE"]} +{"text": "\té", "tokens": 2, "pieces": ["\té"]} +{"text": "　ḍ̇…㋿ꟲ३,𐞁İ\n-'M字mfi𐞁😀🏽ع👍🏽́!!!!d\"\u000b'VE ३ (　", "tokens": 48, "pieces": ["　ḍ̇", "…", "㋿ꟲ", "३", ",𐞁", "İ", "\n", "-'", "M字mfi𐞁", "😀🏽", "ع", "👍🏽́!!!!", "d", "\"", "\u000b", "'VE", " ", "३", " ", " (", "　"]} +{"text": "\r\n0'M\r😀🏽e½
\r\n\r\n\n(.#$% \nß'llعⅣéſⅣs　㍿\r\n\r\n#$%\"EOT'S　👍🏽🙂'VE\n'ſ", "tokens": 54, "pieces": ["\r\n", "0", "'", "M", "\r", "😀🏽", "e", "½", "
\r\n\r\n\n", "(.#$%", " \n", "ß'll", "ع", "Ⅳ", "éſ", "", "Ⅳ", "s", "　", "㍿\r\n\r\n", "#$%\"", "EOT'S", "　", "👍🏽🙂'", "VE", "\n", "'ſ"]} +{"text": "𐞁㋿!\n'DＡ\n\tßé‍Ⅳ>İ.<|fim_prefix|> ", "tokens": 30, "pieces": ["𐞁", "㋿!\n", "'DＡ", "\n", "\tßé", "‍", "Ⅳ", ">İ", ".<|", "fim", "_prefix", "|>", " "]} +{"text": "́㋿ḍ̇!字'ſ EOT​fiⅣ\t漢𐞁'sfi'T'M ‍ \n㍿é'Re-0", "tokens": 42, "pieces": ["́", "㋿ḍ̇", "!字'ſ", " EOT", "​fi", "Ⅳ", "\t漢𐞁's", "fi'T", "'M", " ", " ‍", " \n", "㍿é'Re", "-<", "EOT", ">", "0"]} +{"text": "'Re‍sEOT#$%字'ſ,😀🏽d漢 \nＡ \n!!'M're…!!'T ㍿ İ(a 𐞁s\rß٣٤٥٦\t'VE'Mİa𐞁𐞁", "tokens": 61, "pieces": ["'Re", "‍s", "EOT", "#$%", "字'ſ", ",😀🏽", "d漢", " \n", "Ａ", " \n", "!!'", "M're", "…", "!!'", "T", " ㍿", " İ", "(a", " 𐞁s", "\r", "ß", "٣٤٥", "٦", "\t", "'VE'M", "İa𐞁𐞁"]} +{"text": "𐞁#$%dDž漢", "tokens": 13, "pieces": ["𐞁", "#$%<", "EOT", ">d", "Dž漢"]} +{"text": "<|fim_prefix|>ḍ̇'Séſ\n're\r\n\r\n", "tokens": 16, "pieces": ["<|", "fim", "_prefix", "|>", "ḍ̇'S", "éſ", "\n", "'re", "\r\n\r\n"]} +{"text": "ſ٣٤٥٦Ａ'T'll'ſ٣٤٥٦​ İ漢t", "tokens": 23, "pieces": ["ſ", "", "٣٤٥", "٦", "Ａ'T", "'ll'ſ", "٣٤٥", "٦", "​", " İ漢t"]} +{"text": "­\r'M'VE\u000b0𐞁'D\r\n\r\né😀🏽'…\u000b<Dž!!…
\"　d👍🏽ßm'ſ9½-", "tokens": 50, "pieces": ["­\r", "'M'VE", "\u000b", "0", "𐞁'D", "\r\n\r\n", "é", "😀🏽'", "…", "\u000b", "<Dž", "!!", "…", "
", "\"<", "é", "<|", "fim", "_prefix", "|>", "　d", "👍🏽", "ßm'ſ", "9½", "-"]} +{"text": "9", "tokens": 1, "pieces": ["9"]} +{"text": "(\"字Džé(ꟲ-", "tokens": 14, "pieces": ["(\"", "字Džé", "(<", "EOT", ">ꟲ", "-"]} +{"text": "\r\n­m½ \r\n \n#$%ꟲ><|endoftext|>　0d.'s漢😀🏽é", "tokens": 29, "pieces": ["\r\n", "­m", "½", " \r\n \n", "#$%", "ꟲ", "><|", "endoftext", "|>", "　", "0", "d", ".'", "s漢", "😀🏽", "é"]} +{"text": "ſEOT\r\n9\r \na", "tokens": 8, "pieces": ["ſ", "EOT", "\r\n", "9", "\r \n", "a"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "👍🏽\n'D", "tokens": 5, "pieces": ["👍🏽\n", "'D"]} +{"text": "'S'S\n
-Z
३.½'ſ'VE½ İⅣ\n\t𐞁\">\n Ａ'VEs㋿\r!!'S🙂!'DⅣ", "tokens": 45, "pieces": ["'S'S", "\n", "
", "-Z", "
", "", "३", ".", "½", "'ſ'VE", "½", " İ", "Ⅳ", "\n", "\t𐞁", "\">\n", " ", " Ａ'VE", "s", "㋿\r", "!!'", "S", "🙂!'", "D", "Ⅳ"]} +{"text": "
\r\n\r\n㍿\r\n\r\n\r\n#$%ꟲ", "tokens": 11, "pieces": ["
\r\n\r\n", "㍿\r\n\r\n\r\n", "#$%", "ꟲ"]} +{"text": "\t'Re#$%ꟲ👍🏽　'ꟲ'llé\té,'ſß", "tokens": 27, "pieces": ["\t", "'Re", "#$%", "ꟲ", "👍🏽<", "EOT", ">", "　", "'ꟲ'll", "é", "\té", ",'", "ſß"]} +{"text": "<
\r-('VEDž\r'S\ré'Ree\r\n😀🏽's\né\r\nm\t", "tokens": 28, "pieces": ["<", "
\r", "-('", "VEDž", "\r", "'S", "\r", "é'Re", "e", "\r\n", "😀🏽'", "s", "\n", "é", "\r\n", "m", "\t"]} +{"text": "<|fim_prefix|>𐞁\r\n'll\t's9fi12345678'12345678…-0\n > \n<|fim_prefix|>-é
😀🏽's'S!!🙂\n㋿‍\n0ḍ̇", "tokens": 57, "pieces": ["<|", "fim", "_prefix", "|>", "𐞁", "\r\n", "'ll", "\t", "'s", "9", "fi", "123", "456", "78", "'", "123", "456", "78", "…", "-", "0", "\n", " ", ">", " \n", "<|", "fim", "_prefix", "|>-", "é", "
", "😀🏽'", "s'S", "!!🙂\n", "㋿‍\n", "0", "ḍ̇"]} +{"text": "𐞁🙂𐞁ꟲⅣDž½a0,㍿éḍ̇½'s٣٤٥٦٣٤٥٦", "tokens": 43, "pieces": ["𐞁", "🙂𐞁ꟲ", "Ⅳ", "Dž", "½", "a", "0", ",<", "EOT", "><", "EOT", ">㍿", "éḍ̇", "½", "'s", "٣٤٥", "٦٣٤", "٥٦"]} +{"text": "''re​> \n\r\n\r\n>字s'M­fi㋿'s\r\n\r\n\r\nſ0.t", "tokens": 21, "pieces": ["''", "re", "​>", " \n\r\n\r\n", ">字s'M", "­fi", "㋿'", "s", "\r\n\r\n\r\n", "ſ", "0", ".t"]} +{"text": "9 fiZꟲＡ👍🏽'M­ع<|endoftext|>🙂#$%­ḍ̇t🙂　 字Z ½ꟲ\r'<|endoftext|>́tDž­", "tokens": 52, "pieces": ["9", " fi", "Zꟲ", "Ａ", "👍🏽'", "M", "­ع", "<|", "endoftext", "|>🙂#$%­", "ḍ̇t", "🙂", "　", " 字", "Z", " ", " ", "½", "ꟲ", "\r", "'<|", "endoftext", "|>́", "t", "Dž", "­"]} +{"text": "𐞁'S(㍿!!e'll'Tsfi㍿𐞁EOT're", "tokens": 29, "pieces": ["𐞁'S", "(㍿!!", "e", "'", "ll'T", "sfi", "㍿𐞁", "EOT're"]} +{"text": "aⅣEOT字㍿́\"​ع㍿EOT'D\r<|fim_prefix|>'Sé\r𐞁'Dعİ㍿''VEå‍'Ree0 \u000bDž", "tokens": 57, "pieces": ["a", "Ⅳ", "EOT字", "㍿́", "\"​", "ع", "㍿EOT'D", "\r", "<|", "fim", "_prefix", "|><", "META", "_START", ">'", "Sé", "\r", "𐞁'D", "ع", "İ", "㍿''", "VE", "å", "‍'", "Ree", "0", " ", "\u000bDž"]} +{"text": "å,🙂<|endoftext|>. \n​.\r\n字\r\nİ\r\n\r\n'MＡ㋿½9", "tokens": 56, "pieces": ["å", ",🙂<|", "endoftext", "|>.", " \n", "​.\r\n", "字", "\r\n", "İ", "\r\n\r\n", "'MＡ", "㋿", "½9"]} +{"text": "
<|fim_prefix|>Dž", "tokens": 9, "pieces": ["
", "<|", "fim", "_prefix", "|>", "Dž"]} +{"text": "-\n
(\r\n\r\n\r\n　fi'seß\r漢㋿s ­", "tokens": 16, "pieces": ["-\n", "
", "(\r\n\r\n\r\n", "　fi's", "eß", "\r", "漢", "㋿s", " ­"]} +{"text": "'VE٣٤٥٦👍🏽's
e😀🏽ß🙂>ß <ＡＡ'S!!㍿㋿ '‍㋿é>ḍ̇fi\r\n\r\n'ſ", "tokens": 46, "pieces": ["'VE", "٣٤٥", "٦", "👍🏽'", "s", "
e", "😀🏽", "ß", "🙂>", "ß", " <", "ＡＡ'S", "!!㍿㋿", " ", " '‍㋿", "é", ">ḍ̇fi", "\r\n\r\n", "'ſ"]} +{"text": "'D'ſ…'S9", "tokens": 7, "pieces": ["'D'ſ", "…", "'S", "9"]} +{"text": "\rm३　'S🙂漢㍿.㍿,'EOT!漢 EOTå", "tokens": 23, "pieces": ["\r", "m", "३", "　", "'S", "🙂漢", "㍿.㍿,'", "EOT", "!漢", " EOTå"]} +{"text": "­İ㍿!!🙂 ​EOT'S0㋿éßå㋿ \n", "tokens": 25, "pieces": ["­İ", "㍿!!🙂", " ", " ​", "EOT'S", "0", "㋿éßå", "㋿", " \n"]} +{"text": "s", "tokens": 1, "pieces": ["s"]} +{"text": "'llⅣ''<|endoftext|>a½\r\n\r\n m　!d!!aＡ　'Dß㋿Z<|fim_prefix|>>'red", "tokens": 41, "pieces": ["'ll", "Ⅳ", "''<|", "endoftext", "|>", "a", "½", "\r\n\r\n", " ", " m", "　", "!d", "!!", "a", "Ａ", "　", "'D", "ß", "㋿Z", "<|", "fim", "_prefix", "|>>'", "red"]} +{"text": "𐞁𐞁t\r\n-́'Te'Dſ…<|fim_prefix|>　9ḍ̇", "tokens": 29, "pieces": ["𐞁𐞁t", "\r\n", "-́'T", "e'D", "ſ", "…", "<|", "fim", "_prefix", "|>", "　", "9", "ḍ̇"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": " #$%'ſé('ll ‍­'Re'ReDža", "tokens": 16, "pieces": [" ", "#$%'", "ſé", "('", "ll", " ", " ‍­'", "Re'Re", "Dža"]} +{"text": "٣٤٥٦\u000b<|fim_prefix|>　fi㋿", "tokens": 16, "pieces": ["٣٤٥", "٦", "\u000b", "<|", "fim", "_prefix", "|>", "　fi", "㋿"]} +{"text": "ꟲعae\r\n\r\n'ſ
字(-\t!!!!½åßéＡ!​'Reſ", "tokens": 24, "pieces": ["ꟲعae", "\r\n\r\n", "'ſ", "
字", "(-", "\t", "!!!!", "½", "åßé", "Ａ", "!​'", "Reſ"]} +{"text": "-३ #$%!!👍🏽", "tokens": 10, "pieces": ["-", "३", " ", " #$%!!👍🏽"]} +{"text": "𐞁'\t- 'M.𐞁​", "tokens": 20, "pieces": ["𐞁", "'", "\t", "-", " ", " '", "M", ".𐞁", "​"]} +{"text": " å　<|fim_prefix|>EOT字d‍㍿‍-12345678're\"t0👍🏽<|endoftext|>s>\r\n\r\n9Ⅳad'VE're\"­", "tokens": 48, "pieces": [" ", " å", "　", "<|", "fim", "_prefix", "|>", "EOT字d", "‍㍿‍-", "123", "456", "78", "'re", "\"t", "0", "👍🏽<|", "endoftext", "|>", "s", ">\r\n\r\n", "9Ⅳ", "ad'VE", "'re", "\"­"]} +{"text": "ſ", "tokens": 1, "pieces": ["ſ"]} +{"text": "å٣٤٥٦Z😀🏽>'T\r\n\r\n३ß\"½éZ㋿‍ḍ̇9s'D字\n漢ſdt३e", "tokens": 40, "pieces": ["å", "٣٤٥", "٦", "Z", "😀🏽<", "EOT", ">>'", "T", "\r\n\r\n", "३", "ß", "\"", "½", "é", "Z", "㋿‍", "ḍ̇", "9", "s'D", "字", "\n", "漢ſdt", "३", "e"]} +{"text": "㍿-.'s.e　­'VE ­\u000b\r\nå", "tokens": 16, "pieces": ["㍿-.'", "s", ".e", "　", "­'", "VE", " ­", "\u000b\r\n", "å"]} +{"text": "'T'T", "tokens": 2, "pieces": ["'T'T"]} +{"text": "-.('", "tokens": 2, "pieces": ["-.('"]} +{"text": "Z'ſ'VEé'T…ß'ſ\n're　­.>𐞁́å'VE😀🏽 -d\u000bＡ'T", "tokens": 36, "pieces": ["Z'ſ", "'VEé'T", "…ß'ſ", "\n", "'re", "　", "­.>", "𐞁́å'VE", "😀🏽", " ", "-d", "\u000bＡ'T"]} +{"text": "ḍ̇३३㋿e'M'Re\r\n
\"d \u000b'D-漢 漢e\r\n\r\n\u000b㍿a㍿", "tokens": 35, "pieces": ["ḍ̇", "", "३३", "㋿e'M", "'Re", "\r\n", "
", "\"d", " ", "\u000b", "'D", "-漢", " 漢e", "\r\n\r\n", "\u000b", "㍿a", "㍿"]} +{"text": "'ll'VE'VEå👍🏽é9\r\n\r\n<漢", "tokens": 22, "pieces": ["'ll'VE", "'", "VEå", "👍🏽", "é", "", "9", "\r\n\r\n", "<漢"]} +{"text": "عⅣ'M'Md٣٤٥٦0!
're ‍<>måm३\"ع-㍿㋿'re🙂0's'Ś​m<|endoftext|>½ \n­m-🙂", "tokens": 51, "pieces": ["ع", "Ⅳ", "'M'M", "d", "٣٤٥", "٦0", "!", "
", "'re", " ‍<>", "måm", "३", "\"ع", "-㍿㋿'", "re", "🙂", "0", "'s'S", "́", "​m", "<|", "endoftext", "|>", "½", " \n", "­m", "-🙂"]} +{"text": "\u000b字'éfi'VEDžⅣ\tDžda\r\n\r\n 'VE½<|fim_prefix|><|endoftext|>🙂-'ll漢Ⅳ", "tokens": 41, "pieces": ["\u000b字", "'éfi'VE", "Dž", "Ⅳ", "\tDžda", "\r\n\r\n", " ", " <", "EOT", ">'", "VE", "½", "<|", "fim", "_prefix", "|><|", "endoftext", "|>🙂-'", "ll漢", "Ⅳ"]} +{"text": "İeꟲＡ", "tokens": 6, "pieces": ["İeꟲ", "Ａ"]} +{"text": "9'Reعd(𐞁\n㋿३a\r\n\r\nⅣ(Ａ \n'S字\"a
fi\u000b're-!", "tokens": 38, "pieces": ["9", "'Reعd", "(𐞁", "\n", "㋿", "३", "a", "\r\n\r\n", "Ⅳ", "(Ａ", " \n", "'S字", "\"a", "
fi", "\u000b", "'re", "-!<", "META", "_START", ">"]} +{"text": "Dž٣٤٥٦#$%", "tokens": 8, "pieces": ["Dž", "٣٤٥", "٦", "#$%"]} +{"text": "㍿\r\n\r\nd　e>…Ａſé<|endoftext|>\tḍ̇\r\n\r\n\r", "tokens": 27, "pieces": ["㍿\r\n\r\n", "d", "　e", ">", "…Ａſé", "<|", "endoftext", "|>", "\tḍ̇", "\r\n\r\n\r"]} +{"text": "!fi>'re'VEſ­å'Re.EOTſa 
㋿'ſ\r\n'Mß'M'Sem\r\n\r\n12345678ḍ̇ 𐞁!!­é12345678́👍🏽12345678", "tokens": 58, "pieces": ["!fi", ">'", "re'VE", "ſ", "­å'Re", ".EOTſa", " ", "
", "㋿'", "ſ", "\r\n", "'Mß", "'", "M'S", "em", "\r\n\r\n", "123", "456", "78", "ḍ̇", " ", " 𐞁", "!!­", "é", "123", "456", "78", "́", "👍🏽", "123", "456", "78"]} +{"text": "e.́عſe​\r\nt -㋿½\n'VE('re0 0 ㍿'T ßＡ𐞁!!ḍ̇<|fim_prefix|>字‍½t㋿", "tokens": 59, "pieces": ["e", ".́عſe", "​\r\n", "t", " ", " -㋿", "½", "\n", "'VE", "('", "re", "", "0", " ", " ", "0", " ", " ㍿'", "T", " ", " ß", "Ａ𐞁", "!!", "ḍ̇", "<|", "fim", "_prefix", "|>", "字", "‍", "½", "t", "㋿"]} +{"text": "#$% 's‍'ſém ſ'T0½'M㋿éZ", "tokens": 20, "pieces": ["#$%", " ", " '", "s", "‍'", "ſém", " ſ'T", "0½", "'M", "㋿é", "Z"]} +{"text": "　m٣٤٥٦ \n­'D㋿", "tokens": 16, "pieces": ["　m", "٣٤٥", "٦", " \n", "­'", "D", "㋿"]} +{"text": "d٣٤٥٦́­́३<\r\n..‍'S-.㍿́
İ<'T", "tokens": 30, "pieces": ["d", "٣٤٥", "٦", "́", "­́", "३", "<\r\n", "..<", "EOT", ">‍'", "S", "-.㍿́", "
İ", "<'", "T"]} +{"text": "'S\u000bs're>Ⅳt'VE<'Re><|endoftext|>​'Té
<|fim_prefix|>'s\r\n0½.'VE\u000b'M#$%'M​ſZeḍ̇.< !!d're", "tokens": 52, "pieces": ["'S", "\u000bs're", ">", "Ⅳ", "t'VE", "<'", "Re", "><|", "endoftext", "|>​'", "Té", "
", "<|", "fim", "_prefix", "|>'", "s", "\r\n", "0½", ".'", "VE", "\u000b", "'M", "#$%'", "M", "​ſ", "Zeḍ̇", ".<", " ", "!!", "d're"]} +{"text": "
́عé'll‍ḍ̇!!'ſ're\r'VE…‍'VEſe'VE😀🏽🙂sع", "tokens": 34, "pieces": ["
́عé'll", "‍ḍ̇", "!!'", "ſ're", "\r", "'VE", "…", "‍'", "VEſe'VE", "😀🏽🙂", "sع"]} +{"text": "!!́ fie 
12345678-\u000bé0're(", "tokens": 19, "pieces": ["!!́", " fie", " ", "
", "123", "456", "78", "-", "\u000bé", "0", "'re", "("]} +{"text": "\r\n.́'Dt<\"　٣٤٥٦'Sd'Md'll😀🏽 ḍ̇İ9\t<|fim_prefix|>ém​Ａ　漢å", "tokens": 43, "pieces": ["\r\n", ".́'D", "t", "<\"", "　", "٣٤٥", "٦", "'Sd'M", "d'll", "😀🏽", " ", " ḍ̇", "İ", "9", "\t", "<|", "fim", "_prefix", "|>", "ém", "​Ａ", "　漢å"]} +{"text": "Zfi
åZ👍🏽.'VE're…é123456780ع㋿<|endoftext|>İ'!(9'(㋿
👍🏽'Re　㋿>", "tokens": 53, "pieces": ["Zfi", "
å", "Z", "👍🏽.'", "VE're", "", "…é", "123", "456", "780", "ع", "㋿<|", "endoftext", "|>", "İ", "'!(", "9", "'(㋿", "
", "👍🏽'", "Re", "　", "㋿>"]} +{"text": "'M'llEOT'll", "tokens": 9, "pieces": ["'M", "'", "ll", "EOT'll"]} +{"text": "…éd EOTع're'DDž>㍿>İ\r\n\r\n(　<­½é-'ſع're​ ", "tokens": 30, "pieces": ["…éd", " ", " EOTع're", "'DDž", ">㍿>", "İ", "\r\n\r\n", "(", "　", "<­", "½", "é", "-'", "ſع're", "​", " "]} +{"text": "ßſßfi'St", "tokens": 6, "pieces": ["ßſßfi'S", "t"]} +{"text": "Dž0t'VEm(fi\r", "tokens": 10, "pieces": ["Dž", "0", "t'VE", "m", "(fi", "\r"]} +{"text": "<#$%'Re \n \n字\r\n\r\n…́<|fim_prefix|>12345678EOT­9Dž-'' ​m𐞁३'T'T\n", "tokens": 41, "pieces": ["<#$%'", "Re", " \n \n", "字", "\r\n\r\n", "…́", "<|", "fim", "_prefix", "|>", "123", "456", "78", "EOT", "­", "9", "Dž", "-''", " ​", "m𐞁", "३", "'T'T", "\n"]} +{"text": "<|fim_prefix|>!(EOTm👍🏽Z\"", "tokens": 18, "pieces": ["<|", "fim", "_prefix", "|>!(", "EOTm", "👍🏽", "Z", "\"<", "EOT", ">"]} +{"text": "\r\n\r\nḍ̇'M३'Dع😀🏽><|endoftext|>!>\r'Re漢'VE", "tokens": 25, "pieces": ["\r\n\r\n", "ḍ̇'M", "३", "'Dع", "😀🏽><|", "endoftext", "|>!>\r", "'Re漢'VE"]} +{"text": "३'S🙂é,​'reİ㍿ꟲa-<|endoftext|>éZ­-😀🏽\u000b", "tokens": 33, "pieces": ["३", "'S", "🙂é", ",​'", "re", "İ", "㍿ꟲa", "-<|", "endoftext", "|>", "é", "Z", "­-😀🏽", "\u000b"]} +{"text": "　'VEعＡßꟲ…ſ'M​­d\ré're!\t!#$%\n😀🏽", "tokens": 32, "pieces": ["　", "'VEعＡßꟲ", "…ſ'M", "​­", "d", "\r", "é're", "!", "\t", "!#$%<", "EOT", ">\n", "😀🏽"]} +{"text": "'ſ…'llåé'Re.…m'reſ(\"e'Ⅳéİ­'D#$%a
'S½½३́\rعＡ३ḍ̇㍿", "tokens": 54, "pieces": ["'ſ", "…", "'llåé'Re", ".", "…m're", "ſ", "(\"", "e", "'", "Ⅳ", "é", "İ", "­'", "D", "#$%", "a", "
", "'S", "½½३", "́", "\r", "ع", "Ａ", "३", "ḍ̇", "㍿"]} +{"text": ",'ſ(,'é'Re😀🏽字Ａ㍿㍿(🙂m>'ſ٣٤٥٦å12345678३‍('T㍿‍", "tokens": 49, "pieces": [",'", "ſ", "(,'", "é", "'", "Re", "😀🏽", "字", "Ａ", "㍿㍿(<", "EOT", ">🙂", "m", ">'", "ſ", "٣٤٥", "٦", "å", "123", "456", "78३", "‍('", "T", "㍿‍"]} +{"text": "<🙂٣٤٥٦ⅣEOTEOTt'ſ12345678 \n , ́\"\u000b,\r\n\r\n字'VEfi.'s#$%", "tokens": 32, "pieces": ["<🙂", "٣٤٥", "٦Ⅳ", "EOTEOTt'ſ", "123", "456", "78", " \n", " ,", " ́", "\"", "\u000b", ",\r\n\r\n", "字'VE", "fi", ".'", "s", "#$%"]} +{"text": ".ß -<|fim_prefix|>99\r\n\n👍🏽ḍ̇㋿t㋿<|fim_prefix|>'refi", "tokens": 32, "pieces": [".ß", " -<|", "fim", "_prefix", "|>", "99", "\r\n\n", "👍🏽", "ḍ̇", "㋿t", "㋿<|", "fim", "_prefix", "|>'", "refi"]} +{"text": "­'s12345678.İ‍fit'D(Ａé\t'T9…a३İ", "tokens": 24, "pieces": ["­'", "s", "123", "456", "78", ".İ", "‍fit'D", "(Ａé", "\t", "'T", "9", "…a", "३", "İ"]} +{"text": "​'Sſ漢t e,.'S ½ \nsſé'M㋿'llſ'Dž\t　\"🙂tZḍ̇…", "tokens": 42, "pieces": ["​'", "Sſ", "漢t", " e", ",.'", "S", " ", " ", "½", " \n", "sſé'M", "㋿'", "llſ", "'Dž", "\t", "　", "\"🙂", "t", "Zḍ̇", "…"]} +{"text": ",9\tḍ̇", "tokens": 6, "pieces": [",", "9", "\tḍ̇"]} +{"text": "٣٤٥٦عßé\r\n…,ḍ̇\n㍿'T>　<|fim_prefix|>ع<|endoftext|>s<", "tokens": 45, "pieces": ["'Dḍ̇", "\t字", "\n", "s'M", "a漢", "　", "㋿'", "re", "\t", ",🙂", "0", ".", "
é", ">", "　", "<|", "fim", "_prefix", "|>", "ع", "<|", "endoftext", "|>", "s", "<"]} +{"text": "!‍'D9Dž'llḍ̇​Ａ㋿३\r३té'llå'D'Re…­३0漢<|fim_prefix|>9(­½<|fim_prefix|>́ ́t", "tokens": 52, "pieces": ["!‍'", "D", "9", "Dž'll", "ḍ̇", "​Ａ", "㋿", "३", "\r", "३", "té'll", "å'D", "'Re", "…", "­", "३0", "漢", "<|", "fim", "_prefix", "|>", "9", "(­", "½", "<|", "fim", "_prefix", "|>́", " ́t"]} +{"text": "12345678<|fim_prefix|>\r\nå👍🏽#$%é漢12345678Ⅳ٣٤٥٦t🙂ع𐞁'sİ٣٤٥٦…漢👍🏽ḍ̇EOT're", "tokens": 52, "pieces": ["123", "456", "78", "<|", "fim", "_prefix", "|>\r\n", "å", "👍🏽#$%", "é漢", "123", "456", "78Ⅳ", "٣٤٥", "٦", "t", "🙂ع𐞁's", "İ", "٣٤٥", "٦", "…漢", "👍🏽", "ḍ̇", "EOT're"]} +{"text": "ßعs'㍿\"\r\n\r\n字👍🏽0afi'ſ12345678tİ<9　😀🏽\r\n\r\n-", "tokens": 32, "pieces": ["ßعs", "'㍿\"\r\n\r\n", "字", "👍🏽", "0", "afi'ſ", "123", "456", "78", "t", "İ", "<", "9", "　", "😀🏽\r\n\r\n", "-"]} +{"text": "'M字t12345678…#$%­,\n \nDž9字m\r\n", "tokens": 19, "pieces": ["'M字t", "123", "456", "78", "…", "#$%­,\n", " \n", "Dž", "9", "字m", "\r\n"]} +{"text": "字's👍🏽\u000b\r\n\r\nع<\u000b३ \nDž>㋿åßfi \n", "tokens": 49, "pieces": ["字's", "👍🏽", "\u000b\r\n\r\n", "ع", "<", "\u000b", "", "३", " \n", "Dž", ">㋿", "åßfi", " \n"]} +{"text": "'D'll fi👍🏽EOT‍s👍🏽'VEEOT a😀🏽\r!!
mZ\r\n(­😀🏽ḍ̇EOT\r\n \n", "tokens": 50, "pieces": ["'D'll", "", " fi", "👍🏽", "EOT", "‍s", "👍🏽'", "VEEOT", " ", " <", "EOT", ">a", "😀🏽\r", "!!", "
m", "Z", "\r\n", "(­<", "META", "_START", ">😀🏽", "ḍ̇", "EOT", "\r\n \n"]} +{"text": "ßꟲ‍­<|fim_prefix|>12345678\r\n\r\n", "tokens": 16, "pieces": ["ßꟲ", "‍­<|", "fim", "_prefix", "|>", "123", "456", "78", "\r\n\r\n"]} +{"text": "́", "tokens": 1, "pieces": ["́"]} +{"text": "…é­<|endoftext|>d字,\nع'M9ḍ̇'Re<'sd", "tokens": 26, "pieces": ["…é", "­<|", "endoftext", "|>", "d字", ",\n", "ع'M", "9", "ḍ̇'Re", "<'", "sd"]} +{"text": "\r\n\r\n", "tokens": 1, "pieces": ["\r\n\r\n"]} +{"text": "🙂 12345678Z𐞁‍'ſZ٣٤٥٦'re😀🏽👍🏽#$%>'ſ90‍12345678m's", "tokens": 39, "pieces": ["🙂", " <", "EOT", ">", "123", "456", "78", "Z𐞁", "‍'", "ſ", "Z", "٣٤٥", "٦", "'re", "😀🏽👍🏽#$%>'", "ſ", "90", "‍", "123", "456", "78", "m's"]} +{"text": "!fi.''D​'S​aé🙂'ſ🙂‍eé'Re½!!\u000b!!'T㍿t", "tokens": 36, "pieces": ["!fi", ".<", "EOT", ">''", "D", "​'", "S", "​aé", "🙂'", "ſ", "🙂‍", "eé'Re", "½", "!!<", "META", "_START", ">", "\u000b", "!!'", "T", "㍿t"]} +{"text": "'S'", "tokens": 6, "pieces": ["'", "S", "'"]} +{"text": "\"ſİ#$%½-½ 𐞁'Re'ſ　e-\r\n'M'll
e👍🏽a'ſ", "tokens": 29, "pieces": ["\"ſ", "İ", "#$%", "½", "-", "½", " 𐞁'Re", "'ſ", "　e", "-\r\n", "'M'll", "
e", "👍🏽", "a'ſ"]} +{"text": "👍🏽'VE\"fi-t's'll漢ße'Tåefis\r\r\n're
ع!å㍿å𐞁 \né😀🏽9Ａ(é'T㋿\"-", "tokens": 48, "pieces": ["👍🏽'", "VE", "\"fi", "-t's", "'ll漢ße'T", "åefis", "\r\r\n", "'re", "
ع", "!å", "㍿å𐞁", " \n", "é", "😀🏽", "9", "Ａ", "(é'T", "㋿\"-"]} +{"text": "𐞁​㍿", "tokens": 8, "pieces": ["𐞁", "​㍿"]} +{"text": "ꟲEOT­<|endoftext|>m​ſ \n👍🏽<|fim_prefix|>EOT>#$%t \n'llfi😀🏽Ze,", "tokens": 46, "pieces": ["ꟲ", "EOT", "­<", "META", "_START", "><|", "endoftext", "|>", "m", "​ſ", " \n", "👍🏽<|", "fim", "_prefix", "|>", "EOT", "><", "EOT", ">#$%", "t", " \n", "'llfi", "😀🏽", "Ze", ","]} +{"text": "a\r\n\r\n‍ s'ſ­é \n!!t字0\t,'S", "tokens": 17, "pieces": ["a", "\r\n\r\n", "‍", " s'ſ", "­é", " \n", "!!", "t字", "0", "\t", ",'", "S"]} +{"text": "'M!<٣٤٥٦\nDž'ſ-12345678👍🏽'Sß字Z३\r\né.'T#$%-'t𐞁​㍿<é\u000bⅣ​", "tokens": 48, "pieces": ["'M", "!<", "٣٤٥", "٦", "\n", "Dž'ſ", "-", "123", "456", "78", "👍🏽'", "Sß字", "Z", "३", "\r\n", "é", ".'", "T", "#$%-'", "t𐞁", "​㍿<", "é", "\u000b", "Ⅳ", "​"]} +{"text": "​ 'll\r\n're", "tokens": 9, "pieces": ["​", " ", " '", "ll", "\r\n", "'re"]} +{"text": " \"e'T漢<|endoftext|>Dž(…'VE㋿ſİ\r\n\r\nＡ'reḍ̇'Re.", "tokens": 49, "pieces": [" ", "\"e'T", "漢", "<|", "endoftext", "|>", "Dž", "(", "…", "'VE", "㋿ſ", "İ", "\r\n\r\n", "Ａ", "'", "reḍ̇'Re", "."]} +{"text": "३٣٤٥٦½", "tokens": 6, "pieces": ["३٣٤", "٥٦½"]} +{"text": "ß'Re\"fi ३İ \n<<|endoftext|>12345678🙂'ſ12345678
٣٤٥٦", "tokens": 33, "pieces": ["ß'Re", "\"fi", " ", " <", "META", "_START", ">", "३", "İ", " \n", "<<|", "endoftext", "|>", "123", "456", "78", "🙂'", "ſ", "123", "456", "78", "
", "٣٤٥", "٦"]} +{"text": "<|fim_prefix|>'s\t ḍ̇'re🙂　ß漢's\r\n\r\nZ9'Resḍ̇\n'T\"漢\"Ⅳ", "tokens": 38, "pieces": ["<|", "fim", "_prefix", "|>'", "s", "\t ", " ḍ̇'re", "🙂", "　ß漢's", "\r\n\r\n", "Z", "9", "'Resḍ̇", "\n", "'T", "\"漢", "\"", "Ⅳ"]} +{"text": "‍‍㍿\u000b<|fim_prefix|> 𐞁t-​'aＡ>ḍ̇　'llß٣٤٥٦da'e㍿ꟲſ", " 𐞁t", "-​'", "a", "Ａ", ">ḍ̇", "　", "'llß", "٣٤٥", "٦", "da", "'e", "㍿ꟲſ", "<|fim_prefix|>'Refié𐞁½\u000b३ 'Dſḍ̇12345678 
", "tokens": 31, "pieces": ["‍😀🏽><|", "fim", "_prefix", "|>'", "Refié𐞁", "½", "\u000b", "३", " '", "Dſḍ̇", "123", "456", "78", " 
"]} +{"text": "\t'T　İ​Z😀🏽", "tokens": 9, "pieces": ["\t", "'T", "　İ", "​Z", "😀🏽"]} +{"text": "'D!!Ⅳ​s're ­😀🏽'Re٣٤٥٦́ ㋿٣٤٥٦٣٤٥٦ Dž'M<|endoftext|>
", "tokens": 50, "pieces": ["'D", "!!", "Ⅳ", "​<", "EOT", ">s're", " ", " ­😀🏽'", "Re", "٣٤٥", "٦", "́", " ㋿", "٣٤٥", "٦٣٤", "٥٦", " Dž'M", "<|", "endoftext", "|>", "
"]} +{"text": "ßåéḍ̇ꟲfi👍🏽'll\"", "tokens": 24, "pieces": ["ßåé", "<", "EOT", ">ḍ̇ꟲfi", "👍🏽'", "ll", "\""]} +{"text": "\"t'reع漢́٣٤٥٦Ⅳ\t٣٤٥٦<́s'Sع​m\tⅣ.\r'Déé", "tokens": 31, "pieces": ["\"t're", "ع漢́", "٣٤٥", "٦Ⅳ", "\t", "٣٤٥", "٦", "<́s'S", "ع", "​m", "\t", "Ⅳ", ".\r", "'Déé"]} +{"text": "('M\r字…-३ \u000b…
é<|fim_prefix|>'M.'Re", "tokens": 23, "pieces": ["('", "M", "\r", "字", "…", "-", "३", " \u000b…", "
é", "<|", "fim", "_prefix", "|>'", "M", ".'", "Re"]} +{"text": "'VE.<|endoftext|>fi字\t​\r\n\r\n字​ꟲ\r Dž́𐞁'D12345678٣٤٥٦'T<\r\n'ſ12345678👍🏽漢
𐞁'll­12345678 ع½😀🏽#$%　\r\n\r\n", "tokens": 69, "pieces": ["'VE", ".<|", "endoftext", "|>", "fi字", "\t", "​\r\n\r\n", "字", "​ꟲ", "\r", " ", " Dž́𐞁'D", "123", "456", "78٣", "٤٥٦", "'T", "<\r\n", "'ſ", "123", "456", "78", "👍🏽", "漢", "
𐞁'll", "­", "123", "456", "78", " ", " ع", "½", "😀🏽#$%", "　\r\n\r\n"]} +{"text": "漢!!'Re !!", "tokens": 6, "pieces": ["漢", "!!'", "Re", " ", " !!"]} +{"text": "-EOT", "tokens": 5, "pieces": ["-", "EOT"]} +{"text": "­
ꟲfi㋿٣٤٥٦👍🏽(३'M\"'T('re😀🏽>Ａfi12345678", "tokens": 36, "pieces": ["­", "
ꟲfi", "㋿", "٣٤٥", "٦", "👍🏽(", "३", "'M", "\"'", "T", "('", "re", "😀🏽>", "Ａfi", "", "123", "456", "78"]} +{"text": "㍿'s'ſ字\u000b'Re😀🏽,'T́­ 𐞁
!!fi　,漢漢३\t३ß́\rⅣ\r'ſ'ſå\r\r\n\r\n👍🏽", "tokens": 49, "pieces": ["㍿'", "s'ſ", "字", "\u000b", "'Re", "😀🏽,'", "T́", "­", " 𐞁", "
", "!!", "fi", "　", ",漢漢", "३", "\t", "३", "ß́", "\r", "Ⅳ", "\r", "'ſ'ſ", "å", "\r\r\n\r\n", "👍🏽"]} +{"text": "\t12345678🙂İ­Dža­< Ａ", "tokens": 14, "pieces": ["\t", "123", "456", "78", "🙂İ", "­Dža", "­<", " Ａ"]} +{"text": "a<|fim_prefix|>\r\n\r\n\r\n\r\n
­👍🏽 m'så'VE", "tokens": 19, "pieces": ["a", "<|", "fim", "_prefix", "|>\r\n\r\n\r\n\r\n", "
", "­👍🏽", " m's", "å'VE"]} +{"text": "　\r\n\r\n😀🏽‍'VE-İ\naＡ<字<|endoftext|>#$%a🙂é>ع!! ꟲ​dſ½'T", "tokens": 47, "pieces": ["　\r\n\r\n", "😀🏽‍'", "VE", "-İ", "\n", "a", "Ａ", "<字", "<|", "endoftext", "|>#$%", "a", "🙂é", ">ع", "!!<", "EOT", ">", " ꟲ", "​dſ", "½", "'T"]} +{"text": "'D३a漢𐞁🙂12345678<|fim_prefix|>>㍿'VE٣٤٥٦ EOT'T\t🙂e字(", "tokens": 39, "pieces": ["'D", "३", "a漢𐞁", "🙂", "123", "456", "78", "<|", "fim", "_prefix", "|>>㍿<", "META", "_START", ">'", "VE", "٣٤٥", "٦", " EOT'T", "\t", "🙂e字", "("]} +{"text": "́　 \r\nfi", "tokens": 5, "pieces": ["́", "　 \r\n", "fi"]} +{"text": "'Sİt­.'D's𐞁\r\n-'ll'll.İ́\r\n\r\n'\n字éåt \nß", "tokens": 30, "pieces": ["'Sİt", "­.'", "D's", "𐞁", "\r\n", "-'", "ll'll", ".<", "EOT", ">İ́", "\r\n\r\n", "'\n", "字éåt", " \n", "ß"]} +{"text": "🙂t\"ꟲ\r\nꟲİ<|endoftext|> ḍ̇é 'T​𐞁漢t　ſع12345678fiع", "tokens": 40, "pieces": ["🙂t", "\"ꟲ", "\r\n", "ꟲ", "İ", "<|", "endoftext", "|>", " ḍ̇é", " ", "'T", "​𐞁漢t", "　ſع", "123", "456", "78", "fiع"]} +{"text": "'Re'D'VEDž.!!İꟲ", "tokens": 12, "pieces": ["'Re'D", "'VEDž", ".!!", "İꟲ"]} +{"text": "fiDž­s㍿'
㋿'reſé👍🏽漢𐞁<|fim_prefix|>ḍ̇tDž'sa", "tokens": 42, "pieces": ["fi", "Dž", "­s", "㍿'", "
", "㋿'", "reſé", "👍🏽<", "META", "_START", ">漢𐞁", "<|", "fim", "_prefix", "|>", "ḍ̇t", "Dž's", "a"]} +{"text": "éꟲ漢٣٤٥٦ 'M\tfi\u000b're字", "tokens": 17, "pieces": ["éꟲ漢", "٣٤٥", "٦", " ", " '", "M", "\tfi", "\u000b", "'re字"]} +{"text": " …ſe'9漢­字Ⅳ😀🏽㋿\u000b\r\n\r\nd'D🙂…　d#$%'Reḍ̇e㍿Zt…\u000bİ>e\r­a\r", "tokens": 52, "pieces": [" ", "…ſe", "'", "9", "漢", "­字", "", "Ⅳ", "😀🏽㋿", "\u000b\r\n\r\n", "d'D", "🙂", "…", "　d", "#$%'", "Reḍ̇e", "㍿Zt", "…", "\u000bİ", ">e", "\r", "­a", "\r"]} +{"text": ">,<|endoftext|>İ漢 \r\n\r\ns字!ſ0漢Z\u000bEOT'VE
Ａ👍🏽,", "tokens": 29, "pieces": [">,<|", "endoftext", "|>", "İ漢", " \r\n\r\n", "s字", "!ſ", "0", "漢", "Z", "\u000bEOT'VE", "
Ａ", "👍🏽,"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'TdⅣfi­t\t-­'s<\"'VE'M　'Re'VE漢ſ
'D
‍㍿9#$%ſ'd
\r", "tokens": 42, "pieces": ["'Td", "Ⅳ", "fi", "­t", "\t", "-­'", "s", "<<", "META", "_START", ">\"'", "VE'M", "　", "'Re'VE", "漢ſ", "
", "'D", "
", "‍㍿", "9", "#$%", "ſ'd", "
\r"]} +{"text": "eḍ̇ ,'M0tEOT \n'T\"👍🏽t漢
!!㍿\r\nſİ😀🏽<|endoftext|>'reZ's", "tokens": 39, "pieces": ["eḍ̇", " ,'", "M", "0", "t", "EOT", " \n", "'T", "\"👍🏽", "t漢", "
", "!!㍿\r\n", "ſ", "İ", "😀🏽<|", "endoftext", "|>'", "re", "Z's"]} +{"text": "𐞁'M'ſZ'T'Då\u000bßfi'VE'S\n\n漢'S\r\nſ'SEOT\" Ⅳ", "tokens": 33, "pieces": ["𐞁'M", "'ſ", "Z'T", "'Då", "\u000bßfi'VE", "'", "S", "\n\n", "漢'S", "\r\n", "ſ'S", "EOT", "\"", " ", "Ⅳ"]} +{"text": "😀🏽  -0'M \n㋿9", "tokens": 13, "pieces": ["😀🏽", " ", " ", "-", "0", "'M", " \n", "㋿", "9"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "½'Da'D-<|fim_prefix|>'re'D​m'å३\r\n\r\n३Ａfi(, e.'re'Reḍ̇'T", "tokens": 37, "pieces": ["½", "'Da'D", "-<|", "fim", "_prefix", "|>'", "re'D", "​m", "'å", "३", "\r\n\r\n", "३", "Ａfi", "(,", " ", " e", ".'", "re'Re", "ḍ̇", "'", "T"]} +{"text": "\rm‍㍿0('Séع<|endoftext|>­ ", "tokens": 20, "pieces": ["\r", "m", "‍㍿", "0", "('", "Séع", "<|", "endoftext", "|>­", " "]} +{"text": "'ſaEOT \n\t🙂é漢're​Dž🙂㍿(dm㍿éfiEOT..́…,(0.👍🏽…İ­!㋿", "tokens": 46, "pieces": ["'ſa", "EOT", " \n", "\t", "🙂é漢're", "​Dž", "🙂㍿(", "dm", "㍿éfi", "EOT", "..́", "…", ",(", "0", ".👍🏽", "…İ", "­!㋿"]} +{"text": ",'VEéDž", "tokens": 6, "pieces": [",'", "VEé", "Dž"]} +{"text": "'llＡ́㋿12345678 é𐞁Ａa9𐞁fi㋿ḍ̇'ſſ'D\nḍ̇ß́>('VE'VE,㋿", "tokens": 48, "pieces": ["'ll", "Ａ́", "㋿", "123", "456", "78", " ", " é𐞁", "Ａa", "9", "𐞁fi", "㋿ḍ̇'ſ", "ſ'D", "\n", "ḍ̇ß́", ">('", "VE'VE", ",㋿"]} +{"text": "'ll-ḍ̇ \r'ſd!! ३,𐞁½ſ　éḍ̇sḍ̇é'ſ!\t<|fim_prefix|>\t­
字😀🏽ß<,'re…\u000b.Ａ𐞁'M", "tokens": 62, "pieces": ["'ll", "-ḍ̇", " \r", "'ſd", "!!", " ", "३", ",𐞁", "½", "ſ", "　éḍ̇sḍ̇é'ſ", "!", "\t", "<|", "fim", "_prefix", "|>", "\t", "­", "
字", "😀🏽", "ß", "<,'", "re", "…", "\u000b", ".Ａ𐞁'M"]} +{"text": "🙂s\n\r­'re EOT!漢🙂漢'reß'D<.'ſ　Ⅳ'T9\n's(m're12345678>Ａ<|endoftext|>#$%", "tokens": 46, "pieces": ["🙂s", "\n\r", "­'", "re", " EOT", "!漢", "🙂<", "EOT", ">漢're", "ß'D", "<.'", "ſ", "　", "Ⅳ", "'T", "9", "\n", "'s", "(m're", "123", "456", "78", ">Ａ", "<|", "endoftext", "|>#$%"]} +{"text": "fiİ​ſ'VEſ0-0 \n, 9<|fim_prefix|>漢½", "tokens": 25, "pieces": ["fi", "İ", "​ſ'VE", "ſ", "0", "-", "0", " \n", ",", " ", "9", "<|", "fim", "_prefix", "|>", "漢", "½"]} +{"text": "\r\n-\rs३\t́'T#$%\r\ne​½'ll<|fim_prefix|>", "tokens": 24, "pieces": ["\r\n", "-\r", "s", "३", "\t́'T", "#$%<", "META", "_START", ">\r\n", "e", "​", "½", "'ll", "<|", "fim", "_prefix", "|>"]} +{"text": "漢\"9'ſ're", "tokens": 6, "pieces": ["漢", "\"", "9", "'ſ're"]} +{"text": "(Zs\u000b 
.<'VE\r\n\r\n㍿🙂㍿‍\t(🙂½\r're
 😀🏽🙂!Z \n😀🏽", "tokens": 39, "pieces": ["(Zs", "\u000b ", "
", ".<'", "VE", "\r\n\r\n", "㍿🙂㍿‍", "\t", "(🙂", "½", "\r", "'re", "
 ", " 😀🏽🙂!", "Z", "", " \n", "😀🏽"]} +{"text": "é<|fim_prefix|>d", "tokens": 9, "pieces": ["é", "<|", "fim", "_prefix", "|>", "d"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "12345678Ａ🙂𐞁👍🏽.<|fim_prefix|>\u000b's\u000b!!!'\n\rEOTdꟲsta'S9fi<Ⅳ\u000b ع\r\n\r\n½ 'S", "tokens": 44, "pieces": ["123", "456", "78", "Ａ", "🙂𐞁", "👍🏽.<|", "fim", "_prefix", "|>", "\u000b", "'s", "\u000b", "!!!'\n\r", "EOTdꟲsta'S", "9", "fi", "<", "Ⅳ", "\u000b ", " ع", "\r\n\r\n", "½", " ", "'S"]} +{"text": "(­ꟲ'llZ😀🏽́½㋿< \nå\rßå \n", "tokens": 25, "pieces": ["(­", "ꟲ'll", "Z", "😀🏽́", "½", "㋿<", " \n", "å", "\r", "ßå", " \n"]} +{"text": "!!#$%Ⅳḍ̇漢'llfißeDž'D　‍㍿.(字<|fim_prefix|>ee👍🏽12345678eDžé'T!'VE㋿ m", "tokens": 51, "pieces": ["!!#$%", "Ⅳ", "ḍ̇漢'll", "fiße", "Dž'D", "　", "‍㍿.(", "字", "<|", "fim", "_prefix", "|>", "ee", "👍🏽", "123", "456", "78", "e", "Džé'T", "!'", "VE", "㋿", " ", " m"]} +{"text": "\r'😀🏽'Re　 İ'ſ\"🙂　𐞁\r\n\t#$%　­.a!\t　 \n\u000bſ", "tokens": 33, "pieces": ["\r", "'😀🏽'", "Re", "　", " İ'ſ", "\"🙂", "　𐞁", "\r\n", "\t", "#$%", "　", "­.", "a", "!", "\t　 \n", "\u000bſ"]} +{"text": "ßعe're9'sßſ漢'VE'VE½\"👍🏽'ReZ😀🏽'VE12345678ḍ̇👍🏽
㋿Z…\t (\r\n'reⅣ'T'12345678'S
", "tokens": 55, "pieces": ["ßعe're", "9", "'sßſ漢'VE", "'VE", "½", "\"👍🏽'", "Re", "Z", "😀🏽'", "VE", "123", "456", "78", "ḍ̇", "👍🏽", "
", "㋿Z", "…\t", " ", "(\r\n", "'re", "Ⅳ", "'T", "'", "123", "456", "78", "'S", "
"]} +{"text": "ḍ̇
ꟲ'T'VE
'ſ.Dž\r\n'…𐞁 !'D\r\n\r\n'Mt字 ſ​🙂,­s\r\n\r\nꟲ'Mt(㋿½३", "tokens": 53, "pieces": ["ḍ̇", "
ꟲ'T", "'VE", "
", "'ſ", ".", "Dž", "\r\n", "'", "…𐞁", " ", "!'", "D", "\r\n\r\n", "'Mt字", " ſ", "​🙂,­", "s", "\r\n\r\n", "ꟲ'M", "t", "(㋿", "½३"]} +{"text": "åＡⅣ🙂ßfi'", "tokens": 9, "pieces": ["å", "Ａ", "Ⅳ", "🙂ßfi", "'"]} +{"text": "'s(\nd👍🏽'll Ａ(\n.", "tokens": 12, "pieces": ["'s", "(\n", "d", "👍🏽'", "ll", " Ａ", "(\n", "."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "#$%😀🏽𐞁Ａ\r\n\"'re0‍s㍿­㋿'s
0½­fi\r\n>́", "tokens": 36, "pieces": ["#$%😀🏽", "𐞁", "Ａ", "\r\n", "\"'", "re", "0", "‍s", "㍿­㋿'", "s", "
", "0½", "­fi", "\r\n", ">́"]} +{"text": "'se'M!,‍eع\r\n,9", "tokens": 10, "pieces": ["'se'M", "!,‍", "eع", "\r\n", ",", "9"]} +{"text": "<|fim_prefix|>,EOT<…-dfie'reZå 912345678>　'Tfi'D​٣٤٥٦!!🙂EOT\r\n\r\n'ſfi­\r٣٤٥٦‍'re㍿", "tokens": 52, "pieces": ["<|", "fim", "_prefix", "|>,", "EOT", "<", "…", "-dfie're", "Zå", " ", " ", "912", "345", "678", ">", "　", "'Tfi'D", "​", "٣٤٥", "٦", "!!🙂", "EOT", "\r\n\r\n", "'ſfi", "­\r", "٣٤٥", "٦", "‍'", "re", "㍿"]} +{"text": "ع!'Dß", "tokens": 8, "pieces": ["ع", "!'", "Dß", ""]} +{"text": "漢tZ<'ReDž-㍿('Re0'M\u000b-fi½\r\n!!Z", "tokens": 22, "pieces": ["漢t", "Z", "<'", "Re", "Dž", "-㍿('", "Re", "0", "'M", "\u000b", "-fi", "½", "\r\n", "!!", "Z"]} +{"text": "'s'ſs,　-\r'séZDž<|fim_prefix|> \ns३a
👍🏽½!👍🏽's
!!\r㋿tİꟲ \nsåꟲ\u000b", "tokens": 52, "pieces": ["'s'ſ", "s", ",", "　", "-\r", "'sé", "ZDž", "<|", "fim", "_prefix", "|>", " \n", "s", "३", "a", "
", "👍🏽", "½", "!👍🏽'", "s", "
", "!!\r", "㋿t", "İꟲ", " \n", "såꟲ", "\u000b"]} +{"text": "t‍!!fi 'VE", "tokens": 7, "pieces": ["t", "‍!!", "fi", " ", "'VE"]} +{"text": "㋿ \"'s​EOT#$%\ne", "tokens": 30, "pieces": ["'reعfi", ".­", "eعé", "\tDžＡ're", "Dž'S", "e", "!<", "EOT", ">'", "s", "​EOT", "#$%\n", "e"]} +{"text": "½Dž \n½,e'D٣٤٥٦𐞁\r\n\r\ne
<|endoftext|>\u000bſ 0<('Tfi", "tokens": 52, "pieces": ["½", "Dž", " \n", "½", ",e'D", "٣٤٥", "٦", "𐞁", "\r\n\r\n", "e", "
", "<|", "endoftext", "|>", "\u000bſ", " ", "0", "<('", "Tfi"]} +{"text": "​Ａ　​٣٤٥٦́a're'VE‍ ㍿å0'VÉ9're!!'D'S0\"!½'Re", "tokens": 35, "pieces": ["​Ａ", "　", "​", "٣٤٥", "٦", "́a're", "'VE", "‍", " ", " ㍿", "å", "0", "'VÉ", "9", "'re", "!!'", "D'S", "0", "\"!", "½", "'Re"]} +{"text": "'S½漢́ \r\n'S.,ḍ̇\"'SZ t<|fim_prefix|>😀🏽e'T㋿dé㋿Ⅳ9'ſ字fi!", "tokens": 45, "pieces": ["'S", "½", "漢́", " \r\n", "'S", ".,", "ḍ̇", "\"'", "SZ", " ", " t", "<|", "fim", "_prefix", "|>😀🏽", "e'T", "㋿dé", "㋿", "Ⅳ9", "'", "ſ字fi", "!"]} +{"text": "🙂fi‍',
é​ \n12345678𐞁…'VEＡ'Re", "tokens": 22, "pieces": ["🙂fi", "‍',", "
é", "​", " \n", "123", "456", "78", "𐞁", "…", "'VEＡ'Re"]} +{"text": "'Re漢fi\"12345678'ſ'ſ<|endoftext|>㍿𐞁३字9\u000b\" ſ漢m'VEİ<|endoftext|>,'llé 9DžDž٣٤٥٦\n", "tokens": 60, "pieces": ["'Re漢fi", "\"", "123", "456", "78", "'ſ'ſ", "<|", "endoftext", "|>㍿", "𐞁", "३", "字", "9", "\u000b", "\"", " ſ漢m'VE", "İ", "<|", "endoftext", "|>,'", "llé", " ", "9", "DžDž", "٣٤٥", "٦", "\n"]} +{"text": "عꟲ<|endoftext|>é\r\nİDž'Re​ꟲ>-漢字ع́!!ß'ree \n'👍🏽­İß𐞁٣٤٥٦", "tokens": 46, "pieces": ["عꟲ", "<|", "endoftext", "|>", "é", "\r\n", "İDž'Re", "​ꟲ", ">-", "漢字ع́", "!!", "ß're", "e", " \n", "'👍🏽­", "İß𐞁", "٣٤٥", "٦"]} +{"text": "…́㍿'Re 'llt ३ \nmZ''re'ſ", "tokens": 22, "pieces": ["…́", "㍿'", "Re", " '", "llt", " ", "३", " \n", "m", "Z", "'<", "EOT", ">'", "re'ſ"]} +{"text": " ㍿Dž <|fim_prefix|>m's٣٤٥٦-éDž ع‍fi\"٣٤٥٦'Re'𐞁𐞁,.\r\n'Re漢", "tokens": 43, "pieces": [" ", "㍿Dž", " <|", "fim", "_prefix", "|>", "m's", "٣٤٥", "٦", "-é", "Dž", " ع", "‍fi", "\"", "٣٤٥", "٦", "'Re", "'𐞁𐞁", ",.\r\n", "'Re漢"]} +{"text": "'ReEOT'llꟲDžé0\tfi\"🙂'S'ſ'DZés'll' tḍ̇.", "tokens": 33, "pieces": ["'Re", "EOT'll", "ꟲDžé", "0", "\tfi", "\"🙂<", "EOT", ">'", "S'ſ", "'DZés'll", "'", " tḍ̇", "."]} +{"text": "\u000b<-<|fim_prefix|>#$%ſ'D'ſEOTＡḍ̇
12345678!'Re'ſİ'VE're's'll'T0'S'Re\r\n<'re,>٣٤٥٦😀🏽½½\"\t…ع", "tokens": 57, "pieces": ["\u000b", "<-<|", "fim", "_prefix", "|>#$%", "ſ'D", "'ſ", "EOTＡḍ̇", "
", "123", "456", "78", "!'", "Re'ſ", "İ'VE", "'re's", "'ll'T", "0", "'S'Re", "\r\n", "<'", "re", ",>", "٣٤٥", "٦", "😀🏽", "½½", "\"", "\t", "…ع"]} +{"text": "9 9İ३ſ\n'ree<|fim_prefix|>'llḍ̇…𐞁9m>ع'D!!\nse‍​å12345678漢'ſ
.Ⅳ", "tokens": 46, "pieces": ["9", " ", "9", "İ", "३", "ſ", "\n", "'ree", "<|", "fim", "_prefix", "|>'", "llḍ̇", "…𐞁", "9", "m", ">ع'D", "!!\n", "se", "‍​", "å", "123", "456", "78", "漢'ſ", "
", ".", "Ⅳ"]} +{"text": "\n99İ​a'Re漢　éſ\r\n\r\nDž", "tokens": 13, "pieces": ["\n", "99", "İ", "​a'Re", "漢", "　éſ", "\r\n\r\n", "Dž"]} +{"text": "́\"\r\n\r\n'VE.́Z٣٤٥٦😀🏽a<|endoftext|>\"'ſfié 　'S\r\n\r\n's12345678'T‍eDžéع", "tokens": 47, "pieces": ["́", "\"\r\n\r\n", "'VE", ".́", "Z", "٣٤٥", "٦", "😀🏽", "a", "<|", "endoftext", "|>\"'", "ſfié", " ", " <", "META", "_START", ">", " ", "　", "'S", "\r\n\r\n", "'s", "123", "456", "78", "'T", "‍e", "Džéع"]} +{"text": "٣٤٥٦<|fim_prefix|>''s㍿Ⅳ́  \n३ſ🙂\r\n\r\n\n㍿'s\r\nå", "tokens": 33, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>''", "s", "㍿", "Ⅳ", "́", "  \n", "३", "ſ", "🙂\r\n\r\n\n", "㍿'", "s", "\r\n", "å"]} +{"text": "!३ \n", "tokens": 3, "pieces": ["!", "३", " \n"]} +{"text": "𐞁'Z㋿㋿Ⅳ‍!'Re 　", "tokens": 18, "pieces": ["𐞁", "'Z", "㋿㋿", "Ⅳ", "‍!'", "Re", " 　"]} +{"text": "-
", "tokens": 2, "pieces": ["-", "
"]} +{"text": "‍­ꟲ,<㋿'s🙂", "tokens": 12, "pieces": ["‍­", "ꟲ", ",<㋿'", "s", "🙂"]} +{"text": "\rİⅣ漢0\"​éétḍ̇\"İ\u000b!!'reⅣ　12345678\t㋿0<|endoftext|>12345678…́\r\n\u000bſ\"!éⅣ-", "tokens": 53, "pieces": ["\r", "İ", "Ⅳ", "漢", "0", "\"​", "éétḍ̇", "\"İ", "\u000b", "!!'", "re", "Ⅳ", "　", "123", "456", "78", "\t", "㋿", "0", "<|", "endoftext", "|>", "123", "456", "78", "…́", "\r\n", "\u000bſ", "\"!", "é", "Ⅳ", "-"]} +{"text": "a<|fim_prefix|>", "tokens": 7, "pieces": ["a", "<|", "fim", "_prefix", "|>"]} +{"text": "é>Ź👍🏽­'T‍ …ß<İ'st\r\né­㋿\r\n😀🏽Ⅳfie\u000b", "tokens": 38, "pieces": ["é", ">Ź", "👍🏽­'", "T", "‍", " ", "…ß", "<İ's", "t", "\r\n", "é", "­㋿\r\n", "😀🏽", "Ⅳ", "fie", "\u000b"]} +{"text": "s\"å!Ⅳİe\r\n\r\n𐞁9\r㋿½\n'M-é ­\u000b#$%", "tokens": 43, "pieces": ["s", "\"å", "!", "Ⅳ", "İe", "\r\n\r\n", "𐞁", "9", "\r", "㋿", "½", "\n", "'M", "-é", " ", " <", "s漢'S", " ", "'s", "­", " \n", "'MDžꟲ", "­>­", "\u000b", "#$%"]} +{"text": "…'re\n\r9Ⅳ \n㋿åå\u000b's'Tꟲ­ꟲ\n'T\r\n\r\n12345678 😀🏽's'll12345678e½", "tokens": 47, "pieces": ["…", "'re", "\n\r", "9Ⅳ", " \n", "㋿åå", "\u000b", "'s'T", "ꟲ", "­ꟲ", "\n", "'T", "\r\n\r\n", "123", "456", "78", " ", " 😀🏽<", "META", "_START", ">'", "s'll", "123", "456", "78", "e", "½"]} +{"text": "字漢\u000b‍fit'VE\r'S'😀🏽ع…\t", "tokens": 18, "pieces": ["字漢", "\u000b", "‍fit'VE", "\r", "'S", "'😀🏽", "ع", "…\t"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": ".0३'MＡ३ḍ̇\r<|endoftext|>12345678
㍿t", "tokens": 25, "pieces": [".", "0३", "'MＡ", "३", "ḍ̇", "\r", "<|", "endoftext", "|>", "123", "456", "78", "
", "㍿t"]} +{"text": "\r\n\r\n'll㍿>\n字é<|endoftext|>\t\t12345678>-Dž", "tokens": 24, "pieces": ["\r\n\r\n", "'ll", "㍿>\n", "字é", "<|", "endoftext", "|>", "\t", "\t", "123", "456", "78", ">-", "Dž"]} +{"text": "­ …!ḍ̇m𐞁'Tعſ\r\n\r\nſ́ 0m", "tokens": 29, "pieces": ["­", " ", "…", "!ḍ̇m𐞁'T", "عſ", "\r\n\r\n", "ſ́", "", " ", "0", "m"]} +{"text": "字\"e9ع'M­tſ👍🏽>ßDž­", "tokens": 16, "pieces": ["字", "\"e", "9", "ع'M", "­tſ", "👍🏽>", "ß", "Dž", "­"]} +{"text": "ع…Ａ'D🙂'reEOT#$%! 'M'VE ٣٤٥٦'T‍-👍🏽#$%'Re\n\n 字", "tokens": 38, "pieces": ["ع", "…Ａ'D", "🙂'", "re", "EOT", "#$%!<", "META", "_START", ">", " ", " '", "M'VE", " ", "٣٤٥", "٦", "'T", "‍-👍🏽#$%'", "Re", "\n\n", " ", " 字"]} +{"text": "'Tt𐞁ḍ̇ḍ̇३عEOT'Sſ'd.s​t🙂ݽ‍漢t<|fim_prefix|>", "tokens": 34, "pieces": ["'Tt𐞁ḍ̇ḍ̇", "३", "ع", "EOT'S", "ſ'd", ".s", "​t", "🙂İ", "½", "‍漢t", "<|", "fim", "_prefix", "|>"]} +{"text": " 😀🏽é'll!३'VEZs\r\n\r\n\t\n<|endoftext|>\n\r're", "tokens": 22, "pieces": [" 😀🏽", "é'll", "!", "३", "'VEZs", "\r\n\r\n\t\n", "<|", "endoftext", "|>\n\r", "'re"]} +{"text": "mdİa½a'VE-d​#$% ३", "tokens": 14, "pieces": ["md", "İa", "½", "a'VE", "-d", "​#$%", " ", " ", "३"]} +{"text": "\u000b", "tokens": 1, "pieces": ["\u000b"]} +{"text": "😀🏽'VE㍿.--\u000b're㍿eé'S<|fim_prefix|>🙂𐞁'S-Ａ́İ<|endoftext|>\u000b12345678　Z'Ms\r\n0ſ字", "tokens": 52, "pieces": ["😀🏽'", "VE", "㍿.--", "\u000b", "'re", "㍿eé'S", "<|", "fim", "_prefix", "|>🙂", "𐞁'S", "-Ａ́", "İ", "<|", "endoftext", "|>", "\u000b", "123", "456", "78", "　Z'M", "s", "\r\n", "0", "ſ字"]} +{"text": "'D́عe
ꟲt'Re\"­.!!‍‍é
#$%ḍ̇#$%ꟲ>s漢e…mEOTꟲ…", "tokens": 41, "pieces": ["'D́عe", "
ꟲt'Re", "\"­.!!‍‍", "é", "
", "#$%", "ḍ̇", "#$%", "ꟲ", ">s漢e", "…m", "EOTꟲ", "…"]} +{"text": "a0­\r\n\r\n (s(字mع t'ſ'T!!", "tokens": 17, "pieces": ["a", "0", "­\r\n\r\n", " ", " (", "s", "(字mع", " t'ſ", "'T", "!!"]} +{"text": "‍'sꟲßs!!٣٤٥٦0ꟲ!!́㍿ſ'D\r\n½!!\r\n é\t😀🏽́́'S
…fiⅣ", "tokens": 48, "pieces": ["‍'", "sꟲßs", "!!", "٣٤٥", "٦0", "ꟲ", "!!́㍿", "ſ", "'", "D", "\r\n", "½", "!!\r\n", " ", " é", "\t", "😀🏽́́'", "S", "
", "…fi", "Ⅳ"]} +{"text": "\r\n\r\n<|fim_prefix|>\"‍-''", "tokens": 10, "pieces": ["\r\n\r\n", "<|", "fim", "_prefix", "|>\"‍-''"]} +{"text": " <|fim_prefix|>!!0ſ'Re'ſ<|endoftext|> \n\n ㋿'ſ'Reå'll\r\n9३\"-字.d<Ⅳå", "tokens": 43, "pieces": [" ", "<|", "fim", "_prefix", "|>!!", "0", "ſ'Re", "'ſ", "<|", "endoftext", "|>", " \n\n", " ", " ㋿'", "ſ'Re", "å'll", "\r\n", "9३", "\"-", "字", ".d", "<", "Ⅳ", "å"]} +{"text": "(Ａ <|fim_prefix|>🙂'll,漢 å㋿'VE9'VEDž.𐞁​Ａ\"\r'se", "tokens": 52, "pieces": ["(Ａ", "", " ", "<|", "fim", "_prefix", "|><", "META", "_START", ">🙂<", "EOT", ">'", "ll", ",漢", " ", " å", "㋿'", "VE", "9", "'VEDž", ".𐞁", "​<", "META", "_START", ">Ａ", "\"\r", "'se"]} +{"text": "e字'ſ㋿'ſſḍ̇Ⅳt字ſ'Ś…😀🏽\u000b\nİa\n​🙂'M,٣٤٥٦fi \u000b漢", "tokens": 50, "pieces": ["e字'ſ", "㋿'", "ſſḍ̇", "Ⅳ", "t字", "ſ'S", "́", "…", "😀🏽", "\u000b\n", "İa", "\n", "​🙂'", "M", ",", "٣٤٥", "٦", "fi", " ", "\u000b漢"]} +{"text": " '\u000bḍ̇'llß👍🏽­㋿Ⅳſ𐞁漢½漢­३#$%(😀🏽\"½ſⅣDž'VE'VEé a", "tokens": 50, "pieces": [" ", "'", "\u000bḍ̇'ll", "ß", "👍🏽­㋿", "Ⅳ", "ſ", "𐞁漢", "½", "漢", "­", "३", "#$%(😀🏽\"", "½", "ſ", "Ⅳ", "Dž'VE", "'VEé", " ", " a"]} +{"text": "<|fim_prefix|>!!ß're'reDž#$%12345678\rmå㋿'D\nḍ̇-12345678fi㍿'Re​12345678é\t", "tokens": 45, "pieces": ["<|", "fim", "_prefix", "|>!!", "ß're", "'re", "Dž", "#$%", "123", "456", "78", "\r", "må", "㋿'", "D", "\n", "ḍ̇", "-", "123", "456", "78", "fi", "㍿'", "Re", "​", "123", "456", "78", "é", "\t"]} +{"text": "fi㋿s'ſ́'T'<|fim_prefix|>-İe'll Dž߅'Re ㍿EOT #$%㍿…\u000bZ'VE'S
ſ\r\n\r\n (", "tokens": 55, "pieces": ["fi", "㋿s'ſ", "́'T", "'<|", "fim", "_prefix", "|>-<", "META", "_START", ">İe'll", " Džß", "…", "'Re", " ", " ㍿", "EOT", " ", " #$%㍿", "…", "\u000bZ'VE", "'S", "
ſ", "\r\n\r\n", " ", "("]} +{"text": "　🙂#$%#$%9'VE!!'VE…eDž㋿d\"'SEOT 漢're
'#$%!'Ret0\u000b'S　EOT字åعa912345678\r\n\r\n\u000béDž<|fim_prefix|>", "tokens": 26, "pieces": ["é", "<", "META", "_START", ">a", "912", "345", "678", "\r\n\r\n", "\u000bé", "Dž", "<|", "fim", "_prefix", "|>"]} +{"text": "👍🏽\r0s३🙂漢ée \n𐞁DžsⅣ(\n'ſ'll½,ſ<|fim_prefix|>İ́>ḍ̇<👍🏽!!'ll'S٣٤٥٦字'VE>", "tokens": 55, "pieces": ["👍🏽\r", "0", "s", "३", "🙂漢ée", " \n", "𐞁Džs", "Ⅳ", "(\n", "'ſ'll", "½", ",ſ", "<|", "fim", "_prefix", "|>", "İ́", ">ḍ̇", "<👍🏽!!'", "ll'S", "٣٤٥", "٦", "字'VE", ">"]} +{"text": "\n!m- \n'll'Tt'VE…Dž👍🏽ع३'ſß\r\nſ å漢漢\u000bＡ! ́㋿0ع", "tokens": 46, "pieces": ["\n", "!m", "-", " \n", "'", "ll'T", "t'VE", "…Dž", "👍🏽", "ع", "३", "'ſß", "\r\n", "ſ", " å漢漢", "\u000bＡ", "!", " ", " ́", "㋿", "0", "ع"]} +{"text": " 'T!!'sDž", "tokens": 7, "pieces": [" '", "T", "!!'", "s", "Dž"]} +{"text": "…\u000b's Ⅳß", "tokens": 11, "pieces": ["…", "\u000b", "'s", "", " ", "Ⅳ", "ß"]} +{"text": "#$%Ⅳ\r\n9 Ⅳ0'll𐞁
,'M'll‍'Dḍ̇Dž㍿٣٤٥٦ \n३,Z🙂é\r\n\r\ns", "tokens": 45, "pieces": ["#$%", "Ⅳ", "\r\n", "9", " ", " <", "META", "_START", ">", "Ⅳ0", "'ll𐞁", "
", ",'", "M'll", "‍'", "Dḍ̇", "Dž", "㍿", "٣٤٥", "٦", " \n", "३", ",Z", "🙂é", "\r\n\r\n", "s"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿'ll<|fim_prefix|>s're're(<'S'Re㍿ Ａ​'D", "tokens": 29, "pieces": ["㍿'", "ll", "<|", "fim", "_prefix", "|>", "s're", "'re", "(<'", "S'Re", "㍿", " Ａ", "​'", "D"]} +{"text": " 'S('ſ……'D'll🙂'٣٤٥٦­🙂<|endoftext|>​t　<|fim_prefix|>> #$%‍ßⅣ're\r
9३'D'Mİ́漢s", "tokens": 52, "pieces": [" '", "S", "('", "ſ", "…", "…", "'D'll", "🙂'", "٣٤٥", "٦", "­🙂<|", "endoftext", "|>​", "t", "　", "<|", "fim", "_prefix", "|>>", " ", "#$%‍", "ß", "Ⅳ", "'re", "\r", "
", "9३", "'D'M", "İ́漢s"]} +{"text": "👍🏽s​<|endoftext|>'S \nm!!å٣٤٥٦\"fi", "tokens": 27, "pieces": ["👍🏽", "s", "​<|", "endoftext", "|>'", "S", " \n", "m", "!!", "å", "٣٤٥", "٦", "\"<", "META", "_START", ">fi"]} +{"text": "́'Réd-𐞁\t㍿mdé'́
é३", "tokens": 21, "pieces": ["́'Re", "́d", "-𐞁", "\t", "㍿mdé", "'́", "
é", "३"]} +{"text": "ß​'D<'D\"🙂fi 'M…é‍ \n12345678𐞁字㍿ \n9ß😀🏽té0fi३ḍ̇'VE", "tokens": 47, "pieces": ["ß", "​'", "D", "<'", "D", "\"🙂", "fi", " ", " '", "M", "…é", "‍", " \n", "123", "456", "78", "𐞁", "字", "㍿", " \n", "9", "ß", "😀🏽", "té", "0", "fi", "३", "ḍ̇'VE"]} +{"text": "½-ع12345678\n漢é'T'S\u000b𐞁dåå🙂عİ\r 
'ſḍ̇.'VE
　", "tokens": 36, "pieces": ["½", "-ع", "123", "456", "78", "\n", "漢é'T", "'S", "\u000b𐞁dåå", "🙂ع", "İ", "\r", " ", "
", "'ſḍ̇", ".'", "VE", "
　"]} +{"text": ".😀🏽'Sm're'ſ<|fim_prefix|>Ａḍ̇.­.0'llⅣ<ß.<|endoftext|>漢𐞁😀🏽'res\u000b३‍", "tokens": 48, "pieces": [".😀🏽'", "Sm're", "'ſ", "<|", "fim", "_prefix", "|>", "Ａḍ̇", ".­.", "0", "'ll", "Ⅳ", "<ß", ".<|", "endoftext", "|>", "漢𐞁", "😀🏽'", "res", "\u000b", "३", "‍"]} +{"text": "Z\u000b", "tokens": 2, "pieces": ["Z", "\u000b"]} +{"text": "s­\r\n'Mm…EOTa'VE😀🏽- \n…𐞁<|endoftext|>é'Tm㍿ 'Ret<|fim_prefix|>12345678'T३🙂!é'llEOT㋿s'VE\r\n\r\n​", "tokens": 66, "pieces": ["s", "­\r\n", "'Mm", "…EOT", "a'VE", "😀🏽-", " \n", "…𐞁", "<|", "endoftext", "|>", "é'T", "m", "㍿", " '", "Ret", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "३", "🙂!", "é'll", "EOT", "㋿s'VE", "\r\n\r\n", "​"]} +{"text": "字　0,\u000bZé('VE's\"\r\n\r\nꟲ漢'ReDž\nfiع㍿12345678(㍿", "tokens": 34, "pieces": ["字", "　", "0", ",", "\u000bZé", "('", "VE's", "\"\r\n\r\n", "ꟲ漢", "'", "Re", "Dž", "\n", "fiع", "㍿", "123", "456", "78", "(㍿"]} +{"text": "'Ms-09ḍ̇DžⅣ d0'<ſß \n", "tokens": 22, "pieces": ["'Ms", "-", "09", "ḍ̇", "Dž", "Ⅳ", " ", " d", "0", "'<<", "META", "_START", ">ſß", " \n"]} +{"text": "'ſ𐞁\r're\r\n\r\n\n", "tokens": 15, "pieces": ["'ſ𐞁", "\r", "'", "re", "\r\n\r\n\n"]} +{"text": "'VE eéＡ'Sa😀🏽
Ⅳ-'M𐞁,　 >'VEtd½\t'Mḍ̇‍\r\n\r\n !ꟲ \t३", "tokens": 46, "pieces": ["'VE", " e", "é", "Ａ'S", "a", "😀🏽", "
", "Ⅳ", "-'", "M𐞁", ",", "　", " ", ">'", "VEtd", "½", "\t", "'Mḍ̇", "‍\r\n\r\n", " ", " !", "ꟲ", " ", "\t", "३"]} +{"text": "İ'S0
", "tokens": 4, "pieces": ["İ'S", "0", "
"]} +{"text": "<|fim_prefix|><|fim_prefix|>ß 's'Sm😀🏽fi'Tḍ̇\u000bs\"‍'ſ \nmEOT‍", "tokens": 40, "pieces": ["<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "ß", " ", " '", "s'S", "m", "😀🏽", "fi'T", "ḍ̇", "\u000bs", "\"‍'", "ſ", " \n", "m", "EOT", "‍"]} +{"text": "'Reḍ̇'s!𐞁d'VEſ👍🏽㍿a('St'ſ㋿𐞁  <|fim_prefix|>\r\n½Dž!!字<|endoftext|>'re字12345678'Reaa aEOT", "tokens": 63, "pieces": ["'Reḍ̇'s", "!𐞁d'VE", "ſ", "👍🏽㍿", "a", "('", "St'ſ", "㋿𐞁", " ", " ", "<|", "fim", "_prefix", "|>\r\n", "½", "Dž", "!!", "字", "<|", "endoftext", "|>'", "re字", "123", "456", "78", "'Reaa", " ", " a", "EOT"]} +{"text": "'VE­s\"'M\r\n\r\nZ\r\n\r\n'VE'D9ꟲ", "tokens": 23, "pieces": ["'VE", "­s", "\"'", "M", "\r\n\r\n", "Z", "\r\n\r\n", "'VE'D", "9", "ꟲ", ""]} +{"text": ".d🙂", "tokens": 2, "pieces": [".d", "🙂"]} +{"text": "EOTEOTEOTß<|fim_prefix|>㋿ ́fi‍\nİ!! 
½ß", "tokens": 29, "pieces": ["EOTEOTEOTß", "<|", "fim", "_prefix", "|>㋿", " ", " ́fi", "‍\n", "İ", "!!", " ", "
", "½", "ß"]} +{"text": "'sdDžsſ ㍿ \n🙂\t!'T\u000bse.‍\n<|fim_prefix|>🙂ſ㍿ ­'D'S'Re٣٤٥٦Ⅳ'S", "tokens": 45, "pieces": ["'sd", "Džsſ", " ", " ㍿", " \n", "🙂", "\t", "!'", "T", "\u000bse", ".‍\n", "<|", "fim", "_prefix", "|>🙂", "ſ", "㍿", " ", " ­'", "D'S", "'Re", "٣٤٥", "٦Ⅳ", "'S"]} +{"text": "'ll😀🏽eEOT. \n \r'sعع́\u000b\rⅣé", "tokens": 21, "pieces": ["'ll", "😀🏽", "e", "EOT", ".", " \n \r", "'sعع́", "\u000b\r", "Ⅳ", "é"]} +{"text": "<|endoftext|>😀🏽\r\n!İ\t \n'T'#$%m ­😀🏽m9<mꟲå\t \ne\n㍿", "tokens": 46, "pieces": ["<|", "endoftext", "|>😀🏽\r\n", "!İ", "\t \n", "'T", "'#$%", "m", " ", " ­😀🏽", "m", "9", "<<", "EOT", ">mꟲå", "\t \n", "e", "\n", "㍿"]} +{"text": "'reع\tfi''MEOT㍿9\r\né'T‍eع\r\n\r\n😀🏽<|fim_prefix|>\r\n>​\n\"", "tokens": 37, "pieces": ["'reع", "\tfi", "''", "MEOT", "㍿", "9", "\r\n", "é'T", "‍eع", "\r\n\r\n", "😀🏽<|", "fim", "_prefix", "|><", "EOT", ">\r\n", ">​\n", "\""]} +{"text": "'ſ>\t", "tokens": 4, "pieces": ["'ſ", ">", "\t"]} +{"text": "'ſ\n𐞁'Re,-<|endoftext|>'VEſ…<|fim_prefix|>EOT​'Mſ'S0٣٤٥٦12345678👍🏽'llİ12345678!!\tſé'ſ‍漢'ſå\u000b're'T'D\t㍿½", "tokens": 72, "pieces": ["'ſ", "\n", "𐞁'Re", ",-<|", "endoftext", "|>'", "VEſ", "…", "<|", "fim", "_prefix", "|>", "EOT", "​'", "Mſ'S", "0٣٤", "٥٦1", "234", "567", "8", "👍🏽'", "ll", "İ", "123", "456", "78", "!!", "\tſé'ſ", "‍漢'ſ", "å", "\u000b", "'re'T", "'D", "\t", "㍿", "½"]} +{"text": "㋿𐞁é'VE­­-å‍'漢,EOT字Ａ३<|fim_prefix|>​🙂🙂'VE''s…t​'re'St㍿　!!", "tokens": 51, "pieces": ["㋿𐞁é'VE", "­­-", "å", "‍'", "漢", ",EOT字", "Ａ", "३", "<|", "fim", "_prefix", "|>​🙂🙂'", "VE", "''", "s", "…t", "​'", "re'S", "t", "㍿", "　", "!!"]} +{"text": "'ll字", "tokens": 2, "pieces": ["'ll字"]} +{"text": "\t㍿s㋿#$%12345678,e#$%'T'Re٣٤٥٦😀🏽👍🏽s
!", "tokens": 31, "pieces": ["\t", "㍿s", "㋿#$%", "123", "456", "78", ",e", "#$%'", "T'Re", "٣٤٥", "٦", "😀🏽👍🏽", "s", "
", "!"]} +{"text": "t'ſ‍\t🙂éd\r㋿😀🏽a\r\n\r\n9　'llＡ'll-(0‍٣٤٥٦­　字#$%(å'ſ
Ⅳ'SⅣa", "tokens": 54, "pieces": ["t'ſ", "‍", "\t", "🙂éd", "\r", "㋿😀🏽", "a", "\r\n\r\n", "9", "　", "'ll", "Ａ'll", "-(", "0", "‍", "٣٤٥", "٦", "­", "　字", "#$%<", "META", "_START", ">(", "å'ſ", "
", "Ⅳ", "'S", "Ⅳ", "a"]} +{"text": "'\r\n\r​ḍ̇­a. EOT'VE­\r\n\r\n'Re​㋿ 👍🏽<|fim_prefix|>>㋿>\r\n\r\n\r\n!Dž'VE　'llſ'ré\t'ſ'sḍ̇,", "tokens": 55, "pieces": ["'\r\n\r", "​ḍ̇", "­a", ".", " ", " EOT'VE", "­\r\n\r\n", "'Re", "​㋿", " ", " 👍🏽<|", "fim", "_prefix", "|>>㋿>\r\n\r\n\r\n", "!Dž", "'", "VE", "　", "'llſ're", "́", "\t", "'ſ's", "ḍ̇", ","]} +{"text": "👍🏽ＡéEOT\"Z\ré́\u000bＡعaZ𐞁<|fim_prefix|>9d३", "tokens": 31, "pieces": ["👍🏽", "Ａé", "EOT", "\"Z", "\r", "é́", "\u000bＡعa", "Z𐞁", "<|", "fim", "_prefix", "|>", "9", "d", "३"]} +{"text": "9‍'T字'reⅣt'Re'Dꟲå0'reꟲé's.ع\"'ll字㍿>", "tokens": 33, "pieces": ["9", "‍'", "T字're", "Ⅳ", "t'Re", "'Dꟲå", "0", "'reꟲé's", ".ع", "\"'", "ll字", "㍿>"]} +{"text": "#$%㋿é", "tokens": 6, "pieces": ["#$%㋿", "é"]} +{"text": "é🙂‍'re
å'VE३s.…'T漢.!#$%12345678㋿٣٤٥٦m㋿#$%\r", "tokens": 46, "pieces": ["é", "🙂‍'", "re", "", "
å'VE", "३", "s", ".", "…", "'T漢", ".!#$%", "123", "456", "78", "㋿", "٣٤٥", "٦", "m", "㋿<", "EOT", ">#$%\r"]} +{"text": "👍🏽!!", "tokens": 4, "pieces": ["👍🏽!!"]} +{"text": "'sḍ̇Z\r\n\r\n½İ🙂\r\n\r\n'Tß", "tokens": 29, "pieces": ["ße", "0", "Ａꟲ'D", "𐞁", "\u000b", "
", "Ⅳ", "'S", "<'", "Re", "-s", "<|", "fim", "_prefix", "|>🙂\r\n\r\n", "'Tß"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'S>éZſ<|endoftext|>\">३-ع#$%Z!'VE", "tokens": 21, "pieces": ["'S", ">é", "Zſ", "<|", "endoftext", "|>\">", "३", "-ع", "#$%", "Z", "!'", "VE"]} +{"text": "d'ſ<\r🙂<|endoftext|>\nǻ#$%ع́ \n\n½\tå'Re‍🙂ḍ̇'S,m👍🏽EOTfi​㍿\u000b😀🏽<|fim_prefix|>", "tokens": 52, "pieces": ["d'ſ", "<\r", "🙂<|", "endoftext", "|>\n", "ǻ", "#$%", "ع́", " \n\n", "½", "\tå'Re", "‍🙂", "ḍ̇'S", ",m", "👍🏽", "EOTfi", "​㍿", "\u000b", "😀🏽<|", "fim", "_prefix", "|>"]} +{"text": "e…é'VE9漢é!!<|fim_prefix|>", "tokens": 17, "pieces": ["e", "…é'VE", "9", "漢é", "!!<|", "fim", "_prefix", "|>"]} +{"text": "ḍ̇
\t३", "tokens": 6, "pieces": ["ḍ̇", "
", "\t", "३"]} +{"text": "\t 字é㍿İ<|endoftext|>'VEḍ̇Ⅳ", "tokens": 26, "pieces": ["\t", " 字é", "㍿<", "EOT", ">İ", "<|", "endoftext", "|>'", "VEḍ̇", "Ⅳ"]} +{"text": "Ⅳ​😀🏽İ\"'D🙂'T​9d\råe12345678\r\n\r\n \nع's", "tokens": 34, "pieces": ["Ⅳ", "​😀🏽", "İ", "\"'", "D", "🙂'", "T", "​<", "EOT", ">", "9", "d", "\r", "åe", "123", "456", "78", "\r\n\r\n \n", "ع's"]} +{"text": "<|endoftext|>👍🏽
́\"Z0عZꟲ'Re🙂…<٣٤٥٦'MDž㍿'D½<|fim_prefix|>(ⅣZd́　٣٤٥٦㋿ß'Ma'\r\n'ReⅣ㍿\r\n\r\n.", "tokens": 68, "pieces": ["<|", "endoftext", "|>👍🏽", "
́", "\"Z", "0", "عZꟲ'Re", "🙂", "…", "<", "٣٤٥", "٦", "'MDž", "㍿'", "D", "½", "<|", "fim", "_prefix", "|>(", "Ⅳ", "Zd́", "　", "٣٤٥", "٦", "㋿ß'M", "a", "'\r\n", "'Re", "Ⅳ", "㍿\r\n\r\n", "."]} +{"text": "​9ſＡ㋿\r\n\r\n😀🏽Dž字'VE\tİ'reḍ̇#$%<|endoftext|>\"#$%‍\n", "tokens": 38, "pieces": ["​", "9", "ſ", "Ａ", "㋿\r\n\r\n", "😀🏽", "Dž字'VE", "\tİ're", "ḍ̇", "#$%<|", "endoftext", "|><", "META", "_START", ">\"#$%‍\n"]} +{"text": "(<|fim_prefix|>", "tokens": 6, "pieces": ["(<|", "fim", "_prefix", "|>"]} +{"text": "ſ'ſ'T­Z. \n
<|endoftext|>!!३9m٣٤٥٦9 \r\n𐞁
ßſ's'३'ſ
'VEḍ̇", "tokens": 44, "pieces": ["ſ'ſ", "'T", "­Z", ".", " \n", "
", "<|", "endoftext", "|>!!", "३9", "m", "٣٤٥", "٦9", " \r\n", "𐞁", "
ßſ's", "'", "३", "'ſ", "
", "'VEḍ̇"]} +{"text": "'s३'S \n ſ!'D­🙂㋿\rḍ̇e#$% \n,ss '", "tokens": 24, "pieces": ["'s", "३", "'S", " \n", " ſ", "!'", "D", "­🙂㋿\r", "ḍ̇e", "#$%", " \n", ",ss", " '"]} +{"text": "'VE\u000b\t\t漢\u000b<|endoftext|>åꟲ😀🏽.'sm's!!'ſ́३", "tokens": 30, "pieces": ["'VE", "\u000b\t", "\t漢", "\u000b", "<|", "endoftext", "|>", "åꟲ", "😀🏽.'", "sm's", "!!'", "ſ́", "३"]} +{"text": ".\"<\t'S>　ſ!!eå👍🏽'M'Re!😀🏽 \n!!…ß", "tokens": 38, "pieces": [".\"<", "\t", "'S", ">", "　ſ", "!!", "eå", "👍🏽'", "M'Re", "!😀🏽", " \n", "!!", "…ß"]} +{"text": "!!'ſ'ſ\r\n->'ś'D0é0㍿<'ſ'VE漢\"é­", "tokens": 29, "pieces": ["!!<", "EOT", ">'", "ſ'ſ", "\r\n", "->'", "ś'D", "0", "é", "0", "㍿<'", "ſ'VE", "漢", "\"é", "­"]} +{"text": "12345678ſ'M\n'T㍿d\t", "tokens": 12, "pieces": ["123", "456", "78", "ſ'M", "\n", "'T", "㍿d", "\t"]} +{"text": " ſfi🙂0
½0\u000b
\u000b ‍\r\nⅣ
𐞁12345678'VE😀🏽e'S>fi\r", "tokens": 39, "pieces": [" ", " ſfi", "🙂", "0", "
", "½0", "\u000b
\u000b ", " ‍\r\n", "Ⅳ", "
𐞁", "123", "456", "78", "'VE", "😀🏽", "e'S", ">fi", "\r"]} +{"text": "'VEå(ḍ̇0#$%
's Ａꟲ", "tokens": 18, "pieces": ["'VEå", "(ḍ̇", "0", "#$%", "
", "'s", " Ａꟲ"]} +{"text": "Dž're
漢ß(\" (é(åfi㍿'VE", "tokens": 22, "pieces": ["Dž're", "
漢", "ß", "(\"", " ", "(é", "(åfi", "㍿'", "VE"]} +{"text": "<|endoftext|>m​㋿", "tokens": 12, "pieces": ["<|", "endoftext", "|>", "m", "​㋿"]} +{"text": "ḍ̇#$% \nt'e㍿\n \n'", "tokens": 21, "pieces": ["ḍ̇", "#$%", " \n", "t", "'", "e", "㍿\n", " \n", "'"]} +{"text": "­́ع0'''ſsḍ̇", "tokens": 41, "pieces": ["­́", "ع", "0", "'''", "ſsḍ̇"]} +{"text": "'T'D'SꟲaDž字字'MaDžmfim ́ß\"
eعm!…m!٣٤٥٦‍s\r\n\r\n0👍🏽Ａ", "tokens": 57, "pieces": ["'T'D", "'Sꟲa", "Dž字字'M", "a", "Dž", "mfim", " <", "META", "_START", ">́ß", "\"", "
eعm", "!", "…m", "!", "٣٤٥", "٦", "‍s", "\r\n\r\n", "0", "👍🏽", "Ａ"]} +{"text": "'Reḍ̇ß𐞁Ⅳ'ſ \n'De­ع0ḍ̇.'\r\nå\u000b　", "tokens": 28, "pieces": ["'Reḍ̇ß𐞁", "Ⅳ", "'ſ", " \n", "'De", "­ع", "0", "ḍ̇", ".'\r\n", "å", "\u000b　"]} +{"text": "İ's\nⅣ12345678Džd9\r\n
's<|fim_prefix|>(Dž
'Re'VE'll ß'M‍\n''s漢é३'ſaéⅣḍ̇EOT", "tokens": 52, "pieces": ["İ's", "\n", "Ⅳ12", "345", "678", "Džd", "9", "\r\n", "
", "'s", "<|", "fim", "_prefix", "|>(", "Dž", "
", "'Re'VE", "'ll", " ß'M", "‍\n", "''", "s漢é", "३", "'ſaé", "Ⅳ", "ḍ̇", "EOT"]} +{"text": "amé'VEſZ𐞁'M🙂.e字", "tokens": 15, "pieces": ["amé'VE", "ſ", "Z𐞁'M", "🙂.", "e字"]} +{"text": "ꟲ0s.​EOTꟲ\r\n\r\n\r\nع字 0Z　🙂ſ<|fim_prefix|> \nfi👍🏽İ,Ａ\nİt!å‍", "tokens": 41, "pieces": ["ꟲ", "0", "s", ".​", "EOTꟲ", "\r\n\r\n\r\n", "ع字", " ", "0", "Z", "　", "🙂ſ", "<|", "fim", "_prefix", "|>", " \n", "fi", "👍🏽", "İ", ",Ａ", "\n", "İt", "!å", "‍"]} +{"text": "'T9Ａ!'D𐞁's
\"٣٤٥٦\t٣٤٥٦'M'T<<'Så३'re­ .EOT­'　eꟲm<|endoftext|>　,'Så", "tokens": 63, "pieces": ["å", " ", " '", "D're", "\r\n\r\n", "漢", "<|", "fim", "_prefix", "|>'", "s", "
", "\"", "٣٤٥", "٦", "\t", "٣٤٥", "٦", "'M'T", "<<'", "Så", "३", "'re", "­", " ", ".EOT", "­<", "EOT", ">'", "　eꟲm", "<|", "endoftext", "|>", "　", ",'", "Så"]} +{"text": "\r🙂'Tß字#$%\r\n\r\n\t㍿\r\n\r\n.\u000bİ㋿
12345678'T", "tokens": 25, "pieces": ["\r", "🙂'", "Tß字", "#$%\r\n\r\n", "\t", "㍿\r\n\r\n", ".", "\u000bİ", "㋿", "
", "123", "456", "78", "'T"]} +{"text": "'M
 ß!ſ'Red('ll­\n'S!!Ⅳs#$%.'ss'VE­
'''Ḿa're'M'D", "tokens": 39, "pieces": ["'M", "
", " ß", "!ſ'Re", "d", "('", "ll", "­\n", "'S", "!!", "Ⅳ", "s", "#$%.'", "s", "s'VE", "­", "
", "'''", "M", "́a're", "'M'D"]} +{"text": "<|endoftext|><字fiŹ\n12345678㍿EOT#$%Džé👍🏽
\n'S…'D'M'ſ​ع\n<‍", "tokens": 42, "pieces": ["<|", "endoftext", "|><", "字fi", "Ź", "\n", "123", "456", "78", "㍿EOT", "#$%", "Džé", "👍🏽", "
\n", "'S", "…", "'D'M", "'ſ", "​ع", "\n", "<‍"]} +{"text": "<|endoftext|>ß\r\nİ#$%9ß'ſ'T½", "tokens": 18, "pieces": ["<|", "endoftext", "|>", "ß", "\r\n", "İ", "#$%", "9", "ß'ſ", "'T", "½"]} +{"text": "9,\tſé٣٤٥٦é'reſ'ſZDžsİDž'S-'s漢0!
're<|endoftext|>𐞁's
é٣٤٥٦㍿!!\r\n　's", "tokens": 57, "pieces": ["9", ",", "\tſé", "٣٤٥", "٦", "é're", "ſ'ſ", "ZDžs", "İDž'S", "-'", "s漢", "", "0", "!", "
", "'re", "<|", "endoftext", "|>", "𐞁's", "
é", "٣٤٥", "٦", "㍿!!\r\n", "　", "'s"]} +{"text": "½m½́🙂Ze \nt‍́ 
sꟲå‍'Reé", "tokens": 22, "pieces": ["½", "m", "½", "́", "🙂Ze", " \n", "t", "‍́", " ", "
sꟲå", "‍'", "Reé"]} +{"text": "12345678 \n'Re漢(.e字're‍#$%'VE!!aEOT😀🏽­'S🙂\"'VE\re漢\r\n\r\neZ", "tokens": 33, "pieces": ["123", "456", "78", " \n", "'Re漢", "(.", "e字're", "‍#$%'", "VE", "!!", "a", "EOT", "😀🏽­'", "S", "🙂\"'", "VE", "\r", "e漢", "\r\n\r\n", "e", "Z"]} +{"text": "'ll字,", "tokens": 3, "pieces": ["'ll字", ","]} +{"text": "s,'M\rſå", "tokens": 7, "pieces": ["s", ",'", "M", "\r", "ſå"]} +{"text": "s'\"'ſeع …<'Reꟲ'T­é\r\n\r\n", "tokens": 19, "pieces": ["s", "'\"'", "ſeع", " ", "…", "<'", "Reꟲ'T", "­é", "\r\n\r\n"]} +{"text": "😀🏽#$%\r\n…12345678'Re<\".漢m٣٤٥٦عå'ſe\u000bt12345678å", "tokens": 38, "pieces": ["😀🏽#$%\r\n", "…", "", "123", "456", "78", "'Re", "<\".", "漢m", "٣٤٥", "٦", "ع", "å'ſ", "e", "\u000bt", "123", "456", "78", "å"]} +{"text": "'VE'lla　 \r\n\r\n'llt<", "tokens": 9, "pieces": ["'VE'll", "a", "　 \r\n\r\n", "'llt", "<"]} +{"text": "éé­İ'S‍<", "tokens": 8, "pieces": ["éé", "­İ'S", "‍<"]} +{"text": "'ll'st漢EOT​Ａeİ𐞁0éḍ̇EOT­", "tokens": 23, "pieces": ["'ll's", "t漢", "EOT", "​Ａe", "İ𐞁", "0", "éḍ̇", "EOT", "­"]} +{"text": ">\n𐞁sⅣ𐞁#$%t'VE#$%<|endoftext|>-<|endoftext|>ſ­>'D­३d'VE\t👍🏽𐞁tꟲ\n12345678'D#$%ⅣZ'll㍿ 漢a'M", "tokens": 76, "pieces": [">\n", "𐞁s", "Ⅳ", "𐞁", "#$%", "t'VE", "#$%<|", "endoftext", "|>-<|", "endoftext", "|>", "ſ", "­>'", "D", "­", "३", "d'VE", "\t", "👍🏽", "𐞁tꟲ", "\n", "123", "456", "78", "'D", "#$%", "Ⅳ", "Z'll", "㍿", " 漢a'M"]} +{"text": "'Ḿ's \né! ,'T
's \"'D9ع'llDžEOT­Ａ ½漢…", "tokens": 31, "pieces": ["'Ḿ's", " \n", "é", "!", " ,'", "T", "
", "'s", " ", "\"'", "D", "9", "ع'll", "DžEOT", "­Ａ", " ", "½", "漢", "…"]} +{"text": "ḍ̇ſe\t \n߅'re're'Sfi'عİ­​​'ll🙂ḍ̇㍿'lle.><|endoftext|>(0,漢ḍ̇½e​\u000b", "tokens": 47, "pieces": ["ḍ̇ſe", "\t \n", "ß", "…", "'re're", "'Sfi", "'ع", "İ", "­​​'", "ll", "🙂ḍ̇", "㍿'", "lle", ".><|", "endoftext", "|>(", "0", ",漢ḍ̇", "½", "e", "​", "\u000b"]} +{"text": "३ßZ字0́'Re'T'ſſ12345678३>'remé\r\n­…fit \n!İ….\n'ſ#$% ", "tokens": 35, "pieces": ["३", "ß", "Z字", "0", "́'Re", "'T'ſ", "ſ", "123", "456", "78३", ">'", "remé", "\r\n", "­", "…fit", " \n", "!İ", "…", ".\n", "'ſ", "#$%", " "]} +{"text": "'VE字<|endoftext|><|endoftext|>\nＡع㋿", "tokens": 21, "pieces": ["'VE字", "<|", "endoftext", "|><|", "endoftext", "|>\n", "Ａع", "㋿"]} +{"text": "!é \nⅣa!­\"!m\r\n#$%'字𐞁'ع㍿<|endoftext|>'se( ꟲḍ̇Džd漢!. 'S!!d", "tokens": 53, "pieces": ["!é", " \n", "Ⅳ", "a", "!­\"!", "m", "\r\n", "#$%'", "字𐞁", "'ع", "㍿<|", "endoftext", "|>'", "se", "(", " ꟲḍ̇", "Džd漢", "!.", " ", " '", "S", "!!", "d", ""]} +{"text": "!'SDžİⅣ\r\n\r\n\r!㋿-é\ntéé🙂İ​\r\nſ👍🏽!‍​㋿
३e‍ a", "tokens": 44, "pieces": ["!'", "SDžİ", "Ⅳ", "\r\n\r\n\r", "!㋿<", "META", "_START", ">-", "é", "\n", "téé", "🙂İ", "​\r\n", "ſ", "👍🏽!‍​㋿", "
", "३", "e", "‍", " a"]} +{"text": "\u000b'S", "tokens": 2, "pieces": ["\u000b", "'S"]} +{"text": "<\"é ſ ­ꟲ Ⅳ🙂'ſ३ع٣٤٥٦aé𐞁'M
'T<|endoftext|>", "tokens": 38, "pieces": ["<\"", "é", " ſ", " ­", "ꟲ", " ", "Ⅳ", "🙂'", "ſ", "३", "ع", "٣٤٥", "٦", "aé𐞁'M", "
", "'T", "<|", "endoftext", "|>"]} +{"text": "!.eعt\r\n\r\n\"<|fim_prefix|>fi EOT\r\n\r\nꟲ
'Reḍ̇　sfi,漢mtm\t", "tokens": 39, "pieces": ["!.<", "META", "_START", ">eعt", "\r\n\r\n", "\"<", "EOT", "><|", "fim", "_prefix", "|>", "fi", " EOT", "\r\n\r\n", "ꟲ", "
", "'Reḍ̇", "　sfi", ",漢mtm", "\t"]} +{"text": "ßé're912345678t", "tokens": 7, "pieces": ["ßé're", "912", "345", "678", "t"]} +{"text": "İ\"!!", "tokens": 3, "pieces": ["İ", "\"!!"]} +{"text": " .\u000bå🙂t😀🏽<|fim_prefix|>​​,0'(é'ſm'Re", "tokens": 26, "pieces": [" ", ".", "\u000bå", "🙂t", "😀🏽<|", "fim", "_prefix", "|>​​,", "0", "'(", "é'ſ", "m'Re"]} +{"text": "Ⅳ'ſİ😀🏽e0𐞁 😀🏽#$%t'S>३9㍿٣٤٥٦'㋿'De'Re'!<|endoftext|>e½​‍ \r\n\r\n'…​ß ' ", "tokens": 63, "pieces": ["Ⅳ", "'ſ", "İ", "😀🏽", "e", "0", "𐞁", " 😀🏽#$%", "t'S", ">", "३9", "㍿", "٣٤٥", "٦", "'㋿'", "De'Re", "'!<|", "endoftext", "|>", "e", "½", "​‍", " \r\n\r\n", "'<", "EOT", ">", "…", "​ß", " '", " "]} +{"text": "ſ", "tokens": 1, "pieces": ["ſ"]} +{"text": "​३ ('reḍ̇漢ḍ̇m-'𐞁​'D-ꟲ\r\n\r\nEOT字Ⅳ㋿𐞁 \n> \u000b", "tokens": 45, "pieces": ["​", "३", " ", " ('", "reḍ̇漢ḍ̇m", "-'", "𐞁", "​'", "D", "-ꟲ", "\r\n\r\n", "EOT字", "Ⅳ", "㋿𐞁", " \n", "><", "META", "_START", ">", " \u000b"]} +{"text": "­Ⅳ-ſ! 'sß", "tokens": 12, "pieces": ["­", "Ⅳ", "-", "ſ", "!", " '", "sß"]} +{"text": "漢­­9🙂🙂𐞁😀🏽0 d'TZ", "tokens": 17, "pieces": ["漢", "­­", "9", "🙂🙂", "𐞁", "😀🏽", "0", " ", " d'T", "Z"]} +{"text": "İé0<|endoftext|>İ'll🙂́fi　​s-efi<|endoftext|>'ll­́३,", "tokens": 39, "pieces": ["İé", "0", "<|", "endoftext", "|>", "İ'll", "🙂́", "fi", "　", "​s", "-efi", "<|", "endoftext", "|>'", "ll", "­́", "३", ","]} +{"text": "12345678're　漢́é\n0mßḍ̇!!Ze12345678're'VE\tm'M\r\né'll😀🏽Ⅳt½\t👍🏽ḍ̇fi'T<|endoftext|>­㍿Z'Re", "tokens": 53, "pieces": ["\u000b́'re", "-.‍", "eß", "Ze", "123", "456", "78", "'re'VE", "\tm'M", "\r\n", "é'll", "😀🏽", "Ⅳ", "t", "½", "\t", "👍🏽", "ḍ̇fi'T", "<|", "endoftext", "|>­㍿", "Z'Re"]} +{"text": "(ع👍🏽ſⅣ'e…t㋿fifi३'T🙂dＡém'ss\r\nꟲ㍿9 \n!…Z\" 're٣٤٥٦'re\u000bt!!", "tokens": 52, "pieces": ["(ع", "👍🏽", "ſ", "Ⅳ", "'e", "…t", "㋿fi", "fi", "३", "'T", "🙂d", "Ａém's", "s", "\r\n", "ꟲ", "㍿", "9", " \n", "!", "…Z", "\"", " '", "re", "٣٤٥", "٦", "'re", "\u000bt", "!!"]} +{"text": "𐞁漢å　<|endoftext|>'VE㋿ＡZ\u000bm🙂İ\r\n\r\n'sع12345678å🙂\t#$%'-ß'Reع> d…é", "tokens": 47, "pieces": ["𐞁漢å", "　", "<|", "endoftext", "|>'", "VE", "㋿ＡZ", "\u000bm", "🙂İ", "\r\n\r\n", "'sع", "123", "456", "78", "å", "🙂", "\t", "#$%'-", "ß'Re", "ع", ">", " d", "…é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\n<|endoftext|>'VE½'re𐞁\u000b漢12345678eé<㍿漢Z'M'VE
'M👍🏽s'D ḍ̇-㍿<|fim_prefix|>", "tokens": 52, "pieces": ["\n", "<|", "endoftext", "|>'", "VE", "½", "'re𐞁", "\u000b漢", "123", "456", "78", "eé", "<㍿", "漢", "Z'M", "'VE", "
", "'M", "👍🏽", "s'D", " ḍ̇", "-㍿<|", "fim", "_prefix", "|>"]} +{"text": "12345678\r\n\r\nſ<½12345678ḍ̇'ſع<'ReＡ'D(\n漢‍ #$%​٣٤٥٦🙂­!! ​\r\n\r\n<<|endoftext|>>!!ⅣDžſſ!字", "tokens": 58, "pieces": ["123", "456", "78", "\r\n\r\n", "ſ", "<", "½12", "345", "678", "ḍ̇'ſ", "ع", "<'", "Re", "Ａ'D", "(\n", "漢", "‍", " ", " #$%​", "٣٤٥", "٦", "🙂­!!", " ​\r\n\r\n", "<<|", "endoftext", "|>>!!", "Ⅳ", "Džſſ", "!字"]} +{"text": "
s𐞁tⅣ(٣٤٥٦åZ½​'D ḍ̇<‍ ‍'VEeعfi\r\ń½'S9'T\tſ漢​'re'Ms漢㍿'M", "tokens": 56, "pieces": ["
s𐞁t", "Ⅳ", "(", "٣٤٥", "٦", "å", "Z", "½", "​'", "D", " ḍ̇", "<<", "META", "_START", ">‍", " ‍'", "VEeعfi", "\r\n", "́", "½", "'S", "9", "'T", "\tſ漢", "​'", "re'M", "s漢", "㍿'", "M"]} +{"text": "fiꟲḍ̇", "tokens": 7, "pieces": ["fiꟲḍ̇"]} +{"text": "字😀🏽
İⅣＡ​é<|fim_prefix|>DžEOT😀🏽!<\u000b㋿\"　", "tokens": 32, "pieces": ["字", "😀🏽", "
İ", "Ⅳ", "Ａ", "​é", "<|", "fim", "_prefix", "|>", "DžEOT", "😀🏽!<", "\u000b", "㋿\"", "　"]} +{"text": "0㍿<ḍ̇İ'T!…　\r\n \n'M٣٤٥٦İ́漢\rtꟲ…éİ٣٤٥٦é're 'M‍ß.", "tokens": 49, "pieces": ["0", "㍿<", "ḍ̇", "İ'T", "!", "…　\r\n \n", "'M", "٣٤٥", "٦", "İ́漢", "\r", "tꟲ", "…é", "İ", "٣٤٥", "٦", "é're", " ", " '", "M", "‍ß", "."]} +{"text": "0'llt ", "tokens": 4, "pieces": ["0", "'llt", " "]} +{"text": "🙂'9ß(å\u000ba'Re\r\nḍ̇.\rꟲ㋿ع9'VEsZDž…'M\r\n\r\n\rİＡ'Mfia.३ḍ̇", "tokens": 51, "pieces": ["🙂<", "META", "_START", ">'<", "META", "_START", ">", "9", "ß", "(å", "\u000ba'Re", "\r\n", "ḍ̇", ".\r", "ꟲ", "㋿ع", "9", "'VEs", "ZDž", "…", "'M", "\r\n\r\n\r", "İＡ'M", "fia", ".", "३", "ḍ̇"]} +{"text": "   \n'VEt<|fim_prefix|>é㋿Džts,'lĺ'Re\r\n<,", "tokens": 27, "pieces": ["   \n", "'VEt", "<|", "fim", "_prefix", "|>", "é", "㋿Džts", ",'", "lĺ'Re", "\r\n", "<,"]} +{"text": "(- 'Té𐞁ǻ…EOTعꟲ'Re\u000b,٣٤٥٦㍿s\u000b", "tokens": 35, "pieces": ["(-", " ", " '", "Té𐞁ǻ", "…EOTعꟲ", "'", "Re", "\u000b", ",", "٣٤٥", "٦", "㍿s", "\u000b"]} +{"text": "\"Ⅳ-𐞁-e!'llåİ 漢Zfi㋿", "tokens": 21, "pieces": ["\"", "Ⅳ", "-𐞁", "-e", "!'", "llå", "İ", " ", " 漢Zfi", "㋿"]} +{"text": "d<|endoftext|>'TⅣ\r😀🏽ſⅣd‍!㋿", "tokens": 24, "pieces": ["d", "<|", "endoftext", "|>'", "T", "Ⅳ", "\r", "😀🏽", "ſ", "Ⅳ", "d", "‍!㋿"]} +{"text": "👍🏽#$% EOT<|endoftext|>,عſéſ\t\r½'sd…😀🏽é½<\t\n'VEé👍🏽", "tokens": 41, "pieces": ["👍🏽#$%", " ", " EOT", "<|", "endoftext", "|>,", "عſéſ", "\t\r", "½", "'sd", "…", "😀🏽", "é", "½", "<", "\t\n", "'VEé", "👍🏽"]} +{"text": "(åع🙂ع🙂Dž­'S!'s're­㋿'ſfi😀🏽½'D9", "tokens": 27, "pieces": ["(åع", "🙂ع", "🙂Dž", "­'", "S", "!'", "s're", "­㋿'", "ſfi", "😀🏽", "½", "'D", "9"]} +{"text": "\t٣٤٥٦'T३\rع…𐞁fi٣٤٥٦🙂'll'll
", "tokens": 25, "pieces": ["\t", "٣٤٥", "٦", "'T", "३", "\r", "ع", "…𐞁fi", "٣٤٥", "٦", "🙂'", "ll'll", "
"]} +{"text": "'­\"́́'ll're🙂👍🏽　字

…0ꟲ'llⅣ\" \n३　,\u000b'Re \n", "tokens": 36, "pieces": ["'­\"́́'", "ll're", "🙂👍🏽", "　字", "
", "", "
", "…", "0", "ꟲ'll", "Ⅳ", "\"", " \n", "३", "　", ",", "\u000b", "'Re", " \n"]} +{"text": "
Z<|fim_prefix|>!!\n‍é\u000b㋿ \n'Re　　0d(d㍿Z'lls12345678 👍🏽'T ​́> >👍🏽　", "tokens": 58, "pieces": ["
Z", "<|", "fim", "_prefix", "|>!!\n", "‍<", "META", "_START", ">é", "\u000b", "㋿<", "EOT", ">", " \n", "'Re", "　", "　", "0", "d", "(d", "㍿Z'll", "s", "123", "456", "78", " ", "👍🏽'", "T", " ", " ​́>", " >👍🏽", "　"]} +{"text": "'S漢​'Sİḍ̇‍\r\nＡ", "tokens": 15, "pieces": ["'S漢", "​<", "META", "_START", ">'", "Sİḍ̇", "‍\r\n", "Ａ"]} +{"text": "㍿ <<|fim_prefix|>å\r\n<|fim_prefix|>㍿'T\r\nſ!", "tokens": 27, "pieces": ["㍿", " ", " <<|", "fim", "_prefix", "|>", "å", "\r\n", "<|", "fim", "_prefix", "|>㍿'", "T", "\r\n", "ſ", "!"]} +{"text": ".a
'll'llm<|endoftext|>Z\r\"-'ſ😀🏽>字'sⅣ<|endoftext|>d\tİ­", "tokens": 36, "pieces": [".a", "
", "'ll'll", "m", "<|", "endoftext", "|>", "Z", "\r", "\"-'", "ſ", "😀🏽>", "字's", "Ⅳ", "<|", "endoftext", "|>", "d", "\tİ", "­"]} +{"text": "😀🏽'Re🙂'sعm'VE🙂's('re㋿daa'Dß🙂\u000bİ'll!! åع'så'St12345678\r\n\r\n\neé", "tokens": 43, "pieces": ["😀🏽'", "Re", "🙂'", "sعm'VE", "🙂'", "s", "('", "re", "㋿daa'D", "ß", "🙂", "\u000bİ'll", "!!", " åع's", "å'S", "t", "123", "456", "78", "\r\n\r\n\n", "eé"]} +{"text": "'D're\r\n123456789㍿t‍.​<|endoftext|>!ß㍿\r\nDž<|fim_prefix|>㋿ḍ̇́'M…", "tokens": 46, "pieces": ["'D're", "\r\n", "123", "456", "789", "㍿t", "‍.​<|", "endoftext", "|>!", "ß", "㍿<", "EOT", ">\r\n", "Dž", "<|", "fim", "_prefix", "|>㋿", "ḍ̇́'M", "…"]} +{"text": "t́e9　\t'VE漢ꟲ12345678　Džİ'ZZ\rع𐞁é.😀🏽İ​\u000b­(0'\rZ…m!!EOT　́😀🏽", "tokens": 53, "pieces": ["t́e", "9", "　", "\t", "'VE漢ꟲ", "123", "456", "78", "　Džİ", "'ZZ", "\r", "ع𐞁é", ".😀🏽", "İ", "​", "\u000b", "­(", "0", "'\r", "Z", "…m", "!!", "EOT", "　́", "😀🏽"]} +{"text": "EOT٣٤٥٦!!<|fim_prefix|>fifiaß́\n'VE𐞁\n­🙂 \n(t㋿'D\n\r\né", "tokens": 38, "pieces": ["EOT", "٣٤٥", "٦", "!!<|", "fim", "_prefix", "|>", "fifiaß́", "\n", "'VE𐞁", "\n", "­🙂", " \n", "(t", "㋿'", "D", "\n\r\n", "é"]} +{"text": "! \n…#$%\r\nſ😀🏽३\"́!…😀🏽́é‍#$%é.ém'll'S", "tokens": 37, "pieces": ["!", " \n", "…", "#$%<", "META", "_START", ">\r\n", "ſ", "😀🏽", "३", "\"́", "!", "…", "😀🏽́", "é", "‍#$%", "é", ".ém'll", "'S"]} +{"text": "İ", "tokens": 1, "pieces": ["İ"]} +{"text": "'Md३🙂'llİ🙂\"!　‍Dž<½\r\n\r\n'ſ­s漢!! ع 12345678\"½\n👍🏽\r\n३½.𐞁३", "tokens": 45, "pieces": ["'Md", "३", "🙂'", "ll", "İ", "🙂<", "META", "_START", ">\"!", "　", "‍Dž", "<", "½", "\r\n\r\n", "'ſ", "­s漢", "!!", " ع", " ", "123", "456", "78", "\"", "½", "\n", "👍🏽\r\n", "३½", ".𐞁", "३"]} +{"text": "\u000b(-<|endoftext|>.३　👍🏽#$%'VE'res\r\n", "tokens": 20, "pieces": ["\u000b", "(-<|", "endoftext", "|>.", "३", "　", "👍🏽#$%'", "VE're", "s", "\r\n"]} +{"text": "🙂 \n😀🏽<|endoftext|>​('🙂mt'sm\"ع́é­m'S9!㍿𐞁é'ßＡm'ſ'S½'ſ
", "tokens": 53, "pieces": ["🙂", " \n", "😀🏽<|", "endoftext", "|>​('🙂", "m", "t's", "m", "\"ع́é", "­m'S", "9", "!㍿", "𐞁é", "'ß", "Ａm'ſ", "'S", "½", "'ſ", "
"]} +{"text": "-,漢<|endoftext|>漢\r\n𐞁<|fim_prefix|>𐞁𐞁<|endoftext|>​ع'D㋿'ReⅣ-\"'ſ'Z'll​'Tå　 ​Dž", "tokens": 69, "pieces": ["<|", "endoftext", "|>", "漢", "\r\n", "𐞁", "<|", "fim", "_prefix", "|>", "𐞁𐞁", "<|", "endoftext", "|>​", "ع'D", "㋿'", "Re", "Ⅳ", "-\"'", "ſ", "'Z'll", "​'", "Tå", "　 ", " ​", "Dž", ""]} +{"text": "ع漢EOT३<|fim_prefix|>ع'llİ 　é<|endoftext|>sß!!🙂㋿- ꟲå0ع'Dfi३>٣٤٥٦ 'S\r\n\r\n\rfi", "tokens": 54, "pieces": ["ع漢", "EOT", "३", "<|", "fim", "_prefix", "|>", "ع'll", "İ", " ", "　é", "<|", "endoftext", "|>", "sß", "!!🙂㋿-", " ꟲå", "0", "ع'D", "fi", "३", ">", "٣٤٥", "٦", " ", "'S", "\r\n\r\n\r", "fi"]} +{"text": "'ſEOT\u000b​t́,漢字,ḍ̇🙂d'T'llİḍ̇‍'VE😀🏽12345678s漢٣٤٥٦漢İ 　", "tokens": 42, "pieces": ["'ſ", "EOT", "\u000b", "​t́", ",漢字", ",ḍ̇", "🙂d'T", "'ll", "İḍ̇", "‍'", "VE", "😀🏽", "123", "456", "78", "s漢", "٣٤٥", "٦", "漢", "İ", " 　"]} +{"text": "Z'Så(ḍ̇'VEe‍", "tokens": 12, "pieces": ["Z'S", "å", "(ḍ̇'VE", "e", "‍"]} +{"text": "sßꟲsa\u000baſtع㍿́ع३漢\r\n\tEOTßſé\"­\u000bع'S ́ß", "tokens": 37, "pieces": ["sßꟲsa", "\u000baſtع", "㍿́ع", "३", "漢", "\r\n", "\tEOTß", "ſé", "\"­", "\u000bع'S", " ́ß"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Džd३\"İ12345678'll字'llfi\r\ńé३Dž'VE'VE.a<'lldt", "tokens": 31, "pieces": ["Džd", "३", "\"İ", "123", "456", "78", "'ll字'll", "fi", "\r\n", "́é", "३", "Dž'VE", "'VE", ".a", "<'", "lldt"]} +{"text": "'S <|endoftext|>é'S𐞁<|endoftext|>​ſ…­　''re", "tokens": 31, "pieces": ["'S", " ", " <|", "endoftext", "|>", "é'S", "𐞁", "<|", "endoftext", "|>​", "ſ", "…", "­", "　", "''", "re"]} +{"text": "'s\t'ſZ \"​d👍🏽>!><İſ𐞁'İ<|fim_prefix|>'D-　🙂.'ll'S𐞁sEOT😀🏽dEOT'M　Z", "tokens": 52, "pieces": ["'s", "\t", "'ſ", "Z", " ", "\"​", "d", "👍🏽>!><", "İſ𐞁", "'İ", "<|", "fim", "_prefix", "|>'", "D", "-", "　", "🙂.'", "ll'S", "𐞁s", "EOT", "😀🏽", "d", "EOT'M", "　Z"]} +{"text": "　\u000be,ſs😀🏽字\"ع<|fim_prefix|>'re… \ns\r\r\n ", "tokens": 28, "pieces": ["　", "\u000be", ",ſ", "s", "😀🏽", "字", "\"ع", "<|", "fim", "_prefix", "|>'", "re", "… \n", "s", "\r\r\n", " "]} +{"text": " \t<<|fim_prefix|> \n>㋿ḍ̇½t🙂'T👍🏽\t<|endoftext|>\r\n\r\n 
🙂", "tokens": 35, "pieces": [" ", "\t", "<<|", "fim", "_prefix", "|>", " \n", ">㋿", "ḍ̇", "½", "t", "🙂'", "T", "👍🏽", "\t", "<|", "endoftext", "|>\r\n\r\n", " ", "
", "🙂"]} +{"text": " 0\t's'M<,👍🏽'ſ\t's…👍🏽0", "tokens": 21, "pieces": [" ", " ", "0", "\t", "'s'M", "<,👍🏽'", "ſ", "\t", "'s", "…", "👍🏽", "0"]} +{"text": "ḍ̇\nİ12345678३㋿", "tokens": 12, "pieces": ["ḍ̇", "\n", "İ", "123", "456", "78३", "㋿"]} +{"text": "'M　9<|endoftext|>fi('𐞁a<𐞁😀🏽½½a-'VE\tm\r é　½𐞁ⅣDžéd<|fim_prefix|>…\t'sZİع
Ⅳ", "tokens": 68, "pieces": ["'M", "　", "9", "<|", "endoftext", "|>", "fi", "('", "𐞁a", "<𐞁", "😀🏽", "½½", "a", "-'", "VE", "\tm", "\r", " ", " é", "　", "½", "𐞁", "Ⅳ", "Džéd", "<|", "fim", "_prefix", "|>", "…", "\t", "'s", "Z", "İع", "
", "Ⅳ"]} +{"text": "
m \n!!ſⅣ‍ß𐞁's#$%ع-d'́\t!!<|endoftext|>aDžfi㍿😀🏽#$%#$% ḍ̇‍,", "tokens": 49, "pieces": ["
m", " \n", "!!", "ſ", "Ⅳ", "‍ß𐞁's", "#$%", "ع", "-d", "'́", "\t", "!!<|", "endoftext", "|>", "a", "Džfi", "㍿😀🏽#$%#$%", " ", " ḍ̇", "‍,"]} +{"text": "́a-　३ḍ̇'reꟲ㋿> 'll字ع😀🏽'M\t-'M\r", "tokens": 33, "pieces": ["́a", "-<", "EOT", ">", "　", "३", "ḍ̇'re", "ꟲ", "㋿>", " '", "ll字ع", "😀🏽'", "M", "\t", "-'", "M", "\r"]} +{"text": "'S
('S🙂Z<'VE tm", "tokens": 10, "pieces": ["'S", "
", "('", "S", "🙂Z", "<'", "VE", " ", " tm"]} +{"text": "-0! \u000b'M0EOT(t'Re　e𐞁>dEOT\"​eꟲmm٣٤٥٦!!ḍ̇٣٤٥٦👍🏽12345678", "tokens": 48, "pieces": ["-", "0", "!", " ", "\u000b", "'M", "0", "EOT", "(t'Re", "　e", "𐞁", ">d", "EOT", "\"​", "eꟲmm", "٣٤٥", "٦", "!!", "ḍ̇", "٣٤٥", "٦", "👍🏽", "123", "456", "78"]} +{"text": "٣٤٥٦́a9><Ａ​\t㍿Ａ!!'<fi>é,0㋿漢Ⅳ'Re‍å字m🙂 'VE", "tokens": 39, "pieces": ["٣٤٥", "٦", "́a", "9", "><", "Ａ", "​", "\t", "㍿Ａ", "!!'<", "fi", ">é", ",", "0", "㋿漢", "Ⅳ", "'Re", "‍å字m", "🙂", " ", " '", "VE"]} +{"text": "ع", "tokens": 1, "pieces": ["ع"]} +{"text": " \n0㍿𐞁Dž㋿é字'Re\tḍ̇\r\nꟲ\r　'S'D½", "tokens": 37, "pieces": [" \n", "0", "㍿𐞁", "Dž", "㋿é字'Re", "\tḍ̇", "\r\n", "ꟲ", "\r", "　", "'S'D", "½", ""]} +{"text": "å'S dfi㍿fi　'0Z-'re३
Ａ'ſ<|endoftext|>Ⅳeå…(ſ\r\nm\r\n\r\n<|endoftext|>Dž漢عḍ̇\r\n\r\n-", "tokens": 64, "pieces": ["å'S", "", " ", " dfi", "㍿fi", "　", "'", "0", "Z", "-<", "EOT", ">'", "re", "३", "
Ａ", "'", "ſ", "<|", "endoftext", "|>", "Ⅳ", "eå", "…", "(ſ", "\r\n", "m", "\r\n\r\n", "<|", "endoftext", "|>", "Dž漢عḍ̇", "\r\n\r\n", "-"]} +{"text": "<'M'M!!(ꟲ½12345678!İ🙂'VEDž३#$%\"㍿'D 'Re>må'D\t
漢<|endoftext|><|endoftext|>'s<|fim_prefix|>'T", "tokens": 57, "pieces": ["<'", "M'M", "!!(", "ꟲ", "½12", "345", "678", "!İ", "🙂'", "VEDž", "३", "#$%\"㍿'", "D", " '", "Re", ">må'D", "\t", "
漢", "<|", "endoftext", "|><|", "endoftext", "|>'", "s", "<|", "fim", "_prefix", "|>'", "T"]} +{"text": "Ⅳ'M.'Tḍ̇­-'ſt㍿㋿é­ḍ̇0'D9#$%<|fim_prefix|>\r\n\r\nEOT'Re 'Re😀🏽,dꟲ𐞁#$%é", "tokens": 58, "pieces": ["Ⅳ", "'", "M", ".'", "Tḍ̇", "­-'", "ſt", "㍿㋿", "é", "­ḍ̇", "0", "'D", "9", "#$%<|", "fim", "_prefix", "|>\r\n\r\n", "EOT'Re", " '", "Re", "😀🏽,", "dꟲ𐞁", "#$%", "é"]} +{"text": "'Mİ åＡ\r\n㋿'ſm \n\t>👍🏽!!!ḍ̇İ9𐞁12345678​EOT 字漢fi'ſ9३½ !!\nꟲ", "tokens": 52, "pieces": ["'Mİ", " ", " å", "Ａ", "\r\n", "㋿'", "ſm", " \n", "\t", ">👍🏽!!!", "ḍ̇", "İ", "9", "𐞁", "123", "456", "78", "​EOT", " 字漢fi'ſ", "9३½", " ", " !!\n", "ꟲ"]} +{"text": "'Re　- 𐞁å'ſꟲeå\r\n's　🙂­ع🙂ع\r\n\r\nꟲ#$%
>ḍ̇é", "tokens": 37, "pieces": ["'Re", "　", "-", " 𐞁å'ſ", "ꟲeå", "\r\n", "'s", "　", "🙂­", "ع", "🙂ع", "\r\n\r\n", "ꟲ", "#$%", "
", ">ḍ̇é"]} +{"text": "­(- sådＡ😀🏽éDž#$%<|endoftext|>字", "tokens": 23, "pieces": ["­(-", " såd", "Ａ", "😀🏽", "é", "Dž", "#$%<|", "endoftext", "|>", "字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n\r\n-'Sİe \n'MⅣ́👍🏽‍字9🙂漢fi're­a12345678字'M<|endoftext|>\r\n\r\n漢 …,\"Džß", "tokens": 52, "pieces": ["\r\n\r\n", "-'", "Sİe", " \n", "'M", "", "Ⅳ", "́", "👍🏽‍", "字", "9", "🙂<", "EOT", ">漢fi're", "­a", "123", "456", "78", "字'M", "<|", "endoftext", "|>\r\n\r\n", "漢", " ", "…", ",\"", "Džß"]} +{"text": "İs㋿‍fí👍🏽9‍>", "tokens": 14, "pieces": ["İs", "㋿‍", "fí", "👍🏽", "9", "‍>"]} +{"text": "ḍ̇㋿é'ſ'ſ'D12345678 ß‍!", "tokens": 23, "pieces": ["ḍ̇", "㋿é", "'", "ſ'ſ", "'D", "123", "456", "78", " ", " ß", "‍!"]} +{"text": "Z.'ſ (
sd'ſ's\r\n\r\n\r<👍🏽'Re'D>'Re\n\r\n\r\nZ 'Res‍
\t.ع", "tokens": 31, "pieces": ["Z", ".'", "ſ", " (", "
sd'ſ", "'s", "\r\n\r\n\r", "<👍🏽'", "Re'D", ">'", "Re", "\n\r\n\r\n", "Z", " ", " '", "Res", "‍", "
", "\t", ".ع"]} +{"text": "\r\n\r\n\r\n\r\nＡd\t'Sع ", "tokens": 7, "pieces": ["\r\n\r\n\r\n\r\n", "Ａd", "\t", "'Sع", " "]} +{"text": "'ſꟲ​seaع#$%fi'll0'ſZꟲ<|fim_prefix|>9<ꟲ'D'D", "tokens": 32, "pieces": ["'ſꟲ", "​seaع", "#$%", "fi'll", "0", "'ſ", "Zꟲ", "<|", "fim", "_prefix", "|>", "9", "<ꟲ'D", "'D"]} +{"text": "éß'll'T㋿　\r\n\r\n\n　<|fim_prefix|>‍́<|endoftext|>㍿ꟲ\"'字\t ­👍🏽\t'Re'D‍ \nß", "tokens": 53, "pieces": ["éß'll", "'T", "㋿", "　\r\n\r\n\n", "　", "<|", "fim", "_prefix", "|>‍́<|", "endoftext", "|>㍿", "ꟲ", "\"'<", "EOT", ">字", "\t", " ­👍🏽", "\t", "'Re", "'", "D", "‍", " \n", "ß"]} +{"text": "0ſ9Ａß\rꟲéⅣ\r㋿\u000bß'Reé\t­<|endoftext|>9\r\n 's'­'M", "tokens": 44, "pieces": ["0", "ſ", "9", "Ａß", "\r", "ꟲé", "<", "EOT", ">", "Ⅳ", "\r", "㋿", "\u000bß'Re", "é", "\t", "­<|", "endoftext", "|>", "9", "\r\n", " ", "'s", "'­'", "M"]} +{"text": "12345678​\r\n漢 \n'Reḍ̇9EOT('\r\n\r\nＡ'Re.ḍ̇　　　ſ9'D𐞁字\"e🙂12345678ꟲ'reſe३㋿!!<|fim_prefix|>,å", "tokens": 60, "pieces": ["123", "456", "78", "​\r\n", "漢", " \n", "'Reḍ̇", "9", "EOT", "('\r\n\r\n", "Ａ'Re", ".ḍ̇", "　　", "　ſ", "", "9", "'D𐞁字", "\"e", "🙂", "123", "456", "78", "ꟲ're", "ſe", "३", "㋿!!<|", "fim", "_prefix", "|>,", "å"]} +{"text": "३<|endoftext|>!́𐞁३漢#$% 漢>
a9‍", "tokens": 26, "pieces": ["३", "<|", "endoftext", "|>!́", "𐞁", "३", "漢", "#$%", " ", " 漢", ">", "
a", "9", "‍"]} +{"text": "­'Re-…de!!9…'T'D\n\r\néⅣm🙂\ts\ta <'ll", "tokens": 24, "pieces": ["­'", "Re", "-", "…de", "!!", "9", "…", "'T'D", "\n\r\n", "é", "Ⅳ", "m", "🙂", "\ts", "\ta", " <'", "ll"]} +{"text": "å\u000b́٣٤٥٦<|fim_prefix|>\"", "tokens": 38, "pieces": ["å", "", "\u000b́", "", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\""]} +{"text": "ſ'ſ> d'ſ …İ'ſm's#$%. 9 ", "tokens": 21, "pieces": ["ſ'ſ", ">", " ", " d'ſ", " ", "…İ'ſ", "m's", "#$%.", " ", "9", " "]} +{"text": "('M🙂\r½'ſdEOTDžＡfi　<#$%", "tokens": 43, "pieces": ["('", "M", "🙂\r", "½", "'ſd", "EOTDžＡ", "", "fi", "　", "<#$%"]} +{"text": " \nd½!'ReDžİ'VEd٣٤٥٦🙂e \n\n'Tß　'S
dDž", "tokens": 29, "pieces": [" \n", "d", "½", "!'", "Re", "Džİ'VE", "d", "٣٤٥", "٦", "🙂e", " \n\n", "'Tß", "　", "'S", "
", "d", "Dž"]} +{"text": ">\u000b㍿#$%'fi\t'll!!é9>,­'VE'​'Re  'll(\u000b'S字👍🏽\n \n're٣٤٥٦ !!\r ع", "tokens": 47, "pieces": [">", "\u000b", "㍿#$%'", "fi", "\t", "'ll", "!!", "é", "9", ">,­'", "VE", "'​'", "Re", "  ", " '", "ll", "(", "\u000b", "'S", "字", "👍🏽\n", " \n", "'re", "٣٤٥", "٦", " ", "!!\r", " ع"]} +{"text": "­३(!", "tokens": 3, "pieces": ["­", "३", "(!"]} +{"text": " t　‍Ⅳa'D'reåd㋿İdZ…½Ⅳ\nعꟲ\td!!", "tokens": 33, "pieces": [" t", "　", "‍", "Ⅳ", "a'D", "'reåd", "㋿İd", "Z", "…", "½", "", "Ⅳ", "\n", "عꟲ", "\td", "!!"]} +{"text": "…a's<'Mع😀🏽'M…9ꟲ'ſ\"Ａ\ta\u000bḍ̇('T-12345678e \n 🙂<|endoftext|>́'D", "tokens": 50, "pieces": ["…a's", "<'", "Mع", "😀🏽'", "M", "…", "9", "ꟲ'ſ", "\"Ａ", "\ta", "\u000bḍ̇", "(<", "META", "_START", ">'", "T", "-", "123", "456", "78", "e", " \n", " ", "🙂<|", "endoftext", "|>́'", "D"]} +{"text": "d½t'll㋿\r\n
>字字mع,12345678d'Sé३'VE!!><|endoftext|>EOT́å½12345678'Ａ,Ａ­½\r\n👍🏽é", "tokens": 59, "pieces": ["d", "½", "t'll", "㋿\r\n", "
", ">字字mع", ",", "123", "456", "78", "d'S", "é", "३", "'", "VE", "!!><|", "endoftext", "|>", "EOT́å", "½12", "345", "678", "'Ａ", ",Ａ", "­", "½", "\r\n", "👍🏽", "é", ""]} +{"text": "'ſDž…㍿#$%>'re,fi \ńß ( \n漢#$%.", "tokens": 24, "pieces": ["'ſ", "Dž", "…", "㍿#$%>'", "re", ",fi", " \n", "́ß", " ", " (", " \n", "漢", "#$%."]} +{"text": "e ३عm('s漢 \n \né漢m#$%Ａé'Sع12345678e\"ſ३'ll😀🏽Z'Re>👍🏽३…­\r\n\r\n😀🏽", "tokens": 49, "pieces": ["e", "", " ", " ", "३", "عm", "('", "s漢", " \n \n", "é漢m", "#$%", "Ａé'S", "ع", "123", "456", "78", "e", "\"ſ", "३", "'ll", "😀🏽", "Z'Re", ">👍🏽", "३", "…", "­\r\n\r\n", "😀🏽"]} +{"text": "\r\n\r\n'ſ\r\n\r\n!👍🏽\t \n㋿#$%.å\r\n're漢\"! ", "tokens": 27, "pieces": ["\r\n\r\n", "'ſ", "\r\n\r\n", "!👍🏽", "\t \n", "㋿#$%.", "å", "\r\n", "'re漢", "\"!", " "]} +{"text": "​fi漢.\"ع\nعa'lls'ſ", "tokens": 12, "pieces": ["​fi漢", ".\"", "ع", "\n", "عa'll", "s'ſ"]} +{"text": "\r'S'D　a字fi­<9EOT½…>d,\r😀🏽٣٤٥٦😀🏽'M0ḿ𐞁med's𐞁​", "tokens": 44, "pieces": ["\r", "'S'D", "　a字fi", "­<", "9", "EOT", "½", "…", ">d", ",\r", "😀🏽", "٣٤٥", "٦", "😀🏽'", "M", "0", "ḿ𐞁med's", "𐞁", "​"]} +{"text": "٣٤٥٦'VE\r\n\r\n　\r\n\r\n\r\n\r\ń\r\n𐞁(́'ſ​३!!\u000b Z漢,ع　­'EOT́😀🏽Ａſ­Dž <|fim_prefix|> >fi", "tokens": 54, "pieces": ["٣٤٥", "٦", "'VE", "\r\n\r\n　\r\n\r\n\r\n\r\n", "́", "\r\n", "𐞁", "(́'ſ", "​", "३", "!!", "\u000b", " Z漢", ",ع", "　", "­'", "EOT́", "😀🏽", "Ａſ", "­Dž", " <|", "fim", "_prefix", "|>", " ", " >", "fi"]} +{"text": "(å \n‍(-12345678'D٣٤٥٦é<… İ　🙂字,٣٤٥٦EOT'VE,ſ>字'll'll<|endoftext|>fi", "tokens": 48, "pieces": ["(å", " \n", "‍(-", "123", "456", "78", "'D", "٣٤٥", "٦", "é", "<", "…", " İ", "　", "🙂字", ",", "٣٤٥", "٦", "EOT'VE", ",ſ", ">", "字'll", "'ll", "<|", "endoftext", "|>", "fi"]} +{"text": "𐞁👍🏽.㋿('Sfi\nd\r\n<|fim_prefix|>…ꟲ字(​ḍ̇<|fim_prefix|>\u000b𐞁e漢", "tokens": 50, "pieces": ["𐞁", "👍🏽.㋿('", "Sfi", "\n", "d", "\r\n", "<|", "fim", "_prefix", "|><", "EOT", ">", "…ꟲ字", "(​", "ḍ̇", "<|", "fim", "_prefix", "|>", "\u000b𐞁e漢"]} +{"text": "\u000b漢😀🏽\t \r\n\r\n'T!½🙂Ａ9,éfie🙂(\r\n\r\n\r\n!Ⅳ٣٤٥٦>a\u000b'­!!㍿", "tokens": 35, "pieces": ["\u000b漢", "😀🏽", "\t \r\n\r\n", "'T", "!", "½", "🙂Ａ", "9", ",éfie", "🙂(\r\n\r\n\r\n", "!", "Ⅳ٣٤", "٥٦", ">a", "\u000b", "'­!!㍿"]} +{"text": "<|fim_prefix|>🙂, 😀🏽<|endoftext|>　!!'llDž字㍿\u000b''M㋿'llⅣ#$%!! 漢 0", "tokens": 46, "pieces": ["<|", "fim", "_prefix", "|>🙂,", " ", " 😀🏽<|", "endoftext", "|>", "　", "!!'", "ll", "Dž字", "㍿", "\u000b", "''", "M", "㋿'", "ll", "Ⅳ", "#$%!!", " 漢", " ", "0"]} +{"text": "'T're#$%İ-!!Ź9'VE<|fim_prefix|>٣٤٥٦'ll,Ａs­d'ſ", "tokens": 32, "pieces": ["'T're", "#$%", "İ", "-!!", "Ź", "9", "'VE", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "'", "ll", ",Ａs", "­d'ſ"]} +{"text": "٣٤٥٦ß٣٤٥٦­d'Dž　é >''ſm\t　<|fim_prefix|>fi½\t'T \nſ d", "tokens": 36, "pieces": ["٣٤٥", "٦", "ß", "٣٤٥", "٦", "­d", "'Dž", "　é", " ", ">''", "ſm", "\t", "　", "<|", "fim", "_prefix", "|>", "fi", "½", "\t", "'T", " \n", "ſ", " d"]} +{"text": "'Dع12345678­ſ㋿<|fim_prefix|>㍿. ‍½ꟲs. ꟲ'M'llß(
<|endoftext|>\ts!
漢", "tokens": 54, "pieces": ["'Dع", "123", "456", "78", "­ſ", "㋿<|", "fim", "_prefix", "|>㍿.", " ‍", "½", "ꟲs", ".", " ꟲ'M", "'llß", "(", "
", "<|", "endoftext", "|>", "\t", "<", "EOT", ">s", "!", "
漢"]} +{"text": "\tß　​𐞁<|endoftext|> ㋿\rⅣ>'s\"->,'D
​
Ａ\n Ⅳ!!s\r\n🙂's👍🏽(漢áع", "tokens": 54, "pieces": ["\tß", "　", "​𐞁", "<|", "endoftext", "|>", " ", "㋿<", "EOT", ">\r", "Ⅳ", ">'", "s", "\"->,'", "D", "
", "​", "
Ａ", "\n", " ", "Ⅳ", "!!", "s", "\r\n", "🙂'", "s", "👍🏽(", "漢áع"]} +{"text": "㋿ e👍🏽s'Refi(👍🏽<|endoftext|>'S½><#$% \r\nＡ \r­🙂🙂
fi 0ꟲ'D 👍🏽(<|endoftext|>漢", "tokens": 58, "pieces": ["㋿", " ", "e", "👍🏽", "s'Re", "fi", "(👍🏽<|", "endoftext", "|>'", "S", "½", "><#$%", " \r\n", "Ａ", " \r", "­🙂🙂", "
fi", " ", "0", "ꟲ'D", " ", "👍🏽(<|", "endoftext", "|>", "漢"]} +{"text": "🙂㋿½", "tokens": 5, "pieces": ["🙂㋿", "½"]} +{"text": " 😀🏽㍿ 𐞁'llß\rßé12345678aع'S-\r's\r'#$%!\"eß𐞁", "tokens": 37, "pieces": [" ", " 😀🏽㍿", " 𐞁'll", "ß", "\r", "ßé", "123", "456", "78", "aع'S", "-\r", "'s", "\r", "'#$%!\"", "eß𐞁"]} +{"text": "‍㍿'Mſ㍿½ ​9s漢ß漢afi
ſå12345678\r\n\r\n­ꟲ…­ＡＡ字td漢​eع\"<|fim_prefix|>'ll", "tokens": 54, "pieces": ["‍㍿'", "Mſ", "㍿", "½", " ", " ​", "9", "s漢ß漢afi", "
ſå", "123", "456", "78", "\r\n\r\n", "­ꟲ", "…", "­ＡＡ字td漢", "​eع", "\"<|", "fim", "_prefix", "|>'", "ll"]} +{"text": "12345678'D́ſ\"عfi'llＡ ㋿'Me㋿'ſZع\r\n\r\n<
\"! ­'D\n é😀🏽ع字sé", "tokens": 44, "pieces": ["123", "456", "78", "'D́ſ", "\"عfi'll", "Ａ", " ㋿'", "Me", "㋿'", "ſ", "Zع", "\r\n\r\n", "<", "
", "\"!<", "EOT", ">", " ", " ­'", "D", "\n", " é", "😀🏽", "ع字sé"]} +{"text": "å…9㍿ḍ̇Ａ<|endoftext|>­
İꟲmEOT'Mİét'VE'VE🙂\r字ſ", "tokens": 41, "pieces": ["å", "…", "9", "㍿ḍ̇", "Ａ", "<|", "endoftext", "|>­", "
İꟲm", "EOT'M", "İét'VE", "'VE", "🙂\r", "字ſ"]} +{"text": "½fi­9½​m㋿😀🏽\r\nḍ̇#$%字𐞁'ſ#$%字Ａe \nZ", "tokens": 33, "pieces": ["½", "fi", "­", "9½", "​m", "㋿😀🏽\r\n", "ḍ̇", "#$%", "字𐞁'ſ", "#$%", "字Ａe", " \n", "Z"]} +{"text": "🙂sé<|fim_prefix|>'ll 'M漢!!!!Dž㍿٣٤٥٦　Zḍ̇<|endoftext|>'ſ <|endoftext|>Ａ,İ#$%", "tokens": 47, "pieces": ["🙂sé", "<|", "fim", "_prefix", "|>'", "ll", " '", "M漢", "!!!!", "Dž", "㍿", "٣٤٥", "٦", "　Zḍ̇", "<|", "endoftext", "|>'", "ſ", " <|", "endoftext", "|>", "Ａ", ",İ", "#$%"]} +{"text": "('Re'ſ'Re", "tokens": 8, "pieces": ["('", "Re'ſ", "'", "Re"]} +{"text": " å\r\n\r\n<ßfifié<|fim_prefix|>'reＡ'DžⅣ́!!#$%'<|fim_prefix|>ع㍿", "tokens": 40, "pieces": [" å", "\r\n\r\n", "<ßfifié", "<|", "fim", "_prefix", "|>'", "re", "Ａ", "'Dž", "Ⅳ", "́", "!!#$%'<|", "fim", "_prefix", "|><", "META", "_START", ">ع", "㍿"]} +{"text": "\r\n\r\n­
३\n字'ſ'T𐞁\u000b's\rsd<|endoftext|>fifiDž!'s ", "tokens": 33, "pieces": ["\r\n\r\n", "­", "
", "३", "\n", "字'ſ", "'T𐞁", "\u000b", "'s", "\r", "sd", "<|", "endoftext", "|>", "fifi", "Dž", "!'", "s", " "]} +{"text": "\u000b ,é\r\n\r\n\r 'S
😀🏽-'re𐞁\r\n\r\n\r\n\r\n", "tokens": 21, "pieces": ["\u000b ", " ,", "é", "\r\n\r\n\r", " ", " '", "S", "
", "😀🏽-'", "re𐞁", "\r\n\r\n\r\n\r\n"]} +{"text": "‍-ß
te👍🏽…d-́\r\n<|endoftext|>🙂!!\n<|endoftext|>fi\r\n\r\n​Ⅳ\n", "tokens": 39, "pieces": ["‍-", "ß", "
", "te", "👍🏽", "…d", "-́", "\r\n", "<|", "endoftext", "|>🙂!!\n", "<|", "endoftext", "|>", "fi", "\r\n\r\n", "​", "Ⅳ", "\n"]} +{"text": "\ntſ\t'll३e'ſ'T'',🙂0Dž'TsDž'llḍ̇'D३½👍🏽‍-", "tokens": 34, "pieces": ["\n", "tſ", "\t", "'ll", "३", "e'ſ", "'T", "'',🙂", "0", "Dž'T", "s", "Dž'll", "ḍ̇'D", "३", "", "½", "👍🏽‍-"]} +{"text": "m \t<㋿#$%́12345678🙂'S漢 \n's‍㍿'VE​ßs12345678t'SZ0,,9'ſ<|endoftext|>!12345678'ſ३🙂<|fim_prefix|><>Ａ", "tokens": 62, "pieces": ["m", " ", "\t", "<㋿#$%́", "123", "456", "78", "🙂'", "S漢", " \n", "'s", "‍㍿'", "VE", "​ßs", "123", "456", "78", "t'S", "Z", "0", ",,", "9", "'ſ", "<|", "endoftext", "|>!", "123", "456", "78", "'ſ", "३", "🙂<|", "fim", "_prefix", "|><>", "Ａ"]} +{"text": "aḍ̇ꟲ!!é(\n'Re ㋿!!>!'\u000bé\r\n\r\n'ſ \nＡ,é\néfi9 👍🏽,<👍🏽ꟲd'Tḍ̇", "tokens": 53, "pieces": ["aḍ̇ꟲ", "!!", "é", "(\n", "'Re", " ㋿!!>!<", "META", "_START", ">'", "\u000bé", "\r\n\r\n", "'ſ", " \n", "Ａ", ",é", "\n", "éfi", "9", " ", " 👍🏽,<👍🏽", "ꟲ", "d'T", "ḍ̇"]} +{"text": "'s漢0'D 𐞁.>漢….\"\r½'M'sé<|endoftext|>>12345678ع12345678𐞁", "tokens": 38, "pieces": ["'s漢", "0", "'D", " 𐞁", ".>", "漢", "…", ".\"\r", "½", "'M's", "é", "<|", "endoftext", "|>>", "123", "456", "78", "ع", "123", "456", "78", "𐞁"]} +{"text": "!!'S\"'ſfi.　'", "tokens": 12, "pieces": ["!!'", "S", "\"<", "META", "_START", ">'", "ſfi", ".", "　", "'"]} +{"text": "<ع's", "tokens": 3, "pieces": ["<ع's"]} +{"text": "㍿漢te
'D　m \n३<|endoftext|>ⅣDž,.३ \nß字ſß㍿''>9ع.", "tokens": 45, "pieces": ["㍿漢t", "e", "
", "'D", "　m", " \n", "३", "<|", "endoftext", "|>", "Ⅳ", "Dž", ",.", "३", " \n", "ß字ſß", "㍿''>", "9", "ع", "."]} +{"text": ">🙂😀🏽'Rees're( '12345678‍9​\"'D👍🏽
", "tokens": 34, "pieces": [">🙂😀🏽'", "Rees", "'", "re", "(", " '", "123", "456", "78", "‍", "9", "​\"'", "D", "👍🏽", "
"]} +{"text": " ३'Re\r\n\r9‍
ſ\r\nع\r\n\r\nå'T'Z𐞁'VE ­‍.0!é👍🏽fiEOT\nt🙂EOT><­ꟲ ", "tokens": 55, "pieces": [" ", " ", "३", "'Re", "\r\n\r", "9", "‍", "
", "ſ", "\r\n", "ع", "\r\n\r\n", "å'T", "'Z𐞁'VE", " ", " ­‍.", "0", "!", "é", "👍🏽", "fi", "EOT", "\n", "t", "🙂EOT", "><­", "ꟲ", " "]} +{"text": "Z'VEEOT<|endoftext|>㋿'s😀🏽9#$%😀🏽d　'Re((Džꟲ>< ع'\u000bé'VE㍿#$%'ReEOT​'T!! ३-'M", "tokens": 59, "pieces": ["Z'VE", "EOT", "<|", "endoftext", "|>㋿'", "s", "😀🏽", "9", "#$%😀🏽", "d", "　", "'Re", "((", "Džꟲ", "><", " ع", "'", "\u000bé'VE", "㍿#$%'", "Re", "EOT", "​'", "T", "!!", " ", "३", "-'", "M"]} +{"text": "mꟲ\"aé,ḍ̇'VE'Ma‍d👍🏽a…(字\rd'VE9é'Re<|endoftext|> mꟲ're‍½Džꟲ‍,'s 漢", "tokens": 56, "pieces": ["mꟲ", "\"aé", ",ḍ̇'VE", "'Ma", "‍d", "👍🏽", "a", "…", "(字", "\r", "d'VE", "9", "é'Re", "<|", "endoftext", "|>", " mꟲ're", "‍", "½", "Džꟲ", "‍,'", "s", " 漢"]} +{"text": "t<|fim_prefix|>9㍿½EOTå.Z'T‍漢Ⅳ'ſe𐞁", "tokens": 33, "pieces": ["t", "<|", "fim", "_prefix", "|>", "9", "㍿", "½", "EOTå", ".", "Z'T", "‍漢", "Ⅳ", "'ſe𐞁"]} +{"text": "\r\n\r\n0'S,!!aEOTéå're\r\n\r\n", "tokens": 14, "pieces": ["\r\n\r\n", "0", "'S", ",!!", "a", "EOTéå're", "\r\n\r\n"]} +{"text": "٣٤٥٦ \néåfiع🙂𐞁>'Reḍ̇'M'lleＡ'llＡ", "tokens": 29, "pieces": ["٣٤٥", "٦", " \n", "éåfi", "ع", "🙂𐞁", ">'", "Reḍ̇'M", "'lle", "Ａ'll", "Ａ"]} +{"text": " \n­\u000b­!　ḍ̇​!Dž\"𐞁!!12345678­\u000bs٣٤٥٦é'så㍿\r\ńZ…", "tokens": 42, "pieces": [" \n", "­", "\u000b", "­!", "　ḍ̇", "​!", "Dž", "\"𐞁", "!!", "123", "456", "78", "­", "\u000bs", "٣٤٥", "٦", "é's", "å", "㍿\r\n", "́", "Z", "…"]} +{"text": "'s<|fim_prefix|>İ!漢<|fim_prefix|>(ꟲ'reé å𐞁", "tokens": 29, "pieces": ["'s", "<|", "fim", "_prefix", "|>", "İ", "!漢", "<|", "fim", "_prefix", "|>(", "ꟲ're", "é", " ", " å𐞁"]} +{"text": " 😀🏽<|endoftext|>‍👍🏽\".>\r\n𐞁Dž\n", "tokens": 27, "pieces": [" ", " 😀🏽<|", "endoftext", "|>‍👍🏽\".>\r\n", "𐞁", "Dž", "\n"]} +{"text": "\r\nḍ̇\r\n\r\nＡ'D\n….­", "tokens": 12, "pieces": ["\r\n", "ḍ̇", "\r\n\r\n", "Ａ'D", "\n", "…", ".­"]} +{"text": "'D ㍿'refi!!<|fim_prefix|>\n👍🏽…𐞁…'re0<|endoftext|>>𐞁'VE‍́ \n'll", "tokens": 46, "pieces": ["'D", " ", " ㍿'", "refi", "!!<|", "fim", "_prefix", "|>\n", "👍🏽", "…𐞁", "…", "'re", "0", "<|", "endoftext", "|>>", "𐞁'VE", "‍́", " \n", "'ll"]} +{"text": "\rå", "tokens": 3, "pieces": ["\r", "å"]} +{"text": "㍿İ'ſ#$%́\r\n\r\n\r\n\r\n\u000b字Z
.a'llEOT 9 <'D\"!é'D'Sm'S‍Z'Dꟲİa#$%<|fim_prefix|> \n", "tokens": 52, "pieces": ["㍿İ'ſ", "#$%́\r\n\r\n\r\n\r\n", "\u000b字", "Z", "
", ".a'll", "EOT", " ", " ", "9", " ", "<'", "D", "\"!", "é'D", "'Sm", "'", "S", "‍Z'D", "ꟲİa", "#$%<|", "fim", "_prefix", "|>", " \n"]} +{"text": "'re>'ſ<|fim_prefix|>́
é ", "tokens": 14, "pieces": ["'re", ">'", "ſ", "<|", "fim", "_prefix", "|>́", "
é", " "]} +{"text": "\r\n\r\n'Td(İ'T>dꟲ\r\nm", "tokens": 12, "pieces": ["\r\n\r\n", "'Td", "(İ'T", ">dꟲ", "\r\n", "m"]} +{"text": ".'Re're'S'Ss'ſＡⅣé٣٤٥٦#$%
!-㍿'D 'VEa's㋿'De\"
 　", "tokens": 44, "pieces": [".'", "Re're", "'S'S", "s'ſ", "Ａ", "Ⅳ", "é", "٣٤٥", "٦", "#$%", "
", "!-㍿'", "D", " ", "'VEa's", "㋿'", "D", "e", "\"", "
 　"]} +{"text": "३Ａ#$%0m", "tokens": 10, "pieces": ["३", "Ａ", "#$%", "0", "m"]} +{"text": ",\u000b-'ll \n'𐞁Ａée'll9fi㋿Dž​\rs字", "tokens": 31, "pieces": [",", "\u000b", "-'", "ll", " \n", "'<", "EOT", ">𐞁Ａée'll", "9", "fi", "㋿", "Dž", "​\r", "s字"]} +{"text": "é<|fim_prefix|>\r\n\r\n㍿　‍é", "tokens": 17, "pieces": ["é", "<|", "fim", "_prefix", "|>\r\n\r\n", "㍿", "　", "‍é"]} +{"text": "<|endoftext|>", "tokens": 7, "pieces": ["<|", "endoftext", "|>"]} +{"text": "ſ عfi\r‍-EOT🙂‍'S
 ­'ꟲ#$%\re\"ꟲ#$%<|endoftext|>👍🏽\"09
Ⅳ <|endoftext|>>🙂é", "tokens": 65, "pieces": ["ſ", " عfi", "\r", "‍-", "EOT", "🙂‍'", "S", "
 ", " ­'", "ꟲ", "#$%\r", "e", "\"<", "META", "_START", ">ꟲ", "#$%<|", "endoftext", "|><", "META", "_START", ">👍🏽\"", "09", "
", "Ⅳ", " ", "<|", "endoftext", "|>><", "EOT", ">🙂", "é"]} +{"text": "㋿fi\tDž \n'Tḍ̇éa🙂 \n!!ß😀🏽\r<'reŹ0​\u000b­efi#$%'Sa'S٣٤٥٦
d", "tokens": 43, "pieces": ["㋿fi", "\tDž", " \n", "'Tḍ̇éa", "🙂", " \n", "!!", "ß", "😀🏽\r", "<'", "re", "Ź", "0", "​", "\u000b", "­efi", "#$%'", "Sa'S", "٣٤٥", "٦", "
d"]} +{"text": "0a'såEOTEOTḍ̇'Tß's'VE'D,<|fim_prefix|>…'Dž'S<\r…fim👍🏽ḍ̇३", "tokens": 42, "pieces": ["0", "a's", "å", "EOTEOTḍ̇'T", "ß's", "'VE'D", ",<|", "fim", "_prefix", "|>", "…", "'Dž'S", "<\r", "…fim", "👍🏽", "ḍ̇", "३"]} +{"text": "㍿!!Ａ'D!'ſ\n٣٤٥٦‍é", "tokens": 16, "pieces": ["㍿!!", "Ａ'D", "!'", "ſ", "\n", "٣٤٥", "٦", "‍é"]} +{"text": "m😀🏽½'T𐞁​ß🙂 \n,ß­ \n(́'ſDž­३'EOT>'ſ\rع'M㋿tfiZ'S🙂 ", "tokens": 46, "pieces": ["m", "😀🏽", "½", "'T𐞁", "​<", "EOT", ">ß", "🙂", " \n", ",ß", "­", " \n", "(́'ſ", "Dž", "­", "३", "'EOT", ">'", "ſ", "\r", "ع'M", "㋿tfi", "Z'S", "🙂", " "]} +{"text": "åé ½ é\t#$%\r\n'D're'D('st　å<|fim_prefix|>t'Td
d \n,㍿'s9'Mḍ̇", "tokens": 45, "pieces": ["åé", " ", "½", " ", " é", "\t", "#$%\r\n", "'D're", "'D", "('", "st", "　å", "<|", "fim", "_prefix", "|>", "t'T", "d", "
d", " \n", ",㍿'", "s", "", "9", "'Mḍ̇"]} +{"text": "<ꟲ<|endoftext|>å
'll'ſé字d<|endoftext|>\neeḍ̇\r\n\r\n\r\t\t　\"\"td \n're", "tokens": 39, "pieces": ["<ꟲ", "<|", "endoftext", "|>", "å", "
", "'ll'ſ", "é字d", "<|", "endoftext", "|>\n", "eeḍ̇", "\r\n\r\n\r", "\t\t", "　", "\"\"", "td", " \n", "'re"]} +{"text": ",‍e😀🏽
('M٣٤٥٦㋿fi㋿", "fi", ",fi👍🏽…e'Reع 'ſ'Re Ⅳ'M'S ", "tokens": 32, "pieces": [".漢", "
d", "㋿🙂<", "META", "_START", ">,", "fi", "👍🏽", "…e'Re", "ع", " ", "'ſ'Re", " ", " ", "Ⅳ", "'M'S", " "]} +{"text": "EOT d\r ſ‍
d३ \n㋿12345678Zé", "tokens": 20, "pieces": ["EOT", " d", "\r", " ſ", "‍", "
d", "३", " \n", "㋿", "123", "456", "78", "Zé"]} +{"text": "-ＡEOT!!fi \n🙂<漢,½३é字'ś 字'ſtsd漢12345678", "tokens": 27, "pieces": ["-ＡEOT", "!!", "fi", " \n", "🙂<", "漢", ",", "½३", "é字's", "́", " 字'ſ", "tsd漢", "123", "456", "78"]} +{"text": "​(fi#$%­\n\r<|endoftext|><\r\n'VE漢　𐞁٣٤٥٦mDž'ſ́'TEOT<|fim_prefix|>ſ", "tokens": 46, "pieces": ["​(", "fi", "#$%­\n\r", "<|", "endoftext", "|><\r\n", "'VE漢", "　𐞁", "٣٤٥", "٦", "m", "Dž'ſ", "́'T", "EOT", "<|", "fim", "_prefix", "|>", "ſ"]} +{"text": "9字٣٤٥٦t٣٤٥٦ ­​\r\n३", "tokens": 16, "pieces": ["9", "字", "٣٤٥", "٦", "t", "٣٤٥", "٦", " ", "­​\r\n", "३"]} +{"text": "ع'M!½<|fim_prefix|>EOTeꟲ \nſ!<👍🏽½३a!", "tokens": 29, "pieces": ["ع'M", "!", "½", "<|", "fim", "_prefix", "|><", "EOT", ">EOTeꟲ", " \n", "ſ", "!<👍🏽", "½३", "a", "!"]} +{"text": "'re!!İ٣٤٥٦İ( ,\tZ#$%
İ😀🏽…İ.'Re　'ReEOT#$%'T#$%字<|fim_prefix|>m­İ", "tokens": 48, "pieces": ["'re", "!!", "İ", "٣٤٥", "٦", "İ", "(", " ", ",", "\tZ", "#$%", "
İ", "😀🏽", "…İ", ".'", "Re", "　", "'Re", "EOT", "#$%'", "T", "#$%", "字", "<|", "fim", "_prefix", "|><", "EOT", ">m", "­İ"]} +{"text": "s\u000b‍'re 'ſfi0ma.​9́\t.\n字'Ré'M'll'Sİ,
EOT…👍🏽\t'VE​s,", "tokens": 47, "pieces": ["٣٤٥", "٦", "ß", "<|", "fim", "_prefix", "|>", " '", "ſfi", "0", "ma", ".​", "9", "́", "\t", ".\n", "字'Re", "́'M", "'", "ll'S", "İ", ",", "
EOT", "…", "👍🏽", "\t", "'VE", "​s", ","]} +{"text": "'S\"Ⅳ.\ta३ Ａ(㋿'VEß!!é'VE😀🏽-'VE٣٤٥٦🙂　e'VE'\t0ß👍🏽字#$% \r\n", "tokens": 50, "pieces": ["'S", "\"", "Ⅳ", ".", "\ta", "३", " Ａ", "(㋿'", "VEß", "!!", "é'VE", "😀🏽-'", "VE", "٣٤٥", "٦", "🙂", "　e'VE", "'", "\t", "0", "ß", "👍🏽", "字", "#$%", " \r\n"]} +{"text": "12345678ſ​ #$%ꟲ !'VEm", "tokens": 16, "pieces": ["123", "456", "78", "ſ", "​", " ", " #$%", "ꟲ", " !'", "VEm"]} +{"text": "!　 \n字'Ａ 'ſ\r\nå", "tokens": 54, "pieces": ["!", "　 \n", "字", "'<", "META", "_START", ">Ａ", "", " ", " '", "ſ", "\r\n", "å"]} +{"text": "'ſ\tꟲ𐞁m9\"…\u000b­‍éع'reⅣ,‍٣٤٥٦('sDžZ'm👍🏽👍🏽", "tokens": 51, "pieces": ["'ſ", "\tꟲ𐞁m", "9", "\"", "…", "\u000b", "­<", "META", "_START", ">‍", "éع", "'", "re", "Ⅳ", ",<", "EOT", ">‍", "٣٤٥", "٦", "('", "s", "DžZ'm", "👍🏽👍🏽"]} +{"text": "'sＡ 'T'sfi9­'Re ḍ̇e'M­\r\"　EOT\u000btDž", "tokens": 34, "pieces": ["'s", "Ａ", " <", "EOT", ">'", "T's", "fi", "9", "­'", "Re", " ", " ḍ̇e'M", "­\r", "\"", "　EOT", "\u000bt", "Dž", ""]} +{"text": "'S'VE😀🏽'D,½㍿a <<|endoftext|>'reß㍿👍🏽!'re
é…-
½ع𐞁\"fi,
́é>'D-😀🏽0,㋿\t", "tokens": 64, "pieces": ["'S'VE", "😀🏽'", "D", ",", "½", "㍿a", " ", " <<|", "endoftext", "|>'", "reß", "㍿👍🏽!'", "re", "
é", "…", "-", "
", "½", "ع𐞁", "\"fi", ",", "
́é", ">'", "D", "-😀🏽", "0", ",㋿", "\t"]} +{"text": "'S‍𐞁٣٤٥٦३Džaé　t<|fim_prefix|>Ａ\"-'Rea\r\né\r\n\u000b \u000b9'S'S>", "tokens": 41, "pieces": ["'S", "‍𐞁", "٣٤٥", "٦३", "Džaé", "　t", "<|", "fim", "_prefix", "|>", "Ａ", "\"-<", "EOT", ">'", "Rea", "\r\n", "é", "\r\n", "\u000b ", "\u000b", "9", "'S'S", ">"]} +{"text": "e'VE'Då'Sḍ̇👍🏽", "tokens": 20, "pieces": ["e'VE", "'Då'S", "ḍ̇", "👍🏽<", "EOT", "><", "EOT", ">"]} +{"text": "㍿\r\n'll٣٤٥٦at㋿>Dž😀🏽\n ", "tokens": 21, "pieces": ["㍿\r\n", "'ll", "٣٤٥", "٦", "at", "㋿>", "Dž", "😀🏽\n", " "]} +{"text": "'śé0३😀🏽 \n#$%,#$%<|endoftext|>-İ
!३", "tokens": 25, "pieces": ["'śé", "0३", "😀🏽", " \n", "#$%,#$%<|", "endoftext", "|>-", "İ", "
", "!", "३"]} +{"text": "ſéⅣ\r\n🙂Dž㍿", "tokens": 12, "pieces": ["ſé", "Ⅳ", "\r\n", "🙂Dž", "㍿"]} +{"text": ".́\", fi\r\n\r\n३9 é!!㋿aé👍🏽!!'s३éꟲ\r!!'ſḍ̇ß🙂", "tokens": 35, "pieces": [".́", "\",", " fi", "\r\n\r\n", "३9", " ", " é", "!!㋿", "aé", "👍🏽!!'", "s", "३", "éꟲ", "\r", "!!'", "ſḍ̇ß", "🙂"]} +{"text": "m'VEs's'll<Ａ👍🏽㋿'ſ\r\r\n\r\n.12345678\tع\r\n\r\n字🙂ḍ̇½'reſ\"!!!!́Ⅳfiꟲ<|fim_prefix|>\r‍s", "tokens": 51, "pieces": ["m'VE", "s's", "'ll", "<Ａ", "👍🏽㋿'", "ſ", "\r\r\n\r\n", ".", "123", "456", "78", "\tع", "\r\n\r\n", "字", "🙂ḍ̇", "½", "'reſ", "\"!!!!́", "Ⅳ", "fiꟲ", "<|", "fim", "_prefix", "|>\r", "‍s"]} +{"text": " \n\tDžḍ̇EOTe ㋿'Re𐞁\r'VEaDž<|endoftext|>9#$%ꟲå字're
!\r\n😀🏽'reＡaḍ̇ Dž 'M漢\r\n'ſ're", "tokens": 70, "pieces": [" \n", "\tDžḍ̇", "EOTe", " ", " <", "META", "_START", ">㋿'", "Re𐞁", "\r", "'VEa", "Dž", "<|", "endoftext", "|>", "9", "#$%", "ꟲå字're", "
", "!\r\n", "😀🏽'", "re", "Ａaḍ̇", " ", " Dž", " ", "'M漢", "\r\n", "'ſ're"]} +{"text": "漢'Ré'reEOT​m", "tokens": 8, "pieces": ["漢'Re", "́'re", "EOT", "​m"]} +{"text": "‍é'Dd……!e३>.<Ⅳ३३", "tokens": 25, "pieces": ["‍", "é'D", "d", "…", "…", "!<", "EOT", ">e", "३", ">.<", "Ⅳ३३"]} +{"text": "'TDž're0're\r\n'Re
Ⅳḍ̇ ㋿‍ß-'TdEOTfi å!(fiḍ̇ \n", "tokens": 34, "pieces": ["'TDž're", "0", "'re", "\r\n", "'Re", "
", "Ⅳ", "ḍ̇", " ", "㋿‍", "ß", "-'", "Td", "EOTfi", " ", " å", "!(", "fiḍ̇", " \n"]} +{"text": "'s½,­३d‍ꟲ\t漢㋿𐞁.t… \r\n\r\n9fi字­٣٤٥٦👍🏽fi👍🏽('s12345678\r\n\r\nEOT \n'ع#$%'ll½", "tokens": 55, "pieces": ["'s", "½", ",­", "३", "d", "‍ꟲ", "\t漢", "㋿𐞁", ".t", "… \r\n\r\n", "9", "fi字", "­", "٣٤٥", "٦", "👍🏽", "fi", "👍🏽('", "s", "123", "456", "78", "\r\n\r\n", "EOT", " \n", "'ع", "#$%'", "ll", "½"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\nå,a字字'Re'S😀🏽12345678 ſfiḍ̇'Re#$%㋿'D", "tokens": 32, "pieces": ["\r\n", "å", ",", "a字字'Re", "'S", "😀🏽", "123", "456", "78", " ſfiḍ̇'Re", "#$%㋿'", "D"]} +{"text": "\r\néDž👍🏽漢", "tokens": 8, "pieces": ["\r\n", "é", "Dž", "👍🏽", "漢"]} +{"text": "Ａ\r\n\r\n\r\n㍿عtZⅣ​d'retſ!<|fim_prefix|> 'VE,", "tokens": 34, "pieces": ["Ａ", "\r\n\r\n\r\n", "㍿عt", "Z", "Ⅳ", "​<", "EOT", ">d're", "tſ", "!<", "EOT", "><", "META", "_START", "><|", "fim", "_prefix", "|>", " '", "VE", ","]} +{"text": " \n", "tokens": 1, "pieces": [" \n"]} +{"text": "t-\t", "tokens": 3, "pieces": ["t", "-", "\t"]} +{"text": "'M12345678é<|endoftext|>'Sḍ̇㍿é!ſḍ̇a'ſfi're#$%́", "tokens": 34, "pieces": ["'M", "123", "456", "78", "é", "<|", "endoftext", "|>'", "Sḍ̇", "㍿é", "!ſḍ̇a'ſ", "fi're", "#$%́"]} +{"text": "'T　ع(㋿\"­İ", "tokens": 13, "pieces": ["'T", "　ع", "(㋿<", "META", "_START", ">\"­", "İ"]} +{"text": "字'lĺ­!㍿'ſ<|fim_prefix|>½a0\t<|endoftext|>३😀🏽\r\n\r\n.m​'ſfiåEOT'‍'M\r\n\r\n字\r\n\r\n㋿s're", "tokens": 61, "pieces": ["字'll", "́", "­!㍿'", "ſ", "<|", "fim", "_prefix", "|>", "½", "a", "0", "", "\t", "<|", "endoftext", "|>", "३", "😀🏽\r\n\r\n", ".m", "​'", "ſfiå", "EOT", "'‍'", "M", "\r\n\r\n", "字", "\r\n\r\n", "㋿s", "'", "re"]} +{"text": ",…<|fim_prefix|>­　'D­\r\ne\u000b'SⅣ<|fim_prefix|>.\u000b😀🏽s'VEa\t𐞁'Rea Ⅳ", "tokens": 46, "pieces": [",", "…", "<|", "fim", "_prefix", "|>­", "　", "'D", "­\r\n", "e", "\u000b", "'S", "Ⅳ", "<|", "fim", "_prefix", "|>.", "\u000b", "😀🏽", "s'VE", "a", "", "\t𐞁'Re", "a", " ", "Ⅳ"]} +{"text": "­'s#$%\u000bİ", "tokens": 7, "pieces": ["­'", "s", "#$%", "\u000bİ"]} +{"text": "㋿'M​", "tokens": 6, "pieces": ["㋿'", "M", "​"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "字12345678㋿ß\u000b\u000b \n½t字'S9½\r🙂å३0½< ſ\r'sſ'sع\r-d‍　'refi's", "tokens": 51, "pieces": ["字", "123", "456", "78", "㋿ß", "\u000b\u000b \n", "½", "t字'S", "9½", "\r", "🙂å", "३0½", "<", " ", " ſ", "\r", "'s", "ſ's", "ع", "\r", "-d", "‍", "　", "'re", "fi's"]} +{"text": "0‍٣٤٥٦
0­éß漢d漢
漢's-ꟲås\r字'll-.İsＡ ​́", "tokens": 34, "pieces": ["0", "‍", "٣٤٥", "٦", "
", "0", "­éß漢d漢", "
漢's", "-ꟲås", "\r", "字'll", "-.", "İs", "Ａ", " ​́"]} +{"text": ". é३'re.>३'Mḍ̇'re👍🏽<|fim_prefix|>'T́漢12345678'VE\r\n\r\n👍🏽­'ſDž", "tokens": 39, "pieces": [".", " é", "३", "'re", ".>", "३", "'Mḍ̇'re", "👍🏽<|", "fim", "_prefix", "|>'", "T́漢", "123", "456", "78", "'VE", "\r\n\r\n", "👍🏽­'", "ſ", "Dž"]} +{"text": "\"ⅣEOT'VE
'VE!!…\u000bd'S>\"", "tokens": 17, "pieces": ["\"", "Ⅳ", "EOT'VE", "
", "'VE", "!!", "…", "\u000bd'S", ">\""]} +{"text": "­\nŹ\r\n", "tokens": 4, "pieces": ["­\n", "Ź", "\r\n"]} +{"text": ">'T㍿fi#$%­字Ⅳ0'M'İmⅣEOT-👍🏽 Ａ!EOTⅣ\n0\r\n'VEe\u000b ع\u000b漢🙂­", "tokens": 53, "pieces": [">'", "T", "㍿", "fi", "#$%­", "字", "Ⅳ0", "'M", "'<", "EOT", ">İm", "Ⅳ", "EOT", "-👍🏽", " ", " Ａ", "!EOT", "Ⅳ", "\n", "0", "\r\n", "'VEe", "\u000b ", " ع", "\u000b漢", "🙂­"]} +{"text": "tＡ \"s's\n's　­<\n𐞁,ＡEOT㍿Ａ\r\n\r\n½漢\rEOT'EOTs.", "tokens": 32, "pieces": ["t", "Ａ", " ", "\"s's", "\n", "'s", "　", "­<\n", "𐞁", ",ＡEOT", "㍿Ａ", "\r\n\r\n", "½", "漢", "\r", "EOT", "'EOTs", "."]} +{"text": "Džſ😀🏽😀🏽<>​<‍é 'ſEOT<|endoftext|>‍­'Sfi
", "tokens": 36, "pieces": ["Džſ", "😀🏽😀🏽<>​<‍", "é", " ", " '", "ſ", "EOT", "<|", "endoftext", "|>‍­'", "Sfi", "", "
"]} +{"text": " Zåİ12345678字", "tokens": 8, "pieces": [" Zå", "İ", "123", "456", "78", "字"]} +{"text": "#$%٣٤٥٦İé'Se‍fi!!Dž<'ſ ½", "tokens": 25, "pieces": ["#$%", "٣٤٥", "٦", "İé'S", "e", "‍fi", "!!", "Dž", "<'", "ſ", " <", "META", "_START", ">", "½", ""]} +{"text": "👍🏽s", "tokens": 4, "pieces": ["👍🏽", "s"]} +{"text": "#$%\r\n\r\ns٣٤٥٦<|fim_prefix|>३\u000bm9'Re\ta½<|fim_prefix|>\"\nZ​!!\n#$%ß\rm's\"'T", "tokens": 42, "pieces": ["#$%\r\n\r\n", "s", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "३", "\u000bm", "9", "'Re", "\ta", "", "½", "<|", "fim", "_prefix", "|>\"\n", "Z", "​!!\n", "#$%", "ß", "\r", "m's", "\"'", "T"]} +{"text": "'re'T'S\ń\r\n​fi<ع!字12345678\n9'M‍😀🏽́'re\u000b漢😀🏽́'", "re", "\u000b漢", "
'rea \néEOT😀🏽", "tokens": 29, "pieces": ["e字", "<字're", "٣٤٥", "٦ⅣⅣ", "<|", "fim", "_prefix", "|>", "
", "'rea", " \n", "é", "EOT", "😀🏽"]} +{"text": "٣٤٥٦'ſ<|endoftext|>0EOT👍🏽0  ­…å\r\"'T\r\nḍ̇éa-'re<|endoftext|>'S\r\n\r\n", "tokens": 48, "pieces": ["٣٤٥", "٦", "'ſ", "<|", "endoftext", "|>", "0", "EOT", "👍🏽", "0", " ", " ", "­", "…å", "\r", "\"'", "T", "\r\n", "ḍ̇éa", "-'", "re", "<|", "endoftext", "|>'", "S", "\r\n\r\n"]} +{"text": "Dž​fi're'll", "tokens": 6, "pieces": ["Dž", "​fi're", "'ll"]} +{"text": "\r\nå…,漢
fiß\" \n\r-𐞁'VE>\tm㋿'s
>- \n㍿", "tokens": 33, "pieces": ["\r\n", "å", "…", ",漢", "
fiß", "\"", " \n\r", "-𐞁'VE", ">", "\tm", "㋿'", "s", "
", ">-", " \n", "㍿"]} +{"text": "ma'0½३'D'rea'ſ\u000b😀🏽漢'Ḿ'D!…𐞁 \nee٣٤٥٦ǻ>", "tokens": 35, "pieces": ["ma", "'", "0½३", "'D're", "a'ſ", "\u000b", "😀🏽", "漢'M", "́'D", "!", "…𐞁", " \n", "ee", "٣٤٥", "٦", "ǻ", ">"]} +{"text": "​ع<|endoftext|>㍿ \nZ\u000b\tß'Re\r\n\r\n३EOT", "tokens": 22, "pieces": ["​ع", "<|", "endoftext", "|>㍿", " \n", "Z", "\u000b", "\tß'Re", "\r\n\r\n", "३", "EOT"]} +{"text": "'s\tḍ̇ t
 ㋿-٣٤٥٦ße🙂😀🏽ß12345678😀🏽m<|endoftext|>٣٤٥٦字ſ(\r\néfi0é'T", "tokens": 54, "pieces": ["'s", "\tḍ̇", " t", "
 ", " ㋿<", "META", "_START", ">-", "٣٤٥", "٦", "ße", "🙂😀🏽", "ß", "123", "456", "78", "😀🏽", "m", "<|", "endoftext", "|>", "٣٤٥", "٦", "字ſ", "(\r\n", "éfi", "0", "é'T"]} +{"text": "Ａ'VE<|fim_prefix|>", "tokens": 9, "pieces": ["Ａ'VE", "<|", "fim", "_prefix", "|>"]} +{"text": "fi‍Ａa'S-9㍿!­EOT.'M>\r\né", "tokens": 22, "pieces": ["fi", "‍", "Ａa'S", "-", "9", "㍿!­", "EOT", ".'", "M", ">\r\n", "é"]} +{"text": "ſ'VEḍ̇३å漢Ⅳ0m🙂👍🏽ꟲ㍿", "tokens": 24, "pieces": ["ſ'VE", "ḍ̇", "३", "å漢", "Ⅳ0", "m", "🙂👍🏽", "ꟲ", "㍿"]} +{"text": "'ſ'll<|endoftext|>< tm३ Z-<|endoftext|>'St.m", "tokens": 25, "pieces": ["'ſ'll", "<|", "endoftext", "|><", " ", " tm", "३", " Z", "-<|", "endoftext", "|>'", "St", ".m"]} +{"text": "\"-Z<|endoftext|>'rea\n  Zéé'Taé0'Méå　Dž<|fim_prefix|>ⅣDž9's", "tokens": 40, "pieces": ["\"-", "Z", "<|", "endoftext", "|>'", "rea", "\n", " ", " Zéé'T", "aé", "0", "'Méå", "　Dž", "<|", "fim", "_prefix", "|>", "Ⅳ", "Dž", "9", "'s"]} +{"text": "Ⅳ😀🏽\r\ndé½\r\n\r\né'D's(", "tokens": 14, "pieces": ["Ⅳ", "😀🏽\r\n", "dé", "½", "\r\n\r\n", "é'D", "'s", "("]} +{"text": " (🙂're漢İⅣ \nß'Tİ'T'M", "tokens": 50, "pieces": [" ", "(🙂'", "re漢", "İ", "Ⅳ", "", " \n", "ß'T", "İ'T", "'M"]} +{"text": "\r\n\r\n!!\"\t　́\tſEOT\t漢𐞁\"ع's㋿(­३'MDž\t\nع\té>d'Sİ\"\n", "tokens": 39, "pieces": ["\r\n\r\n", "!!\"", "\t", "　́", "\tſ", "EOT", "\t漢𐞁", "\"<", "EOT", ">ع's", "㋿(­", "३", "'MDž", "\t\n", "ع", "\té", ">d'S", "İ", "\"\n"]} +{"text": "\r\n\r\nİ字'T㍿å \n‍.fi\r!!!t!<,Z​́Z", "tokens": 22, "pieces": ["\r\n\r\n", "İ字'T", "㍿å", " \n", "‍.", "fi", "\r", "!!!", "t", "!<,", "Z", "​́", "Z"]} +{"text": "s 'D٣٤٥٦Z\" 's<|endoftext|>\t#$%\n \n'llé'MZm𐞁'D'T", "tokens": 45, "pieces": ["s", " ", " '", "D", "٣٤٥", "٦", "Z", "\"", " ", " '", "s", "<|", "endoftext", "|>", "\t", "#$%\n", " \n", "'llé", "'", "MZm𐞁'D", "'", "T", ""]} +{"text": "🙂<|fim_prefix|>🙂'Re12345678\n>Dž#$%'s12345678!é字عa👍🏽Ａ字a漢\u000b0\t#$%", "tokens": 56, "pieces": ["🙂<", "EOT", "><", "mé", "!!'", "M", "<|", "endoftext", "|><|", "fim", "_prefix", "|>🙂'", "Re", "123", "456", "78", "\n", ">Dž", "#$%'", "s", "123", "456", "78", "!é字عa", "👍🏽", "Ａ字a漢", "\u000b", "0", "\t", "#$%"]} +{"text": "‍½'D'D‍sDž㋿-'s\"́é'D d\u000bعEOT​㋿", "tokens": 27, "pieces": ["‍", "½", "'D'D", "‍s", "Dž", "㋿-'", "s", "\"́é'D", " d", "\u000bع", "EOT", "​㋿"]} +{"text": "å'Tt<|fim_prefix|>字𐞁𐞁\u000b'TⅣ12345678é字", "tokens": 32, "pieces": ["å'T", "t", "<|", "fim", "_prefix", "|>", "字𐞁𐞁", "\u000b", "'T", "Ⅳ12", "345", "678", "é字"]} +{"text": ">😀🏽ꟲ", "tokens": 7, "pieces": [">😀🏽", "ꟲ"]} +{"text": "ß🙂　!İ'ſ<|endoftext|>'re-Zß9ḍ̇\n\téع٣٤٥٦(éEOT>㍿é#$%-m12345678ſDžİ", "tokens": 48, "pieces": ["ß", "🙂", "　", "!İ'ſ", "<|", "endoftext", "|>'", "re", "-Zß", "9", "ḍ̇", "\n", "\téع", "٣٤٥", "٦", "(é", "EOT", ">㍿", "é", "#$%-", "m", "123", "456", "78", "ſ", "Džİ"]} +{"text": "9'́<|fim_prefix|>🙂\"!​ꟲꟲ\r\n\r\n'reße0Z'D漢", "tokens": 25, "pieces": ["9", "'́", "<|", "fim", "_prefix", "|>🙂\"!​", "ꟲꟲ", "\r\n\r\n", "'reße", "0", "Z'D", "漢"]} +{"text": "!0're,'s'D.'VE#$%👍🏽😀🏽İ​🙂'Sé 'MEOT<٣٤٥٦'s\u000bfi0<|endoftext|>عé", "tokens": 43, "pieces": ["!", "0", "'re", ",'", "s'D", ".'", "VE", "#$%👍🏽😀🏽", "İ", "​🙂'", "Sé", " ", " '", "MEOT", "<", "٣٤٥", "٦", "'s", "\u000bfi", "0", "<|", "endoftext", "|>", "عé"]} +{"text": "\u000be㋿👍🏽'D٣٤٥٦'reع'M'll
12345678Ⅳa漢㍿>Ⅳ.-'D'VEs ३­\n", "tokens": 42, "pieces": ["\u000be", "㋿👍🏽'", "D", "٣٤٥", "٦", "'reع'M", "'ll", "
", "123", "456", "78Ⅳ", "a漢", "㍿>", "Ⅳ", ".-'", "D'VE", "s", " ", " ", "३", "­\n"]} +{"text": "é>'12345678㋿½٣٤٥٦३'Re\"t　!!‍ ́", "tokens": 25, "pieces": ["é", ">'", "123", "456", "78", "㋿", "½٣٤", "٥٦३", "'Re", "\"t", "　", "!!‍", " ́", ""]} +{"text": "Ź's𐞁e字Ａ9­ ßＡ漢tå\rd'MDž<", "tokens": 24, "pieces": ["Ź's", "𐞁e字", "Ａ", "9", "­", " ß", "Ａ漢tå", "\r", "d'M", "Dž", "<"]} +{"text": "\r\n\u000bEOTa'sEOT½0​½'VE-\r\t(👍🏽\u000bꟲ\"İ३\néİ <|endoftext|>'M0", "tokens": 45, "pieces": ["\r\n", "\u000bEOTa's", "EOT", "½0", "​", "½", "'VE", "-\r", "\t", "(👍🏽<", "META", "_START", ">", "\u000bꟲ", "\"İ", "३", "\n", "é", "İ", " ", "<|", "endoftext", "|>'", "M", "0"]} +{"text": "㋿'T ae🙂ſ0٣٤٥٦👍🏽m,‍\ré'VE Ⅳ'Så­>\n'Re३'Re<​.㍿'Tß'T👍🏽", "tokens": 47, "pieces": ["㋿'", "T", " ae", "🙂ſ", "0٣٤", "٥٦", "👍🏽", "m", ",‍\r", "é'VE", " ", "Ⅳ", "'Så", "­>\n", "'Re", "३", "'Re", "<​.㍿'", "Tß'T", "👍🏽"]} +{"text": "<|fim_prefix|>", "tokens": 6, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "a!!EOT'", "tokens": 5, "pieces": ["a", "!!", "EOT", "'"]} +{"text": "ſ🙂å漢ſéfi ", "tokens": 14, "pieces": ["ſ", "🙂å漢ſé", "fi", " "]} +{"text": "́́
0Dž‍㍿sع'Re'sm'Re
<|fim_prefix|>३å३字👍🏽\n", "tokens": 32, "pieces": ["́́", "
", "0", "Dž", "‍㍿", "sع'Re", "'sm'Re", "
", "<|", "fim", "_prefix", "|>", "३", "å", "३", "字", "👍🏽\n"]} +{"text": "t\né\t'å \n​,'M\"\u000b½m㋿ \n<‍ß12345678d　Z'VE'ſＡ٣٤٥٦'M#$%éDž", "tokens": 43, "pieces": ["t", "\n", "é", "\t", "'å", " \n", "​,'", "M", "\"", "\u000b", "½", "m", "㋿", " \n", "<‍", "ß", "123", "456", "78", "d", "　Z'VE", "'ſ", "Ａ", "٣٤٥", "٦", "'M", "#$%", "é", "Dž"]} +{"text": "'T<'S!!\r\nZDž.㍿ås'em\n'T 12345678'T9🙂 \n's㍿\"'D9's\r.ſ12345678👍🏽", "tokens": 43, "pieces": ["'T", "<'", "S", "!!\r\n", "ZDž", ".㍿", "ås", "'em", "\n", "'T", " ", "123", "456", "78", "'T", "9", "🙂", " \n", "'s", "㍿\"'", "D", "9", "'s", "\r", ".ſ", "123", "456", "78", "👍🏽"]} +{"text": "me<|fim_prefix|>#$%é​", "tokens": 12, "pieces": ["me", "<|", "fim", "_prefix", "|>#$%", "é", "​"]} +{"text": "\tst…\reßé 'ſé'S\tḍ̇", "tokens": 17, "pieces": ["\tst", "…\r", "eßé", " ", " '", "ſé'S", "\tḍ̇"]} +{"text": "İaé<|endoftext|>#$%<'ſs\n", "tokens": 16, "pieces": ["İaé", "<|", "endoftext", "|>#$%<'", "ſs", "\n"]} +{"text": "é\n eZ's­👍🏽 ​'T.mm🙂d<|endoftext|> 's\"!é­ ‍\".", "tokens": 33, "pieces": ["é", "\n", " e", "Z's", "­👍🏽", " ", " ​'", "T", ".mm", "🙂d", "<|", "endoftext", "|>", " '", "s", "\"!", "é", "­", " ", " ‍\"."]} +{"text": "🙂​​\u000bⅣ-<|fim_prefix|>'VE½½'ſ…å'll\"😀🏽 \n३Ａta<½ \n<", "tokens": 34, "pieces": ["🙂​​", "\u000b", "Ⅳ", "-<|", "fim", "_prefix", "|>'", "VE", "½½", "'ſ", "…å'll", "\"😀🏽", " \n", "३", "Ａta", "<", "½", " \n", "<"]} +{"text": " e'VE👍🏽­EOT<|fim_prefix|>漢ßå​.'re<|fim_prefix|>​😀🏽‍½ع𐞁,é　9­㍿
.…\n ٣٤٥٦aＡ<|fim_prefix|>s ", "tokens": 68, "pieces": [" ", " e'VE", "👍🏽­", "EOT", "<|", "fim", "_prefix", "|>", "漢ßå", "​.'", "re", "<|", "fim", "_prefix", "|>​😀🏽‍", "½", "ع𐞁", ",é", "　", "9", "­㍿", "
", ".", "…\n", " ", "٣٤٥", "٦", "a", "Ａ", "<|", "fim", "_prefix", "|>", "s", " "]} +{"text": "é३!!-a12345678!!", "tokens": 9, "pieces": ["é", "३", "!!-", "a", "123", "456", "78", "!!"]} +{"text": " \n're.", "tokens": 7, "pieces": [" \n", "'", "re", "."]} +{"text": "'T'DＡ३𐞁'T", "tokens": 9, "pieces": ["'T'D", "Ａ", "३", "𐞁'T"]} +{"text": " <|endoftext|>", "tokens": 8, "pieces": [" ", "<|", "endoftext", "|>"]} +{"text": "👍🏽…­漢\r\nⅣ åZＡZ'T<|fim_prefix|>\n\nsZm\" EOTZ\r\n\r\n
'll 
Z👍🏽漢㋿٣٤٥٦😀🏽EOTꟲ0're½‍", "tokens": 58, "pieces": ["👍🏽", "…", "­漢", "\r\n", "Ⅳ", " å", "ZＡZ'T", "<|", "fim", "_prefix", "|>\n\n", "s", "Zm", "\"", " ", " EOTZ", "\r\n\r\n", "
", "'ll", " ", "
Z", "👍🏽", "漢", "㋿", "٣٤٥", "٦", "😀🏽", "EOTꟲ", "0", "'re", "½", "‍"]} +{"text": "🙂as\nå\r.å​Dž(\t​!!Ⅳd‍>㋿#$%​\u000b字'll<|endoftext|>fi!,", "tokens": 47, "pieces": ["🙂<", "META", "_START", ">as", "\n", "å", "\r", ".å", "​Dž", "(", "\t", "​!!", "Ⅳ", "d", "‍<", "EOT", ">>㋿#$%​", "\u000b字'll", "<|", "endoftext", "|>", "fi", "!,"]} +{"text": "\"٣٤٥٦å٣٤٥٦'s éZ!!👍🏽( 12345678İ́ <|endoftext|>\r\n\r\n< …\r\n\r\n'T'٣٤٥٦㋿😀🏽<|endoftext|>٣٤٥٦te'MmꟲＡ", "tokens": 67, "pieces": ["\"", "٣٤٥", "٦", "å", "٣٤٥", "٦", "'s", " ", " é", "Z", "!!👍🏽(", " ", "123", "456", "78", "İ́", " <|", "endoftext", "|>\r\n\r\n", "<", " …\r\n\r\n", "'T", "'", "٣٤٥", "٦", "㋿😀🏽<|", "endoftext", "|>", "٣٤٥", "٦", "te'M", "mꟲ", "Ａ"]} +{"text": ">\u000b😀🏽漢́s-'M'sEOT 漢9#$%ß👍🏽,", "tokens": 23, "pieces": [">", "\u000b", "😀🏽", "漢́s", "-'", "M's", "EOT", " ", " 漢", "9", "#$%", "ß", "👍🏽,"]} +{"text": "́> DžDžEOTfi\r\n🙂a'Re >EOTＡEOT.Z0<|fim_prefix|><|endoftext|>.\"é ss'D0‍ å…", "tokens": 47, "pieces": ["́", ">", " ", " DžDžEOTfi", "\r\n", "🙂a'Re", " ", ">EOTＡEOT", ".Z", "0", "<|", "fim", "_prefix", "|><|", "endoftext", "|>.\"", "é", " ss'D", "0", "‍", " ", " å", "…"]} +{"text": "İ'T'sd!å½ḍ̇㋿ع're👍🏽é漢ß\n'VE​Ⅳ\t㍿'VE!,٣٤٥٦'ſ", "tokens": 55, "pieces": ["İ'T", "'sd", "!<", "m漢ß", "<|", "endoftext", "|>", "å", "½", "ḍ̇", "㋿ع're", "👍🏽", "é漢ß", "\n", "'VE", "​", "Ⅳ", "\t", "㍿'", "VE", "!,", "٣٤٥", "٦", "'ſ"]} +{"text": "EOT𐞁afi　 \taDžfí 字‍ß \n", "tokens": 19, "pieces": ["EOT𐞁afi", "　 ", "\ta", "Džfí", " 字", "‍ß", " \n"]} +{"text": "'VE\n'ſ'Deéḍ̇\"𐞁Z'ſ ,12345678 \n
Zꟲ 9😀🏽.…!!!eDž<-,　", "tokens": 44, "pieces": ["'VE", "\n", "'ſ'D", "eéḍ̇", "\"𐞁", "Z'ſ", " ,", "123", "456", "78", " \n", "
Zꟲ", " ", "9", "😀🏽.", "…", "!!!", "e", "Dž", "<-,", "　"]} +{"text": "EOTe'T'ſ𐞁<|endoftext|>\n0ZZع(\u000b>½m ", "tokens": 29, "pieces": ["EOTe'T", "'ſ𐞁", "<|", "endoftext", "|>\n", "0", "Z", "Zع", "(", "\u000b", ">", "½", "m", " "]} +{"text": "sZ‍<|fim_prefix|>'Re!'Té　\r🙂ea'D'D<㍿é́ e'D'T'Tſ(EOT😀🏽\r\n\r\n", "tokens": 41, "pieces": ["s", "Z", "‍<|", "fim", "_prefix", "|>'", "Re", "!'", "Té", "　\r", "🙂ea'D", "'D", "<㍿", "é́", " <", "EOT", ">e'D", "'T'T", "ſ", "(EOT", "😀🏽\r\n\r\n"]} +{"text": "'T<İ'S
​('sＡ३'lled(", "tokens": 37, "pieces": ["ſe", "-", "٣٤٥", "٦", "'s", "Ａ", "३", "'lled", "("]} +{"text": "''Re'D\t​eſ́.<12345678'S'VE\"ḍ̇'Re​'Reḍ̇s<fiDž''Te🙂ꟲſ'D(​'VEé9́", "tokens": 50, "pieces": ["''", "Re'D", "\t", "​eſ́", ".<", "123", "456", "78", "'S'VE", "\"ḍ̇'Re", "​'", "Reḍ̇s", "<fi", "Dž", "''", "Te", "🙂ꟲſ", "'", "D", "(​'", "VEé", "9", "́"]} +{"text": ",\t\"\re>Ａé!!\r\n\r\n'ś#$%'reEOT9!'S'llEOT\u000b", "tokens": 24, "pieces": [",", "\t", "\"\r", "e", ">Ａé", "!!\r\n\r\n", "'ś", "#$%'", "re", "EOT", "9", "!'", "S'll", "EOT", "\u000b"]} +{"text": "#$%İa'Dß \n
­
㋿…'Re ", "tokens": 17, "pieces": ["#$%", "İa'D", "ß", " \n", "
", "­", "
", "㋿", "…", "'Re", " "]} +{"text": "'s　'DEOTé0'll👍🏽mⅣ\u000b'Re'D\n're", "tokens": 22, "pieces": ["'s", "　", "'DEOTé", "0", "'ll", "👍🏽", "m", "Ⅳ", "\u000b", "'Re", "'", "D", "\n", "'re"]} +{"text": "a'Ds'VEⅣ're　'VE 9​'Re", "tokens": 21, "pieces": ["a'D", "s'VE", "", "Ⅳ", "'re", "　", "'VE", " ", " ", "9", "​'", "Re"]} +{"text": "'s­👍🏽…fi\u000b漢,\r\n'reع!!-fis", "tokens": 17, "pieces": ["'s", "­👍🏽", "…fi", "\u000b漢", ",\r\n", "'reع", "!!-", "fis"]} +{"text": ".m　 \u000b㋿d'ſ́'D𐞁\r\n'Mså​9EOT㋿\"9.
عa", "tokens": 35, "pieces": [".m", "　 ", "\u000b", "㋿d'ſ", "́'D", "𐞁", "\r\n", "'Mså", "​", "9", "EOT", "㋿\"", "9", ".", "
عa"]} +{"text": " \nßＡ'M 9'S　­'Re're're漢漢's<|fim_prefix|>éé\rDž…ß#$%👍🏽­
'D\r\t#$%Z,'VE३#$%,'re-", "tokens": 56, "pieces": [" \n", "ß", "Ａ'M", " ", " ", "9", "'S", "　", "­'", "Re're", "'re漢漢's", "<|", "fim", "_prefix", "|>", "éé", "\r", "Dž", "…ß", "#$%👍🏽­", "
", "'D", "\r", "\t", "#$%", "Z", ",'", "VE", "३", "#$%,'", "re", "-"]} +{"text": "㍿ß½m\u000b<㋿'ſ 'Ａ　ſ漢d\råtEOT
‍>", "tokens": 31, "pieces": ["㍿ß", "½", "m", "\u000b", "<㋿'", "ſ", " '", "Ａ", "　ſ", "漢d", "\r", "åt", "EOT", "
", "‍>"]} +{"text": "\"é३🙂'Reḍ̇!Ａ're", "tokens": 12, "pieces": ["\"é", "३", "🙂'", "Reḍ̇", "!Ａ're"]} +{"text": "३Ａ\"m<|fim_prefix|><12345678t'M<|endoftext|>'s㋿٣٤٥٦…\r\n\r\n12345678ś", "tokens": 38, "pieces": ["३", "Ａ", "\"m", "<|", "fim", "_prefix", "|><", "123", "456", "78", "t'M", "<|", "endoftext", "|>'", "s", "㋿", "٣٤٥", "٦", "…\r\n\r\n", "123", "456", "78", "ś"]} +{"text": "'ll🙂'٣٤٥٦éEOTfi漢EOT<|fim_prefix|>\r\n\r\n \n'VEDž\n<|fim_prefix|>", "tokens": 33, "pieces": ["'ll", "🙂'", "٣٤٥", "٦", "é", "EOTfi漢", "EOT", "<|", "fim", "_prefix", "|>\r\n\r\n", " \n", "'VEDž", "\n", "<|", "fim", "_prefix", "|>"]} +{"text": "'re३Dž's!! \n𐞁'll'SEOT9!!㍿éDž\r0é", "tokens": 27, "pieces": ["'re", "३", "Dž's", "!!", " \n", "𐞁'll", "'SEOT", "9", "!!㍿", "é", "Dž", "\r", "0", "é"]} +{"text": "'D-
'12345678́<|endoftext|>#$%٣٤٥٦㍿<|endoftext|>'VE\"åDž,é
\r…'𐞁​́Dž
字　Dž", "tokens": 56, "pieces": ["'D", "-", "
", "'", "123", "456", "78", "́", "<|", "endoftext", "|>#$%", "٣٤٥", "٦", "㍿<|", "endoftext", "|>'", "VE", "\"å", "Dž", ",é", "
\r", "…", "'𐞁", "​́", "Dž", "
字", "　Dž"]} +{"text": "​‍\r\n\r\n\"EOT\n\r\n🙂 . ㋿ع́éعع>İsعİ́ 'ſ…́é字", "tokens": 36, "pieces": ["​‍\r\n\r\n", "\"EOT", "\n", "\r\n", "🙂", " ", ".", " ", "㋿ع́éعع", ">İsع", "İ́", " ", "'ſ", "…́é字"]} +{"text": "\n\n#$% \n\r\tع \nⅣmعs३٣٤٥٦
<\r\n.ḍ̇👍🏽", "tokens": 28, "pieces": ["\n\n", "#$%", " \n\r", "\tع", " \n", "Ⅳ", "mعs", "३٣٤", "٥٦", "
", "<\r\n", ".ḍ̇", "👍🏽"]} +{"text": "('VE㋿fiḍ̇　", "tokens": 14, "pieces": ["('", "VE", "㋿<", "META", "_START", ">fiḍ̇", "　"]} +{"text": "m't", "tokens": 2, "pieces": ["m't"]} +{"text": "9\r#$% 'Re𐞁\rt#$%'SEOTİ𐞁é'0sḍ̇!!'ſtꟲ9a<|fim_prefix|>\"㋿'Re㋿, ٣٤٥٦👍🏽-e👍🏽Ⅳ㋿​", "tokens": 72, "pieces": ["9", "\r", "#$%", " ", "'Re𐞁", "\r", "t", "#$%'", "SEOTİ𐞁é", "'", "0", "sḍ̇", "!!'", "ſtꟲ", "9", "a", "<|", "fim", "_prefix", "|>\"㋿'", "Re", "㋿,", " ", "٣٤٥", "٦", "👍🏽-", "e", "👍🏽", "Ⅳ", "㋿​"]} +{"text": "𐞁​\n12345678.𐞁0d𐞁㍿३e½Ａ३'re", "tokens": 28, "pieces": ["𐞁", "​\n", "123", "456", "78", ".𐞁", "0", "d𐞁", "㍿", "३", "e", "½", "Ａ", "३", "'re"]} +{"text": "ßé'Re ́#$%'Sm
0'12345678's0\r\ńⅣEOT㍿ \naſİꟲ漢(", "tokens": 35, "pieces": ["ßé'Re", " ", " ́", "#$%'", "Sm", "
", "0", "'", "123", "456", "78", "'s", "0", "\r\n", "́", "Ⅳ", "EOT", "㍿", " \n", "aſ", "İꟲ漢", "("]} +{"text": "t\rſ#$%'Mꟲ'ſ​!!EOT0", "tokens": 16, "pieces": ["t", "\r", "ſ", "#$%'", "Mꟲ'ſ", "​!!", "EOT", "0"]} +{"text": "‍afi\n0漢", "tokens": 6, "pieces": ["‍afi", "\n", "0", "漢"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'Smع'll\r\n'VE\r\n\r\n😀🏽\t\nſ漢㍿
'Re.ع\r\n\r\n", "😀🏽", "\t\n", "ſ漢", "㍿", "
", "'Re", ".ع", "½ḍ̇Dž​(ＡDž'T9𐞁.'Ts㋿<|fim_prefix|>‍Dž½>ſ,Z'Ms.٣٤٥٦d>'D're'ſ😀🏽9<|endoftext|>a", "tokens": 63, "pieces": ["", "½", "ḍ̇", "Dž", "​(", "ＡDž'T", "9", "𐞁", ".'", "Ts", "㋿<|", "fim", "_prefix", "|>‍", "Dž", "½", ">ſ", ",Z'M", "s", ".", "٣٤٥", "٦", "d", ">'", "D're", "'ſ", "😀🏽", "9", "<|", "endoftext", "|>", "a"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "　İ­İſDž\u000b", "tokens": 8, "pieces": ["　İ", "­İſ", "Dž", "\u000b"]} +{"text": "३\r\n\r\n㋿Ａ٣٤٥٦eعs!!'s㋿½Ⅳ'Té\r\n\r\n\"'D!<|fim_prefix|>عſ\r\n\r\n\r\ń́\r,\n　\n<३", "tokens": 53, "pieces": ["३", "\r\n\r\n", "㋿Ａ", "٣٤٥", "٦", "eعs", "!!'", "s", "㋿", "½Ⅳ", "'Té", "\r\n\r\n", "\"'", "D", "!<|", "fim", "_prefix", "|>", "عſ", "\r\n\r\n\r\n", "́́", "\r", ",\n", "　\n", "<", "३", ""]} +{"text": "Z>sé\t\r\nſß<|endoftext|>…!😀🏽0#$%😀🏽é <|endoftext|> \n99'VE
å٣٤٥٦-fi漢Ⅳ‍'D're㋿'Re,", "tokens": 65, "pieces": ["Z", ">sé", "\t\r\n", "ſß", "<|", "endoftext", "|>", "…", "!😀🏽", "0", "#$%😀🏽", "é", " <|", "endoftext", "|>", " \n", "99", "'VE", "
å", "", "٣٤٥", "٦", "-fi漢", "Ⅳ", "‍'", "D're", "㋿'", "Re", ","]} +{"text": "é 👍🏽عdꟲ'T'Re", "tokens": 12, "pieces": ["é", " ", "👍🏽", "عdꟲ'T", "'Re"]} +{"text": "'Da\u000b#$% \nZ're <|endoftext|>…​Z 0.\u000b㋿e", "tokens": 27, "pieces": ["'Da", "\u000b", "#$%", " \n", "Z're", " <|", "endoftext", "|>", "…", "​Z", " ", "0", ".", "\u000b", "㋿e"]} +{"text": "\u000b३𐞁'M…­\n㋿", "tokens": 13, "pieces": ["\u000b", "३", "𐞁'M", "…", "­\n", "㋿"]} +{"text": "'s\n'SZ㍿", "tokens": 7, "pieces": ["'s", "\n", "'SZ", "㍿"]} +{"text": "👍🏽!!‍\"ß\"ḍ̇\r\n\r\n३fi", "tokens": 14, "pieces": ["👍🏽!!‍\"", "ß", "\"ḍ̇", "\r\n\r\n", "३", "fi"]} +{"text": "éé'll \nＡ !-", "tokens": 10, "pieces": ["éé'll", " \n", "Ａ", " ", " !-"]} +{"text": "'Re 'ſtḍ̇… 9漢ß字!#$%𐞁…a12345678㋿9", "tokens": 36, "pieces": ["'Re", " ", " '", "ſtḍ̇", "… ", " ", "9", "漢ß字", "!#$%", "𐞁", "…a", "", "123", "456", "78", "㋿", "9"]} +{"text": "#$%字　Dž\r", "tokens": 7, "pieces": ["#$%", "字", "　Dž", "\r"]} +{"text": "\" \n㋿<|fim_prefix|>EOT\r\n\r\n　", "tokens": 15, "pieces": ["\"", " \n", "㋿<|", "fim", "_prefix", "|>", "EOT", "\r\n\r\n", "　"]} +{"text": "\r\n\r\n㋿👍🏽!'VEß12345678'll\r'ſ!!ß'DⅣ", "tokens": 26, "pieces": ["\r\n\r\n", "㋿👍🏽!'", "VEß", "123", "456", "78", "'ll", "\r", "'ſ", "!!<", "META", "_START", ">ß'D", "Ⅳ"]} +{"text": "EOT😀🏽👍🏽<|fim_prefix|>12345678t'D0!ꟲ0fi…12345678👍🏽é㋿12345678.", "tokens": 46, "pieces": ["EOT", "😀🏽👍🏽<|", "fim", "_prefix", "|>", "123", "456", "78", "t'D", "0", "!ꟲ", "", "0", "fi", "…", "123", "456", "78", "👍🏽", "é", "㋿", "123", "456", "78", "."]} +{"text": "<|endoftext|> <|endoftext|>0　'reع'Ma½", "tokens": 22, "pieces": ["<|", "endoftext", "|>", " ", " <|", "endoftext", "|>", "0", "　", "'reع'M", "a", "½"]} +{"text": "\r's‍(. .\n👍🏽0Ａ<|endoftext|>'D\ré­\r\n\r\ń​'D'T", "tokens": 32, "pieces": ["\r", "'s", "‍(.", " ", ".\n", "👍🏽", "0", "Ａ", "<|", "endoftext", "|>'", "D", "\r", "é", "­\r\n\r\n", "́", "​'", "D'T"]} +{"text": "'ReDž-…🙂漢Z漢
　'll.\r३é​'VE\u000b\"'reé'S 'VE漢0ß👍🏽😀🏽ع\t‍😀🏽ꟲ", "tokens": 56, "pieces": ["'Re", "Dž", "-", "…", "🙂漢Z漢", "
", "　", "'", "ll", ".\r", "३", "é", "​'", "VE", "\u000b", "\"'", "reé'S", " '", "VE漢", "0", "ß", "👍🏽😀🏽", "ع", "\t", "‍<", "EOT", ">😀🏽", "ꟲ"]} +{"text": "0<>'Tİ.ḍ̇åDž\r\n\r\nعＡ", "tokens": 16, "pieces": ["0", "<>'", "Tİ", ".ḍ̇å", "Dž", "\r\n\r\n", "ع", "Ａ"]} +{"text": "s9ḍ̇字𐞁'll'३Ａ é　Z‍½s>'Mİa'Sé#$%té'<|endoftext|>", "tokens": 38, "pieces": ["s", "9", "ḍ̇字𐞁'll", "'", "३", "Ａ", " é", "　Z", "‍", "½", "s", ">'", "Mİa'S", "é", "#$%", "té", "'<|", "endoftext", "|>"]} +{"text": "𐞁å
t'llZéDž…e'TعEOT#$%#$%\u000b​. EOTa> ́'(ß<|endoftext|>㋿'re😀🏽-.", "tokens": 59, "pieces": ["𐞁å", "
t'll", "Zé", "Dž", "…e'T", "ع", "EOT", "#$%#$%", "\u000b", "​.", " EOTa", ">", " ", " ́", "'(", "ß", "<|", "endoftext", "|>㋿'", "re", "😀🏽<", "EOT", ">-."]} +{"text": "Dž٣٤٥٦EOT\r٣٤٥٦　字å𐞁👍🏽ſ0​ '㋿<|fim_prefix|>𐞁#$%字'llＡ\t#$%å \né<|fim_prefix|>㍿#$%s'VEع're", "tokens": 74, "pieces": ["Dž", "٣٤٥", "٦", "EOT", "\r", "٣٤٥", "٦", "　字å𐞁", "👍🏽", "ſ", "0", "​", " ", " '㋿<|", "fim", "_prefix", "|>", "𐞁", "#$%", "字'll", "Ａ", "\t", "#$%", "å", " \n", "é", "<|", "fim", "_prefix", "|>㍿#$%", "s'VE", "ع're", ""]} +{"text": "…\n½ >'D\r\n\r\n\t­-é\t9​'ll٣٤٥٦mſ漢-Dž٣٤٥٦İEOT'½\u000b\r\n٣٤٥٦漢.ḍ̇ß½", "tokens": 52, "pieces": ["…\n", "½", " >'", "D", "\r\n\r\n", "\t", "­-", "é", "\t", "9", "​'", "ll", "٣٤٥", "٦", "mſ漢", "-Dž", "٣٤٥", "٦", "İEOT", "'", "½", "\u000b\r\n", "٣٤٥", "٦", "漢", ".ḍ̇ß", "½"]} +{"text": " \n'Re㋿👍🏽éZ½aſ́-ſ'll'Re🙂(<|fim_prefix|>'\rfi'VEDžİ㋿d!é>", "tokens": 39, "pieces": [" \n", "'Re", "㋿👍🏽", "é", "Z", "½", "aſ́", "-ſ'll", "'Re", "🙂(<|", "fim", "_prefix", "|>'\r", "fi'VE", "Džİ", "㋿d", "!é", ">"]} +{"text": " \n''VEḍ̇'T", "tokens": 11, "pieces": [" \n", "''", "VE", "ḍ̇'T"]} +{"text": "!!#$%\n0é٣٤٥٦å.(<
12345678'ſa\n漢Ａḍ̇e　t('ſå'll\"Z'VE \"\té#$%mt", "tokens": 49, "pieces": ["!!#$%<", "EOT", ">\n", "0", "é", "٣٤٥", "٦", "å", ".(<", "
", "123", "456", "78", "'ſa", "\n", "漢Ａḍ̇e", "　t", "('", "ſå'll", "\"Z'VE", " \"", "\té", "#$%", "mt"]} +{"text": "'e߅!å''MⅣ㋿'VE", "tokens": 16, "pieces": ["'eß", "…", "!å", "''", "M", "Ⅳ", "㋿'", "VE"]} +{"text": "🙂fi<|fim_prefix|>👍🏽é'D'Re\r'ſ", "tokens": 18, "pieces": ["🙂fi", "<|", "fim", "_prefix", "|>👍🏽", "é'D", "'Re", "\r", "'ſ"]} +{"text": "　́\r\ń\n😀🏽'ſfi'Re0éß", "tokens": 15, "pieces": ["　́", "\r\n", "́", "\n", "😀🏽'", "ſfi'Re", "0", "éß"]} +{"text": "㍿'re'SＡ字", "tokens": 8, "pieces": ["㍿'", "re'S", "Ａ字"]} +{"text": "é漢\"m#$%!!\r\n\r\n३👍🏽EOT\t漢 9 'D\"٣٤٥٦\r\n\r\n\r\nꟲ#$%́'M- \nعDž \n😀🏽9!", "tokens": 56, "pieces": ["é漢", "\"m", "#$%!!<", "EOT", ">\r\n\r\n", "३", "👍🏽", "EOT", "\t", "漢", " ", " ", "9", " '", "D", "\"", "٣٤٥", "٦", "\r\n\r\n\r\n", "ꟲ", "#$%́'", "M", "-", " \n", "ع", "Dž", " \n", "😀🏽", "9", "!<", "EOT", ">"]} +{"text": "d-\te d👍🏽,½‍åå٣٤٥٦", "tokens": 19, "pieces": ["d", "-", "\te", " d", "👍🏽,", "½", "‍åå", "٣٤٥", "٦"]} +{"text": "\r\nEOTś٣٤٥٦'D0'S👍🏽字", "tokens": 18, "pieces": ["\r\n", "EOT", "ś", "٣٤٥", "٦", "'D", "0", "'S", "👍🏽", "字"]} +{"text": ",é½", "tokens": 3, "pieces": [",é", "½"]} +{"text": "eEOT👍🏽'ſ <|endoftext|>", "tokens": 16, "pieces": ["e", "EOT", "👍🏽'", "ſ", " ", " <|", "endoftext", "|>"]} +{"text": "ḍ̇\tß𐞁\r'VE\r\n12345678'llſ\u000bé३🙂.­'VEꟲ<|fim_prefix|>\u000b㍿9Z㍿\r\n\r\n0ſ >́'Mfi,'S漢\u000b'<|fim_prefix|>", "tokens": 53, "pieces": ["-", "9", "'ll", "‍!", "漢t", "Dž", "<|", "endoftext", "|>'", "VEꟲ", "<|", "fim", "_prefix", "|>", "\u000b", "㍿", "9", "Z", "㍿\r\n\r\n", "0", "ſ", " ", ">́'M", "fi", ",'", "S漢", "\u000b", "'<|", "fim", "_prefix", "|>"]} +{"text": "'D!'ll㍿aZ>­EOT­", "EOT", "　\t,'ll<'SⅣ漢'!!(a½#$%é㋿eEOT😀🏽t#$%9 \t٣٤٥٦<|endoftext|>'Re　a", "tokens": 63, "pieces": ["EOTs'VE", "#$%<|", "fim", "_prefix", "|>", "　", "\t", ",'", "ll", "<'", "S", "Ⅳ", "漢", "'!!(", "a", "½", "#$%", "é", "㋿e", "EOT", "😀🏽", "t", "#$%", "9", " ", "\t", "٣٤٥", "٦", "<|", "endoftext", "|>'", "Re", "　a"]} +{"text": "\"09\r\n'T", "tokens": 4, "pieces": ["\"", "09", "\r\n", "'T"]} +{"text": "́😀🏽EOT\r<|fim_prefix|>Z'T٣٤٥٦é㍿ß
 ½\"''VE
12345678عtZ<|endoftext|>", "tokens": 50, "pieces": ["́", "😀🏽", "EOT", "\r", "<|", "fim", "_prefix", "|>", "Z'T", "٣٤٥", "٦", "é", "㍿ß", "
", " ", "½", "\"''", "VE", "
", "123", "456", "78", "عt", "Z", "<|", "endoftext", "|>"]} +{"text": "d!< \n㋿t😀🏽12345678㋿…e<(‍>ع'Tſ", "tokens": 30, "pieces": ["d", "!<", " \n", "㋿t", "😀🏽", "123", "456", "78", "㋿", "…e", "<(‍>", "ع'T", "ſ"]} +{"text": "🙂", "tokens": 1, "pieces": ["🙂"]} +{"text": "dfi​'DⅣéZ- \nſ", "tokens": 13, "pieces": ["dfi", "​'", "D", "Ⅳ", "é", "Z", "-", " \n", "ſ"]} +{"text": "'Tm👍🏽0🙂12345678́!!½s \n٣٤٥٦\re#$%㋿漢EOT", "tokens": 29, "pieces": ["'Tm", "👍🏽", "0", "🙂", "123", "456", "78", "́", "!!", "½", "s", " \n", "٣٤٥", "٦", "\r", "e", "#$%㋿", "漢", "EOT"]} +{"text": "㍿mméZ-ع,DžDž'S½\r\n\r\n'T㋿\"tm<ꟲ'Re\r\n\r\nEOT<|endoftext|>​", "tokens": 41, "pieces": ["㍿mmé", "Z", "-ع", ",<", "EOT", ">DžDž'S", "½", "\r\n\r\n", "'T", "㋿\"", "tm", "<ꟲ'Re", "\r\n\r\n", "EOT", "<|", "endoftext", "|>​"]} +{"text": "d\t𐞁m\tém३½👍🏽 \"\tⅣ-'ſ'M", "tokens": 23, "pieces": ["d", "\t𐞁m", "\tém", "३½", "👍🏽", " ", " \"", "\t", "Ⅳ", "-'", "ſ'M"]} +{"text": "  m३!!'Re　́e!!ßḍ̇ß İ\n'Té<|endoftext|>9 ", "tokens": 32, "pieces": [" ", " m", "३", "!!'", "Re", "　́", "e", "!!", "ßḍ̇ß", " İ", "\n", "'Té", "<|", "endoftext", "|>", "9", " "]} +{"text": "ḍ̇s\r\r\n\r\n<\t\n🙂>0ß'Re\r\n\r\nＡEOTdＡt9'T.😀🏽!fi'llé<|endoftext|>'T‍ꟲ
", "tokens": 51, "pieces": ["ḍ̇s", "\r\r\n\r\n", "<", "\t\n", "🙂>", "0", "ß'Re", "\r\n\r\n", "ＡEOTd", "Ａt", "9", "'T", ".😀🏽<", "META", "_START", ">!", "fi'll", "é", "<|", "endoftext", "|><", "EOT", ">'", "T", "‍ꟲ", "
"]} +{"text": "\n\r\n#$% ́\u000bm.\t<|endoftext|>　ꟲa Ａ­㍿́㋿(́'ll'😀🏽t​", "tokens": 48, "pieces": ["\n\r\n", "#$%", " ́", "\u000b", "m", ".", "\t", "<|", "endoftext", "|>", "　ꟲa", " ", " Ａ", "­㍿́㋿(́'", "ll", "'<", "META", "_START", ">😀🏽", "t", "​"]} +{"text": "ſ\"½é>‍#$%🙂're　½ \n<👍🏽am漢­.12345678 \n\t'ſ", "tokens": 29, "pieces": ["ſ", "\"", "½", "é", ">‍#$%🙂'", "re", "　", "½", " \n", "<👍🏽", "am漢", "­.", "123", "456", "78", " \n", "\t", "'ſ"]} +{"text": "㍿'VEå'VE \nEOT <|fim_prefix|>\nåaå'S\n\r‍", "tokens": 27, "pieces": ["㍿'", "VEå'VE", " \n", "EOT", " ", " <|", "fim", "_prefix", "|>\n", "åaå'S", "\n\r", "‍"]} +{"text": "'Re'll#$%!-'s<­d'Re​a0😀🏽 'S漢\t", "tokens": 24, "pieces": ["'Re'll", "#$%!-<", "META", "_START", ">'", "s", "<­", "d'Re", "​a", "0", "😀🏽", " '", "S漢", "\t"]} +{"text": "fiDž😀🏽m''Re!! t>a'å\t(́ݽꟲ-DžＡt㍿İ's😀🏽9‍m\ta \"", "tokens": 41, "pieces": ["fi", "Dž", "😀🏽", "m", "''", "Re", "!!", " t", ">a", "'å", "\t", "(́", "İ", "½", "ꟲ", "-DžＡt", "㍿İ's", "😀🏽", "9", "‍m", "\ta", " ", "\""]} +{"text": "­'D're", "tokens": 4, "pieces": ["­'", "D're"]} +{"text": "mḍ̇9ḍ̇fi'M", "tokens": 17, "pieces": ["m", "ḍ̇", "9", "ḍ̇fi'M", ""]} +{"text": "́ \n㍿\r\n\r\n👍🏽👍🏽!!Ａa㋿½Dže­🙂\r\n\r\n'T　!!dZt👍🏽\u000b>ع'S\"mt", "tokens": 47, "pieces": ["́", " \n", "㍿\r\n\r\n", "👍🏽👍🏽!!", "Ａa", "㋿", "½", "Dže", "­🙂\r\n\r\n", "'T", "　", "!!", "d", "Zt", "👍🏽", "\u000b", ">", "ع'S", "\"mt"]} +{"text": "👍🏽#$%\t Z<|endoftext|>'Reḍ̇,0😀🏽! \n!́½\r\n\r\n\tſ'​ꟲ\"\u000bß‍EOT\"​(ß👍🏽", "tokens": 49, "pieces": ["👍🏽#$%", "\t ", " Z", "<|", "endoftext", "|>'", "Reḍ̇", ",", "0", "😀🏽!", " \n", "!́", "½", "\r\n\r\n", "\tſ", "'​", "ꟲ", "\"", "\u000bß", "‍EOT", "\"​(", "ß", "👍🏽"]} +{"text": "
'Sé‍Ⅳ<|fim_prefix|>漢9\r\n\r\n​ßs#$%- …​漢1234567812345678-Ⅳ#$%,,㍿é'saß(字'ſ\u000b", "tokens": 57, "pieces": ["
", "'", "Sé", "‍", "Ⅳ", "<|", "fim", "_prefix", "|>", "漢", "9", "\r\n\r\n", "​ßs", "#$%-", " ", "…", "​漢", "123", "456", "781", "234", "567", "8", "-", "Ⅳ", "#$%,,㍿<", "META", "_START", ">é's", "aß", "(字'ſ", "\u000b"]} +{"text": "(d\r\n\r\n,'Re\u000bꟲ'VE　<|endoftext|>
ḍ̇'ſ 'Re\r\n\r\nİ字👍🏽'M漢 ٣٤٥٦'漢‍ſ​ \n<|fim_prefix|>", "tokens": 57, "pieces": ["(d", "\r\n\r\n", ",'", "Re", "\u000bꟲ'VE", "　", "<|", "endoftext", "|>", "
ḍ̇'ſ", " ", " <", "META", "_START", ">'", "Re", "\r\n\r\n", "İ字", "👍🏽'", "M漢", " ", " ", "٣٤٥", "٦", "'漢", "‍ſ", "​", " \n", "<|", "fim", "_prefix", "|>"]} +{"text": " ३", "tokens": 2, "pieces": [" ", "३"]} +{"text": "​'S<|fim_prefix|>s🙂0-'ſع­t\"㋿　\r\n", "tokens": 22, "pieces": ["​'", "S", "<|", "fim", "_prefix", "|>", "s", "🙂", "0", "-'", "ſع", "­t", "\"㋿", "　\r\n"]} +{"text": "ⅣEOT ", "tokens": 8, "pieces": ["Ⅳ", "EOT", "", " "]} +{"text": "EOT", "tokens": 2, "pieces": ["EOT"]} +{"text": "a'ſ12345678🙂漢​ſḍ̇Z-‍m\n'ſDž", "tokens": 22, "pieces": ["a'ſ", "123", "456", "78", "🙂漢", "​ſḍ̇", "Z", "-‍", "m", "\n", "'ſ", "Dž"]} +{"text": "EOT字'VE", "tokens": 9, "pieces": ["EOT", "字'VE"]} +{"text": "­'re\"\r\n𐞁ع.a'DEOT\"‍\u000b'Re", "tokens": 22, "pieces": ["­'", "re", "\"\r", "\n", "𐞁ع", ".a'D", "EOT", "\"‍", "\u000b", "'Re"]} +{"text": "…,e'ſd''VEＡ٣٤٥٦​DždEOT漢'ſ'll\rſ㍿", "tokens": 34, "pieces": ["…", ",e'ſ", "d", "''", "VE", "Ａ", "٣٤٥", "٦", "​", "Džd", "EOT漢'ſ", "'ll", "\r", "ſ", "㍿"]} +{"text": "㋿,", "tokens": 4, "pieces": ["㋿,"]} +{"text": "ḍ̇㍿٣٤٥٦<|fim_prefix|>t \"İ''Tfí‍å‍'retd'İé\"d", "tokens": 37, "pieces": ["ḍ̇", "㍿", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "t", " \"", "İ", "''", "Tfí", "‍å", "‍'", "ret", "d", "'İé", "\"d"]} +{"text": "'s🙂'D 
عé 's'VE.İ'İ..12345678Ａ\r㋿é ½'ſ'VE-ꟲ #$%<|endoftext|>", "tokens": 67, "pieces": ["'s", "🙂'", "D", " ", "
", "عé", " ", " '", "s'VE", ".İ", "'İ", "..", "123", "456", "78", "Ａ", "\r", "㋿é", " ", "½", "'ſ'VE", "-ꟲ", " ", " #$%<|", "endoftext", "|>"]} +{"text": "mＡ🙂​㍿漢'll( eＡß12345678\t#$%३t's　'T!!!!><😀🏽漢Dž're", "tokens": 37, "pieces": ["m", "Ａ", "🙂​㍿", "漢", "'", "ll", "(", " e", "Ａß", "123", "456", "78", "\t", "#$%", "३", "t's", "　", "'T", "!!!!><😀🏽", "漢", "Dž're"]} +{"text": "('S'å's\n‍Ⅳ½\r's'VE 字", "tokens": 16, "pieces": ["('", "S", "'å's", "\n", "‍", "Ⅳ½", "\r", "'s'VE", " ", " 字"]} +{"text": "fi'D㋿", "tokens": 5, "pieces": ["fi'D", "㋿"]} +{"text": "'re", "tokens": 1, "pieces": ["'re"]} +{"text": "'D9…٣٤٥٦!!t
🙂字🙂漢-e'Re　'M​\u000b'ſ", "tokens": 23, "pieces": ["'D", "9", "…", "٣٤٥", "٦", "!!", "t", "
", "🙂字", "🙂漢", "-e'Re", "　", "'M", "​", "\u000b", "'ſ"]} +{"text": "\nḍ̇fiEOT12345678㍿👍🏽㋿Džع\u000b३<'re𐞁'VE'Mßḍ̇\r>𐞁 90'T½ع𐞁 ‍é<|endoftext|>'reDž३12345678.", "tokens": 70, "pieces": ["\n", "ḍ̇fi", "EOT", "123", "456", "78", "㍿👍🏽㋿", "Džع", "\u000b", "३", "<'", "re𐞁'VE", "'Mßḍ̇", "\r", ">𐞁", " ", "90", "'T", "½", "ع𐞁", " ", "‍é", "<|", "endoftext", "|>'", "re", "Dž", "३12", "345", "678", "."]} +{"text": "　e'Re​é9㍿s​'Mİ,Ⅳ12345678ع'sＡs Ⅳ\t.12345678", "tokens": 35, "pieces": ["　e'Re", "​é", "9", "㍿<", "EOT", ">s", "​'", "Mİ", ",", "Ⅳ12", "345", "678", "ع's", "Ａs", " ", "Ⅳ", "\t", ".", "123", "456", "78"]} +{"text": "'EOT\"…𐞁9é🙂're9,>a,'VEmꟲ-ß٣٤٥٦", "tokens": 30, "pieces": ["'EOT", "\"", "…𐞁", "9", "é", "🙂'", "re", "9", ",>", "a", ",'", "VEmꟲ", "-ß", "٣٤٥", "٦"]} +{"text": "ꟲ­　!!­'S's😀🏽…­Z#$%'re　sDž'll12345678'Sعs٣٤٥٦́३t­ ", "tokens": 40, "pieces": ["ꟲ", "­", "　", "!!­'", "S's", "😀🏽", "…", "­Z", "#$%'", "re", "　s", "Dž'll", "123", "456", "78", "'Sعs", "٣٤٥", "٦", "́", "३", "t", "­", " "]} +{"text": "s\"é<|endoftext|>👍🏽!漢>ḍ̇\u000b\tⅣ\u000b12345678𐞁\r\n字\u000bm!㍿('VE'ſ0 \n㍿​'M\r\n\r\n'M12345678fi㋿tⅣ", "tokens": 64, "pieces": ["s", "\"é", "<|", "endoftext", "|>👍🏽!", "漢", ">ḍ̇", "\u000b", "\t", "Ⅳ", "\u000b", "123", "456", "78", "𐞁", "\r\n", "字", "\u000bm", "!㍿('", "VE'ſ", "0", " \n", "㍿​'", "M", "\r\n\r\n", "'M", "123", "456", "78", "fi", "㋿t", "Ⅳ"]} +{"text": "12345678\n(ß㋿Ａ́12345678'S½<|endoftext|>", "tokens": 25, "pieces": ["123", "456", "78", "\n", "(ß", "㋿Ａ́", "123", "456", "78", "'S", "½", "<|", "endoftext", "|>"]} +{"text": "‍㍿", "tokens": 4, "pieces": ["‍㍿"]} +{"text": "é#$%9s\r\nſ'VE<|endoftext|>'ll👍🏽sfi'Re'reå", "tokens": 27, "pieces": ["é", "#$%", "9", "s", "\r\n", "ſ'VE", "<|", "endoftext", "|>'", "ll", "👍🏽", "sfi'Re", "'reå"]} +{"text": "Ⅳꟲåå(å<|fim_prefix|>!sİ!\t \n'ſ'ſ३　éDž
́é㍿Dž\r\ńs٣٤٥٦<|endoftext|><|fim_prefix|>", "tokens": 58, "pieces": ["Ⅳ", "ꟲåå", "(å", "<|", "fim", "_prefix", "|>!", "s", "İ", "!", "\t \n", "'ſ'ſ", "३", "　é", "Dž", "
́é", "㍿Dž", "\r\n", "́s", "٣٤٥", "٦", "<|", "endoftext", "|><|", "fim", "_prefix", "|>"]} +{"text": "𐞁\n½٣٤٥٦Ａ㋿0\"å!fi字́'ll ", "tokens": 23, "pieces": ["𐞁", "\n", "½٣٤", "٥٦", "Ａ", "㋿", "0", "\"å", "!fi字́'ll", " "]} +{"text": "ꟲⅣ-ꟲaé漢­!'Sḍ̇EOT!İas", "tokens": 23, "pieces": ["ꟲ", "Ⅳ", "-ꟲaé漢", "­!'", "Sḍ̇", "EOT", "!İas"]} +{"text": "Dž'T<|fim_prefix|>字㋿́漢'Dm12345678 ", "tokens": 21, "pieces": ["Dž'T", "<|", "fim", "_prefix", "|>", "字", "㋿́漢'D", "m", "123", "456", "78", " "]} +{"text": ">", "tokens": 1, "pieces": [">"]} +{"text": "'re>'m#$%…!!𐞁\t\n9ſ㍿EOTİ'漢", "tokens": 23, "pieces": ["'re", ">'", "m", "#$%", "…", "!!", "𐞁", "\t\n", "9", "ſ", "㍿EOTİ", "'漢"]} +{"text": "s#$% fi<|fim_prefix|>\u000b 漢Ａ😀🏽ßfi'M'T'T'MZßＡꟲ>\n0 !#$%٣٤٥٦\r\nDž'VE٣٤٥٦é'll'ret", "tokens": 55, "pieces": ["s", "#$%", " ", " fi", "<|", "fim", "_prefix", "|>", "\u000b", " 漢", "Ａ", "😀🏽", "ßfi'M", "'T'T", "'MZß", "Ａꟲ", ">\n", "0", " !#$%", "٣٤٥", "٦", "\r\n", "Dž'VE", "٣٤٥", "٦", "é'll", "'ret"]} +{"text": "0…'T's\u000bZfi", "tokens": 8, "pieces": ["0", "…", "'T's", "\u000bZfi"]} +{"text": "e漢dé\r\n\r\n'M", "tokens": 10, "pieces": ["e漢", "dé", "\r\n\r\n", "'M"]} +{"text": "😀🏽", "tokens": 3, "pieces": ["😀🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "-é'S<Ⅳ३ ع\tſ!! \n", "tokens": 12, "pieces": ["-é'S", "<", "Ⅳ३", " ع", "\tſ", "!!", " \n"]} +{"text": "<", "tokens": 1, "pieces": ["<"]} +{"text": "Z㋿0!!½'ll('Tt'remDžEOT9>'S \u000bDž. \u000bⅣfia३'re漢'M'Re", "tokens": 36, "pieces": ["Z", "㋿", "0", "!!", "½", "'ll", "('", "Tt're", "m", "DžEOT", "9", ">'", "S", " ", "\u000bDž", ".", " ", "\u000b", "Ⅳ", "fia", "३", "'re漢'M", "'Re"]} +{"text": "😀🏽!‍>'D \na‍ Ａ", "tokens": 12, "pieces": ["😀🏽!‍>'", "D", " \n", "a", "‍", " Ａ"]} +{"text": "Z<|fim_prefix|>𐞁'llé", "tokens": 17, "pieces": ["Z", "<|", "fim", "_prefix", "|>", "𐞁", "'", "llé"]} +{"text": "ＡEOT<Dž's👍🏽'VE'VEİ\r\n\r\nEOT! \n३9t", "tokens": 23, "pieces": ["ＡEOT", "<Dž's", "👍🏽'", "VE'VE", "İ", "\r\n\r\n", "EOT", "!", " \n", "३9", "t"]} +{"text": "\r\n½\ne", "tokens": 4, "pieces": ["\r\n", "½", "\n", "e"]} +{"text": "'DEOTeꟲs\rEOT㋿­'ſ12345678'", "tokens": 26, "pieces": ["'", "DEOTeꟲs", "\r", "EOT", "㋿­<", "EOT", ">'", "ſ", "123", "456", "78", "'"]} +{"text": "­ſ٣٤٥٦å12345678", "tokens": 11, "pieces": ["­ſ", "٣٤٥", "٦", "å", "123", "456", "78"]} +{"text": "12345678'VEＡ½sDž字", "tokens": 11, "pieces": ["123", "456", "78", "'VEＡ", "½", "s", "Dž字"]} +{"text": "३'M
(…ZꟲⅣ'VE字½𐞁 ‍", "tokens": 25, "pieces": ["३", "'M", "
", "(", "…Zꟲ", "Ⅳ", "'VE字", "½", "𐞁", " ", "‍"]} +{"text": "३'(é𐞁३!½'Sfié!#$%'ſ㍿ß '(ſ'Refi,!𐞁'llt'ſ　🙂 ß😀🏽𐞁és#$%'", "tokens": 50, "pieces": ["३", "'(", "é𐞁", "३", "!", "½", "'Sfié", "!#$%'", "ſ", "㍿ß", " ", "'(", "ſ'Re", "fi", ",!", "𐞁'll", "t'ſ", "　", "🙂", " ß", "😀🏽", "𐞁és", "#$%'"]} +{"text": "👍🏽𐞁é\r\n\r\n!!'ReſßعZ'T!!İ字ſEOT́-é३　漢'll字漢ſmⅣ,👍🏽m(", "tokens": 41, "pieces": ["👍🏽", "𐞁é", "\r\n\r\n", "!!'", "Reſßع", "Z'T", "!!", "İ字ſ", "EOT́", "-é", "३", "　漢'll", "字漢ſm", "Ⅳ", ",👍🏽", "m", "("]} +{"text": "　\t<|fim_prefix|>𐞁'll'Tdꟲḍ̇🙂'T\r\né́e​Ⅳ", "tokens": 32, "pieces": ["　", "\t", "<|", "fim", "_prefix", "|>", "𐞁'll", "'Tdꟲḍ̇", "🙂'", "T", "\r\n", "é́e", "​", "Ⅳ"]} +{"text": "ſZß漢!!ꟲ!!㋿㋿ Ⅳ", "tokens": 22, "pieces": ["ſ", "Z", "ß漢", "!!", "ꟲ", "!!㋿㋿", " ", "Ⅳ"]} +{"text": "!ḍ̇​dsåm\r\n\r\n><|endoftext|>,'Re<|fim_prefix|>'llfis\r\n\r\n½", "tokens": 33, "pieces": ["!ḍ̇", "​dsåm", "\r\n\r\n", "><|", "endoftext", "|>,<", "EOT", ">'", "Re", "<|", "fim", "_prefix", "|>'", "llfis", "\r\n\r\n", "½"]} +{"text": "tEOT'D('Re'M><|endoftext|>\u000b‍,0İ's're'T\tＡ", "tokens": 23, "pieces": ["t", "EOT'D", "('", "Re'M", "><|", "endoftext", "|>", "\u000b", "‍,", "0", "İ's", "'re'T", "\tＡ"]} +{"text": "㋿'M'll-ḍ̇é
9Ⅳfi!!><|endoftext|>é字　#$%​0'Re!!İ'll9<字<…ſå", "tokens": 49, "pieces": ["㋿'", "M'll", "-ḍ̇é", "
", "", "9Ⅳ", "fi", "!!><|", "endoftext", "|>", "é字", "　", "#$%​", "0", "'Re", "!!", "İ'll", "9", "<字", "<", "…ſå"]} +{"text": "Z'\u000b٣٤٥٦ſ
\u000b''D'D‍m½\nZ……<'T >ém'MaＡ\u000b\r\n \n å\u000bå½\r#$%9", "tokens": 48, "pieces": ["Z", "'", "\u000b", "٣٤٥", "٦", "ſ", "
", "\u000b", "''", "D'D", "‍m", "½", "\n", "Z", "…", "…", "<'", "T", " ", " >", "ém'M", "a", "Ａ", "\u000b\r\n \n", " ", " å", "\u000bå", "", "½", "\r", "#$%", "9"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "😀🏽İDž٣٤٥٦\r're''S\r😀🏽é.0s🙂'ع½ \n's\"(😀🏽\u000b'ſ!!😀🏽İZ漢ſ عé-", "tokens": 50, "pieces": ["😀🏽", "İDž", "٣٤٥", "٦", "\r", "'", "re", "''", "S", "\r", "😀🏽", "é", ".", "0", "s", "🙂'", "ع", "½", " \n", "'s", "\"(😀🏽", "\u000b", "'ſ", "!!😀🏽", "İZ漢ſ", " عé", "-"]} +{"text": "\r\nEOT
å(٣٤٥٦\t#$%'lld ٣٤٥٦9#$%㋿- ,
 \n\rtḍ̇>​Z", "tokens": 47, "pieces": ["\r\n", "EOT", "
å", "(", "٣٤٥", "٦", "\t", "#$%'", "lld", " ", "٣٤٥", "٦9", "#$%㋿-", " ", " ,", "
 \n\r", "tḍ̇", ">​", "Z", ""]} +{"text": "9​ ſ𐞁Ⅳ'ſꟲDž😀🏽عßZ㋿afi…'re'lla\t …字́३Ⅳ'st", "tokens": 44, "pieces": ["9", "​", " ", " ſ𐞁", "Ⅳ", "'ſꟲ", "Dž", "😀🏽", "عß", "Z", "㋿afi", "…", "'re'll", "a", "\t ", "…字́", "३Ⅳ", "'st"]} +{"text": "👍🏽!!\n\nⅣＡm,٣٤٥٦,Ａ,Zß'ſ 12345678\n9ḍ̇Ⅳß \n🙂é 👍🏽<|fim_prefix|><|endoftext|>", "tokens": 51, "pieces": ["👍🏽!!\n\n", "Ⅳ", "Ａm", ",", "٣٤٥", "٦", ",Ａ", ",Zß'ſ", " ", "123", "456", "78", "\n", "9", "ḍ̇", "Ⅳ", "ß", " \n", "🙂é", " ", " 👍🏽<|", "fim", "_prefix", "|><|", "endoftext", "|>"]} +{"text": ".३'S'S\"a'D­٣٤٥٦
'llＡＡ𐞁0字9", "tokens": 22, "pieces": [".", "३", "'S'S", "\"a'D", "­", "٣٤٥", "٦", "
", "'ll", "ＡＡ𐞁", "0", "字", "9"]} +{"text": "\r\n'!!é'll<|endoftext|>\t'MⅣt漢‍‍𐞁Ａ'ſe>​ EOT'Reſ'-𐞁Z's", "tokens": 45, "pieces": ["\r\n", "'!!", "é'll", "<|", "endoftext", "|>", "\t", "'M", "Ⅳ", "t漢", "‍‍", "𐞁", "Ａ'ſ", "e", ">​", " EOT'Re", "ſ", "'-", "𐞁", "Z's"]} +{"text": "\t<|endoftext|>👍🏽ß'Re漢a9>éå'D12345678İDž…9\t㍿Ａt", "tokens": 37, "pieces": ["\t", "<|", "endoftext", "|>👍🏽", "ß'Re", "漢a", "9", ">éå'D", "123", "456", "78", "İDž", "…", "9", "\t", "㍿Ａt"]} +{"text": "漢Dž!é'M‍'VEꟲ\t㋿'ſ…'så're㋿İ9fi!!<|endoftext|>", "tokens": 38, "pieces": ["漢", "Dž", "!é'M", "‍'", "VEꟲ", "\t", "㋿'", "ſ", "…", "'så're", "㋿İ", "9", "fi", "!!<|", "endoftext", "|>"]} +{"text": "́
fi'Rem\u000b!å'Reſ㋿ …", "tokens": 20, "pieces": ["́", "
fi'Re", "m", "\u000b", "!å'Re", "ſ", "㋿", " …"]} +{"text": "<|endoftext|>a(,'DDž\r\n 'T'Så‍　a ㍿\r<|endoftext|>", "tokens": 35, "pieces": ["<|", "endoftext", "|>", "a", "(,'", "DDž", "\r\n", " '", "T'S", "å", "‍", "　a", " ", " ㍿\r", "<|", "endoftext", "|>"]} +{"text": "‍<<|fim_prefix|>9", "tokens": 8, "pieces": ["‍<<|", "fim", "_prefix", "|>", "9"]} +{"text": "​‍ع٣٤٥٦ḍ̇👍🏽!३ Dž​
\n(字　٣٤٥٦t,\r\n'M'T's'Dfi𐞁-ḍ̇dß𐞁", "tokens": 51, "pieces": ["​‍", "ع", "٣٤٥", "٦", "ḍ̇", "👍🏽!", "३", " Dž", "​", "
\n", "(字", "　", "٣٤٥", "٦", "t", ",\r\n", "'M'T", "'s'D", "fi𐞁", "-", "ḍ̇dß𐞁"]} +{"text": "́'reⅣß'll👍🏽\r\n\r\n​👍🏽'\r\n\r\nß㍿", "tokens": 22, "pieces": ["́", "'", "re", "Ⅳ", "ß'll", "👍🏽\r\n\r\n", "​👍🏽'\r\n\r\n", "ß", "㍿"]} +{"text": "å…　'M,eع'Ḿ\r‍🙂\" å‍İع𐞁m'TtEOT'S　　'D", "tokens": 33, "pieces": ["å", "…", "　", "'M", ",eع'M", "́", "\r", "‍🙂\"", " å", "‍İع𐞁m'T", "t", "EOT'S", "　", "　", "'D"]} +{"text": "'ReＡ…,'s\n　", "tokens": 8, "pieces": ["'Re", "Ａ", "…", ",'", "s", "\n", "　"]} +{"text": "aé é're👍🏽a\r\n\r\n字ḍ̇‍12345678('Re's0‍>éé,", "tokens": 33, "pieces": ["aé", " ", " é're", "👍🏽", "a", "\r\n\r\n", "字ḍ̇", "‍", "123", "456", "78", "('", "Re's", "0", "‍>", "éé", ","]} +{"text": "0́…½ße‍a𐞁٣٤٥٦<|endoftext|>
", "tokens": 24, "pieces": ["0", "́", "…", "½", "ße", "‍a𐞁", "٣٤٥", "٦", "<|", "endoftext", "|>", "
"]} +{"text": "fimİ<|endoftext|>'M<|fim_prefix|>३ \n
.字Ａİt漢é'Re12345678", "tokens": 32, "pieces": ["fim", "İ", "<|", "endoftext", "|>'", "M", "<|", "fim", "_prefix", "|>", "३", " \n", "
", ".字Ａİt漢é'Re", "123", "456", "78"]} +{"text": "٣٤٥٦ſ Ａ\u000bİſ're<|fim_prefix|>ع'll…\r\n\r\n!!'Dİ9'Mعåd३　0‍‍\r\n\r\nd'D­", "tokens": 46, "pieces": ["٣٤٥", "٦", "ſ", " ", " Ａ", "\u000bİſ're", "<|", "fim", "_prefix", "|>", "ع'll", "…\r\n\r\n", "!!'", "Dİ", "9", "'M", "عåd", "३", "　", "0", "‍‍\r\n\r\n", "d'D", "­"]} +{"text": "ꟲe0\r😀🏽 \nḍ̇'S
<|endoftext|>EOT\u000b!!Ａİ\n'Reſ", "tokens": 31, "pieces": ["ꟲe", "0", "\r", "😀🏽", " \n", "ḍ̇'S", "
", "<|", "endoftext", "|>", "EOT", "\u000b", "!!", "Ａİ", "\n", "'Reſ"]} +{"text": "('T \n𐞁 \nå👍🏽aꟲ!!­'ſ\r\n\r\na'VE‍'Se㍿\"<|fim_prefix|>>İ'S,", "tokens": 40, "pieces": ["('", "T", " \n", "𐞁", " \n", "å", "👍🏽", "aꟲ", "!!­'", "ſ", "\r\n\r\n", "a'VE", "‍'", "Se", "㍿\"<|", "fim", "_prefix", "|>>", "İ'S", ","]} +{"text": "'T'T👍🏽…<|fim_prefix|>'VE\té㋿", "tokens": 19, "pieces": ["'T'T", "👍🏽", "…", "<|", "fim", "_prefix", "|>'", "VE", "\té", "㋿"]} +{"text": "­'re…'T,a…३½ \n-\r \na'VE-ꟲ", "tokens": 22, "pieces": ["­'", "re", "…", "'T", ",a", "…", "३½", " \n", "-\r", " \n", "a'VE", "-ꟲ"]} +{"text": "İ!!é \nⅣ\n…\r\n\r\n
m　", "tokens": 18, "pieces": ["İ", "!!", "é", " \n", "", "Ⅳ", "\n…\r\n\r\n", "
m", "　"]} +{"text": "ع😀🏽'ſ㋿.t åfiZ", "tokens": 15, "pieces": ["ع", "😀🏽'", "ſ", "㋿.", "t", " åfi", "Z"]} +{"text": "'s\r\n!td'M'D!​́ \u000b \n'll😀🏽'Re!!>́9́", "9", "EOT'sع\r\n\r\ndꟲé<㍿efi 🙂'T…'Re", "tokens": 43, "pieces": [".-", "İé", "#$%", "é", "-!!", "e", "　漢", "­", "　", "\u000b", ">EOT's", "ع", "\r\n\r\n", "dꟲé", "<㍿", "efi", " ", "🙂'", "T", "…", "'", "Re"]} +{"text": "\"\u000b're!!\nsſ👍🏽e
…ſ<|endoftext|>'D́s!!é9३\"㍿,
é😀🏽Ⅳ\t<\net-漢fiDžt", "tokens": 50, "pieces": ["\"", "\u000b", "'re", "!!\n", "sſ", "👍🏽", "e", "
", "…ſ", "<|", "endoftext", "|>'", "D́s", "!!", "é", "9३", "\"㍿,", "
é", "😀🏽", "Ⅳ", "\t", "<\n", "et", "-漢fi", "Džt"]} +{"text": "Ⅳ३㋿'ll'S𐞁 fi\u000b<|fim_prefix|>漢'ſ#$% \nm t‍'VE<|endoftext|> ३ 
", "tokens": 44, "pieces": ["Ⅳ३", "㋿'", "ll'S", "𐞁", " fi", "\u000b", "<|", "fim", "_prefix", "|>", "漢'ſ", "#$%", " \n", "m", " t", "‍'", "VE", "<|", "endoftext", "|>", " ", "३", " 
"]} +{"text": "< ꟲ…'s#$%\u000b🙂İs\r\n\r\n㋿dßm>\"​\r\n\r\nⅣ'D𐞁-​'Śtß\r­!漢d…३👍🏽", "tokens": 55, "pieces": ["<", " ", " ꟲ", "…", "'s", "#$%", "\u000b", "🙂İs", "\r\n\r\n", "㋿", "dßm", ">\"​\r\n\r\n", "Ⅳ", "'D𐞁", "-​'", "Śtß", "\r", "­<", "EOT", ">!", "漢d", "…", "३", "👍🏽"]} +{"text": "٣٤٥٦㋿0am<|fim_prefix|>åé
#$%Ⅳ9eⅣ12345678 \n…mßعe㍿ع'", "tokens": 48, "pieces": ["٣٤٥", "٦", "㋿", "0", "am", "<|", "fim", "_prefix", "|>", "åé", "", "
", "#$%", "Ⅳ9", "e", "Ⅳ12", "345", "678", " \n", "…mßعe", "㍿ع", "'"]} +{"text": "<½'Re'm12345678\nm \n漢Džſ", "tokens": 14, "pieces": ["<", "½", "'Re'm", "123", "456", "78", "\n", "m", " \n", "漢Džſ"]} +{"text": "漢<|endoftext|>　'S'sꟲZꟲas<|endoftext|>Ａ㍿́fiſ0'M​912345678ſ", "tokens": 40, "pieces": ["漢", "<|", "endoftext", "|>", "　", "'S's", "ꟲZꟲas", "<|", "endoftext", "|>", "Ａ", "㍿́fiſ", "0", "'M", "​", "912", "345", "678", "ſ"]} +{"text": " 'D३éDžİ'Sd
EOT. t", "tokens": 15, "pieces": [" '", "D", "३", "é", "Džİ'S", "d", "
EOT", ".", " t"]} +{"text": "'ll😀🏽'DfiZ'VE 　-ꟲ
\rꟲe…
", "tokens": 25, "pieces": ["'ll", "😀🏽'", "Dfi", "Z'VE", " ", "　", "-ꟲ", "
\r", "ꟲe", "…
"]} +{"text": "३!\u000bds'll'M\r\n", "tokens": 10, "pieces": ["३", "!", "\u000bds'll", "'M", "\r\n"]} +{"text": "𐞁'll½ḍ̇", "tokens": 9, "pieces": ["𐞁'll", "½", "ḍ̇"]} +{"text": "!!ꟲß ß'D(12345678👍🏽\td \r\n\r\nİＡſ½dd㋿😀🏽 é0éå<|endoftext|>é३", "tokens": 50, "pieces": ["!!", "ꟲß", " ", " ß'D", "(", "123", "456", "78", "👍🏽", "\td", " ", " <", "META", "_START", ">\r\n\r\n", "İＡſ", "½", "dd", "㋿😀🏽", " é", "0", "éå", "<|", "endoftext", "|>", "é", "३"]} +{"text": "''llma㍿'s<|fim_prefix|>ꟲDž\n!​(-#$%…Z👍🏽eé tꟲ.\"\r…Dž😀🏽'll ", "tokens": 50, "pieces": ["''", "llma", "㍿'", "s", "<|", "fim", "_prefix", "|>", "ꟲ", "Dž", "\n", "!​(-#$%", "…Z", "👍🏽", "eé", " tꟲ", ".\"\r", "…Dž", "😀🏽'", "ll", " "]} +{"text": "\r\nt'Re३'re!!\"㍿éß'M\r\n\r\n'Rea½㍿'M<|endoftext|>'ſ'D'Mß🙂Zd字'Re9e \nmé'!!'Re's", "tokens": 47, "pieces": ["\r\n", "t'Re", "३", "'re", "!!\"㍿", "éß'M", "\r\n\r\n", "'Rea", "½", "㍿'", "M", "<|", "endoftext", "|>'", "ſ'D", "'Mß", "🙂Zd字'Re", "9", "e", " \n", "mé", "'!!'", "Re's"]} +{"text": "ß>'Refi12345678ßꟲé🙂\"'S٣٤٥٦!३!𐞁\u000b 👍🏽", "tokens": 32, "pieces": ["ß", ">'", "Refi", "123", "456", "78", "ßꟲé", "🙂\"'", "S", "٣٤٥", "٦", "!", "३", "!𐞁", "\u000b ", " 👍🏽"]} +{"text": "'SZ­t. (𐞁 \n'Reé
EOT\r'll…\r\n㍿ſ!!!İ ſ,\r\nfi\r㍿㍿\"<|fim_prefix|>ع", "tokens": 50, "pieces": ["'SZ", "­t", ".", " ", "(𐞁", " \n", "'Reé", "
EOT", "\r", "'ll", "…\r\n", "㍿ſ", "!!!", "İ", " ", " ſ", ",\r\n", "fi", "\r", "㍿㍿\"<|", "fim", "_prefix", "|>", "ع", ""]} +{"text": "#$%>é!!", "tokens": 6, "pieces": ["#$%>", "é", "!!"]} +{"text": "İḍ̇́9m'T12345678'S12345678s½३́é\r\n\r\n👍🏽'ſ㋿ꟲd 'Reḍ̇‍́😀🏽'Dß're EOT! \r\n\r\n", "tokens": 56, "pieces": ["İḍ̇́", "9", "m'T", "123", "456", "78", "'S", "123", "456", "78", "s", "½३", "́é", "\r\n\r\n", "👍🏽'", "ſ", "㋿ꟲd", " ", "'Reḍ̇", "‍́", "😀🏽'", "Dß're", " EOT", "!", " \r\n\r\n"]} +{"text": ">漢ß🙂're‍'VE!!<|fim_prefix|>ḍ̇", "tokens": 19, "pieces": [">漢ß", "🙂'", "re", "‍'", "VE", "!!<|", "fim", "_prefix", "|>", "ḍ̇"]} +{"text": "'lld'Re!𐞁>!漢Dž12345678\t漢٣٤٥٦㍿", "tokens": 33, "pieces": ["'lld'Re", "!𐞁", ">!", "漢", "Dž", "123", "456", "78", "\t漢", "٣٤٥", "٦", "㍿"]} +{"text": "é<|fim_prefix|>Dž٣٤٥٦½👍🏽½'Re9'll𐞁d12345678́ſa ß>\",'TaＡ \n😀🏽ß!<\r\n\r\nå \n漢<|endoftext|>'llＡ", "tokens": 64, "pieces": ["é", "<|", "fim", "_prefix", "|>", "Dž", "٣٤٥", "٦½", "👍🏽", "½", "'Re", "9", "'ll𐞁d", "123", "456", "78", "́ſa", " ", " ß", ">\",'", "Ta", "Ａ", " \n", "😀🏽", "ß", "!<\r\n\r\n", "å", " \n", "漢", "<|", "endoftext", "|>'", "ll", "Ａ"]} +{"text": "­\t　Z<|endoftext|>EOTé(amd'M㋿ 字a \nḍ̇　é'VEſＡḍ̇'ſß'ſ'　<'s'D<", "tokens": 47, "pieces": ["­", "\t", "　Z", "<|", "endoftext", "|>", "EOTé", "(amd'M", "㋿", " 字a", " \n", "ḍ̇", "　é'VE", "ſ", "Ａḍ̇'ſ", "ß'ſ", "'", "　", "<'", "s'D", "<"]} +{"text": "!!m𐞁", "tokens": 6, "pieces": ["!!", "m𐞁"]} +{"text": "Z३\u000b\r'am! \n,<|endoftext|>㋿🙂EOT㋿字#$%!…٣٤٥٦字<|fim_prefix|>!\rſ'S'D½​½ſ>\u000b
\r\n\r\n0", "tokens": 53, "pieces": ["Z", "३", "\u000b\r", "'am", "!", " \n", ",<|", "endoftext", "|>㋿🙂", "EOT", "㋿字", "#$%!", "…", "٣٤٥", "٦", "字", "<|", "fim", "_prefix", "|>!\r", "ſ'S", "'D", "½", "​", "½", "ſ", ">", "\u000b
\r\n\r\n", "0"]} +{"text": "\r🙂'D!!'lla-­🙂\u000b.'S ​dfi\rꟲİ㋿-\nⅣ\n é 0", "tokens": 33, "pieces": ["\r", "🙂'", "D", "!!'", "lla", "-­🙂", "\u000b", ".'", "S", " ", "​dfi", "\r", "ꟲ", "İ", "㋿-\n", "Ⅳ", "\n", " é", " ", "0"]} +{"text": "\n'll\r'ſ\n're0👍🏽d", "tokens": 15, "pieces": ["\n", "'ll", "\r", "'ſ", "\n", "'re", "0", "👍🏽", "d"]} +{"text": "!!Ａ

😀🏽\r\n३'M'Reꟲ㍿éDž, Ａ<|endoftext|>", "tokens": 31, "pieces": ["!!", "Ａ", "
", "
", "😀🏽\r\n", "३", "'M'Re", "ꟲ", "㍿é", "Dž", ",", " ", " Ａ", "<|", "endoftext", "|>"]} +{"text": "𐞁0-'ſ'VE<|fim_prefix|>a", "tokens": 16, "pieces": ["𐞁", "0", "-'", "ſ'VE", "<|", "fim", "_prefix", "|>", "a"]} +{"text": "éＡ😀🏽
EOT㍿　😀🏽'VE! ½å'lléⅣ,　<\u000b#$%漢.\tts\"'T<|fim_prefix|>\r,ßé'SédＡDž", "tokens": 57, "pieces": ["é", "Ａ", "😀🏽", "
EOT", "㍿", "　", "😀🏽'", "VE", "!", " ", " ", "½", "å'll", "é", "Ⅳ", ",", "　", "<", "\u000b", "#$%", "漢", ".", "\tts", "\"'", "T", "<|", "fim", "_prefix", "|>\r", ",ßé'S", "éd", "ＡDž"]} +{"text": "'T३​漢'll0٣٤٥٦ꟲ ", "tokens": 14, "pieces": ["'T", "३", "​漢'll", "0٣٤", "٥٦", "ꟲ", " "]} +{"text": "㍿m,㍿t\t
<|endoftext|>0tt123456780t'ſß're\u000be㍿٣٤٥٦'llⅣ'ſ½dåꟲ३!!9ḍ̇🙂", "tokens": 55, "pieces": ["㍿m", ",㍿", "t", "\t", "
", "<|", "endoftext", "|>", "0", "tt", "123", "456", "780", "t'ſ", "ß're", "\u000be", "㍿", "٣٤٥", "٦", "'ll", "Ⅳ", "'ſ", "½", "dåꟲ", "३", "!!", "9", "ḍ̇", "🙂"]} +{"text": "!!t,'reDž12345678,s \r\n\r\n<|endoftext|>'Sḍ̇Ａ'S,🙂é'VE12345678éß
\r\n\r\n\r\n\r\n'S\r\nDž'T<|fim_prefix|>é३ſⅣ!!३
İ🙂", "tokens": 60, "pieces": ["!!", "t", ",'", "re", "Dž", "123", "456", "78", ",s", " \r\n\r\n", "<|", "endoftext", "|>'", "Sḍ̇", "Ａ'S", ",🙂", "é'VE", "123", "456", "78", "éß", "
\r\n\r\n\r\n\r\n", "'S", "\r\n", "Dž'T", "<|", "fim", "_prefix", "|>", "é", "३", "ſ", "Ⅳ", "!!", "३", "
İ", "🙂"]} +{"text": "'M0\r\n\r\n\r0ßEOT'reＡ
", "tokens": 11, "pieces": ["'M", "0", "\r\n\r\n\r", "0", "ß", "EOT're", "Ａ", "
"]} +{"text": "å‍Ａ's9'ſ🙂漢's
\u000b\t
a\"👍🏽عع-0Džfi\u000bt", "tokens": 29, "pieces": ["å", "‍Ａ's", "9", "'ſ", "🙂漢's", "
\u000b\t", "
a", "\"👍🏽", "عع", "-", "0", "Džfi", "\u000bt"]} +{"text": "'set", "tokens": 2, "pieces": ["'set"]} +{"text": "…𐞁fiꟲ", "tokens": 10, "pieces": ["…𐞁fiꟲ"]} +{"text": "'re漢\r\n\u000b'Re㍿ee!!fisḍ̇​३\n<|endoftext|>😀🏽,t'ſ", "tokens": 32, "pieces": ["'re漢", "\r\n", "\u000b", "'Re", "㍿ee", "!!", "fisḍ̇", "​", "३", "\n", "<|", "endoftext", "|>😀🏽,", "t'ſ"]} +{"text": "🙂 \n𐞁'T­'re(İ'reDž-'reDžs½\r\n>🙂😀🏽ḍ̇'re\r漢́", "tokens": 34, "pieces": ["🙂", " \n", "𐞁'T", "­'", "re", "(İ're", "Dž", "-'", "re", "Džs", "½", "\r\n", ">🙂😀🏽", "ḍ̇'re", "\r", "漢́"]} +{"text": "'lĺ -d㋿\n!!'Re\r\n.é'D­å𐞁\u000bꟲ\r\n\r\n'S!!<|fim_prefix|>­e٣٤٥٦", "tokens": 41, "pieces": ["'lĺ", " ", " -", "d", "㋿\n", "!!'", "Re", "\r\n", ".é'D", "­å𐞁", "\u000bꟲ", "\r\n\r\n", "'S", "!!<|", "fim", "_prefix", "|>­", "e", "٣٤٥", "٦"]} +{"text": "m'T­>d\u000bＡ'sⅣ fia🙂\u000b👍🏽fi👍🏽ſtع", "tokens": 25, "pieces": ["m'T", "­>", "d", "\u000bＡ's", "Ⅳ", " fia", "🙂", "\u000b", "👍🏽", "fi", "👍🏽", "ſtع"]} +{"text": "'!\u000b㋿.9३t\u000ba.'Te 'D٣٤٥٦Ⅳ​ İ\"m 'Re'Re𐞁　<|endoftext|> ㋿\rꟲ😀🏽", "tokens": 57, "pieces": ["'!", "\u000b", "㋿.", "9३", "t", "\u000ba", ".'", "Te", " ", "'D", "٣٤٥", "٦Ⅳ", "​", " İ", "\"m", " ", " '", "Re'Re", "𐞁", "　", "<|", "endoftext", "|>", " ㋿\r", "ꟲ", "😀🏽"]} +{"text": "\n'Re㍿\nm‍'½ḍ̇Dž \n…  ३.é", "tokens": 27, "pieces": ["\n", "'Re", "㍿\n", "m", "‍'", "½", "ḍ̇", "Dž", " \n", "…  ", " ", "३", ".é"]} +{"text": "ḍ̇㍿'S🙂>\"<|endoftext|>Ａ'३é!!　<|fim_prefix|>'३\u000b'T\r\" e🙂", "tokens": 44, "pieces": ["ḍ̇", "㍿'", "S", "🙂>\"<|", "endoftext", "|>", "Ａ", "'", "३", "é", "!!<", "EOT", ">", "　", "<|", "fim", "_prefix", "|>'", "३", "\u000b", "'T", "\r", "\"", " ", " e", "🙂"]} +{"text": "३𐞁m㋿ＡⅣ", "tokens": 12, "pieces": ["३", "𐞁m", "㋿Ａ", "Ⅳ"]} +{"text": "fi ", "tokens": 2, "pieces": ["fi", " "]} +{"text": " s're​EOT'reꟲ½'sfié́", "tokens": 19, "pieces": [" ", " s're", "​EOT're", "ꟲ", "½", "'sfié́"]} +{"text": " \t­'D🙂'ſ漢d", "tokens": 35, "pieces": [" ", "\t", "­'", "D", "🙂'", "ſ", "<", "EOT", ">漢d"]} +{"text": "字ꟲDž́🙂<|endoftext|>tＡ's12345678­<|fim_prefix|>'ſ́Z👍🏽mＡś's'D'VE#$%'VE\téß㍿ß'M", "tokens": 59, "pieces": ["字ꟲDž́", "🙂<|", "endoftext", "|>", "t", "Ａ's", "123", "456", "78", "­<|", "fim", "_prefix", "|>'", "ſ́", "Z", "👍🏽", "m", "Ａs", "́'s", "'", "D'VE", "#$%'", "VE", "\téß", "㍿ß'M"]} +{"text": "0
३'ll🙂'll", "tokens": 7, "pieces": ["0", "
", "३", "'ll", "🙂'", "ll"]} +{"text": "\r!!'T!!ꟲe٣٤٥٦fi<|fim_prefix|>👍🏽-#$%́𐞁\r\n#$%㍿'sé㍿", "tokens": 43, "pieces": ["\r", "!!'", "T", "!!", "ꟲe", "٣٤٥", "٦", "fi", "<|", "fim", "_prefix", "|>👍🏽-#$%́", "𐞁", "\r\n", "#$%㍿'", "sé", "㍿"]} +{"text": "'D", "tokens": 4, "pieces": ["'", "D"]} +{"text": "ꟲ.­Dž's\r\n 'VE \n٣٤٥٦é!!9fi-𐞁\r\u000b12345678\r\n.EOT12345678İ'ſaſＡ-㍿ \n0", "tokens": 59, "pieces": ["ꟲ", ".­", "Dž's", "\r\n", " '", "VE", " \n", "٣٤٥", "٦", "é", "!!", "9", "fi", "-𐞁", "\r", "\u000b", "123", "456", "78", "\r\n", ".EOT", "", "123", "456", "78", "İ'ſ", "a", "ſ", "Ａ", "-㍿", " \n", "0"]} +{"text": "😀🏽s.​㍿,…å-e'VE9
fi  ", "tokens": 21, "pieces": ["😀🏽", "s", ".​㍿,", "…å", "-e'VE", "9", "
fi", "  "]} +{"text": "\u000b12345678<|endoftext|>", "tokens": 11, "pieces": ["\u000b", "123", "456", "78", "<|", "endoftext", "|>"]} +{"text": "ḍ̇Dž's.\n,㍿𐞁㍿s9'Ta \n<|endoftext|>́👍🏽e'Re'lls \n'll.d
.…m", "tokens": 46, "pieces": ["ḍ̇", "Dž's", ".\n", ",㍿", "𐞁", "㍿s", "9", "'Ta", " \n", "<|", "endoftext", "|>́👍🏽", "e'Re", "'lls", " \n", "'ll", ".d", "
", ".", "…m"]} +{"text": "(㋿#$%'M\r 'DmEOT're'reeå­\u000b,å字aİ👍🏽‍‍fiaḍ̇'re'VE \n,'T!!'så\r", "tokens": 48, "pieces": ["(㋿#$%'", "M", "\r", " ", "'Dm", "EOT're", "'reeå", "­", "\u000b", ",å字", "a", "İ", "👍🏽‍‍", "fiaḍ̇'re", "'VE", " \n", ",'", "T", "!!'", "så", "\r"]} +{"text": ".mİéå😀🏽\r½𐞁ßİ३…<عé'Re\r\r\n\r\n ß​åddع0", "tokens": 34, "pieces": [".m", "İéå", "😀🏽\r", "½", "𐞁ß", "İ", "३", "…", "<عé'Re", "\r\r\n\r\n", " ", " ß", "​åddع", "0"]} +{"text": "#$%\nſd३!!३ḍ̇㋿12345678'll, \n'ſ!! <|fim_prefix|>'ſé㍿,0…'ll३'TDž!٣٤٥٦Z😀🏽00ＡZ", "tokens": 58, "pieces": ["#$%<", "META", "_START", ">\n", "ſd", "३", "!!", "३", "ḍ̇", "㋿", "123", "456", "78", "'ll", ",", " \n", "'ſ", "!!", " <|", "fim", "_prefix", "|>'", "ſé", "㍿,", "0", "…", "'ll", "३", "'TDž", "!", "٣٤٥", "٦", "Z", "😀🏽", "00", "ＡZ"]} +{"text": "\t'DZ(><,!!عdDžm\nat-👍🏽\tß😀🏽३­́½'Sfi漢mع", "tokens": 42, "pieces": ["\t", "'DZ", "(><,!!", "عd", "Džm", "\n", "at", "-👍🏽", "\tß", "😀🏽", "३", "­́", "½", "'Sfi", "漢", "mع"]} +{"text": "ß٣٤٥٦ꟲ fi
a", "tokens": 11, "pieces": ["ß", "٣٤٥", "٦", "ꟲ", " fi", "
a"]} +{"text": "　'Ḿ​t　é㋿Ⅳ𐞁 ſ漢d‍fim<|fim_prefix|>", "tokens": 30, "pieces": ["　", "'Ḿ", "​t", "　é", "㋿", "Ⅳ", "𐞁", " ſ漢d", "‍fim", "<|", "fim", "_prefix", "|>"]} +{"text": "'Re('S\n", "tokens": 4, "pieces": ["'Re", "('", "S", "\n"]} +{"text": "Dž👍🏽Ⅳ\u000bfi'VE㍿at 字eḍ̇\"字…\t字　é'Reſ", "tokens": 34, "pieces": ["Dž", "👍🏽", "Ⅳ", "\u000bfi'VE", "㍿at", "", " ", " 字eḍ̇", "\"字", "…", "\t字", "　é'Re", "ſ"]} +{"text": "'D<|fim_prefix|>d𐞁\r'll'Re٣٤٥٦३㍿'ll('ſ́\u000bs…'S\r\n\r\n👍🏽'D'<'VE'SZ09'D'S㋿<|endoftext|>", "tokens": 57, "pieces": ["'D", "<|", "fim", "_prefix", "|>", "d𐞁", "\r", "'ll'Re", "٣٤٥", "٦३", "㍿'", "ll", "('", "ſ́", "\u000bs", "…", "'S", "\r\n\r\n", "👍🏽'", "D", "'<'", "VE'S", "Z", "09", "'D'S", "㋿<|", "endoftext", "|>"]} +{"text": "ḍ̇0‍🙂​​'Re0㍿>", "tokens": 14, "pieces": ["ḍ̇", "0", "‍🙂​​'", "Re", "0", "㍿>"]} +{"text": "é'M\n\r'M'DＡ\"", "tokens": 9, "pieces": ["é'M", "\n\r", "'M'D", "Ａ", "\""]} +{"text": "e㍿' EOTéåDž fiß's'll🙂\u000b'Tİ\r\nİ!", "tokens": 27, "pieces": ["e", "㍿'", " EOTéå", "Dž", " fiß", "'", "s'll", "🙂", "\u000b", "'Tİ", "\r\n", "İ", "!"]} +{"text": "\r0'VE
'VEZعs　\t'VE㋿İ　ḍ̇́-té😀🏽Ａ'ſ\n\r \n0é'S​", "tokens": 42, "pieces": ["\r", "0", "'VE", "", "
", "'VEZعs", "　", "\t", "'VE", "㋿İ", "　ḍ̇́", "-té", "😀🏽", "Ａ'ſ", "\n\r \n", "0", "é'S", "​"]} +{"text": "　ſt \r\nś'M'D", "tokens": 9, "pieces": ["　ſt", " \r\n", "ś'M", "'D"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "9'M漢漢३!!é½''Dß字'll''ſ\r㍿Ⅳ", "tokens": 21, "pieces": ["9", "'M漢漢", "३", "!!", "é", "½", "''", "Dß字'll", "''", "ſ", "\r", "㍿", "Ⅳ"]} +{"text": "́", "tokens": 1, "pieces": ["́"]} +{"text": "->\r\n\r\n­'Re​ꟲ<|endoftext|>‍m…‍㋿0
eDž'S字\"<|endoftext|>Ⅳé㍿‍ \nع'M' >́'VEfi", "tokens": 55, "pieces": ["->\r\n\r\n", "­'", "Re", "​ꟲ", "<|", "endoftext", "|>‍", "m", "…", "‍㋿", "0", "
e", "Dž'S", "字", "\"<|", "endoftext", "|>", "Ⅳ", "é", "㍿‍", " \n", "ع'M", "'", " ", ">́'VE", "fi"]} +{"text": "0㍿'é're.!!ß0", "tokens": 12, "pieces": ["0", "㍿'", "é're", ".!!", "ß", "0"]} +{"text": "'ſ'T🙂 'M\u000b½a'ſ‍字!!m(!!😀🏽ع", "tokens": 32, "pieces": ["'ſ'T", "🙂", " ", " '", "M", "", "\u000b", "½", "a'ſ", "‍<", "EOT", "><", "EOT", ">字", "!!", "m", "(!!😀🏽", "ع"]} +{"text": "\r\ń12345678Ⅳ😀🏽'Re😀🏽<|fim_prefix|>漢", "tokens": 25, "pieces": ["\r\n", "́", "123", "456", "78", "", "Ⅳ", "😀🏽'", "Re", "😀🏽<|", "fim", "_prefix", "|>", "漢"]} +{"text": "\u000b'Re-<|endoftext|>𐞁-👍🏽'D\u000b!!🙂\u000b\r", "tokens": 28, "pieces": ["\u000b", "'Re", "-<|", "endoftext", "|>", "𐞁", "-👍🏽'", "D", "\u000b", "!!🙂", "\u000b\r"]} +{"text": "'s\t#$%EOT0漢. .(t‍s>-'re\rİ12345678EOT-'", "re", "\r", "İ", "123", "456", "78", "EOT", "'ll👍🏽>,", "tokens": 17, "pieces": [",t", "㍿‍\r\n\r\n", "m", "…", "'", "ll", "👍🏽>,"]} +{"text": "'ll>३\r\r\n\r\ń", "tokens": 6, "pieces": ["'ll", ">", "३", "\r\r\n\r\n", "́"]} +{"text": "​", "tokens": 1, "pieces": ["​"]} +{"text": "'D३😀🏽<|fim_prefix|><'T\r\n\r\nDž!\"tDž\r\n漢.　(a<|fim_prefix|>\r\n\r\n-㋿\n.\né<|fim_prefix|>DžⅣ字ad!!٣٤٥٦", "tokens": 55, "pieces": ["'D", "३", "😀🏽<|", "fim", "_prefix", "|><'", "T", "\r\n\r\n", "Dž", "!\"", "t", "Dž", "\r\n", "漢", ".", "　", "(a", "<|", "fim", "_prefix", "|>\r\n\r\n", "-㋿\n", ".\n", "é", "<|", "fim", "_prefix", "|>", "Dž", "Ⅳ", "字ad", "!!", "٣٤٥", "٦"]} +{"text": "\r", "tokens": 1, "pieces": ["\r"]} +{"text": "!!", "tokens": 1, "pieces": ["!!"]} +{"text": ".𐞁​㍿\"e३'M \ń \nfifi\t 😀🏽mt <|fim_prefix|>", "tokens": 33, "pieces": [".𐞁", "​㍿\"", "e", "३", "'M", " \n", "́", "", " \n", "fifi", "\t ", " 😀🏽", "mt", " <|", "fim", "_prefix", "|>"]} +{"text": "Z'MamＡ　'ſd½㋿㋿a\"d字 Ⅳ<|endoftext|>́İ'ſ'VE'ſ\n12345678㍿　👍🏽𐞁İ", "tokens": 56, "pieces": ["Z'M", "am", "Ａ", "　", "'ſd", "½", "㋿㋿<", "META", "_START", ">a", "\"d字", " ", " ", "Ⅳ", "<|", "endoftext", "|>́", "İ'ſ", "'VE'ſ", "\n", "123", "456", "78", "㍿", "　", "👍🏽", "𐞁", "İ"]} +{"text": "٣٤٥٦!Zſꟲ-…'ll ", "tokens": 15, "pieces": ["٣٤٥", "٦", "!Zſꟲ", "-", "…", "'ll", " "]} +{"text": "'ll‍ İdİ\naEOTd!\r\n\r\n\r\n\r\n\u000b'M'VEꟲ㍿<|endoftext|>0 \n<|endoftext|>'ſ­㋿'ſ", "tokens": 46, "pieces": ["'ll", "‍", " ", " İd", "İ", "\n", "a", "EOTd", "!\r\n\r\n\r\n\r\n", "\u000b", "'M'VE", "ꟲ", "㍿<|", "endoftext", "|>", "0", " \n", "<|", "endoftext", "|>'", "ſ", "­㋿'", "ſ"]} +{"text": "fi\r\n\r\n…", "tokens": 4, "pieces": ["fi", "\r\n\r\n", "…"]} +{"text": "ſ>'s'Sa9ع٣٤٥٦\r\n\r\n-Ⅳ\nß", "tokens": 17, "pieces": ["ſ", ">'", "s'S", "a", "9", "ع", "٣٤٥", "٦", "\r\n\r\n", "-", "Ⅳ", "\n", "ß"]} +{"text": "٣٤٥٦<|fim_prefix|>0\u000b​(-३é字'S!!!!字fi'ſ<|endoftext|>0 's\u000bZ㍿'Red\r>👍🏽s\n👍🏽'll'll
e", "tokens": 60, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "0", "\u000b", "​(-", "३", "é字'S", "!!!!", "字fi'ſ", "<|", "endoftext", "|>", "0", " ", "'s", "\u000bZ", "㍿'", "Red", "\r", ">👍🏽", "s", "\n", "👍🏽'", "ll'll", "
e"]} +{"text": "­t漢 (\nfi >t\rs!!'T're!!½'refi　­­!!\nḍ̇😀🏽#$%­EOT'T>", "tokens": 32, "pieces": ["­t漢", " (\n", "fi", " >", "t", "\r", "s", "!!'", "T're", "!!", "½", "'refi", "　", "­­!!\n", "ḍ̇", "😀🏽#$%­", "EOT'T", ">"]} +{"text": "'S<|fim_prefix|>.㋿\"👍🏽½​ḍ̇\u000b>9", "tokens": 25, "pieces": ["'S", "<|", "fim", "_prefix", "|>.㋿\"👍🏽", "½", "​ḍ̇", "", "\u000b", ">", "9"]} +{"text": "٣٤٥٦😀🏽㍿-0!!'VE''S!!Z(-0'!!́३
", "tokens": 29, "pieces": ["٣٤٥", "٦", "😀🏽㍿-", "0", "!!'", "VE", "''", "S", "!!<", "EOT", ">Z", "(-", "0", "'!!́", "३", "
"]} +{"text": "'ſ㍿!İ're　t\r\n‍½ſfi\u000b'Dé!漢­½‍'ſ\"ḍ̇éＡ<'Ss,", "tokens": 41, "pieces": ["'ſ", "㍿!", "İ're", "　t", "\r\n", "‍<", "EOT", ">", "½", "ſfi", "\u000b", "'Dé", "!漢", "­", "½", "‍'", "ſ", "\"ḍ̇é", "Ａ", "<'", "Ss", ","]} +{"text": ".ḍ̇åİ\"!!ḍ̇ꟲ漢  ­Ａ\nſ>३12345678#$%a३\u000b'ſ9,", "tokens": 38, "pieces": [".ḍ̇å", "İ", "\"!!", "ḍ̇ꟲ漢", " ", " ­", "Ａ", "\n", "ſ", ">", "३12", "345", "678", "#$%", "a", "३", "\u000b", "'ſ", "9", ","]} +{"text": "㋿12345678>٣٤٥٦<|fim_prefix|>,'Re<|fim_prefix|>>́😀🏽ＡDž9'VE \n're", "tokens": 37, "pieces": ["㋿", "123", "456", "78", ">", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>,'", "Re", "<|", "fim", "_prefix", "|>>́😀🏽", "ＡDž", "9", "'VE", " \n", "'re"]} +{"text": "Džſ­< \n　\rme12345678dعſ­#$%-ḿ…\"　(Dž 's‍fi!٣٤٥٦<|endoftext|>\r\n'll
… \n're", "tokens": 53, "pieces": ["Džſ", "­<", " \n　\r", "me", "123", "456", "78", "dعſ", "­#$%-", "ḿ", "…", "\"", "　", "(Dž", " ", " '", "s", "‍fi", "!", "٣٤٥", "٦", "<|", "endoftext", "|>\r\n", "'ll", "
… \n", "'re"]} +{"text": "İſ,㍿!a0'M‍'ll漢'VE<|endoftext|>Dž३…!Dž ḍ̇'Dé'll🙂ع३aåß \n,½'Re\r\n\r\n漢'll'ß", "tokens": 56, "pieces": ["İſ", ",㍿!", "a", "0", "'M", "‍'", "ll漢", "'", "VE", "<|", "endoftext", "|>", "Dž", "३", "…", "!Dž", " ḍ̇'D", "é'll", "🙂ع", "३", "aåß", " \n", ",", "½", "'Re", "\r\n\r\n", "漢'll", "'ß"]} +{"text": "٣٤٥٦Ａ㋿EOT​\r\n\r\n\"‍'Sa'VE٣٤٥٦Z!ꟲ🙂 'ſ'll", "tokens": 31, "pieces": ["٣٤٥", "٦", "Ａ", "㋿EOT", "​\r\n\r\n", "\"‍'", "Sa'VE", "٣٤٥", "٦", "Z", "!ꟲ", "🙂", " '", "ſ'll"]} +{"text": "\t(s(", "tokens": 3, "pieces": ["\t", "(s", "("]} +{"text": "ＡDž𐞁a-😀🏽é\r 漢
​'\u000b𐞁EOTḍ̇'VE…>㍿\t
字🙂!'s­Ａ#$%'VE漢0ß'M're㍿", "tokens": 60, "pieces": ["ＡDž𐞁a", "-😀🏽", "é", "\r", " 漢", "
", "​'", "\u000b", "𐞁EOTḍ̇'VE", "…", ">㍿", "\t", "
字", "🙂!'", "s", "­Ａ", "#$%'", "VE漢", "0", "ß'M", "'re", "㍿"]} +{"text": "-!🙂'll٣٤٥٦ '.㍿Dž!!eſe", "tokens": 20, "pieces": ["-!🙂'", "ll", "٣٤٥", "٦", " ", "'.㍿", "Dž", "!!", "eſe"]} +{"text": " 😀🏽9İ\t㋿…ß m\t", "tokens": 14, "pieces": [" 😀🏽", "9", "İ", "\t", "㋿", "…ß", " m", "\t"]} +{"text": "…㋿🙂tm🙂字>EOT 漢'T­\r\r\n\r\n ٣٤٥٦éé", "tokens": 24, "pieces": ["…", "㋿🙂", "tm", "🙂字", ">EOT", " 漢'T", "­\r\r\n\r\n", " ", "٣٤٥", "٦", "éé"]} +{"text": "'ſ\r漢
<|endoftext|>😀🏽\rå😀🏽<|endoftext|>a‍<|fim_prefix|>　ع\u000bå", "tokens": 44, "pieces": ["'ſ", "\r", "漢", "
", "<|", "endoftext", "|>😀🏽\r", "å", "😀🏽<", "EOT", "><|", "endoftext", "|>", "a", "‍<|", "fim", "_prefix", "|>", "　ع", "\u000bå"]} +{"text": "ß'T'🙂EOTZdEOT🙂 ½!!'D<|endoftext|>fi0\" \n㋿٣٤٥٦-é\"'VE　'S", "tokens": 40, "pieces": ["ß'T", "'🙂", "EOTZd", "EOT", "🙂", " ", " ", "½", "!!'", "D", "<|", "endoftext", "|>", "fi", "0", "\"", " \n", "㋿", "٣٤٥", "٦", "-é", "\"'", "VE", "　", "'S"]} +{"text": "İ\n's12345678ß<|fim_prefix|>\r\n\r\ns漢 -٣٤٥٦#$% 'Re0!!!㋿\tDž", "tokens": 43, "pieces": ["İ", "\n", "'s", "123", "456", "78", "ß", "<|", "fim", "_prefix", "|>\r\n\r\n", "s漢", " -", "٣٤٥", "٦", "#$%", " ", " '", "Re", "0", "!!!㋿", "\t", "Dž", ""]} +{"text": "\"́ع'T\rḍ̇é㍿👍🏽ḍ̇\r\n\r\n​🙂's#$%,½'T<|endoftext|>ßDž ३'M​å<|endoftext|>''T", "tokens": 56, "pieces": ["\"́ع'T", "\r", "ḍ̇é", "㍿👍🏽", "ḍ̇", "\r\n\r\n", "​🙂'", "s", "#$%,", "½", "'T", "<|", "endoftext", "|>", "ß", "Dž", " ", "३", "'M", "​å", "<|", "endoftext", "|>''", "T"]} +{"text": "12345678ééİ Dž<|endoftext|>!!𐞁३㋿👍🏽漢‍<|fim_prefix|><|endoftext|>s", "tokens": 49, "pieces": ["123", "456", "78", "éé", "İ", " ", " Dž", "<|", "endoftext", "|>!!", "𐞁", "", "३", "㋿👍🏽", "漢", "‍<|", "fim", "_prefix", "|><|", "endoftext", "|>", "s"]} +{"text": "㍿🙂Ａ<­> éع123456780\r\n𐞁12345678'VE'S漢<|fim_prefix|>éDžİ'D're…Zeé>٣٤٥٦ \t'VE>fiéß", "tokens": 61, "pieces": ["㍿🙂", "Ａ", "<­>", " éع", "123", "456", "780", "\r\n", "𐞁", "123", "456", "78", "'VE'S", "漢", "<|", "fim", "_prefix", "|><", "EOT", ">é", "Džİ'D", "'re", "…Zeé", ">", "٣٤٥", "٦", " ", "\t", "'VE", ">fiéß"]} +{"text": "👍🏽a'VE9ſعDž ٣٤٥٦­Dž\r\n३fiß‍‍'De<|fim_prefix|>ſع漢<|fim_prefix|>\"­Ａ", "tokens": 52, "pieces": ["👍🏽", "a'VE", "9", "ſ", "ع", "Dž", " ", " ", "٣٤٥", "٦", "­Dž", "\r\n", "३", "fiß", "‍‍'", "De", "<|", "fim", "_prefix", "|>", "ſع漢", "<|", "fim", "_prefix", "|>\"<", "META", "_START", ">­", "Ａ"]} +{"text": "å,\r\n\r\nع½㋿\r\né३-'M<|fim_prefix|>'M­ 'Téßİ'st३'s\"tＡ\r\n\r\nd\u000b'sḍ̇ſ٣٤٥٦\r\n", "tokens": 47, "pieces": ["å", ",\r\n\r\n", "ع", "½", "㋿\r\n", "é", "३", "-'", "M", "<|", "fim", "_prefix", "|>'", "M", "­", " '", "Téß", "İ's", "t", "३", "'s", "\"t", "Ａ", "\r\n\r\n", "d", "\u000b", "'sḍ̇ſ", "٣٤٥", "٦", "\r\n"]} +{"text": "ß 'M#$%'s ३㍿　sdDž'll'sEOT
\r\n\r\n<|endoftext|> ‍\u000bEOT\u000b\n㍿Dž éEOT'S㋿ \né", "tokens": 53, "pieces": ["ß", " '", "M", "#$%'", "s", " ", "३", "㍿", "　sd", "Dž'll", "'s", "EOT", "
\r\n\r\n", "<|", "endoftext", "|>", " ", "‍", "\u000bEOT", "\u000b\n", "㍿Dž", " <", "EOT", ">é", "EOT'S", "㋿", " \n", "é"]} +{"text": "٣٤٥٦åſ ‍és'ſ'lĺ12345678'T#$%٣٤٥٦!ع ́­", "tokens": 34, "pieces": ["٣٤٥", "٦", "åſ", " ", "‍és'ſ", "'lĺ", "123", "456", "78", "'T", "#$%<", "META", "_START", ">", "٣٤٥", "٦", "!ع", " ", " ́", "­"]} +{"text": ">EOT'reꟲß\nſ'S's'llm\u000b👍🏽'㋿'lĺ\t㋿Ⅳ>漢👍🏽 Ａ", "tokens": 38, "pieces": [">EOT're", "ꟲß", "\n", "ſ'S", "'s'll", "m", "\u000b", "👍🏽'㋿'", "lĺ", "\t", "㋿", "Ⅳ", ">漢", "👍🏽", " ", " Ａ"]} +{"text": "'re٣٤٥٦EOTEOT<३😀🏽're-Ⅳ'D­ß\té३​漢٣٤٥٦-'S>ḍ̇ſ0sſ9-d0İ", "tokens": 44, "pieces": ["'re", "٣٤٥", "٦", "EOTEOT", "<", "३", "😀🏽'", "re", "-", "Ⅳ", "'D", "­ß", "\té", "३", "​漢", "٣٤٥", "٦", "-'", "S", ">ḍ̇ſ", "0", "sſ", "9", "-d", "0", "İ"]} +{"text": "12345678\tſ", "tokens": 5, "pieces": ["123", "456", "78", "\tſ"]} +{"text": " EOT\"Ⅳ😀🏽'ſ", "tokens": 11, "pieces": [" ", " EOT", "\"", "Ⅳ", "😀🏽'", "ſ"]} +{"text": "½ßs\u000b\"😀🏽'llDžＡ .9İ'Re#$%12345678é", "tokens": 24, "pieces": ["½", "ßs", "\u000b", "\"😀🏽'", "ll", "DžＡ", " ", ".", "9", "İ'Re", "#$%", "123", "456", "78", "é"]} +{"text": " 👍🏽\u000b३ !!​!!漢<|fim_prefix|>\n
fi a½\t12345678'Re' \u000b12345678", "tokens": 53, "pieces": ["", " ", " 👍🏽", "\u000b", "", "३", " ", " !!​!!", "漢", "<|", "fim", "_prefix", "|>\n", "
fi", " a", "½", "\t", "123", "456", "78", "'Re", "'", " ", "\u000b", "123", "456", "78", "<", "EOT", ">"]} +{"text": "'re😀🏽EOTß\ns'll\n#$%0<́é>ع<Ⅳ'ع", "tokens": 44, "pieces": ["'re", "😀🏽", "EOTß", "\n", "s'll", "\n", "#$%", "0", "<<", "EOT", ">́é", ">ع", "<", "Ⅳ", "'ع"]} +{"text": "(字as'ſ!ꟲ
9m👍🏽\"ⅣZḿ\"'Tİá12345678Dž'll😀🏽0\nDž!㋿ḍ̇Ａa'll'M٣٤٥٦ꟲ 
12345678", "tokens": 53, "pieces": ["Dž", "123", "456", "78", "'re'll", "㋿d", "9", "\"👍🏽<", "META", "_START", ">Dž'll", "😀🏽", "0", "\n", "Dž", "!㋿", "ḍ̇", "Ａa'll", "'M", "٣٤٥", "٦", "ꟲ", " ", "
", "123", "456", "78"]} +{"text": "aa‍!Z㍿👍🏽!!>٣٤٥٦ ꟲDžＡ! ½ꟲéé.Dždafi𐞁Dž­ \n…Ａ-👍🏽, 😀🏽𐞁👍🏽", "tokens": 66, "pieces": ["aa", "‍!", "Z", "㍿👍🏽!!>", "٣٤٥", "٦", " ꟲ", "DžＡ", "!", " ", "½", "ꟲéé", ".Dždafi", "𐞁", "Dž", "­", " \n", "…Ａ", "-👍🏽,", " ", " 😀🏽", "𐞁", "👍🏽"]} +{"text": "<|endoftext|>½३㋿#$%a9,.åDžDž'S ß漢12345678….'TDž", "tokens": 37, "pieces": ["<|", "endoftext", "|>", "½३", "㋿#$%", "a", "9", ",.", "å", "DžDž'S", " ", " ß漢", "123", "456", "78", "…", ".'", "TDž"]} +{"text": "dⅣfifi'MⅣ३‍'s\n­'ll (!!'s é\"-", "tokens": 25, "pieces": ["d", "Ⅳ", "fifi'M", "Ⅳ३", "‍'", "s", "\n", "­'", "ll", " ", "(!!'", "s", " ", " é", "\"-"]} +{"text": "㋿🙂 ḍ̇#$%\r\n ٣٤٥٦½ꟲ🙂'ſß\u000b\rعa‍<|endoftext|> Ａ \na­Z'll\r\n'T\r\n\r\nsⅣ'ſſEOT #$%​", "tokens": 64, "pieces": ["㋿🙂", " ḍ̇", "#$%\r\n", " ", " ", "٣٤٥", "٦½", "ꟲ", "🙂'", "ſß", "\u000b\r", "عa", "‍<|", "endoftext", "|>", " <", "EOT", ">Ａ", " \n", "a", "­Z'll", "\r\n", "'T", "\r\n\r\n", "s", "Ⅳ", "'ſſ", "EOT", " <", "EOT", ">#$%​"]} +{"text": "漢ع Ⅳ🙂​'ll<|endoftext|> 'ḍ̇​İ…३\r\n٣٤٥٦", "tokens": 31, "pieces": ["漢ع", " ", "Ⅳ", "🙂​'", "ll", "<|", "endoftext", "|>", " ", " '", "ḍ̇", "​İ", "…", "३", "\r\n", "٣٤٥", "٦"]} +{"text": "s​🙂9ſ0\r's👍🏽'VE 9(Ａ", "tokens": 17, "pieces": ["s", "​🙂", "9", "ſ", "0", "\r", "'s", "👍🏽'", "VE", " ", "9", "(Ａ"]} +{"text": ".'S'VE字漢'lĺaDž!!éEOT\u000b<　Ａ㋿aådع'SEOT\u000b​­<'Re", "tokens": 37, "pieces": [".'", "S'VE", "字漢'll", "́a", "Dž", "!!", "é", "EOT", "\u000b", "<", "　Ａ", "㋿aådع", "'", "SEOT", "\u000b", "​­<'", "Re"]} +{"text": "fi!! ", "tokens": 3, "pieces": ["fi", "!!", " "]} +{"text": "\"‍'ſ'Ma9'VE>\r\n'llfi", "tokens": 17, "pieces": ["\"‍'", "ſ'M", "a", "9", "'", "VE", ">\r\n", "'ll", "fi"]} +{"text": "mß'll'M 9عd9e'retEOT\r\n\r\n३'ſ9
㋿", "tokens": 23, "pieces": ["mß'll", "'M", " ", "9", "عd", "9", "e're", "t", "EOT", "\r\n\r\n", "३", "'ſ", "9", "
", "㋿"]} +{"text": "🙂0!!!‍½'a½½>s字‍dfi-'\r'ſ'reⅣZß'Re…", "tokens": 31, "pieces": ["🙂", "0", "!!!‍", "½", "'a", "½½", ">s字", "‍dfi", "-'\r", "'ſ", "'", "re", "Ⅳ", "Zß'Re", "…"]} +{"text": "!!eⅣ!<|fim_prefix|>", "tokens": 11, "pieces": ["!!", "e", "Ⅳ", "!<|", "fim", "_prefix", "|>"]} +{"text": "a'ſ ", "tokens": 4, "pieces": ["a'ſ", " "]} +{"text": "<|endoftext|>½és're‍", "tokens": 23, "pieces": ["<|", "endoftext", "|>", "½", "és're", "‍"]} +{"text": "'M's​\r­　're'S'sdfi're'VE(​Ⅳ", "tokens": 18, "pieces": ["'M's", "​\r", "­", "　", "'re'S", "'sdfi're", "'VE", "(​", "Ⅳ"]} +{"text": "
३EOT éfi'll\r!…İ", "tokens": 13, "pieces": ["
", "३", "EOT", " ", " éfi'll", "\r", "!", "…İ"]} +{"text": "'Ts,m", "tokens": 3, "pieces": ["'Ts", ",m"]} +{"text": "𐞁Z12345678é\"\u000b'ſ ́٣٤٥٦ꟲZ", "tokens": 31, "pieces": ["𐞁", "Z", "123", "456", "78", "é", "\"<", "EOT", ">", "\u000b", "'ſ", " ", " <", "EOT", ">́", "٣٤٥", "٦", "ꟲ", "Z"]} +{"text": "'M're Ⅳ𐞁­\"", "tokens": 11, "pieces": ["'M're", " ", "Ⅳ", "𐞁", "­\""]} +{"text": "
㋿\r\n \nés…", "tokens": 9, "pieces": ["
", "㋿\r\n", " \n", "és", "…"]} +{"text": "<(٣٤٥٦.m0字İ're٣٤٥٦m\r\n\r\n\r\n　00Z'M \n'ſZ­Z's㋿ \n🙂 \n\nDž<|fim_prefix|>mİ's<|endoftext|>", "tokens": 51, "pieces": ["<(", "٣٤٥", "٦", ".m", "0", "字", "İ're", "٣٤٥", "٦", "m", "\r\n\r\n\r\n", "　", "00", "Z'M", " \n", "'ſ", "Z", "­Z's", "㋿", " \n", "🙂", " \n\n", "Dž", "<|", "fim", "_prefix", "|>", "m", "İ's", "<|", "endoftext", "|>"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'ll३\u000b<|endoftext|>🙂\"!…Z字'T㋿s 'T字!👍🏽…🙂…㍿,fi\"ḍ̇!ع\nå ​Ⅳ　½\"t", "tokens": 60, "pieces": ["'ll", "३", "\u000b", "<|", "endoftext", "|>🙂\"!", "…Z字'T", "㋿s", "", " ", "'T字", "!👍🏽", "…", "🙂", "…", "㍿,", "fi", "\"ḍ̇", "!ع", "\n", "å", " ​", "Ⅳ", "　", "½", "\"t"]} +{"text": " ꟲ \u000b12345678 \r
👍🏽𐞁", "tokens": 19, "pieces": [" ꟲ", " ", "\u000b", "123", "456", "78", " \r", "
", "👍🏽", "𐞁"]} +{"text": "½9🙂ß", "tokens": 4, "pieces": ["½9", "🙂ß"]} +{"text": "'Re9́<\n漢t\"½#$%\u000b\u000b字٣٤٥٦('M'M#$%'‍　\"'S字​<|fim_prefix|> 'VE\u000b👍🏽'TZ​d", "tokens": 44, "pieces": ["'Re", "9", "́", "<\n", "漢t", "\"", "½", "#$%", "\u000b", "\u000b字", "٣٤٥", "٦", "('", "M'M", "#$%'‍", "　", "\"'", "S字", "​<|", "fim", "_prefix", "|>", " '", "VE", "\u000b", "👍🏽'", "TZ", "​d"]} +{"text": "!\r\n​　　0ſ'T'reḍ̇​ ㋿#$%9EOT​", "tokens": 23, "pieces": ["!\r\n", "​", "　", "　", "0", "ſ'T", "'reḍ̇", "​", " ", " ㋿#$%", "9", "EOT", "​"]} +{"text": "'VE'S.\r\n\r\n٣٤٥٦<|fim_prefix|>\r'VEＡ\nＡZ\u000b.", "tokens": 27, "pieces": ["'VE'S", ".\r\n\r\n", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>\r", "'VEＡ", "\n", "ＡZ", "\u000b", "."]} +{"text": "'VE!.12345678éＡ'fiea \nḍ̇\t'Re0'm#$%𐞁!!‍EOT<|endoftext|>　Ａ,३Dž", "tokens": 46, "pieces": ["'VE", "!.", "123", "456", "78", "é", "Ａ", "'<", "EOT", ">fiea", " \n", "ḍ̇", "\t", "'Re", "0", "'m", "#$%", "𐞁", "!!‍", "EOT", "<|", "endoftext", "|>", "　Ａ", ",", "३", "Dž"]} +{"text": ".9aＡ!!åé字#$%Ⅳé 
EOT​mm(…‍'字字EOT \n👍🏽\r\n'.'VE🙂", "tokens": 41, "pieces": [".", "9", "a", "Ａ", "!!", "åé字", "#$%", "Ⅳ", "é", " ", "
EOT", "​mm", "(", "…", "‍'", "字字", "EOT", " \n", "👍🏽\r\n", "'.'", "VE", "🙂"]} +{"text": "\r\n👍🏽.sesEOT㋿\"'De!'VE9", "tokens": 18, "pieces": ["\r\n", "👍🏽.", "ses", "EOT", "㋿\"'", "De", "!'", "VE", "9"]} +{"text": "\t!!…'Re<|endoftext|> å12345678Ⅳ漢<'re!!mEOT\tEOTß　<|endoftext|>EOTm-𐞁Ａ ḍ̇漢ⅣZ!é'ſ😀🏽EOT", "tokens": 68, "pieces": ["\t", "!!", "…", "'Re", "<|", "endoftext", "|>", " å", "123", "456", "78Ⅳ", "漢", "<'", "re", "!!", "m", "EOT", "\tEOT", "ß", "　", "<|", "endoftext", "|>", "EOTm", "-𐞁", "Ａ", " ḍ̇漢", "Ⅳ", "Z", "!é'ſ", "😀🏽", "EOT"]} +{"text": "\u000b\rm­s'T𐞁Ⅳ're…३é>Ⅳ‍'s'ſ(\r\nfi<字tſEOT½éd́-'reDž\"<|endoftext|>m㋿'M", "tokens": 55, "pieces": ["\u000b\r", "m", "­s'T", "𐞁", "Ⅳ", "'re", "…", "३", "é", ">", "Ⅳ", "‍'", "s'ſ", "(\r\n", "fi", "<字tſ", "EOT", "½", "éd́", "-'", "re", "Dž", "\"<|", "endoftext", "|>", "m", "㋿'", "M"]} +{"text": "Ⅳ#$%'Ree\u000b½<|endoftext|>ß​½Ａ½s 'sEOT𐞁\r\u000b½ḍ̇'٣٤٥٦", "tokens": 40, "pieces": ["Ⅳ", "#$%'", "Ree", "\u000b", "½", "<|", "endoftext", "|>", "ß", "​", "½", "Ａ", "½", "s", " ", "'s", "EOT𐞁", "\r", "\u000b", "½", "ḍ̇", "'", "٣٤٥", "٦"]} +{"text": "ſé٣٤٥٦0‍t­Ⅳ½ꟲ'llⅣe\"½ ", "tokens": 24, "pieces": ["ſé", "٣٤٥", "٦0", "‍t", "­", "Ⅳ½", "ꟲ'll", "Ⅳ", "e", "\"", "½", " "]} +{"text": "
>'T½émå9e>😀🏽ém İéEOT \n#$%Ⅳ'Rea<|endoftext|>å12345678𐞁ZEOT　字t'T", "tokens": 52, "pieces": ["
", ">'", "T", "½", "émå", "9", "e", ">😀🏽", "ém", " ", " İé", "EOT", " \n", "#$%", "Ⅳ", "'Rea", "<|", "endoftext", "|>", "å", "123", "456", "78", "𐞁", "ZEOT", "　字t'T"]} +{"text": "'D.é㋿!\r12345678٣٤٥٦\"fi\ts'M३'Dع㋿\r\u000bⅣm\n-𐞁㍿", "tokens": 54, "pieces": ["字ß'Re", "\n", "𐞁åḍ̇漢å", "\t", "'S", "!!'", "ſ", "9", "<|", "fim", "_prefix", "|>", "\ts'M", "३", "'Dع", "㋿<", "EOT", ">\r", "\u000b", "Ⅳ", "m", "\n", "-𐞁", "㍿"]} +{"text": "\n're'reDžfi
Dž  \r\n", "tokens": 11, "pieces": ["\n", "'re're", "Džfi", "
Dž", "  \r\n"]} +{"text": "!EOT.'ſ \n'T're'VE'Dſ  字 ", "tokens": 22, "pieces": ["!EOT", ".'", "ſ", " \n", "'T", "'", "re'VE", "'Dſ", " ", " 字", " <", "EOT", ">"]} +{"text": "'sm!　 t\"٣٤٥٦漢Zé'll 'll!٣٤٥٦ -00㋿́\u000b \n m　́'Ddع .Dž𐞁", "tokens": 44, "pieces": ["'sm", "!", "　 ", " t", "\"", "٣٤٥", "٦", "漢Zé'll", " '", "ll", "!", "٣٤٥", "٦", " ", " -", "00", "㋿́", "\u000b \n", " m", "　́'D", "dع", " .", "Dž𐞁"]} +{"text": "'S 'ſ \nt
​fi\nſ㍿9'VE12345678Z㍿<|endoftext|>Ⅳ👍🏽's<\n\r\n\r\n", "tokens": 46, "pieces": ["'S", " ", " '", "ſ", " \n", "t", "
", "​", "fi", "\n", "ſ", "㍿", "9", "'VE", "123", "456", "78", "Z", "㍿<|", "endoftext", "|>", "Ⅳ", "👍🏽'", "s", "<<", "META", "_START", ">\n\r\n\r\n"]} +{"text": "漢👍🏽.Džſ0!!\r\n'T9å åaåEOTİꟲ㍿‍㋿\t\tḍ̇", "tokens": 41, "pieces": ["漢", "👍🏽.", "Džſ", "0", "!!\r\n", "'T", "9", "å", " ", "åaå", "EOTİꟲ", "㍿‍㋿", "\t", "\tḍ̇"]} +{"text": "Z\r\n\r\n\r'ſ'MZ\r,12345678㍿㍿", "tokens": 18, "pieces": ["Z", "\r\n\r\n\r", "'ſ'M", "Z", "\r", ",", "123", "456", "78", "㍿㍿"]} +{"text": "　\rd'ſa😀🏽're३<㋿\r\n\r\nꟲZ<🙂's <|fim_prefix|>mꟲ👍🏽字", "tokens": 40, "pieces": ["　\r", "d'ſ", "a", "😀🏽'", "re", "३", "<㋿\r\n\r\n", "ꟲ", "Z", "<🙂'", "s", " ", " <|", "fim", "_prefix", "|>", "mꟲ", "👍🏽", "字"]} +{"text": "ßİm(\r\n'lléＡ're\rs'DDž!fi३عdＡ'D-
​e㍿!!‍", "tokens": 30, "pieces": ["ß", "İm", "(\r\n", "'llé", "Ａ're", "\r", "s'D", "Dž", "!fi", "३", "عd", "Ａ'D", "-", "
", "​e", "㍿!!‍"]} +{"text": "㍿ꟲ0dßt's漢 \n🙂😀🏽­ꟲ's👍🏽'Dtm<|endoftext|>é٣٤٥٦#$%!字éع'll👍🏽Ａ", "tokens": 57, "pieces": ["㍿ꟲ", "0", "dßt's", "漢", " \n", "🙂😀🏽­", "ꟲ's", "👍🏽'", "Dtm", "<|", "endoftext", "|>", "é", "٣٤٥", "٦", "#$%!<", "EOT", ">字éع", "'", "ll", "👍🏽", "Ａ"]} +{"text": "\u000b<|endoftext|>Ａ‍ß0#$%ⅣtZ>
're　Dž-ع!e\tß'ſ　é عß,<　
ḍ̇", "tokens": 47, "pieces": ["\u000b", "<|", "endoftext", "|>", "Ａ", "‍ß", "0", "#$%", "Ⅳ", "t", "Z", ">", "
", "'re", "　Dž", "-ع", "!e", "\tß'ſ", "　é", " ", " ع", "ß", ",<", "　", "
ḍ̇"]} +{"text": "éḍ̇\r\n\t0!'T\ts字㋿Dž\u000bع'reEOT'Re!.Z'ſ", "tokens": 27, "pieces": ["éḍ̇", "\r\n", "\t", "0", "!'", "T", "\ts字", "㋿Dž", "\u000bع're", "EOT'Re", "!.", "Z'ſ"]} +{"text": "ḍ̇ée!㍿\"'D👍🏽\"fidＡ'M'VE'!!🙂½\r\n\r\n>'s's
Zḍ̇'ll9漢m٣٤٥٦Z#$%(­", "tokens": 50, "pieces": ["ḍ̇é", "e", "!㍿\"'", "D", "👍🏽\"", "fid", "Ａ'M", "'VE", "'!!🙂", "½", "\r\n\r\n", ">'", "s's", "
Zḍ̇'ll", "9", "漢m", "٣٤٥", "٦", "Z", "#$%(­"]} +{"text": "0'lléa😀🏽 \r\n\r\n字… e'Tt> e 'Té​12345678EOTZ㋿
.½0Ａ㍿字𐞁́漢m'sa", "tokens": 49, "pieces": ["0", "'lléa", "😀🏽", " \r\n\r\n", "字", "…", " e'T", "t", ">", " e", " ", " '", "Té", "​", "123", "456", "78", "EOTZ", "㋿", "
", ".", "½0", "Ａ", "㍿字𐞁́漢m's", "a"]} +{"text": "<|fim_prefix|>漢　fi𐞁ع#$%.😀🏽#$%#$%12345678½ꟲ'Re!å'Reع😀🏽\n'sfi\ńＡ½\t.>　'M's🙂", "tokens": 53, "pieces": ["<|", "fim", "_prefix", "|>", "漢", "　fi𐞁ع", "#$%.😀🏽#$%#$%", "123", "456", "78½", "ꟲ'Re", "!å'Re", "ع", "😀🏽\n", "'sfi", "\n", "́", "Ａ", "½", "\t", ".>", "　", "'M's", "🙂"]} +{"text": "½'s
　　12345678 \nſEOT\r㍿Zḍ̇!EOTſ'T\r\nß 𐞁", "tokens": 35, "pieces": ["½", "'s", "
　", "　", "123", "456", "78", " \n", "ſ", "EOT", "\r", "㍿Zḍ̇", "!EOTſ'T", "\r\n", "ß", " 𐞁"]} +{"text": "'ſ<|fim_prefix|>'Rea12345678'll­٣٤٥٦Z0३9'Res", "tokens": 29, "pieces": ["'ſ", "<|", "fim", "_prefix", "|>'", "Rea", "123", "456", "78", "'ll", "­", "٣٤٥", "٦", "Z", "0३9", "'Res", ""]} +{"text": "३12345678e-Zİe Ⅳ'(\r\n\r\n‍\r\n\r\né \n漢㍿\nＡ㍿å<
 \nEOT­Ⅳ,\"9<|endoftext|>\t9\u000b\r", "tokens": 50, "pieces": ["३12", "345", "678", "e", "-Zİe", " ", " ", "Ⅳ", "'(\r\n\r\n", "‍\r\n\r\n", "é", " \n", "漢", "㍿\n", "Ａ", "㍿å", "<", "
 \n", "EOT", "­", "Ⅳ", ",\"", "9", "<|", "endoftext", "|>", "\t", "9", "\u000b\r"]} +{"text": "é'VE''Ta 
<.­👍🏽\nİ\u000b#$%d٣٤٥٦'ſ'll '漢'Re‍½😀🏽'VE٣٤٥٦", "tokens": 41, "pieces": ["é'VE", "''", "Ta", " ", "
", "<.­👍🏽\n", "İ", "\u000b", "#$%", "d", "٣٤٥", "٦", "'ſ'll", " '", "漢'Re", "‍", "½", "😀🏽'", "VE", "٣٤٥", "٦"]} +{"text": "'Mfi \r\n12345678EOT  ٣٤٥٦éZİſ …,👍🏽!", "tokens": 27, "pieces": ["'Mfi", " \r\n", "123", "456", "78", "EOT", "  ", " ", "٣٤٥", "٦", "é", "Zİſ", " ", "…", ",👍🏽!"]} +{"text": "'🙂😀🏽a\"- 0 '𐞁'VEßé'D३👍🏽İ'D👍🏽ßＡ'VEt ㋿ \tİé", "tokens": 49, "pieces": ["'🙂😀🏽", "a", "\"-", " ", " ", "0", " ", " <", "EOT", ">'", "𐞁'VE", "ßé'D", "३", "👍🏽", "İ'D", "👍🏽", "ß", "Ａ'VE", "t", " ", "㋿", " ", "\tİé"]} +{"text": "👍🏽aßa\" .å", "tokens": 10, "pieces": ["👍🏽", "aßa", "\"", " .", "å"]} +{"text": " \nꟲ½\te\u000b३ع'Re漢'T9Ⅳ'>३-<|fim_prefix|><­-'D'll'D'sİ<|endoftext|>12345678EOT
", "tokens": 47, "pieces": [" \n", "ꟲ", "½", "\te", "\u000b", "३", "ع'Re", "漢'T", "9Ⅳ", "'>", "३", "-<", "EOT", "><|", "fim", "_prefix", "|><­-'", "D'll", "'D's", "İ", "<|", "endoftext", "|>", "123", "456", "78", "EOT", "
"]} +{"text": "İåeEOT\n‍👍🏽‍😀🏽 \r\n\r\n12345678", "tokens": 19, "pieces": ["İåe", "EOT", "\n", "‍👍🏽‍😀🏽", " \r\n\r\n", "123", "456", "78"]} +{"text": "#$%'ſ\r\n 90😀🏽<|fim_prefix|>'ſ \n \n­㋿då", "tokens": 25, "pieces": ["#$%'", "ſ", "\r\n", " ", " ", "90", "😀🏽<|", "fim", "_prefix", "|>'", "ſ", " \n \n", "­㋿", "då"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'s'<|endoftext|>EOTعdİ\"å\ta", "tokens": 16, "pieces": ["'s", "'<|", "endoftext", "|>", "EOTعd", "İ", "\"å", "\ta"]} +{"text": "'Ａ🙂 \n's\r12345678're", "tokens": 10, "pieces": ["'Ａ", "🙂", " \n", "'s", "\r", "123", "456", "78", "'re"]} +{"text": "ꟲ<'M>", "tokens": 6, "pieces": ["ꟲ", "<'", "M", ">"]} +{"text": "👍🏽fißå<İ㍿#$%'Re٣٤٥٦0́٣٤٥٦Z<|endoftext|>‍<|endoftext|>ß👍🏽ßfi,", "tokens": 52, "pieces": ["👍🏽<", "META", "_START", ">fißå", "<İ", "㍿#$%'", "Re", "٣٤٥", "٦0", "́", "٣٤٥", "٦", "Z", "<|", "endoftext", "|>‍<|", "endoftext", "|>", "ß", "👍🏽", "ßfi", ","]} +{"text": "​<|fim_prefix|>\r'T㋿'VE字.​EOT,'T३", "tokens": 21, "pieces": ["​<|", "fim", "_prefix", "|>\r", "'T", "㋿'", "VE字", ".​", "EOT", ",'", "T", "३"]} +{"text": "\u000b", "tokens": 5, "pieces": ["", "\u000b"]} +{"text": "12345678㋿Džé½>\"!!e'Re…'M٣٤٥٦EOTİ😀🏽", "tokens": 31, "pieces": ["123", "456", "78", "㋿Džé", "½", ">\"!!", "e'Re", "…", "'M", "٣٤٥", "٦", "EOTİ", "😀🏽"]} +{"text": "\"å\u000bⅣ.e é३", "tokens": 9, "pieces": ["\"å", "\u000b", "Ⅳ", ".e", " ", " é", "३"]} +{"text": "👍🏽ZDž<|endoftext|> ", "tokens": 14, "pieces": ["👍🏽", "ZDž", "<|", "endoftext", "|>", " "]} +{"text": "​\r\nḍ̇́å'T　Z漢0

'Re\r\nå٣٤٥٦\t½㋿ꟲ'T㋿　㍿😀🏽 'VEß'refi>\nḍ̇m's12345678-ß\r\n", "tokens": 59, "pieces": ["​\r\n", "ḍ̇́å'T", "　Z漢", "0", "
", "
", "'Re", "\r\n", "å", "٣٤٥", "٦", "\t", "½", "㋿ꟲ'T", "㋿", "　", "㍿😀🏽", " '", "VEß're", "fi", ">\n", "ḍ̇m's", "123", "456", "78", "-ß", "\r\n"]} +{"text": "½ 12345678('ſ😀🏽<|endoftext|>𐞁😀🏽\r\n\r\n(!'Sſ<|fim_prefix|>Z's👍🏽 👍🏽,", "tokens": 49, "pieces": ["½", " ", " ", "123", "456", "78", "('", "ſ", "😀🏽<|", "endoftext", "|>", "𐞁", "😀🏽\r\n\r\n", "(!'", "Sſ", "<|", "fim", "_prefix", "|>", "Z's", "👍🏽", " ", "👍🏽,"]} +{"text": "'llⅣ३>½12345678ḍ̇́é>a>'D(\r\n\r\n,12345678'M", "tokens": 25, "pieces": ["'ll", "Ⅳ३", ">", "½12", "345", "678", "ḍ̇́é", ">a", ">'", "D", "(\r\n\r\n", ",", "123", "456", "78", "'M"]} +{"text": "İ字㋿㋿Z'Sع
ḍ̇EOT.'D
㋿e 🙂12345678\u000b,ßå'Reé🙂, \ne'ſ३'s>‍\"ꟲ's'", "tokens": 55, "pieces": ["İ字", "㋿㋿", "Z'S", "ع", "
ḍ̇", "EOT", ".'", "D", "
", "㋿e", " 🙂", "123", "456", "78", "\u000b", ",<", "META", "_START", ">ßå'Re", "é", "🙂,", " \n", "e'ſ", "३", "'s", ">‍\"", "ꟲ's", "'"]} +{"text": "m👍🏽", "tokens": 4, "pieces": ["m", "👍🏽"]} +{"text": "字ⅣDž​ -0漢12345678", "tokens": 13, "pieces": ["字", "Ⅳ", "Dž", "​", " ", " -", "0", "漢", "123", "456", "78"]} +{"text": "'DEOT' \n 0字Z३", "tokens": 10, "pieces": ["'DEOT", "'", " \n", " ", "0", "字", "Z", "३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "字Ⅳm​d‍½'re👍🏽,\nDžs㍿𐞁Z12345678\r\n", "tokens": 40, "pieces": ["字", "Ⅳ", "m", "​d", "‍", "½", "'re", "👍🏽,\n", "Džs", "㍿𐞁", "Z", "123", "456", "78", "\r\n"]} +{"text": "'ſ<|endoftext|>t'Tdaع👍🏽字\rZ'Mꟲ\r 漢\t0fi<|endoftext|>'MⅣe',३­Ａt­​Ⅳé½><|fim_prefix|>", "tokens": 59, "pieces": ["'ſ", "<|", "endoftext", "|>", "t'T", "daع", "👍🏽", "字", "\r", "Z'M", "ꟲ", "\r", " ", " 漢", "\t", "0", "fi", "<|", "endoftext", "|>'", "M", "Ⅳ", "e", "',", "३", "­Ａt", "­​", "Ⅳ", "é", "½", "><|", "fim", "_prefix", "|>"]} +{"text": "İ'Ré　Ⅳ ſ'reZsm\ń 'Tḍ̇'reEOT　​  \n𐞁\r'Sa'Té٣٤٥٦­́٣٤٥٦DžⅣＡ", "tokens": 51, "pieces": ["İ'Re", "́", "　", "Ⅳ", " ſ're", "Zsm", "\n", "́", " '", "Tḍ̇'re", "EOT", "　", "​", "  \n", "𐞁", "\r", "'", "Sa'T", "é", "٣٤٥", "٦", "­́", "٣٤٥", "٦", "Dž", "Ⅳ", "Ａ"]} +{"text": "…字\u000btå>'T字'VEḍ̇½'", "T字'VE", "ḍ̇", "½", "İ'sé\r\n\r\n­ßs\r\n\r\nfi<9're\n'MéⅣ<|fim_prefix|>#$%.\"'Te३ \r\n\r\n>　e٣٤٥٦\u000bed", "tokens": 52, "pieces": ["Ź's", "123", "456", "78", "Z", "İ's", "é", "\r\n\r\n", "­ßs", "\r\n\r\n", "fi", "<", "9", "'re", "\n", "'Mé", "Ⅳ", "<|", "fim", "_prefix", "|>#$%.\"'", "Te", "३", " \r\n\r\n", ">", "　e", "٣٤٥", "٦", "\u000bed"]} +{"text": "'s'll ३Džad'll-　 ㍿'T'ſ\"Ⅳ 字漢 ३ !!İa\r\n\r\n३0s", "tokens": 35, "pieces": ["'s'll", " ", " ", "३", "Džad'll", "-", "　", " ", "㍿'", "T'ſ", "\"", "Ⅳ", " ", " 字漢", " ", "३", " ", "!!", "İa", "\r\n\r\n", "३0", "s"]} +{"text": " ‍㍿'VEEOT-\t🙂́12345678\r\n\r\n'Re#$%😀🏽​🙂-\r\n\r\n😀🏽're#$%ع,'VEEOT㍿👍🏽é𐞁m'D　👍🏽٣٤٥٦\"(ſ", "tokens": 62, "pieces": [" ", "‍㍿'", "VEEOT", "-", "\t", "🙂́", "123", "456", "78", "\r\n\r\n", "'Re", "#$%😀🏽​🙂-\r\n\r\n", "😀🏽'", "re", "#$%", "ع", ",'", "VEEOT", "㍿👍🏽", "é𐞁m'D", "　", "👍🏽", "٣٤٥", "٦", "\"(", "ſ"]} +{"text": ".a\r\n
'Sꟲ㋿fiDž\n9fi 9🙂'Re👍🏽 d……३", "tokens": 30, "pieces": [".a", "\r\n", "
", "'Sꟲ", "㋿fi", "Dž", "\n", "9", "fi", " ", "9", "🙂'", "Re", "👍🏽", " d", "…", "…", "३"]} +{"text": "d👍🏽fi!m漢漢!!‍EOT\r\n<|fim_prefix|>'Re­'ll㍿　🙂\r\n\r\n,'Dḍ̇e­　­étع́'llß<|endoftext|>\r　-", "tokens": 59, "pieces": ["d", "👍🏽", "fi", "!m漢", "漢", "!!‍", "EOT", "\r\n", "<|", "fim", "_prefix", "|>'", "Re", "­'", "ll", "㍿", "　", "🙂\r\n\r\n", ",'", "Dḍ̇e", "­", "　", "­étع́'ll", "ß", "<|", "endoftext", "|>\r", "　", "-"]} +{"text": "<|endoftext|>'S!eⅣ\r\naéEOT'ſ\r\n\r\n­<|endoftext|>sß　👍🏽ⅣéEOT​", "tokens": 42, "pieces": ["<|", "endoftext", "|>'", "S", "!e", "Ⅳ", "\r\n", "aé", "EOT'ſ", "\r\n\r\n", "­<|", "endoftext", "|>", "sß", "　", "👍🏽", "Ⅳ", "é", "EOT", "​"]} +{"text": "\ŕ<|fim_prefix|>'D'M 
ḍ̇s \n😀🏽#$%Ａéİ　㋿\n ", "tokens": 31, "pieces": ["\r", "́", "<|", "fim", "_prefix", "|>'", "D'M", " ", "
ḍ̇s", " \n", "😀🏽#$%", "Ａé", "İ", "　", "㋿\n", " "]} +{"text": "ꟲ\"Zḍ̇ååfiꟲ", "tokens": 16, "pieces": ["ꟲ", "\"Zḍ̇ååfiꟲ"]} +{"text": "\r\n\r\nm'll're'VE\u000bEOTd\t🙂're٣٤٥٦ꟲ漢\t字#$%ḍ̇\"de", "tokens": 34, "pieces": ["\r\n\r\n", "m'll", "'re'VE", "\u000b", "EOTd", "\t", "🙂'", "re", "٣٤٥", "٦", "ꟲ漢", "\t字", "#$%", "ḍ̇", "\"de"]} +{"text": ".漢\t'㋿12345678ßZEOT'VE <\r \n…㍿(fis㍿'VEDž's-ſ12345678٣٤٥٦", "tokens": 44, "pieces": [".漢", "\t", "'㋿", "123", "456", "78", "ß", "ZEOT'VE", " ", "<\r", " \n", "…", "㍿(", "fis", "㍿'", "VEDž's", "-ſ", "123", "456", "78٣", "٤٥٦"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'TEOT\"-ſ㋿ \n㋿", "tokens": 12, "pieces": ["'TEOT", "\"-", "ſ", "㋿", " \n", "㋿"]} +{"text": ".'llåm0'T­‍'VE're m½\ŕ\t", "tokens": 17, "pieces": [".'", "llåm", "0", "'T", "­‍'", "VE're", " ", " m", "½", "\r", "́", "\t"]} +{"text": "'Re😀🏽३'Mt", "tokens": 11, "pieces": ["'Re", "😀🏽", "३", "'Mt", ""]} +{"text": "ⅣZå\r!('llDž\r\n
ßſ're 𐞁'sꟲ", "tokens": 24, "pieces": ["Ⅳ", "Zå", "\r", "!('", "ll", "Dž", "\r\n", "
ßſ're", " 𐞁's", "ꟲ"]} +{"text": "👍🏽d́.\r\n३ \n\"　\t😀🏽e Z'ḍ̇EOT👍🏽>", "tokens": 27, "pieces": ["👍🏽", "d́", ".\r\n", "३", " \n", "\"", "　", "\t", "😀🏽", "e", " ", " Z", "'ḍ̇", "EOT", "👍🏽>"]} +{"text": "9m㋿12345678's'S'Ts'Re'Re\t<|fim_prefix|>.\"\r\n", "tokens": 22, "pieces": ["9", "m", "㋿", "123", "456", "78", "'s'S", "'Ts'Re", "'Re", "\t", "<|", "fim", "_prefix", "|>.\"\r\n"]} +{"text": "EOT!字,'ſ字\r😀🏽éİ#$%٣٤٥٦३m🙂\".é'S🙂ḍ̇ſ \n \n\" 9EOTe", "tokens": 45, "pieces": ["EOT", "!字", ",'", "ſ字", "\r", "😀🏽", "é", "İ", "#$%", "٣٤٥", "٦३", "m", "🙂\"<", "META", "_START", ">.", "é'S", "🙂ḍ̇ſ", " \n \n", "\"", " ", "9", "EOTe"]} +{"text": " \na\r\n\r\n9'll<'VE'Re漢
­\r\nⅣ\u000b㋿字🙂(>Ａ", "tokens": 27, "pieces": [" \n", "a", "\r\n\r\n", "9", "'", "ll", "<'", "VE'Re", "漢", "
", "­\r\n", "Ⅳ", "\u000b", "㋿字", "🙂(>", "Ａ"]} +{"text": "㋿ſ Ⅳ ><|fim_prefix|>
Z🙂
d\t", "tokens": 23, "pieces": ["㋿", "ſ", " ", "Ⅳ", " ", "><|", "fim", "_prefix", "|>", "
Z", "🙂", "
d", "\t"]} +{"text": "
((½Ⅳ 𐞁‍#$%(é'T<#$%éEOTZ12345678's\n‍\u000b.ſe<|endoftext|>㋿​́>㋿<|fim_prefix|>
字Dž٣٤٥٦", "tokens": 66, "pieces": ["
", "((", "½Ⅳ", " 𐞁", "‍#$%(", "é'T", "<#$%", "é", "EOTZ", "123", "456", "78", "'s", "\n", "‍", "\u000b", ".ſe", "<|", "endoftext", "|>㋿​́>㋿<|", "fim", "_prefix", "|>", "
字", "Dž", "٣٤٥", "٦"]} +{"text": "­漢EOT'D३é \r\n\r\n‍'s㍿ \n'll\n12345678ßEOTtd İع‍e#$% İ'VEßＡ", "tokens": 37, "pieces": ["­漢", "EOT'D", "३", "é", " \r\n\r\n", "‍'", "s", "㍿", " \n", "'ll", "\n", "123", "456", "78", "ß", "EOTtd", " İع", "‍e", "#$%", " ", " İ'VE", "ß", "Ａ"]} +{"text": "fi'S,\r\n\"​<😀🏽", "tokens": 9, "pieces": ["fi'S", ",\r\n", "\"​<😀🏽"]} +{"text": "ſ­ EOT'ſ
'Re👍🏽😀🏽Dž\u000bad🙂s", "tokens": 24, "pieces": ["ſ", "­", " EOT'ſ", "", "
", "'Re", "👍🏽😀🏽", "Dž", "\u000bad", "🙂s"]} +{"text": "<|endoftext|>İ\n…-👍🏽㍿m's　­Džt <|fim_prefix|>12345678'T<|fim_prefix|>dḍ̇a", "tokens": 47, "pieces": ["<|", "endoftext", "|>", "İ", "\n", "…", "-👍🏽㍿", "m's", "　", "­Džt", " ", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "<|", "fim", "_prefix", "|>", "dḍ̇a"]} +{"text": "('D\r'ſ", "tokens": 5, "pieces": ["('", "D", "\r", "'ſ"]} +{"text": "!!aée\"漢 漢 .㍿　", "tokens": 17, "pieces": ["!!", "aé", "e", "\"漢", " ", " 漢", " .㍿", "　"]} +{"text": "å½­12345678\t,ع,as'VE\r ٣٤٥٦ḍ̇३ \n㍿٣٤٥٦　ſå½𐞁ßḍ̇\n12345678ǻ", "tokens": 53, "pieces": ["å", "½", "­", "123", "456", "78", "\t", ",ع", ",as'VE", "\r", " ", " ", "٣٤٥", "٦", "ḍ̇", "३", " \n", "㍿", "٣٤٥", "٦", "　ſå", "½", "𐞁ßḍ̇", "\n", "123", "456", "78", "ǻ"]} +{"text": "é'Tm
.́字
're​ \né'll", "tokens": 15, "pieces": ["é'T", "m", "
", ".́字", "
", "'re", "​", " \n", "é'll"]} +{"text": "㋿9EOT½,'re'S!'ſ<|endoftext|>🙂३İ're'T\r\n…ꟲ#$%'Sḍ̇😀🏽12345678'ſḍ̇\r's𐞁!!\n
漢#$%", "tokens": 58, "pieces": ["㋿", "9", "EOT", "½", ",'", "re'S", "!'", "ſ", "<|", "endoftext", "|>🙂", "३", "İ're", "'T", "\r\n", "…ꟲ", "#$%'", "Sḍ̇", "😀🏽", "123", "456", "78", "'ſḍ̇", "\r", "'s𐞁", "!!\n", "
漢", "#$%"]} +{"text": ".fi　ꟲt12345678\n㍿ḍ̇\r\n\r\ń\n", "㍿ḍ̇", "\r\n\r\n", "́", "EOTＡ\"e>　 \"㍿", "tokens": 19, "pieces": ["İ", "<|", "endoftext", "|>", "EOTＡ", "\"e", ">", "　", " \"㍿"]} +{"text": "ſ<|endoftext|>9Ａ-'re>½>́ ", "tokens": 17, "pieces": ["ſ", "<|", "endoftext", "|>", "9", "Ａ", "-'", "re", ">", "½", ">́", " "]} +{"text": "Ａ \n(Ⅳ'VEſ(ZZå\r\n\r\nå\" s㍿ḍ̇㍿9e.!!́\n", "tokens": 33, "pieces": ["Ａ", " \n", "(", "Ⅳ", "'VEſ", "(ZZå", "\r\n\r\n", "å", "\"", " ", " s", "㍿ḍ̇", "㍿", "9", "e", ".!!́\n"]} +{"text": "ꟲDž !<\r\n're -", "tokens": 11, "pieces": ["ꟲ", "Dž", " !<\r\n", "'re", " ", " -"]} +{"text": "ß9<|fim_prefix|>'sⅣ'sḍ̇éſ\t#$%Ａ'Mé­9\u000bés>　​漢's😀🏽'll ½!!\r\n'M'T‍\n", "tokens": 51, "pieces": ["ß", "9", "<|", "fim", "_prefix", "|>'", "s", "Ⅳ", "'sḍ̇éſ", "\t", "#$%", "Ａ'M", "é", "­", "9", "\u000bés", ">", "　", "​<", "EOT", ">漢's", "😀🏽'", "ll", " ", "½", "!!\r\n", "'M'T", "‍\n"]} +{"text": "́,字́\"d's'S\r\n　½9'D ḍ̇½🙂ꟲ'́漢!!🙂", "tokens": 27, "pieces": ["́", ",字́", "\"d's", "'S", "\r\n", "　", "½9", "'D", " ", " ḍ̇", "½", "🙂ꟲ", "'́漢", "!!🙂"]} +{"text": "#$%ZꟲＡam!9Ⅳß½́12345678!!", "tokens": 19, "pieces": ["#$%", "ZꟲＡam", "!", "9Ⅳ", "ß", "½", "́", "123", "456", "78", "!!"]} +{"text": "
\r\n\r\n㍿<0\r\nß'9<|endoftext|>ſ", "tokens": 19, "pieces": ["
\r\n\r\n", "㍿<", "0", "\r\n", "ß", "'", "9", "<|", "endoftext", "|>", "ſ"]} +{"text": "
ع…-å'S9👍🏽EOT́\"'ḍ̇.'é­!́EOT<|fim_prefix|>字åå ", "tokens": 41, "pieces": ["
ع", "…", "-å'S", "9", "👍🏽", "EOT́", "\"'", "ḍ̇", ".'", "é", "­<", "META", "_START", ">!́", "EOT", "<|", "fim", "_prefix", "|>", "字åå", " "]} +{"text": "<|fim_prefix|>𐞁'ſ9ḍ̇
字ꟲع éEOT𐞁 #$%!!'ſ​३mEOT\r\nZ(​'s㍿#$%!!\"", "tokens": 57, "pieces": ["<|", "fim", "_prefix", "|>", "𐞁'ſ", "9", "ḍ̇", "
字ꟲع", " é", "EOT𐞁", " ", "#$%!!<", "EOT", ">'", "ſ", "​", "३", "m", "EOT", "\r\n", "Z", "(​'", "s", "㍿#$%!!\""]} +{"text": "Ⅳ½​m\r\nＡ'T…\u000bdDžaİ　<|fim_prefix|>‍'llⅣ‍\"\u000b'S-", "tokens": 40, "pieces": ["Ⅳ½", "​m", "\r\n", "Ａ'T", "…", "\u000bd", "Dža", "İ", "　", "<|", "fim", "_prefix", "|>‍'", "ll", "", "Ⅳ", "‍\"", "\u000b", "'S", "-"]} +{"text": "ſe… EOT", "tokens": 7, "pieces": ["ſe", "…", " EOT"]} +{"text": "३('VE漢\r\n\r\n 👍🏽-́a‍#$%<\u000b>😀🏽𐞁ſꟲ'Re
ꟲte'llİ", "tokens": 37, "pieces": ["३", "('", "VE漢", "\r\n\r\n", " ", " 👍🏽-́", "a", "‍#$%<", "\u000b", ">😀🏽", "𐞁ſꟲ'Re", "
ꟲte'll", "İ"]} +{"text": "½‍\r­'reåⅣİé㍿,'s'S", "tokens": 17, "pieces": ["½", "‍\r", "­'", "reå", "Ⅳ", "İé", "㍿,'", "s'S"]} +{"text": "((
Dž'… 0d\r>aé're'll<|endoftext|>'D'llع<|fim_prefix|>'>عé're9\t \nＡa", "tokens": 46, "pieces": ["((", "
Dž", "'", "…", " ", "0", "d", "\r", ">aé're", "'ll", "<|", "endoftext", "|>'", "D'll", "ع", "<|", "fim", "_prefix", "|>'>", "عé're", "", "9", "\t \n", "Ａa"]} +{"text": "ع'll😀🏽're'Re㋿'M字 ", "tokens": 19, "pieces": ["ع'll", "😀🏽'", "re'Re", "㋿'", "M", "字", " "]} +{"text": "fi㋿(d< 0\"Z漢ß‍😀🏽漢''ſé<|fim_prefix|>\r\n!9\"#$%…­EOT👍🏽\t👍🏽'TDž-", "tokens": 49, "pieces": ["fi", "㋿(", "d", "<", " ", "0", "\"Z漢ß", "‍😀🏽", "漢", "''", "ſé", "<|", "fim", "_prefix", "|>\r\n", "!", "9", "\"#$%", "…", "­EOT", "👍🏽", "\t", "👍🏽'", "TDž", "-"]} +{"text": "ſm🙂…Ⅳ𐞁½́st!'re", "tokens": 16, "pieces": ["ſm", "🙂", "…", "Ⅳ", "𐞁", "½", "́st", "!'", "re"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n\r\n😀🏽ع́字🙂ß३'Re\"𐞁\"'ll \n㋿12345678🙂\n's\n #$%!!ꟲ'ſ#$%!!ſ\r\n👍🏽's're'T", "tokens": 51, "pieces": ["\r\n\r\n", "😀🏽", "ع́字", "🙂ß", "३", "'Re", "\"𐞁", "\"'", "ll", " \n", "㋿", "123", "456", "78", "🙂\n", "'s", "\n", " ", " #$%!!", "ꟲ'ſ", "#$%!!", "ſ", "\r\n", "👍🏽'", "s're", "'T"]} +{"text": "㍿㍿<👍🏽\"🙂 déEOTåḍ̇\r\n\r\n's\t<|fim_prefix|>><'S漢㋿å're \n½'D-", "tokens": 50, "pieces": ["㍿㍿<👍🏽\"🙂", " ", " dé", "EOTå", "ḍ̇", "\r\n\r\n", "'s", "\t", "<|", "fim", "_prefix", "|>><'", "S漢", "㋿", "å're", " \n", "½", "'D", "-"]} +{"text": "'VE.́　…\rⅣİ'Re\"é٣٤٥٦'VE\u000b字٣٤٥٦ḍ̇👍🏽0!!Ａ३é12345678mat\r\n'S👍🏽å漢‍'sDžＡ'S", "tokens": 61, "pieces": ["'VE", ".́", "　…\r", "Ⅳ", "İ'Re", "\"é", "٣٤٥", "٦", "'VE", "\u000b字", "٣٤٥", "٦", "ḍ̇", "👍🏽", "0", "!!", "Ａ", "३", "é", "123", "456", "78", "mat", "\r\n", "'S", "👍🏽", "å漢", "‍'", "s", "DžＡ'S"]} +{"text": "!!字é<|fim_prefix|>'D\r(Ⅳ'Dꟲ,🙂…EOT३'D( \n \n­ \n'lléas
'Ree\r\n- ", "tokens": 41, "pieces": ["!!", "字é", "<|", "fim", "_prefix", "|>'", "D", "\r", "(", "Ⅳ", "'Dꟲ", ",🙂", "…EOT", "३", "'D", "(", " \n \n", "­", " \n", "'lléas", "
", "'Ree", "\r\n", "-", " "]} +{"text": "…", "tokens": 2, "pieces": ["…"]} +{"text": "'re#$%Dž'😀🏽㋿ع \nå0­
'lla字''s'S😀🏽'ſ \u000bZ-é \"'ll­👍🏽<|fim_prefix|>ꟲ!!\n", "tokens": 56, "pieces": ["'re", "#$%", "Dž", "'😀🏽㋿", "ع", " \n", "å", "0", "­", "
", "'lla字", "''", "s'S", "😀🏽'", "ſ", " ", "\u000bZ", "-é", " ", "\"'", "ll", "­👍🏽<|", "fim", "_prefix", "|>", "ꟲ", "!!\n"]} +{"text": "'D…\u000b🙂a🙂 EOT!!٣٤٥٦12345678\r'Té३\r\n\r\n9'VE", "tokens": 25, "pieces": ["'D", "…", "\u000b", "🙂a", "🙂", " EOT", "!!", "٣٤٥", "٦12", "345", "678", "\r", "'Té", "३", "\r\n\r\n", "9", "'VE"]} +{"text": "'Så\u000be𐞁'ſ㍿'s㋿İ'S㍿d\t㋿'S'ſ're's😀🏽३#$%İⅣfi　fifidꟲ'", "ſ", "㍿'", "s", "㋿İ'S", "㍿d", "\t", "㋿'", "S'ſ", "'re's", "😀🏽", "३", "#$%", "İ", "Ⅳ", "fi", "　fifidꟲ", "éå㍿'s\té\r٣٤٥٦å­å'Re\r\n🙂٣٤٥٦12345678­Ⅳ", "tokens": 39, "pieces": ["t", "-a'D", ">éå", "㍿'", "s", "\té", "\r", "٣٤٥", "٦", "å", "­å'Re", "\r\n", "🙂", "٣٤٥", "٦12", "345", "678", "­", "Ⅳ"]} +{"text": "½'Re'ſ9>d \n<'llé0dß12345678'D­ſ#$%'s", "tokens": 25, "pieces": ["½", "'", "Re'ſ", "9", ">d", " \n", "<'", "llé", "0", "dß", "123", "456", "78", "'D", "­ſ", "#$%'", "s"]} +{"text": "'VE", "tokens": 6, "pieces": ["'VE", ""]} +{"text": "'re's३'Re12345678eＡ'VE\nＡ,'s#$%\n𐞁\"Dž", "tokens": 28, "pieces": ["'re's", "३", "'Re", "123", "456", "78", "e", "Ａ'VE", "\n", "Ａ", ",'", "s", "#$%\n", "𐞁", "\"Dž"]} +{"text": "\n'VE🙂½<|endoftext|><\n'M!'Mꟲ­𐞁ꟲ're\u000bfis! 9'Da\r\na👍🏽😀🏽're( 12345678'", "tokens": 57, "pieces": ["\n", "'VE", "🙂", "½", "<|", "endoftext", "|><\n", "'M", "!'", "Mꟲ", "­<", "EOT", ">𐞁ꟲ're", "\u000bfis", "!", " ", "9", "'Da", "\r\n", "a", "👍🏽😀🏽'", "re", "(", " ", " ", "123", "456", "78", "'"]} +{"text": "fi \t👍🏽12345678're​ꟲ👍🏽EOT٣٤٥٦'ll٣٤٥٦ å('.", "tokens": 31, "pieces": ["fi", " ", "\t", "👍🏽", "123", "456", "78", "'re", "​ꟲ", "👍🏽", "EOT", "٣٤٥", "٦", "'ll", "٣٤٥", "٦", " å", "('."]} +{"text": "\"\rd㋿\t \t　'S\tså'VE🙂", "tokens": 15, "pieces": ["\"\r", "d", "㋿", "\t \t", "　", "'S", "\tså'VE", "🙂"]} +{"text": "m३\r\n 𐞁👍🏽\r's,s😀🏽́­ \r\n\r\n🙂𐞁", "tokens": 25, "pieces": ["m", "३", "\r\n", " 𐞁", "👍🏽\r", "'s", ",s", "😀🏽́­", " \r\n\r\n", "🙂𐞁"]} +{"text": "0ſ\r!EOT​é'Ḿ\r'M'S​et'reſ\u000b", "tokens": 22, "pieces": ["", "0", "ſ", "\r", "!EOT", "​é'M", "́", "\r", "'M'S", "​et're", "ſ", "\u000b"]} +{"text": "#$%,d
EOT漢\n​Ⅳꟲe'M", "tokens": 16, "pieces": ["#$%,", "d", "
EOT漢", "\n", "​", "Ⅳ", "ꟲe'M"]} +{"text": "><|endoftext|>!ḍ̇'Ré.漢>'M", "tokens": 20, "pieces": ["><|", "endoftext", "|>!", "ḍ̇'Re", "́", ".漢", ">'", "M", ""]} +{"text": "Zéſ--Dž's\r\n\r\nꟲ́'VE字漢're!!s\r\n'-<", "EOT", ">'", "s", "\r\n\r\n", "ꟲ́'VE", "字漢're", "!!", "s", "\r\n", "'-<", "EOT", "🙂", " ", "　", "#$%"]} +{"text": " 👍🏽
!!\t!!'sꟲ́<ꟲ🙂é'VEＡ\"Z𐞁'-İ<|endoftext|>", "
", "!!", "\t", "!!'", "sꟲ́", "<ꟲ", "🙂é'VE", "Ａ", "\"Z𐞁", "'-", "İ", "<|", "endoftext", "|><", "s", "9", "​"]} +{"text": ",tع'D \nḍ̇㍿'Re\r\n 𐞁'VEd👍🏽​'ll'ReＡ𐞁 \n🙂", "tokens": 35, "pieces": [",tع'D", " \n", "ḍ̇", "㍿'", "Re", "\r\n", " 𐞁'VE", "d", "👍🏽​'", "ll'Re", "Ａ𐞁", " \n", "🙂"]} +{"text": "́'VEDž\nⅣé𐞁‍'M", "tokens": 16, "pieces": ["́'VE", "Dž", "\n", "Ⅳ", "é𐞁", "‍'", "M"]} +{"text": "é \nEOT
<'‍'Ttå!", "tokens": 13, "pieces": ["é", " \n", "EOT", "
", "<'‍'", "Ttå", "!"]} +{"text": "!!字\r\n\r\n'ſ'D　Z㋿'T( Ⅳ0eé㋿a're \n", "tokens": 27, "pieces": ["!!", "字", "\r\n\r\n", "'ſ'D", "　Z", "㋿'", "T", "(", " ", " ", "Ⅳ0", "eé", "㋿a're", " \n"]} +{"text": "aſ'D👍🏽­㍿aḍ̇\u000b​! \n'S<|endoftext|>s're9fi>", "tokens": 31, "pieces": ["aſ'D", "👍🏽­㍿", "aḍ̇", "\u000b", "​!", " \n", "'S", "<|", "endoftext", "|>", "s're", "9", "fi", ">"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "Ⅳ३Ａ٣٤٥٦0éſ\u000b\r𐞁 Džé½ḍ̇sd,'Sa\r\n\r\n३é", "tokens": 32, "pieces": ["Ⅳ३", "Ａ", "٣٤٥", "٦0", "éſ", "\u000b\r", "𐞁", " Džé", "½", "ḍ̇sd", ",'", "Sa", "\r\n\r\n", "३", "é"]} +{"text": "t d​ß'ſfi-!😀🏽a३'sḍ̇12345678Z'll­'Re#$%
 ,s'Så'Té'ſ(\r\n\r\ntſm'\r\n", "tokens": 47, "pieces": ["t", " ", " d", "​ß'ſ", "fi", "-!😀🏽", "a", "३", "'sḍ̇", "123", "456", "78", "Z'll", "­'", "Re", "#$%", "
 ", " ,", "s'S", "å'T", "é'ſ", "(\r\n\r\n", "tſm", "'\r\n"]} +{"text": "​漢<'DtEOT­ḍ̇<𐞁<字́ 'M're0", "tokens": 25, "pieces": ["​", "漢", "<'", "Dt", "EOT", "­ḍ̇", "<𐞁", "<字́", " '", "M're", "0"]} +{"text": ">dé,!!-\rEOT- \nt Ⅳ \n\tḍ̇Z字", "tokens": 29, "pieces": [">", "dé", ",!!-\r", "EOT", "-", " \n", "t", " ", " ", "Ⅳ", " \n", "", "\tḍ̇", "Z字"]} +{"text": "\r\n
'Sſ٣٤٥٦!!…DžZ३'Tع#$%e", "tokens": 24, "pieces": ["\r\n", "
", "'Sſ", "٣٤٥", "٦", "!!", "…", "DžZ", "३", "'Tع", "#$%", "e"]} +{"text": "fi🙂'👍🏽'M \n'Ta🙂t \u000b　>Ａfi", "🙂'👍🏽'", "M", " \n", "'Ta", "🙂t", " \u000b", "　", ">Ａ", "#$%å'ſ𐞁<|fim_prefix|>ß!!🙂….ꟲ\"", "tokens": 48, "pieces": [".", "123", "456", "78", "'𐞁", "३", "ع'M", "(,", "½", "́", "\n", "३", ">#$%", "å'ſ", "𐞁", "<|", "fim", "_prefix", "|>", "ß", "!!🙂", "…", ".ꟲ", "\""]} +{"text": "<|fim_prefix|>> \n𐞁e", "tokens": 12, "pieces": ["<|", "fim", "_prefix", "|>>", " \n", "𐞁e"]} +{"text": "'Dt-ḍ̇३'ſ'dZ.m\r\n‍\r\n\r\n,Dž\r''M'VE", "tokens": 23, "pieces": ["'Dt", "-ḍ̇", "३", "'ſ'd", "Z", ".m", "\r\n", "‍\r\n\r\n", ",Dž", "\r", "''", "M'VE"]} +{"text": "'VE'TＡ<\r\n'S'VEfi漢-#$%\u000b½sééd.½　'S.'ll>🙂Ａ's('T​漢'T", "tokens": 36, "pieces": ["'VE'T", "Ａ", "<\r\n", "'S'VE", "fi漢", "-#$%", "\u000b", "½", "sééd", ".", "½", "　", "'S", ".'", "ll", ">🙂", "Ａ's", "('", "T", "​漢'T"]} +{"text": "'S\"𐞁\r\n\r\n字‍,
\"🙂‍.aa!9'Dta'M​\t\nZ\t　…m­İ𐞁,Za㋿'re", "tokens": 47, "pieces": ["'S", "\"", "𐞁", "\r\n\r\n", "字", "‍,", "
", "\"🙂‍.", "aa", "!", "9", "'Dta'M", "​", "\t\n", "Z", "\t　", "…m", "­İ", "𐞁", ",Za", "㋿'", "re"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㋿å㍿12345678(-<|endoftext|>>!!\r\n'ſ'VEſ
'sZ<|endoftext|>😀🏽9ßİ‍…EOTⅣع's(ſ <|fim_prefix|>", "tokens": 69, "pieces": ["㋿å", "㍿", "123", "456", "78", "(-<|", "endoftext", "|>><", "EOT", ">!!\r\n", "'ſ", "'", "VEſ", "
", "'s", "Z", "<|", "endoftext", "|>😀🏽", "9", "ß", "İ", "‍", "…EOT", "Ⅳ", "ع's", "(ſ", " <|", "fim", "_prefix", "|>"]} +{"text": " '‍\r\n9-'ree'rea\r\n\r\n३𐞁(́Dž…'Ree😀🏽\"🙂'D0,fi", "tokens": 37, "pieces": [" ", " '‍\r\n", "9", "-'", "ree're", "a", "\r\n\r\n", "३", "𐞁", "(́", "Dž", "", "…", "'Ree", "😀🏽\"🙂'", "D", "0", ",fi"]} +{"text": "🙂0漢(Dž'VE<|fim_prefix|> \"ḍ̇­Z👍🏽٣٤٥٦漢(å字EOT\"12345678's\n'D", "tokens": 43, "pieces": ["🙂", "0", "漢", "(Dž'VE", "<|", "fim", "_prefix", "|>", " ", "\"ḍ̇", "­Z", "👍🏽", "٣٤٥", "٦", "漢", "(å字", "EOT", "\"", "123", "456", "78", "'s", "\n", "'D"]} +{"text": "<|fim_prefix|><|endoftext|>'Re…'T#$% d<|endoftext|><|fim_prefix|>12345678", "tokens": 34, "pieces": ["<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "Re", "…", "'T", "#$%", " d", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "123", "456", "78"]} +{"text": "　㋿𐞁İ\nmḍ̇'re<|fim_prefix|>'Sḍ̇mfi9<٣٤٥٦'Dß're \n ḍ̇㍿\r‍'👍🏽<|endoftext|>🙂ſ'S", "tokens": 60, "pieces": ["　", "㋿𐞁", "İ", "\n", "mḍ̇'re", "<|", "fim", "_prefix", "|>'", "Sḍ̇mfi", "9", "<", "٣٤٥", "٦", "'Dß're", " \n", " ḍ̇", "㍿\r", "‍'👍🏽<|", "endoftext", "|>🙂", "ſ'S"]} +{"text": "'D're½½Ａs9>'M,😀🏽.'Re　́🙂'Ms's३­\r\n­9'", "re", "½½", "Ａs", "9", ">'", "M", ",😀🏽.'", "Re", "　́", "🙂'", "Ms's", "३", "­\r\n", "­", "9", "é́t𐞁Ⅳ#$%'VE'sfi,ſḍ̇'M-,EOT \nＡ's", "tokens": 44, "pieces": ["İ", "‍​", "
", "㋿", "　漢", "‍\r", "३", "!!́<", "EOT", ">é́t𐞁", "Ⅳ", "#$%'", "VE's", "fi", ",ſḍ̇'M", "-,", "EOT", " \n", "Ａ's"]} +{"text": "'re…0!!٣٤٥٦ßdZ‍'sZ", "tokens": 16, "pieces": ["'re", "…", "0", "!!", "٣٤٥", "٦", "ßd", "Z", "‍'", "s", "Z"]} +{"text": "fiＡ \nEOT\t½'ll,s Ａ9", "tokens": 12, "pieces": ["fi", "Ａ", " \n", "EOT", "\t", "½", "'ll", ",s", " Ａ", "9"]} +{"text": "𐞁!!漢٣٤٥٦　㋿'S\r\n\r\nſ", "tokens": 18, "pieces": ["𐞁", "!!", "漢", "٣٤٥", "٦", "　", "㋿'", "S", "\r\n\r\n", "ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "!\r\n\r\n.\u000b­ſ.åe", "tokens": 9, "pieces": ["!\r\n\r\n", ".", "\u000b", "­ſ", ".åe"]} +{"text": " \t9'Mſ9!!\"\"9'llꟲ\u000bİ㋿!!fiZع'ſⅣⅣſ", "tokens": 29, "pieces": [" ", "\t", "9", "'Mſ", "9", "!!\"\"", "9", "'llꟲ", "\u000bİ", "㋿!!", "fi", "Zع'ſ", "ⅣⅣ", "ſ"]} +{"text": "EOT३ 12345678'T'Reḍ̇ \n'Dm'sd😀🏽\t㍿,t 'Tع\rå", "tokens": 41, "pieces": ["EOT", "३", "", " ", "123", "456", "78", "'", "T'Re", "ḍ̇", " \n", "'Dm's", "d", "😀🏽", "\t", "㍿,", "t", "", " ", "'Tع", "\r", "å"]} +{"text": "\u000bＡé
e\r'T <|endoftext|>t!!e'D٣٤٥٦𐞁👍🏽Ⅳ\rſ'lltDž('", "tokens": 44, "pieces": ["\u000bＡé", "
e", "\r", "'T", " ", " <|", "endoftext", "|>", "t", "!!<", "EOT", ">e'D", "٣٤٥", "٦", "𐞁", "👍🏽", "Ⅳ", "\r", "ſ'll", "t", "Dž", "('"]} +{"text": "\nḍ̇㋿㍿😀🏽漢a", "tokens": 15, "pieces": ["\n", "ḍ̇", "㋿㍿😀🏽", "漢a"]} +{"text": "s ㍿'reＡt!!𐞁½d㋿EOT…d!!㍿㍿e
éİ.​‍ßZ
३0Ⅳ,　é'S'VE漢३0's", "tokens": 54, "pieces": ["s", " ", "㍿'", "re", "Ａt", "!!", "𐞁", "½", "d", "㋿EOT", "…d", "!!㍿㍿", "e", "
é", "İ", ".​‍", "ß", "Z", "
", "३0Ⅳ", ",", "　é'S", "'VE漢", "३0", "'s"]} +{"text": " <㍿ع😀🏽Z𐞁ſ
mm​dß'Re漢\r\n\r\n \n\"…\"", "tokens": 27, "pieces": [" <㍿", "ع", "😀🏽", "Z𐞁ſ", "
mm", "​dß'Re", "漢", "\r\n\r\n \n", "\"", "…", "\""]} +{"text": " \ném'S½'s
字­'ſ\r\n\r\n‍ſ٣٤٥٦\r\n\r\n
'VE​éDž㍿\u000bååع\r\n'se\rعt''ll", "tokens": 48, "pieces": [" \n", "ém'S", "½", "'s", "
字", "­<", "EOT", ">'", "ſ", "\r\n\r\n", "‍ſ", "٣٤٥", "٦", "\r\n\r\n", "
", "'VE", "​é", "Dž", "㍿", "\u000bååع", "\r\n", "'se", "\r", "عt", "''", "ll"]} +{"text": "字mſ🙂'", "tokens": 5, "pieces": ["字mſ", "🙂'"]} +{"text": "🙂 ", "tokens": 2, "pieces": ["🙂", " "]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'s<|endoftext|>㋿\"👍🏽\rEOT́!e", "tokens": 21, "pieces": ["'s", "<|", "endoftext", "|>㋿\"👍🏽\r", "EOT́", "!e"]} +{"text": ".㋿at字<'Sm'M9字e
\u000bع!👍🏽ß㍿..-d ḍ̇🙂.'S\r'ſ\r\n\r\nＡ'ſ12345678'VE
é'ſ\n", "tokens": 52, "pieces": [".㋿", "at字", "<'", "Sm'M", "9", "字e", "
", "\u000bع", "!👍🏽", "ß", "㍿..-", "d", " ", " ḍ̇", "🙂.'", "S", "\r", "'ſ", "\r\n\r\n", "Ａ'ſ", "123", "456", "78", "'VE", "
é'ſ", "\n"]} +{"text": "ḍ̇e'ſ👍🏽<|fim_prefix|><|endoftext|>s!­ é😀🏽ſ½…'Mꟲ\u000b -㋿ 'S\r#$%fi字-<\u000b\r(-\reé㋿", "tokens": 68, "pieces": ["ḍ̇e'ſ", "👍🏽<|", "fim", "_prefix", "|><|", "endoftext", "|>", "s", "!­", " ", "é", "😀🏽", "ſ", "½", "…", "'Mꟲ", "\u000b", " -㋿", " ", " '", "S", "\r", "#$%", "fi字", "-<", "\u000b\r", "(-\r", "eé", "㋿"]} +{"text": "fid <|endoftext|>!­ 'D३Z🙂<|endoftext|>#$%字'ſé", "tokens": 34, "pieces": ["fid", " ", " <|", "endoftext", "|><", "META", "_START", ">!­", " ", "'D", "३", "Z", "🙂<|", "endoftext", "|>#$%", "字'ſ", "é"]} +{"text": "'ll !!​<|fim_prefix|>ع‍'ll‍‍''S.EOT#$%'
\u000b👍🏽Dž'\r\n\r\nİ́", "tokens": 38, "pieces": ["'ll", " ", " !!​<", "EOT", "><|", "fim", "_prefix", "|>", "ع", "‍'", "ll", "‍‍''", "S", ".EOT", "#$%'", "
", "\u000b", "👍🏽", "Dž", "'\r\n\r\n", "İ́"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "<|fim_prefix|>a­fi\"'S'ſ३<0'Sé>'VE㋿a.'Re'T\r\nſDža🙂0\"ſ", "tokens": 37, "pieces": ["<|", "fim", "_prefix", "|>", "a", "­fi", "\"'", "S'ſ", "३", "<", "0", "'Sé", ">'", "VE", "㋿a", ".'", "Re'T", "\r\n", "ſ", "Dža", "🙂", "0", "\"ſ"]} +{"text": "㍿12345678'S!!#$%fi'ß\t㋿åe.­😀🏽'VE\r\n\r\nDž‍漢٣٤٥٦İ㍿ >9'Re\u000b9!\u000b0 \nß‍'Re'ſſ", "tokens": 57, "pieces": ["㍿", "123", "456", "78", "'S", "!!#$%", "fi", "'ß", "\t", "㋿åe", ".­😀🏽'", "VE", "\r\n\r\n", "Dž", "‍漢", "٣٤٥", "٦", "İ", "㍿", " ", " >", "9", "'Re", "\u000b", "9", "!", "\u000b", "0", " \n", "ß", "‍'", "Re'ſ", "ſ"]} +{"text": "tm'll\t12345678<,३Ａ🙂12345678Dž'll𐞁İå𐞁\u000b\u000b
", "tokens": 31, "pieces": ["tm'll", "\t", "123", "456", "78", "<,", "३", "Ａ", "🙂", "123", "456", "78", "Dž'll", "𐞁İå𐞁", "\u000b\u000b
"]} +{"text": "'Reßå>12345678å­m😀🏽", "tokens": 15, "pieces": ["'Reßå", ">", "123", "456", "78", "å", "­m", "😀🏽"]} +{"text": "​ ſå㍿ꟲ<|fim_prefix|>-EOT<|fim_prefix|>'T,(㍿ 
ßعé'Mḍ̇.", "tokens": 44, "pieces": ["​", " ſå", "㍿ꟲ", "<|", "fim", "_prefix", "|>-", "EOT", "<|", "fim", "_prefix", "|>'", "T", ",(㍿", " ", "", "
ßعé'M", "ḍ̇", "."]} +{"text": "　㋿㋿\u000bꟲ", "tokens": 11, "pieces": ["　", "㋿㋿", "\u000bꟲ"]} +{"text": "　ꟲ३'ſ'ſ\n'll!!'M'S1234567812345678🙂\r\nZ\r", "tokens": 25, "pieces": ["　ꟲ", "३", "'ſ'ſ", "\n", "'ll", "!!'", "M'S", "123", "456", "781", "234", "567", "8", "🙂\r\n", "Z", "\r"]} +{"text": "fi​'D\"Džmſ 
>३ꟲ\r\n'ſa", "tokens": 20, "pieces": ["fi", "​'", "D", "\"Džmſ", " ", "
", ">", "३", "ꟲ", "\r\n", "'ſa"]} +{"text": "é>…‍́漢́ꟲ09…12345678d-　<|endoftext|> \n\r'' EOTå- ,'sEOT#$% \n<|endoftext|>9!!…😀🏽", "tokens": 66, "pieces": ["é", ">", "…", "‍́漢́ꟲ", "09", "…", "123", "456", "78", "d", "-", "　", "<|", "endoftext", "|>", " \n\r", "''", " ", " EOTå", "-<", "META", "_START", ">", " ", ",'", "s", "EOT", "#$%", " \n", "<|", "endoftext", "|>", "9", "!!", "…", "😀🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "å'lls<|fim_prefix|>'M'-12345678 \n'D é", "tokens": 20, "pieces": ["å'll", "s", "<|", "fim", "_prefix", "|>'", "M", "'-", "123", "456", "78", " \n", "'D", " ", " é"]} +{"text": "३漢t!!!!es\u000b\u000b\r\n​'ſ<|fim_prefix|>'TZ\"å!!!!", "es", "\u000b\u000b\r\n", "​'", "ſ", "<|", "fim", "_prefix", "|>'", "TZ", "\"å", "99 ㍿'D🙂-", "tokens": 14, "pieces": ["🙂", "　", "", "99", " ㍿'", "D", "🙂-"]} +{"text": "'T \n'T‍mZ'llmＡ!\r\n\r\n(ds\t", "tokens": 13, "pieces": ["'T", " \n", "'T", "‍m", "Z'll", "m", "Ａ", "!\r\n\r\n", "(ds", "\t"]} +{"text": "­
Dž\"ßİ#$%'M'sꟲ(\r\n\r\n<ꟲ'S'M😀🏽d३….​½ꟲ٣٤٥٦", "tokens": 41, "pieces": ["­", "
Dž", "\"", "ß", "İ", "#$%'", "M's", "ꟲ", "(\r\n\r\n", "<ꟲ'S", "'M", "😀🏽", "d", "३", "…", ".​", "½", "ꟲ", "٣٤٥", "٦"]} +{"text": "'Da‍㍿,'Reß12345678'DeZ-'re'D's,-12345678'VE'll३(", "tokens": 28, "pieces": ["'Da", "‍㍿,'", "Reß", "123", "456", "78", "'De", "Z", "-'", "re'D", "'s", ",-", "123", "456", "78", "'VE'll", "३", "("]} +{"text": "​ \n'VEſ", "tokens": 5, "pieces": ["​", " \n", "'VEſ"]} +{"text": "-ß'Re३å .", "tokens": 7, "pieces": ["-ß'Re", "३", "å", " ."]} +{"text": "åḍ̇!!<|fim_prefix|>\r's'S‍'sſ٣٤٥٦'D12345678.#$%-!fi　fi< 'é‍😀🏽३å> ", "tokens": 47, "pieces": ["åḍ̇", "!!<|", "fim", "_prefix", "|>\r", "'s'S", "‍'", "sſ", "٣٤٥", "٦", "'D", "123", "456", "78", ".#$%-!", "fi", "　fi", "<", " ", " '", "é", "‍😀🏽", "३", "å", ">", " "]} +{"text": "ß😀🏽ſ'llt'T''T'M0́!a'S,㍿٣٤٥٦>
…s>́'re\n's", "tokens": 34, "pieces": ["ß", "😀🏽", "ſ'll", "t'T", "''", "T'M", "0", "́", "!a'S", ",㍿", "٣٤٥", "٦", ">", "
", "…s", ">́'re", "\n", "'s"]} +{"text": "
\r\n
İ0'Red#$% 12345678åe \nع'VE'Re d'T漢> 'll'Tİ́㍿३𐞁ß12345678 ㋿", "tokens": 52, "pieces": ["
\r\n", "
İ", "0", "'Red", "#$%<", "EOT", ">", " ", "123", "456", "78", "åe", " \n", "ع'VE", "'Re", " d'T", "漢", ">", " '", "ll'T", "İ́", "㍿", "३", "𐞁ß", "123", "456", "78", " ", " ㋿"]} +{"text": "\u000bعع<|endoftext|>㍿ ​<|endoftext|>EOT'ſtd㍿'s‍'M\r(#$%9\n<|fim_prefix|>! \n'ſ9٣٤٥٦'T\tⅣ", "tokens": 60, "pieces": ["\u000bعع", "<|", "endoftext", "|>㍿", " ", " ​<|", "endoftext", "|>", "EOT'ſ", "td", "㍿'", "s", "‍'", "M", "\r", "(#$%", "9", "\n", "<|", "fim", "_prefix", "|>!", " \n", "'ſ", "9٣٤", "٥٦", "'T", "\t", "Ⅳ"]} +{"text": "eEOT''ress'VEm>'S- 'M字\r\n\r\n'Sḍ̇İ0're'Re
́dEOT", "''", "ress'VE", "m", ">'", "S", "-", " '", "M字", "\r\n\r\n", "'Sḍ̇", "İ", "0", "'re'Re", "
́d", "EOT‍ß\r
\n字<\u000bꟲḍ̇Dž'D́ésⅣ!\u000b<…(<|endoftext|>0's'Re 're'Sſa", "tokens": 60, "pieces": ["'re", "0", "Ａ", ".㋿<", "EOT", ">EOT", "‍ß", "\r
\n", "字", "<", "\u000bꟲḍ̇", "Dž'D", "́és", "Ⅳ", "!", "\u000b", "<", "…", "(<|", "endoftext", "|>", "0", "'", "s'Re", " <", "META", "_START", ">'", "re'S", "ſa"]} +{"text": "\" é", "tokens": 4, "pieces": ["\"", " ", " é"]} +{"text": "😀🏽's…½\t३#$%ḍ̇'T\r'Reé-0İ㍿<|fim_prefix|>Dž \n　", "tokens": 55, "pieces": ["ꟲع", " ß", "Ａe", "(\r\n\r\n", "eß", "㍿", " ", " ḍ̇'s", "ꟲ", "<|", "endoftext", "|>", "ḍ̇'T", "\r", "'Reé", "-", "0", "İ", "㍿<|", "fim", "_prefix", "|>", "Dž", " \n", "　"]} +{"text": "عe 'D३'s'reéa#$%9字\r!'s", "tokens": 17, "pieces": ["عe", " ", "'D", "३", "'s're", "éa", "#$%", "9", "字", "\r", "!'", "s"]} +{"text": "'ſⅣⅣ12345678>Dž​٣٤٥٦३", "tokens": 19, "pieces": ["'ſ", "ⅣⅣ1", "234", "567", "8", ">Dž", "​", "٣٤٥", "٦३"]} +{"text": "\rſ'ſ.ſ", "tokens": 6, "pieces": ["\r", "ſ'ſ", ".ſ"]} +{"text": "​(ꟲ'M🙂'VE\re'VE\n12345678İ's!!>٣٤٥٦Z\r\n\r\n\tＡ३\r\nZ㍿\r,­٣٤٥٦-\r !!'VE<|endoftext|><|endoftext|>", "tokens": 61, "pieces": ["​(", "ꟲ'M", "🙂'", "VE", "\r", "e'VE", "\n", "123", "456", "78", "İ's", "!!>", "٣٤٥", "٦", "Z", "\r\n\r\n", "\tＡ", "३", "\r\n", "Z", "㍿\r", ",­", "٣٤٥", "٦", "-\r", " ", "!!'", "VE", "<|", "endoftext", "|><|", "endoftext", "|>"]} +{"text": "ⅣⅣZé'T　\tm…'T<|fim_prefix|>ḍ̇- \n㋿Z'sé漢३​m \nd", "tokens": 36, "pieces": ["ⅣⅣ", "Zé'T", "　", "\tm", "…", "'T", "<|", "fim", "_prefix", "|>", "ḍ̇", "-", " \n", "㋿Z's", "é漢", "३", "​m", " \n", "d"]} +{"text": "!!ꟲ'Ts, #$%,…🙂>​aİ\r\n\r\n
ée

é‍,a'VE​aß\r\n\t9t<|fim_prefix|>\rſ٣٤٥٦<|fim_prefix|>Džae", "tokens": 57, "pieces": ["ſ", "\n", "😀🏽'", "re", "<|", "fim", "_prefix", "|>🙂>​", "a", "İ", "\r\n\r\n", "
ée", "
", "
é", "‍,", "a'VE", "​aß", "\r\n", "\t", "9", "t", "<|", "fim", "_prefix", "|>\r", "ſ", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "Džae"]} +{"text": "<|endoftext|>㍿­ås½ ع\r\n(ع𐞁👍🏽-'ll!'llm㋿Z", "tokens": 36, "pieces": ["<|", "endoftext", "|>㍿­", "ås", "½", " ", " ع", "\r\n", "(ع𐞁", "👍🏽-'", "ll", "!'", "llm", "㋿Z"]} +{"text": "<|fim_prefix|>İd㍿ß ­🙂!>,\r\n\r\n𐞁٣٤٥٦\" ,ꟲ…tå　a½é<|endoftext|>,٣٤٥٦é>", "tokens": 56, "pieces": ["<|", "fim", "_prefix", "|>", "İd", "㍿ß", " ­🙂!>,\r\n\r\n", "𐞁", "٣٤٥", "٦", "\"", " ,", "ꟲ", "…tå", "　a", "½", "é", "<|", "endoftext", "|>,", "٣٤٥", "٦", "é", ">"]} +{"text": "> ́!\t're'T㍿ſ㍿EOT\r\n\r\n­'ReEOTⅣ'M", "tokens": 29, "pieces": [">", " ́", "!<", "META", "_START", ">", "\t", "'re'T", "㍿ſ", "㍿EOT", "\r\n\r\n", "­'", "Re", "EOT", "Ⅳ", "'M"]} +{"text": "m-½𐞁Zꟲ漢㋿'D<|fim_prefix|>.<|fim_prefix|>'re'T0''lle\ńmsd12345678!!'Reſ(Ⅳ", "tokens": 48, "pieces": ["m", "-", "½", "𐞁Zꟲ漢", "㋿'", "D", "<|", "fim", "_prefix", "|>.<|", "fim", "_prefix", "|>'", "re'T", "0", "''", "lle", "\n", "́msd", "123", "456", "78", "!!'", "Reſ", "(", "Ⅳ"]} +{"text": "ꟲ👍🏽٣٤٥٦　Dž‍9'M㋿fi<|endoftext|>'ſs\té(㋿", "tokens": 43, "pieces": ["ꟲ", "👍🏽<", "META", "_START", ">", "٣٤٥", "٦", "　Dž", "‍<", "META", "_START", ">", "9", "'M", "㋿fi", "<|", "endoftext", "|>'", "ſs", "\té", "(㋿"]} +{"text": "𐞁'VE\r\n🙂‍字字(", "tokens": 12, "pieces": ["𐞁'VE", "\r\n", "🙂‍", "字字", "("]} +{"text": "́­'Re>Ａ́å.'VE½\n'Tt9åⅣ\"fi🙂Ａ", "tokens": 27, "pieces": ["́", "­'", "Re", "><", "EOT", ">Ａ́å", ".'", "VE", "½", "\n", "'Tt", "9", "å", "Ⅳ", "\"fi", "🙂Ａ"]} +{"text": "<ſå½EOTeDžé ", "tokens": 12, "pieces": ["<ſå", "½", "EOTe", "Džé", " "]} +{"text": "éİ(\r㍿ \né३Z!!9\u000bⅣ\u000b<|fim_prefix|>'ſ　ḍ̇漢-\tm'Té", "tokens": 37, "pieces": ["é", "İ", "(\r", "㍿", " \n", "é", "३", "Z", "!!", "9", "", "\u000b", "Ⅳ", "\u000b", "<|", "fim", "_prefix", "|>'", "ſ", "　ḍ̇漢", "-", "\tm'T", "é"]} +{"text": "'ll \"\u000b 🙂३<‍🙂12345678Dž <㋿e漢'ſⅣ0ع<|endoftext|>a㍿", "tokens": 39, "pieces": ["'ll", " ", " \"", "\u000b ", " 🙂", "३", "<‍🙂", "123", "456", "78", "Dž", " ", " <㋿", "e漢'ſ", "Ⅳ0", "ع", "<|", "endoftext", "|>", "a", "㍿"]} +{"text": "!!'ſa'Re\rＡ!Ａ9m", "tokens": 11, "pieces": ["!!'", "ſa'Re", "\r", "Ａ", "!Ａ", "9", "m"]} +{"text": "ع12345678", "tokens": 4, "pieces": ["ع", "123", "456", "78"]} +{"text": "​字ع're\n ㍿'VEé'llfi", "tokens": 14, "pieces": ["​字ع're", "\n", " ㍿'", "VEé'll", "fi"]} +{"text": " \n.عİ12345678d'fia\n​३ḍ̇m!>३'re#$%(​👍🏽𐞁
'S­'llſſ\n👍🏽 \nDž'Re 'sfi", "tokens": 56, "pieces": [" \n", ".ع", "İ", "123", "456", "78", "d", "'fia", "\n", "​", "३", "ḍ̇m", "!><", "META", "_START", ">", "३", "'", "re", "#$%(​👍🏽", "𐞁", "
", "'S", "­'", "llſſ", "\n", "👍🏽", " \n", "Dž'Re", " ", "'sfi"]} +{"text": "d", "tokens": 1, "pieces": ["d"]} +{"text": "\t㋿", "tokens": 4, "pieces": ["\t", "㋿"]} +{"text": ">  ㋿'llZ\t<|endoftext|><|endoftext|>'VE­ḍ̇!!'s­-\r'", "tokens": 35, "pieces": [">", " ", " ", "㋿'", "ll", "Z", "\t", "<|", "endoftext", "|><|", "endoftext", "|>'", "VE", "­ḍ̇", "!!'", "s", "­-\r", "'"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ꟲDž㍿𐞁d<|endoftext|><|fim_prefix|>< (عＡ字‍‍'T👍🏽's's#$%!!<Ⅳß ٣٤٥٦>'Dع　", "tokens": 56, "pieces": ["ꟲ", "Dž", "㍿𐞁d", "<|", "endoftext", "|><|", "fim", "_prefix", "|><", " ", " (", "عＡ字", "‍‍'", "T", "👍🏽'", "s's", "#$%!!<", "Ⅳ", "ß", " ", "٣٤٥", "٦", ">'", "Dع", "　"]} +{"text": "m \n's9İé(!>३㋿­<|fim_prefix|> !漢𐞁ḍ̇s#$%åß12345678'D", "tokens": 46, "pieces": ["m", " \n", "'s", "9", "İé", "(!>", "३", "㋿­<|", "fim", "_prefix", "|>", " ", " <", "META", "_START", ">!", "漢𐞁", "ḍ̇s", "#$%", "åß", "123", "456", "78", "'D"]} +{"text": "!é🙂\r\n३s𐞁'llEOT>
㋿>,́<𐞁عḍ̇\"عté0sam", "tokens": 35, "pieces": ["!é", "🙂\r\n", "३", "s𐞁'll", "EOT", ">", "
", "㋿>,́<", "𐞁عḍ̇", "\"عté", "0", "sam"]} +{"text": "fiZ'ſ!\nß\u000bsea0'ſ
Ⅳ🙂👍🏽 \n‍
('M　t('DＡ!éⅣ‍ m\té9d😀🏽'ſ", "tokens": 48, "pieces": ["fi", "Z'ſ", "!\n", "ß", "\u000bsea", "0", "'ſ", "
", "Ⅳ", "🙂👍🏽", " \n", "‍", "
", "('", "M", "　t", "('", "DＡ", "!é", "Ⅳ", "‍", " m", "\té", "9", "d", "😀🏽'", "ſ"]} +{"text": "12345678\n'…><|fim_prefix|>Ａ(d'ſ!'ll<|fim_prefix|>'s", "tokens": 26, "pieces": ["123", "456", "78", "\n", "'", "…", "><|", "fim", "_prefix", "|>", "Ａ", "(d'ſ", "!'", "ll", "<|", "fim", "_prefix", "|>'", "s"]} +{"text": "\r\n Z>'ſåſ\"ſ½­-(<　ع\r\n\r\n
", "tokens": 18, "pieces": ["\r\n", " Z", ">'", "ſåſ", "\"ſ", "½", "­-(<", "　ع", "\r\n\r\n", "
"]} +{"text": "!😀🏽🙂EOT'VEſ", "tokens": 10, "pieces": ["!😀🏽🙂", "EOT'VE", "ſ"]} +{"text": "é#$%m\r\n\r\nß𐞁>\t(‍Ａ\r\n\r\n'ſḍ̇३'S🙂İ(#$%", "tokens": 32, "pieces": ["é", "#$%", "m", "\r\n\r\n", "ß", "𐞁", ">", "\t", "(‍", "Ａ", "\r\n\r\n", "'ſḍ̇", "३", "'S", "🙂İ", "(#$%"]} +{"text": ".'Sḍ̇'S…", "tokens": 12, "pieces": [".'", "Sḍ̇'S", "", "…"]} +{"text": "ع …Ａ🙂'T. \n!'Re🙂é12345678Z\r\n", "tokens": 24, "pieces": ["ع", "", " ", "…Ａ", "🙂'", "T", ".", " \n", "!'", "Re", "🙂é", "123", "456", "78", "Z", "\r\n"]} +{"text": "\t३‍Ａ🙂Ⅳ㋿İḍ̇
<\r\n\r\n½<|fim_prefix|>ꟲ'DعＡDž'D'VE<|endoftext|>'ſ", "tokens": 43, "pieces": ["\t", "३", "‍Ａ", "🙂", "Ⅳ", "㋿İḍ̇", "
", "<\r\n\r\n", "½", "<|", "fim", "_prefix", "|>", "ꟲ'D", "ع", "ＡDž'D", "'VE", "<|", "endoftext", "|>'", "ſ"]} +{"text": "maſ\r\n\r\n'TDž'D Dž\rⅣ'T12345678å\t
Dž漢(!ع(!! ½EOT\r\nḍ̇!!　<|fim_prefix|>", "tokens": 50, "pieces": ["maſ", "\r\n\r\n", "'TDž'D", " ", " Dž", "\r", "Ⅳ", "'T", "123", "456", "78", "å", "\t", "
Dž漢", "(!", "ع", "(!!", " ", " ", "½", "EOT", "\r\n", "ḍ̇", "!!<", "EOT", ">", "　", "<|", "fim", "_prefix", "|>"]} +{"text": "​<|fim_prefix|>\t'lls漢Ⅳ >'T٣٤٥٦! Z#$%\"'re>
ع'M'ſEOT's👍🏽'\r\n\r\n‍​s\n­<|endoftext|>​!!ḍ̇0,", "tokens": 59, "pieces": ["​<|", "fim", "_prefix", "|>", "\t", "'lls漢", "Ⅳ", " ", ">'", "T", "٣٤٥", "٦", "!", " Z", "#$%\"'", "re", ">", "
ع'M", "'ſ", "EOT's", "👍🏽'\r\n\r\n", "‍​", "s", "\n", "­<|", "endoftext", "|>​!!", "ḍ̇", "0", ","]} +{"text": "½!t३\n .d㍿'Res​12345678 \n\r\n.漢…ع…ß'Re漢'VE<|endoftext|>٣٤٥٦ß
.\u000b'Séd
漢", "tokens": 60, "pieces": ["½", "!t", "३", "\n", " ", " .", "d", "㍿'", "Res", "​", "123", "456", "78", " \n\r\n", ".", "漢", "…ع", "…ß'Re", "漢'VE", "<|", "endoftext", "|>", "٣٤٥", "٦", "ß", "", "
", ".", "\u000b", "'Séd", "
漢"]} +{"text": "…s👍🏽e३'VE''reZ　 \n", "tokens": 14, "pieces": ["٣٤٥", "٦", "<|", "endoftext", "|>", "Z", "　 \n"]} +{"text": "m>'S12345678\r\nḍ̇m‍ḍ̇漢d9<|fim_prefix|>ds, ( EOT!'D9é, a
#$%𐞁d́🙂", "tokens": 52, "pieces": ["m", ">'", "S", "123", "456", "78", "\r\n", "ḍ̇m", "‍ḍ̇漢d", "9", "<|", "fim", "_prefix", "|>", "ds", ",", " (", " ", " EOT", "!<", "META", "_START", ">'", "D", "9", "é", ",", " a", "
", "#$%", "𐞁d́", "🙂"]} +{"text": "(", "tokens": 1, "pieces": ["("]} +{"text": ">​-<|fim_prefix|>㍿'Tt(<　'D🙂 Z\r\n\r\n
(s'VE.#$%Dž​'T㋿\t​
Dž'-", "tokens": 47, "pieces": [">​-<|", "fim", "_prefix", "|>㍿'", "Tt", "(<", "　", "'D", "🙂", " Z", "\r\n\r\n", "
", "(s'VE", ".#$%", "Dž", "​'", "T", "㋿", "\t", "​", "
Dž", "'-"]} +{"text": "'Ḿ9", "tokens": 3, "pieces": ["'Ḿ", "9"]} +{"text": "'s'S>", "tokens": 3, "pieces": ["'s'S", ">"]} +{"text": "‍㋿Ⅳ", "tokens": 6, "pieces": ["‍㋿", "Ⅳ"]} +{"text": "٣٤٥٦\r\n\r\n­å ­0​½½9字e \u000b", "tokens": 20, "pieces": ["٣٤٥", "٦", "\r\n\r\n", "­å", " ­", "0", "​", "½½9", "字e", " \u000b"]} +{"text": "ßⅣ> m'VEfi　ḍ̇", "tokens": 13, "pieces": ["ß", "Ⅳ", ">", " ", " m'VE", "fi", "　ḍ̇"]} +{"text": "'M<|fim_prefix|>ع\t<ꟲ'S><|endoftext|>‍'ſ\u000b́
EOTé­
 \n9", "tokens": 37, "pieces": ["'M", "<|", "fim", "_prefix", "|>", "ع", "\t", "<", "ꟲ'S", "><|", "endoftext", "|>‍'", "ſ", "\u000b́", "
EOTé", "­", "
 \n", "9"]} +{"text": "\rß\rZa\tm　👍🏽0　🙂漢EOT'll're''Ａ\u000b< \naa-'llå\r\nḍ̇t­<12345678'Mſ", "tokens": 40, "pieces": ["\r", "ß", "\r", "Za", "\tm", "　", "👍🏽", "0", "　", "🙂漢", "EOT'll", "'re", "''", "Ａ", "\u000b", "<", " \n", "aa", "-'", "llå", "\r\n", "ḍ̇t", "­<", "123", "456", "78", "'Mſ"]} +{"text": "　ßd٣٤٥٦EOT\r\n\r\n9-🙂'ſEOT\r\u000b<|fim_prefix|>'🙂 !!字mZꟲ<|endoftext|>!!٣٤٥٦'ſ!'ſ!!ſ", "tokens": 51, "pieces": ["　ßd", "٣٤٥", "٦", "EOT", "\r\n\r\n", "9", "-🙂'", "ſ", "EOT", "\r", "\u000b", "<|", "fim", "_prefix", "|>'🙂", " !!", "字m", "Zꟲ", "<|", "endoftext", "|>!!", "٣٤٥", "٦", "'ſ", "!'", "ſ", "!!", "ſ"]} +{"text": "0're<…,12345678­Džå'Tḍ̇'Re👍🏽\nDž漢𐞁", "tokens": 34, "pieces": ["0", "'re", "<", "…", ",", "123", "456", "78", "­", "Džå'T", "ḍ̇'Re", "👍🏽\n", "Dž漢𐞁"]} +{"text": "-'VE…<|fim_prefix|>'́́'0e‍s,''ll'ſ\t…s0👍🏽é", "tokens": 34, "pieces": ["-'", "VE", "…", "<|", "fim", "_prefix", "|>'́́'", "0", "e", "‍s", ",''", "ll'ſ", "\t", "…s", "0", "👍🏽", "é"]} +{"text": "ḍ̇ 　३'Tḍ̇漢 ſ's…'D… 　<\rعİétḍ̇å🙂0é\u000bع<|endoftext|>'T\u000b३", "tokens": 54, "pieces": ["ḍ̇", " <", "EOT", ">", "　", "३", "'Tḍ̇漢", " ſ's", "…", "'D", "… ", "　", "<\r", "عİétḍ̇å", "🙂", "0", "é", "\u000bع", "<|", "endoftext", "|>'", "T", "\u000b", "३", ""]} +{"text": "٣٤٥٦(é👍🏽e9 \n(😀🏽0\r'Sſ're漢é\r\né👍🏽,0 ſds's㍿9ḍ̇'re(🙂ſa　", "tokens": 47, "pieces": ["٣٤٥", "٦", "(é", "👍🏽", "e", "9", " \n", "(😀🏽", "0", "\r", "'Sſ're", "漢é", "\r\n", "é", "👍🏽,", "0", " ſds's", "㍿", "9", "ḍ̇'re", "(🙂", "ſa", "　"]} +{"text": "ßİ0​'ſ#$%​\"㋿́'Sꟲ<ع-٣٤٥٦", "tokens": 26, "pieces": ["ß", "İ", "0", "​'", "ſ", "#$%​\"㋿́'", "Sꟲ", "<ع", "-", "٣٤٥", "٦"]} +{"text": "😀🏽<|fim_prefix|>s!'M!!😀🏽\u000bea", "tokens": 18, "pieces": ["😀🏽<|", "fim", "_prefix", "|>", "s", "!'", "M", "!!😀🏽", "\u000bea"]} +{"text": ".½\r\n𐞁ſ㋿½👍🏽Dž''ll㋿'S's'ſ½0 mfi　Z,s'Redİ", "tokens": 38, "pieces": [".", "½", "\r\n", "𐞁ſ", "㋿", "½", "👍🏽", "Dž", "''", "ll", "㋿'", "S's", "'ſ", "½0", " mfi", "　Z", ",s'Re", "d", "İ"]} +{"text": "㍿", "tokens": 3, "pieces": ["㍿"]} +{"text": "é12345678fi 𐞁'ſ<|fim_prefix|>३!!İ\t0Z!d.<|fim_prefix|>🙂#$%>…!", "tokens": 43, "pieces": ["é", "123", "456", "78", "fi", " ", " 𐞁'ſ", "<|", "fim", "_prefix", "|><", "META", "_START", ">", "३", "!!", "İ", "\t", "0", "Z", "!d", ".<|", "fim", "_prefix", "|>🙂#$%>", "…", "!"]} +{"text": "#$%㍿'Mm
,!!0'S'ſ'VE'D", "tokens": 22, "pieces": ["#$%㍿'", "M", "m", "
", ",!!", "0", "'S'ſ", "'VE'D"]} +{"text": "<|endoftext|>'S字m'VEt́9#$%''ret\r'll­", "tokens": 22, "pieces": ["<|", "endoftext", "|>'", "S字m'VE", "t́", "9", "#$%''", "ret", "\r", "'ll", "­"]} +{"text": "mⅣé", "tokens": 4, "pieces": ["m", "Ⅳ", "é"]} +{"text": "DžEOT ('Re'll½🙂㋿ⅣＡ́(👍🏽>́", "tokens": 29, "pieces": ["DžEOT", " ", " (<", "EOT", ">'", "Re'll", "½", "🙂㋿", "Ⅳ", "Ａ́", "(👍🏽><", "META", "_START", ">́"]} +{"text": "३ Ａ(­'D　s　Dž🙂Dž \n㍿ 'ſ　\t🙂Ａ ३#$%Dž0'T'0\u000b", "tokens": 38, "pieces": [" ", "😀🏽", "Ａ", "<|", "endoftext", "|>", " \n", "㍿", " ", "'ſ", "　", "\t", "🙂Ａ", " ", "३", "#$%", "Dž", "", "0", "'T", "'", "0", "\u000b"]} +{"text": "Ａ<|endoftext|>fiß9ꟲ<|endoftext|>e ½ >㋿İ😀🏽' ㍿😀🏽́\u000b३\n\u000b9d'M'Tße", "tokens": 53, "pieces": ["Ａ", "<|", "endoftext", "|>", "fiß", "9", "ꟲ", "<|", "endoftext", "|>", "e", " ", "½", " ", ">㋿", "İ", "😀🏽'", " ", "㍿😀🏽́", "\u000b", "३", "\n", "\u000b", "9", "d'M", "'Tße"]} +{"text": "…½é\r\ne\t9 #$%'ll(s漢'ſ<|endoftext|>Ａ<|fim_prefix|>'S<|endoftext|>'D>EOT㋿٣٤٥٦🙂<½'M½té>'T", "tokens": 59, "pieces": ["…", "½", "é", "\r\n", "e", "\t", "9", " ", " #$%'", "ll", "(s漢'ſ", "<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>'", "S", "<|", "endoftext", "|>'", "D", ">EOT", "㋿", "٣٤٥", "٦", "🙂<", "½", "'M", "½", "té", ">'", "T"]} +{"text": "　Zma<‍és'VE'sm'S \n👍🏽ꟲ9😀🏽", "tokens": 25, "pieces": ["　Zma", "<‍", "és'VE", "'sm'S", " \n", "👍🏽", "ꟲ", "9", "😀🏽"]} +{"text": "ß'VE<|fim_prefix|>d\t!!ḍ̇å\ńſ\"́é​　'D'VE㋿!'sé😀🏽EOT 'lld😀🏽́'s", "tokens": 52, "pieces": ["ß'VE", "<|", "fim", "_prefix", "|>", "d", "\t", "!!", "ḍ̇å", "\n", "́ſ", "\"́é", "​", "　", "'D'VE", "㋿!'", "sé", "😀🏽", "EOT", " ", "'lld", "😀🏽́'", "s", ""]} +{"text": "é<|endoftext|>EOT́<|fim_prefix|> \n( fiß-㋿'ſ<|fim_prefix|>,'s \n#$%-
\r", "tokens": 42, "pieces": ["é", "<|", "endoftext", "|>", "EOT́", "<|", "fim", "_prefix", "|>", " \n", "(", " fiß", "-㋿'", "ſ", "<|", "fim", "_prefix", "|>,'", "s", " \n", "#$%-", "
\r"]} +{"text": "
é٣٤٥٦'D \n'll\u000bå'MEOT 'll'ſⅣ'\rfifid😀🏽­…字😀🏽३'ll'Dd,ḍ̇", "tokens": 46, "pieces": ["
é", "٣٤٥", "٦", "'D", " \n", "'ll", "\u000bå'M", "EOT", " ", " '", "ll'ſ", "Ⅳ", "'\r", "fifid", "😀🏽­", "…字", "😀🏽", "३", "'ll'D", "d", ",ḍ̇"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿½<ſ'll \n(½'ll字'Re!!!!<…ꟲ½a\tＡ'VE'lltⅣDžm'Re ", "tokens": 35, "pieces": ["㍿", "½", "<ſ'll", " \n", "(", "½", "'ll字'Re", "!!!!<", "…ꟲ", "½", "a", "\tＡ'VE", "'llt", "Ⅳ", "Džm'Re", " "]} +{"text": "'s…\u000b\r-'s'll's'ſ​\r\n\r\n'Sm\u000b½ !\ré9😀🏽s　ſ", "tokens": 30, "pieces": ["'s", "…\u000b\r", "-'", "s'll", "'s'ſ", "​\r\n\r\n", "'Sm", "\u000b", "½", " !\r", "é", "9", "😀🏽<", "EOT", ">s", "　ſ"]} +{"text": "😀🏽ſ'ſİ
字ém'Re \n​<́३ 'T", "tokens": 24, "pieces": ["😀🏽", "ſ'ſ", "İ", "
字", "ém'Re", " \n", "​<́", "३", " ", "'T"]} +{"text": "'VE<#$%'re<|fim_prefix|>!fié٣٤٥٦'s>fis\"<|fim_prefix|>　३'ll\r\n12345678𐞁>,", "tokens": 42, "pieces": ["'VE", "<#$%'", "re", "<|", "fim", "_prefix", "|>!", "fié", "٣٤٥", "٦", "'s", ">fis", "\"<|", "fim", "_prefix", "|>", "　", "३", "'ll", "\r\n", "123", "456", "78", "𐞁", ">,"]} +{"text": " 
a éꟲ👍🏽
ꟲ'ſ 'VEtZ🙂½漢", "tokens": 30, "pieces": [" ", "
a", " éꟲ", "👍🏽", "
ꟲ'ſ", " '", "VEt", "<", "EOT", ">Z", "🙂", "½", "漢"]} +{"text": "Ⅳİ''ll\t…å😀🏽 \n\u000b!عDž", "tokens": 19, "pieces": ["Ⅳ", "İ", "''", "ll", "\t", "…å", "😀🏽", " \n", "\u000b", "!ع", "Dž"]} +{"text": "\r\nfi 👍🏽!!,😀🏽İ字EOT\r\n\r\nⅣⅣ.ḍ̇…🙂Ⅳ
(('Ss t́Ⅳ \nZ\t 's\r\n\r\n!!", "tokens": 44, "pieces": ["\r\n", "fi", " ", "👍🏽!!,😀🏽", "İ字", "EOT", "\r\n\r\n", "ⅣⅣ", ".ḍ̇", "…", "🙂", "Ⅳ", "
", "(('", "Ss", " ", " t́", "Ⅳ", " \n", "Z", "\t ", " '", "s", "\r\n\r\n", "!!"]} +{"text": "é", "tokens": 1, "pieces": ["é"]} +{"text": "­-👍🏽Ａ😀🏽s\t.12345678½ \r\nḍ̇''Re​‍.
sḍ̇a", "tokens": 30, "pieces": ["­-👍🏽", "Ａ", "😀🏽", "s", "\t", ".", "123", "456", "78½", " \r\n", "ḍ̇", "''", "Re", "​‍.", "
sḍ̇a"]} +{"text": "Ａ‍👍🏽\t​ås字 ع'Ds", "tokens": 14, "pieces": ["Ａ", "‍👍🏽", "\t", "​ås字", " ع'D", "s"]} +{"text": "9", "tokens": 1, "pieces": ["9"]} +{"text": "Dž", "tokens": 2, "pieces": ["Dž"]} +{"text": "́,İt'VE!!å'ſⅣ🙂'ſ's9\r\nſ𐞁ḍ̇　<|fim_prefix|>Ź٣٤٥٦'Dİfiꟲß,s0a 字", "tokens": 55, "pieces": ["́", ",İt'VE", "!!", "å'ſ", "Ⅳ", "🙂'", "ſ's", "9", "\r\n", "ſ", "𐞁ḍ̇", "　", "<|", "fim", "_prefix", "|>", "Ź", "٣٤٥", "٦", "'Dİfiꟲß", ",s", "0", "a", " ", " 字"]} +{"text": "'S<|fim_prefix|><<|fim_prefix|>…-\r\n\"é\rZ'llع(fi", "tokens": 28, "pieces": ["'S", "<|", "fim", "_prefix", "|><<|", "fim", "_prefix", "|><", "META", "_START", ">", "…", "-\r\n", "\"é", "\r", "Z'll", "ع", "(fi"]} +{"text": "s'½ \nééſ'll👍🏽'ſ😀🏽fi\r\n9é‍'VE\r…　𐞁\r'Dİſ\"é 字İé'M'DＡ🙂,ſ", "tokens": 52, "pieces": ["s", "'", "½", " \n", "ééſ'll", "👍🏽'", "ſ", "😀🏽", "fi", "\r\n", "9", "é", "‍'", "VE", "\r", "…", "", "　𐞁", "\r", "'Dİſ", "\"é", " ", " 字İé'M", "'DＡ", "🙂,", "ſ"]} +{"text": "'re fi e'D('D", "tokens": 8, "pieces": ["'re", " ", " fi", " e'D", "('", "D"]} +{"text": "9're字­'D字'🙂12345678å>🙂's३ſ \n", "tokens": 21, "pieces": ["9", "'re字", "­'", "D字", "'🙂", "123", "456", "78", "å", ">🙂'", "s", "३", "ſ", " \n"]} +{"text": "12345678😀🏽!字\r\n'Tḍ̇٣٤٥٦­½'T'VEⅣ\"", "tokens": 29, "pieces": ["123", "456", "78", "😀🏽!", "字", "\r\n", "'Tḍ̇", "٣٤٥", "٦", "­", "½", "'T'VE", "", "Ⅳ", "\""]} +{"text": ".👍🏽\r\n\r\nd­‍<|endoftext|>😀🏽　Ⅳ<|endoftext|>12345678m\r\n\r\n'S漢ꟲ", "tokens": 38, "pieces": [".👍🏽\r\n\r\n", "d", "­‍<|", "endoftext", "|>😀🏽", "　", "Ⅳ", "<|", "endoftext", "|>", "123", "456", "78", "m", "\r\n\r\n", "'S漢ꟲ"]} +{"text": ".>#$%Z12345678…㍿", "tokens": 13, "pieces": [".>#$%", "Z", "123", "456", "78", "…", "㍿"]} +{"text": "#$% \n'Re \n<́(ꟲ㍿​fiéfi\t​Ⅳ'S ́ßemd\t0\n", "tokens": 33, "pieces": ["#$%", " \n", "'Re", " \n", "<́", "(<", "EOT", ">ꟲ", "㍿​", "fiéfi", "\t", "​", "Ⅳ", "'S", " ́ßemd", "\t", "0", "\n"]} +{"text": "(…'lleá#$%EOT!! ​9ß­'res\"ⅣⅣ-'M३e#$%'s12345678'll<|fim_prefix|>…Džfi‍åZ<|endoftext|>", "tokens": 60, "pieces": ["(", "…", "'lleá", "#$%", "EOT", "!!", " ", "​", "9", "ß", "­'", "res", "\"", "ⅣⅣ", "-'", "M", "३", "e", "#$%'", "s", "123", "456", "78", "'ll", "<|", "fim", "_prefix", "|>", "…Džfi", "‍å", "Z", "<|", "endoftext", "|>"]} +{"text": "
‍३'re0\r\nDž­🙂a👍🏽­㍿-ḍ̇Dž", "tokens": 28, "pieces": ["
", "‍", "३", "'re", "0", "\r\n", "Dž", "­🙂", "a", "👍🏽­<", "EOT", ">㍿-", "ḍ̇", "Dž"]} +{"text": "\té'ſ \nḍ̇aꟲ''VE'VEé 漢…' Ａ'Re's.", "tokens": 32, "pieces": ["\té'ſ", " \n", "ḍ̇aꟲ", "''", "VE'VE", "é", " ", " 漢", "…", "'", " Ａ'Re", "'s", ".<", "META", "_START", ">"]} +{"text": "12345678'stⅣع.… ,EOT字İ漢㋿ !!'M🙂㋿
", "tokens": 32, "pieces": ["123", "456", "78", "'st", "Ⅳ", "ع", ".", "…", " ", ",EOT字İ漢", "㋿", " ", "!!<", "META", "_START", ">'", "M", "🙂㋿", "
"]} +{"text": " \n.Ａİ,½👍🏽're\t'VE𐞁عéDžſ(#$%'reſ ㋿😀🏽ع!å𐞁‍s👍🏽å­Z'", "tokens": 58, "pieces": [" \n", ".Ａ", "İ", ",", "½", "👍🏽'", "re", "\t", "'VE𐞁عé", "Džſ", "(#$%'", "reſ", " ", " ㋿😀🏽", "ع", "!å𐞁", "‍s", "👍🏽", "å", "­Z", "'"]} +{"text": "ḍ̇ꟲms12345678e'S9٣٤٥٦字ḍ̇\"ع㍿İ'M'llſ\r\n 🙂a<|endoftext|>fi漢!३<|fim_prefix|>　\u000b٣٤٥٦>", "tokens": 62, "pieces": ["ḍ̇ꟲms", "123", "456", "78", "e'S", "9", "", "٣٤٥", "٦", "字ḍ̇", "\"ع", "㍿İ'M", "'llſ", "\r\n", " ", " 🙂", "a", "<|", "endoftext", "|>", "fi漢", "!", "३", "<|", "fim", "_prefix", "|>", "　", "\u000b", "٣٤٥", "٦", ">"]} +{"text": "ßåع0ع12345678½㋿　Džꟲtſ-sꟲ#$%'ſ'ſt''D sDž\"a‍Ⅳ字12345678 \n", "tokens": 49, "pieces": ["ßå", "ع", "0", "ع", "123", "456", "78½", "㋿", "　Džꟲtſ", "-sꟲ", "#$%'", "ſ'ſ", "t", "''", "D", " ", " s", "Dž", "\"a", "‍", "Ⅳ", "字", "123", "456", "78", " \n"]} +{"text": "m<|endoftext|>Ⅳ\r\n\r\n!\r\n\r\n'漢  'S'Ses'D're𐞁'VE.aDžm(", "tokens": 36, "pieces": ["m", "<|", "endoftext", "|>", "Ⅳ", "\r\n\r\n", "!\r\n\r\n", "'漢", " ", " ", "'S'S", "es'D", "'re𐞁'VE", ".a", "Džm", "("]} +{"text": "\téa\t0t漢ꟲEOT\u000bEOT́e­-\u000b​", "tokens": 24, "pieces": ["\téa", "\t", "0", "t漢ꟲ", "EOT", "\u000bEOT́e", "­<", "EOT", ">-", "\u000b", "​"]} +{"text": "<|endoftext|>½EOTfiḍ̇​½́
\"ḍ̇#$%Dž​Ａꟲ\"<", "tokens": 32, "pieces": ["<|", "endoftext", "|>", "½", "EOTfiḍ̇", "​", "½", "́", "
", "\"ḍ̇", "#$%", "Dž", "​Ａꟲ", "\"<"]} +{"text": "٣٤٥٦0(😀🏽'lld0\r.<İ'Re'Re#$%<|endoftext|>#$%<|endoftext|>ß­'Re>sa", "tokens": 44, "pieces": ["٣٤٥", "٦0", "(😀🏽'", "lld", "0", "\r", ".<", "EOT", "><", "İ'Re", "'Re", "#$%<|", "endoftext", "|>#$%<|", "endoftext", "|>", "ß", "­'", "Re", ">sa"]} +{"text": "😀🏽12345678㋿ḍ̇.𐞁\r\n<|endoftext|>,ß -😀🏽é.'S9\r\n\r\n　!!漢", "tokens": 38, "pieces": ["😀🏽", "123", "456", "78", "㋿ḍ̇", ".𐞁", "\r\n", "<|", "endoftext", "|>,", "ß", " -😀🏽", "é", ".'", "S", "9", "\r\n\r\n", "　", "!!", "漢"]} +{"text": "👍🏽\u000b ſ>‍!!字٣٤٥٦\"ḍ̇'३ß9é३\"e\nß0
!Ａd  Ａ9\n", "tokens": 40, "pieces": ["👍🏽", "\u000b ", " ſ", ">‍!!", "字", "٣٤٥", "٦", "\"ḍ̇", "'", "३", "ß", "9", "é", "३", "\"e", "\n", "ß", "0", "
", "!Ａd", " ", " Ａ", "9", "\n"]} +{"text": "t12345678 'ſfiḍ̇ſfiⅣ㍿>", "tokens": 18, "pieces": ["t", "123", "456", "78", " '", "ſfiḍ̇ſfi", "Ⅳ", "㍿>"]} +{"text": "('Mꟲ字'Dß-<|endoftext|>٣٤٥٦\u000bd\r\nſ😀🏽\n…aémＡ…㍿½👍🏽३s
 ", "tokens": 63, "pieces": [" ", " <'", "s", "Z", "　", "‍", "½", "e'ſ", "e", "
e", "9", " \n", "<|", "fim", "_prefix", "|>'", "Dß", "-<|", "endoftext", "|>", "٣٤٥", "٦", "\u000bd", "\r\n", "ſ", "😀🏽\n", "…aém", "Ａ", "…", "㍿", "½", "👍🏽", "३", "s", "
 "]} +{"text": " ſ>!'Re㋿'M's'T,İ!ع9-(­å㍿ع-'ll𐞁'Re​d😀🏽ꟲ !!😀🏽", "tokens": 49, "pieces": [" ſ", ">!'", "Re", "㋿'", "M's", "'T", ",İ", "!ع", "9", "-(­", "å", "㍿ع", "-'", "ll", "𐞁'Re", "​d", "😀🏽", "ꟲ", " ", "!!😀🏽"]} +{"text": "३- \n٣٤٥٦é ſ!!'ll'ſ \n>'å­'ll½'M𐞁́Dž​<|endoftext|>!fißſa12345678<|endoftext|>́9'", "å", "­'", "ll", "½", "'M𐞁́", "Dž", "​<|", "endoftext", "|>!", "fißſa", "123", "456", "78", "<|", "endoftext", "|>́", "9", "字<|endoftext|>(0​\rＡ　0İ㍿'D㋿ḍ̇㋿\"Z", "tokens": 44, "pieces": [" \n", "9", "👍🏽", "é're", "字", "<|", "endoftext", "|>(", "0", "​\r", "Ａ", "　", "0", "İ", "㍿'", "D", "㋿ḍ̇", "㋿\"", "Z"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'😀🏽ع!𐞁　s\"😀🏽字٣٤٥٦٣٤٥٦İ,٣٤٥٦ꟲ#$%,\r\n\r\n字a", "tokens": 39, "pieces": ["'😀🏽", "ع", "!𐞁", "　s", "\"😀🏽", "字", "٣٤٥", "٦٣٤", "٥٦", "İ", ",", "٣٤٥", "٦", "ꟲ", "#$%,\r\n\r\n", "字a"]} +{"text": "\rDž'ſ're12345678‍'Re'T\"\r'M
ع\u000b\tt", "tokens": 24, "pieces": ["Z漢", "-'", "VEDž", " ", " 🙂,", "9", "字", "9", "…", "\"\r", "'M", "
ع", "\u000b", "\tt"]} +{"text": "<ḍ̇漢ꟲ'T<('Reꟲع\r\n\r\nDžⅣ'll㍿!!Dž \n'T'Dعfi<|fim_prefix|>漢é­m'ſ㋿ \n㋿0\u000b'refi", "tokens": 63, "pieces": ["<ḍ̇漢", "ꟲ'T", "<('", "Reꟲع", "\r\n\r\n", "Dž", "Ⅳ", "'ll", "㍿!!", "Dž", " \n", "'T'D", "عfi", "<|", "fim", "_prefix", "|>", "漢é", "­m'ſ", "㋿", " \n", "㋿", "0", "\u000b", "'", "refi"]} +{"text": "0'Tt漢漢 <|endoftext|><|fim_prefix|><|fim_prefix|>'T\"m'S½\u000bt'ſḍ̇>ßſ'Tt>s éZ 12345678'Ma­'Re😀🏽", "tokens": 56, "pieces": ["0", "'Tt漢漢", " ", "<|", "endoftext", "|><|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "T", "\"m'S", "½", "\u000bt'ſ", "ḍ̇", ">ßſ'T", "t", ">s", " é", "Z", " ", "123", "456", "78", "'Ma", "­'", "Re", "😀🏽"]} +{"text": "'VE'll\r\n\r\n's'M're­'ss'D३ß㍿", "tokens": 16, "pieces": ["'VE'll", "\r\n\r\n", "'s'M", "'re", "­'", "ss'D", "३", "ß", "㍿"]} +{"text": "​㍿㍿ 字३३Dž!12345678s'VEt漢­EOT", "tokens": 25, "pieces": ["​㍿㍿", " ", " 字", "३३", "Dž", "!", "123", "456", "78", "s'VE", "t漢", "­EOT"]} +{"text": "'S‍s😀🏽'Re ㋿émfi0", "tokens": 18, "pieces": ["'S", "‍s", "😀🏽'", "Re", " ", " ㋿", "émfi", "0"]} +{"text": "#$%dé'ſ́Z'MEOT.'VE>", "tokens": 14, "pieces": ["#$%", "dé'ſ", "́", "Z'M", "EOT", ".'", "VE", ">"]} +{"text": "t>Dž'Re aEOT­\r\n\r\n字9EOT-!(\u000b'Mm-", "tokens": 23, "pieces": ["t", ">Dž'Re", " a", "EOT", "­\r\n\r\n", "字", "9", "EOT", "-!(", "\u000b", "'Mm", "-"]} +{"text": "字㍿　!('T \r9'll😀🏽'D EOT‍<|endoftext|>(Dž-'D​\r\nEOT", "tokens": 43, "pieces": ["字", "㍿", "　", "!('", "T", " \r", "9", "'ll", "😀🏽'", "D", "", " EOT", "‍<|", "endoftext", "|><", "EOT", ">(", "Dž", "-'", "D", "​\r\n", "EOT"]} +{"text": ">漢㋿d t😀🏽‍Dž \n'TsdEOT\"<|endoftext|>ع🙂\r👍🏽(ß9's,", "tokens": 36, "pieces": [">漢", "㋿d", " t", "😀🏽‍", "Dž", " \n", "'Tsd", "EOT", "\"<|", "endoftext", "|>", "ع", "🙂\r", "👍🏽(", "ß", "9", "'s", ","]} +{"text": "㋿\u000b‍EOTåt字\r\né\r #$%ꟲ>㋿é'S 'm.𐞁", "tokens": 35, "pieces": ["㋿", "\u000b", "‍EOTåt字", "\r\n", "é", "\r", " ", " #$%", "ꟲ", ">㋿", "é'S", " '", "m", ".𐞁"]} +{"text": "\r\n 'ſḍ̇​'Re,", "tokens": 13, "pieces": ["\r\n", " <", "META", "_START", ">'", "ſḍ̇", "​'", "Re", ","]} +{"text": "'ſ!
\"'S>'sſ'Re.
　½<|fim_prefix|> \nd'<'ſé'll", "tokens": 31, "pieces": ["'ſ", "!", "
", "\"'", "S", ">'", "s", "ſ'Re", ".", "
", "　", "½", "<|", "fim", "_prefix", "|>", " \n", "d", "'<'", "ſé'll"]} +{"text": "(Ź\n<|fim_prefix|>́'VEＡ<|fim_prefix|>", "tokens": 19, "pieces": ["(Ź", "\n", "<|", "fim", "_prefix", "|>́'", "VEＡ", "<|", "fim", "_prefix", "|>"]} +{"text": "😀🏽'VEfiḍ̇\r\n\r\nßéſ#$%Ａm!'re½ 'Re­३\nⅣ (>'reعe\r漢", "tokens": 37, "pieces": ["😀🏽'", "VEfiḍ̇", "\r\n\r\n", "ßéſ", "#$%", "Ａm", "!'", "re", "½", " ", "'Re", "­", "३", "\n", "Ⅳ", " (>'", "reعe", "\r", "漢"]} +{"text": "'s㍿m'VE>ßß'ſ (Ⅳ‍'Dßß", "'", "ſ", " ", "(", "Ⅳ", "‍'", "D", "字", "tokens": 37, "pieces": ["३", "𐞁", "-", "\t", "…", "㍿Dž字", "!!👍🏽", "ꟲ'VE", "é", "#$%\r\n", "<'", "s", "<|", "endoftext", "|>", "字"]} +{"text": "㋿9e,🙂å\r\n­'M>.eꟲ.'st'ſ३0Z'M.", "tokens": 32, "pieces": ["㋿", "9", "e", ",🙂", "å", "\r\n", "­'", "M", ">.", "eꟲ", ".'", "st'ſ", "३", "", "0", "Z'M", "."]} +{"text": " \n'Dž.'ssé
9́(#$%'M'S㍿Ⅳ漢éé's \n­\r\u000b'M😀🏽", "tokens": 44, "pieces": [" \n", "'Dž", ".'", "ssé", "
", "9", "́", "(#$%'", "M'S", "㍿", "Ⅳ", "漢éé's", " \n", "­\r", "", "\u000b", "'", "M", "😀🏽"]} +{"text": "ḍ̇é漢", "tokens": 5, "pieces": ["ḍ̇é漢"]} +{"text": "Z12345678\u000b('D,é>'<|endoftext|>'\r\n漢'sm<|endoftext|>Zt", "tokens": 33, "pieces": ["Z", "123", "456", "78", "\u000b", "('", "D", ",é", ">'<|", "endoftext", "|>'\r\n", "漢's", "m", "<|", "endoftext", "|>", "Zt"]} +{"text": "!!'re'VE'll're　ع'D　\u000b9!!", "tokens": 14, "pieces": ["!!'", "re'VE", "'ll're", "　ع'D", "　", "\u000b", "9", "!!"]} +{"text": "
å å‍>👍🏽\r\n<|endoftext|>ݽe­ꟲDž-ḍ̇!!ꟲfi<|fim_prefix|>\t'VEs <é\r\n\r\n'll٣٤٥٦", "tokens": 55, "pieces": ["
å", " ", " å", "‍>👍🏽\r\n", "<|", "endoftext", "|>", "İ", "½", "e", "­ꟲ", "Dž", "-ḍ̇", "!!", "ꟲfi", "<|", "fim", "_prefix", "|>", "\t", "'VEs", " <", "é", "\r\n\r\n", "'ll", "٣٤٥", "٦"]} +{"text": "\r\n'VE>dd́字'Re'lle's\né  t३ 漢'ſ!!#$%漢३\n\"
'👍🏽😀🏽EOT", "tokens": 38, "pieces": ["\r\n", "'VE", ">dd́字'Re", "'lle's", "\n", "é", "  ", " t", "३", " 漢'ſ", "!!#$%", "漢", "३", "\n", "\"", "
", "'👍🏽😀🏽", "EOT"]} +{"text": "etd12345678 ", "tokens": 9, "pieces": ["etd", "123", "456", "78", "", " "]} +{"text": "\n½'reİİ\"\n 'M>'ſ's\u000b'VE㋿d　🙂a\nEOTé​", "tokens": 38, "pieces": ["\n", "½", "'re", "İİ", "\"\n", " '", "M", ">'", "ſ's", "\u000b", "'VE", "㋿d", "", "　", "🙂a", "\n", "EOTé", "​"]} +{"text": ".\r\n\r\n٣٤٥٦", "tokens": 5, "pieces": [".\r\n\r\n", "٣٤٥", "٦"]} +{"text": "å字( å .s-　m३9( ́!​<'Tm", "tokens": 38, "pieces": ["å字", "(", " å", " ", " .", "s", "-", "　m", "३9", "(<", "EOT", ">", " ́", "!​<<", "META", "_START", "><", "EOT", ">'", "Tm"]} +{"text": "#$%ع\"\n", "tokens": 4, "pieces": ["#$%", "ع", "\"\n"]} +{"text": " \nſꟲ-👍🏽(mm#$%!!(\t
'D\" 
'Ré#$%>", "tokens": 37, "pieces": [" \n", "ſ", "ꟲ", "-👍🏽(", "mm", "#$%!!(", "\t", "
", "'D", "\"", " ", " <", "EOT", ">", "
", "'Re", "́", "#$%>"]} +{"text": "!! \nßåå", "tokens": 7, "pieces": ["!!", " \n", "ßåå"]} +{"text": "ꟲ'VE­́Ⅳte!-'M#$%", "tokens": 15, "pieces": ["ꟲ'VE", "­́", "Ⅳ", "te", "!-'", "M", "#$%"]} +{"text": "​㋿0\r'M漢é́Dždåe s \nå😀🏽'Resd \n३>'ll\u000b 'refi'lla'VE½\n", "tokens": 40, "pieces": ["​㋿", "0", "\r", "'M漢é́", "Dždåe", " s", " \n", "å", "😀🏽'", "Resd", " \n", "३", ">'", "ll", "\u000b ", " '", "refi'll", "a'VE", "½", "\n"]} +{"text": "e'Re9٣٤٥٦​٣٤٥٦'T", "tokens": 13, "pieces": ["e'Re", "9٣٤", "٥٦", "​", "٣٤٥", "٦", "'T"]} +{"text": "🙂m", "tokens": 2, "pieces": ["🙂m"]} +{"text": " 
́'ll字İ", "tokens": 6, "pieces": [" ", "
́'ll", "字", "İ"]} +{"text": "å'Mꟲ'S\t٣٤٥٦३…e­　­å३é<|endoftext|>　　\r\t.", "tokens": 35, "pieces": ["å'M", "ꟲ'S", "\t", "٣٤٥", "٦३", "…e", "­", "　", "­å", "३", "é", "<|", "endoftext", "|>", "　　\r", "\t", "."]} +{"text": "٣٤٥٦t𐞁 \n\r\n!éDž>'T'S.'s0'ſ ", "tokens": 24, "pieces": ["٣٤٥", "٦", "t𐞁", " \n\r\n", "!é", "Dž", ">'", "T'S", ".'", "s", "0", "'ſ", " "]} +{"text": "٣٤٥٦'reſ \n\r\n<Ⅳ　\r३0
s½'VE \n#$%٣٤٥٦\u000b𐞁", "tokens": 41, "pieces": ["٣٤٥", "٦", "'reſ", " \n\r\n", "<", "Ⅳ", "　", "\r", "३0", "
s", "½", "'VE", " \n", "#$%", "٣٤٥", "٦", "\u000b", "𐞁"]} +{"text": "'　ḍ̇…d'é<|endoftext|>EOT \u000b㋿'Re'll­ \n😀🏽½𐞁<|endoftext|>#$%é9'VE9's(İ", "tokens": 55, "pieces": ["'", "　ḍ̇", "…d", "'é", "<|", "endoftext", "|>", "EOT", " ", "\u000b", "㋿'", "Re'll", "­", " \n", "😀🏽", "½", "𐞁", "<|", "endoftext", "|>#$%", "é", "9", "'", "VE", "9", "'s", "(İ"]} +{"text": "​'漢00
…㍿!😀🏽‍-'𐞁 😀🏽\r\n\r\n👍🏽Ａ漢字<<𐞁m\r\n", "tokens": 42, "pieces": ["​'", "漢", "00", "
", "", "…", "㍿!😀🏽‍-'", "𐞁", " ", " 😀🏽\r\n\r\n", "👍🏽", "Ａ漢字", "<<", "𐞁m", "\r\n"]} +{"text": "fi‍(", "tokens": 3, "pieces": ["fi", "‍("]} +{"text": "­!ꟲ字é㍿m　 ㍿éＡⅣEOTfi　'ſ𐞁a \n‍ſ é\r\n\r\n\"å'S", "tokens": 48, "pieces": ["­!", "ꟲ字é", "㍿m", "　", " ", "㍿é", "Ａ", "Ⅳ", "EOTfi", "　", "'ſ𐞁a", " \n", "‍<", "EOT", ">ſ", " é", "\r\n\r\n", "\"å'S"]} +{"text": "Z \n!!ß(<|endoftext|>EOT३　ḍ̇>åEOTß!字ꟲ½字.㍿!!é 'VE\t", "tokens": 43, "pieces": ["Z", " \n", "!!", "ß", "(<|", "endoftext", "|>", "EOT", "३", "　", "ḍ̇", ">å", "EOTß", "!字ꟲ", "½", "字", ".㍿!!", "é", " '", "VE", "\t"]} +{"text": "'s\t#$%‍‍  \n　Dž ٣٤٥٦ 0.fiꟲ<\u000b'D…'re12345678ꟲ…㍿ع'T<|fim_prefix|>\tå­'T'D>!!'ſm", "tokens": 59, "pieces": ["'s", "\t", "#$%‍‍", "  \n", "　Dž", " ", "٣٤٥", "٦", " ", "0", ".fiꟲ", "<", "\u000b", "'D", "…", "'re", "123", "456", "78", "ꟲ", "…", "㍿ع'T", "<|", "fim", "_prefix", "|>", "\tå", "­'", "T'D", ">!!'", "ſm"]} +{"text": "　'VEع'reeⅣ \n\r\n٣٤٥٦३ ‍ EOT-.　😀🏽'sa", "tokens": 29, "pieces": ["　", "'VEع're", "e", "Ⅳ", " \n", "\r\n", "٣٤٥", "٦३", " ‍", " ", " EOT", "-.", "　", "😀🏽'", "sa"]} +{"text": "㍿EOTſ<12345678𐞁\tßa .​t漢<|fim_prefix|><ع😀🏽\u000b\n
　字Z0Dž<|fim_prefix|>", "tokens": 47, "pieces": ["㍿EOTſ", "<", "123", "456", "78", "𐞁", "\tßa", " ", " .​", "t漢", "<|", "fim", "_prefix", "|><", "ع", "😀🏽", "\u000b\n", "
", "　字", "Z", "0", "Dž", "<|", "fim", "_prefix", "|>"]} +{"text": "\"'T'D'Re ́'sİ<|fim_prefix|>\r३\r\nꟲaßſ½́½'ll", "tokens": 31, "pieces": ["\"'", "T'D", "'Re", " ", " <", "META", "_START", ">́'s", "İ", "<|", "fim", "_prefix", "|>\r", "३", "\r\n", "ꟲaßſ", "½", "́", "½", "'ll"]} +{"text": "<'s\nİ३#$%İd ‍ſ字㋿㍿ſ-字> 😀🏽́ß\na", "tokens": 30, "pieces": ["<'", "s", "\n", "İ", "३", "#$%", "İd", " ‍", "ſ字", "㋿㍿", "ſ", "-字", ">", " ", " 😀🏽́", "ß", "\n", "a"]} +{"text": "字 ", "tokens": 2, "pieces": ["字", " "]} +{"text": "-'lltİ12345678ꟲd½İ0ßḍ̇t-< 👍🏽", "tokens": 28, "pieces": ["-'", "llt", "İ", "123", "456", "78", "ꟲd", "½", "İ", "0", "ß", "ḍ̇t", "-<", " ", " 👍🏽"]} +{"text": " ㋿\r<|fim_prefix|> 0é!!.㋿'ſḍ̇é'll 'T#$%\t\r\n\r\n12345678fi\"\"Džé'Reع<", "tokens": 44, "pieces": [" ", "㋿\r", "<|", "fim", "_prefix", "|>", " ", "0", "é", "!!.㋿'", "ſḍ̇é'll", " ", " '", "T", "#$%", "\t\r\n\r\n", "123", "456", "78", "fi", "\"\"", "Džé'Re", "ع", "<"]} +{"text": "'llå,\r­ \u000b㍿\r\n\r\n\r'👍🏽\r\n\r\né", "tokens": 20, "pieces": ["'llå", ",\r", "­", " ", "\u000b", "㍿\r\n\r\n\r", "'👍🏽\r\n\r\n", "é"]} +{"text": "'ſ", "tokens": 2, "pieces": ["'ſ"]} +{"text": " 're'éé㍿ḍ̇DžZ(m😀🏽
عé\rⅣ'M㋿ḍ̇İé½\u000b", "tokens": 37, "pieces": [" '", "re", "'éé", "㍿ḍ̇", "DžZ", "(m", "😀🏽", "
عé", "\r", "Ⅳ", "'M", "㋿ḍ̇", "İé", "½", "\u000b"]} +{"text": "'ſⅣt𐞁­\n👍🏽🙂a🙂'M<|fim_prefix|>字Dž𐞁ع́㋿", "Ⅳ", "t𐞁", "­\n", "👍🏽🙂", "a", "🙂'", "M", "<|", "fim", "_prefix", "|>", "字Dž𐞁ع́", "㋿<", "é漢", "…", ",e", "(𐞁"]} +{"text": "​.'ś‍­.'ll9m٣٤٥٦ßⅣ'ſḍ̇'Re\tmß>𐞁#$%Ⅳ(\t !ꟲＡZ 0d0", "tokens": 46, "pieces": ["​.'", "ś", "‍­.'", "ll", "9", "m", "٣٤٥", "٦", "ß", "Ⅳ", "'ſḍ̇'Re", "\tmß", ">𐞁", "#$%", "Ⅳ", "(", "\t ", " !", "ꟲ", "ＡZ", " ", "0", "d", "0"]} +{"text": "́!'DZ Dž<|fim_prefix|> \n'Re", "tokens": 14, "pieces": ["́", "!'", "DZ", " Dž", "<|", "fim", "_prefix", "|>", " \n", "'Re"]} +{"text": "Ⅳ\r🙂'M9-́ \n!!İ­\t!字'D<|fim_prefix|>ꟲe'VE \n's😀🏽mعꟲet　👍🏽'D字🙂'Re", "tokens": 53, "pieces": ["Ⅳ", "\r", "🙂'", "M", "9", "-́", " \n", "!!", "İ", "­", "\t", "!字'D", "<|", "fim", "_prefix", "|>", "ꟲe'VE", " \n", "'s", "😀🏽", "mعꟲet", "　", "👍🏽'", "D字", "🙂<", "EOT", ">'", "Re"]} +{"text": "　,> Z٣٤٥٦(‍'s…\r'Re>…é\u000bİꟲ'S'Re!\ndḍ̇'DéZ字 ſ'Re0ع'S \u000bİ", "tokens": 49, "pieces": ["　", ",>", " ", " Z", "٣٤٥", "٦", "(‍'", "s", "…\r", "'Re", ">", "…é", "\u000bİꟲ", "'", "S'Re", "!\n", "dḍ̇'D", "é", "Z字", " ſ'Re", "0", "ع'S", " ", "\u000bİ"]} +{"text": "'MZ‍!!'S३ß<|fim_prefix|>(<|fim_prefix|>e'red٣٤٥٦'VE \n'VEe0👍🏽 ", "tokens": 40, "pieces": ["'MZ", "‍!!'", "S", "३", "ß", "<|", "fim", "_prefix", "|>(<|", "fim", "_prefix", "|>", "e're", "d", "٣٤٥", "٦", "'VE", " \n", "'VEe", "0", "👍🏽", " "]} +{"text": "#$%‍ å𐞁é㍿ <EOTd'ſ12345678\r!漢­'Tfi\n.'Sm#$%", "tokens": 44, "pieces": ["#$%‍", " å𐞁é", "㍿", " ", "<<", "META", "_START", ">EOTd'ſ", "123", "456", "78", "\r", "!漢", "<", "META", "_START", ">­'", "Tfi", "\n", ".'", "Sm", "#$%"]} +{"text": ",漢½m'T३👍🏽Dž\r\n
́-s\r\n\r\n<|fim_prefix|>a३éİ­🙂عfi­ſ'Mfi٣٤٥٦\n", "tokens": 39, "pieces": [",漢", "½", "m'T", "३", "👍🏽", "Dž", "\r\n", "
́", "-s", "\r\n\r\n", "<|", "fim", "_prefix", "|>", "a", "३", "é", "İ", "­🙂", "عfi", "­ſ'M", "fi", "٣٤٥", "٦", "\n"]} +{"text": "…#$%", "tokens": 4, "pieces": ["…", "#$%"]} +{"text": "<|fim_prefix|>", "tokens": 6, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "👍🏽fi \nes'T　é0å😀🏽ع\t12345678İd́Ａ­ḍ̇<|fim_prefix|>\rＡß! ", "tokens": 43, "pieces": ["👍🏽", "fi", " \n", "es'T", "　é", "0", "å", "😀🏽", "ع", "\t", "", "123", "456", "78", "İd́", "Ａ", "­ḍ̇", "<|", "fim", "_prefix", "|>\r", "Ａß", "!", " "]} +{"text": " ­<", "tokens": 3, "pieces": [" ", "­<"]} +{"text": "\"e🙂>ع-é12345678Džḍ̇.t👍🏽!३", "tokens": 21, "pieces": ["\"e", "🙂>", "ع", "-é", "123", "456", "78", "Džḍ̇", ".t", "👍🏽!", "३"]} +{"text": "३\r\nå!\nZ㋿ſ'Re,𐞁'll(\n'D!'re'ſ 字m", "tokens": 30, "pieces": ["३", "\r\n", "å", "!\n", "Z", "㋿ſ'Re", ",𐞁'll", "(\n", "'D", "!<", "META", "_START", ">'", "re'ſ", " ", " 字m"]} +{"text": "<|fim_prefix|>字𐞁'VEİs \nعⅣ'ReéfiⅣDž\u000bfi", "tokens": 32, "pieces": ["<|", "fim", "_prefix", "|>", "字", "𐞁'VE", "İs", " \n", "ع", "Ⅳ", "'Reéfi", "Ⅳ", "Dž", "\u000bfi"]} +{"text": "m-<\r\n\r\n'ſ\u000b\"", "tokens": 8, "pieces": ["m", "-<\r\n\r\n", "'ſ", "\u000b", "\""]} +{"text": "'ſ're㍿m'S'Reꟲ\r
🙂.12345678sfi,'<|fim_prefix|>9ꟲ\u000b漢\nt>字\r\nsſ'Red'TmZEOT\t'T", "tokens": 56, "pieces": ["'ſ're", "㍿m'S", "'Reꟲ", "\r", "
", "🙂.", "123", "456", "78", "sfi", ",'<|", "fim", "_prefix", "|>", "9", "ꟲ", "\u000b漢", "\n", "t", ">字", "\r\n", "sſ'Re", "d'T", "m", "ZEOT", "\t", "'T"]} +{"text": ",　", "tokens": 2, "pieces": [",", "　"]} +{"text": "EOT🙂!EOT're,ß'Ms\r\n9\r\n\r\nZ#$%\"'sEOT'ſ \n'll😀🏽㍿'M- ½'ſ!a'D'll0\n", "tokens": 45, "pieces": ["EOT", "🙂!", "EOT're", ",ß'M", "s", "\r\n", "9", "\r\n\r\n", "Z", "#$%\"'", "s", "EOT'ſ", " \n", "'ll", "😀🏽㍿'", "M", "-", " ", " ", "½", "'ſ", "!a'D", "'ll", "0", "\n"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " \n Ⅳ0e 🙂", "tokens": 11, "pieces": [" \n", " ", "Ⅳ", "", "0", "e", " 🙂"]} +{"text": "d<|endoftext|>'sİ!'VEß,́Dž!tⅣ'M<ꟲ字 \n\u000b!!㍿٣٤٥٦　!!<\rDž'T''VE<|endoftext|>(漢㋿", "tokens": 60, "pieces": ["d", "<|", "endoftext", "|>'", "s", "İ", "!'", "VEß", ",́", "Dž", "!t", "Ⅳ", "'M", "<ꟲ字", " \n", "\u000b", "!!㍿", "٣٤٥", "٦", "　", "!!<\r", "Dž'T", "''", "VE", "<|", "endoftext", "|>(", "漢", "㋿"]} +{"text": "#$%\r\n​aé　 !३'é\r­\rZ字!!é0٣٤٥٦'Re'M ", "tokens": 29, "pieces": ["#$%\r\n", "​aé", "", "　 ", " !", "३", "'é", "\r", "­\r", "Z字", "!!", "é", "0٣٤", "٥٦", "'Re'M", " "]} +{"text": "e!EOTع­<|endoftext|>0t½½ع́'ſ", "tokens": 24, "pieces": ["e", "!EOTع", "­<|", "endoftext", "|>", "0", "t", "½½", "ع́'ſ"]} +{"text": "éＡ㍿ḍ̇e'VEDž\r 12345678½é're'Re0\t\r\n\r\ń½!!\r\n\r\n👍🏽'Dfi'S<|endoftext|>s\r\n-ꟲa<|fim_prefix|>'ſ 'res", "tokens": 64, "pieces": ["é", "Ａ", "㍿ḍ̇e'VE", "Dž", "\r", " ", "123", "456", "78½", "é're", "'Re", "0", "\t\r\n\r\n", "́", "½", "!!\r\n\r\n", "👍🏽'", "Dfi'S", "<|", "endoftext", "|>", "s", "\r\n", "-ꟲa", "<|", "fim", "_prefix", "|>'", "ſ", "", " ", "'res"]} +{"text": "\"‍٣٤٥٦é0!!ꟲＡ", "tokens": 14, "pieces": ["\"‍", "٣٤٥", "٦", "é", "0", "!!", "ꟲ", "Ａ"]} +{"text": "és", "tokens": 2, "pieces": ["és"]} +{"text": "Ⅳع­𐞁#$%\t😀🏽fi ꟲfi'ſ!!t'D'MZꟲ", "tokens": 41, "pieces": ["Ⅳ", "ع", "­<", "META", "_START", ">𐞁", "#$%", "\t", "😀🏽", "fi", " ꟲfi'ſ", "!!", "t", "'", "D'M", "Zꟲ"]} +{"text": "🙂DžＡ9fi'D𐞁EOT.0‍<漢,'M>३㍿‍漢'fi>'S", "tokens": 38, "pieces": ["🙂DžＡ", "9", "<", "META", "_START", ">fi'D", "𐞁", "EOT", ".", "0", "‍<", "漢", ",'", "M", ">", "३", "㍿‍", "漢", "'fi", ">'", "S"]} +{"text": "​ <|endoftext|>字(​ 'ſEOT'Mß‍㍿<|endoftext|>字ſ　'sꟲ12345678Ⅳ'Dꟲ३'reⅣs'll'll\n ㋿Ａ　a
㍿३ \n", "tokens": 66, "pieces": ["​", " ", " <|", "endoftext", "|>", "字", "(​", " '", "ſ", "EOT'M", "ß", "‍㍿<|", "endoftext", "|>", "字ſ", "　", "'sꟲ", "123", "456", "78Ⅳ", "'Dꟲ", "३", "'re", "Ⅳ", "s'll", "'ll", "\n", " ㋿", "Ａ", "　a", "
", "㍿", "३", " \n"]} +{"text": "𐞁's㍿dſ'ſ　👍🏽\t#$%Ⅳſ<
🙂\"m‍🙂ſ", "tokens": 30, "pieces": ["𐞁's", "㍿dſ'ſ", "　", "👍🏽", "\t", "#$%", "Ⅳ", "ſ", "<", "
", "🙂\"", "m", "‍🙂", "ſ"]} +{"text": "\r", "tokens": 1, "pieces": ["\r"]} +{"text": "m\"'T<　　👍🏽'T ꟲ字㋿é! #$%\u000b!!٣٤٥٦漢'M", "tokens": 36, "pieces": ["m", "\"'", "T", "<<", "META", "_START", ">", "　", "　", "👍🏽'", "T", " ꟲ字", "㋿é", "!", " ", " #$%", "\u000b", "!!", "٣٤٥", "٦", "漢'M"]} +{"text": "<'så漢'M'ſ🙂٣٤٥٦\nfi😀🏽d #$%(", "tokens": 22, "pieces": ["<'", "så漢'M", "'ſ", "🙂", "٣٤٥", "٦", "\n", "fi", "😀🏽", "d", " ", " #$%("]} +{"text": "d'S
 ", "tokens": 4, "pieces": ["d'S", "
 "]} +{"text": " EOT́😀🏽­d\tＡ123456780.'VE😀🏽éåé9<|endoftext|>­ß'S'EOTDž'llßİ", "tokens": 44, "pieces": [" EOT́", "😀🏽­", "d", "\tＡ", "123", "456", "780", ".'", "VE", "😀🏽", "é", "åé", "9", "<|", "endoftext", "|>­", "ß'S", "'EOTDž'll", "ß", "İ"]} +{"text": "👍🏽İ<|fim_prefix|>", "tokens": 10, "pieces": ["👍🏽", "İ", "<|", "fim", "_prefix", "|>"]} +{"text": "<|endoftext|>\r\n-d\r\n\r\n<|endoftext|> \n'D>½…\t🙂", "tokens": 24, "pieces": ["<|", "endoftext", "|>\r\n", "-d", "\r\n\r\n", "<|", "endoftext", "|>", " \n", "'D", ">", "½", "…", "\t", "🙂"]} +{"text": "ع'reå", "tokens": 4, "pieces": ["ع're", "å"]} +{"text": "EOT'll're　åt\r\nZ 9<|fim_prefix|>'M…>­㍿!!fiعEOT\u000b'Re", "tokens": 33, "pieces": ["EOT'll", "'re", "　åt", "\r\n", "Z", " ", "9", "<|", "fim", "_prefix", "|>'", "M", "…", ">­㍿!!", "fiع", "EOT", "\u000b", "'Re"]} +{"text": "㋿ß漢…Z㍿\"ß<#$%
\r\n\tZEOTZ9́>ＡZ'M­'Re　\r é字!!é", "tokens": 43, "pieces": ["㋿ß漢", "…Z", "㍿\"", "ß", "<#$%", "
\r\n", "\tZEOTZ", "9", "́", "><", "EOT", ">ＡZ'M", "­'", "Re", "　\r", " é字", "!!", "é"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿'ſ'VE'VE'Re'Mḍ̇　­​\u000b(ḍ̇​'Re㍿㋿　\u000ba'reé\r㍿\n,12345678 𐞁", "tokens": 59, "pieces": ["㍿'", "ſ'VE", "'VE'Re", "'", "Mḍ̇", "　", "­​", "\u000b", "(<", "EOT", ">ḍ̇", "​'", "Re", "㍿㋿", "　", "\u000ba're", "é", "\r", "㍿\n", ",", "123", "456", "78", " ", " 𐞁"]} +{"text": "​
字>㍿😀🏽\"'D'reéDž 're‍é'Re\"'T٣٤٥٦12345678٣٤٥٦Z㍿😀🏽\"'", "D're", "é", "Dž", " ", " '", "re", "‍é'Re", "\"'", "T", "٣٤٥", "٦12", "345", "678", "٣٤٥", "٦", "Z", "'ſ　s漢́\t>'Re<­", "tokens": 20, "pieces": ["ß", "#$%🙂", "३", "a'VE", "\r\n", ">'", "ſ", "　s漢́", "\t", ">'", "Re", "<­"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "é\rDž12345678　'll\tḍ̇'T\"!'llعs'll'll ㋿", "tokens": 25, "pieces": ["é", "\r", "Dž", "123", "456", "78", "　", "'ll", "\tḍ̇'T", "\"!'", "llعs'll", "'ll", " ", "㋿"]} +{"text": ">Ａ#$%åEOT's字ß‍ \t9'ſ>'M 'S'Res<|endoftext|><|endoftext|>İ㍿é9Džé's漢'", "tokens": 53, "pieces": [">Ａ", "#$%", "å", "EOT's", "字ß", "‍", " ", "\t", "9", "'ſ", ">'", "M", " ", " <", "META", "_START", ">'", "S'Re", "s", "<|", "endoftext", "|><|", "endoftext", "|>", "İ", "㍿é", "9", "Džé's", "漢", "'"]} +{"text": "Ⅳ!!0\u000bé\r\n\r\nＡ!!Džꟲ'9…9'D㍿\n0Z12345678'M#$%
\r\nfi'reß字", "tokens": 56, "pieces": ["Ⅳ", "!!", "0", "\u000bé", "\r\n\r\n", "Ａ", "!!", "Džꟲ", "'", "9", "…", "9", "'D", "㍿\n", "", "0", "Z", "123", "456", "78", "'M", "#$%", "
\r\n", "fi're", "ß", "字"]} +{"text": "عſ३es9​㍿\r\n'D\r\n\r\n", "tokens": 21, "pieces": ["عſ", "३", "es", "9", "​㍿\r\n", "'D", "\r\n\r\n"]} +{"text": "éDž.­ …>…\r\n­", "tokens": 14, "pieces": ["é", "Dž", ".­", " ", "…", ">", "…\r\n", "­"]} +{"text": "​㋿'T'D㋿㍿>字 ㍿", "tokens": 19, "pieces": ["​㋿'", "T'D", "㋿㍿>", "字", " ", "㍿"]} +{"text": "\r!!'re's‍!m(字­
\téع́d😀🏽EOTaDž\"e'VE\rå0\nꟲＡ漢>\n9… 'Déعع", "tokens": 49, "pieces": ["\r", "!!'", "re's", "‍!", "m", "(字", "­", "
", "\téع́d", "😀🏽", "EOTa", "Dž", "\"e'VE", "\r", "å", "0", "\n", "ꟲＡ漢", ">\n", "9", "…", " ", "'Déعع"]} +{"text": "Ａ\r​a\t🙂　'Dع​t𐞁'llfiİ", "tokens": 18, "pieces": ["Ａ", "\r", "​a", "\t", "🙂", "　", "'Dع", "​t𐞁'll", "fi", "İ"]} +{"text": "EOT😀🏽<|endoftext|>‍\r\n\"Džꟲ'\rd.<9\r\r\n\r\n!\n…9　٣٤٥٦", "tokens": 39, "pieces": ["EOT", "😀🏽<|", "endoftext", "|>‍\r\n", "\"Dž", "ꟲ", "'\r", "d", ".<", "9", "\r\r\n\r\n", "!\n", "…", "9", "　", "٣٤٥", "٦"]} +{"text": "'re 👍🏽字\r!!<|endoftext|>sⅣ'T(　Z\r\n😀🏽٣٤٥٦0🙂👍🏽'M<|endoftext|>\r漢ḍ̇\r\n\r\n İ<9Ⅳ'Ds", "tokens": 58, "pieces": ["'re", " ", "👍🏽", "字", "\r", "!!<|", "endoftext", "|>", "s", "Ⅳ", "'T", "(", "　Z", "\r\n", "😀🏽", "٣٤٥", "٦0", "🙂👍🏽'", "M", "<|", "endoftext", "|>\r", "漢ḍ̇", "\r\n\r\n", " İ", "<", "9Ⅳ", "'Ds"]} +{"text": "\u000b\tſ漢é", "tokens": 6, "pieces": ["\u000b", "\tſ漢é"]} +{"text": "Dž#$% t'Re‍ ꟲ", "tokens": 10, "pieces": ["Dž", "#$%", " t'Re", "‍", " ꟲ"]} +{"text": "Zꟲ½e#$%e", "tokens": 9, "pieces": ["Zꟲ", "½", "e", "#$%", "e"]} +{"text": "　…‍​'s'ſ🙂'll <|fim_prefix|>ع\t!é٣٤٥٦ḍ̇e…漢Dž", "tokens": 58, "pieces": ["　", "…", "‍​'", "s'ſ", "🙂'", "ll", " <|", "fim", "_prefix", "|>", "ع", "", "\t", "!é", "٣٤٥", "٦", "ḍ̇e", "…漢", "Dž"]} +{"text": "'T<|fim_prefix|>12345678'\r\n
á İ\" İ 's", "tokens": 21, "pieces": ["'T", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'\r\n", "
á", " İ", "\"", " ", " İ", " ", "'s"]} +{"text": "'s\t9EOT'ſ<😀🏽​>0漢sſ ꟲ'VEǻ‍", "tokens": 26, "pieces": ["'s", "\t", "9", "EOT'ſ", "<😀🏽​>", "0", "漢sſ", " ꟲ'VE", "ǻ", "‍"]} +{"text": ">'M", "tokens": 5, "pieces": ["><", "META", "_START", ">'", "M"]} +{"text": ".!<|endoftext|> ३#$%", "tokens": 12, "pieces": [".!<|", "endoftext", "|>", " ", "३", "#$%"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'T​å३t𐞁å½­​0a", "tokens": 17, "pieces": ["'T", "​å", "३", "t𐞁å", "½", "­​", "0", "a"]} +{"text": "‍", "tokens": 1, "pieces": ["‍"]} +{"text": "\u000bétＡ\r\nå \n​0'T EOT9\r\n\r\n \nt\"'Sa", "tokens": 21, "pieces": ["\u000bét", "Ａ", "\r\n", "å", " \n", "​", "0", "'T", " EOT", "9", "\r\n\r\n \n", "t", "\"'", "Sa"]} +{"text": "(-‍
'S­Z漢Z🙂12345678 'VEs'Re漢👍🏽​>　 ​́s😀🏽Ⅳ'D
Z'Z", "tokens": 48, "pieces": ["(-‍", "
", "'", "S", "­<", "META", "_START", ">Z漢", "Z", "🙂<", "EOT", ">", "123", "456", "78", " ", "'VEs'Re", "漢", "👍🏽​>", "　", " ​́", "s", "😀🏽", "Ⅳ", "'D", "
Z", "'Z"]} +{"text": "…fi'll's　#$%㋿std#$%.Ⅳ😀🏽👍🏽'T㍿​ß'M", "tokens": 39, "pieces": ["…fi'll", "'s", "　", "#$%㋿", "s", "td", "#$%.", "Ⅳ", "😀🏽👍🏽'", "T", "㍿​", "ß'M"]} +{"text": "㍿é0\r\n\r\n㋿d　'…12345678漢㋿Z ſ 'Re ( 'reDž\n\r\n\r\n", "tokens": 41, "pieces": ["㍿", "é", "0", "\r\n\r\n", "㋿d", "　", "'", "…", "123", "456", "78", "漢", "㋿Z", " ſ", " ", " '", "Re", " ", " (", " ", " '", "re", "Dž", "\n\r\n\r\n"]} +{"text": "'llDž12345678aé<'T-'S ३eEOTſ\r'ſ\u000bꟲEOT‍ .DžⅣZ!!-'", "S", " ", "३", "e", "EOTſ", "\r", "'ſ", "\u000bꟲ", "EOT", "‍", " .", "Dž", "Ⅳ", "Z", "!!<", "EOTtع'D", "　é", ",ǻ", "Ａ", "३"]} +{"text": "#$%‍½İ'T㋿‍漢e ", "tokens": 13, "pieces": ["#$%‍", "½", "İ'T", "㋿‍", "漢e", " "]} +{"text": "'VE!Z३fi३9Dž　'T
'T>‍'Re'Re ,.㋿'s#$%'T\r\n㍿9½<|endoftext|>'lle \n漢 字​", "tokens": 48, "pieces": ["'VE", "!Z", "३", "fi", "३9", "Dž", "　", "'T", "
", "'T", ">‍'", "Re'Re", " ", ",.㋿'", "s", "#$%'", "T", "\r\n", "㍿", "9½", "<|", "endoftext", "|>'", "lle", " \n", "漢", " 字", "​"]} +{"text": "é👍🏽ꟲ𐞁 漢Z'ſſ'll!!<|fim_prefix|>\r\nſ­ \n­\r\n\r\n'll٣٤٥٦\tfi12345678ḍ̇é \n\"12345678İt'Sḍ̇🙂-", "tokens": 57, "pieces": ["é", "👍🏽", "ꟲ𐞁", " 漢", "Z'ſ", "ſ'll", "!!<|", "fim", "_prefix", "|>\r\n", "ſ", "­", " \n", "­\r\n\r\n", "'ll", "٣٤٥", "٦", "\tfi", "123", "456", "78", "ḍ̇é", " \n", "\"", "123", "456", "78", "İt'S", "ḍ̇", "🙂-"]} +{"text": "-㍿́#$%ꟲ-㍿\r\n\r\n\r\n'D　
‍ꟲ ꟲعé#$%
​🙂ßع👍🏽'ſ'D​<|endoftext|>!'ll\r\n\r\nDž½ßd", "tokens": 57, "pieces": ["-㍿́#$%", "ꟲ", "-㍿\r\n\r\n\r\n", "'D", "　", "
", "‍ꟲ", " ꟲعé", "#$%", "
", "​🙂", "ßع", "👍🏽'", "ſ'D", "​<|", "endoftext", "|>!'", "ll", "\r\n\r\n", "Dž", "½", "ßd"]} +{"text": "'sſ 's
…ع-٣٤٥٦>e!　٣٤٥٦
㍿<|endoftext|>
<fi😀🏽'VE>İé\n<ꟲ­ßſ🙂12345678'DZ ㋿ Ⅳ", "tokens": 58, "pieces": ["\n\n", ">-", "٣٤٥", "٦", ">e", "!", "　", "٣٤٥", "٦", "
", "㍿<|", "endoftext", "|>", "
", "<fi", "😀🏽'", "VE", ">İé", "\n", "<ꟲ", "­ßſ", "🙂", "123", "456", "78", "'DZ", " ", "㋿", " ", "Ⅳ"]} +{"text": "\td#$%#$%>\r\n,t\t0<|fim_prefix|>\nع३½\r\n", "tokens": 25, "pieces": ["\td", "#$%#$%><", "EOT", ">\r\n", ",t", "\t", "0", "<|", "fim", "_prefix", "|>\n", "ع", "", "३½", "\r\n"]} +{"text": "́-­'T'lld0.e'M,'Se\r\n\r\n\r\n!!㍿'reDž字", "tokens": 25, "pieces": ["́", "-­'", "T'll", "d", "0", ".e'M", ",'", "Se", "\r\n", "\r\n\r\n", "!!㍿'", "re", "Dž字"]} +{"text": ".'sEOT\r\n\r\nⅣå!s", "tokens": 11, "pieces": [".'", "s", "EOT", "\r\n\r\n", "Ⅳ", "å", "!s"]} +{"text": "३İ'll३漢", "tokens": 5, "pieces": ["३", "İ'll", "३", "漢"]} +{"text": "Z \n‍Ａ'VE\r\nméå\t㋿\té​‍\"é'ſt㋿!", "tokens": 29, "pieces": ["Z", " \n", "‍Ａ'VE", "\r\n", "méå", "\t", "㋿", "\té", "​‍\"", "é'ſ", "t", "㋿!"]} +{"text": "fi'ſ\r\n\r\n!!. \"ع", "tokens": 8, "pieces": ["fi'ſ", "\r\n\r\n", "!!.", " ", " \"", "ع"]} +{"text": "𐞁ꟲ're\n<\"İém­'re", "tokens": 18, "pieces": ["𐞁ꟲ're", "\n", "<\"", "İém", "­'", "re"]} +{"text": "\r\nå0​👍🏽Ａ're'TEOT", "tokens": 13, "pieces": ["\r\n", "å", "0", "​👍🏽", "Ａ're", "'TEOT"]} +{"text": "\n>'Sfí㋿ed
'Re<|fim_prefix|>aZ<|fim_prefix|>漢
", "tokens": 27, "pieces": ["\n", ">'", "Sfí", "㋿ed", "
", "'Re", "<|", "fim", "_prefix", "|>", "a", "Z", "<|", "fim", "_prefix", "|>", "漢", "
"]} +{"text": "123456789
ꟲ­<|fim_prefix|>㋿٣٤٥٦­'ſ'S!ß'Smḍ̇ḍ̇", "tokens": 35, "pieces": ["123", "456", "789", "
ꟲ", "­<|", "fim", "_prefix", "|>㋿", "٣٤٥", "٦", "­'", "ſ'S", "!ß'S", "mḍ̇ḍ̇"]} +{"text": "३", "tokens": 1, "pieces": ["३"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿​٣٤٥٦12345678'Dع🙂9ſ😀🏽ß㋿İ'Ret३Zع9İ'Reſ‍字‍!३​'s<|fim_prefix|><|fim_prefix|>𐞁12345678​\n'llm", "tokens": 66, "pieces": ["㍿​", "٣٤٥", "٦12", "345", "678", "'Dع", "🙂", "9", "ſ", "😀🏽", "ß", "㋿İ'Re", "t", "३", "Zع", "9", "İ'Re", "ſ", "‍字", "‍!", "३", "​'", "s", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "𐞁", "123", "456", "78", "​\n", "'llm"]} +{"text": "字as'ś'sＡ漢12345678å", "tokens": 12, "pieces": ["字as's", "́'s", "Ａ漢", "123", "456", "78", "å"]} +{"text": "<|endoftext|>s012345678<|endoftext|>0>\"\r\n\r\n㋿İ𐞁.漢,'Ta'Dd\n \n㋿'S½漢\r\n \n漢", "tokens": 47, "pieces": ["<|", "endoftext", "|>", "s", "012", "345", "678", "<|", "endoftext", "|>", "0", ">\"\r\n\r\n", "㋿İ𐞁", ".漢", ",'", "Ta'D", "d", "\n \n", "㋿'", "S", "½", "漢", "\r\n \n", "漢"]} +{"text": " ㋿३ßda'T!!", "tokens": 9, "pieces": [" ㋿", "३", "ßda'T", "!!"]} +{"text": "<Dž 𐞁㋿\u000bع‍", "tokens": 14, "pieces": ["<Dž", " 𐞁", "㋿", "\u000bع", "‍"]} +{"text": "🙂e٣٤٥٦'llZ<|fim_prefix|>'T", "tokens": 15, "pieces": ["🙂e", "٣٤٥", "٦", "'ll", "Z", "<|", "fim", "_prefix", "|>'", "T"]} +{"text": "m,'\"e字㋿字\u000b'll\u000b0å\r'M9ds𐞁Ａ\r\n 're12345678ḍ̇éaEOT\r½dZ\r\n\r\n \n>e,9é", "tokens": 49, "pieces": ["m", ",'\"", "e字", "㋿字", "\u000b", "'ll", "\u000b", "0", "å", "\r", "'M", "9", "ds𐞁", "Ａ", "\r\n", " ", " '", "re", "123", "456", "78", "ḍ̇éa", "EOT", "\r", "½", "d", "Z", "\r\n\r\n \n", ">e", ",", "9", "é"]} +{"text": "12345678'M😀🏽.٣٤٥٦'T'Re३e(<㍿åd('T'll\r\n\r\n'VE㋿🙂m​,ḍ̇ \n#$%ß \n‍<|endoftext|>ß'T½'
Dž#$%", "tokens": 65, "pieces": ["123", "456", "78", "'M", "😀🏽.", "٣٤٥", "٦", "'T'Re", "३", "e", "(<㍿", "åd", "('", "T", "'", "ll", "\r\n\r\n", "'VE", "㋿🙂", "m", "​,", "ḍ̇", " \n", "#$%", "ß", " \n", "‍<|", "endoftext", "|>", "ß'T", "½", "'", "
Dž", "#$%"]} +{"text": "a\r\n\r\nEOT \nḍ̇>12345678's<|fim_prefix|>'ſ.EOT12345678'Rema'D𐞁'T<|endoftext|>fi.", "tokens": 43, "pieces": ["a", "\r\n\r\n", "EOT", " \n", "ḍ̇", ">", "123", "456", "78", "'s", "<|", "fim", "_prefix", "|>'", "ſ", ".EOT", "123", "456", "78", "'Rema'D", "𐞁'T", "<|", "endoftext", "|>", "fi", "."]} +{"text": "!!!!fi漢İＡe.ḍ̇t漢㍿​!!'reda'M.'ſ‍é> 'VÉ", "tokens": 38, "pieces": ["!!<", "EOT", ">!!", "fi漢", "İ", "Ａe", ".ḍ̇t漢", "㍿​!!'", "reda'M", ".'", "ſ", "‍é", ">", " ", " '", "VÉ"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "­>\"ådſ　 ", "tokens": 7, "pieces": ["­>\"", "ådſ", "　 "]} +{"text": "'D'D 'Re12345678'Re'T ZEOT ع㍿\t's㍿­ſ, ", "tokens": 26, "pieces": ["'D'D", " ", " '", "Re", "123", "456", "78", "'Re'T", " ZEOT", " ع", "㍿", "\t", "'s", "㍿­", "ſ", ",", " "]} +{"text": "'Re'S,-👍🏽 'T
é　#$%", "tokens": 15, "pieces": ["'Re'S", ",-👍🏽", " ", " '", "T", "
é", "　", "#$%"]} +{"text": "😀🏽\u000b'llＡ\r\n\r\n\r\n<|endoftext|>(㍿.'s👍🏽½dDž\"", "tokens": 27, "pieces": ["😀🏽", "\u000b", "'ll", "Ａ", "\r\n\r\n\r\n", "<|", "endoftext", "|>(㍿.'", "s", "👍🏽", "½", "d", "Dž", "\""]} +{"text": "9'Re0İ0'M\n<|endoftext|>'T😀🏽٣٤٥٦0", "tokens": 23, "pieces": ["9", "'Re", "0", "İ", "0", "'M", "\n", "<|", "endoftext", "|>'", "T", "😀🏽", "٣٤٥", "٦0"]} +{"text": "<|endoftext|>s㋿\r\n\r\n👍🏽éad.'sſ'S", "tokens": 28, "pieces": ["<|", "endoftext", "|><", "EOT", ">s", "㋿\r\n\r\n", "👍🏽", "éad", ".'", "sſ", "'", "S"]} +{"text": "a \u000b'ſꟲfi<'VE'Re\r\n", "tokens": 16, "pieces": ["a", " ", "\u000b", "'ſꟲfi", "<'", "VE'Re", "\r\n"]} +{"text": "\r\n\r\n­Dž's'M'S<́'VE'Re\t　'३'🙂é👍🏽\u000b<|endoftext|>Ⅳ३‍㋿'re", "tokens": 39, "pieces": ["\r\n\r\n", "­Dž's", "'M'S", "<́'VE", "'Re", "\t", "　", "'", "३", "'🙂", "é", "👍🏽", "\u000b", "<|", "endoftext", "|>", "Ⅳ३", "‍㋿'", "re"]} +{"text": "é漢a t'Z\tع> #$%\tİ…'ReDž…٣٤٥٦'D!!é\nⅣmＡt sZ😀🏽'Mfi'll", "tokens": 51, "pieces": ["é漢a", " ", " t", "'Z", "\tع", ">", " ", " #$%", "\tİ", "…", "'Re", "Dž", "…", "٣٤٥", "٦", "'D", "!!", "é", "\n", "Ⅳ", "m", "Ａt", " s", "Z", "😀🏽'", "Mfi'll"]} +{"text": "٣٤٥٦ß'Re…!'re\tZmé字9fi!'TEOT -'ſ's(漢a", "tokens": 26, "pieces": ["٣٤٥", "٦", "ß'Re", "…", "!'", "re", "\tZmé字", "9", "fi", "!'", "TEOT", " -'", "ſ's", "(漢a"]} +{"text": "<|endoftext|>m½!!‍0　<|fim_prefix|>-\r12345678'T'Zs12345678.　'sZ‍<|endoftext|>d'VEd漢'S'VEⅣ
\u000bé漢'Re \n", "tokens": 63, "pieces": ["<|", "endoftext", "|>", "m", "½", "!!‍", "0", "　", "<|", "fim", "_prefix", "|>-\r", "", "123", "456", "78", "'T", "'Zs", "123", "456", "78", ".", "　", "'s", "Z", "‍<|", "endoftext", "|>", "d'VE", "d漢'S", "'VE", "Ⅳ", "
", "\u000bé漢'Re", " \n"]} +{"text": "ſ‍Ａ(ſ३\t'D \tⅣ's\r\"EOT.٣٤٥٦ \nİ \"", "tokens": 24, "pieces": ["ſ", "‍Ａ", "(ſ", "३", "\t", "'D", " ", "\t", "Ⅳ", "'s", "\r", "\"EOT", ".", "٣٤٥", "٦", " \n", "İ", " \""]} +{"text": "　(!'Re'Re<|fim_prefix|>\rع\u000bDž're　ß‍'T99d!!åⅣ,‍'D!!㍿
a'll\nŹ'D　", "tokens": 51, "pieces": ["　", "(!'", "Re", "'", "Re", "<|", "fim", "_prefix", "|>\r", "ع", "\u000bDž're", "　ß", "‍'", "T", "99", "d", "!!", "å", "Ⅳ", ",‍'", "D", "!!㍿", "
a'll", "\n", "Ź'D", "　"]} +{"text": "'D're㋿0!", "tokens": 7, "pieces": ["'D're", "㋿", "0", "!"]} +{"text": "Ⅳ'ſ‍<|fim_prefix|>9t'Re\rḍ̇🙂'D\r\n\r\nſ'Re字! ع'D\"…　e'll\n!İ !<|fim_prefix|>　tＡ\"fi", "tokens": 50, "pieces": ["Ⅳ", "'ſ", "‍<|", "fim", "_prefix", "|>", "9", "t'Re", "\r", "ḍ̇", "🙂'", "D", "\r\n\r\n", "ſ'Re", "字", "!", " ع'D", "\"", "…", "　e'll", "\n", "!İ", " !<|", "fim", "_prefix", "|>", "　t", "Ａ", "\"fi"]} +{"text": "🙂\u000bİ12345678\r\n12345678", "tokens": 10, "pieces": ["🙂", "\u000bİ", "123", "456", "78", "\r\n", "123", "456", "78"]} +{"text": "åe\u000b0m <\t'T㍿…'VE𐞁e9<|fim_prefix|>'sſ!!\n\r\n㋿Ⅳ'D　<0é", "tokens": 50, "pieces": ["åe", "\u000b", "0", "m", " <", "\t", "'T", "㍿", "…", "'VE𐞁e", "9", "<|", "fim", "_prefix", "|>'", "sſ", "!!<", "EOT", ">\n\r\n", "㋿", "Ⅳ", "'D", "　", "<", "0", "é"]} +{"text": "'ll'Ddm>㋿'İ're'S-'S>𐞁 …Ａ", "tokens": 25, "pieces": ["'ll'D", "dm", ">㋿'", "İ're", "'S", "-'", "S", ">", "𐞁", " ", "…Ａ"]} +{"text": "12345678'M\r,Ａ 字'sm\r\n\r\n m>𐞁 \n عſe٣٤٥٦", "tokens": 31, "pieces": ["123", "456", "78", "'M", "\r", ",Ａ", " ", " 字's", "m", "\r\n\r\n", " m", ">𐞁", " \n", " عſe", "٣٤٥", "٦"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿　'DZ. \n!\r\n\r\n\n-👍🏽'ſ\"'D\"㍿ \n'D,'ſ", "tokens": 28, "pieces": ["㍿", "　", "'", "DZ", ".", " \n", "!\r\n\r\n\n", "-👍🏽'", "ſ", "\"'", "D", "\"㍿", " \n", "'D", ",'", "ſ"]} +{"text": "ZaعeéEOT🙂(!!'T٣٤٥٦ع 0éⅣ\n'ſ<|endoftext|>'re👍🏽\r\n\r\nå.< 字", "tokens": 50, "pieces": ["Zaعeé", "EOT", "🙂(!!<", "META", "_START", ">'", "T", "٣٤٥", "٦", "ع", " ", "0", "é", "Ⅳ", "\n", "'ſ", "<", "EOT", "><|", "endoftext", "|>'", "re", "👍🏽\r\n\r\n", "å", ".<", " ", " 字"]} +{"text": "İm12345678\r''12345678'T-t　ß12345678
<|fim_prefix|>‍", "tokens": 28, "pieces": ["İm", "123", "456", "78", "\r", "''", "123", "456", "78", "'T", "-t", "　ß", "123", "456", "78", "
", "<|", "fim", "_prefix", "|>‍"]} +{"text": "ḍ̇ea\" 字ع\r
👍🏽
👍🏽😀🏽åḍ̇३Ａ\u000b<|endoftext|>'M­'TZ'ſ👍🏽(‍㋿३0", "tokens": 51, "pieces": ["ḍ̇ea", "\"", " 字ع", "\r", "
", "👍🏽", "
", "👍🏽😀🏽", "åḍ̇", "३", "Ａ", "\u000b", "<|", "endoftext", "|>'", "M", "­'", "TZ'ſ", "👍🏽(‍㋿", "३0"]} +{"text": "'D\n½½㍿\r\nꟲ́12345678 a\t#$%٣٤٥٦\n!d0'ſ12345678٣٤٥٦㍿ßé😀🏽.'", "tokens": 48, "pieces": ["'D", "\n", "½½", "㍿<", "META", "_START", ">\r\n", "ꟲ́", "123", "456", "78", " a", "\t", "#$%", "٣٤٥", "٦", "\n", "!d", "0", "'ſ", "123", "456", "78٣", "٤٥٦", "㍿ßé", "😀🏽.'"]} +{"text": "Dž>Ａa㍿́<(ḍ̇İ#$%\r' \n　e!!!!9<'T", "tokens": 28, "pieces": ["Dž", ">Ａa", "㍿́", "<(", "ḍ̇", "İ", "#$%\r", "'<", "META", "_START", ">", " \n", "　e", "!!!!", "9", "<'", "T"]} +{"text": "#$%!字\"👍🏽㋿a'Re\t​Ａ​İ'M\rſ'T\u000bİ字<'D!!
", "tokens": 28, "pieces": ["#$%!", "字", "\"👍🏽㋿", "a'Re", "\t", "​Ａ", "​İ'M", "\r", "ſ'T", "\u000bİ字", "<'", "D", "!!", "
"]} +{"text": "ꟲ're 😀🏽\"'", "tokens": 9, "pieces": ["ꟲ're", " ", " 😀🏽\"'"]} +{"text": "　…'s \nßEOTé", "tokens": 10, "pieces": ["　", "…", "'s", " \n", "ß", "EOTé"]} +{"text": "́(", "tokens": 2, "pieces": ["́", "("]} +{"text": "\r\n\r\n
å<|fim_prefix|>EOT'Ss \r\nt…㋿'ll!!'VE're 0Zt👍🏽ع'D", "tokens": 37, "pieces": ["\r\n\r\n", "
å", "<|", "fim", "_prefix", "|>", "EOT'S", "s", " \r\n", "t", "…", "㋿'", "ll", "!!'", "VE're", " ", "0", "Zt", "👍🏽", "ع'D"]} +{"text": "'Dß字9\r\n\r\n३…d'VEfi👍🏽Z…ſ(\tDže३", "tokens": 28, "pieces": ["'Dß字", "9", "\r\n\r\n", "३", "…d'VE", "fi", "👍🏽", "Z", "…ſ", "(", "\tDže", "", "३"]} +{"text": "\"ḍ̇ع\"!,!!　''T　12345678漢́\"漢 <|endoftext|>🙂'ſ's㋿👍🏽>‍", "tokens": 53, "pieces": ["\"ḍ̇", "ع", "\"!,!!", "　", "''", "T", "　", "123", "456", "78", "漢́", "\"漢", " <|", "endoftext", "|>🙂'", "ſ's", "㋿👍🏽>‍"]} +{"text": "'\r\nع'Dé'<|endoftext|>'Re12345678\r\n\r\nfi#$%t\r\n\r\n字EOT<<|fim_prefix|> \n-'VE'Re​ \n", "tokens": 39, "pieces": ["'\r\n", "ع'D", "é", "'<|", "endoftext", "|>'", "Re", "123", "456", "78", "\r\n\r\n", "fi", "#$%", "t", "\r\n\r\n", "字", "EOT", "<<|", "fim", "_prefix", "|>", " \n", "-'", "VE'Re", "​", " \n", ""]} +{"text": "'ReEOT𐞁ḍ̇<|fim_prefix|>e½İ\u000b½å", "tokens": 23, "pieces": ["'Re", "EOT𐞁ḍ̇", "<|", "fim", "_prefix", "|>", "e", "½", "İ", "\u000b", "½", "å"]} +{"text": "ſe​", "tokens": 3, "pieces": ["ſe", "​"]} +{"text": "'Re😀🏽'\r😀🏽12345678\tß\r\n\r\n　<|fim_prefix|>́ſ<|endoftext|>#$%('s\r\n\r\n,!!ꟲ𐞁👍🏽­́ß ", "tokens": 52, "pieces": ["'Re", "😀🏽'\r", "😀🏽", "123", "456", "78", "\tß", "\r\n\r\n", "　", "<|", "fim", "_prefix", "|>́", "ſ", "<|", "endoftext", "|>#$%('", "s", "\r\n\r\n", ",!!", "ꟲ𐞁", "👍🏽­́", "ß", " "]} +{"text": "'D👍🏽́'s'", "s", "#$%m", "tokens": 49, "pieces": ["\u000b", "#$%<", "d字́'ſ", "ß", "m"]} +{"text": "å 's　\"<|fim_prefix|>12345678'll. \n -　\u000bé½ß'Dſ\"!!ꟲm😀🏽'Rea12345678Ⅳ", "tokens": 43, "pieces": ["å", " ", " '", "s", "　", "\"<|", "fim", "_prefix", "|>", "123", "456", "78", "'ll", ".", " \n", " -", "　", "\u000bé", "½", "ß'D", "ſ", "\"!!", "ꟲm", "😀🏽'", "Rea", "123", "456", "78Ⅳ"]} +{"text": "'ḍ̇ ३'ReＡ३\r\n
s( .​9
-\n½…½­😀🏽'D(fi", "tokens": 32, "pieces": ["'ḍ̇", " ", " ", "३", "'Re", "Ａ", "३", "\r\n", "
s", "(", " ", " .​", "9", "
", "-\n", "½", "…", "½", "­😀🏽'", "D", "(fi"]} +{"text": "'Mḍ̇s(sfiZ٣٤٥٦d'D‍漢\r\n\r\n!12345678३EOT'S0́㋿漢　½", "tokens": 33, "pieces": ["'Mḍ̇s", "(sfi", "Z", "٣٤٥", "٦", "d'D", "‍漢", "\r\n\r\n", "!", "123", "456", "78३", "EOT'S", "0", "́", "㋿漢", "　", "½"]} +{"text": "漢! \r\ń😀🏽'😀🏽 'SZ\"t,ع'ſ𐞁<|endoftext|>Z#$%\"", "tokens": 36, "pieces": ["漢", "!", " \r\n", "́", "😀🏽'😀🏽", " <", "META", "_START", ">'", "SZ", "\"t", ",ع'ſ", "𐞁", "<|", "endoftext", "|>", "Z", "#$%\""]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ſ
s👍🏽㍿­å<|fim_prefix|>ßa", "tokens": 23, "pieces": ["ſ", "
s", "👍🏽㍿­", "å", "<|", "fim", "_prefix", "|>", "ß", "a"]} +{"text": "Ⅳſİ'S'll'D'llé-ß'S ​́­m", "tokens": 17, "pieces": ["Ⅳ", "ſ", "İ'S", "'ll'D", "'llé", "-ß'S", " ​́­", "m"]} +{"text": "0EOT-३s𐞁 \nḍ̇'M\r\n㋿fi!𐞁a'D''se<|fim_prefix|>\"İ<|fim_prefix|>Ａ­ḍ̇", "tokens": 47, "pieces": ["0", "EOT", "-", "३", "s𐞁", " \n", "ḍ̇'M", "\r\n", "㋿fi", "!𐞁a'D", "''", "se", "<|", "fim", "_prefix", "|>\"", "İ", "<|", "fim", "_prefix", "|>", "Ａ", "­ḍ̇"]} +{"text": "­\neḍ̇fi\"s.'S漢  'VE😀🏽é\r\n\r\n!!s Z-ع(mé\r\ń's.　t३!", "tokens": 35, "pieces": ["­\n", "eḍ̇fi", "\"s", ".'", "S漢", " ", " ", "'VE", "😀🏽", "é", "\r\n\r\n", "!!", "s", " ", " Z", "-ع", "(mé", "\r\n", "́'s", ".", "　t", "३", "!"]} +{"text": "漢#$%9fiⅣ½㍿s😀🏽fi\r", "tokens": 17, "pieces": ["漢", "#$%", "9", "fi", "Ⅳ½", "㍿s", "😀🏽", "fi", "\r"]} +{"text": "<|endoftext|>\r\n\r\nꟲ\u000b'VE\n…Ⅳ𐞁 \n👍🏽9'll12345678​", "tokens": 38, "pieces": ["<|", "endoftext", "|>\r\n\r\n", "ꟲ", "\u000b", "'VE", "\n", "", "…", "Ⅳ", "𐞁", " \n", "👍🏽", "9", "'ll", "123", "456", "78", "​"]} +{"text": "'sDž'Re", "tokens": 7, "pieces": ["'s", "Dž", "'", "Re"]} +{"text": "🙂 ­'M d'ſ 😀🏽㍿eå<0'll\"'Dm", "tokens": 24, "pieces": ["🙂", " ", " ­'", "M", " d'ſ", " 😀🏽㍿", "e", "å", "<", "0", "'ll", "\"'", "Dm"]} +{"text": "\n…'Re'll👍🏽maé٣٤٥٦åİ'VE㋿🙂٣٤٥٦<|endoftext|>ée'Re\t\r\n!'Mfi
字🙂", "tokens": 57, "pieces": ["é'll", "'lle", "İ", " <|", "fim", "_prefix", "|>", "…", "'Re'll", "👍🏽", "maé", "٣٤٥", "٦", "å", "İ'VE", "㋿🙂", "٣٤٥", "٦", "<|", "endoftext", "|>", "ée'Re", "\t\r\n", "!'", "Mfi", "
字", "🙂"]} +{"text": "Džå0‍.'M\u000b'Se‍漢!!'T\u000bⅣ́", "tokens": 24, "pieces": ["Džå", "0", "‍.'", "M", "\u000b", "'Se", "‍漢", "!!'", "T", "\u000b", "Ⅳ", "́"]} +{"text": "Z\"", "tokens": 2, "pieces": ["Z", "\""]} +{"text": "aſ'ſ(e>'T Z12345678
<'Reé👍🏽s!!٣٤٥٦'D\r\ns👍🏽dDž\"ß'ع\rꟲa", "tokens": 49, "pieces": ["aſ'ſ", "(e", ">'", "T", " Z", "123", "456", "78", "
", "<'", "Reé", "👍🏽", "s", "!!", "٣٤٥", "٦", "'D", "\r\n", "s", "👍🏽", "d", "Dž", "\"ß", "'ع", "\r", "ꟲa"]} +{"text": ".t(ta'M‍😀🏽'ré🙂'Sé12345678　é", "tokens": 24, "pieces": [".t", "(ta", "'", "M", "‍😀🏽'", "ré", "🙂'", "Sé", "123", "456", "78", "　é"]} +{"text": "'M å​'ſ漢é!", "tokens": 10, "pieces": ["'M", " å", "​'", "ſ漢é", "!"]} +{"text": "e…d\r\n\r\n½　'S'M‍\"\tİe- \n!!'re३(\n字 ​fi 's ,\n9ꟲ­", "tokens": 36, "pieces": ["e", "…d", "\r\n\r\n", "½", "　", "'S'M", "‍\"", "\tİe", "-", " \n", "!!'", "re", "३", "(<", "META", "_START", ">\n", "字", " ", "​fi", " ", "'s", " ,\n", "9", "ꟲ", "­"]} +{"text": "#$%\r \n½ a'S­'D", "tokens": 11, "pieces": ["#$%\r", " \n", "½", " a'S", "­'", "D"]} +{"text": "🙂aß㍿<|fim_prefix|>'VE 𐞁#$%ſ'll#$%\r\nＡ<|endoftext|>", "tokens": 33, "pieces": ["🙂aß", "㍿<|", "fim", "_prefix", "|>'", "VE", " ", " 𐞁", "#$%", "ſ'll", "#$%\r\n", "Ａ", "<|", "endoftext", "|>"]} +{"text": "𐞁 å!ae(\r\ns\t字'M'll'TＡ😀🏽٣٤٥٦ع\t́😀🏽éZİ\u000b", "tokens": 34, "pieces": ["𐞁", " å", "!ae", "(\r\n", "s", "\t字'M", "'ll'T", "Ａ", "😀🏽", "٣٤٥", "٦", "ع", "\t́", "😀🏽", "é", "Zİ", "\u000b"]} +{"text": "0m٣٤٥٦Ⅳ'T", "tokens": 9, "pieces": ["0", "m", "٣٤٥", "٦Ⅳ", "'T"]} +{"text": "å'M", "tokens": 3, "pieces": ["å'M"]} +{"text": "å'Ret'Retꟲ𐞁
m🙂🙂\n𐞁e", "tokens": 23, "pieces": ["å'Re", "t'Re", "tꟲ𐞁", "
m", "🙂🙂\n", "𐞁e"]} +{"text": "'re're>åſ㍿​\u000bé'M0'M\r", "tokens": 16, "pieces": ["'re're", ">åſ", "㍿​", "\u000bé'M", "0", "'M", "\r"]} +{"text": "e　\u000b'S.a", "tokens": 5, "pieces": ["e", "　", "\u000b", "'S", ".a"]} +{"text": " <  \n漢d're​eİe\"\t㋿
ع́\t(", "tokens": 21, "pieces": [" ", "<", "  \n", "漢d're", "​e", "İe", "\"", "\t", "㋿", "
ع́", "\t", "("]} +{"text": "!!'s'S'Tt", "tokens": 9, "pieces": ["!!'", "s'S", "'Tt"]} +{"text": ",'ſḍ̇<|fim_prefix|>d é字'ſ㋿㋿e", "tokens": 32, "pieces": [",'", "ſḍ̇", "<|", "fim", "_prefix", "|>", "d", " ", " é", "字'ſ", "㋿<", "META", "_START", ">㋿", "e"]} +{"text": "Dž 'D𐞁
👍🏽#$%𐞁!٣٤٥٦Z!!t", "tokens": 30, "pieces": ["Dž", " '", "D𐞁", "
", "👍🏽#$%", "𐞁", "!<", "EOT", ">", "٣٤٥", "٦", "Z", "!!", "t"]} +{"text": "\t,Ⅳ", "tokens": 4, "pieces": ["\t", ",", "Ⅳ"]} +{"text": " ​'re \nعt𐞁'Dta \nåé'D漢👍🏽så㍿́DžDžt Z'S< ", "tokens": 42, "pieces": [" ", " ​'", "re", " \n", "عt𐞁'D", "ta", " \n", "åé'D", "漢", "👍🏽", "så", "㍿́DžDžt", " ", " Z'S", "<", " "]} +{"text": "<|fim_prefix|>…'s.𐞁عfi字 >\r\n're", "tokens": 23, "pieces": ["<|", "fim", "_prefix", "|>", "…", "'s", ".𐞁عfi字", "", " ", " >\r\n", "'re"]} +{"text": "'VE9👍🏽", "tokens": 6, "pieces": ["'VE", "9", "👍🏽"]} +{"text": "Ⅳfi(\"'D\r\nİ>Ａ漢😀🏽'S　.mEOTZ
……!!9字ḍ̇𐞁<ꟲ\r\n<", "tokens": 48, "pieces": ["Ⅳ", "fi", "(\"'", "D", "\r\n", "İ", ">Ａ漢", "😀🏽'", "S", "　", ".m", "EOTZ", "
", "", "…", "…", "!!", "9", "字ḍ̇𐞁", "<ꟲ", "\r\n", "<"]} +{"text": "'M#$%٣٤٥٦(#$%.m㍿㋿9#$%'T㋿\r\n\r\n!!Ａḍ̇", "tokens": 30, "pieces": ["'M", "#$%", "٣٤٥", "٦", "(#$%.", "m", "㍿㋿", "9", "#$%'", "T", "㋿\r\n\r\n", "!!", "Ａḍ̇"]} +{"text": "'s<|fim_prefix|>'ReZ😀🏽㋿漢s𐞁Ⅳḍ̇t İ ́e👍🏽…ḍ̇fi \n　漢ß'VE", "tokens": 47, "pieces": ["'s", "<|", "fim", "_prefix", "|>'", "Re", "Z", "😀🏽㋿", "漢s𐞁", "Ⅳ", "ḍ̇t", " ", " İ", " ́e", "👍🏽", "…ḍ̇fi", " \n", "　漢ß'VE"]} +{"text": "'re'Dİ字9​.🙂​㋿\t-
İé12345678‍
<|endoftext|>㍿12345678👍🏽\r㍿Zfi'VE>　é", "tokens": 52, "pieces": ["'re'D", "İ字", "9", "​.🙂​㋿", "\t", "-", "
İé", "123", "456", "78", "‍", "
", "<|", "endoftext", "|>㍿", "123", "456", "78", "👍🏽\r", "㍿Zfi'VE", ">", "　é", ""]} +{"text": "0 \n­  …'re漢­'T\r٣٤٥٦9s9d. 're\n'sⅣ\r\n'T<|fim_prefix|>", "tokens": 41, "pieces": ["0", " \n", "­", "  ", "…", "'re漢", "­'", "T", "\r", "٣٤٥", "٦9", "s", "", "9", "d", ".", " '", "re", "\n", "'s", "Ⅳ", "\r\n", "'T", "<|", "fim", "_prefix", "|>"]} +{"text": "🙂㋿'re字'll,s\r½　ḍ̇𐞁漢'D<|fim_prefix|>٣٤٥٦٣٤٥٦'re'Re,👍🏽!!­\r9", "tokens": 48, "pieces": ["🙂㋿'", "re字'll", ",<", "EOT", ">s", "\r", "½", "　ḍ̇𐞁漢'D", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦٣٤", "٥٦", "'re'Re", ",👍🏽!!­\r", "9"]} +{"text": "'VE", "tokens": 2, "pieces": ["'VE"]} +{"text": "
ß 's'Re#$% \r\n㍿́m 'M12345678!!
🙂<|fim_prefix|>ß>é'M'D!!DžݽEOT", "tokens": 30, "pieces": ["!ß", "🙂", "٣٤٥", "٦", "!", "
", "'T'M", "ß", ">é'M", "'D", "!!", "Džİ", "", "½", "EOT"]} +{"text": "mZas　!Ⅳ>
'S㍿0\r­ꟲ,#$%漢'T'll
-½'ſ'D!!Zté\"漢9字字's
½", "tokens": 42, "pieces": ["m", "Zas", "　", "!", "Ⅳ", ">", "
", "'S", "㍿", "0", "\r", "­ꟲ", ",#$%", "漢'T", "'ll", "
", "-", "½", "'ſ'D", "!!", "Zté", "\"漢", "9", "字字's", "
", "½"]} +{"text": "<'Re­ 'T㋿'VE0 \nEOT\n　mééİ.\r\n\r\né<|endoftext|>å<|endoftext|>🙂!12345678\"İ'S-Dž", "tokens": 49, "pieces": ["<'", "Re", "­", " ", " '", "T", "㋿'", "VE", "0", " \n", "EOT", "\n", "　méé", "İ", ".\r\n\r\n", "é", "<|", "endoftext", "|>", "å", "<|", "endoftext", "|>🙂!", "123", "456", "78", "\"İ'S", "-Dž"]} +{"text": "­'Tå'D're\r\n\r\n-é,12345678.<,٣٤٥٦ſ٣٤٥٦\t́\"'VE½m\u000bꟲ", "tokens": 40, "pieces": ["­'", "T", "å'D", "'re", "\r\n\r\n", "-é", ",", "123", "456", "78", ".<,", "٣٤٥", "٦", "ſ", "", "٣٤٥", "٦", "\t́", "\"'", "VE", "½", "m", "\u000bꟲ"]} +{"text": "'M🙂12345678e\r\n9,12345678İ'reعＡ", "tokens": 16, "pieces": ["'M", "🙂", "123", "456", "78", "e", "\r\n", "9", ",", "123", "456", "78", "İ're", "ع", "Ａ"]} +{"text": " \"-m'३㍿字🙂 \tꟲ‍ḍ̇éḍ̇#$%­å,'s", "tokens": 37, "pieces": [" ", " \"-", "m", "'<", "EOT", ">", "३", "㍿字", "🙂", " ", "\tꟲ", "‍", "ḍ̇éḍ̇", "#$%­", "å", ",'", "s"]} +{"text": "㍿\rEOTḍ̇'S ßİ​d,½٣٤٥٦Ⅳ' ٣٤٥٦漢'Dé　'S𐞁ḍ̇mfi\u000b\rm'Re ½<|fim_prefix|><|endoftext|>> \n'", "tokens": 71, "pieces": ["㍿\r", "EOTḍ̇'S", " ß", "İ", "​d", ",<", "EOT", ">", "½٣٤", "٥٦Ⅳ", "'", " ", " ", "٣٤٥", "٦", "漢'D", "é", "　", "'S𐞁ḍ̇mfi", "\u000b\r", "m'Re", " ", " ", "½", "<|", "fim", "_prefix", "|><|", "endoftext", "|>>", " \n", "'"]} +{"text": "\r\n\r\né<　(ḍ̇ ", "tokens": 10, "pieces": ["\r\n\r\n", "é", "<", "　", "(ḍ̇", " "]} +{"text": "t,!㍿😀🏽éé", "tokens": 20, "pieces": ["t", "Ⅳ", "İ𐞁'T", "Ａ", ".🙂'", "ll", "Ⅳ", "́", ">éé"]} +{"text": "é́٣٤٥٦Džmſ\r𐞁<|fim_prefix|>­s\r\n9ß \n å'Re…", "tokens": 32, "pieces": ["é́", "٣٤٥", "٦", "Džmſ", "\r", "𐞁", "<|", "fim", "_prefix", "|>­", "s", "\r\n", "9", "ß", " \n", " å'Re", "…"]} +{"text": "!!­d(ſe<|endoftext|>Ⅳſ'D\r\n\r\n#$%dd漢㋿(ع(漢EOT.'s½字", "tokens": 35, "pieces": ["!!­", "d", "(ſe", "<|", "endoftext", "|>", "Ⅳ", "ſ'D", "\r\n\r\n", "#$%", "dd漢", "㋿(", "ع", "(漢", "EOT", ".'", "s", "½", "字"]} +{"text": "ꟲ'Re fi漢!!å>'D", "tokens": 16, "pieces": ["ꟲ'Re", " ", " fi漢", "!!", "å", ">'", "D"]} +{"text": "12345678­ꟲ!! 漢́'reéZ's", "tokens": 20, "pieces": ["123", "456", "78", "­ꟲ", "!!", " 漢́'re", "é", "Z's", ""]} +{"text": "́éé.㋿İé३​><…漢­🙂'VE\t \n\n!Ⅳ", "tokens": 27, "pieces": ["́éé", ".㋿", "İé", "३", "​><", "…漢", "­🙂'", "VE", "\t \n\n", "!", "Ⅳ"]} +{"text": "eⅣ😀🏽'D're३'s's9-0!", "tokens": 16, "pieces": ["e", "Ⅳ", "😀🏽'", "D're", "३", "'s's", "9", "-", "0", "!"]} +{"text": "0 \n(½å<|fim_prefix|> m عd'llé!Z字EOTå😀🏽㍿é.İ　Ａ \nfi
.'ßfi​Z'VEfi \n", "tokens": 50, "pieces": ["0", " \n", "(", "½", "å", "<|", "fim", "_prefix", "|>", " m", " عd'll", "é", "!Z字EOTå", "😀🏽㍿", "é", ".İ", "　Ａ", " \n", "fi", "
", ".'", "ßfi", "​Z'VE", "fi", " \n"]} +{"text": "<|endoftext|> \n…Ａİ‍ ,", "tokens": 15, "pieces": ["<|", "endoftext", "|>", " \n", "…Ａİ", "‍", " ", " ,"]} +{"text": "a'S!", "tokens": 3, "pieces": ["a'S", "!"]} +{"text": "👍🏽\r\n\r\n<|fim_prefix|>字'M.ḍ̇ꟲ", "tokens": 19, "pieces": ["👍🏽\r\n\r\n", "<|", "fim", "_prefix", "|>", "字'M", ".ḍ̇ꟲ"]} +{"text": " ½字\nm\r\n\r\n'T<>>mſaå㋿ 'T!𐞁ع<|endoftext|>\r\n\r\na're,123456789'S \n-", "tokens": 45, "pieces": ["", " ", "½", "字", "\n", "m", "\r\n\r\n", "'T", "<>>", "mſaå", "㋿", " ", "'T", "!𐞁ع", "<|", "endoftext", "|>\r\n\r\n", "a're", ",", "123", "456", "789", "'S", " \n", "-"]} +{"text": "'Re<|endoftext|>Dž\n٣٤٥٦ḍ̇ta!fi…a٣٤٥٦　-Ⅳ😀🏽é\n#$%漢👍🏽  　😀🏽'T'Mſ'ſé", "tokens": 60, "pieces": ["'Re", "<|", "endoftext", "|>", "Dž", "\n", "٣٤٥", "٦", "ḍ̇ta", "!fi", "…a", "٣٤٥", "٦", "　", "-", "Ⅳ", "😀🏽", "é", "\n", "#$%", "漢", "👍🏽", "  ", "　", "😀🏽'", "T", "'", "Mſ'ſ", "é"]} +{"text": "́㍿'Re9d#$%#$%ꟲＡꟲ<|fim_prefix|>\r\n\n<|endoftext|>EOTZ㍿m", "tokens": 40, "pieces": ["́", "㍿'", "Re", "9", "d", "#$%#$%", "ꟲＡꟲ", "<|", "fim", "_prefix", "|>\r\n\n", "<|", "endoftext", "|>", "EOTZ", "㍿m"]} +{"text": "'Tt㍿😀🏽efi'T12345678sİ 'S​(​'S'T", "tokens": 24, "pieces": ["'Tt", "㍿😀🏽", "efi'T", "123", "456", "78", "s", "İ", " '", "S", "​(​'", "S'T"]} +{"text": "9'<ⅣⅣ 0'll　d'ſDž", "tokens": 19, "pieces": ["9", "'<", "ⅣⅣ", " ", " ", "0", "'", "ll", "　d'ſ", "Dž"]} +{"text": "\r\n.d<|fim_prefix|>字­,ꟲꟲ!EOT's", "tokens": 21, "pieces": ["\r\n", ".d", "<|", "fim", "_prefix", "|>", "字", "­,", "ꟲꟲ", "!EOT's"]} +{"text": "ſ.'ſ<|endoftext|>👍🏽EOT👍🏽Ａ Ⅳ👍🏽9'ſ　å…\"m'M>\u000b.<'VEm", "tokens": 47, "pieces": ["ſ", ".'", "ſ", "<|", "endoftext", "|>👍🏽", "EOT", "👍🏽", "Ａ", " ", "Ⅳ", "👍🏽", "9", "'ſ", "　", "<", "EOT", ">å", "…", "\"m'M", ">", "\u000b", ".<'", "VEm"]} +{"text": " 'll12345678'D\u000bſ\t'Re𐞁…㍿e'ſ­EOT𐞁.Ａ'ReEOT\n", "tokens": 40, "pieces": [" ", " '", "ll", "123", "456", "78", "'", "D", "\u000bſ", "\t", "'Re𐞁", "…", "㍿e'ſ", "­EOT𐞁", ".Ａ'Re", "EOT", "\n"]} +{"text": "s३漢㍿\r\n\r\nꟲع
DžEOT\"ſ३ḍ̇mDž", "tokens": 28, "pieces": ["s", "३", "漢", "㍿\r\n\r\n", "ꟲع", "
DžEOT", "\"ſ", "३", "ḍ̇m", "Dž"]} +{"text": " ſ…'MåDžḍ̇\t'Re,<|endoftext|>ⅣＡ字 \n İfi.\tétſsꟲ<|endoftext|>㍿>ꟲ", "tokens": 53, "pieces": [" ", " ſ", "…", "'Må", "Džḍ̇", "\t", "'Re", ",<|", "endoftext", "|>", "Ⅳ", "Ａ字", " \n", " ", " İfi", ".", "\tétſsꟲ", "<|", "endoftext", "|>㍿>", "ꟲ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "<|endoftext|> ", "tokens": 8, "pieces": ["<|", "endoftext", "|>", " "]} +{"text": "12345678're'½'ſ😀🏽漢‍(ſ m𐞁漢->Ⅳ😀🏽'ſ(字\u000bm9Džعḍ̇漢ḍ̇ \n½ß ", "tokens": 48, "pieces": ["123", "456", "78", "'re", "'", "½", "'ſ", "😀🏽", "漢", "‍(", "ſ", " m𐞁漢", "->", "Ⅳ", "😀🏽'", "ſ", "(字", "\u000bm", "9", "Džعḍ̇漢ḍ̇", " \n", "½", "ß", " "]} +{"text": "'Re -e𐞁EOT9'VE<|endoftext|>\r'D㋿e!\r\n😀🏽‍́ḍ̇…ع字é<|fim_prefix|>'ſ", "tokens": 54, "pieces": ["'Re", " ", "-e", "<", "META", "_START", ">𐞁", "EOT", "9", "'VE", "<|", "endoftext", "|>\r", "'D", "㋿e", "!\r\n", "😀🏽‍́", "ḍ̇", "…ع字é", "<|", "fim", "_prefix", "|>'", "ſ"]} +{"text": "<|endoftext|>>a#$%dZEOT m\naع", "tokens": 17, "pieces": ["<|", "endoftext", "|>>", "a", "#$%", "d", "ZEOT", " m", "\n", "aع"]} +{"text": "🙂‍DžDž㍿字\u000b's字'é're", "tokens": 16, "pieces": ["🙂‍", "DžDž", "㍿字", "\u000b", "'s字", "'é're"]} +{"text": "ع,<|fim_prefix|>'e \n>ḍ̇\u000bd'D12345678're a#$%…Dž𐞁\nå㍿\u000bⅣ!!\"½\n\"​#$%​Ⅳ½́́t", "tokens": 56, "pieces": ["ع", ",<|", "fim", "_prefix", "|>'", "e", " \n", ">ḍ̇", "\u000bd'D", "123", "456", "78", "'re", " ", " a", "#$%", "…Dž𐞁", "\n", "å", "㍿", "\u000b", "Ⅳ", "!!\"", "½", "\n", "\"​#$%​", "Ⅳ½", "́́t"]} +{"text": "-.'Sfi🙂9İ­#$%Ⅳ9\"m\r'll३s\r\n\r\nfiZe३'ll", "tokens": 32, "pieces": ["-<", "META", "_START", ">.'", "Sfi", "🙂", "9", "İ", "­#$%", "Ⅳ9", "\"m", "\r", "'", "ll", "३", "s", "\r\n\r\n", "fi", "Ze", "३", "'ll"]} +{"text": "​\rＡ(٣٤٥٦d👍🏽<ع'DعＡé'Re\r\n\r\n \t'ſt#$%Z\"Dž‍.'Sa\r　𐞁 \n\r\n", "tokens": 43, "pieces": ["​\r", "Ａ", "(", "٣٤٥", "٦", "d", "👍🏽<", "ع'D", "عＡé'Re", "\r\n\r\n", " ", "\t", "'ſt", "#$%", "Z", "\"Dž", "‍.'", "Sa", "\r", "　𐞁", " \n\r\n"]} +{"text": "ꟲ,'M", "tokens": 5, "pieces": ["ꟲ", ",'", "M"]} +{"text": "㋿　fi,ع'll12345678'reé\r\n,\ns'", "tokens": 18, "pieces": ["㋿", "　fi", ",ع'll", "123", "456", "78", "'reé", "\r\n", ",\n", "s", "'"]} +{"text": "'re-Ⅳ\"ḍ̇ع!!s", "tokens": 11, "pieces": ["'re", "-", "Ⅳ", "\"ḍ̇ع", "!!", "s"]} +{"text": "'lléꟲ\n12345678ḍ̇Ⅳ0㍿'VE'D12345678e㍿ß㋿'sß-½s\u000b\t'll…#$%\r Z'TtEOT 𐞁,ꟲd", "tokens": 65, "pieces": ["'lléꟲ", "\n", "123", "456", "78", "ḍ̇", "Ⅳ0", "㍿'", "VE'D", "123", "456", "78", "e", "㍿ß", "㋿'", "sß", "-", "½", "s", "\u000b", "\t", "'ll", "…", "#$%\r", " <", "EOT", ">Z'T", "t", "EOT", " 𐞁", ",ꟲd"]} +{"text": "\u000b
åZ‍ \r\n\r\n\r\n\r\nEOT \n‍tع…'Re\r\n!ⅣſⅣ!!😀🏽9<(\r\n>'​fi\u000b🙂", "tokens": 36, "pieces": ["\u000b", "
å", "Z", "‍", " \r\n\r\n\r\n\r\n", "EOT", " \n", "‍tع", "…", "'Re", "\r\n", "!", "Ⅳ", "ſ", "Ⅳ", "!!😀🏽", "9", "<(\r\n", ">'​", "fi", "\u000b", "🙂"]} +{"text": " …३\r'M
!漢- 
12345678t<|endoftext|>
'D,\na'Red ٣٤٥٦漢\"é㋿ s‍>\r\n\r\ns9DžZ½😀🏽", "tokens": 53, "pieces": [" ", "…", "३", "\r", "'M", "
", "!漢", "-", " ", "
", "123", "456", "78", "t", "<|", "endoftext", "|>", "
", "'D", ",\n", "a'Re", "d", " ", "٣٤٥", "٦", "漢", "\"é", "㋿", " s", "‍>\r\n\r\n", "s", "9", "DžZ", "½", "😀🏽"]} +{"text": "'ll𐞁\r…'ll!Ⅳ-<|endoftext|>ß \r\n\r\n\r\n\r\nét㋿\n'Tt12345678", "tokens": 35, "pieces": ["'ll𐞁", "\r", "…", "'ll", "!", "Ⅳ", "-<|", "endoftext", "|>", "ß", " \r\n\r\n\r\n\r\n", "ét", "㋿\n", "'Tt", "123", "456", "78"]} +{"text": "'s\u000b-'s‍EOT#$%ſ\u000bßEOT\u000b(åعfi", "tokens": 19, "pieces": ["'s", "\u000b", "-'", "s", "‍EOT", "#$%", "ſ", "\u000bß", "EOT", "\u000b", "(åعfi"]} +{"text": "\t< '漢e漢12345678'Re'D'D912345678\u000b éad　ḍ̇İmm!!12345678३㋿\r\n", "tokens": 36, "pieces": ["\t", "<", " ", " '", "漢e漢", "123", "456", "78", "'Re'D", "'D", "912", "345", "678", "\u000b ", " éad", "　ḍ̇", "İmm", "!!", "123", "456", "78३", "㋿\r\n"]} +{"text": "Z09٣٤٥٦0é'T're 'T", "tokens": 12, "pieces": ["Z", "09٣", "٤٥٦", "0", "é'T", "'re", " ", "'T"]} +{"text": "㍿٣٤٥٦\nt㍿字\r\n\r\n漢٣٤٥٦\n<'re‍>\t㍿㍿Ⅳ\r\n!'VE.9e'.ع", "tokens": 41, "pieces": ["㍿", "٣٤٥", "٦", "\n", "t", "㍿字", "\r\n\r\n", "漢", "٣٤٥", "٦", "\n", "<'", "re", "‍>", "\t", "㍿㍿", "Ⅳ", "\r\n", "!'", "VE", ".", "9", "e", "'.", "ع"]} +{"text": "!! ‍é½३Ａ'VEſ<|fim_prefix|>ع​é😀🏽'T\r<|fim_prefix|> d!!a'll٣٤٥٦½Ａ<字s😀🏽#$%d½㋿e#$%", "tokens": 62, "pieces": ["!!", " ", "‍é", "½३", "Ａ'VE", "ſ", "<|", "fim", "_prefix", "|>", "ع", "​é", "😀🏽'", "T", "\r", "<|", "fim", "_prefix", "|>", " d", "!!", "a", "'", "ll", "٣٤٥", "٦½", "Ａ", "<字s", "😀🏽#$%", "d", "½", "㋿e", "#$%"]} +{"text": "ع​Ａ(t12345678>'S'T😀🏽a'Re٣٤٥٦\r\nd", "tokens": 28, "pieces": ["ع", "​Ａ", "(<", "META", "_START", ">t", "123", "456", "78", ">'", "S'T", "😀🏽", "a'Re", "٣٤٥", "٦", "\r\n", "d"]} +{"text": "!!\"
#$%\"'lléß
ß　12345678->t👍🏽\r\n\r\n're漢　‍<|fim_prefix|>d0 \n 0's㋿e#$%0😀🏽", "tokens": 55, "pieces": ["!!\"", "
", "#$%\"'", "lléß", "
ß", "　", "123", "456", "78", "->", "t", "👍🏽\r\n\r\n", "'re漢", "　", "‍<|", "fim", "_prefix", "|>", "d", "0", " \n", " ", "0", "'s", "㋿e", "#$%<", "META", "_START", ">", "0", "😀🏽"]} +{"text": "'Da<३éſd'Ret'Re𐞁字字'DEOT\r\n\r\nEOT\"½'S­ -\r.tå\r\n\r\n​​ع㋿😀🏽", "tokens": 41, "pieces": ["'Da", "<", "३", "éſd'Re", "t'Re", "𐞁字字'D", "EOT", "\r\n\r\n", "EOT", "\"", "½", "'S", "­", " ", "-\r", ".tå", "\r\n\r\n", "​​", "ع", "㋿😀🏽"]} +{"text": "\r\n'llＡ", "tokens": 3, "pieces": ["\r\n", "'ll", "Ａ"]} +{"text": ">🙂\"'Rea", "tokens": 5, "pieces": [">🙂\"'", "Rea"]} +{"text": "'D'ſ'٣٤٥٦ <\u000b漢㋿0'llＡ.\t'D🙂'T٣٤٥٦ ٣٤٥٦-<|endoftext|>!! <|fim_prefix|><|endoftext|>字Džé㍿ع\r\nd\u000b're㍿​ḍ̇", "tokens": 76, "pieces": ["'D'ſ", "'", "٣٤٥", "٦", " ", " <", "\u000b漢", "㋿", "0", "'ll", "Ａ", ".", "\t", "'D", "🙂'", "T", "٣٤٥", "٦", " ", " ", "٣٤٥", "٦", "-<|", "endoftext", "|>!!", " ", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "字Džé", "㍿ع", "\r\n", "d", "\u000b", "'re", "㍿​", "ḍ̇"]} +{"text": "'　字'D٣٤٥٦dſZ \u000b­　字é'reéé
ḍ̇fi😀🏽İḍ̇é‍३md𐞁<|endoftext|>­👍🏽​'VE'‍\r", "tokens": 61, "pieces": ["'", "　字'D", "٣٤٥", "٦", "dſ", "Z", " ", "\u000b", "­", "　字é're", "éé", "
", "ḍ̇fi", "😀🏽", "İḍ̇é", "‍", "३", "md𐞁", "<|", "endoftext", "|>­👍🏽​'", "VE", "'‍\r"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "😀🏽 ㍿'ſ😀🏽👍🏽ḍ̇'Dİ'S字\r9😀🏽漢字fi<|endoftext|>s(\t­\r(", "tokens": 43, "pieces": ["😀🏽", " ", "㍿'", "ſ", "😀🏽👍🏽", "ḍ̇'D", "İ'S", "字", "\r", "9", "😀🏽", "漢字fi", "<|", "endoftext", "|>", "s", "(", "\t", "­\r", "("]} +{"text": "åm½s\"'Re'D字\r\n''ſ字ع0s12345678😀🏽Ａ\u000b<|endoftext|>\"
EOT're", "tokens": 35, "pieces": ["åm", "½", "s", "\"'", "Re'D", "字", "\r\n", "''", "ſ字ع", "0", "s", "123", "456", "78", "😀🏽", "Ａ", "\u000b", "<|", "endoftext", "|>\"", "
EOT're"]} +{"text": " \ń👍🏽1234567812345678", "tokens": 11, "pieces": [" \n", "́", "👍🏽", "123", "456", "781", "234", "567", "8"]} +{"text": "\r\"\"é\r\n\r\n#$%😀🏽-𐞁…\r\n de…t­s>EOT!!", "tokens": 27, "pieces": ["\r", "\"\"", "é", "\r\n\r\n", "#$%😀🏽-", "𐞁", "…\r\n", " ", " de", "…t", "­s", ">EOT", "!!"]} +{"text": "́ḍ̇", "tokens": 4, "pieces": ["́ḍ̇"]} +{"text": "''字9ꟲ 'VE\u000b😀🏽\n'ſ!0Zß٣٤٥٦m　😀🏽<<|fim_prefix|>字å 'Re\r\n३'T'D<", "字å", " ", " '", "Re", "\r\n", "३", "'T'D", "<<", "d", "👍🏽", "İ", "…", "("]} +{"text": "ع<", "tokens": 2, "pieces": ["ع", "<"]} +{"text": "
\t\u000bḍ̇'\"", "tokens": 7, "pieces": ["
\t", "\u000bḍ̇", "'\""]} +{"text": "\"'M३ds😀🏽'T​\r\n.\t d🙂٣٤٥٦
३'Ss\r\n\r\nfi'T\r\n\r\n\u000b", "tokens": 28, "pieces": ["\"'", "M", "३", "ds", "😀🏽'", "T", "​\r\n", ".", "\t ", " d", "🙂", "٣٤٥", "٦", "
", "३", "'Ss", "\r\n\r\n", "fi'T", "\r\n\r\n", "\u000b"]} +{"text": "0‍'SZ(12345678㋿🙂'D12345678
\"ⅣEOT'T\r\n\r\n字<|endoftext|>#$%<|endoftext|>'D\u000b<|fim_prefix|>​　Ａ'Refi­Dž0'<ſfi-,🙂", "tokens": 64, "pieces": ["0", "‍'", "SZ", "(", "123", "456", "78", "㋿🙂'", "D", "123", "456", "78", "
", "\"", "Ⅳ", "EOT'T", "\r\n\r\n", "字", "<|", "endoftext", "|>#$%<|", "endoftext", "|>'", "D", "\u000b", "<|", "fim", "_prefix", "|>​", "　Ａ'Re", "fi", "­Dž", "0", "'<", "ſfi", "-,🙂"]} +{"text": "< \"'Re<|fim_prefix|>", "tokens": 15, "pieces": ["<", " ", "\"'", "Re", "<|", "fim", "_prefix", "|><", "META", "_START", ">"]} +{"text": "½\u000b'ſＡDž.\r­><|fim_prefix|>!!½s　ß🙂", "tokens": 22, "pieces": ["½", "\u000b", "'ſ", "ＡDž", ".\r", "­><|", "fim", "_prefix", "|>!!", "½", "s", "　ß", "🙂"]} +{"text": "0.३é!90🙂d9'VE​字 t.'S३😀🏽0ß字<\rＡ٣٤٥٦🙂字m<<|fim_prefix|>👍🏽", "tokens": 48, "pieces": ["0", ".", "३", "é", "!", "90", "🙂d", "9", "'VE", "​字", " t", ".'", "S", "३", "😀🏽", "0", "ß字", "<\r", "Ａ", "٣٤٥", "٦", "🙂字m", "<<|", "fim", "_prefix", "|>👍🏽"]} +{"text": "ḍ̇漢'Dé​#$%㋿'re''re<>\r\né😀🏽12345678", "tokens": 27, "pieces": ["ḍ̇漢'D", "é", "​#$%㋿'", "re", "''", "re", "<>\r\n", "é", "😀🏽", "123", "456", "78"]} +{"text": "'ll३-té½e'…\n​ß\u000bⅣ😀🏽Ⅳ'M9!३'", "tokens": 25, "pieces": ["'ll", "३", "-té", "½", "e", "'", "…\n", "​ß", "\u000b", "Ⅳ", "😀🏽", "Ⅳ", "'M", "9", "!", "३", "'"]} +{"text": "'llع😀🏽 -'s Džع99( 'Re9até9٣٤٥٦㍿漢->\"EOTéEOT½!!ßa12345678​
'VE'Re", "tokens": 51, "pieces": ["'llع", "😀🏽", " -'", "s", " Džع", "99", "(", " '", "Re", "9", "até", "9٣٤", "٥٦", "㍿漢", "->\"", "EOTé", "EOT", "½", "!!", "ßa", "123", "456", "78", "​", "
", "'VE'Re"]} +{"text": "ع'VE字", "tokens": 4, "pieces": ["ع'VE", "字"]} +{"text": "٣٤٥٦𐞁12345678'…​'VE𐞁(ß", "tokens": 26, "pieces": ["٣٤٥", "٦", "𐞁", "123", "456", "78", "'<", "META", "_START", ">", "…", "​'", "VE𐞁", "(ß"]} +{"text": "12345678字(İ 字😀🏽३\u000bm𐞁<|fim_prefix|>👍🏽 \"fi'T'Msꟲ́½​\"漢fi\u000b<|endoftext|>㍿fi", "tokens": 57, "pieces": ["123", "456", "78", "字", "(İ", " ", " 字", "😀🏽", "३", "\u000bm𐞁", "<|", "fim", "_prefix", "|>👍🏽", " ", " \"", "fi'T", "'Msꟲ́", "½", "​\"<", "EOT", ">漢fi", "\u000b", "<|", "endoftext", "|>㍿", "fi"]} +{"text": "㍿12345678­ꟲ'VE'DꟲDž'Sḍ̇ḍ̇.'T'\r\n\r\n'll\rDž \n'llZåſ<|endoftext|>𐞁.", "tokens": 49, "pieces": ["㍿", "123", "456", "78", "­ꟲ'VE", "'Dꟲ", "Dž'S", "ḍ̇ḍ̇", ".'", "T", "'\r\n\r\n", "'ll", "\r", "Dž", " \n", "'ll", "Zåſ", "<|", "endoftext", "|>", "𐞁", "."]} +{"text": "<ꟲ३é<|fim_prefix|>0(​😀🏽", "tokens": 19, "pieces": ["<ꟲ", "३", "é", "<|", "fim", "_prefix", "|>", "0", "(​😀🏽"]} +{"text": " 's\"'ll\r'M… 'D'\t­　‍Ａ㋿\r\n\r\n \n.", "tokens": 27, "pieces": [" '", "s", "\"'", "ll", "\r", "'M", "… ", " '", "D", "'", "\t", "­", "　", "‍Ａ", "㋿\r\n\r\n", " \n", "."]} +{"text": " 'ReEOTİ́é­s𐞁'Dع٣٤٥٦'M0½㋿\n'VEa", "tokens": 28, "pieces": [" '", "Re", "EOTİ́é", "­s𐞁'D", "ع", "٣٤٥", "٦", "'M", "0½", "㋿\n", "'VEa"]} +{"text": "­\"Ａ", "tokens": 3, "pieces": ["­\"", "Ａ"]} +{"text": "!!\r\na㋿'reéDž\ré!!!!t字tḍ̇!!'DDž .\r\u000be漢,", "tokens": 40, "pieces": ["!!\r\n", "a", "㋿'", "reé", "Dž", "\r", "é", "!!!!", "t字tḍ̇", "!!<", "m", "'", "DDž", " ", ".\r", "\u000be漢", ","]} +{"text": " 12345678!((\rm‍e ½ſ'VE 'S३\t‍.9<", "tokens": 23, "pieces": [" ", "123", "456", "78", "!((\r", "m", "‍e", " ", "½", "ſ'VE", " ", " '", "S", "३", "\t", "‍.", "9", "<"]} +{"text": "!<|endoftext|>12345678\ns½0<\"٣٤٥٦​
Ａ㍿ \n
İ字é​", "tokens": 33, "pieces": ["!<|", "endoftext", "|>", "123", "456", "78", "\n", "s", "½0", "<\"", "٣٤٥", "٦", "​", "
Ａ", "㍿", " \n", "
İ字é", "​"]} +{"text": "🙂ꟲ\u000bt\n
​㋿㍿.\r\"​12345678㋿字e\"'ſ漢'T\t!ſ​
're<…​", "tokens": 50, "pieces": ["🙂ꟲ", "", "\u000bt", "\n", "
", "​㋿㍿.\r", "\"​", "123", "456", "78", "㋿字e", "\"'", "ſ漢'T", "\t", "!ſ", "​", "
", "'re", "<", "…", "​"]} +{"text": "ßEOT½ 0½عZ　'‍‍'VEé\u000b", "tokens": 18, "pieces": ["ß", "EOT", "½", " ", "0½", "ع", "Z", "　", "'‍‍'", "VEé", "\u000b"]} +{"text": "åİa\t字9㋿9…<|fim_prefix|>,　‍'re…\t'12345678-'D<|fim_prefix|>漢 \n!'M　'D\t字…åe", "tokens": 51, "pieces": ["å", "İa", "\t字", "9", "㋿", "9", "…", "<|", "fim", "_prefix", "|>,", "　", "‍'", "re", "…", "\t", "'", "123", "456", "78", "-'", "D", "<|", "fim", "_prefix", "|>", "漢", " \n", "!'", "M", "　", "'D", "\t字", "…åe"]} +{"text": "ḍ̇\nḍ̇​٣٤٥٦", "tokens": 12, "pieces": ["ḍ̇", "\n", "ḍ̇", "​", "٣٤٥", "٦"]} +{"text": "fi ,ß\"İZ09\u000b \né\"'Reİꟲ\"ß'T́!!('ree-'S'reḍ̇Z
'VE", "tokens": 35, "pieces": ["fi", " ", " ,", "ß", "\"İZ", "09", "\u000b \n", "é", "\"'", "Re", "İꟲ", "\"ß'T", "́", "!!('", "ree", "-'", "S're", "ḍ̇", "Z", "
", "'VE"]} +{"text": "漢ꟲ🙂İ👍🏽\r\n\r\n字' Ａ😀🏽(ꟲ𐞁<|endoftext|>ＡZéa👍🏽٣٤٥٦\r", "tokens": 48, "pieces": ["漢ꟲ", "🙂İ", "👍🏽\r\n\r\n", "字", "'", " Ａ", "😀🏽(", "ꟲ𐞁", "<|", "endoftext", "|>", "ＡZéa", "👍🏽", "٣٤٥", "٦", "\r"]} +{"text": "12345678​
\n‍­9½­'re<|endoftext|>İ👍🏽fi'ſé½'ſ​\u000b㋿0
字é<|endoftext|>d३0Ａ㍿­٣٤٥٦12345678>EOT-🙂", "tokens": 67, "pieces": ["123", "456", "78", "​", "
\n", "‍­", "9½", "­'", "re", "<|", "endoftext", "|>", "İ", "👍🏽", "fi'ſ", "é", "½", "'ſ", "​", "\u000b", "㋿", "0", "
字é", "<|", "endoftext", "|>", "d", "३0", "Ａ", "㍿­", "٣٤٥", "٦12", "345", "678", ">EOT", "-🙂"]} +{"text": "عt\"é<|fim_prefix|>Dž!!é㋿'VEé½\n-٣٤٥٦", "tokens": 30, "pieces": ["عt", "\"é", "<|", "fim", "_prefix", "|>", "Dž", "!!", "é", "㋿'", "VEé", "½", "\n", "-", "٣٤٥", "٦"]} +{"text": "İ㋿'T'VE'SⅣ0EOT٣٤٥٦,.٣٤٥٦Ａ½", "tokens": 25, "pieces": ["İ", "㋿'", "T'VE", "'S", "Ⅳ0", "EOT", "٣٤٥", "٦", ",.", "٣٤٥", "٦", "Ａ", "½"]} +{"text": "ße'D३Z‍", "tokens": 5, "pieces": ["ße'D", "३", "Z", "‍"]} +{"text": "é<|fim_prefix|>9dåſ!!Z ", "tokens": 15, "pieces": ["é", "<|", "fim", "_prefix", "|>", "9", "dåſ", "!!", "Z", " "]} +{"text": " 'SEOTDž\r\n\u000b
‍ع😀🏽\r\nع'sḍ̇å
", "tokens": 25, "pieces": [" '", "SEOTDž", "\r\n", "\u000b", "
", "‍ع", "😀🏽\r\n", "ع", "'", "sḍ̇å", "
"]} +{"text": "𐞁㋿ḍ̇\n　½ſḍ̇m漢", "tokens": 22, "pieces": ["𐞁", "㋿ḍ̇", "\n", "　", "½", "ſḍ̇m漢", ""]} +{"text": "'T३!!9'T'MⅣ\r\n\r\n🙂½.ꟲ-Z…'s0漢عع\r'Re(ع'll123456780㋿(<|endoftext|>\r\n३é", "tokens": 43, "pieces": ["'T", "३", "!!", "9", "'T'M", "Ⅳ", "\r\n\r\n", "🙂", "½", ".ꟲ", "-Z", "…", "'s", "0", "漢عع", "\r", "'Re", "(ع'll", "123", "456", "780", "㋿(<|", "endoftext", "|>\r\n", "३", "é"]} +{"text": "(
\"
>", "tokens": 5, "pieces": ["(", "
", "\"", "
", ">"]} +{"text": "漢́\r\n\r\n\t́字字a 'Dé", "tokens": 15, "pieces": ["漢́", "\r\n\r\n", "\t", "́字字a", " ", " '", "Dé"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "　Dž 'refi\r'\u000b<|endoftext|>Z\t0", "tokens": 19, "pieces": ["　Dž", " '", "refi", "\r", "'", "\u000b", "<|", "endoftext", "|>", "Z", "\t", "0"]} +{"text": "'Re㋿İ-éEOTＡſa‍字\"㍿…İع​'VE'VE㋿Ａḍ̇ꟲ 's.👍🏽>\r\n\r\n३ \n\r\n…12345678ad \n㍿", "tokens": 60, "pieces": ["'", "Re", "㋿İ", "-é", "EOTＡſa", "‍字", "\"㍿", "…İع", "​'", "VE'VE", "㋿Ａḍ̇ꟲ", " '", "s", ".👍🏽>\r\n\r\n", "३", " \n\r\n", "…", "123", "456", "78", "ad", " \n", "㍿"]} +{"text": "\",\r\n\r\nḍ̇\r\n\r\n<|endoftext|>(tİ<|fim_prefix|>'re\r\n\r\n🙂'llea\u000bＡe", "tokens": 34, "pieces": ["\",\r\n\r\n", "ḍ̇", "\r\n\r\n", "<|", "endoftext", "|>(", "t", "İ", "<|", "fim", "_prefix", "|>'", "re", "\r\n\r\n", "🙂'", "llea", "\u000b", "Ａe"]} +{"text": "sZ0's😀🏽
३", "tokens": 9, "pieces": ["s", "Z", "0", "'s", "😀🏽", "
", "३"]} +{"text": "\rs字 \n漢'D​m𐞁e0,Z0½", "tokens": 17, "pieces": ["\r", "s字", " \n", "漢'D", "​m𐞁e", "0", ",Z", "0½"]} +{"text": "12345678　\r\n'Re>ſ\ne Ａ\t­ \n…mＡfi#$%éå'", "tokens": 26, "pieces": ["123", "456", "78", "　\r\n", "'Re", ">ſ", "\n", "e", " Ａ", "\t", "­", " \n", "…m", "Ａfi", "#$%", "éå", "'"]} +{"text": "<|fim_prefix|> \na\r\n\r\n'.'𐞁🙂३'re.0\r\n\r\n👍🏽,<|endoftext|>< éDžé'T𐞁㍿\r'll.𐞁👍🏽\u000bİ‍", "tokens": 57, "pieces": ["<|", "fim", "_prefix", "|>", " \n", "a", "\r\n\r\n", "'.'", "𐞁", "🙂", "३", "'re", ".", "0", "\r\n\r\n", "👍🏽,<|", "endoftext", "|><", " é", "Džé'T", "𐞁", "㍿\r", "'ll", ".𐞁", "👍🏽", "\u000bİ", "‍"]} +{"text": "‍'ſ
 'ſ​!!.EOT عİ#$%\r\n\r\n漢's'Sfi", "tokens": 21, "pieces": ["‍'", "ſ", "
 ", " '", "ſ", "​!!.", "EOT", " ", " ع", "İ", "#$%\r\n\r\n", "漢's", "'Sfi"]} +{"text": "! \n#$%ع'llmꟲDž'VE\t\r\n\r\n-½­'VE𐞁EOT're's🙂", "tokens": 29, "pieces": ["!", " \n", "#$%", "ع'll", "mꟲ", "Dž'VE", "\t\r\n\r\n", "-", "½", "­'", "VE𐞁", "EOT're", "'s", "🙂"]} +{"text": "fi\nⅣ'Re㍿!!­", "tokens": 10, "pieces": ["fi", "\n", "Ⅳ", "'Re", "㍿!!­"]} +{"text": "'M!!a0'VE\r\n\r\nß漢(Ⅳm ع>(!", "tokens": 17, "pieces": ["'M", "!!", "a", "0", "'VE", "\r\n\r\n", "ß漢", "(", "Ⅳ", "m", " ", " ع", ">(!"]} +{"text": "'VE#$%efi're.'re'ſ'T'VEḍ̇㋿'S㋿<|endoftext|>٣٤٥٦👍🏽", "tokens": 39, "pieces": ["'VE", "#$%", "efi're", ".'", "re'ſ", "'T'VE", "ḍ̇", "㋿'", "S", "㋿<|", "endoftext", "|>", "٣٤٥", "٦", "👍🏽"]} +{"text": "<|endoftext|>'ſ 'T0'll𐞁<|endoftext|>'S<|endoftext|>'s👍🏽🙂👍🏽9d🙂ßعZ, 's<|endoftext|>😀🏽'T's\u000bŹ \n<<<\ré𐞁", "tokens": 75, "pieces": ["<|", "endoftext", "|>'", "ſ", " ", " '", "T", "0", "'ll𐞁", "<|", "endoftext", "|>'", "S", "<|", "endoftext", "|>'", "s", "👍🏽🙂👍🏽", "9", "d", "🙂ßع", "Z", ",", " ", " '", "s", "<|", "endoftext", "|>😀🏽'", "T's", "\u000bŹ", " \n", "<<<\r", "é𐞁"]} +{"text": "fi'ſ.#$%\r\n\r\n'Md‍åsfié !!å\r12345678! <|fim_prefix|>é\"漢字漢\né½> \r㍿", "tokens": 46, "pieces": ["fi'ſ", ".#$%\r\n\r\n", "'Md", "‍åsfié", " ", "!!", "å", "\r", "123", "456", "78", "!", " ", "<|", "fim", "_prefix", "|>", "é", "\"漢字漢", "\n", "é", "½", ">", " \r", "㍿"]} +{"text": "e'ReⅣm　'T#$%👍🏽\u000b,", "tokens": 14, "pieces": ["e'Re", "Ⅳ", "m", "　", "'T", "#$%👍🏽", "\u000b", ","]} +{"text": "'ſع9ꟲfi", "tokens": 8, "pieces": ["'ſع", "9", "ꟲfi"]} +{"text": "(å字téعⅣé,", "tokens": 10, "pieces": ["(å字téع", "Ⅳ", "é", ","]} +{"text": "<́'Re9'㍿ ½\"<|fim_prefix|>\r\n.<|endoftext|>  're\r\n\r\nms٣٤٥٦é😀🏽'VE漢ßdd́>sḍ̇é'Sm", "tokens": 49, "pieces": ["<́'Re", "9", "'㍿", " ", "½", "\"<|", "fim", "_prefix", "|>\r\n", ".<|", "endoftext", "|>", " ", " ", "'re", "\r\n\r\n", "ms", "٣٤٥", "٦", "é", "😀🏽'", "VE漢ßdd́", ">sḍ̇é'S", "m"]} +{"text": ">‍\n'ſ-　Z字'ſſ12345678åꟲعs'llDž\r\n\r\nfiⅣß'Mé👍🏽Dž漢㋿👍🏽'S\u000b'Så", "tokens": 51, "pieces": [">‍\n", "'ſ", "-", "　Z字'ſ", "ſ", "123", "456", "78", "åꟲعs'll", "Dž", "\r\n\r\n", "fi", "Ⅳ", "ß'M", "é", "👍🏽", "Dž漢", "㋿👍🏽'", "S", "\u000b", "'Så"]} +{"text": "漢ꟲ…\r\n\r\nés'Sİ12345678ſaع'MEOTⅣ
'Tꟲ", "tokens": 26, "pieces": ["漢ꟲ", "…\r\n\r\n", "és'S", "İ", "123", "456", "78", "ſaع'M", "EOT", "Ⅳ", "
", "'Tꟲ"]} +{"text": "Ⅳ 'VE>ḍ̇\n'S\n🙂<|endoftext|>.Dž''MZ㍿'ll\r\n\r\né‍12345678٣٤٥٦😀🏽'llé'Mt㋿'ll\n\tEOTß", "tokens": 57, "pieces": ["Ⅳ", " ", " '", "VE", ">ḍ̇", "\n", "'S", "\n", "🙂<|", "endoftext", "|>.", "Dž", "''", "MZ", "㍿'", "ll", "\r\n\r\n", "é", "‍", "123", "456", "78٣", "٤٥٦", "😀🏽'", "llé'M", "t", "㋿'", "ll", "\n", "\tEOTß"]} +{"text": "ḍ̇fi", "tokens": 4, "pieces": ["ḍ̇fi"]} +{"text": "åDžßDžⅣ.'Ré-\"12345678fifi!!<|endoftext|>", "tokens": 29, "pieces": ["å", "Džß", "Dž", "Ⅳ", ".'", "Re", "́", "-\"", "123", "456", "78", "fifi", "!!<|", "endoftext", "|>"]} +{"text": "½<|fim_prefix|>​12345678a", "tokens": 12, "pieces": ["½", "<|", "fim", "_prefix", "|>​", "123", "456", "78", "a"]} +{"text": "e EOTß👍🏽<'VE𐞁(<|fim_prefix|>#$%<ḍ̇'Re're#$%#$%å'ſſ́\rå字 \n>'M're\t", "tokens": 53, "pieces": ["e", " ", " EOTß", "👍🏽<'", "VE𐞁", "<", "EOT", ">(<|", "fim", "_prefix", "|>#$%<", "ḍ̇'Re", "'re", "#$%#$%", "å'ſ", "ſ́", "\r", "å字", " \n", ">'", "M're", "\t"]} +{"text": "
ḍ̇'s\r\n0 \n㍿'T!'T,'ll🙂<|endoftext|>Dž!'s🙂's'll'VE's'\u000b", "tokens": 38, "pieces": ["
ḍ̇'s", "\r\n", "0", " \n", "㍿'", "T", "!'", "T", ",'", "ll", "🙂<|", "endoftext", "|>", "Dž", "!'", "s", "🙂'", "s'll", "'VE's", "'", "\u000b"]} +{"text": "\u000b\t👍🏽'll#$%m<٣٤٥٦‍\t 'D😀🏽<\n​­EOT३.㋿!㋿Z", "tokens": 37, "pieces": ["\u000b", "\t", "👍🏽'", "ll", "#$%", "m", "<", "٣٤٥", "٦", "‍", "\t ", " '", "D", "😀🏽<\n", "​­", "EOT", "३", ".㋿!㋿", "Z"]} +{"text": "(9'll", "tokens": 3, "pieces": ["(", "9", "'ll"]} +{"text": "🙂
…ſd#$%\u000b'VEé'Re\u000ba(<|endoftext|>mＡ́🙂字​'S　\r\n'llß'llⅣ", "tokens": 38, "pieces": ["🙂", "
", "…ſd", "#$%", "\u000b", "'VEé'Re", "\u000ba", "(<|", "endoftext", "|>", "m", "Ａ́", "🙂字", "​'", "S", "　\r\n", "'llß'll", "Ⅳ"]} +{"text": " EOT'Re\u000b0字", "tokens": 6, "pieces": [" EOT'Re", "\u000b", "0", "字"]} +{"text": "Z. 
9عt 'Rea🙂字m!<㍿ꟲḍ̇fi'D　ḍ̇9're('ſß\r\n\r\nss", "tokens": 44, "pieces": ["Z", ".<", "EOT", ">", " ", "
", "9", "عt", " '", "Rea", "🙂字m", "!<㍿", "ꟲḍ̇fi'D", "　ḍ̇", "9", "'re", "('", "ſß", "\r\n\r\n", "ss", ""]} +{"text": "ⅣEOT<|endoftext|>fi'll<|fim_prefix|>e \tZ 'T\"ꟲ字,EOT漢Dž\"\r\n'ſ\r\nſ><\r\n\r\n'S­👍🏽\råé#$%", "tokens": 55, "pieces": ["Ⅳ", "EOT", "<|", "endoftext", "|>", "fi'll", "<|", "fim", "_prefix", "|>", "e", " ", "\tZ", " ", "'T", "\"ꟲ字", ",EOT漢", "Dž", "\"\r\n", "'ſ", "\r\n", "ſ", "><\r\n\r\n", "'S", "­👍🏽\r", "åé", "#$%"]} +{"text": "0'ſ㋿'T　>\n㍿'re字!!'reDž\t👍🏽 dd-​‍0İ ", "tokens": 31, "pieces": ["0", "'ſ", "㋿'", "T", "　", ">\n", "㍿'", "re字", "!!'", "re", "Dž", "\t", "👍🏽", " dd", "-​‍", "0", "İ", " "]} +{"text": "\r-'>12345678\"!!漢㍿٣٤٥٦­ＡmＡع", "tokens": 21, "pieces": ["\r", "-'>", "123", "456", "78", "\"!!", "漢", "㍿", "٣٤٥", "٦", "­Ａm", "Ａع"]} +{"text": "\u000b٣٤٥٦\n'D😀🏽Z'S\"'ſ٣٤٥٦​9\r\n\r\n(#$%!!🙂ع<>İt३'ḍ̇　　́Ⅳ㍿㍿", "tokens": 50, "pieces": ["\u000b", "٣٤٥", "٦", "\n", "'D", "😀🏽", "Z'S", "\"'", "ſ", "٣٤٥", "٦", "​", "9", "\r\n\r\n", "(#$%<", "EOT", ">!!🙂", "ع", "<>", "İt", "३", "'ḍ̇", "　", "　́", "Ⅳ", "㍿㍿"]} +{"text": "'s\r\n", "tokens": 5, "pieces": ["'", "s", "\r\n"]} +{"text": "… 'ſ​>Dž,'s<'Re<|fim_prefix|>fißß<'S'VE<|fim_prefix|>t🙂Ⅳ\u000b\r'VE<|fim_prefix|>e,a<|endoftext|>字ＡİDž", "tokens": 68, "pieces": ["… ", " '", "ſ", "​>", "Dž", ",'", "s", "<'", "Re", "<|", "fim", "_prefix", "|>", "fißß", "<'", "S'VE", "<|", "fim", "_prefix", "|>", "t", "🙂<", "EOT", ">", "Ⅳ", "\u000b\r", "'VE", "<|", "fim", "_prefix", "|>", "e", ",a", "<|", "endoftext", "|>", "字", "ＡİDž"]} +{"text": "漢's'T𐞁\u000b\t12345678'D9ſḍ̇9'VE'S㋿Z👍🏽d­漢éd", "tokens": 38, "pieces": ["漢's", "'T𐞁", "\u000b", "\t", "123", "456", "78", "'", "D", "9", "ſḍ̇", "9", "'VE'S", "㋿Z", "👍🏽", "d", "­漢éd"]} +{"text": "é٣٤٥٦­#$%ḍ̇½ḍ̇ ㍿ḍ̇\t­½٣٤٥٦'Re𐞁EOTع<|fim_prefix|><|fim_prefix|>!Ⅳfié<|fim_prefix|>\r…0#$%​'M \nt're", "tokens": 77, "pieces": ["é", "٣٤٥", "٦", "­#$%", "ḍ̇", "½", "ḍ̇", " ", " ㍿", "ḍ̇", "\t", "­", "½٣٤", "٥٦", "'Re𐞁", "EOTع", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>!", "Ⅳ", "fié", "<|", "fim", "_prefix", "|>\r", "…", "0", "#$%​'", "M", " \n", "t're"]} +{"text": "\n­Ⅳ\r\n\r\n 'M,
\r\n\r\n,é e'Sfi👍🏽", "tokens": 20, "pieces": ["\n", "­", "Ⅳ", "\r\n\r\n", " ", "'M", ",", "
", "\r\n\r\n", ",é", " e'S", "fi", "👍🏽"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b(́'res ꟲm\r\néé\r\n\r\nḍ̇!.'S字
\r\nfia'ſDž'T's­'VEm३d字'll", "tokens": 34, "pieces": ["👍🏽", "123", "456", "78३", "­​<", "META", "_START", ">.'", "S字", "
\r\n", "fia'ſ", "Dž'T", "'s", "­'", "VEm", "३", "d字'll"]} +{"text": "<\r\n\r\nå<'Tfi9#$%Dž\r\n😀🏽🙂‍😀🏽\t,'T.'s\u000b字e#$% \u000b t9٣٤٥٦\t'ſ \n", "tokens": 50, "pieces": ["<<", "META", "_START", ">\r\n\r\n", "å", "<'", "Tfi", "9", "#$%<", "EOT", ">Dž", "\r\n", "😀🏽🙂‍😀🏽", "\t", ",'", "T", ".'", "s", "\u000b字e", "#$%", " \u000b ", " t", "9٣٤", "٥٦", "\t", "'ſ", " \n"]} +{"text": ">\ń٣٤٥٦-a‍<|endoftext|>\u000bé'T12345678Dž0eß\r\nfi\r\n\r\n　ß́́ ⅣDž<|fim_prefix|>t​…åZ​a<|fim_prefix|>", "tokens": 64, "pieces": [">\n", "́", "٣٤٥", "٦", "-a", "‍<|", "endoftext", "|>", "\u000bé'T", "123", "456", "78", "Dž", "0", "eß", "\r\n", "fi", "\r\n\r\n", "　ß́", "́", " ", "Ⅳ", "Dž", "<|", "fim", "_prefix", "|>", "t", "​", "…å", "Z", "​a", "<|", "fim", "_prefix", "|>"]} +{"text": "­..😀🏽're漢\t\t\r\n😀🏽\u000bß'VE𐞁é漢<|endoftext|>​Ⅳ'Ś…<|fim_prefix|>(Ⅳ\tm३", "tokens": 49, "pieces": ["­..😀🏽'", "re漢", "\t\t\r\n", "😀🏽", "\u000bß'VE", "𐞁é漢", "<|", "endoftext", "|>​", "Ⅳ", "'", "Ś", "…", "<|", "fim", "_prefix", "|>(", "Ⅳ", "\tm", "३"]} +{"text": "😀🏽漢😀🏽 's-\"\r'ſDž0#$%!!👍🏽! Dž㋿'<|endoftext|>'VEع m\r字Ⅳ٣٤٥٦>Dž½<​<|fim_prefix|>", "tokens": 71, "pieces": ["😀🏽", "漢", "😀🏽", " ", " '", "s", "-\"\r", "'ſ", "Dž", "0", "#$%!!👍🏽!<", "META", "_START", ">", " Dž", "㋿'<|", "endoftext", "|>'", "VEع", " ", " m", "\r", "字", "Ⅳ٣٤", "٥٦", ">Dž", "½", "<​<", "EOT", "><|", "fim", "_prefix", "|><", "EOT", ">"]} +{"text": "‍#$%éİ
३>0\"'VEé", "tokens": 13, "pieces": ["‍#$%", "é", "İ", "
", "३", ">", "0", "\"'", "VEé"]} +{"text": "‍,'res'VE‍DžⅣ's٣٤٥٦\r\n\r\n'Re 🙂'M\r\n\r\n\rdſ \nZ½é", "tokens": 29, "pieces": ["‍,'", "res'VE", "‍Dž", "Ⅳ", "'s", "٣٤٥", "٦", "\r\n\r\n", "'Re", " ", " 🙂'", "M", "\r\n\r\n\r", "dſ", " \n", "Z", "½", "é"]} +{"text": "'re!ع\rmDž😀🏽#$%…é
\nİ\"é'VEß(sd'VE <漢­ꟲ\r\n\r\nså'M", "tokens": 38, "pieces": ["'re", "!ع", "\r", "m", "Dž", "😀🏽#$%", "…é", "
\n", "İ", "\"é'VE", "ß", "(sd'VE", " ", "<漢", "­ꟲ", "\r\n\r\n", "så'M"]} +{"text": "'D, \r\n\r\nعꟲ'ſ", "tokens": 9, "pieces": ["'D", ",", " \r\n\r\n", "عꟲ'ſ"]} +{"text": "漢<|fim_prefix|>aéع \na'VE­‍ſDž字­t'\te𐞁0́'VEsİ
ß字'll!İ(-ꟲ", "tokens": 51, "pieces": ["漢", "<|", "fim", "_prefix", "|>", "aéع", " \n", "a'VE", "­‍", "ſ", "Dž", "字", "­t", "'", "\te𐞁", "0", "́'VE", "s", "İ", "
ß字'll", "!İ", "(-<", "EOT", ">ꟲ"]} +{"text": "ßİꟲ😀🏽'\r\ne<|fim_prefix|>Dž٣٤٥٦t(ſ'ReＡ", "tokens": 27, "pieces": ["ß", "İꟲ", "😀🏽'\r\n", "e", "<|", "fim", "_prefix", "|>", "Dž", "٣٤٥", "٦", "t", "(ſ'Re", "Ａ"]} +{"text": "'ll字,aß'Re½ Ａ", "tokens": 8, "pieces": ["'ll字", ",aß'Re", "½", " Ａ"]} +{"text": "Ⅳ's漢İ
(𐞁0ée\r\n\r\nd½\rḍ̇9!9
​,٣٤٥٦\r\nع'ſ३'Re", "tokens": 38, "pieces": ["Ⅳ", "'s漢", "İ", "
", "(𐞁", "0", "ée", "\r\n\r\n", "d", "½", "\r", "ḍ̇", "9", "!", "9", "
", "​,", "٣٤٥", "٦", "\r\n", "ع'ſ", "३", "'Re"]} +{"text": "e'M　9　(…'Re!Z<|fim_prefix|>'D", "tokens": 21, "pieces": ["e'M", "　", "9", "　", "(", "…", "'Re", "!", "Z", "<|", "fim", "_prefix", "|>'", "D"]} +{"text": "Z \r😀🏽‍'VE字Ａ😀🏽-\t#$%'>́Dž\r-\r\n're漢'ſ", "tokens": 28, "pieces": ["Z", " \r", "😀🏽‍'", "VE字", "Ａ", "😀🏽-", "\t", "#$%'>́", "Dž", "\r", "-\r\n", "'re漢'ſ"]} +{"text": "é
٣٤٥٦ 'M🙂fim 'S<", "tokens": 15, "pieces": ["é", "
", "٣٤٥", "٦", " ", " '", "M", "🙂fim", " '", "S", "<"]} +{"text": "å
🙂ⅣDž\n👍🏽s's'res \n's'll!fi😀🏽​a're", "tokens": 27, "pieces": ["å", "
", "🙂", "Ⅳ", "Dž", "\n", "👍🏽", "s's", "'res", " \n", "'s'll", "!fi", "😀🏽​", "a're"]} +{"text": "9EOT‍ 're\"<|endoftext|>३t'T\n.İ\n'T́s🙂<'D\ta", "tokens": 28, "pieces": ["9", "EOT", "‍", " ", "'re", "\"<|", "endoftext", "|>", "३", "t'T", "\n", ".İ", "\n", "'T́s", "🙂<'", "D", "\ta"]} +{"text": "'ſ'llⅣ", "tokens": 8, "pieces": ["'ſ'll", "Ⅳ", ""]} +{"text": "🙂ß𐞁'll<|endoftext|> \n<|endoftext|>Ａ<|fim_prefix|>'M字㍿'Re's!!!fiEOTⅣ", "tokens": 48, "pieces": ["🙂ß𐞁'll", "<|", "endoftext", "|>", " \n", "<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>'", "M字", "㍿'", "Re's", "!<", "META", "_START", ">!!", "fi", "EOT", "Ⅳ"]} +{"text": "\u000bé‍'re
é́漢 　t<|endoftext|>'Re12345678\"­ ́㋿\u000b字e\r'll", "tokens": 47, "pieces": ["\u000bé", "‍'", "re", "
é́漢", " ", "　t", "<|", "endoftext", "|>'", "Re", "123", "456", "78", "\"­", " ́", "<", "EOT", ">㋿<", "META", "_START", ">", "\u000b字e", "\r", "'ll"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi'Dİ'reéåع\"Ⅳ(téİ", "tokens": 17, "pieces": ["fi'D", "İ're", "é", "åع", "\"", "Ⅳ", "(té", "İ"]} +{"text": "9å'reⅣ're­\u000b'Téd ,　字😀🏽'Såع٣٤٥٦ḍ̇\r,½ꟲ漢‍ꟲ…Ⅳ's", "tokens": 57, "pieces": ["9", "å're", "Ⅳ", "'re", "­", "\u000b", "'Téd", " ", ",", "　字", "😀🏽'", "Såع", "", "٣٤٥", "٦", "ḍ̇", "\r", ",", "½", "ꟲ漢", "‍ꟲ", "…", "Ⅳ", "'s"]} +{"text": "㋿‍!!​'re-EOT\r\n\r\n-​,'ll​éd\n!ß­ m'Rea'ſ \r'Red…0", "tokens": 41, "pieces": ["㋿‍!!​'", "re", "-EOT", "\r\n\r\n", "-​,'", "ll", "​éd", "\n", "!", "ß", "­", " m'Re", "a'ſ", " \r", "'Re", "d", "…", "0"]} +{"text": "'M'ſ'D½\t…𐞁….\n12345678İ
.'M‍㍿<|fim_prefix|>", "tokens": 35, "pieces": ["'M'ſ", "'D", "½", "\t", "…𐞁", "…", ".\n", "123", "456", "78", "İ", "
", ".'", "M", "‍㍿<|", "fim", "_prefix", "|>"]} +{"text": "ß\ra
9😀🏽'ſ>…😀🏽Ⅳ­'T<㋿İꟲ‍'ſ 0'll.t", "tokens": 36, "pieces": ["ß", "\r", "a", "
", "9", "😀🏽'", "ſ", ">", "…", "😀🏽", "Ⅳ", "­'", "T", "<㋿", "İꟲ", "‍'", "ſ", " ", "0", "'ll", ".t"]} +{"text": "'sſ'Re\"\n123456780'ſa ", "tokens": 11, "pieces": ["'sſ'Re", "\"\n", "123", "456", "780", "'ſa", " "]} +{"text": "\r\n 漢t ḍ̇😀🏽sꟲ'Dt\rfi\" 漢td ", "tokens": 25, "pieces": ["\r\n", " 漢t", " ḍ̇", "😀🏽", "sꟲ'D", "t", "\r", "fi", "\"", " ", " 漢td", " "]} +{"text": "ع㋿Z'M𐞁 \n…'ll'VE,e.Džs", "tokens": 21, "pieces": ["ع", "㋿Z'M", "𐞁", " \n", "…", "'ll'VE", ",e", ".Džs"]} +{"text": " 'DéＡ<\r\n\r\n>Dž!d字#$%'D!½e‍­s!<\réＡعfi'Re", "tokens": 29, "pieces": [" '", "Dé", "Ａ", "<\r\n\r\n", ">Dž", "!d字", "#$%'", "D", "!", "½", "e", "‍­", "s", "!<\r", "é", "Ａعfi'Re"]} +{"text": "(aſİ'ſ
字Ａ\"३e9!\r ½
's​\r\n\r\n'VE", "tokens": 23, "pieces": ["(aſ", "İ'ſ", "
字", "Ａ", "\"", "३", "e", "9", "!\r", " ", " ", "½", "
", "'s", "​\r\n\r\n", "'VE"]} +{"text": "Ａ \nꟲ'T\"fis
's𐞁🙂å\r\na\r\n\r\n", "tokens": 21, "pieces": ["Ａ", " \n", "ꟲ'T", "\"fis", "
", "'s𐞁", "🙂å", "\r\n", "a", "\r\n\r\n"]} +{"text": "m 'VE-'.👍🏽EOTé\r\nétå", "tokens": 17, "pieces": ["m", " ", "'VE", "-'.👍🏽", "EOTé", "\r\n", "étå"]} +{"text": "'S'D-ſ‍#$%'Dß́'S s!'MZé>é", "tokens": 19, "pieces": ["'S'D", "-ſ", "‍#$%'", "Dß́'S", " ", " s", "!'", "MZé", ">é"]} +{"text": "'ll,'VEs'VE're​a​\"-Zع'Séå…ꟲ字", "tokens": 24, "pieces": ["'ll", ",'", "VEs'VE", "'re", "​a", "​\"-", "Zع'S", "éå", "…ꟲ字"]} +{"text": "\n \n s mfi !\t'Dt!!ꟲ,dİ12345678㍿…'Reع<|endoftext|>​漢ع😀🏽'ſ 　m", "tokens": 44, "pieces": ["\n \n", " ", " s", " ", " mfi", " !", "\t", "'Dt", "!!", "ꟲ", ",d", "İ", "123", "456", "78", "㍿", "…", "'Reع", "<|", "endoftext", "|>​", "漢ع", "😀🏽'", "ſ", " ", "　m"]} +{"text": "㍿ßßm'DDž ٣٤٥٦'Re
 𐞁(½\n \n٣٤٥٦<|endoftext|>å'Mß́0\"…,'Re'M(fi<|fim_prefix|>EOT㋿Ⅳ٣٤٥٦", "tokens": 69, "pieces": ["㍿ßßm'D", "Dž", " ", " ", "٣٤٥", "٦", "'Re", "
", " 𐞁", "(", "½", "\n \n", "٣٤٥", "٦", "<|", "endoftext", "|>", "å'M", "ß́", "0", "\"", "…", ",'", "Re'M", "(", "fi", "<|", "fim", "_prefix", "|>", "EOT", "㋿", "Ⅳ٣٤", "٥٦"]} +{"text": "(ꟲé　<|fim_prefix|> \n<|fim_prefix|> 🙂😀🏽fi0a>\n٣٤٥٦éꟲ😀🏽́é", " \n", "<|", "fim", "_prefix", "|>", " ", " 🙂😀🏽", "fi", "0", "a", ">\n", "٣٤٥", "٦", "éꟲ", "😀🏽́", "é", "½12345678…😀🏽Ⅳ<|endoftext|>\"३ع㍿té
d​m½", "tokens": 49, "pieces": ["…", "'lléع", " \n", "<'", "re", "Ⅳ", "<ß", "<|", "fim", "_prefix", "|>", "½12", "345", "678", "…", "😀🏽", "Ⅳ", "<|", "endoftext", "|>\"", "३", "ع", "㍿té", "
d", "​m", "½"]} +{"text": "٣٤٥٦Ⅳ ​Ⅳ'reſ́'Re٣٤٥٦éé<<|endoftext|> <|fim_prefix|>عåⅣ😀🏽 ḍ̇'D'D
٣٤٥٦㍿'s'VEs'Re👍🏽0'VEå\r\n>字", "tokens": 74, "pieces": ["٣٤٥", "٦Ⅳ", " ", " ​", "Ⅳ", "'reſ́'Re", "٣٤٥", "٦", "éé", "<<|", "endoftext", "|>", " ", " <|", "fim", "_prefix", "|>", "عå", "Ⅳ", "😀🏽", " ", " ḍ̇'D", "'D", "
", "٣٤٥", "٦", "㍿'", "s'VE", "s'Re", "👍🏽", "0", "'VEå", "\r\n", ">字"]} +{"text": "#$%!!㋿Z½٣٤٥٦ḍ̇\tＡtعd㋿漢'Téع9Ａ'é\né­#$%Dž…", "tokens": 44, "pieces": ["#$%!!㋿", "Z", "½٣٤", "٥٦", "ḍ̇", "\tＡtعd", "㋿<", "META", "_START", ">漢'T", "éع", "9", "Ａ", "'é", "\n", "é", "­#$%", "Dž", "…"]} +{"text": "m
ḍ̇ع're're漢 \n\r\nＡⅣ\n'res<|fim_prefix|>‍عé12345678", "tokens": 30, "pieces": ["m", "
ḍ̇ع're", "'re漢", " \n\r\n", "Ａ", "Ⅳ", "\n", "'res", "<|", "fim", "_prefix", "|>‍", "عé", "123", "456", "78"]} +{"text": "é'SDž'sعß'T.", "tokens": 9, "pieces": ["é'S", "Dž's", "عß'T", "."]} +{"text": "\"　--Dž-‍mＡ\r\n\n\" \n're", "tokens": 15, "pieces": ["\"", "　", "--", "Dž", "-‍", "m", "Ａ", "\r\n\n", "\"", " \n", "'re"]} +{"text": "\u000bmé0‍­'S㍿­३𐞁Dž‍", "tokens": 19, "pieces": ["\u000bmé", "0", "‍­'", "S", "㍿­", "३", "𐞁", "Dž", "‍"]} +{"text": "!!字㍿ Ⅳ #$% !!漢<|fim_prefix|>😀🏽,
\"ꟲEOT  Z\"'Tعå٣٤٥٦0>😀🏽-tfi\t㋿😀🏽", "tokens": 62, "pieces": ["!!", "字", "㍿", " ", "Ⅳ", " ", " #$%", " ", "!!", "漢", "<|", "fim", "_prefix", "|>😀🏽,", "
", "\"ꟲ", "EOT", " ", " Z", "\"'", "Tعå", "٣٤٥", "٦0", "><", "EOT", ">😀🏽-", "tfi", "\t", "㋿😀🏽"]} +{"text": "EOT'RedⅣZ🙂('Re'D\rDž'VE\t\t.'T㋿\u000b​㍿​😀🏽é0٣٤٥٦\r\nZ-'re🙂fi漢½'re.ع", "tokens": 50, "pieces": ["EOT'Re", "d", "Ⅳ", "Z", "🙂('", "Re'D", "\r", "Dž'VE", "\t", "\t", ".'", "T", "㋿", "\u000b", "​㍿​😀🏽", "é", "0٣٤", "٥٦", "\r\n", "Z", "-'", "re", "🙂fi漢", "½", "'re", ".ع"]} +{"text": "#$%½sé", "tokens": 4, "pieces": ["#$%", "½", "sé"]} +{"text": "\u000bat<İ…ß🙂عꟲ字're३,'T٣٤٥٦'Re…Ａ\r\n ́t>\r\n's​
éDž9'Dع‍", "tokens": 48, "pieces": ["\u000bat", "<İ", "", "…ß", "🙂عꟲ字're", "३", ",'", "T", "٣٤٥", "٦", "'Re", "…Ａ", "\r\n", " ", " ́t", ">\r\n", "'s", "​<", "EOT", ">", "
é", "Dž", "9", "'Dع", "‍"]} +{"text": "a漢\u000bİ​\n\r's!'Re'D👍🏽İ\r\n'VEİ'St ", "tokens": 21, "pieces": ["a漢", "\u000bİ", "​\n\r", "'s", "!'", "Re'D", "👍🏽", "İ", "\r\n", "'VEİ'S", "t", " "]} +{"text": "𐞁d'Sع㋿éݽ'S\u000bſ'T\"9ſ123456780\r\nß'smＡ,½　", "tokens": 31, "pieces": ["𐞁d'S", "ع", "㋿é", "İ", "½", "'S", "\u000bſ'T", "\"", "9", "ſ", "123", "456", "780", "\r\n", "ß's", "m", "Ａ", ",", "½", "　"]} +{"text": "́m#$%'T　('VE\r,Ⅳ😀🏽!!½ ́\t-　0'D३å‍>eſd", "tokens": 37, "pieces": ["́m", "#$%'", "T", "　", "('", "VE", "\r", ",", "Ⅳ", "😀🏽!!", "½", " ", " ́", "\t", "-", "　", "0", "'D", "३", "å", "‍>", "eſd"]} +{"text": " \n ", "tokens": 2, "pieces": [" \n", " "]} +{"text": "\t \n<|endoftext|>\r\n9,\r\n12345678EOT", "tokens": 15, "pieces": ["\t \n", "<|", "endoftext", "|>\r\n", "9", ",\r\n", "123", "456", "78", "EOT"]} +{"text": "'Dſ'S9३İ ,'T🙂\u000b're\t'e'Re<fi​🙂<|fim_prefix|>'D,‍<|fim_prefix|>　-12345678字12345678é　ꟲ", "tokens": 52, "pieces": ["'Dſ'S", "9", "", "३", "İ", " ,'", "T", "🙂", "\u000b", "'re", "\t", "'e'Re", "<fi", "​🙂<|", "fim", "_prefix", "|>'", "D", ",‍<|", "fim", "_prefix", "|>", "　", "-", "123", "456", "78", "字", "123", "456", "78", "é", "　ꟲ"]} +{"text": "(㋿\u000b0\r\n\r\n'M", "tokens": 8, "pieces": ["(㋿", "\u000b", "0", "\r\n\r\n", "'M"]} +{"text": "\t
'ſⅣ㍿ 字'D　>", "tokens": 18, "pieces": ["\t", "
", "'ſ", "Ⅳ", "㍿", " ", "字'D", "　", ">"]} +{"text": "m\r㍿( Dž\n३-­\"\t's", "tokens": 19, "pieces": ["m", "\r", "㍿(", " Dž", "\n", "३", "-­\"<", "EOT", ">", "\t", "'s"]} +{"text": "३!İ🙂(,字d0​漢a", "tokens": 12, "pieces": ["३", "!İ", "🙂(,", "字d", "0", "​漢a"]} +{"text": "!!e \n<#$% a >ع \n​३\u000b'VE­\r", "tokens": 18, "pieces": ["!!", "e", " \n", "<#$%", " a", " >", "ع", " \n", "​", "३", "\u000b", "'VE", "­\r"]} +{"text": ",'Dž", "tokens": 3, "pieces": [",'", "Dž"]} +{"text": "Ａ'ſ \n", "tokens": 4, "pieces": ["Ａ'ſ", " \n"]} +{"text": "ſEOT0‍-éZع\r", "tokens": 10, "pieces": ["ſ", "EOT", "0", "‍-", "é", "Zع", "\r"]} +{"text": "'re EOTŹ'३عś\r.ع
… ß👍🏽", "tokens": 22, "pieces": ["'re", " EOTŹ", "'", "३", "عś", "\r", ".ع", "
…", " ß", "👍🏽"]} +{"text": "'DEOTＡ'Re\r\n\u000b \n😀🏽ae,ſDžꟲ.㍿'llİ३𐞁Ａ'M'VE…é'll9<|endoftext|><|endoftext|>'", "tokens": 53, "pieces": ["'DEOTＡ'Re", "\r\n\u000b \n", "😀🏽", "ae", ",ſ", "Džꟲ", ".㍿'", "ll", "İ", "३", "𐞁", "Ａ'M", "'VE", "…é'll", "9", "<|", "endoftext", "|><|", "endoftext", "|>'"]} +{"text": "Ⅳ('s\"!!'S#$%'llé'll\u000b ㋿!a's<|fim_prefix|>('re　<|fim_prefix|>\r\né
½'Re\t½½'s!́'ll३ \n ", "tokens": 55, "pieces": ["Ⅳ", "('", "s", "\"!!'", "S", "#$%'", "ll", "é'll", "\u000b", " ㋿!", "a's", "<|", "fim", "_prefix", "|>('", "re", "　", "<|", "fim", "_prefix", "|>\r\n", "é", "
", "½", "'Re", "\t", "½½", "'s", "!́'ll", "३", " \n", " "]} +{"text": "e12345678", "tokens": 4, "pieces": ["e", "123", "456", "78"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "fi0‍ḍ̇<|endoftext|>ḍ̇>9 '<|fim_prefix|>\nß́'VE\r\n​(𐞁𐞁", "tokens": 43, "pieces": ["fi", "0", "‍ḍ̇", "<|", "endoftext", "|>", "ḍ̇", ">", "9", " ", " '<|", "fim", "_prefix", "|>\n", "ß́'VE", "\r\n", "​(", "𐞁𐞁"]} +{"text": "­-'D9👍🏽👍🏽é\"👍🏽e's<|fim_prefix|>12345678İt åé३'D >('VE\n", "tokens": 43, "pieces": ["­-<", "EOT", ">'", "D", "9", "👍🏽👍🏽", "é", "\"👍🏽", "e's", "<|", "fim", "_prefix", "|>", "123", "456", "78", "İt", " åé", "३", "'D", " ", ">('", "VE", "\n"]} +{"text": "'lls\r\n\r\n \"'TＡ'Re٣٤٥٦sé 😀🏽", "tokens": 17, "pieces": ["'lls", "\r\n\r\n", " \"'", "TＡ'Re", "٣٤٥", "٦", "sé", " ", "😀🏽"]} +{"text": "s\n'VE\rEOT\n", "tokens": 8, "pieces": ["s", "\n", "'VE", "\r", "EOT", "\n"]} +{"text": "fiⅣ
>", "tokens": 5, "pieces": ["fi", "Ⅳ", "
", ">"]} +{"text": "½!!عDžⅣⅣİ😀🏽'T \néß'Dm'Tعß!, ع>", "tokens": 30, "pieces": ["½", "!!", "ع", "Dž", "ⅣⅣ", "İ", "😀🏽'", "T", " \n", "éß'D", "m'T", "عß", "!,", " ع", ">"]} +{"text": " \r\n\r\nⅣ'Rea", "tokens": 5, "pieces": [" \r\n\r\n", "Ⅳ", "'Rea"]} +{"text": "#$%å's🙂㍿'Re\t😀🏽Z　­e‍'s漢're>!-'sß٣٤٥٦'s<|fim_prefix|>㍿0!! \ń<|endoftext|>👍🏽9m­Ａ", "tokens": 68, "pieces": ["#$%", "å's", "🙂㍿'", "Re", "\t", "😀🏽", "Z", "", "　", "­e", "‍'", "s漢're", ">!-'", "sß", "٣٤٥", "٦", "'s", "<|", "fim", "_prefix", "|>㍿", "0", "!!", " \n", "́", "<|", "endoftext", "|>👍🏽", "9", "m", "­<", "META", "_START", ">Ａ"]} +{"text": "Ａ>㋿Ⅳ#$%­­'VEs\rİ'Md　 \r\n…<'s.ſ٣٤٥٦EOT\"9​<　t\u000b", "tokens": 43, "pieces": ["Ａ", ">㋿", "Ⅳ", "#$%­­'", "VEs", "\r", "İ'M", "d", "　 \r\n", "…", "<'", "s", ".", "ſ", "٣٤٥", "٦", "EOT", "\"", "9", "​<", "　t", "\u000b"]} +{"text": "Ａé-'\n", "tokens": 8, "pieces": ["Ａé", "-'\n"]} +{"text": "<|fim_prefix|>a'Re<𐞁漢
½", "tokens": 19, "pieces": ["<|", "fim", "_prefix", "|>", "a'Re", "<<", "META", "_START", ">𐞁漢", "
", "½"]} +{"text": "<|endoftext|>٣٤٥٦Džéś9Ａ　🙂s!<,㍿12345678\n'M'VE<|endoftext|>😀🏽", "tokens": 47, "pieces": ["<|", "endoftext", "|>", "٣٤٥", "٦", "Džéś", "9", "Ａ", "　", "🙂s", "!<,㍿", "123", "456", "78", "\n", "'", "M'VE", "<|", "endoftext", "|>😀🏽"]} +{"text": "é🙂漢𐞁Ⅳfi'VEd.­ \n…\u000b㋿ \n𐞁㋿d\"EOT--'ſZ\tm", "tokens": 50, "pieces": ["é", "🙂漢", "𐞁", "Ⅳ", "fi'VE", "d", ".­", " \n", "…", "\u000b", "㋿", " \n", "𐞁", "㋿d", "\"EOT", "--'", "ſ", "Z", "\tm"]} +{"text": "'M'ret👍🏽ꟲ\u000bt12345678'M -٣٤٥٦!'re", "tokens": 23, "pieces": ["'M're", "t", "👍🏽", "ꟲ", "\u000bt", "123", "456", "78", "'M", " ", " -", "٣٤٥", "٦", "!'", "re"]} +{"text": "\r\n\r\nİ'VE٣٤٥٦­éZ\t \n", "tokens": 15, "pieces": ["\r\n\r\n", "İ'VE", "٣٤٥", "٦", "­é", "Z", "\t \n"]} +{"text": "<|endoftext|>'s🙂𐞁​‍d", "tokens": 16, "pieces": ["<|", "endoftext", "|>'", "s", "🙂𐞁", "​‍", "d"]} +{"text": "'sßꟲ ,12345678㍿🙂're<|fim_prefix|>'MＡſ'll㋿½e!,'ll‍ꟲⅣ>", "tokens": 45, "pieces": ["'", "sßꟲ", " ,", "123", "456", "78", "㍿🙂'", "re", "<|", "fim", "_prefix", "|>'", "MＡſ'll", "㋿", "½", "e", "!,'", "ll", "‍ꟲ", "Ⅳ", ">"]} +{"text": "<|endoftext|>\"'sfi", "tokens": 10, "pieces": ["<|", "endoftext", "|>\"'", "sfi"]} +{"text": "(m12345678㍿३'Re(İ'D( #$%sZ'llé0🙂‍­9", "tokens": 25, "pieces": ["(m", "123", "456", "78", "㍿", "३", "'Re", "(İ'D", "(", " ", "#$%", "s", "Z'll", "é", "0", "🙂‍­", "9"]} +{"text": "ſ(字\t​!ſＡ٣٤٥٦😀🏽　 ३'TfiＡ…'s㋿<'VE", "tokens": 32, "pieces": ["ſ", "(字", "\t", "​!", "ſ", "Ａ", "٣٤٥", "٦", "😀🏽", "　", " ", "३", "'Tfi", "Ａ", "…", "'s", "㋿<'", "VE"]} +{"text": "' d字😀🏽­Z<|fim_prefix|>Ⅳ
fi", "tokens": 19, "pieces": ["'", " ", " d字", "😀🏽­", "Z", "<|", "fim", "_prefix", "|>", "Ⅳ", "
fi"]} +{"text": "😀🏽㍿", "tokens": 6, "pieces": ["😀🏽㍿"]} +{"text": "\t\n<|endoftext|>m,\r\n\r\n\u000b🙂\"<|endoftext|>\na", "tokens": 23, "pieces": ["\t\n", "<|", "endoftext", "|><", "EOT", ">m", ",\r\n\r\n", "\u000b", "🙂\"<|", "endoftext", "|>\n", "a"]} +{"text": "s,\"…!!'res‍Dž ḍ̇漢ⅣDž12345678'D'M😀🏽👍🏽 \n'D 
", "tokens": 35, "pieces": ["s", ",\"", "…", "!!'", "res", "‍Dž", " ", " ḍ̇漢", "Ⅳ", "Dž", "123", "456", "78", "'D'M", "😀🏽👍🏽", " \n", "'D", " 
"]} +{"text": "ḍ̇㋿ع'M''M m漢\r　'D0s'TZEOTfi­", "tokens": 26, "pieces": ["ḍ̇", "㋿ع'M", "''", "M", " m漢", "\r", "　", "'D", "0", "s'T", "ZEOTfi", "­"]} +{"text": "🙂åe㋿'D….ḍ̇́\n.DžⅣḍ̇😀🏽'VE'S漢😀🏽Ａ!!d(a𐞁 \nZ<|fim_prefix|>٣٤٥٦EOT", "tokens": 60, "pieces": ["🙂<", "EOT", ">åe", "㋿'", "D", "…", ".ḍ̇́", "\n", ".Dž", "Ⅳ", "ḍ̇", "😀🏽'", "VE'S", "漢", "😀🏽", "Ａ", "!!", "d", "(a𐞁", " \n", "Z", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "EOT"]} +{"text": "‍ aé<|endoftext|>Ａ٣٤٥٦ſ\u000bꟲ\nt'll㋿ß\"!!\nséå- ع'M漢\r\n\r\n 'DEOT㍿12345678½Ａ", "tokens": 50, "pieces": ["‍", " aé", "<|", "endoftext", "|>", "Ａ", "٣٤٥", "٦", "ſ", "\u000bꟲ", "\n", "t'll", "㋿ß", "\"!!\n", "séå", "-", " ع'M", "漢", "\r\n\r\n", " '", "DEOT", "㍿", "123", "456", "78½", "Ａ"]} +{"text": "dét\u000b \n#$%\t#$%'S12345678\r\n𐞁\u000b'T‍ 'ree㍿㍿\"İＡ. \nd\r\n\r\nééaß\n", "tokens": 45, "pieces": ["dét", "\u000b \n", "#$%", "\t", "#$%'", "S", "123", "456", "78", "\r\n", "𐞁", "\u000b", "'T", "‍", " ", " '", "ree", "㍿㍿\"", "İＡ", ".", " \n", "d", "\r\n\r\n", "éé", "aß", "\n"]} +{"text": "12345678İ\u000b'VE\u000b<|fim_prefix|>99,👍🏽Ａ'D😀🏽½ 👍🏽'll३-á'Reß'll", "tokens": 37, "pieces": ["123", "456", "78", "İ", "\u000b", "'VE", "\u000b", "<|", "fim", "_prefix", "|>", "99", ",👍🏽", "Ａ'D", "😀🏽", "½", " ", " 👍🏽'", "ll", "३", "-á'Re", "ß'll"]} +{"text": " ㋿\"", "tokens": 6, "pieces": [" ", " ㋿\""]} +{"text": "!!é's.m>'M🙂İ'<|fim_prefix|>\r\n(　\u000b's ́,٣٤٥٦.'T'll
½m<'VEd'ſ", "tokens": 37, "pieces": ["!!", "é's", ".m", ">'", "M", "🙂İ", "'<|", "fim", "_prefix", "|>\r\n", "(", "　", "\u000b", "'s", " ́", ",", "٣٤٥", "٦", ".'", "T'll", "
", "½", "m", "<'", "VEd'ſ"]} +{"text": "-\n😀🏽(dd३!!'ſå'T\r\n\u000bſ'VE'VEa…\u000b​ 're", "tokens": 27, "pieces": ["-\n", "😀🏽(", "dd", "३", "!!'", "ſå'T", "\r\n", "\u000bſ'VE", "'VEa", "…", "\u000b", "​", " '", "re"]} +{"text": "\r\n\r\n<|endoftext|>12345678­‍\r\nع'sé'T>Ⅳ
ꟲ!́<|endoftext|>", "tokens": 35, "pieces": ["\r\n\r\n", "<|", "endoftext", "|>", "123", "456", "78", "­‍\r\n", "ع's", "é'T", ">", "Ⅳ", "
ꟲ", "!́", "<|", "endoftext", "|>"]} +{"text": "'D'VEt\r㋿\r", "tokens": 12, "pieces": ["'D'VE", "t", "\r", "㋿\r"]} +{"text": "'Reå", "tokens": 9, "pieces": ["'Re", "å", ""]} +{"text": "t'll٣٤٥٦­\ra　Ａ ḍ̇🙂-𐞁'reⅣDžåé㍿\r\n\r\n'Re!\n👍🏽\r\n\r\n\r'S👍🏽å३‍\u000b㍿
­", "tokens": 55, "pieces": ["t'll", "٣٤٥", "٦", "­\r", "a", "　Ａ", " ḍ̇", "🙂-", "𐞁're", "Ⅳ", "Džåé", "㍿\r\n\r\n", "'Re", "!\n", "👍🏽\r\n\r\n\r", "'S", "👍🏽", "å", "३", "‍", "\u000b", "㍿", "
", "­"]} +{"text": "!!Ⅳ🙂Dž\r\n\r\n're \ńd漢½'S🙂må\"'S٣٤٥٦!🙂ß\t‍é", "tokens": 30, "pieces": ["!!", "Ⅳ", "🙂Dž", "\r\n\r\n", "'re", " \n", "́d漢", "½", "'S", "🙂må", "\"'", "S", "٣٤٥", "٦", "!🙂", "ß", "\t", "‍é"]} +{"text": ".­é 12345678'Re½👍🏽ꟲ字,!ſ'ſDž
👍🏽🙂३\n ", "tokens": 31, "pieces": [".­", "é", " ", "123", "456", "78", "'Re", "½", "👍🏽", "ꟲ字", ",!", "ſ'ſ", "Dž", "
", "👍🏽🙂", "३", "\n", " "]} +{"text": "eعé٣٤٥٦🙂 >­é ꟲ🙂\r\n\r\n\r", "tokens": 20, "pieces": ["eعé", "٣٤٥", "٦", "🙂", " ", " >­", "é", " ", " ꟲ", "🙂\r\n\r\n\r"]} +{"text": "åd<|endoftext|>'́\r\n\r\n's㋿😀🏽ḍ̇😀🏽'Re🙂å0'D‍#$%'VEḍ̇🙂　(🙂sd#$%éa\"\u000b­ſİ\r\n\r\n", "tokens": 54, "pieces": ["åd", "<|", "endoftext", "|>'́\r\n\r\n", "'s", "㋿😀🏽", "ḍ̇", "😀🏽'", "Re", "🙂å", "0", "'D", "‍#$%'", "VEḍ̇", "🙂", "　", "(🙂", "sd", "#$%", "éa", "\"", "\u000b", "­ſ", "İ", "\r\n\r\n"]} +{"text": "\rDžZmZ'Dt!'s!㍿", "tokens": 13, "pieces": ["\r", "DžZm", "Z'D", "t", "!'", "s", "!㍿"]} +{"text": "ݽ#$%ß👍🏽'll­'ſ\n字'Dé㍿a'३'D 😀🏽\t <|endoftext|>Dž३fiꟲEOTtdé…>'ſ", "tokens": 51, "pieces": ["İ", "½", "#$%", "ß", "👍🏽'", "ll", "­'", "ſ", "\n", "字'D", "é", "㍿a", "'", "३", "'D", " 😀🏽", "\t ", " <|", "endoftext", "|>", "Dž", "३", "fiꟲ", "EOTtdé", "…", ">'", "ſ"]} +{"text": "EOT ́<|fim_prefix|>a'llßع \n!,😀🏽é😀🏽-İ'sß#$%㋿é", "tokens": 34, "pieces": ["EOT", " ́", "<|", "fim", "_prefix", "|>", "a'll", "ßع", " \n", "!,😀🏽", "é", "😀🏽-", "İ's", "ß", "#$%㋿", "é"]} +{"text": "½<😀🏽'Re", "tokens": 7, "pieces": ["½", "<😀🏽'", "Re"]} +{"text": "३'Re<|endoftext|>😀🏽!३ḍ̇!!.😀🏽\u000b'M>㍿", "tokens": 27, "pieces": ["३", "'Re", "<|", "endoftext", "|>😀🏽!", "३", "ḍ̇", "!!.😀🏽", "\u000b", "'M", ">㍿"]} +{"text": "((", "tokens": 1, "pieces": ["(("]} +{"text": "'sée 'S‍\t.,Džm½….‍㍿Ⅳ　𐞁'll٣٤٥٦عa🙂
-ꟲ\r\n\r\n३'ſ<'T's!'-EOT
", "tokens": 52, "pieces": ["'sée", " '", "S", "‍", "\t", ".,", "Džm", "½", "…", ".‍㍿", "Ⅳ", "　𐞁'll", "٣٤٥", "٦", "عa", "🙂", "
", "-ꟲ", "\r\n\r\n", "३", "'ſ", "<'", "T's", "!'-", "EOT", "
"]} +{"text": "\t字…'s字em sae\" 'ſ12345678é'reİ'ss㋿<|fim_prefix|>#$%　e漢'VE.fi>…é½ꟲ…", "tokens": 54, "pieces": ["Ⅳ", "'Re", "\u000b\n", "'ſ", "\tt", "", " sae", "\"", " ", " '", "ſ", "123", "456", "78", "é're", "İ's", "s", "㋿<|", "fim", "_prefix", "|>#$%", "　e漢'VE", ".fi", ">", "…é", "½", "ꟲ", "…"]} +{"text": "ḍ̇Dž<|endoftext|>'MEOTＡ ſ'lls٣٤٥٦<|endoftext|>Zꟲ,a,\t\r\n\r\n\r\n", "tokens": 39, "pieces": ["ḍ̇", "Dž", "<|", "endoftext", "|>'", "MEOTＡ", " ", " ſ'll", "s", "٣٤٥", "٦", "<|", "endoftext", "|>", "Zꟲ", ",a", ",", "\t\r\n\r\n\r\n"]} +{"text": "EOT…㋿'T,t9'Me\u000b( Ａ!!('ſⅣfi🙂're!0(ḍ̇'T'D㍿
's'll", "tokens": 40, "pieces": ["EOT", "…", "㋿'", "T", ",t", "9", "'Me", "\u000b", "(", " Ａ", "!!('", "ſ", "Ⅳ", "fi", "🙂'", "re", "!", "0", "(ḍ̇'T", "'D", "㍿", "
", "'s'll"]} +{"text": "m<<|fim_prefix|>!!å🙂!!<|fim_prefix|>fi😀🏽éꟲſ'Re३ \n0😀🏽\ńå½
\r\n\r\n!!a'VE-ع>", "tokens": 49, "pieces": ["m", "<<|", "fim", "_prefix", "|>!!", "å", "🙂!!<|", "fim", "_prefix", "|>", "fi", "😀🏽", "éꟲſ'Re", "३", " \n", "0", "😀🏽\n", "́å", "½", "
\r\n\r\n", "!!", "a'VE", "-ع", ">"]} +{"text": "<|endoftext|>ß \nfi'Re9­'T\t 'Må\"㋿", "tokens": 23, "pieces": ["<|", "endoftext", "|>", "ß", " \n", "fi'Re", "9", "­'", "T", "\t", " '", "Må", "\"㋿"]} +{"text": "Ⅳ(d­\r>éⅣ", "tokens": 9, "pieces": ["Ⅳ", "(d", "­\r", ">é", "Ⅳ"]} +{"text": "ßİZ,Ａé\"t 'VE‍'>'T're½!㍿\tm.<|endoftext|>ḍ̇,'VEd0!Zm!!e漢😀🏽t­å.­", "tokens": 48, "pieces": ["ß", "İZ", ",Ａé", "\"t", " '", "VE", "‍'>'", "T're", "½", "!㍿", "\tm", ".<|", "endoftext", "|>", "ḍ̇", ",'", "VEd", "0", "!Zm", "!!", "e漢", "😀🏽", "t", "­å", ".­"]} +{"text": "'se'reeDžZ-　e　Ⅳ½Ⅳ👍🏽🙂tat𐞁ḍ̇'VE9<ꟲſİ𐞁(‍étꟲ", "tokens": 49, "pieces": ["'se're", "e", "DžZ", "-", "　e", "　", "Ⅳ½Ⅳ", "👍🏽🙂", "tat𐞁ḍ̇'VE", "9", "<ꟲſ", "İ𐞁", "(‍", "étꟲ"]} +{"text": "ꟲ'ſ<|fim_prefix|>'M½ß\r\n\r\né½å​٣٤٥٦\u000b…'T​Ⅳsع🙂0", "tokens": 35, "pieces": ["ꟲ'ſ", "<|", "fim", "_prefix", "|>'", "M", "½", "ß", "\r\n\r\n", "é", "½", "å", "​", "٣٤٥", "٦", "\u000b", "…", "'T", "​", "Ⅳ", "sع", "🙂", "0"]} +{"text": "é're ३.‍dßع½𐞁<　<\r're'T's(ḍ̇𐞁 \r\n\r\n\n\r'S12345678'\r\n\r\né\r\n", "tokens": 39, "pieces": ["é're", " ", "३", ".‍", "dßع", "½", "𐞁", "<", "　", "<\r", "'re'T", "'s", "(ḍ̇𐞁", " \r\n\r\n\n\r", "'S", "123", "456", "78", "'\r\n\r\n", "é", "\r\n"]} +{"text": "a
. 漢- -<|fim_prefix|>ع字٣٤٥٦fiꟲ'ſ漢'T<㋿'T#$%a#$%dⅣ#$%😀🏽", "tokens": 47, "pieces": ["a", "
", ".", " 漢", "-", " ", " -<|", "fim", "_prefix", "|>", "ع字", "٣٤٥", "٦", "fiꟲ'ſ", "漢'T", "<㋿'", "T", "#$%", "a", "#$%", "d", "Ⅳ", "#$%😀🏽"]} +{"text": "㋿d's'Dž'sDž<|endoftext|>'ll'VE,(­åſ𐞁ßDž\u000bfi>-‍t å", "tokens": 45, "pieces": ["㋿d's", "'Dž's", "Dž", "<|", "endoftext", "|>'", "ll'VE", ",(­", "åſ𐞁ß", "Dž", "\u000bfi", ">-‍", "t", " ", " å", ""]} +{"text": "'re#$%½a'S\t-< \n12345678'VE\r\n\r\n
…,\"t　 ", "tokens": 24, "pieces": ["'re", "#$%", "½", "a'S", "\t", "-<", " \n", "123", "456", "78", "'", "VE", "\r\n\r\n", "
", "…", ",\"", "t", "　 "]} +{"text": "३(漢'ſ­㍿ >", "tokens": 11, "pieces": ["३", "(漢'ſ", "­㍿", " ", ">"]} +{"text": "
'T'ſ٣٤٥٦\"\u000bꟲ<|fim_prefix|>'M'llſ\r\n'D\r\n\r\n'ſ٣٤٥٦", "tokens": 31, "pieces": ["
", "'T'ſ", "٣٤٥", "٦", "\"", "\u000bꟲ", "<|", "fim", "_prefix", "|>'", "M'll", "ſ", "\r\n", "'D", "\r\n\r\n", "'ſ", "٣٤٥", "٦"]} +{"text": "İ​é'S\r𐞁\r\n\r\n<…​!!9'Re", "tokens": 23, "pieces": ["İ", "​", "é'S", "\r", "𐞁", "\r\n\r\n", "<", "…", "​!!", "9", "'Re"]} +{"text": "\r\n\r\n0fi0-\u000b", "tokens": 8, "pieces": ["", "0", "fi", "0", "-", "\u000b"]} +{"text": "'ſꟲZ<|endoftext|>\u000b", "tokens": 14, "pieces": ["'ſꟲ", "Z", "<|", "endoftext", "|>", "\u000b"]} +{"text": "dt!!Z \n'T'D0\rEOT'Re<|fim_prefix|>aß\u000baⅣ !字İ'DＡſ's\"ḍ̇'S漢Ａe३a\r\n\r\nå'Re", "tokens": 47, "pieces": ["dt", "!!", "Z", " \n", "'T'D", "0", "\r", "EOT'Re", "<|", "fim", "_prefix", "|>", "aß", "\u000ba", "Ⅳ", " !", "字", "İ'D", "Ａſ's", "\"ḍ̇'S", "漢Ａe", "३", "a", "\r\n\r\n", "å'Re"]} +{"text": "'S　'T‍\t12345678m(\"字", "tokens": 11, "pieces": ["'S", "　", "'T", "‍", "\t", "123", "456", "78", "m", "(\"", "字"]} +{"text": "\r\n'ſ<漢'VE>,'Re\u000btfié漢
ḍ̇
'Mḍ̇ß's#$%\rع \nＡ!!", "tokens": 38, "pieces": ["\r\n", "'ſ", "<漢'VE", ">,'", "Re", "\u000btfié漢", "
ḍ̇", "
", "'Mḍ̇", "ß's", "#$%\r", "ع", " \n", "Ａ", "!!"]} +{"text": "!字㋿字ß<|fim_prefix|><🙂ḍ̇\r漢 ſfi…\r\n\r\né​EOT'Mḍ̇m
ḿDžé­½<|endoftext|>字9", "tokens": 58, "pieces": ["!", "字", "㋿字ß", "<|", "fim", "_prefix", "|><🙂", "ḍ̇", "\r", "漢", " ſfi", "…\r\n\r\n", "é", "​EOT'M", "ḍ̇m", "
ḿ", "Džé", "­", "½", "<|", "endoftext", "|>", "字", "", "9"]} +{"text": "'M👍🏽字'ḍ̇d'D\r\n\r\nfi \n​åİZ'S<|fim_prefix|>'ſ'ſ\r\n\r\neＡ㍿漢're漢d<|fim_prefix|>9\n#$%😀🏽İ", "tokens": 56, "pieces": ["'M", "👍🏽", "字", "'ḍ̇d'D", "\r\n\r\n", "fi", " \n", "​å", "İ", "Z'S", "<|", "fim", "_prefix", "|>'", "ſ'ſ", "\r\n\r\n", "e", "Ａ", "㍿漢're", "漢d", "<|", "fim", "_prefix", "|>", "9", "\n", "#$%😀🏽", "İ"]} +{"text": "'reع\r\n\nＡ'T😀🏽", "tokens": 31, "pieces": ["'", "reع", "\r\n", "\n", "Ａ'T", "😀🏽"]} +{"text": "　!!12345678\t'MZ'T.\t'VE𐞁\r!!٣٤٥٦!! ", "tokens": 28, "pieces": ["　", "!!", "123", "456", "78", "\t", "'MZ", "'", "T", ".", "\t", "'VE𐞁", "\r", "!!", "٣٤٥", "٦", "!!", " "]} +{"text": "!!ꟲ漢 𐞁½👍🏽9'!!漢,'re漢　​é\r字!'VEefi\n<|fim_prefix|>d\r\nꟲ", "tokens": 43, "pieces": ["!!", "ꟲ漢", " 𐞁", "½", "👍🏽", "9", "'!!", "漢", ",'", "re漢", "　", "​é", "\r", "字", "!'", "VEefi", "\n", "<|", "fim", "_prefix", "|>", "d", "\r\n", "ꟲ"]} +{"text": " \n\u000b", "tokens": 2, "pieces": [" \n", "\u000b"]} +{"text": "<'Reé<|endoftext|>.'ReEOT \n", "tokens": 21, "pieces": ["<<", "EOT", ">'", "Reé", "<|", "endoftext", "|><", "META", "_START", ">.'", "Re", "EOT", " \n"]} +{"text": "m\n­ع é́'S'Mt!!''ll\r\n\r\n<'ſ,'", "tokens": 19, "pieces": ["m", "\n", "­ع", " ", " é́'S", "'Mt", "!!''", "ll", "\r\n\r\n", "<'", "ſ", ",'"]} +{"text": "t 'M
fi.EOTꟲḍ̇\t​\u000be'SＡ'Re\u000b,", "tokens": 22, "pieces": ["t", " '", "M", "
fi", ".EOTꟲḍ̇", "\t", "​", "\u000be'S", "Ａ'Re", "\u000b", ","]} +{"text": "ßm'D .🙂\u000b<|endoftext|>Ａ'VE'T३9<|fim_prefix|>ḍ̇ḍ̇‍s<|fim_prefix|><|fim_prefix|> #$%12345678å<|endoftext|>\r0tع(字\r", "tokens": 70, "pieces": ["ßm'D", " ", ".🙂", "\u000b", "<|", "endoftext", "|>", "Ａ'VE", "'T", "३9", "<|", "fim", "_prefix", "|>", "ḍ̇ḍ̇", "‍s", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", " ", "#$%", "123", "456", "78", "å", "<|", "endoftext", "|><", "EOT", ">\r", "0", "tع", "(字", "\r"]} +{"text": "0'VE'll 'VE ,'sEOT12345678字<字㍿㋿!'Mꟲ!!Dž\r.åm'S0\tⅣDž#$%s½'ll🙂㍿'ll…㋿", "tokens": 59, "pieces": ["0", "'VE'll", " ", "'VE", " ", " ,'", "s", "EOT", "123", "456", "78", "字", "<字", "㍿㋿!'", "Mꟲ", "!!", "Dž", "\r", ".åm'S", "0", "\t", "Ⅳ", "Dž", "#$%", "s", "½", "'ll", "🙂㍿'", "ll", "…", "㋿"]} +{"text": "<-EOT​ \n㍿漢'VEdꟲ123456789", "tokens": 22, "pieces": ["<-", "EOT", "​", " \n", "㍿", "漢'VE", "dꟲ", "123", "456", "789"]} +{"text": "\r .٣٤٥٦漢㋿'ll👍🏽t🙂'D\t㋿,'VE..'ReＡ'ſ!!<|fim_prefix|>　\u000b <(", "tokens": 46, "pieces": ["\r", " ", ".", "٣٤٥", "٦", "漢", "㋿'", "ll", "👍🏽", "t", "🙂'", "D", "\t", "㋿,'", "VE", "..'", "Re", "Ａ'ſ", "!!<|", "fim", "_prefix", "|>", "　\u000b", " <("]} +{"text": "…då!('T👍🏽,字漢㋿'ll12345678½\u000båå,👍🏽\"", "tokens": 32, "pieces": ["…då", "!('", "T", "👍🏽,", "字漢", "㋿'", "ll", "123", "456", "78½", "\u000båå", ",👍🏽\""]} +{"text": "ꟲéꟲ<|endoftext|>😀🏽s<|fim_prefix|>'Daéعt­'12345678 ३d​\t३'M#$%#$%\tDž09​👍🏽字<|fim_prefix|>漢fiꟲ", "tokens": 64, "pieces": ["ꟲéꟲ", "<|", "endoftext", "|>😀🏽", "s", "<|", "fim", "_prefix", "|>'", "Daéعt", "­'", "123", "456", "78", " ", "३", "d", "​", "\t", "३", "'M", "#$%#$%", "\tDž", "09", "​👍🏽", "字", "<|", "fim", "_prefix", "|>", "漢fiꟲ"]} +{"text": "éfiꟲß'ſ\r\n\r\n'VE\u000b\r ­'llḍ̇𐞁12345678ßꟲ½12345678İＡ漢'S12345678fi9🙂'Mt!!s", "tokens": 56, "pieces": ["éfiꟲ", "ß'ſ", "\r\n\r\n", "'VE", "\u000b\r", " ­'", "llḍ̇𐞁", "123", "456", "78", "ßꟲ", "½12", "345", "678", "İＡ漢'S", "123", "456", "78", "fi", "9", "🙂'", "Mt", "!!", "s"]} +{"text": "#$%\"Džé\r\nEOTⅣ\u000b \n.éaEOTḍ̇㍿ꟲd𐞁㍿Džſ‍s\"İ㋿\"fi<|fim_prefix|>'D\r\n\r\n", "tokens": 57, "pieces": ["#$%\"", "Dž", "é", "\r\n", "EOT", "Ⅳ", "\u000b \n", ".éa", "EOTḍ̇", "㍿ꟲd𐞁", "㍿Džſ", "‍s", "\"İ", "㋿\"", "fi", "<|", "fim", "_prefix", "|>'", "D", "\r\n\r\n"]} +{"text": "\tt'S'SEOT​Z.eꟲ'll'D½İ३字ḿ'!!tå é'll'Dİ(#$%­𐞁'll٣٤٥٦​t'Re\t …", "tokens": 49, "pieces": ["\tt'S", "'SEOT", "​Z", ".eꟲ'll", "'D", "½", "İ", "३", "字ḿ", "'!!", "tå", " ", " é'll", "'Dİ", "(#$%­", "𐞁'll", "٣٤٥", "٦", "​t'Re", "\t …"]} +{"text": "'VE>'ll٣٤٥٦", "tokens": 8, "pieces": ["'VE", ">'", "ll", "٣٤٥", "٦"]} +{"text": "­漢عe𐞁>ts e'!!'Red漢're!d!!ع\rsfi👍🏽", "tokens": 33, "pieces": ["­漢", "عe𐞁", ">", "ts", " ", " e", "'!!'", "Red漢're", "!d", "!!", "ع", "\r", "sfi", "👍🏽"]} +{"text": "㋿३\r\n<|fim_prefix|>Ⅳ'T'Mḍ̇ZZ\r\t \n\",'S'VE", "tokens": 30, "pieces": ["㋿", "३", "\r\n", "<", "META", "_START", "><|", "fim", "_prefix", "|>", "Ⅳ", "'T'M", "ḍ̇", "ZZ", "\r\t \n", "\",'", "S'VE"]} +{"text": "…'ſ\r\n\r\n\r\rⅣ　​'S>'T‍<|fim_prefix|>9ß\r'ſ''VE.!!'VEm'Dé…\r\n㍿ſꟲé\n\raEOT", "tokens": 58, "pieces": ["…", "'ſ", "\r\n\r\n\r\r", "", "Ⅳ", "　", "​'", "S", ">'", "T", "‍<|", "fim", "_prefix", "|><", "EOT", ">", "9", "ß", "\r", "'ſ", "''", "VE", ".!!'", "VEm'D", "é", "…\r\n", "㍿ſꟲé", "\n\r", "a", "EOT"]} +{"text": "#$%\r\n字'St‍'VE", "tokens": 8, "pieces": ["#$%\r\n", "字'S", "t", "‍'", "VE"]} +{"text": "Z​m'sß٣٤٥٦'ſ㋿ \n'́", "tokens": 17, "pieces": ["Z", "​m's", "ß", "٣٤٥", "٦", "'ſ", "㋿", " \n", "'́"]} +{"text": "ſع\u000bⅣ㋿!! -'re३fi<fi", "tokens": 17, "pieces": ["ſع", "\u000b", "Ⅳ", "㋿!!", " ", " -'", "re", "३", "fi", "<fi"]} +{"text": "😀🏽\"ع-́", "tokens": 7, "pieces": ["😀🏽\"", "ع", "-́"]} +{"text": "ꟲ'sⅣ 0́́'T!!㋿#$%३Ａt<", "tokens": 21, "pieces": ["ꟲ's", "Ⅳ", " ", "0", "́́'T", "!!㋿#$%", "३", "Ａt", "<"]} +{"text": "> 'S\r\n\r\n'll''VEꟲꟲ\r\n­ＡZ'Re\r\n漢9<½\r\n<‍İ\r\n­‍\n12345678'M.<|fim_prefix|>\t
tt", "tokens": 44, "pieces": [">", " '", "S", "\r\n\r\n", "'ll", "''", "VEꟲꟲ", "\r\n", "­ＡZ'Re", "\r\n", "漢", "9", "<", "½", "\r\n", "<‍", "İ", "\r\n", "­‍\n", "123", "456", "78", "'M", ".<|", "fim", "_prefix", "|>", "\t", "
tt"]} +{"text": "𐞁", "tokens": 4, "pieces": ["𐞁"]} +{"text": "\t​", "tokens": 2, "pieces": ["\t", "​"]} +{"text": "\u000b́\r\naع'S\r\n'S\u000bḍ̇'S'Re㋿　'S'ſ,👍🏽'll12345678s 😀🏽\t\u000bm 'ḍ̇#$%", "tokens": 48, "pieces": ["\u000b́", "\r\n", "aع'S", "\r\n", "'S", "\u000bḍ̇'S", "'Re", "㋿", "　", "'S'ſ", ",👍🏽'", "ll", "123", "456", "78", "s", " ", " 😀🏽", "\t", "\u000bm", " ", "'<", "EOT", ">ḍ̇", "#$%"]} +{"text": "\"é're<|endoftext|>", "tokens": 10, "pieces": ["\"é're", "<|", "endoftext", "|>"]} +{"text": "👍🏽ꟲ字Ⅳ𐞁9e!!㋿'Mfi㍿'M9!", "tokens": 29, "pieces": ["👍🏽", "ꟲ字", "Ⅳ", "𐞁", "9", "e", "!!㋿'", "Mfi", "㍿'", "M", "9", "!"]} +{"text": "İé́'S'S'VE‍\"'D\r\n\r\n👍🏽 daＡḍ̇ \nＡ'DDž㍿", "tokens": 31, "pieces": ["İé́'S", "'S'VE", "‍\"'", "D", "\r\n\r\n", "👍🏽", " ", " da", "Ａḍ̇", " \n", "Ａ", "'", "DDž", "㍿"]} +{"text": "Dž\r\n\r\n.é　Ａ<|fim_prefix|>es.ع漢'ſ<|endoftext|>ß<|fim_prefix|>ß<|endoftext|><́>(🙂ع<|fim_prefix|>'ll字", "tokens": 56, "pieces": ["Dž", "\r\n\r\n", ".é", "　Ａ", "<|", "fim", "_prefix", "|>", "es", ".ع", "漢'ſ", "<|", "endoftext", "|>", "ß", "<|", "fim", "_prefix", "|>", "ß", "<|", "endoftext", "|><́>(🙂", "ع", "<|", "fim", "_prefix", "|>'", "ll字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'Re字\n#$%!
\r\n\r\n<|fim_prefix|>㍿ꟲＡ9'…
<|fim_prefix|>\r\n\r\n३EOT 
 \né'D0'M‍", "tokens": 42, "pieces": ["'Re字", "\n", "#$%!", "
\r\n\r\n", "<|", "fim", "_prefix", "|>㍿", "ꟲ", "Ａ", "9", "'", "…", "
", "<|", "fim", "_prefix", "|>\r\n\r\n", "३", "EOT", " 
 \n", "é'D", "0", "'M", "‍"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ſ#$%'SＡ!! \n\"‍!!ḍ̇\r\n­å(,#$%😀🏽-…e \r(!! 9ß　m\tꟲ('Re0é🙂㍿-", "tokens": 52, "pieces": ["ſ", "#$%'", "SＡ", "!!", " \n", "\"‍!!", "ḍ̇", "\r\n", "­å", "(,#$%😀🏽<", "EOT", ">-", "…e", " \r", "(!!", " ", "9", "ß", "　m", "\tꟲ", "('", "Re", "0", "é", "🙂㍿-"]} +{"text": "ꟲé>\"'ll'S( ḍ̇\r<|endoftext|>　-<🙂", "tokens": 49, "pieces": ["ꟲé", ">\"'", "ll'S", "(", " ", " ḍ̇", "\r", "<|", "endoftext", "|>", "　", "-<", "ta", "…", "‍\r\n", "́a'S", "å", "Ⅳ", "a", "!!\r", "🙂<|", "endoftext", "|><🙂"]} +{"text": "\"\nsDžte\u000b\u000bé㋿½
", "tokens": 14, "pieces": ["\"\n", "s", "Džte", "\u000b", "\u000bé", "㋿", "½", "
"]} +{"text": "<|endoftext|>>em\u000b\ŕßé(Ⅳ㋿'M'MＡ éḍ̇!fi!\r\n\r\n'T's.🙂'ReZ'Re…#$%", "tokens": 43, "pieces": ["<|", "endoftext", "|>>", "em", "\u000b\r", "́ßé", "(", "Ⅳ", "㋿'", "M'M", "Ａ", " éḍ̇", "!fi", "!\r\n\r\n", "'T's", ".🙂'", "Re", "Z'Re", "…", "#$%"]} +{"text": "Z字9#$%!!efim123456780<|endoftext|>", "tokens": 23, "pieces": ["Z字", "9", "#$%!!", "efi", "m", "123", "456", "780", "<|", "endoftext", "|>"]} +{"text": "👍🏽'Re㍿ſß߅'re'reꟲ\t\"字ſ\"12345678", "tokens": 23, "pieces": ["\r", "<|", "endoftext", "|>", "…", "'re're", "ꟲ", "\t", "\"字ſ", "\"", "123", "456", "78"]} +{"text": "'T'll'VE (‍ſ३½👍🏽𐞁'VE😀🏽Ⅳ'\n👍🏽㍿​'D'lla#$% ", "tokens": 38, "pieces": ["'T'll", "'VE", " (‍", "ſ", "३½", "👍🏽", "𐞁'VE", "😀🏽", "Ⅳ", "'\n", "👍🏽㍿​'", "D'll", "a", "#$%", " "]} +{"text": "d…-\r\n\r\n'S!漢's…!m­­'T12345678'M'Reع
\u000b'S<|fim_prefix|>ع\r", "tokens": 33, "pieces": ["d", "…", "-\r\n\r\n", "'S", "!漢's", "…", "!m", "­­'", "T", "123", "456", "78", "'M'Re", "ع", "
", "\u000b", "'S", "<|", "fim", "_prefix", "|>", "ع", "\r"]} +{"text": "'D👍🏽𐞁<|fim_prefix|>Ⅳḍ̇'M字<|endoftext|>३", "tokens": 31, "pieces": ["'D", "👍🏽", "𐞁", "<|", "fim", "_prefix", "|>", "Ⅳ", "ḍ̇'M", "字", "<|", "endoftext", "|>", "३"]} +{"text": "9㋿e'TEOTafi😀🏽!!<字漢'M<'VE‍åꟲ fi\u000b😀🏽é​'M#$%", "tokens": 43, "pieces": ["9", "㋿e'T", "EOTafi", "😀🏽<", "EOT", ">!!<", "字漢'M", "<'", "VE", "‍åꟲ", " ", " fi", "\u000b", "😀🏽", "é", "​'", "M", "#$%"]} +{"text": "\"ḍ̇!ع٣٤٥٦EOT ,s>­'VE-İ'llⅣ", "tokens": 28, "pieces": ["\"ḍ̇", "!ع", "٣٤٥", "٦", "EOT", " ", " ,", "s", ">­'", "VE", "-<", "META", "_START", ">İ'll", "Ⅳ"]} +{"text": "(\r\n\r\n'MfiⅣ!!ſ\rfim‍9Ａ<|fim_prefix|>'VEＡ　", "tokens": 23, "pieces": ["(\r\n\r\n", "'Mfi", "Ⅳ", "!!", "ſ", "\r", "fim", "‍", "9", "Ａ", "<|", "fim", "_prefix", "|>'", "VEＡ", "　"]} +{"text": "e­-\n12345678字", "tokens": 7, "pieces": ["e", "­-\n", "123", "456", "78", "字"]} +{"text": "Ａé \nß9'TEOT𐞁#$%>\"912345678
12345678\n\"'s", "tokens": 25, "pieces": ["Ａé", " \n", "ß", "9", "'TEOT𐞁", "#$%>\"", "912", "345", "678", "
", "123", "456", "78", "\n", "\"'", "s"]} +{"text": "'VE's \n\r\nİ　३ \nm<|endoftext|>㋿'llZ'll­İ\"9e-ꟲ'Re", "tokens": 37, "pieces": ["'VE", "'", "s", " \n\r\n", "İ", "　", "३", " \n", "m", "<|", "endoftext", "|>㋿'", "ll", "Z'll", "­İ", "\"", "9", "e", "-ꟲ'Re"]} +{"text": "漢\r\n\r\n\r\n\r\n<|endoftext|>EOT
<|endoftext|>\n٣٤٥٦字́'ſ'S< …\u000b\t'a \n'Re　'T𐞁'Re", "tokens": 50, "pieces": ["漢", "\r\n\r\n\r\n\r\n", "<|", "endoftext", "|>", "EOT", "", "
", "<|", "endoftext", "|>\n", "٣٤٥", "٦", "字́'ſ", "'S", "<", " …\u000b", "\t", "'a", "", " \n", "'Re", "　", "'T𐞁'Re"]} +{"text": "́…'D\"s>'lla", "tokens": 7, "pieces": ["́", "…", "'D", "\"s", ">'", "lla"]} +{"text": "s\nét-'M \n'De'ſfiḍ̇'ſ(İa½ EOT🙂ed㋿😀🏽12345678a\té­​\rZe…'", "tokens": 51, "pieces": ["s", "\n", "ét", "-'", "M", " \n", "'De'ſ", "fiḍ̇'ſ", "(İa", "½", " EOT", "🙂ed", "㋿😀🏽", "123", "456", "78", "a", "\té", "­​\r", "Ze", "…", "'"]} +{"text": "½字
字<|endoftext|>-dZ'Re 'ſ'VE‍字#$%0EOT'İ <|endoftext|>>(…Ａ漢‍‍٣٤٥٦.!!.a're", "tokens": 50, "pieces": ["½", "字", "
字", "<|", "endoftext", "|>-", "d", "Z'Re", " ", "'ſ'VE", "‍字", "#$%", "0", "EOT", "'İ", " ", " <|", "endoftext", "|>>(", "…Ａ漢", "‍‍", "٣٤٥", "٦", ".!!.", "a're"]} +{"text": "Z\r\n\r\né<|fim_prefix|>Z字㍿
å'M'Tſm½'Re\rⅣ \n's,9DžéDž字EOT漢ḍ̇ a12345678're ‍0\u000b漢", "tokens": 52, "pieces": ["Z", "\r\n\r\n", "é", "<|", "fim", "_prefix", "|>", "Z字", "㍿", "
å'M", "'Tſm", "½", "'Re", "\r", "Ⅳ", " \n", "'s", ",", "9", "Džé", "Dž字EOT漢ḍ̇", " a", "123", "456", "78", "'re", " ‍", "0", "\u000b漢"]} +{"text": "‍　'ꟲ'T'Re<#$%'S<٣٤٥٦sDž\t \né\n's\r\nm0👍🏽😀🏽🙂　\r\n\r\nⅣ12345678ßt", "tokens": 42, "pieces": ["‍", "　", "'ꟲ'T", "'Re", "<#$%'", "S", "<", "٣٤٥", "٦", "s", "Dž", "\t \n", "é", "\n", "'s", "\r\n", "m", "0", "👍🏽😀🏽🙂", "　\r\n\r\n", "Ⅳ12", "345", "678", "ßt"]} +{"text": "aae 're", "tokens": 3, "pieces": ["aae", " '", "re"]} +{"text": "s'ſع \r\n", "tokens": 5, "pieces": ["s'ſ", "ع", " \r\n"]} +{"text": "३\n🙂.a‍ 　012345678🙂 .…　'VE
½fiſ😀🏽", "tokens": 26, "pieces": ["३", "\n", "🙂.", "a", "‍", " ", "　", "012", "345", "678", "🙂", " ", " .", "…", "　", "'VE", "
", "½", "fiſ", "😀🏽"]} +{"text": "<|fim_prefix|>́.…😀🏽Ⅳ漢👍🏽éḍ̇字-EOTZ'ſḍ̇'#$%s😀🏽३!(ع 　́𐞁👍🏽(ZZ½-", "tokens": 59, "pieces": ["<|", "fim", "_prefix", "|>́.<", "EOT", ">", "…", "😀🏽", "Ⅳ", "漢", "👍🏽", "éḍ̇字", "-EOTZ'ſ", "ḍ̇", "'#$%", "s", "😀🏽", "३", "!(", "ع", " ", "　́𐞁", "👍🏽(", "ZZ", "½", "-"]} +{"text": ">­ ‍Džḍ̇ 　\t٣٤٥٦,\t12345678'S\u000b\tEOT'S\"‍ \nt'll　Dž", "tokens": 34, "pieces": [">­", " ", " ‍", "Džḍ̇", " 　", "\t", "٣٤٥", "٦", ",", "\t", "123", "456", "78", "'S", "\u000b", "\tEOT'S", "\"‍", " \n", "t'll", "　Dž"]} +{"text": "å　mع'S're're!!#$%'VEé\n‍", "tokens": 16, "pieces": ["å", "　mع'S", "'re're", "!!#$%'", "VEé", "\n", "‍"]} +{"text": "٣٤٥٦३　ꟲe.…Zd​Dž\n㍿ ß́٣٤٥٦'re　㋿'㋿ ś\nḍ̇", "tokens": 46, "pieces": ["٣٤٥", "٦३", "　ꟲe", ".", "…Zd", "​Dž", "\n", "㍿", " ß́", "٣٤٥", "٦", "'re", "　", "㋿'㋿", " <", "META", "_START", ">ś", "\n", "ḍ̇"]} +{"text": " \n\t'MZ", "tokens": 7, "pieces": ["", " \n", "\t", "'MZ"]} +{"text": "ß٣٤٥٦㍿ ‍'s><㍿ع's漢
 İꟲ­\t!漢‍'Re \nß'VE", "tokens": 36, "pieces": ["ß", "٣٤٥", "٦", "㍿", " ", " ‍'", "s", "><㍿", "ع's", "漢", "
 ", " İꟲ", "­", "\t", "!漢", "‍'", "Re", " \n", "ß'VE"]} +{"text": "İ३", "tokens": 2, "pieces": ["İ", "३"]} +{"text": " \ń字aİ!'re \r\n\r\né-\rmm,\u000b'S\r\n\u000b​!½'D\"'ſꟲs३\n0​ع　ꟲEOTé🙂", "tokens": 45, "pieces": [" \n", "́字a", "İ", "!'", "re", " \r\n\r\n", "é", "-\r", "mm", ",", "\u000b", "'S", "\r\n", "\u000b", "​!", "½", "'D", "\"'", "ſꟲs", "३", "\n", "0", "​ع", "　ꟲEOTé", "🙂"]} +{"text": "३'D '　åZ12345678ع'S\"३ḍ̇🙂>\tḍ̇Za\"å \n're-é‍𐞁", "tokens": 39, "pieces": ["३", "'D", " ", "'", "　å", "Z", "", "123", "456", "78", "ع'S", "\"", "३", "ḍ̇", "🙂>", "\tḍ̇", "Za", "\"å", " \n", "'re", "-é", "‍𐞁"]} +{"text": "😀🏽İⅣ½…ꟲm　ꟲ", "tokens": 20, "pieces": ["😀🏽", "İ", "Ⅳ½", "…ꟲm", "　", "ꟲ"]} +{"text": "🙂0EOTḍ̇漢(,\réꟲ٣٤٥٦éＡ<<|fim_prefix|> \n\r\n\r\n!!", "tokens": 30, "pieces": ["🙂", "0", "EOTḍ̇漢", "(,\r", "éꟲ", "٣٤٥", "٦", "é", "Ａ", "<<|", "fim", "_prefix", "|>", " \n\r\n\r\n", "!!"]} +{"text": "…é(٣٤٥٦é<|endoftext|>‍>fiſ\r\n0
('D<'D!!!!s're!!🙂\r\n'ſ'VE½३\r\n\r\né!'VE", "tokens": 44, "pieces": ["…é", "(", "٣٤٥", "٦", "é", "<|", "endoftext", "|>‍>", "fiſ", "\r\n", "0", "
", "('", "D", "<'", "D", "!!!!", "s're", "!!🙂\r\n", "'ſ'VE", "½३", "\r\n\r\n", "é", "!'", "VE"]} +{"text": "d!㍿<|endoftext|>‍12345678d­​漢😀🏽é<|fim_prefix|>  ſs\"ḍ̇‍", "tokens": 41, "pieces": ["d", "!㍿<|", "endoftext", "|>‍", "123", "456", "78", "d", "­​", "漢", "😀🏽", "é", "<|", "fim", "_prefix", "|>", "  ", " ſs", "\"ḍ̇", "‍"]} +{"text": "㋿​<|endoftext|>½'ReꟲEOTs", "tokens": 19, "pieces": ["㋿​<|", "endoftext", "|>", "½", "'Reꟲ", "EOTs"]} +{"text": "㍿.ZDž𐞁'VE…½", "tokens": 16, "pieces": ["㍿.", "ZDž𐞁'VE", "…", "½"]} +{"text": "'SＡd­EOT😀🏽ع!a/b,", "tokens": 14, "pieces": ["'SＡd", "­EOT", "😀🏽", "ع", "!a", "/b", ","]} +{"text": "İ'Sé㍿…0ſ'M", "tokens": 11, "pieces": ["İ'S", "é", "㍿", "…", "0", "ſ'M"]} +{"text": "㍿İ　­🙂Ⅳ…!!('s/\r\n½٣٤٥٦𐞁字", "tokens": 25, "pieces": ["㍿İ", "　", "­🙂", "Ⅳ", "…", "!!('", "s", "/\r\n", "½٣٤", "٥٦", "𐞁字"]} +{"text": "#$%EOT!mİſ!!aB́😀🏽/🙂>aB\r字𐞁aa", "tokens": 27, "pieces": ["#$%", "EOT", "!m", "İſ", "!!", "a", "B́", "😀🏽/🙂>", "a", "B", "\r", "字𐞁aa"]} +{"text": "HTTPServer's,", "tokens": 4, "pieces": ["HTTPServer's", ","]} +{"text": "\u000b9-ßß'T३㍿\n/𐞁camelCaset'Ａ/ \n
😀🏽é'Re 0Z😀🏽漢字Z'M0\r\n'T\u000b‍<😀🏽", "tokens": 49, "pieces": ["\u000b", "9", "-ßß'T", "३", "㍿\n/", "𐞁camel", "Caset", "'Ａ", "/", " \n", "
", "😀🏽", "é'Re", " ", "0", "Z", "😀🏽", "漢字", "Z'M", "0", "\r\n", "'T", "\u000b", "‍<😀🏽"]} +{"text": "\r\n\r\n \n ३Ze", "tokens": 5, "pieces": ["\r\n\r\n \n", " ", "३", "Ze"]} +{"text": "eＡABC<\r#$%\r \n EOTꟲ0aBᵃ字< ㍿
<|endoftext|>👍🏽ḍ̇!!fiaİ\r\n\r\nEOTᵃaBABC\r\n\r\n#$%'retBßHTTPServer", "tokens": 61, "pieces": ["e", "ＡABC", "<\r", "#$%\r", " \n", " EOTꟲ", "0", "a", "Bᵃ字", "<", " ", "㍿", "
", "<|", "endoftext", "|>👍🏽", "ḍ̇", "!!", "fia", "İ", "\r\n\r\n", "EOTᵃa", "BABC", "\r\n\r\n", "#$%'", "ret", "Bß", "HTTPServer"]} +{"text": "ꟲ٣٤٥٦t", "tokens": 8, "pieces": ["ꟲ", "٣٤٥", "٦", "t"]} +{"text": "́åm'll12345678👍🏽ḍ̇12345678́fi字HTTPServer　mDžungla\r\naBABC12345678٣٤٥٦/-'Sé字́<|endoftext|>Z#$%<|fim_prefix|>", "tokens": 61, "pieces": ["́åm'll", "123", "456", "78", "👍🏽", "ḍ̇", "123", "456", "78", "́fi字", "HTTPServer", "　m", "Džungla", "\r\n", "a", "BABC", "123", "456", "78٣", "٤٥٦", "/-'", "Sé字́", "<|", "endoftext", "|>", "Z", "#$%<|", "fim", "_prefix", "|>"]} +{"text": "'re\t \n!! \nå…!!ḍ̇<|fim_prefix|>\r\n\r\naB'Mds\n/>\t　…ع<ſꟲiOS𐞁camelCase/\r\n٣٤٥٦>,' 　camelCaseDžunglaé😀🏽", "tokens": 61, "pieces": ["'re", "\t \n", "!!", " \n", "å", "…", "!!", "ḍ̇", "<|", "fim", "_prefix", "|>\r\n\r\n", "a", "B'M", "ds", "\n", "/>", "\t　", "…ع", "<ſꟲi", "OS𐞁camel", "Case", "/\r\n", "٣٤٥", "٦", ">,'", " ", "　camel", "Case", "Džunglaé", "😀🏽"]} +{"text": "camelCase३字iOS/!!-Ａſß-!!'ResDžunglaDž​́<\t", "tokens": 26, "pieces": ["camel", "Case", "३", "字i", "OS", "/!!-", "Ａſß", "-!!'", "Res", "Džungla", "Dž", "​́", "<", "\t"]} +{"text": "fiiOS…'re३HTTPServer\n­'é", "tokens": 14, "pieces": ["fii", "OS", "…", "'re", "३", "HTTPServer", "\n", "­'", "é"]} +{"text": "'re\n \nſ!!'re \n 
Džungla'ſ-'Tå maB", "tokens": 24, "pieces": ["'re", "\n \n", "ſ", "!!'", "re", "", " \n", " ", "
Džungla'ſ", "-'", "Tå", " ", " ma", "B"]} +{"text": "aB\r/.㍿字'SDžungla漢(''Re\u000b/a/bDžEOTe<|fim_prefix|>a/b12345678<|fim_prefix|>\r\na/b㍿<\r\n\r\n\n\n/\t#$%'T'<|endoftext|>é㍿\r\n ", "tokens": 72, "pieces": ["a", "B", "\r", "/.㍿", "字'S", "Džungla漢", "(''", "Re", "\u000b", "/a", "/b", "DžEOTe", "<|", "fim", "_prefix", "|>", "a", "/b", "123", "456", "78", "<|", "fim", "_prefix", "|>\r\n", "a", "/b", "㍿<\r\n\r\n\n\n/", "\t", "#$%'", "T", "'<|", "endoftext", "|>", "é", "㍿\r\n", " "]} +{"text": "ḍ̇a٣٤٥٦\ta/b,,iOS/\r\n😀🏽Z0/\r\nİDžé,<|endoftext|>e'D\" \n 
३ꟲ", "tokens": 63, "pieces": ["ḍ̇a", "٣٤٥", "٦", "\ta", "/b", ",,", "i", "OS", "/\r\n", "😀🏽", "Z", "0", "/\r\n", "İDžé", ",<", "a", "/b", "‍\n", "\t", "㍿!!", "s", "Ab", "
", "!!", "t", "<<|", "endoftext", "|><|", "endoftext", "|>", "e'D", "\"", " \n", " ", "
", "३", "ꟲ"]} +{"text": "㍿Ａ<.camelCaseİ\r\n\r\n éZ/\r\n\r\n\r\nع'll½\u000ba/b\naDžungla<|endoftext|>'reAb漢\n/(HTTPServer　　
<|fim_prefix|>'re, ", "tokens": 51, "pieces": ["㍿Ａ", "<.", "camel", "Case", "İ", "\r\n\r\n", " é", "Z", "/\r\n\r\n\r\n", "ع'll", "½", "\u000ba", "/b", "\n", "a", "Džungla", "<|", "endoftext", "|>'", "re", "Ab漢", "\n", "/(", "HTTPServer", "　　", "
", "<|", "fim", "_prefix", "|>'", "re", ",", " "]} +{"text": "<|endoftext|>Ⅳ \n å字\td're漢😀🏽ᵃ#$%ᵃ\r\nع<9
", "tokens": 32, "pieces": ["<|", "endoftext", "|>", "Ⅳ", " \n", " å字", "\td're", "漢", "😀🏽", "ᵃ", "#$%", "ᵃ", "\r\n", "ع", "<", "9", "
"]} +{"text": "Džunglaꟲ\n<|endoftext|>ß㍿Džungla\"0,　👍🏽ᵃ12345678\na/bḍ̇\u000bdİع'T/\r\ń🙂👍🏽dＡ　Z🙂㋿m!", "tokens": 66, "pieces": ["Džunglaꟲ", "\n", "<|", "endoftext", "|>", "ß", "㍿Džungla", "\"", "0", ",<", "EOT", ">", "　", "👍🏽", "ᵃ", "123", "456", "78", "\n", "a", "/bḍ̇", "\u000bd", "İع'T", "/\r\n", "́", "🙂👍🏽", "d", "Ａ", "　Z", "🙂㋿", "m", "!"]} +{"text": "ḍ̇", "tokens": 3, "pieces": ["ḍ̇"]} +{"text": "👍🏽>\u000b>😀🏽é
Dž,'M漢'D\n\"ꟲ Z\r<|fim_prefix|>'ſⅣ　Džunglaḍ̇½", "tokens": 44, "pieces": ["👍🏽>", "\u000b", ">😀🏽", "é", "
Dž", ",'", "M漢'D", "\n", "\"ꟲ", " ", " Z", "\r", "<|", "fim", "_prefix", "|>'", "ſ", "Ⅳ", "　Džunglaḍ̇", "½"]} +{"text": "camelCased-m٣٤٥٦ＡHTTPServer", "tokens": 15, "pieces": ["camel", "Cased", "-m", "٣٤٥", "٦", "Ａ", "HTTPServer"]} +{"text": "㋿漢'M'VEiOS<|endoftext|>", "tokens": 16, "pieces": ["㋿漢'M", "'VEi", "OS", "<|", "endoftext", "|>"]} +{"text": "/,HTTPServeŕ ḍ̇iOS٣٤٥٦'DHTTPServer'M\r\n're'reꟲ👍🏽'reaBB<|endoftext|>\u000be​ſ'måꟲſſd👍🏽m½", "tokens": 55, "pieces": ["/,", "HTTPServeŕ", " ḍ̇i", "OS", "٣٤٥", "٦", "'DHTTPServer'M", "\r\n", "'re're", "ꟲ", "👍🏽'", "rea", "BB", "<|", "endoftext", "|>", "\u000be", "​ſ'm", "åꟲſſd", "👍🏽", "m", "½"]} +{"text": "Abḍ̇\r\n", "tokens": 5, "pieces": ["Abḍ̇", "\r\n"]} +{"text": "ḍ̇\r\nİ​m#$% ३­HTTPServeré'Re", "tokens": 16, "pieces": ["ḍ̇", "\r\n", "İ", "​m", "#$%", " ", "३", "­HTTPServeré'Re"]} +{"text": "​३0.‍ZⅣ'Td'sßſs㋿ſ㋿ABC🙂𐞁\"", "tokens": 34, "pieces": ["​", "३0", ".‍", "Z", "Ⅳ", "'Td's", "ßſs", "㋿ſ", "㋿ABC", "🙂𐞁", "\"<", "EOT", ">"]} +{"text": "s12345678!'ll \n 'ReⅣs　B(㍿　 'Tß'DABCAb>HTTPServer\r#$%\r\nDžunglaEOT\r\n\r\n12345678字", "tokens": 44, "pieces": ["s", "123", "456", "78", "!'", "ll", " \n", " '", "Re", "Ⅳ", "s", "　B", "(㍿", "　", " ", "'Tß'D", "ABCAb", ">HTTPServer", "\r", "#$%\r\n", "Džungla", "EOT", "\r\n\r\n", "123", "456", "78", "字"]} +{"text": "Ａ'D's /ḍ̇ABCEOTſḍ̇s", "tokens": 15, "pieces": ["Ａ'D", "'s", " /", "ḍ̇", "ABCEOTſḍ̇s"]} +{"text": "<|endoftext|>Ⅳm're0a/b
méé's\n/Dž'Dd\n漢漢<|endoftext|>　é \naB🙂३'ReB‍m'ſ㍿.camelCase", "tokens": 59, "pieces": ["<|", "endoftext", "|>", "Ⅳ", "m're", "0", "a", "/b", "
m", "éé's", "\n", "/Dž'D", "d", "\n", "漢漢", "<|", "endoftext", "|>", "　é", " \n", "a", "B", "🙂", "३", "'Re", "B", "‍m'ſ", "㍿.", "camel", "Case"]} +{"text": "éAb½dḍ̇d'T\rꟲᵃ>HTTPServera/b­'VEé३<…\n/<|endoftext|>Ａ<|fim_prefix|>", "tokens": 46, "pieces": ["é", "Ab", "½", "dḍ̇", "d'T", "\r", "ꟲᵃ", ">HTTPServera", "/b", "­'", "VEé", "३", "<", "…\n", "/<|", "endoftext", "|>", "Ａ", "<|", "fim", "_prefix", "|>"]} +{"text": "㍿\ré ſéſ́ \r\n\r\n!ع(fi-ع
㋿camelCase#$%HTTPServerİ é…字 \n \"", "tokens": 39, "pieces": ["㍿\r", "é", " ſéſ́", " \r\n\r\n", "!ع", "(fi", "-ع", "
", "㋿camel", "Case", "#$%", "HTTPServer", "İ", " é", "", "…字", " \n", " \""]} +{"text": "Ａ　Bé ꟲBAb,de🙂
", "tokens": 13, "pieces": ["Ａ", "　Bé", " ", " ꟲBAb", ",de", "🙂", "
"]} +{"text": "\u000b<𐞁\"ᵃå,'T㍿㋿ABC\n/<|endoftext|>​'T👍🏽ꟲꟲİ'VE'\r\n\r\n,
a/b", "tokens": 52, "pieces": ["\u000b", "<𐞁", "\"ᵃå", ",'", "T", "㍿㋿", "ABC", "\n", "/<|", "endoftext", "|><", "META", "_START", ">​'", "T", "👍🏽", "ꟲꟲ", "İ'VE", "'\r\n\r\n", ",", "
a", "/b"]} +{"text": "
'll/㍿ع<|fim_prefix|>/\r\niOS\n/…\r\n\r\n　éå­ḍ̇'SDžungla𐞁9", "tokens": 43, "pieces": ["
", "'ll", "/㍿", "ع", "<|", "fim", "_prefix", "|><", "META", "_START", ">/\r\n", "i", "OS", "\n", "/", "…\r\n\r\n", "　éå", "­ḍ̇'S", "Džungla𐞁", "9"]} +{"text": "a'll9Z ", "tokens": 5, "pieces": ["a'll", "9", "Z", " "]} +{"text": "Dž'M字fi.'reİ'ſ　 #$%", "tokens": 14, "pieces": ["Dž'M", "字fi", ".'", "re", "İ'ſ", "　 ", " #$%"]} +{"text": "/ß'T½㍿<\rcamelCase<|endoftext|>'re", "tokens": 19, "pieces": ["/ß'T", "½", "㍿<\r", "camel", "Case", "<|", "endoftext", "|>'", "re"]} +{"text": "ß\u000ba/b<|fim_prefix|>½'T…!İ(𐞁漢‍\ta👍🏽'M\r<|endoftext|>t", "tokens": 38, "pieces": ["ß", "\u000ba", "/b", "<|", "fim", "_prefix", "|>", "½", "'T", "…", "!İ", "(𐞁漢", "‍", "\ta", "👍🏽'", "M", "\r", "<|", "endoftext", "|>", "t"]} +{"text": "👍🏽\n/!!iOS12345678", "tokens": 14, "pieces": ["👍🏽\n/", "!!", "i", "OS", "123", "456", "78", ""]} +{"text": "'VE 'll🙂ḍ̇\rs", "tokens": 15, "pieces": ["'VE", " ", " '", "ll", "🙂ḍ̇", "\r", "s", ""]} +{"text": "camelCase½Z…HTTPServer-'llſB's<|fim_prefix|>\n/\r\n \n fifi'.٣٤٥٦at'ſ", "tokens": 31, "pieces": ["camel", "Case", "½", "Z", "…HTTPServer", "-'", "llſ", "B's", "<|", "fim", "_prefix", "|>\n/\r\n", " \n", " fifi", "'.", "٣٤٥", "٦", "at'ſ"]} +{"text": "e'S#$%👍🏽iOSⅣ😀🏽/\r\n­'DžunglaEOT#$%'DiOS/#$%sB\r'res", "tokens": 38, "pieces": ["e'S", "#$%👍🏽", "i", "OS", "Ⅳ", "😀🏽/\r\n", "­'", "Džungla", "EOT", "#$%'", "Di", "OS", "/#$%", "s", "B", "\r", "'res"]} +{"text": "　9㋿a/b㍿'Re३字\n/#$%B
<|fim_prefix|>…a", "/b", "㍿'", "Re", "३", "字", "\n", "/#$%", "B", "
", "<|", "fim", "_prefix", "|>", "…", "mع㍿EOT\n<|fim_prefix|> \nEOT\r\nDžungla\"\r\nⅣ😀🏽Ａ'Re.<'ll<|endoftext|> eDžungla👍🏽漢12345678B\n'Re e​ \n ", "tokens": 62, "pieces": ["mع", "㍿EOT", "\n", "<|", "fim", "_prefix", "|>", " \n", "EOT", "\r\n", "Džungla", "\"\r\n", "Ⅳ", "😀🏽", "Ａ'Re", ".<'", "ll", "<|", "endoftext", "|>", " e", "Džungla", "👍🏽", "漢", "123", "456", "78", "B", "\n", "'Re", " e", "​", " \n", " "]} +{"text": "́٣٤٥٦ \n ‍\u000béaBiOS", "tokens": 12, "pieces": ["́", "٣٤٥", "٦", " \n", " ‍", "\u000béa", "Bi", "OS"]} +{"text": "t🙂.#$%> camelCase ＡHTTPServerté…!Ⅳ𐞁12345678Zsm!‍ ⅣB\n\n/", "tokens": 39, "pieces": ["t", "🙂.#$%>", " ", " camel", "Case", " ", " ＡHTTPServerté", "…", "!", "Ⅳ", "𐞁", "123", "456", "78", "Zsm", "!‍", " ", " ", "Ⅳ", "B", "\n\n", "/"]} +{"text": "́e-👍🏽Ab'Re'ſ'Re.𐞁­𐞁ß's//\r\n٣٤٥٦(🙂0/ \n \r‍
", "tokens": 41, "pieces": ["́e", "-👍🏽<", "EOT", ">Ab'Re", "'ſ'Re", ".𐞁", "­𐞁ß's", "//\r\n", "٣٤٥", "٦", "(🙂", "0", "/", " \n \r", "‍", "
"]} +{"text": "d<'D", "tokens": 3, "pieces": ["d", "<'", "D"]} +{"text": "fi12345678ABC/\r\nᵃAb#$%İ12345678BiOS\n/
/½́
Ａ <|fim_prefix|>́", "tokens": 33, "pieces": ["fi", "123", "456", "78", "ABC", "/\r\n", "ᵃAb", "#$%", "İ", "123", "456", "78", "Bi", "OS", "\n", "/", "
", "/", "½", "́", "
Ａ", " <|", "fim", "_prefix", "|>́"]} +{"text": "Ⅳ́0é'́\r\nDž🙂​\r\n\r\n<|fim_prefix|>\n👍🏽​", "tokens": 30, "pieces": ["Ⅳ", "́", "", "0", "é", "'́", "\r\n", "Dž", "🙂<", "EOT", ">​\r\n\r\n", "<|", "fim", "_prefix", "|>\n", "👍🏽​"]} +{"text": " \n é/\n\r9'M
 \n ㍿ddḍ̇é\u000bع\r\n\r\n", "tokens": 21, "pieces": [" \n", " é", "/\n\r", "9", "'M", "
 \n", " ㍿", "ddḍ̇é", "\u000bع", "\r\n\r\n"]} +{"text": "'VE<|fim_prefix|>½t>\u000bİd­é㍿\n/漢å", "tokens": 28, "pieces": ["'VE", "<|", "fim", "_prefix", "|>", "½", "t", ">", "\u000bİd", "­é", "㍿\n/", "漢å"]} +{"text": "\r\n\r\nᵃcamelCaseZ12345678'D'Tm字㋿å\r\n\r\n'Re!!'ll\r\n12345678३🙂\nABCＡ漢-​㍿\n#$%HTTPServer!!'Tß​<|fim_prefix|>漢å!!/\r\n", "tokens": 63, "pieces": ["\r\n\r\n", "ᵃcamel", "Case", "Z", "123", "456", "78", "'D'T", "m字", "㋿å", "\r\n\r\n", "'Re", "!!'", "ll", "\r\n", "123", "456", "78३", "🙂\n", "ABCＡ漢", "-​㍿\n", "#$%", "HTTPServer", "!!'", "Tß", "​<|", "fim", "_prefix", "|>", "漢å", "!!/\r\n"]} +{"text": "'s\n/", "tokens": 3, "pieces": ["'s", "\n", "/"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'re漢", "tokens": 2, "pieces": ["'re漢"]} +{"text": "ſ'a9Džungla", "tokens": 7, "pieces": ["ſ", "'a", "9", "Džungla"]} +{"text": "\n/\r\nm/'ſ​d'M٣٤٥٦ABC d٣٤٥٦ع\r!\r!ᵃ𐞁 (Dž'\r\n\r\nſ> ​é　", "tokens": 44, "pieces": ["\n", "/\r\n", "m", "/'", "ſ", "​", "d'M", "٣٤٥", "٦", "ABC", " d", "٣٤٥", "٦", "ع", "\r", "!\r", "!ᵃ𐞁", " ", "(Dž", "'\r\n\r\n", "ſ", ">", " ", "​é", "　"]} +{"text": ",'reZꟲᵃ/", "tokens": 10, "pieces": [",'", "re", "Zꟲᵃ", "/"]} +{"text": "\rⅣ'ſDžHTTPServer'Reå12345678㋿9ſ㋿ABC🙂BåaDžungla‍!camelCase12345678<|endoftext|>/' ㋿\"'Ts", "tokens": 53, "pieces": ["\r", "Ⅳ", "'ſ", "DžHTTPServer'Re", "å", "123", "456", "78", "㋿", "9", "ſ", "㋿ABC", "🙂Båa", "Džungla", "‍!", "camel", "Case", "123", "456", "78", "<|", "endoftext", "|>/'", " ", "㋿\"'", "Ts"]} +{"text": "\n'a/b‍<|fim_prefix|>½字EOT३", "tokens": 15, "pieces": ["\n", "'a", "/b", "‍<|", "fim", "_prefix", "|>", "½", "字", "EOT", "३"]} +{"text": " \n ع\r\n\r\nå😀🏽#$%  'M‍İ ㍿\r\n​'S­/\r\ncamelCase'S>…#$%a/b\n/Bé", "tokens": 42, "pieces": [" \n", " ع", "\r\n\r\n", "å", "😀🏽#$%", " ", " ", "'M", "‍İ", " ", " ㍿\r\n", "​'", "S", "­/\r\n", "camel", "Case'S", ">", "…", "#$%", "a", "/b", "\n", "/Bé"]} +{"text": " (#$%\"é.camelCase(-t　\r\n\r\n>㍿\n/\r\n \n 's㋿a/b\t<|fim_prefix|>٣٤٥٦ᵃt\t > ", "tokens": 44, "pieces": [" ", " (#$%\"", "é", ".camel", "Case", "(-", "t", "　\r\n\r\n", ">㍿\n/\r\n", " \n", " '", "s", "㋿a", "/b", "\t", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ᵃt", "\t", " >", " "]} +{"text": "Z㋿EOT㋿'sdZⅣ're(dꟲ㍿'½عß", "tokens": 26, "pieces": ["Z", "㋿EOT", "㋿'", "sd", "Z", "Ⅳ", "'re", "(dꟲ", "㍿'", "½", "عß"]} +{"text": "ABC/\r\na👍🏽ᵃcamelCase½mß́camelCase👍🏽t \n<|fim_prefix|>a 12345678/漢Za/b́m\r>aB
m ́'M'ſ
　>…\u000b'll", "tokens": 56, "pieces": ["ABC", "/\r\n", "a", "👍🏽", "ᵃcamel", "Case", "½", "mß́camel", "Case", "👍🏽", "t", " \n", "<|", "fim", "_prefix", "|>", "a", " ", "123", "456", "78", "/漢Za", "/b́m", "\r", ">a", "B", "
m", " ́'M", "'ſ", "
", "　", ">", "…", "\u000b", "'ll"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r\n३㋿é㋿", "tokens": 10, "pieces": ["\r\n", "३", "㋿é", "㋿"]} +{"text": " \r\n\r\n!ḍ̇ABC'VEå𐞁<|fim_prefix|>'s٣٤٥٦ꟲé<|endoftext|>a/btHTTPServeŕ'VEḍ̇!!\"t\t \n ,camelCase'VE>camelCase\n👍🏽", "tokens": 72, "pieces": [" \r\n\r\n", "!ḍ̇", "ABC'VE", "å𐞁", "<|", "fim", "_prefix", "|>'", "s", "٣٤٥", "٦", "ꟲ", "é", "<|", "endoftext", "|>", "a", "/bt", "HTTPServeŕ'VE", "ḍ̇", "!!\"<", "EOT", ">t", "\t \n", " ,", "camel", "Case'VE", ">camel", "Case", "\n", "👍🏽"]} +{"text": ".㋿!a \n !aBḍ̇‍👍🏽12345678Dž", "tokens": 25, "pieces": [".<", "EOT", ">㋿!", "a", " \n", " !", "a", "Bḍ̇", "‍👍🏽", "123", "456", "78", "Dž"]} +{"text": "عZ \n\nİ\u000b. …'s/İ,👍🏽", "tokens": 16, "pieces": ["ع", "Z", " \n\n", "İ", "\u000b", ".", " ", "…", "'s", "/İ", ",👍🏽"]} +{"text": "s​", "tokens": 2, "pieces": ["s", "​"]} +{"text": "३EOT٣٤٥٦d\t're0iOSABCAbſaB\r\n\r\nİ\t!<|endoftext|>㍿ ع'llꟲaB३\"aB​.ᵃ9 t", "tokens": 52, "pieces": ["३", "EOT", "٣٤٥", "٦", "d", "\t", "'re", "0", "i", "OSABCAbſa", "B", "\r\n\r\n", "İ", "\t", "!<|", "endoftext", "|>㍿<", "EOT", ">", " ع'll", "ꟲa", "B", "३", "\"a", "B", "​.", "ᵃ", "9", " t"]} +{"text": "…‍ß\t字​…aB<|endoftext|>　'ſعAb　/😀🏽", "tokens": 24, "pieces": ["漢", "<|", "endoftext", "|><|", "endoftext", "|>", "　", "'ſع", "Ab", "　", "/😀🏽"]} +{"text": "iOS\r\n\r\n　'Džungla<|endoftext|>'D
EOTſ", "tokens": 21, "pieces": ["i", "OS", "\r\n\r\n", "　", "'Džungla", "<|", "endoftext", "|>'", "D", "
EOTſ"]} +{"text": "!😀🏽字'M‍Dž0 12345678fi½…a", "tokens": 19, "pieces": ["!😀🏽", "字'M", "‍Dž", "0", " ", "123", "456", "78", "fi", "½", "…a"]} +{"text": "'S'Mm'\"#$%'re\t-👍🏽㍿\r\n\r\n.Z'ſEOT AbAb<|fim_prefix|>/. \n 字iOS‍ḍ̇字9", "tokens": 45, "pieces": ["'S'M", "m", "'\"#$%'", "re", "\t", "-<", "META", "_START", ">👍🏽㍿\r\n\r\n", ".Z'ſ", "EOT", " ", " Ab", "Ab", "<|", "fim", "_prefix", "|>/.", " \n", " 字i", "OS", "‍ḍ̇字", "9"]} +{"text": "　's\n/㋿'Re👍🏽/\r\n'　fi'S/DžunglacamelCase \n's(́m<|fim_prefix|>ᵃ👍🏽\r,'Re'M­𐞁\r\n\r\n\"字­🙂e<|endoftext|>漢Z12345678", "tokens": 68, "pieces": ["　", "'s", "\n", "/㋿'", "Re", "👍🏽/\r\n", "'", "　fi'S", "/Džunglacamel", "Case", " \n", "'s", "(́m", "<|", "fim", "_prefix", "|>", "ᵃ", "👍🏽\r", ",'", "Re'M", "­𐞁", "\r\n\r\n", "\"字", "­🙂", "e", "<|", "endoftext", "|>", "漢", "Z", "123", "456", "78"]} +{"text": "camelCase<ꟲ㍿\"ع😀🏽😀🏽", "tokens": 20, "pieces": ["camel", "Case", "<ꟲ", "㍿\"", "ع", "😀🏽😀🏽"]} +{"text": "é'D‍<|fim_prefix|>ḍ̇å½ſ​…ḍ̇'S'VE\r\n\r\n'll字a0ᵃ0Dž३0 \n ß \u000b𐞁ABCDž", "tokens": 50, "pieces": ["é'D", "‍<|", "fim", "_prefix", "|>", "ḍ̇å", "½", "ſ", "​", "…ḍ̇'S", "'VE", "\r\n\r\n", "'ll字a", "0", "ᵃ", "0", "Dž", "३0", " \n", " ß", " ", "\u000b𐞁", "ABCDž"]} +{"text": "!/\r\n‍s­Dža/b😀🏽", "tokens": 12, "pieces": ["!/\r\n", "‍s", "­Dža", "/b", "😀🏽"]} +{"text": "é's‍…٣٤٥٦t<|fim_prefix|>'VE٣٤٥٦,a/bDžungla\nAb/\r\n/\r\n\tacamelCase9éABC", "tokens": 39, "pieces": ["é's", "‍", "…", "٣٤٥", "٦", "t", "<|", "fim", "_prefix", "|>'", "VE", "٣٤٥", "٦", ",a", "/b", "Džungla", "\n", "Ab", "/\r\n/\r\n", "\tacamel", "Case", "9", "é", "ABC"]} +{"text": " ⅣéZİ­'ſå\r\n\r\nAbAb🙂٣٤٥٦'ſᵃꟲ😀🏽ᵃ/ 12345678/\r\nABC𐞁a𐞁ع字12345678😀🏽ع", "tokens": 65, "pieces": [" ", "Ⅳ", "é", "Zİ", "­'", "ſå", "\r\n\r\n", "Ab", "Ab", "🙂", "٣٤٥", "٦", "'ſᵃꟲ", "😀🏽", "ᵃ", "/<", "META", "_START", ">", " ", " ", "123", "456", "78", "/\r\n", "ABC𐞁a𐞁ع字", "123", "456", "78", "😀🏽", "ع", ""]} +{"text": ">'t", "tokens": 2, "pieces": [">'", "t"]} +{"text": "ꟲꟲe!aB­\r漢,é​éHTTPServerDžungla ‍ #$%< #$%ᵃ👍🏽/\r\nHTTPServer🙂　-9­DžZ𐞁", "tokens": 54, "pieces": ["ꟲꟲe", "!a", "B", "­\r", "漢", ",é", "​é", "HTTPServer", "Džungla", " ‍", " #$%<", " ", " #$%", "ᵃ", "👍🏽/\r\n", "HTTPServer", "🙂", "　", "-", "9", "­DžZ𐞁"]} +{"text": "!'s'>'Re9>३🙂Džunglaß字字aB\r<|endoftext|>ＡDžDž'll<|endoftext|>ᵃaB!<|fim_prefix|>ḍ̇́å \n٣٤٥٦", "tokens": 62, "pieces": ["!'", "s", "'>'", "Re", "9", ">", "३", "🙂Džunglaß字字a", "B", "\r", "<|", "endoftext", "|>", "ＡDžDž'll", "<|", "endoftext", "|>", "ᵃa", "B", "!<|", "fim", "_prefix", "|>", "ḍ̇́å", " \n", "٣٤٥", "٦"]} +{"text": "aⅣ<|endoftext|>åß㋿/ \n  \rß-\r\n! \nع'reſDžungla　<|fim_prefix|>İ,‍字", "tokens": 45, "pieces": ["a", "Ⅳ", "<|", "endoftext", "|>", "åß", "㋿/<", "EOT", ">", " \n  \r", "ß", "-\r\n", "!", " \n", "ع're", "ſ", "Džungla", "　", "<|", "fim", "_prefix", "|>", "İ", ",‍", "字"]} +{"text": "'ſ", "tokens": 2, "pieces": ["'ſ"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ABCß/\n/>'aB㋿e0\nع'\n𐞁 \n😀🏽.é𐞁/\r\n　é­ İ'll<|endoftext|>!", "tokens": 44, "pieces": ["ABCß", "/\n/", ">'", "a", "B", "㋿e", "0", "\n", "ع", "'\n", "𐞁", " \n", "😀🏽.", "é𐞁", "/\r\n", "　é", "­", " İ'll", "<|", "endoftext", "|>!"]} +{"text": ".és٣٤٥٦İfi'D\n", "tokens": 10, "pieces": [".és", "٣٤٥", "٦", "İfi'D", "\n"]} +{"text": "/\r\n'a/b'ſZDž \n𐞁ꟲ'T<,,<|endoftext|>㍿'DABCꟲ👍🏽ᵃ\u000b३ \nＡ", "tokens": 49, "pieces": ["/\r\n", "'", "a", "/b'ſ", "ZDž", " \n", "𐞁ꟲ'T", "<,,<|", "endoftext", "|>㍿'", "DABCꟲ", "👍🏽", "ᵃ", "\u000b", "३", " \n", "Ａ"]} +{"text": "😀🏽ꟲtAb/\r\n/\r\n३٣٤٥٦Ⅳ​>/EOTcamelCaseＡ٣٤٥٦", "tokens": 28, "pieces": ["😀🏽", "ꟲt", "Ab", "/\r\n/\r\n", "३٣٤", "٥٦Ⅳ", "​>/", "EOTcamel", "Case", "Ａ", "٣٤٥", "٦"]} +{"text": "å \n　å　字Ａ㋿\ra/bß'Re!­\n/!㋿", "tokens": 24, "pieces": ["å", " \n", "　å", "　字", "Ａ", "㋿\r", "a", "/bß'Re", "!­\n/", "!㋿"]} +{"text": "a/bé9㍿عå'M👍🏽 iOS/ᵃ", "tokens": 23, "pieces": ["a", "/bé", "9", "㍿عå'M", "👍🏽", " i", "OS", "/<", "META", "_START", ">ᵃ"]} +{"text": "ß/\r\nt(‍'S!'TiOS0a/bİDž㋿ \n ,　å'EOT<|endoftext|>a/bm'\r½", "tokens": 51, "pieces": ["ß", "/\r\n", "t", "(‍'", "S", "!'", "Ti", "OS", "0", "a", "/b", "İDž", "㋿", " \n", " ,", "　", "å", "'EOT", "<|", "endoftext", "|>", "a", "/bm", "'\r", "½"]} +{"text": "'ſ'sß'SsABC😀🏽\t'T'll\r\n३'٣٤٥٦ḍ̇\r\n'T漢/İ12345678aBEOT", "tokens": 34, "pieces": ["'ſ's", "ß'S", "s", "ABC", "😀🏽", "\t", "'T'll", "\r\n", "३", "'", "٣٤٥", "٦", "ḍ̇", "\r\n", "'T漢", "/İ", "123", "456", "78", "a", "BEOT"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " 字0a　'VE\t'VE　𐞁'ſ\u000b𐞁​", "tokens": 22, "pieces": [" 字", "0", "a", "　", "'VE", "\t", "'VE", "　𐞁'ſ", "\u000b𐞁", "​"]} +{"text": "\r\n<|endoftext|>", "tokens": 8, "pieces": ["\r\n", "<|", "endoftext", "|>"]} +{"text": "fiEOT🙂.,aB\n/\n/ 🙂, <|fim_prefix|>'Mdſ camelCaseEOT🙂…👍🏽\u000b٣٤٥٦a!!EOT'Re👍🏽fiå'll<|endoftext|>‍m", "tokens": 56, "pieces": ["fi", "EOT", "🙂.,", "a", "B", "\n", "/\n/", " ", "🙂,", " <|", "fim", "_prefix", "|>'", "Mdſ", " camel", "Case", "EOT", "🙂", "…", "👍🏽", "\u000b", "٣٤٥", "٦", "a", "!!", "EOT'Re", "👍🏽", "fiå'll", "<|", "endoftext", "|>‍", "m"]} +{"text": "​12345678m!!éİéiOSABC'll　>Ⅳmİ<|fim_prefix|>\r!Ab /camelCase're'M12345678½'M \n /\r\nm-'ſ HTTPServer🙂", "tokens": 50, "pieces": ["​", "123", "456", "78", "m", "!!", "é", "İéi", "OSABC'll", "　", ">", "Ⅳ", "m", "İ", "<|", "fim", "_prefix", "|>\r", "!Ab", " ", " /", "camel", "Case're", "'", "M", "123", "456", "78½", "'M", " \n", " /\r\n", "m", "-'", "ſ", " HTTPServer", "🙂"]} +{"text": "ABC're\r\n\r\n'TEOT'VEa/b(​字\r\n\t", "tokens": 15, "pieces": ["ABC're", "\r\n\r\n", "'TEOT'VE", "a", "/b", "(​", "字", "\r\n", "\t"]} +{"text": "㍿字EOTİfi'ſ(", "tokens": 11, "pieces": ["㍿字EOTİfi'ſ", "("]} +{"text": "( Džunglá​㋿İ'VE'!", "tokens": 20, "pieces": ["(<", "META", "_START", ">", " ", " Džunglá", "​㋿", "İ'VE", "'!"]} +{"text": " \nfi‍字é‍\"HTTPServer­-é'fi　\n/\n're\u000b㋿d'Ree", "tokens": 29, "pieces": [" <", "EOT", ">", " \n", "fi", "‍字é", "‍\"", "HTTPServer", "­-", "é", "'fi", "　\n", "/\n", "'re", "\u000b", "㋿d'Re", "e"]} +{"text": "/Ａt'T㋿/\r\n
0HTTPServerEOT½ \tß‍𐞁\" ABC‍…'M'D/\r\nAb,​a/b'Mع-𐞁\t", "tokens": 48, "pieces": ["/Ａt'T", "㋿/\r\n", "
", "0", "HTTPServer", "EOT", "", "½", " ", "\tß", "‍𐞁", "\"", " ", " ABC", "‍", "…", "'M'D", "/\r\n", "Ab", ",​", "a", "/b'M", "ع", "-𐞁", "\t"]} +{"text": "'ſＡé\r\n\r\na'S👍🏽<|fim_prefix|>‍漢​'ll", "tokens": 25, "pieces": ["'ſ", "Ａé", "\r\n\r\n", "a'S", "👍🏽<|", "fim", "_prefix", "|>‍<", "EOT", ">漢", "​'", "ll"]} +{"text": " ㍿>'VE\t'Té'VEZعDžsＡ\t>…<Džungla>", "tokens": 27, "pieces": [" ㍿>'", "VE", "\t", "'Té'VE", "ZعDžs", "Ａ", "\t", ">", "…", "<Džungla", ">"]} +{"text": "HTTPServer('ſHTTPServer३camelCaseİBDž\rå>tＡ'M#$%/å٣٤٥٦9ꟲé　ta/\r\n<|fim_prefix|>s.ꟲ\r\n/", "tokens": 54, "pieces": ["HTTPServer", "('", "ſ", "HTTPServer", "३", "camel", "Case", "İBDž", "\r", "å", ">t", "Ａ'M", "#$%/", "å", "٣٤٥", "٦9", "ꟲé", "　ta", "/\r\n", "<|", "fim", "_prefix", "|>", "s", ".ꟲ", "\r\n", "/"]} +{"text": "a\n/Ab", "tokens": 41, "pieces": ["a", "\n", "/Ab", ""]} +{"text": "
/(em'T's(漢'Re　 \n e　're\n/Z'ſå\r\n\r\n#$%#$%٣٤٥٦'Re
camelCase­'VE🙂", "tokens": 39, "pieces": ["
", "/(", "em'T", "'s", "(漢'Re", "　 \n", " e", "　", "'re", "\n", "/Z'ſ", "å", "\r\n\r\n", "#$%#$%", "٣٤٥", "٦", "'Re", "
camel", "Case", "­'", "VE", "🙂"]} +{"text": "<|fim_prefix|>0 <|fim_prefix|>Ⅳ́ZcamelCaseⅣ‍'Re-<|endoftext|> ſ0a/bmᵃ", "tokens": 42, "pieces": ["<|", "fim", "_prefix", "|>", "0", " <|", "fim", "_prefix", "|>", "Ⅳ", "́Zcamel", "Case", "Ⅳ", "‍'", "Re", "-<|", "endoftext", "|>", " ", " ſ", "0", "a", "/bmᵃ"]} +{"text": "字éİ#$%ſZꟲᵃa iOS,camelCase👍🏽a​a0a0\u000bEOT😀🏽é", "tokens": 37, "pieces": ["字é", "İ", "#$%", "ſ", "Zꟲᵃa", " ", " i", "OS", ",camel", "Case", "👍🏽", "a", "​a", "0", "a", "0", "\u000bEOT", "😀🏽", "é"]} +{"text": "\u000ba/bＡdeᵃᵃ(漢½
fi٣٤٥٦a/b", "tokens": 44, "pieces": ["\u000ba", "/b", "Ａdeᵃᵃ", "(漢", "", "½", "
fi", "٣٤٥", "٦", "a", "/b"]} +{"text": "İ'VEİdDžungla​!!Abd\r'll'S'D'VE\n/", "tokens": 21, "pieces": ["İ'VE", "İd", "Džungla", "​!!", "Abd", "\r", "'ll'S", "'D'VE", "\n", "/"]} +{"text": "Ab½#$%'ll­(.𐞁३\r'll­😀🏽-d'D३½ع漢­'😀🏽
ꟲ./\r\n \naB\n\n0'Re \nd<|endoftext|>", "tokens": 55, "pieces": ["Ab", "½", "#$%'", "ll", "­(.<", "EOT", ">𐞁", "३", "\r", "'ll", "­😀🏽-", "d'D", "३½", "ع漢", "­'😀🏽", "
ꟲ", "./\r\n", " \n", "a", "B", "\n\n", "0", "'Re", " \n", "d", "<|", "endoftext", "|>"]} +{"text": "fi😀🏽\r\n\r\n<ع #$%\u000b \n 
", "tokens": 16, "pieces": ["fi", "😀🏽\r\n\r\n", "<", "ع", " ", "#$%", "\u000b \n", " 
"]} +{"text": "#$%'Me.Džunglaa'T٣٤٥٦a…fi\r\n\r\na'Tḍ̇字éAbAbå'Té're're-'\r\n\r\nscamelCasefi", "tokens": 41, "pieces": ["#$%'", "Me", ".Džunglaa'T", "٣٤٥", "٦", "a", "…fi", "\r\n\r\n", "a'T", "ḍ̇字é", "Ab", "Abå'T", "é're", "'re", "-'\r\n\r\n", "scamel", "Casefi"]} +{"text": "#$% \nḍ̇ZsⅣ<|endoftext|>Ⅳ㍿😀🏽", "tokens": 25, "pieces": ["#$%", " \n", "ḍ̇", "Zs", "Ⅳ", "<|", "endoftext", "|>", "Ⅳ", "㍿😀🏽"]} +{"text": "a/b­😀🏽'TＡع㍿>'re٣٤٥٦a/b\"𐞁a/b0 😀🏽ß'Re<|fim_prefix|>", "tokens": 40, "pieces": ["a", "/b", "­😀🏽'", "TＡع", "㍿>'", "re", "٣٤٥", "٦", "a", "/b", "\"𐞁a", "/b", "0", " 😀🏽", "ß'Re", "<|", "fim", "_prefix", "|>"]} +{"text": "sB,'D", "tokens": 4, "pieces": ["s", "B", ",'", "D"]} +{"text": "dt漢m#$%B𐞁fiḍ̇٣٤٥٦\n'sꟲDžunglaa👍🏽'VEſ字Ａ㍿ſſB\u000be>\r\n'DİſtiOS ᵃcamelCase>iOS", "tokens": 63, "pieces": ["dt漢m", "#$%<", "EOT", ">B𐞁fiḍ̇", "٣٤٥", "٦", "\n", "'sꟲ", "Džunglaa", "👍🏽'", "VEſ字", "Ａ", "㍿ſſ", "B", "\u000be", ">\r\n", "'Dİſti", "OS", " ", " ᵃcamel", "Case", ">i", "OS"]} +{"text": "'T-Ａ😀🏽ſ'TEOT \n \r\n\r\n\n/\nİꟲ漢>fimaBſ३", "tokens": 57, "pieces": ["'T", "-Ａ", "😀🏽", "ſ'T", "EOT", "", " \n \r\n\r\n\n", "/\n", "İꟲ漢", ">fima", "B", "", "ſ", "३"]} +{"text": "d\r\nꟲⅣ", "tokens": 7, "pieces": ["d", "\r\n", "ꟲ", "Ⅳ"]} +{"text": "'MiOS(\r\n\r\na/b'Mḍ̇éé\nİ‍fi\r\n\r\nßḍ̇ſⅣ'\"-İ(\u000b'VE(", "tokens": 56, "pieces": ["'Mi", "OS", "(\r\n\r\n", "a", "/b'M", "ḍ̇", "", "éé", "\n", "İ", "‍fi", "\r\n\r\n", "ßḍ̇", "ſ", "Ⅳ", "'\"-", "İ", "(", "\u000b", "'VE", "("]} +{"text": ",…\r‍Dž​'reé0\rDžungla'Re'Dfi/\r\n३­t'Rett'T'TDž字
٣٤٥٦>'VEa/b'Re\u000béé½Džİ'M0", "tokens": 49, "pieces": [",", "…\r", "‍Dž", "​'", "reé", "0", "\r", "Džungla'Re", "'Dfi", "/\r\n", "३", "­t'Re", "tt'T", "'TDž字", "
", "٣٤٥", "٦", ">'", "VEa", "/b'Re", "\u000béé", "½", "Džİ'M", "0"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "t#$%\t㋿EOT", "tokens": 9, "pieces": ["t", "#$%", "\t", "㋿EOT"]} +{"text": "90're \n(#$%-Dž　.ᵃ
'll'ſſ­'DDžungla.漢åaBed👍🏽 \n ع!!0\"/字", "tokens": 44, "pieces": ["90", "'re", "", " \n", "(#$%-", "Dž", "　", ".ᵃ", "
", "'ll'ſ", "ſ", "­'", "DDžungla", ".漢åa", "Bed", "👍🏽", " \n", " ع", "!!", "0", "\"/", "字"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "0>
Ａ😀🏽'ſ٣٤٥٦!/\r\n!\r\n\r\n >é'VEⅣB'Ss😀🏽'sAb…-ᵃḍ̇iOS…/\r\ncamelCase\r\n#$%", "tokens": 52, "pieces": ["0", ">", "
Ａ", "😀🏽'", "ſ", "٣٤٥", "٦", "!/\r\n", "!\r\n\r\n", " ", " >", "é'VE", "Ⅳ", "B'S", "s", "😀🏽'", "s", "Ab", "…", "-ᵃḍ̇i", "OS", "…", "/\r\n", "camel", "Case", "\r\n", "#$%"]} +{"text": "Džfi0é\n/½'T#$%Ⅳ>…é\t‍d𐞁 <|endoftext|>!ß#$%tİİa‍/ع>…'S'D🙂", "tokens": 56, "pieces": ["Džfi", "0", "é", "\n", "/", "½", "'T", "#$%", "Ⅳ", ">", "…", "é", "\t", "‍d𐞁", " ", "<|", "endoftext", "|>!", "ß", "#$%", "t", "İİa", "‍/", "ع", ">", "…", "'S'D", "🙂"]} +{"text": "'M\r\n\u000b\ta/b\n//\r\n\r\nع㍿,Abᵃ\r", "tokens": 18, "pieces": ["'M", "\r\n", "\u000b", "\ta", "/b", "\n", "//\r\n\r\n", "ع", "㍿,", "Abᵃ", "\r"]} +{"text": "‍ 　ſABCå\"\naé👍🏽 å", "\"\n", "aé", "👍🏽", " ", " éꟲ😀🏽camelCase,åDž\n/sé👍🏽Ab\rع'llcamelCase ,-iOS\rDžungla३…ꟲ㍿'re/aB", "tokens": 60, "pieces": ["9", "漢", "<|", "endoftext", "|>", "é", "ꟲ", "😀🏽", "camel", "Case", ",å", "Dž", "\n", "/sé", "👍🏽", "Ab", "\r", "ع'll", "camel", "Case", " ", " ,-", "i", "OS", "\r", "Džungla", "३", "…ꟲ", "㍿'", "re", "/a", "B"]} +{"text": "0<|fim_prefix|>👍🏽'VE'Re!'re,Džunglaſ\r 'ſ>'Reſt
㋿Z!!'S", "tokens": 37, "pieces": ["0", "<|", "fim", "_prefix", "|>👍🏽'", "VE'Re", "!'", "re", ",Džunglaſ", "\r", " ", " '", "ſ", ">'", "Reſt", "
", "㋿Z", "!!'", "S"]} +{"text": "<\" \n 👍🏽HTTPServer'llꟲeém😀🏽👍🏽.<ᵃⅣ\u000b>𐞁s('M\u000bcamelCaseå字éDžd're a/b-㍿'D''T", "tokens": 62, "pieces": ["<\"<", "META", "_START", ">", " \n", " 👍🏽", "HTTPServer'll", "ꟲeém", "😀🏽👍🏽.<", "ᵃ", "Ⅳ", "\u000b", ">𐞁s", "('", "M", "\u000bcamel", "Caseå字é", "Džd're", " ", " a", "/b", "-㍿'", "D", "''", "T"]} +{"text": "𐞁🙂'T<|endoftext|>\nDžungla \n'ReHTTPServer\raHTTPServerB ABC< <|endoftext|>aBⅣ​ écamelCase \n \r\n\r\n9Ａ漢Dž½.t<
fi
\n", "tokens": 61, "pieces": ["𐞁", "🙂'", "T", "<|", "endoftext", "|>\n", "Džungla", " \n", "'Re", "HTTPServer", "\r", "a", "HTTPServer", "B", " ", " ABC", "<", " ", " <|", "endoftext", "|>", "a", "B", "Ⅳ", "​", " écamel", "Case", " \n \r\n\r\n", "9", "Ａ漢", "Dž", "½", ".t", "<", "
fi", "
\n"]} +{"text": "t​ée ", "tokens": 4, "pieces": ["t", "​ée", " "]} +{"text": "iOS \n­< !! \n<㋿ 　\rſcamelCaseⅣꟲ३camelCase- عiOS'S 　㋿é", "tokens": 38, "pieces": ["i", "OS", " \n", "­<", " ", "!!", " \n", "<㋿", " 　\r", "ſcamel", "Case", "Ⅳ", "ꟲ", "३", "camel", "Case", "-", " عi", "OS'S", " ", "　", "㋿é"]} +{"text": "<|fim_prefix|>ḍ̇camelCase'se'TeHTTPServerEOT \r👍🏽d😀🏽'VEDž३", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>", "ḍ̇camel", "Case's", "e'T", "e", "HTTPServer", "EOT", "", " \r", "👍🏽", "d", "😀🏽'", "VE", "Dž", "३"]} +{"text": "HTTPServer'reeé\r\n \nß", "tokens": 8, "pieces": ["HTTPServer're", "eé", "\r\n \n", "ß"]} +{"text": ">'VE0ᵃ>", "tokens": 11, "pieces": [">'", "VE", "", "0", "ᵃ", ">"]} +{"text": "/\r\né'Dß<|fim_prefix|>🙂'T00\r\n\r\n\"ع'TcamelCase-\u000b12345678ꟲa/b", "tokens": 34, "pieces": ["/\r\n", "é'D", "ß", "<|", "fim", "_prefix", "|>🙂'", "T", "00", "\r\n\r\n", "\"ع", "'", "Tcamel", "Case", "-", "\u000b", "123", "456", "78", "ꟲa", "/b"]} +{"text": "Džungla12345678>HTTPServer/\r\n​'re'VE.㋿12345678​ B'll- Zm​'é'SعⅣ'DⅣ/\r\nعB́B'D'Re\u000b٣٤٥٦", "tokens": 52, "pieces": ["Džungla", "123", "456", "78", ">HTTPServer", "/\r\n", "​'", "re'VE", ".㋿", "123", "456", "78", "​", " B'll", "-", " Zm", "​'", "é'S", "ع", "Ⅳ", "'D", "Ⅳ", "/\r\n", "عB́", "B'D", "'Re", "\u000b", "٣٤٥", "٦"]} +{"text": "\t𐞁🙂Dž\nDž
#$%́ ‍ \n \r\n\r\n'MAbmdABC-", "tokens": 32, "pieces": ["\t𐞁", "🙂Dž", "\n", "Dž", "
", "#$%́<", "META", "_START", ">", " ", " ‍", " \n \r\n\r\n", "'MAb", "md", "ABC", "-"]} +{"text": "m½'D\n😀🏽\"🙂‍'Re　'D…éＡ́३ꟲEOT\n/\r'M\u000b\n<́", "tokens": 36, "pieces": ["m", "½", "'D", "\n", "😀🏽\"🙂‍'", "Re", "　", "'D", "…é", "Ａ́", "३", "ꟲ", "EOT", "\n", "/\r", "'M", "\u000b", "\n", "<́"]} +{"text": "<éBABC…👍🏽 'Ds'S0tHTTPServerABC​\r\n'll' 0<|endoftext|>", "tokens": 32, "pieces": ["<é", "BABC", "…", "👍🏽", " ", " '", "Ds'S", "0", "t", "HTTPServer", "ABC", "​\r\n", "'ll", "'", " ", " ", "0", "<|", "endoftext", "|>"]} +{"text": "ßm\t\r\n\r\n
iOSDžungla \n a/b", "tokens": 13, "pieces": ["ßm", "\t\r\n\r\n", "
i", "OSDžungla", " \n", " a", "/b"]} +{"text": "‍\r\n\r\naBع'M㍿t\n 'SZ/\r\n/\r\n½é \n're. (
字", "tokens": 25, "pieces": ["‍\r\n\r\n", "a", "Bع'M", "㍿t", "\n", " ", " '", "SZ", "/\r\n/\r\n", "½", "é", " \n", "'re", ".", " ", "(", "
字"]} +{"text": "漢ſå\r\n\r\n<|fim_prefix|>\r\n😀🏽EOT<|fim_prefix|>½ iOS \n", "tokens": 27, "pieces": ["漢ſå", "\r\n\r\n", "<|", "fim", "_prefix", "|>\r\n", "😀🏽", "EOT", "<|", "fim", "_prefix", "|>", "½", " ", " i", "OS", " \n"]} +{"text": "'sᵃ㋿å'DABC'Teå<-'D½½9å", "tokens": 27, "pieces": ["'sᵃ", "㋿å'D", "ABC'T", "eå", "<-'", "D", "½", "", "½9", "å"]} +{"text": "İ<|fim_prefix|>éeaBa/b\n/iOS'D \n EOT,Ab'll!🙂d\r\n\r\n\r\n\r\nᵃcamelCase \n ABC٣٤٥٦", "tokens": 37, "pieces": ["İ", "<|", "fim", "_prefix", "|>", "éea", "Ba", "/b", "\n", "/i", "OS'D", " \n", " EOT", ",Ab'll", "!🙂", "d", "\r\n\r\n\r\n\r\n", "ᵃcamel", "Case", " \n", " ABC", "٣٤٥", "٦"]} +{"text": "ⅣaBİ<|endoftext|>'M…#$%'VEDž ", "tokens": 21, "pieces": ["Ⅳ", "a", "Bİ", "<|", "endoftext", "|>'", "M", "…", "#$%'", "VEDž", " "]} +{"text": "a/b३t0/\r\n \n\r\n\r\n­\tḍ̇ᵃ'Re><|fim_prefix|>'THTTPServer字­camelCase \n'M ३'T'T", "tokens": 37, "pieces": ["a", "/b", "३", "t", "0", "/\r\n", " \n\r\n\r\n", "­", "\tḍ̇ᵃ'Re", "><|", "fim", "_prefix", "|>'", "THTTPServer字", "­camel", "Case", " \n", "'M", " ", " ", "३", "'T'T"]} +{"text": "​12345678😀🏽<|endoftext|>Dž \n ", "tokens": 28, "pieces": ["i", "OS", "🙂", " ", " B", "/\r\n", "\t", "㋿>", "123", "456", "78", "😀🏽<|", "endoftext", "|>", "Dž", " \n", " "]} +{"text": "ᵃ-\u000b12345678ſiOSa/b9<|endoftext|>'D'll'Dž'll'sſ#$%éſ<|endoftext|>9(EOTfi é#$%ée­­𐞁'llZ ㋿", "tokens": 63, "pieces": ["ᵃ", "-", "\u000b", "123", "456", "78", "ſi", "OSa", "/b", "9", "<|", "endoftext", "|>'", "D'll", "'Dž'll", "'sſ", "#$%", "éſ", "<|", "endoftext", "|>", "9", "(EOTfi", " é", "#$%", "ée", "­­", "𐞁'll", "Z", " ", " ㋿"]} +{"text": "a/b(Dž३<|fim_prefix|>…😀🏽9éᵃ", "tokens": 22, "pieces": ["a", "/b", "(Dž", "३", "<|", "fim", "_prefix", "|>", "…", "😀🏽", "9", "éᵃ"]} +{"text": "-𐞁/ḍ̇!İ'VEDžunglaéfi'Td
(́(ع/", "tokens": 27, "pieces": ["-𐞁", "/ḍ̇", "!İ'VE", "Džunglaéfi'T", "d", "
", "(́", "(ع", "/"]} +{"text": "12345678-\n/aB­/!<|endoftext|>", "tokens": 17, "pieces": ["123", "456", "78", "-\n/", "a", "B", "­/!<|", "endoftext", "|>"]} +{"text": "å㋿d…é'/　 \n İ\"é\u000b漢-😀🏽s　<|endoftext|>½é­ABC,
 ḍ̇\n HTTPServerHTTPServer", "tokens": 52, "pieces": ["å", "㋿d", "", "…é", "'/", "　 \n", " İ", "\"é", "\u000b漢", "-😀🏽", "s", "　", "<|", "endoftext", "|>", "½", "é", "­ABC", ",", "
", " ḍ̇", "\n", " ", " HTTPServer", "HTTPServer"]} +{"text": "İßs
ꟲa/b", "tokens": 9, "pieces": ["İßs", "
ꟲa", "/b"]} +{"text": "'ll'sé<|endoftext|>३å'M>!", "tokens": 16, "pieces": ["'ll's", "é", "<|", "endoftext", "|>", "३", "å'M", ">!"]} +{"text": "sZAb  \n0ſB(\u000bd𐞁\r\n\r\n>'SDž /३عⅣd'M(aBé,…\n<|endoftext|>'S'll0Ab👍🏽
", "tokens": 50, "pieces": ["s", "ZAb", "  \n", "0", "ſ", "B", "(", "\u000bd𐞁", "\r\n\r\n", ">'", "SDž", " ", "/", "३", "ع", "Ⅳ", "d'M", "(a", "Bé", ",", "…\n", "<|", "endoftext", "|>'", "S'll", "0", "Ab", "👍🏽", "
"]} +{"text": "‍e'śع\"\r 12345678ß-́,9#$%\n/ſ \n -d‍字'SaB\na/bcamelCaseİ\n/", "tokens": 40, "pieces": ["‍e's", "́ع", "\"\r", " ", " ", "123", "456", "78", "ß", "-́", ",", "9", "#$%\n/", "ſ", " \n", " -", "d", "‍字'S", "a", "B", "\n", "a", "/bcamel", "Case", "İ", "\n", "/"]} +{"text": "Dž \n 𐞁🙂<|endoftext|>'s9<|endoftext|>\u000b字'T́(\"9\r\n\r\n'siOS字<|fim_prefix|><|endoftext|>'Re'M\r\nd<|fim_prefix|>'reé🙂", "tokens": 61, "pieces": ["Dž", " \n", " 𐞁", "🙂<|", "endoftext", "|>'", "s", "9", "<|", "endoftext", "|>", "\u000b字'T", "́", "(\"", "9", "\r\n\r\n", "'si", "OS字", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "Re'M", "\r\n", "d", "<|", "fim", "_prefix", "|>'", "reé", "🙂"]} +{"text": "　EOT\u000bfidZ#$%HTTPServer/iOSᵃ㍿́\r\"ݽ…\nm.Ab'M'S٣٤٥٦Džungla(éꟲ'll", "tokens": 45, "pieces": ["　EOT", "\u000bfid", "Z", "#$%", "HTTPServer", "/i", "OSᵃ", "㍿́", "\r", "\"İ", "½", "…\n", "m", ".Ab'M", "'S", "٣٤٥", "٦", "Džungla", "(éꟲ'll"]} +{"text": "ᵃ\r\n.#$%!!٣٤٥٦\ndＡB0\n/́ꟲ !!𐞁B(ſABC字HTTPServer<|fim_prefix|>\r \nABCé\n😀🏽HTTPServera/b\t'12345678", "tokens": 63, "pieces": ["ᵃ", "\r\n", ".#$%!!", "٣٤٥", "٦", "\n", "d", "ＡB", "0", "\n", "/́ꟲ", " !!<", "META", "_START", ">𐞁", "B", "(ſ", "ABC字HTTPServer", "<|", "fim", "_prefix", "|>\r", " \n", "ABCé", "\n", "😀🏽", "HTTPServera", "/b", "\t", "'", "123", "456", "78"]} +{"text": "( \n㋿td🙂å'så'M\n/camelCase½\n,ABCå\n0/", "tokens": 26, "pieces": ["(", " \n", "㋿td", "🙂å's", "å'M", "\n", "/camel", "Case", "½", "\n", ",ABCå", "\n", "0", "/"]} +{"text": "\n/>-HTTPServer\nAb'VE'S#$%<|endoftext|>camelCaseß'VEعDžungla>ꟲéß\r'(", "tokens": 50, "pieces": ["\n", "/>-", "HTTPServer", "\n", "Ab'VE", "'S", "#$%<|", "endoftext", "|>", "camel", "Caseß'VE", "ع", "Džungla", ">ꟲéß", "\r", "'(<", "META", "_START", ">"]} +{"text": "ꟲa/b", "tokens": 5, "pieces": ["ꟲa", "/b"]} +{"text": "09HTTPServer's㋿iOS#$%ᵃ,ⅣaB'siOS", "tokens": 22, "pieces": ["09", "HTTPServer's", "㋿i", "OS", "#$%", "ᵃ", ",", "Ⅳ", "a", "B's", "i", "OS"]} +{"text": "é", "tokens": 1, "pieces": ["é"]} +{"text": "Džungla㋿'s", "tokens": 9, "pieces": ["Džungla", "㋿'", "s"]} +{"text": "字…‍HTTPServerＡ‍' \n 'D\n\r\n　a/b…\n😀🏽12345678Bfi'Re🙂s\"㋿!😀🏽\r\n­", "tokens": 48, "pieces": ["字", "", "…", "‍HTTPServer", "Ａ", "‍'", " \n", " '", "D", "\n\r\n", "　a", "/b", "…\n", "😀🏽", "123", "456", "78", "Bfi'Re", "🙂s", "\"㋿!😀🏽\r\n", "­"]} +{"text": " #$%éḍ̇\t३́afi, Ab\r\n\r\na٣٤٥٦\",­'re!!ABC\r\né!\r٣٤٥٦'re\r\r\n\r\n­ᵃ'VE\té<|fim_prefix|>", "tokens": 51, "pieces": [" ", "#$%", "éḍ̇", "\t", "३", "́afi", ",", " Ab", "\r\n\r\n", "a", "٣٤٥", "٦", "\",­'", "re", "!!", "ABC", "\r\n", "é", "!\r", "٣٤٥", "٦", "'re", "\r\r\n\r\n", "­ᵃ'VE", "\té", "<|", "fim", "_prefix", "|>"]} +{"text": "İＡé,", "tokens": 4, "pieces": ["İＡé", ","]} +{"text": "‍ <|fim_prefix|>,'D.'ſ/\r.a0m\n!'reAb\"👍🏽/Ab!!Ab३​\tABC", "tokens": 36, "pieces": ["‍", " ", " <|", "fim", "_prefix", "|>,'", "D", ".'", "ſ", "/\r", ".a", "0", "m", "\n", "!'", "re", "Ab", "\"👍🏽<", "META", "_START", ">/", "Ab", "!!", "Ab", "३", "​", "\tABC"]} +{"text": "ⅣcamelCase<|endoftext|>\" \r\n<|fim_prefix|>'rea/bᵃtfi'reᵃB👍🏽٣٤٥٦ABCe'DBᵃ\rm\"\r\r\n　d عDžungla\r\n0'Re𐞁 \n!ſ", "tokens": 66, "pieces": ["Ⅳ", "camel", "Case", "<|", "endoftext", "|>\"", " \r\n", "<|", "fim", "_prefix", "|>'", "rea", "/bᵃtfi're", "ᵃ", "B", "👍🏽", "٣٤٥", "٦", "ABCe'D", "Bᵃ", "\r", "m", "\"\r\r\n", "　d", " عDžungla", "\r\n", "0", "'Re𐞁", " \n", "!ſ"]} +{"text": "İ漢\"aZ½‍<|fim_prefix|>d12345678B're", "tokens": 18, "pieces": ["İ漢", "\"a", "Z", "½", "‍<|", "fim", "_prefix", "|>", "d", "123", "456", "78", "B're"]} +{"text": "　0字…½\n'ſ/\r\né½t12345678漢ḍ̇.漢漢>é-/\r\n'ḍ̇\"Ⅳ'D👍🏽\n#$%<|endoftext|>", "tokens": 49, "pieces": ["　", "0", "字", "…", "½", "\n", "'ſ", "/\r\n", "é", "½", "t", "123", "456", "78", "漢ḍ̇", ".漢漢", ">é", "-/\r\n", "'ḍ̇", "\"", "Ⅳ", "'D", "👍🏽\n", "#$%<|", "endoftext", "|>"]} +{"text": "
å'VE‍'D \n
ß'VE \n👍🏽٣٤٥٦​𐞁'TAb𐞁…tHTTPServer👍🏽", "tokens": 40, "pieces": ["
å'VE", "‍'", "D", " \n", "
ß'VE", " \n", "👍🏽", "٣٤٥", "٦", "​𐞁'T", "Ab𐞁", "…t", "HTTPServer", "👍🏽"]} +{"text": "'ſDžes\n'ſHTTPServer'S㋿e
  \n HTTPServer…\n
iOSZ㍿ß\n\u000b/👍🏽'reع漢 \n ḍ̇-'VE½", "tokens": 49, "pieces": ["'ſ", "Džes", "\n", "'ſ", "HTTPServer'S", "㋿e", "
  \n", " HTTPServer", "…\n", "
i", "OSZ", "㍿ß", "\n", "\u000b", "/👍🏽'", "reع漢", " \n", " ḍ̇", "-'", "VE", "½"]} +{"text": "'T½\"😀🏽'TBd/'re\r\n\r\nſ é٣٤٥٦'Re…<
", "tokens": 24, "pieces": ["'T", "½", "\"😀🏽'", "TBd", "/'", "re", "\r\n\r\n", "ſ", " ", " é", "٣٤٥", "٦", "'Re", "…", "<", "
"]} +{"text": "-\u000ba0Z9é '½.", "tokens": 11, "pieces": ["-", "\u000ba", "0", "Z", "9", "é", " '", "½", "."]} +{"text": "३'S\nİa/b​Ａ. s \n ㋿/å.e!'Mḍ̇'DiOS", "tokens": 28, "pieces": ["३", "'S", "\n", "İa", "/b", "​Ａ", ".", " s", " \n", " ㋿/", "å", ".e", "!'", "Mḍ̇'D", "i", "OS"]} +{"text": "…/Džunglaİ
é\te \né \n eB <\rＡᵃ'ßBa\n/\n/½\t३sAb𐞁camelCase're/ 🙂", "tokens": 50, "pieces": ["…", "/<", "EOT", ">Džungla", "İ", "
é", "\te", " \n", "é", " \n", " e", "B", "", " ", " <\r", "Ａᵃ", "'ß", "Ba", "\n", "/\n/", "½", "\t", "३", "s", "Ab𐞁camel", "Case're", "/", " 🙂"]} +{"text": "­'ReZ
字Z", "tokens": 7, "pieces": ["­'", "Re", "Z", "
字", "Z"]} +{"text": "ABC\n/漢'll\n\r\n\r\n🙂́ᵃＡꟲ'll'Réd>字-#$%😀🏽e½/\r\naBa/b'Re'", "ll'Re", "́d", ">字", "-#$%😀🏽", "e", "½", "/\r\n", "a", "Ba", "/b'Re", "d9a\r\u000bꟲ'Re
漢éAb're<|endoftext|>\n0
EOTＡd٣٤٥٦😀🏽", "tokens": 62, "pieces": ["a'D", "å", "‍", " ", "'VEa", "/b'll", "漢", " ", "\"", "…", "㋿ꟲ", "\n", "Dž", "d", "9", "a", "\r", "\u000bꟲ'Re", "
漢é", "Ab're", "<|", "endoftext", "|>\n", "0", "
EOTＡd", "٣٤٥", "٦", "😀🏽"]} +{"text": "12345678İ \nꟲEOTaعZ İ'ſ‍ \n12345678Džunglaé㍿eDž<|fim_prefix|>‍‍٣٤٥٦s\téⅣ12345678Z字aB(㋿fiAbé", "tokens": 65, "pieces": ["123", "456", "78", "İ", " \n", "ꟲEOTaع", "Z", " İ'ſ", "‍", " \n", "123", "456", "78", "Džunglaé", "㍿e", "Dž", "<|", "fim", "_prefix", "|>‍‍", "٣٤٥", "٦", "s", "\té", "Ⅳ12", "345", "678", "Z字a", "B", "(㋿", "fi", "Abé"]} +{"text": "DžABC\r\n<|fim_prefix|>🙂‍Ab㍿٣٤٥٦Dž <|fim_prefix|>B#$%HTTPServer", "tokens": 34, "pieces": ["DžABC", "\r\n", "<|", "fim", "_prefix", "|>🙂‍", "Ab", "㍿", "٣٤٥", "٦", "Dž", " ", " <|", "fim", "_prefix", "|>", "B", "#$%", "HTTPServer"]} +{"text": "'sDžunglaHTTPServer  \n \n/漢'VE👍🏽ⅣAb0éⅣaå३12345678🙂'M,\u000b\r…٣٤٥٦Džungla३'Re(ſ'D", "tokens": 56, "pieces": ["'s", "Džungla", "HTTPServer", "  \n \n", "/漢'VE", "👍🏽", "Ⅳ", "Ab", "0", "é", "Ⅳ", "aå", "३12", "345", "678", "🙂'", "M", ",", "\u000b\r", "…", "٣٤٥", "٦", "Džungla", "३", "'", "Re", "(ſ'D"]} +{"text": "'Mİ‍'sＡ.-½　ꟲ'VEḍ̇éᵃ \nAb12345678", "tokens": 26, "pieces": ["'Mİ", "‍'", "s", "Ａ", ".-", "½", "　ꟲ'VE", "ḍ̇éᵃ", " \n", "Ab", "123", "456", "78"]} +{"text": "Ａ　Ａ𐞁å㋿aß#$%a/\r\n,
s½>😀🏽(½
漢EOTABCع\u000b'S'-İ
m👍🏽<|endoftext|>\r\naᵃ", "tokens": 54, "pieces": ["Ａ", "　Ａ𐞁å", "㋿aß", "#$%", "a", "/\r\n", ",", "
s", "½", ">😀🏽(", "½", "
漢EOTABCع", "\u000b", "'S", "'-", "İ", "
m", "👍🏽<|", "endoftext", "|>\r\n", "aᵃ"]} +{"text": "#$%'VE(İ \n Ab­ 0a'Re　- 'S'Re­/\r\nB", "tokens": 22, "pieces": ["#$%'", "VE", "(İ", " \n", " Ab", "­", " ", " ", "0", "a'Re", "　", "-", " ", " '", "S'Re", "­/\r\n", "B"]} +{"text": "!!٣٤٥٦'Re \n👍🏽ꟲDžunglaa/b'ſßcamelCase(\r.'DaZs", "tokens": 33, "pieces": ["!!", "٣٤٥", "٦", "'Re", " \n", "👍🏽", "ꟲDžunglaa", "/b'ſ", "ßcamel", "Case", "(\r", ".'", "Da", "Zs"]} +{"text": "sⅣ\n/ꟲa🙂½'M\n", "tokens": 19, "pieces": ["s", "", "Ⅳ", "\n", "/ꟲa", "🙂", "½", "'M", "\n"]} +{"text": "­​字Dž\t'Tß\r\niOS's'MAbİ >é'll/\r\nAbiOSa.​DžB😀🏽́", "tokens": 35, "pieces": ["­​", "字", "Dž", "\t", "'Tß", "\r\n", "i", "OS's", "'MAb", "İ", " ", ">é'll", "/\r\n", "Abi", "OSa", ".​", "DžB", "😀🏽́"]} +{"text": "B\t/\r\n𐞁a/b<|fim_prefix|>Dž#$%\u000b\n/mcamelCase", "tokens": 27, "pieces": ["B", "\t", "/\r\n", "𐞁a", "/b", "<|", "fim", "_prefix", "|>", "Dž", "#$%", "\u000b\n", "/mcamel", "Case"]} +{"text": "!#$%'VEع<|fim_prefix|>HTTPServer\n<­\r\n\r\n‍字😀🏽é­,B<|fim_prefix|>३ \ne👍🏽\r\n\r\nd>", "tokens": 55, "pieces": ["!#$%'", "VEع", "<|", "fim", "_prefix", "|>", "HTTPServer", "\n", "<­\r\n\r\n", "‍字", "😀🏽", "é", "­,", "B", "<|", "fim", "_prefix", "|>", "३", " \n", "e", "👍🏽\r\n\r\n", "d", ">"]} +{"text": "½tİ\u000b
", "tokens": 5, "pieces": ["½", "t", "İ", "\u000b
"]} +{"text": "'VE!!\n/ع😀🏽", "tokens": 8, "pieces": ["'VE", "!!\n/", "ع", "😀🏽"]} +{"text": "
 \n ABC\tfi\"'VEAb㍿/s🙂ABCa/b‍　'M0t\r\n'sEOT", "tokens": 29, "pieces": ["
 \n", " ABC", "\tfi", "\"'", "VEAb", "㍿/", "s", "🙂ABCa", "/b", "‍", "　", "'M", "0", "t", "\r\n", "'s", "EOT"]} +{"text": "\r\n\r\n#$%!!​½Ａ'M 'TiOS", "tokens": 12, "pieces": ["\r\n\r\n", "#$%!!​", "½", "Ａ'M", " ", " '", "Ti", "OS"]} +{"text": "9éDžunglaEOTtda/bİ12345678<|fim_prefix|>#$%ß𐞁m\r\n/\r\nع9camelCases\u000b'SDžunglaḍ̇éꟲEOT<|fim_prefix|>aſ,/", "tokens": 65, "pieces": ["9", "é", "Džungla", "EOTt", "da", "/b", "İ", "123", "456", "78", "<|", "fim", "_prefix", "|>#$%", "ß𐞁m", "\r\n", "/\r\n", "ع", "9", "camel", "Cases", "\u000b", "'SDžunglaḍ̇éꟲ", "EOT", "<|", "fim", "_prefix", "|>", "aſ", ",/"]} +{"text": "EOT0　0Ⅳꟲ#$%字
Z'ſ'Ta/bḍ̇'llß", "tokens": 25, "pieces": ["EOT", "0", "　", "0Ⅳ", "ꟲ", "#$%", "字", "
Z'ſ", "'Ta", "/bḍ̇'ll", "ß"]} +{"text": " \nDžunglam12345678\n/Ⅳ12345678EOT'lliOS'ſß", "tokens": 23, "pieces": [" \n", "Džunglam", "123", "456", "78", "\n", "/", "Ⅳ12", "345", "678", "EOT'll", "i", "OS'ſ", "ß"]} +{"text": " \nEOTDž‍字\u000b👍🏽😀🏽字🙂", "tokens": 16, "pieces": [" \n", "EOTDž", "‍字", "\u000b", "👍🏽😀🏽", "字", "🙂"]} +{"text": "ꟲé12345678aB \n 12345678EOT \n<३'VE00…Ⅳ字iOSé'Reꟲ're's'siOSt 0aeعé's", "tokens": 54, "pieces": ["ꟲé", "123", "456", "78", "a", "B", " \n", " ", "123", "456", "78", "EOT", " \n", "<", "३", "'VE", "", "00", "…", "Ⅳ", "字i", "OSé'Re", "ꟲ're", "'s's", "i", "OSt", " ", "0", "aeعé's", ""]} +{"text": "'ll'Sé/a/b(d eꟲ/\r\nＡå​'reAb'M#$%'sss\rḍ̇ ㋿/\r\nm", "tokens": 33, "pieces": ["'ll'S", "é", "/a", "/b", "(d", " eꟲ", "/\r\n", "Ａå", "​'", "re", "Ab'M", "#$%'", "sss", "\r", "ḍ̇", " ", " ㋿/\r\n", "m"]} +{"text": "'ll \n/­", "tokens": 9, "pieces": ["'ll", " \n", "/­<", "EOT", ">"]} +{"text": "\"­t'll\"ééfi0Z's\n '㍿aB\"'re #$%…'ll\n/m漢३\u000b'D𐞁.ᵃ!!\t㍿Džungla😀🏽𐞁", "tokens": 61, "pieces": ["\"­<", "META", "_START", ">t'll", "\"ééfi", "0", "Z's", "\n", " ", " '㍿", "a", "B", "\"'", "re", " #$%", "…", "'ll", "\n", "/m漢", "३", "\u000b", "'D𐞁", ".ᵃ", "!!", "\t", "㍿Džungla", "😀🏽", "𐞁"]} +{"text": "sßAb'M\n\"\r\n\r\n<漢\tAb🙂DžＡ\t㍿\n/ ", "tokens": 21, "pieces": ["sß", "Ab'M", "\n", "\"\r\n\r\n", "<漢", "\tAb", "🙂DžＡ", "\t", "㍿\n/", " "]} +{"text": "'s \n 👍🏽㍿ḍ̇🙂", "tokens": 12, "pieces": ["'s", " \n", " 👍🏽㍿", "ḍ̇", "🙂"]} +{"text": "(½　Z9s\r#$%12345678m'S
­\nDžunglafi𐞁ᵃmHTTPServer 'Re㍿㍿ \n B!!!'reZعİ<|endoftext|>", "tokens": 54, "pieces": ["(", "½", "　Z", "9", "s", "\r", "#$%", "123", "456", "78", "m'S", "
", "­\n", "Džunglafi𐞁ᵃm", "HTTPServer", " '", "Re", "㍿㍿", " \n", " B", "!!!'", "re", "Zع", "İ", "<|", "endoftext", "|>"]} +{"text": "\réＡ a/ba­…٣٤٥٦\n12345678漢", "tokens": 19, "pieces": ["\r", "é", "Ａ", " a", "/ba", "­", "…", "٣٤٥", "٦", "\n", "123", "456", "78", "漢"]} +{"text": "\r\n\r\n'T#$%< \n 'VE ", "tokens": 9, "pieces": ["\r\n\r\n", "'T", "#$%<", " \n", " '", "VE", " "]} +{"text": "\n/字iOSع½\r\n\r\n
camelCase#$%٣٤٥٦å<|fim_prefix|>­\r'SfiⅣ", "tokens": 34, "pieces": ["\n", "/字i", "OSع", "½", "\r\n\r\n", "
", "camel", "Case", "#$%", "٣٤٥", "٦", "å", "<|", "fim", "_prefix", "|>­\r", "'Sfi", "Ⅳ"]} +{"text": "'s漢𐞁İiOScamelCasedİe #$%d'MABCs \n 'reİa/b-ع'reHTTPServer \n!३camelCase<|fim_prefix|>😀🏽é.", "tokens": 52, "pieces": ["'s漢𐞁", "İi", "OScamel", "Cased", "İe", " ", " #$%", "d'M", "ABCs", " \n", " '", "re", "İa", "/b", "-", "ع're", "HTTPServer", " \n", "!", "३", "camel", "Case", "<|", "fim", "_prefix", "|>😀🏽", "é", "."]} +{"text": "12345678Džungla'D ½a \n …<12345678ᵃ/aBDžungla\u000b㍿camelCase🙂mع\u000b<|fim_prefix|>'Tm字iOS<|endoftext|><|endoftext|>iOSſABC\tꟲ😀🏽\n/Ab\t", "tokens": 79, "pieces": ["123", "456", "78", "Džungla'D", " ", "½", "a", " \n", " ", "…", "<", "123", "456", "78", "ᵃ", "/a", "BDžungla", "\u000b", "㍿camel", "Case", "🙂mع", "\u000b", "<|", "fim", "_prefix", "|>'", "Tm字i", "OS", "<|", "endoftext", "|><|", "endoftext", "|>", "i", "OSſ", "ABC", "\tꟲ", "😀🏽\n/", "Ab", "\t"]} +{"text": "漢'ſ", "tokens": 3, "pieces": ["漢'ſ"]} +{"text": "… camelCasefiİ<३#$%㍿ḍ̇dB😀🏽", "tokens": 22, "pieces": ["… ", " camel", "Casefi", "İ", "<", "३", "#$%㍿", "ḍ̇d", "B", "😀🏽"]} +{"text": "\u000b\n/!\r\n\r\niOS'ſ12345678/\r\n ㍿'M<|fim_prefix|>Ab\r😀🏽ſ'll­", "tokens": 33, "pieces": ["\u000b\n", "/!\r\n\r\n", "i", "OS'ſ", "123", "456", "78", "/\r\n", " ㍿'", "M", "<|", "fim", "_prefix", "|>", "Ab", "\r", "😀🏽", "ſ'll", "­"]} +{"text": "'Taſ​'ll0३'S'ſ'Sſ!!😀🏽,!!\u000bHTTPServer/​mé
9𐞁ß𐞁\r\n\r\n's'ᵃ🙂s", "tokens": 44, "pieces": ["'Taſ", "​'", "ll", "0३", "'S'ſ", "'Sſ", "!!😀🏽,!!", "\u000bHTTPServer", "/​", "mé", "
", "9", "𐞁ß𐞁", "\r\n\r\n", "'s", "'ᵃ", "🙂s"]} +{"text": "/\r\n👍🏽­​İİß're\n/'llZmB\n/́ſ́aBİع😀🏽're …'ſ '　.㋿camelCaseHTTPServerfit'sm", "tokens": 49, "pieces": ["/\r\n", "👍🏽­​", "İİß're", "\n", "/'", "ll", "Zm", "B", "\n", "/́ſ́a", "Bİع", "😀🏽'", "re", " ", "…", "'ſ", " ", " '", "　", ".㋿", "camel", "Case", "HTTPServerfit's", "m"]} +{"text": "ꟲ!!m'D,.,ع🙂!!iOS#$%\r\n…\"\rAb'SaBABC👍🏽>\u000b👍🏽t#$%Dž½", "tokens": 38, "pieces": ["ꟲ", "!!", "m'D", ",.,", "ع", "🙂!!", "i", "OS", "#$%\r\n", "…", "\"\r", "Ab'S", "a", "BABC", "👍🏽>", "\u000b", "👍🏽", "t", "#$%", "Dž", "½"]} +{"text": "Ab0\r\nſ🙂é,B-,fi'll'VE'BEOT
12345678\n/HTTPServerEOT👍🏽ع \n ⅣcamelCase🙂字🙂ßEOT\u000beiOS…🙂\"٣٤٥٦", "tokens": 53, "pieces": ["Ab", "0", "\r\n", "ſ", "🙂é", ",B", "-,", "fi'll", "'VE", "'BEOT", "
", "123", "456", "78", "\n", "/HTTPServer", "EOT", "👍🏽", "ع", " \n", " ", "Ⅳ", "camel", "Case", "🙂字", "🙂ß", "EOT", "\u000bei", "OS", "…", "🙂\"", "٣٤٥", "٦"]} +{"text": "EOT'reİ😀🏽 ㋿aB
HTTPServer　<३'D", "tokens": 20, "pieces": ["EOT're", "İ", "😀🏽", " ㋿", "a", "B", "
HTTPServer", "　", "<", "३", "'D"]} +{"text": "Ⅳ\r\u000b漢
<|endoftext|> \n 'll½­å Džungla　åⅣᵃiOSḍ̇́ſ!‍aB/\r\nAbع ́aBe\u000b'ſ㋿
", "tokens": 61, "pieces": ["Ⅳ", "\r", "\u000b漢", "
", "<|", "endoftext", "|><", "META", "_START", ">", " \n", " '", "ll", "½", "­å", " ", " Džungla", "　å", "Ⅳ", "ᵃi", "OSḍ̇́ſ", "!‍", "a", "B", "/\r\n", "Abع", " ́a", "Be", "\u000b", "'ſ", "㋿", "
"]} +{"text": "\nZ!!é#$%٣٤٥٦ ㋿\t​­/
ABC㋿", "tokens": 23, "pieces": ["\n", "Z", "!!", "é", "#$%", "٣٤٥", "٦", " ", "㋿", "\t", "​­/", "
ABC", "㋿"]} +{"text": " \n Zm\r\n'sé漢!!#$%a/bEOTꟲ<|endoftext|>'sß👍🏽😀🏽\n/dcamelCase'T'D9s>\niOS\n/ꟲ'VE's/\rᵃDž…iOS", "tokens": 67, "pieces": [" \n", " Zm", "\r\n", "'sé漢", "!!#$%<", "META", "_START", ">a", "/b", "EOTꟲ", "<|", "endoftext", "|>'", "sß", "👍🏽😀🏽\n/", "dcamel", "Case'T", "'D", "9", "s", ">\n", "i", "OS", "\n", "/ꟲ'VE", "'s", "/\r", "ᵃ", "Dž", "…i", "OS"]} +{"text": "\rsＡعsEOT
　\tſ \n \n　 \n a/b\t'VEe>'VEDžungla.e'ReZ…'<|fim_prefix|>0BⅣ\"\ta", "tokens": 47, "pieces": ["\r", "s", "Ａعs", "EOT", "
　", "\t", "ſ", " \n \n　 \n", " a", "/b", "\t", "'VEe", ">'", "VEDžungla", ".e'Re", "Z", "…", "'<|", "fim", "_prefix", "|>", "0", "B", "Ⅳ", "\"", "\ta"]} +{"text": "Ⅳ", "tokens": 6, "pieces": ["Ⅳ", ""]} +{"text": "ſiOSéEOT  \nsß\r\n\r\n字½!!㋿(é😀🏽
>", "tokens": 24, "pieces": ["ſi", "OSé", "EOT", "  \n", "sß", "\r\n\r\n", "字", "½", "!!㋿(", "é", "😀🏽", "
", ">"]} +{"text": "​½ꟲ㋿.dⅣ'M३1234567812345678iOS‍…'Refi'llİ", "tokens": 29, "pieces": ["​", "½", "ꟲ", "㋿.", "d", "Ⅳ", "'M", "३12", "345", "678", "123", "456", "78", "i", "OS", "‍", "…", "'Refi'll", "İ"]} +{"text": "㋿é
'Mſ#$%s#$%
👍🏽 😀🏽३İ‍'VEع-HTTPServer字­", "tokens": 31, "pieces": ["㋿é", "
", "'Mſ", "#$%", "s", "#$%", "
", "👍🏽", " ", " 😀🏽", "३", "İ", "‍'", "VEع", "-HTTPServer字", "­"]} +{"text": "३fi9­HTTPServer", "tokens": 6, "pieces": ["३", "fi", "9", "­HTTPServer"]} +{"text": " Abꟲ\"ABC", "tokens": 7, "pieces": [" Abꟲ", "\"ABC"]} +{"text": "字́d/Ⅳḍ̇'a/bZ,ᵃ
'St​'ll‍३s𐞁İ", "tokens": 30, "pieces": ["字́d", "/", "Ⅳ", "ḍ̇", "'a", "/b", "Z", ",ᵃ", "
", "'St", "​'", "ll", "‍", "३", "s𐞁", "İ"]} +{"text": "漢ꟲéİſfiZ\u000b㍿.\rDž३iOS>\n
EOT㍿ \n'T", "tokens": 30, "pieces": ["漢ꟲé", "İſfi", "Z", "\u000b", "㍿.\r", "Dž", "३", "i", "OS", ">\n", "
EOT", "㍿", " \n", "'T"]} +{"text": ">😀🏽's🙂>'re\r\ń/\r\n<|endoftext|>'DB\r\n9…'sＡ\n/(eß'D/a…- \n m\r\nem
ABC😀🏽'", "s", "🙂>'", "re", "\r\n", "́", "/\r\n", "<|", "endoftext", "|>'", "DB", "\r\n", "9", "…", "'s", "Ａ", "\n", "/(", "eß'D", "/", "a", "…", "-", " \n", " ", " m", "\r\n", "em", "
ABC", "12345678'T\n'VE𐞁𐞁\u000b<ß<|fim_prefix|>diOSaBABC…Ⅳa\r\n​", "tokens": 50, "pieces": ["　", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'T", "\n", "'VE", "𐞁𐞁", "\u000b", "<ß", "<|", "fim", "_prefix", "|>", "d", "i", "OSa", "BABC", "…", "Ⅳ", "a", "\r\n", "​"]} +{"text": "å😀🏽عcamelCase /\r\n ḍ̇ \n Ab漢", "tokens": 23, "pieces": ["å", "😀🏽", "ع", "camel", "Case", " ", " /\r\n", " ", " ḍ̇", " \n", " Ab漢"]} +{"text": "ḍ̇\u000b­Z/Džungla٣٤٥٦!\n　…　>EOT<|fim_prefix|>
'ReAbéعABC\n/'VE½Ａ\u000be(👍🏽", "tokens": 45, "pieces": ["ḍ̇", "\u000b", "­Z", "/Džungla", "٣٤٥", "٦", "!\n", "　…", "　", ">EOT", "<|", "fim", "_prefix", "|>", "
", "'Re", "Abéع", "ABC", "\n", "/'", "VE", "½", "Ａ", "\u000be", "(👍🏽"]} +{"text": "'S>m!B>#$%!!0ſAbABCİ👍🏽é'D ​ B \n\u000bDžungla\n/ aBB \n/　\r'M", "tokens": 38, "pieces": ["'S", ">m", "!B", ">#$%!!", "0", "ſ", "Ab", "ABCİ", "👍🏽", "é'D", " ", "​", " B", " \n", "\u000bDžungla", "\n", "/", " ", " a", "BB", " \n", "/", "　\r", "'M"]} +{"text": "/😀🏽-İ-åع'ſ\r\n dcamelCase😀🏽's㋿(㍿㍿'et…a<|endoftext|>/\r\n!", "tokens": 51, "pieces": ["/😀🏽-", "İ", "-åع'ſ", "\r\n", " dcamel", "Case", "😀🏽'", "s", "㋿(㍿㍿'<", "META", "_START", ">et", "…a", "<|", "endoftext", "|>/\r\n", "!<", "META", "_START", ">"]} +{"text": "Zع'fi\r\n\r\n\r\nABCe0\rZ३ficamelCase'DDžungla'MsEOTd\r0ſꟲ'ReDžungla9…Ab½DžunglaAbDžungla½", "tokens": 50, "pieces": ["Zع", "'fi", "\r\n\r\n\r\n", "ABCe", "0", "\r", "Z", "३", "ficamel", "Case'D", "Džungla'M", "s", "EOTd", "\r", "0", "ſꟲ'Re", "Džungla", "9", "…Ab", "½", "Džungla", "Ab", "Džungla", "½"]} +{"text": "ABCaBß　'VEe'sDžunglaᵃ", "tokens": 16, "pieces": ["ABCa", "Bß", "　", "'VEe's", "Džunglaᵃ"]} +{"text": "Z 🙂é!!漢‍\r\n漢👍🏽B", "tokens": 13, "pieces": ["Z", " ", " 🙂", "é", "!!", "漢", "‍\r\n", "漢", "👍🏽", "B"]} +{"text": "EOTDžungla (iOS'😀🏽 \n /\r\n,\rt\r\n\r\n🙂B'Ⅳ0EOTa/b\r\n\r\n!‍", "tokens": 38, "pieces": ["EOTDžungla", " ", " (", "i", "OS", "'😀🏽", " \n", " /\r\n", ",\r", "t", "\r\n\r\n", "🙂B", "'", "Ⅳ0", "EOTa", "/b", "\r\n\r\n", "!‍<", "EOT", ">"]} +{"text": "\u000bEOT.s
ſ👍🏽👍🏽İع. \n >🙂㍿,s'T𐞁iOSḍ̇‍BDžungla😀🏽Džungla>İ🙂DžunglacamelCase'D'ſ!d㋿", "tokens": 68, "pieces": ["\u000bEOT", ".s", "
ſ", "👍🏽👍🏽", "İع", ".", " \n", " >🙂㍿,", "s", "'", "T𐞁i", "OSḍ̇", "‍BDžungla", "😀🏽", "Džungla", ">İ", "🙂Džunglacamel", "Case'D", "'", "ſ", "!d", "㋿"]} +{"text": "''S9ABC㋿,\n/9\r'Ś(ABCⅣ́", "tokens": 20, "pieces": ["''", "S", "", "9", "ABC", "㋿,\n/", "9", "\r", "'Ś", "(ABC", "Ⅳ", "́"]} +{"text": "漢㋿​'VE​ßḍ̇<㋿iOS…İ㍿ fiＡ  ́㋿ \n Ab'VE", "tokens": 37, "pieces": ["漢", "㋿​'", "VE", "​ßḍ̇", "<㋿", "i", "OS", "…İ", "㍿", " fi", "Ａ", " ", " ́", "㋿", " \n", " Ab'VE"]} +{"text": "d字\r\n\r\n𐞁camelCaseHTTPServer'ſ́9aåå<|fim_prefix|>< .éḍ̇dⅣßfi", "tokens": 36, "pieces": ["d字", "\r\n\r\n", "𐞁camel", "Case", "HTTPServer'ſ", "́", "9", "aåå", "<|", "fim", "_prefix", "|><", " ", " .", "éḍ̇d", "Ⅳ", "ßfi"]} +{"text": "'VEⅣåst \n fi\"( \nİ٣٤٥٦0camelCase٣٤٥٦'D'TＡ/\r\nBdt字३", "tokens": 31, "pieces": ["'VE", "Ⅳ", "åst", " \n", " fi", "\"(", " \n", "İ", "٣٤٥", "٦0", "camel", "Case", "٣٤٥", "٦", "'D'T", "Ａ", "/\r\n", "Bdt字", "३"]} +{"text": "字\rcamelCase\r\n\r\néåt#$%fi\n/,!HTTPServer#$% 'ss", "tokens": 26, "pieces": ["字", "\r", "camel", "Case", "\r\n\r\n", "éåt", "#$%", "fi", "\n", "/<", "EOT", ">,!", "HTTPServer", "#$%", " ", " '", "ss"]} +{"text": "\t𐞁'll….\rⅣ9\u000bᵃ٣٤٥٦𐞁iOS\n\t…​'M　३\t'S👍🏽　d'", "tokens": 44, "pieces": ["\t𐞁'll", "…", ".\r", "Ⅳ9", "\u000bᵃ", "٣٤٥", "٦", "𐞁i", "OS", "\n", "\t", "…", "​'", "M", "　", "३", "\t", "'S", "👍🏽", "　d", "'"]} +{"text": "fi漢👍🏽", "tokens": 5, "pieces": ["fi漢", "👍🏽"]} +{"text": "字<|endoftext|>\r\n>𐞁\t\r\nß AbZ \n 'Sd\r­/\r\n​ \n ", "tokens": 29, "pieces": ["字", "<|", "endoftext", "|>\r\n", ">𐞁", "\t\r\n", "ß", " Ab", "Z", "", " \n", " '", "Sd", "\r", "­/\r\n", "​", " \n", " "]} +{"text": "​Džungla", "tokens": 5, "pieces": ["​Džungla"]} +{"text": "\r\n  éfi㍿Z३Dž", "tokens": 12, "pieces": ["\r\n", " ", " éfi", "㍿Z", "३", "Dž"]} +{"text": "9\u000b'VE'll", "tokens": 5, "pieces": ["9", "\u000b", "'VE'll"]} +{"text": " 🙂!​\n'Mm\n/Bé'D>s\r\n字'M㋿\nEOT's/\r\n　fi('Ta/bmd", "tokens": 28, "pieces": [" 🙂!​\n", "'Mm", "\n", "/Bé'D", ">s", "\r\n", "字'M", "㋿\n", "EOT's", "/\r\n", "　fi", "('", "Ta", "/bmd"]} +{"text": "/\r\n‍å>B's字'VE'𐞁iOS-ꟲå३Džunglae‍ḍ̇​fi>.", "tokens": 39, "pieces": ["/\r\n", "‍å", ">B's", "字'VE", "'𐞁i", "OS", "-ꟲå", "३", "Džunglae", "‍ḍ̇", "​fi", ">."]} +{"text": " \nABCiOS३İ\nABCDžs", "tokens": 11, "pieces": [" \n", "ABCi", "OS", "३", "İ", "\n", "ABCDžs"]} +{"text": "e字,\r\n\r\ncamelCase'llcamelCaseåHTTPServereعtEOT'ReABCé>", "tokens": 22, "pieces": ["e字", ",\r\n\r\n", "camel", "Case'll", "camel", "Caseå", "HTTPServereعt", "EOT'Re", "ABCé", ">"]} +{"text": "009<|endoftext|>a'll\" 'reḍ̇Džungla　/\r\n émİꟲ'M", "tokens": 35, "pieces": ["009", "<|", "endoftext", "|>", "a'll", "\"<", "META", "_START", ">", " ", " '", "reḍ̇", "Džungla", "　", "/\r\n", " ", " ém", "İꟲ'M"]} +{"text": "\n漢'VE \n ", "tokens": 6, "pieces": ["\n", "漢'VE", " \n", " "]} +{"text": "\n/\t\nEOTꟲ0㋿édéB㋿,", "tokens": 23, "pieces": ["\n", "/<", "META", "_START", ">", "\t\n", "EOTꟲ", "0", "㋿édé", "B", "㋿,"]} +{"text": "ᵃ漢 \n­\n/aBédéaB \n#$%fiZ\r\n!!HTTPServer'ſ'llcamelCase", "tokens": 26, "pieces": ["ᵃ漢", " \n", "­\n/", "a", "Bédéa", "B", " \n", "#$%", "fi", "Z", "\r\n", "!!", "HTTPServer'ſ", "'llcamel", "Case"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'llİᵃ9😀🏽s‍'ſⅣ㋿<|endoftext|>ᵃ're🙂𐞁'ſ…\u000beعßB're", "tokens": 44, "pieces": ["'ll", "İᵃ", "9", "😀🏽", "s", "‍'", "ſ", "Ⅳ", "㋿<|", "endoftext", "|>", "ᵃ're", "🙂𐞁'ſ", "…", "\u000beعß", "B're"]} +{"text": "'Re!camelCase字👍🏽(å-!\"٣٤٥٦\u000b/½/​ع<ᵃ \n <|endoftext|>'Se 12345678'll🙂<|fim_prefix|>éDžunglaé'MHTTPServer!!‍!", "tokens": 62, "pieces": ["'Re", "!camel", "Case字", "👍🏽(", "å", "-!\"", "٣٤٥", "٦", "\u000b", "/", "½", "/​", "ع", "<ᵃ", " \n", " <|", "endoftext", "|>'", "Se", " ", "123", "456", "78", "'ll", "🙂<|", "fim", "_prefix", "|>", "é", "Džunglaé'M", "HTTPServer", "!!‍!"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "'VE㋿", "tokens": 5, "pieces": ["'VE", "㋿"]} +{"text": "
iOS''Refi👍🏽\nfi👍🏽\r\n\r\n!! \n ٣٤٥٦'ll٣٤٥٦t!!…😀🏽ᵃmİⅣ", "tokens": 41, "pieces": ["
i", "OS", "''", "Refi", "👍🏽\n", "fi", "👍🏽\r\n\r\n", "!!", " \n", " ", "٣٤٥", "٦", "'ll", "٣٤٥", "٦", "t", "!!", "…", "😀🏽", "ᵃm", "İ", "Ⅳ"]} +{"text": "…'Me<|fim_prefix|>\r\n\r\n 𐞁camelCase३\u000biOS12345678d('D", "tokens": 27, "pieces": ["…", "'Me", "<|", "fim", "_prefix", "|>\r\n\r\n", " 𐞁camel", "Case", "३", "\u000bi", "OS", "123", "456", "78", "d", "('", "D"]} +{"text": "Džungla\rEOT Džungla-…aB\r\n ", "tokens": 20, "pieces": ["Džungla", "\r", "EOT", " ", " Džungla", "-", "…a", "B", "\r\n", " "]} +{"text": "İå'Sm-!fi'S\u000bſBiOS​ \n >camelCase\"912345678\r\n\r\n\tZ", "tokens": 24, "pieces": ["İå'S", "m", "-!", "fi'S", "\u000bſ", "Bi", "OS", "​", " \n", " >", "camel", "Case", "\"", "912", "345", "678", "\r\n\r\n", "\tZ"]} +{"text": "‍9‍­/\r\n-sé,…!!d(\r\n \nſ#$%漢é#$%!㋿ᵃ½ABC\r\t\r\nZed\n/𐞁 ​ ", "tokens": 45, "pieces": ["‍", "9", "‍­/\r\n", "-sé", ",", "…", "!!", "d", "(\r\n", " \n", "ſ", "#$%", "漢é", "#$%!㋿", "ᵃ", "½", "ABC", "\r\t\r\n", "Zed", "\n", "/𐞁", "", " ​", " "]} +{"text": "HTTPServer'DßAb漢m'll9Dž!0漢'ſḍ̇\u000b-ſa/bfiB\r\n\r\n‍'TDž (Ⅳ/\r\n", "tokens": 41, "pieces": ["HTTPServer'D", "ß", "Ab漢m'll", "9", "Dž", "!", "0", "漢'ſ", "ḍ̇", "\u000b", "-ſa", "/bfi", "B", "\r\n\r\n", "‍'", "TDž", " ", " (", "Ⅳ", "/\r\n"]} +{"text": "!å('T ½a/b​0\r\n.DžunglaHTTPServer'ſ㍿fi㋿!字!​漢😀🏽\tᵃZ!Z字 ", "tokens": 45, "pieces": ["!å", "('", "T", " ", "½", "a", "/b", "​", "0", "\r\n", ".Džungla", "HTTPServer'ſ", "㍿fi", "㋿!", "字", "!​", "漢", "😀🏽", "\tᵃ", "Z", "!Z字", " "]} +{"text": "ع EOT!!0ſ'VEＡ", "tokens": 9, "pieces": ["ع", " EOT", "!!", "0", "ſ'VE", "Ａ"]} +{"text": "\r<|fim_prefix|>'s \n DžåAb\t­\n३B'sAbs\n😀🏽…", "tokens": 27, "pieces": ["\r", "<|", "fim", "_prefix", "|>'", "s", " \n", " Džå", "Ab", "\t", "­\n", "३", "B's", "Abs", "\n", "😀🏽", "…"]} +{"text": "m👍🏽e\r\n\r\n9‍a/bⅣ<🙂́(ع(<|fim_prefix|>­- \n \"m<|endoftext|>>㋿\t
­-", " \n", " \"", "m", "<|", "endoftext", "|>>㋿", "\t", "
", "㍿ⅣHTTPServer \nABC", "tokens": 24, "pieces": [" \n", "!!", "İ", "\r\n\r\n", "٣٤٥", "٦", " ", "<|", "fim", "_prefix", "|>㍿", "Ⅳ", "HTTPServer", " \n", "ABC"]} +{"text": "'ll,🙂camelCase…­😀🏽-m'D'D ⅣB'Sᵃ😀🏽'HTTPServer", "tokens": 29, "pieces": ["'ll", ",🙂", "camel", "Case", "…", "­😀🏽-", "m'D", "'D", " ", "Ⅳ", "B'S", "ᵃ", "😀🏽'", "HTTPServer"]} +{"text": "é.Ⅳé
Ⅳ> \n12345678a/b👍🏽Dž\r\n\r\n\r\nBs𐞁<'T>…9,9字 'reᵃᵃ!>­ſ\"Džungla漢", "tokens": 58, "pieces": ["é", ".", "Ⅳ", "é", "
", "Ⅳ", ">", " \n", "123", "456", "78", "a", "/b", "👍🏽", "Dž", "\r\n\r\n\r\n", "Bs𐞁", "<'", "T", ">", "…", "9", ",", "9", "字", " '", "reᵃᵃ", "!>­", "ſ", "\"Džungla漢"]} +{"text": "aEOT'ſ'ſ s \n iOS \n ᵃ'refiABC३\n'D", "tokens": 22, "pieces": ["a", "EOT'ſ", "'ſ", " s", " \n", " i", "OS", " \n", " ᵃ're", "fi", "ABC", "३", "\n", "'D"]} +{"text": "tiOS'SＡ\r\n漢9‍　\t12345678'M٣٤٥٦>𐞁३٣٤٥٦\n/!!d漢'VEHTTPServer'reＡABC", "tokens": 48, "pieces": ["ti", "OS'S", "Ａ", "\r\n", "漢", "9", "‍", "　", "\t", "123", "456", "78", "'M", "٣٤٥", "٦", ">𐞁", "३٣٤", "٥٦", "\n", "/!!", "d漢'VE", "HTTPServer're", "ＡABC"]} +{"text": "'S'T‍12345678字ABC", "tokens": 8, "pieces": ["'S'T", "‍", "123", "456", "78", "字", "ABC"]} +{"text": "㍿​'DB!!<Dž😀🏽'D漢\n//tAb　　ᵃ'ReHTTPServerfi\r\n字saBa字iOS'll#$%Z½㋿", "tokens": 47, "pieces": ["㍿​'", "DB", "!!<", "Dž", "😀🏽'", "D漢", "\n", "/<", "META", "_START", ">/", "t", "Ab", "　", "　ᵃ'Re", "HTTPServerfi", "\r\n", "字sa", "Ba字i", "OS'll", "#$%", "Z", "½", "㋿"]} +{"text": "s,\niOS!#$%Ａ\r…😀🏽,a/bZ's­mZḍ̇३'VE\r\n/'ll\u000b.HTTPServereعİm㋿'S'M'Re", "tokens": 45, "pieces": ["s", ",\n", "i", "OS", "!#$%", "Ａ", "\r", "…", "😀🏽,", "a", "/b", "Z's", "­m", "Zḍ̇", "३", "'VE", "\r\n", "/'", "ll", "\u000b", ".HTTPServereع", "İm", "㋿'", "S'M", "'Re"]} +{"text": "ßt 👍🏽́ع'VE's٣٤٥٦!!12345678<|endoftext|>\r/\r\n½ ḍ̇", "tokens": 31, "pieces": ["ßt", " 👍🏽́", "ع'VE", "'s", "٣٤٥", "٦", "!!", "123", "456", "78", "<|", "endoftext", "|>\r/\r\n", "½", " ḍ̇"]} +{"text": "\n/12345678camelCase/٣٤٥٦EOT'ſ's\n/<\ra/biOSİ!!😀🏽a\r\n\r\n\r\n!!åééa㍿a/b", "tokens": 44, "pieces": ["\n", "/", "123", "456", "78", "camel", "Case", "/", "٣٤٥", "٦", "EOT'ſ", "'s", "\n", "/<\r", "a", "/bi", "OSİ", "!!😀🏽", "a", "\r\n\r\n\r\n", "!!", "åééa", "㍿a", "/b"]} +{"text": "👍🏽½Ⅳ
'M \n字 'T ,½́/é'ſ'", "tokens": 21, "pieces": ["👍🏽", "½Ⅳ", "
", "'M", " \n", "字", " ", "'T", " ", " ,", "½", "́", "/é'ſ", "'"]} +{"text": "\rZ'sEOTHTTPServerfi0½AbHTTPServera/b<|endoftext|>…'Re\n<|fim_prefix|>-", "tokens": 36, "pieces": ["\r", "Z's", "EOTHTTPServer", "fi", "0½", "Ab", "HTTPServera", "/b", "<|", "endoftext", "|>", "…", "'Re", "\n", "<|", "fim", "_prefix", "|>-"]} +{"text": "s'D'rét漢é<|endoftext|>…HTTPServerHTTPServerABC9!!'re½'D😀🏽,ß漢Z", "tokens": 35, "pieces": ["s'D", "'rét漢é", "<|", "endoftext", "|>", "…HTTPServer", "HTTPServer", "ABC", "9", "!!'", "re", "½", "'D", "😀🏽,", "ß漢", "Z"]} +{"text": "😀🏽'VEt३ \t9'Reé́!,𐞁́<|endoftext|><|fim_prefix|>㋿9>­́'ReZ're/\r\n 👍🏽'ſ
é'T㍿\r\n\r\n", "tokens": 61, "pieces": ["😀🏽'", "VEt", "३", " ", "\t", "9", "'Reé́", "!,", "𐞁́", "<|", "endoftext", "|><|", "fim", "_prefix", "|>㋿", "9", ">­́'", "Re", "Z're", "/\r\n", " ", "👍🏽'", "ſ", "
é'T", "㍿\r\n\r\n"]} +{"text": "
<|endoftext|>", "tokens": 8, "pieces": ["
", "<|", "endoftext", "|>"]} +{"text": "s.👍🏽é'M漢ꟲ'ß \n #$%0\r\n‍(", "tokens": 25, "pieces": ["s", ".👍🏽", "é'M", "漢ꟲ", "'ß", " \n", " #$%", "0", "\r\n", "‍("]} +{"text": "å漢EOT👍🏽🙂12345678\n/#$% \n'S　Džungla'D🙂Ⅳ12345678fi🙂EOT'", "tokens": 35, "pieces": ["å漢", "EOT", "👍🏽🙂", "123", "456", "78", "\n", "/#$%", " \n", "'S", "　Džungla'D", "🙂", "Ⅳ12", "345", "678", "fi", "🙂EOT", "'"]} +{"text": "३ſ'VEع\"'VE𐞁­'s٣٤٥٦'SZ/!!'ſꟲ<'M 
/\r\nHTTPServerfi", "tokens": 35, "pieces": ["३", "ſ'VE", "ع", "\"'", "VE𐞁", "­'", "s", "٣٤٥", "٦", "'SZ", "/!!'", "ſꟲ", "<'", "M", " ", "
", "/\r\n", "HTTPServerfi"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\u000b're'SＡ'Re𐞁9\r\n\n/,m\r\n\r\ncamelCase𐞁३é\r\n\r\nع­​…\t'SＡ
aBſ…ḍ̇Z/㍿Z‍a/b", "tokens": 53, "pieces": ["\u000b", "'re'S", "Ａ'Re", "𐞁", "9", "\r\n\n", "/,", "m", "\r\n\r\n", "camel", "Case𐞁", "३", "é", "\r\n\r\n", "ع", "­​", "…", "\t", "'SＡ", "
", "a", "Bſ", "…ḍ̇", "Z", "/㍿", "Z", "‍a", "/b"]} +{"text": "-‍.<|endoftext|>́Ⅳꟲa/b字'S#$%m'T
½", "tokens": 25, "pieces": ["-‍.<|", "endoftext", "|>́", "Ⅳ", "ꟲa", "/b字'S", "#$%", "m'T", "
", "½"]} +{"text": "'T,,e0/ \n' B'S'SDžunglaABCZ\u000b.\r\n9\ts\r\n\r\n'M㋿㋿12345678mſ!!
aB́>-
/\r\n👍🏽", "tokens": 48, "pieces": ["'T", ",,", "e", "0", "/", " \n", "'<", "EOT", ">", " ", " B'S", "'SDžungla", "ABCZ", "\u000b", ".\r\n", "9", "\ts", "\r\n\r\n", "'M", "㋿㋿", "123", "456", "78", "mſ", "!!", "
a", "B́", ">-", "
", "/\r\n", "👍🏽"]} +{"text": "ſe­'VE½'re\"́㍿'.0>d,٣٤٥٦Ab'\r\n ZZs're漢Ⅳ­Ａ \n\u000bZ0aB​/\r\nᵃEOT", "tokens": 44, "pieces": ["ſe", "­'", "VE", "½", "'re", "\"́", "㍿'.", "0", ">d", ",", "٣٤٥", "٦", "Ab", "'\r\n", " ", " ZZs're", "漢", "Ⅳ", "­Ａ", " \n", "\u000bZ", "0", "a", "B", "​/\r\n", "ᵃ", "EOT"]} +{"text": "…,😀🏽\n//\r\nHTTPServer!㋿Z'ſ-", "tokens": 18, "pieces": ["…", ",😀🏽\n//\r\n", "HTTPServer", "!㋿", "Z'ſ", "-"]} +{"text": "Dž<|endoftext|>…HTTPServer-ꟲ'VE　字　ßaBDžع9\u000bEOT9'.ꟲ\r\nssaB  Dž", "tokens": 44, "pieces": ["Dž", "<|", "endoftext", "|>", "…HTTPServer", "-ꟲ'VE", "　字", "　ßa", "BDžع", "9", "\u000bEOT", "9", "'.", "ꟲ", "\r\n", "ssa", "B", " ", " Dž"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "/\r\nİe‍#$%'ScamelCase'llEOT/'Dé
字字fi\tＡ\"'MABC/ع'reꟲ­ꟲ­aEOT𐞁字Džꟲßſ", "tokens": 52, "pieces": ["/\r\n", "İe", "‍#$%'", "Scamel", "Case'll", "EOT", "/'", "Dé", "
", "字字fi", "\tＡ", "\"'", "MABC", "/ع're", "ꟲ", "­ꟲ", "­a", "EOT𐞁字Džꟲßſ"]} +{"text": "'VEᵃİ're.'s, !'ſ\r\r\n\r\n \n 9'T-\r\nm​camelCase/\r\n \n \r\n㋿٣٤٥٦字'ſ'VE>'s \n", "tokens": 46, "pieces": ["'VEᵃ", "İ're", ".'", "s", ",", " ", "!'", "ſ", "\r\r\n\r\n \n", " ", "9", "'T", "-\r\n", "m", "​camel", "Case", "/\r\n", " \n \r\n", "㋿", "٣٤٥", "٦", "字'ſ", "'VE", ">'", "s", " \n", ""]} +{"text": "'M\rå<|fim_prefix|> \n9ḍ̇e/ᵃᵃcamelCase\t‍'D<|endoftext|>漢(", "tokens": 44, "pieces": ["'M", "\r", "å", "<|", "fim", "_prefix", "|>", " \n", "9", "ḍ̇e", "/ᵃᵃcamel", "Case", "\t", "‍'", "D", "<|", "endoftext", "|>", "漢", "("]} +{"text": "d-Ab \nḍ̇­aB漢'　…㋿>", "tokens": 17, "pieces": ["d", "-Ab", " \n", "ḍ̇", "­a", "B漢", "'", "　", "…", "㋿>"]} +{"text": "𐞁camelCase<|fim_prefix|>é'll ", "tokens": 15, "pieces": ["𐞁camel", "Case", "<|", "fim", "_prefix", "|>", "é'll", " "]} +{"text": "<👍🏽camelCasecamelCase", "tokens": 8, "pieces": ["<👍🏽", "camel", "Casecamel", "Case"]} +{"text": "EOT-.́½ꟲ!!漢ABCd/\r\n!Džungla'lléABC'Reſſ'Mfim\"'Z'saBEOT'll‍Z", "tokens": 40, "pieces": ["EOT", "-.́", "½", "ꟲ", "!!", "漢ABCd", "/\r\n", "!Džungla'll", "é", "ABC'Re", "ſſ'M", "fi", "m", "\"'", "Z's", "a", "BEOT'll", "‍Z"]} +{"text": "å 'lla/b\n/\r\n㍿\r'　 \n Dž<|fim_prefix|>BAb字😀🏽Džungla\n/d-'Då\r'sᵃcamelCaseDž,\n'MfiZ\rＡ'ſå", "tokens": 61, "pieces": ["å", " '", "lla", "/b", "\n", "/\r\n", "㍿<", "META", "_START", ">\r", "'", "　 \n", " Dž", "<|", "fim", "_prefix", "|>", "BAb字", "😀🏽", "Džungla", "\n", "/d", "-'", "Då", "\r", "'sᵃcamel", "Case", "Dž", ",\n", "'Mfi", "Z", "\r", "Ａ'ſ", "å"]} +{"text": "👍🏽's", "tokens": 5, "pieces": ["👍🏽'", "s"]} +{"text": ".'Re😀🏽Dž", "tokens": 7, "pieces": [".'", "Re", "😀🏽", "Dž"]} +{"text": "camelCaseᵃé🙂a'll mᵃABC<|fim_prefix|>", "tokens": 20, "pieces": ["camel", "Caseᵃé", "🙂a'll", " mᵃ", "ABC", "<|", "fim", "_prefix", "|>"]} +{"text": "عع\r\naBs\r\n\r\n'Re", "tokens": 10, "pieces": ["عع", "\r\n", "a", "Bs", "\r\n\r\n", "'Re"]} +{"text": "
漢a　ß\r\n\r\n9㋿Ⅳ'Re'sABC‍\tABC<'Re㍿㋿漢,'re\"", "tokens": 30, "pieces": ["
漢a", "　ß", "\r\n\r\n", "9", "㋿", "Ⅳ", "'Re's", "ABC", "‍", "\tABC", "<'", "Re", "㍿㋿", "漢", ",'", "re", "\""]} +{"text": "'ſ३fié- ́!B'T𐞁\n/!<👍🏽0'T0DžcamelCase\"!'​ \n'S　.\r<|fim_prefix|>åmABC😀🏽　", "tokens": 59, "pieces": ["'ſ", "३", "fié", "-", " ́", "!<", "META", "_START", ">B'T", "𐞁", "\n", "/!<👍🏽", "0", "'T", "0", "Džcamel", "Case", "\"!'​", " \n", "'S", "　", ".\r", "<|", "fim", "_prefix", "|>", "åm", "ABC", "😀🏽", "　"]} +{"text": "B㍿İ/s𐞁
", "tokens": 11, "pieces": ["B", "㍿İ", "/s𐞁", "
"]} +{"text": "…<|fim_prefix|>HTTPServer'Re३' \n 'sABC<|fim_prefix|>s \n ',", "tokens": 29, "pieces": ["…", "<|", "fim", "_prefix", "|>", "HTTPServer'Re", "३", "'", " \n", " '", "s", "ABC", "<|", "fim", "_prefix", "|>", "s", " \n", " '<", "META", "_START", ">,"]} +{"text": "!!é\u000ba/bß'llfi字漢(Abé\u000b\r…‍d٣٤٥٦ \r'S.é'llEOT.AbcamelCase-\u000b'Re", "tokens": 39, "pieces": ["!!", "é", "\u000ba", "/bß'll", "fi字漢", "(Abé", "\u000b\r", "…", "‍d", "٣٤٥", "٦", " \r", "'S", ".é'll", "EOT", ".Abcamel", "Case", "-", "\u000b", "'Re"]} +{"text": "㋿.Dž
 /\r\n\t३/\r\niOS‍𐞁㍿́\r\n\r\n٣٤٥٦'0😀🏽<|fim_prefix|>㋿ß's㍿'ll ", "tokens": 57, "pieces": ["㋿.", "Dž", "
", " ", "/\r\n", "\t", "३", "/\r\n", "i", "OS", "‍𐞁", "㍿́", "\r\n\r\n", "٣٤٥", "٦", "'", "0", "😀🏽<|", "fim", "_prefix", "|>㋿", "ß", "'", "s", "㍿'", "ll", " "]} +{"text": ">,iOSHTTPServer\"(­ᵃt😀🏽", "tokens": 14, "pieces": [">,", "i", "OSHTTPServer", "\"(­", "ᵃt", "😀🏽"]} +{"text": "'ll", "tokens": 1, "pieces": ["'ll"]} +{"text": " \n…'MaBt'T'M字'lld\tEOT!! ‍Ⅳ\r\n\r\nEOT😀🏽", "tokens": 28, "pieces": [" \n", "…", "'Ma", "Bt'T", "'M字'll", "d", "\tEOT", "!!", " ", " ‍", "Ⅳ", "\r\n\r\n", "EOT", "😀🏽<", "META", "_START", ">"]} +{"text": "iOS½,a/b',EOT\"㋿
'sEOT", "tokens": 16, "pieces": ["i", "OS", "½", ",a", "/b", "',", "EOT", "\"㋿", "
", "'s", "EOT"]} +{"text": "ABC<|fim_prefix|>#$% <|endoftext|>İéſAb'VEZ㍿0㋿㍿t9'VE😀🏽a/b🙂Džungla0'ReDžungla漢#$%'T🙂😀🏽\t", "tokens": 67, "pieces": ["ABC", "<|", "fim", "_prefix", "|>#$%", " ", " <|", "endoftext", "|>", "İéſ", "Ab'VE", "Z", "㍿", "0", "㋿㍿<", "META", "_START", ">t", "9", "'VE", "😀🏽", "a", "/b", "🙂Džungla", "0", "'Re", "Džungla漢", "#$%'", "T", "🙂😀🏽", "\t"]} +{"text": "㋿<|endoftext|>0ḿs\"-<|endoftext|>mᵃ #$%\"\taB're㍿s/'Re漢👍🏽Ⅳ'Re३. Źᵃ", "tokens": 51, "pieces": ["㋿<|", "endoftext", "|>", "0", "ḿs", "\"-<|", "endoftext", "|>", "mᵃ", " ", "#$%\"", "\ta", "B're", "㍿s", "/'", "Re漢", "👍🏽", "Ⅳ", "'Re", "३", ".", " Źᵃ"]} +{"text": "ḍ̇\t\n/'re!!　 å😀🏽åmt!\né<|fim_prefix|>ع-\"''T\t'T́/\r\n!/#$%😀🏽-㋿㋿　d", "tokens": 53, "pieces": ["ḍ̇", "\t\n", "/'", "re", "!!", "　", " å", "😀🏽", "åmt", "!\n", "é", "<|", "fim", "_prefix", "|>", "ع", "-\"''", "T", "\t", "'T́", "/\r\n", "!/#$%😀🏽-㋿㋿", "　d", ""]} +{"text": " ḍ̇…fiBe\n/\r\nAbⅣ\n/'ll", "tokens": 16, "pieces": [" ḍ̇", "…fi", "Be", "\n", "/\r\n", "Ab", "Ⅳ", "\n", "/'", "ll"]} +{"text": "HTTPServer\n/\n-éaB #$%9½.>'T \n\r\n\r\n", "tokens": 21, "pieces": ["HTTPServer", "\n", "/\n", "-éa", "B", " #$%", "9½", ".>'", "T", " \n", "\r\n\r\n"]} +{"text": "ع-३٣٤٥٦\r\n\r\n012345678t३Ⅳ,漢ſiOS𐞁'Tİå'S12345678EOT'D𐞁ma/béⅣ'D😀🏽İ0漢/\r\n a/b
å", "tokens": 58, "pieces": ["ع", "-", "३٣٤", "٥٦", "\r\n\r\n", "012", "345", "678", "t", "३Ⅳ", ",漢ſi", "OS𐞁'T", "İå'S", "123", "456", "78", "EOT'D", "𐞁ma", "/bé", "Ⅳ", "'D", "😀🏽", "İ", "0", "漢", "/\r\n", " ", " a", "/b", "
å"]} +{"text": "Džİ ᵃ‍'MEOT'll!>…'sAba/bع \n0٣٤٥٦­/ \nß'sＡ　‍.\r,‍", "tokens": 38, "pieces": ["Džİ", " ᵃ", "‍'", "MEOT'll", "!>", "…", "'s", "Aba", "/bع", " \n", "0٣٤", "٥٦", "­/", " \n", "ß's", "Ａ", "　", "‍.\r", ",‍"]} +{"text": " e'M'Stſ'ſ ­Džungla​<|endoftext|>ꟲHTTPServer'D/\r\n​<|fim_prefix|>🙂ꟲ(.>/\r\n'Re'll\té𐞁字 \nABC0're", "tokens": 59, "pieces": [" e'M", "'Stſ'ſ", " ", "­Džungla", "​<|", "endoftext", "|>", "ꟲHTTPServer'D", "/\r\n", "​<|", "fim", "_prefix", "|>🙂", "ꟲ", "(.>/\r\n", "'", "Re'll", "\té𐞁字", " \n", "ABC", "0", "'re"]} +{"text": "'ll(
> \n t🙂\n/>\r\n,éꟲ　aHTTPServer>éAb'VE…", "tokens": 26, "pieces": ["'ll", "(", "
", ">", " \n", " t", "🙂\n/", ">\r\n", ",éꟲ", "　a", "HTTPServer", ">é", "Ab'VE", "…"]} +{"text": "!!½𐞁<­ᵃ\"ع ", "tokens": 14, "pieces": ["!!", "½", "𐞁", "<­", "ᵃ", "\"ع", " "]} +{"text": "\"(­­‍‍漢#$%'D\r\n\r\n \n 'llfi0eİ!aB\rå", "tokens": 22, "pieces": ["\"(­­‍‍", "漢", "#$%'", "D", "\r\n\r\n \n", " '", "llfi", "0", "e", "İ", "!a", "B", "\r", "å"]} +{"text": "'M'T漢ABĆ㍿a/bHTTPServer½a/bém#$%Ａ㍿ABC>
Ⅳ३'ſ𐞁'>aBABCB㍿ \n tABC　sa/bm'reᵃ12345678", "tokens": 59, "pieces": ["'M'T", "漢", "ABC", "́", "㍿a", "/b", "HTTPServer", "½", "a", "/bém", "#$%", "Ａ", "㍿ABC", ">", "
", "Ⅳ३", "'ſ𐞁", "'>", "a", "BABCB", "㍿", " \n", " t", "ABC", "　sa", "/bm're", "ᵃ", "123", "456", "78"]} +{"text": "ع 
é́ꟲ-'M'S\"\t12345678>sİå", "tokens": 20, "pieces": ["ع", " ", "
é́ꟲ", "-'", "M'S", "\"", "\t", "123", "456", "78", ">s", "İå"]} +{"text": " \n /\r\n­'Dß👍🏽t\n/Ⅳ#$% B9​B\"Ⅳ'Re½Ａ<|endoftext|>字́", "tokens": 37, "pieces": [" \n", " /\r\n", "­'", "Dß", "👍🏽", "t", "\n", "/", "Ⅳ", "#$%", " B", "9", "​B", "\"", "Ⅳ", "'Re", "½", "Ａ", "<|", "endoftext", "|>", "字́"]} +{"text": "e/\r\n \n ३sᵃsDžunglaDžß \n 　'Re!ḍ̇́'ſ'D'sABCABC<|endoftext|>ABC0\r\n\r\n\r\n…!'VE👍🏽𐞁'Re'ſé,", "tokens": 63, "pieces": ["e", "/\r\n", " \n", " ", "३", "sᵃs", "Džungla", "Džß", " \n", " ", "　", "'Re", "!ḍ̇́'ſ", "'D's", "ABCABC", "<|", "endoftext", "|>", "ABC", "0", "\r\n\r\n\r\n", "…", "!'", "VE", "👍🏽", "𐞁", "'", "Re'ſ", "é", ","]} +{"text": "\n/B'VE३'a/b\rİ\r­字　　Džunglaſ
ꟲé/\r\nHTTPServer>\">", "tokens": 30, "pieces": ["\n", "/B'VE", "३", "'a", "/b", "\r", "İ", "\r", "­字", "　", "　Džunglaſ", "
ꟲé", "/\r\n", "HTTPServer", ">\">"]} +{"text": "'MHTTPServer\r\n\r\n㍿téᵃt㍿ع!se0\r\nᵃ/\r\nße'Mé😀🏽‍\t́Dž😀🏽,#$%", "tokens": 44, "pieces": ["'MHTTPServer", "\r\n\r\n", "㍿téᵃt", "㍿ع", "!", "se", "0", "\r\n", "ᵃ", "/\r\n", "ße'M", "é", "😀🏽‍", "\t́", "Dž", "😀🏽,#$%"]} +{"text": " EOT'll'ſ́ⅣDžungla
 \n(.a/b\n/字s", "tokens": 24, "pieces": [" EOT'll", "'ſ́", "", "Ⅳ", "Džungla", "
 \n", "(.", "a", "/b", "\n", "/字s"]} +{"text": "‍'re\r\n\r\n字'lla-'s<12345678'D'MiOSé­ \n", "tokens": 24, "pieces": ["‍'", "re", "\r\n\r\n", "字'll", "a", "-'", "s", "<", "123", "456", "78", "'D'M", "i", "OSé", "­", " \n"]} +{"text": "Ａ'VE'M/ 👍🏽Džungla12345678'ſ9m \n \n\n/daDžungla'T'S<|fim_prefix|>ſaB<|endoftext|>m's𐞁‍Z🙂é'S'reBfi", "tokens": 71, "pieces": ["Ａ'VE", "'", "M", "/", " ", "👍🏽", "Džungla", "123", "456", "78", "'ſ", "", "9", "m", " \n \n\n", "/da", "Džungla'T", "'S", "<|", "fim", "_prefix", "|>", "ſ", "a", "B", "<|", "endoftext", "|>", "m's", "𐞁", "‍Z", "🙂é'S", "'re", "Bfi"]} +{"text": "……''re\r\n٣٤٥٦-B<|fim_prefix|>😀🏽(-­'ll(
iOS<|fim_prefix|>'s👍🏽
\r\ne\r\n\r\n​a/b 𐞁,", "tokens": 52, "pieces": ["…", "…", "''", "re", "\r\n", "٣٤٥", "٦", "-B", "<|", "fim", "_prefix", "|>😀🏽(-­'", "ll", "(", "
i", "OS", "<|", "fim", "_prefix", "|>'", "s", "👍🏽", "
\r\n", "e", "\r\n\r\n", "​a", "/b", " 𐞁", ","]} +{"text": "­", "tokens": 1, "pieces": ["­"]} +{"text": "ß\r\n🙂m'T9camelCase'Re\u000bAb'M ", "tokens": 13, "pieces": ["ß", "\r\n", "🙂m'T", "9", "camel", "Case'Re", "\u000bAb'M", " "]} +{"text": "åⅣ<|endoftext|>", "tokens": 11, "pieces": ["å", "Ⅳ", "<|", "endoftext", "|>"]} +{"text": " \n camelCaseß\rḍ̇.٣٤٥٦ſ३३EOT", "tokens": 18, "pieces": [" \n", " camel", "Caseß", "\r", "ḍ̇", ".", "٣٤٥", "٦", "ſ", "३३", "EOT"]} +{"text": "٣٤٥٦aB/\r\n½½'VEDž/\r\nm字Ａm👍🏽9dᵃ\n,\"camelCase'M漢'sß,Ａ'Tع𐞁Z's fiꟲå'D", "tokens": 56, "pieces": ["٣٤٥", "٦", "a", "B", "/\r\n", "½½", "'VEDž", "/\r\n", "m字", "Ａm", "👍🏽", "9", "dᵃ", "\n", ",\"", "camel", "Case'M", "漢's", "ß", ",Ａ'T", "ع𐞁", "Z's", " ", " fiꟲå'D"]} +{"text": "/\r\nfi,0", "tokens": 8, "pieces": ["/\r\n", "fi", ",", "0"]} +{"text": "ᵃ-İe­🙂s'Tå're\r\n\r\nＡ\r\n\r\n👍🏽ḍ̇>", "tokens": 23, "pieces": ["ᵃ", "-İe", "­🙂", "s'T", "å're", "\r\n\r\n", "Ａ", "\r\n\r\n", "👍🏽", "ḍ̇", ">"]} +{"text": " \n B ٣٤٥٦👍🏽👍🏽,'Dᵃ<|endoftext|><|fim_prefix|>iOS'll/fi'D'VEDžungla \n ꟲsDžZ👍🏽𐞁", "tokens": 58, "pieces": [" \n", " B", " ", "٣٤٥", "٦", "👍🏽👍🏽,'", "Dᵃ", "<|", "endoftext", "|><|", "fim", "_prefix", "|>", "i", "OS'll", "/fi'D", "'VEDžungla", " \n", " ꟲs", "DžZ", "👍🏽", "𐞁"]} +{"text": "d½\n/iOS .B-\n'́字iOS<|fim_prefix|>😀🏽camelCase‍å३½ꟲ'Sfi a/b,e'reAb/\r\ńDžcamelCase'S", "tokens": 51, "pieces": ["d", "½", "\n", "/i", "OS", " ", " <", "META", "_START", ">.", "B", "-\n", "'́字i", "OS", "<|", "fim", "_prefix", "|>😀🏽", "camel", "Case", "‍å", "३½", "ꟲ'S", "fi", " a", "/b", ",e're", "Ab", "/\r\n", "́Džcamel", "Case'S"]} +{"text": "'M''T(\u000bé\r\n dⅣ're‍'a(𐞁㍿३a/b\n(m\r\n\r\n/\r\n\r.‍>/\r\n \n /\r\n'ReDžZ", "tokens": 44, "pieces": ["'M", "''", "T", "(", "\u000bé", "\r\n", " d", "Ⅳ", "'re", "‍'", "a", "(", "𐞁", "㍿", "३", "a", "/b", "\n", "(m", "\r\n\r\n", "/\r\n\r", ".‍>/\r\n", " \n", " /\r\n", "'Re", "DžZ"]} +{"text": "́ \r\n", "tokens": 2, "pieces": ["́", " \r\n"]} +{"text": "#$%\n\n.ع́'Rea٣٤٥٦\r\né-/\r\né ", "tokens": 18, "pieces": ["#$%\n\n", ".ع́'Re", "a", "٣٤٥", "٦", "\r\n", "é", "-/\r\n", "é", " "]} +{"text": "<|fim_prefix|>", "tokens": 6, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "ABCDžunglaaB9camelCase
å>s👍🏽 'TＡDžungla­\r.'T👍🏽,", "tokens": 36, "pieces": ["ABCDžunglaa", "B", "9", "camel", "Case", "
å", ">s", "👍🏽", " '", "TＡDžungla", "­\r", ".'", "T", "👍🏽,"]} +{"text": "　\t'D#$%Ab/…", "tokens": 9, "pieces": ["　", "\t", "'D", "#$%", "Ab", "/", "…"]} +{"text": "'DHTTPServermḍ̇<Džungla \"åＡ漢𐞁㋿Džungla>ſEOT٣٤٥٦ⅣＡ'sＡsfiḍ̇sZ\ts0", "tokens": 58, "pieces": ["'DHTTPServermḍ̇", "<Džungla", " ", " \"", "å", "Ａ漢𐞁", "㋿Džungla", ">ſ", "EOT", "٣٤٥", "٦Ⅳ", "Ａ's", "Ａs", "fiḍ̇s", "Z", "\ts", "", "0"]} +{"text": "mİ漢 🙂㍿\n/e'ſ …
camelCaseaBḍ̇Ⅳ", "tokens": 25, "pieces": ["m", "İ漢", " ", "🙂㍿\n/", "e'ſ", " …", "
camel", "Casea", "Bḍ̇", "Ⅳ"]} +{"text": "𐞁(👍🏽漢Dž…　's>ꟲåfi३'T
d!!>\n/t\tß½ⅣＡås\na/b'M३!漢a/b", "tokens": 56, "pieces": ["𐞁", "(👍🏽", "漢", "Dž", "…", "", "　", "'s", ">ꟲå", "fi", "३", "'T", "
d", "!!>\n/", "t", "\tß", "½Ⅳ", "Ａås", "\n", "a", "/b'M", "३", "!漢a", "/b"]} +{"text": "12345678 \n ḍ̇'S'", "S", "'ḍ̇\r\n'M0字!!(🙂'reEOTAba/bḍ̇m\n/　٣٤٥٦\n/-a/b<|fim_prefix|>ß👍🏽åDžungla\r\n\r\n#$%a/b<", "tokens": 59, "pieces": ["<|", "fim", "_prefix", "|>'", "ḍ̇", "\r\n", "'M", "0", "字", "!!(🙂'", "re", "EOTAba", "/bḍ̇m", "\n", "/", "　", "٣٤٥", "٦", "\n", "/-", "a", "/b", "<|", "fim", "_prefix", "|>", "ß", "👍🏽", "å", "Džungla", "\r\n\r\n", "#$%", "a", "/b", "<"]} +{"text": "𐞁<|endoftext|>'½\r\n\r\nḍ̇　ABC字!!ᵃ😀🏽12345678\ré!aB\nAb'T🙂\t\"Dž\t字Džع३\nDž<|endoftext|>t🙂camelCase㋿ḍ̇", "tokens": 68, "pieces": ["𐞁", "<|", "endoftext", "|>'", "½", "\r\n\r\n", "ḍ̇", "　ABC字", "!!", "ᵃ", "😀🏽", "123", "456", "78", "\r", "é", "!a", "B", "\n", "Ab'T", "🙂", "\t", "\"Dž", "\t字Džع", "३", "\n", "Dž", "<|", "endoftext", "|>", "t", "🙂camel", "Case", "㋿ḍ̇"]} +{"text": "'VE'Re٣٤٥٦. \n 'THTTPServerＡ'll३ m🙂#$%𐞁fiiOS
漢\nEOT㍿éiOSa/b0a'
 \n 
", "tokens": 50, "pieces": ["'VE'Re", "٣٤٥", "٦", ".", " \n", " '", "THTTPServer", "Ａ'll", "३", " ", " m", "🙂#$%", "𐞁fii", "OS", "", "
漢", "\n", "EOT", "㍿éi", "OSa", "/b", "0", "a", "'", "
 \n", " 
"]} +{"text": "३Z 9/<|endoftext|>", "tokens": 15, "pieces": ["", "३", "Z", " ", "9", "/<|", "endoftext", "|>"]} +{"text": "'T.½AbİABCḍ̇३iOS'ᵃ'M", "tokens": 17, "pieces": ["'T", ".", "½", "Ab", "İABCḍ̇", "३", "i", "OS", "'ᵃ'M"]} +{"text": "tⅣ'ReiOSḍ̇'M<|endoftext|>,㍿Ａ#$%0\tZ.'‍½a/b9… /\r\n　HTTPServerḍ̇/\r\n're''VEHTTPServer \n漢　Ⅳ", "tokens": 57, "pieces": ["t", "Ⅳ", "'Rei", "OSḍ̇'M", "<|", "endoftext", "|>,㍿", "Ａ", "#$%", "0", "", "\tZ", ".'‍", "½", "a", "/b", "9", "… ", " /\r\n", "　HTTPServerḍ̇", "/\r\n", "'re", "''", "VEHTTPServer", " \n", "漢", "　", "Ⅳ"]} +{"text": "'D\tiOSt'\u000b\reſa 👍🏽 \n .ᵃ'TeＡḍ̇a/b!!", "tokens": 31, "pieces": ["'D", "\ti", "OSt", "'", "\u000b\r", "eſa", " ", " 👍🏽", " \n", " <", "META", "_START", ">.", "ᵃ'T", "e", "Ａḍ̇a", "/b", "!!"]} +{"text": "३'s<ḍ̇ ſ0åß𐞁 🙂camelCase12345678", "tokens": 27, "pieces": ["३", "'", "s", "<ḍ̇", " ", " ſ", "0", "åß𐞁", " ", "🙂camel", "Case", "123", "456", "78"]} +{"text": "عEOT<|endoftext|>'Re\r­½a'ReDžungla0ḍ̇aå  'S‍́ Dž🙂", "tokens": 36, "pieces": ["ع", "EOT", "<|", "endoftext", "|>'", "Re", "\r", "­", "½", "a'Re", "Džungla", "0", "ḍ̇aå", " ", " ", "'S", "‍́", " ", " Dž", "🙂"]} +{"text": "‍㍿ᵃé\"<|endoftext|><|fim_prefix|>\ttᵃ​'Ta<|endoftext|>'½ \té字ᵃEOTABC<|fim_prefix|>'s", "tokens": 56, "pieces": ["‍㍿", "ᵃé", "\"<|", "endoftext", "|><|", "fim", "_prefix", "|>", "\ttᵃ", "​'", "Ta", "<|", "endoftext", "|>'", "½", " ", "\té字ᵃ", "EOTABC", "<|", "fim", "_prefix", "|>'", "s"]} +{"text": "'Bd\"ßsa/b .BAb…>é‍es\n/ \nAb9ḍ̇\r\nBd㍿٣٤٥٦EOT/\r\nABC", "tokens": 41, "pieces": ["'Bd", "\"ßsa", "/b", " ", " .", "BAb", "…", ">é", "‍es", "\n", "/", " \n", "Ab", "9", "ḍ̇", "\r\n", "Bd", "㍿", "٣٤٥", "٦", "EOT", "/\r\n", "ABC"]} +{"text": "
fiåꟲ  ꟲ12345678ᵃ…-‍B'Dm>å#$%­/İ ß'ſ'llＡ.!! /", "tokens": 46, "pieces": ["", "
fiåꟲ", " ", " ꟲ", "123", "456", "78", "ᵃ", "…", "-‍", "B'D", "m", ">å", "#$%­/", "İ", " ß'ſ", "'ll", "Ａ", ".!!", " ", " /"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "عꟲ​/\r\n　s🙂'VE👍🏽٣٤٥٦0Džungla㋿/\r\nAb㋿'TDž/0Ａ३٣٤٥٦漢ß9ꟲ", "tokens": 49, "pieces": ["عꟲ", "​/\r\n", "　s", "🙂'", "VE", "👍🏽", "٣٤٥", "٦0", "Džungla", "㋿/\r\n", "Ab", "㋿'", "TDž", "/", "0", "Ａ", "३٣٤", "٥٦", "漢ß", "9", "ꟲ"]} +{"text": "'ſ/!!漢/\r\nm👍🏽sfi9ſ\r\n👍🏽's\t!!३camelCaseſfi\rꟲ😀🏽>ś㍿éAb \n ,\t㋿", "tokens": 48, "pieces": ["'ſ", "/!!", "漢", "/\r\n", "m", "👍🏽", "sfi", "9", "ſ", "\r\n", "👍🏽'", "s", "\t", "!!", "३", "camel", "Caseſfi", "\r", "ꟲ", "😀🏽>", "ś", "㍿é", "Ab", " \n", " ,", "\t", "㋿"]} +{"text": "camelCase'llⅣe　>12345678\r\n \n ㍿㍿'VEcamelCase​ABC'res𐞁漢ḍ̇ ", "tokens": 37, "pieces": ["camel", "Case'll", "Ⅳ", "e", "　", ">", "123", "456", "78", "\r\n \n", " ㍿㍿'", "VEcamel", "Case", "​ABC're", "s𐞁漢ḍ̇", " "]} +{"text": "eDžungla‍<|fim_prefix|><", "tåé", "Džß", "३", "́t", "-a", "\t", "…ḍ̇𐞁", "9", "/\r\n", "ḍ̇", "👍🏽", "\u000b", "…İ", "
"]} +{"text": "'ll　dm/\r\nDž́𐞁<|fim_prefix|>\n/😀🏽́Dž字\n字\r(Z", "tokens": 29, "pieces": ["'ll", "　dm", "/\r\n", "Dž́𐞁", "<|", "fim", "_prefix", "|>\n/", "😀🏽́", "Dž字", "\n", "字", "\r", "(Z"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "٣٤٥٦'T​🙂s​!
'DžHTTPServerZḍ̇\u000b'ſ!'M　\u000ba/b​👍🏽-​éDžunglaB \n,/ᵃ३", "tokens": 46, "pieces": ["٣٤٥", "٦", "'T", "​🙂", "s", "​!", "
", "'DžHTTPServer", "Zḍ̇", "\u000b", "'ſ", "!'", "M", "　", "\u000ba", "/b", "​👍🏽-​", "é", "Džungla", "B", " \n", ",/", "ᵃ", "३"]} +{"text": "🙂é\r(
'T>12345678­ 'll Ａ́\reaBḍ̇ \n 'VEdé(Z<‍Ⅳꟲ/😀🏽😀🏽'VE\u000bDž\r\tå𐞁", "tokens": 58, "pieces": ["🙂é", "\r", "(", "
", "'T", ">", "123", "456", "78", "­", " '", "ll", " ", " Ａ́", "\r", "ea", "Bḍ̇", " \n", " '", "VEdé", "(Z", "<‍", "Ⅳ", "ꟲ", "/😀🏽😀🏽'", "VE", "\u000bDž", "\r", "\tå𐞁"]} +{"text": "\u000b\n/'Mſ'ſ'd", "tokens": 8, "pieces": ["\u000b\n", "/'", "Mſ'ſ", "'d"]} +{"text": "<|fim_prefix|>Dž><|fim_prefix|> \n/\r\naiOŚİ\r\n\r'M/\rå'ſ'Tع\n ᵃعABC'Re", "tokens": 39, "pieces": ["<|", "fim", "_prefix", "|>", "Dž", "><|", "fim", "_prefix", "|>", " \n", "/\r\n", "ai", "OŚ", "İ", "\r\n\r", "'M", "/\r", "å'ſ", "'Tع", "\n", " ᵃع", "ABC'Re"]} +{"text": "HTTPServerB३EOTⅣtm12345678'll!!", "tokens": 17, "pieces": ["HTTPServer", "B", "", "३", "EOT", "Ⅳ", "tm", "123", "456", "78", "'ll", "!!"]} +{"text": "EOTcamelCase
३å🙂m-\u000b🙂9Ab٣٤٥٦\u000bDžungla㍿!!iOS\n/<|endoftext|>'TaB٣٤٥٦/\r\n\r\n'sa/b…fi<|fim_prefix|>\n‍EOT", "tokens": 60, "pieces": ["EOTcamel", "Case", "
", "३", "å", "🙂m", "-", "\u000b", "🙂", "9", "Ab", "٣٤٥", "٦", "\u000bDžungla", "㍿!!", "i", "OS", "\n", "/<|", "endoftext", "|>'", "Ta", "B", "٣٤٥", "٦", "/\r\n\r\n", "'sa", "/b", "…fi", "<|", "fim", "_prefix", "|>\n", "‍EOT"]} +{"text": "DžunglaABC𐞁\r /\r\n𐞁 \n t👍🏽Ab'sdméé9'M!!é…ꟲ\t-/\r\n'éé", "tokens": 41, "pieces": ["Džungla", "ABC𐞁", "\r", " /\r\n", "𐞁", " \n", " t", "👍🏽", "Ab's", "dméé", "9", "'M", "!!", "é", "…ꟲ", "\t", "-/\r\n", "'éé"]} +{"text": "ß,ᵃs\r\n
B<9'll \n 🙂é9Z
>iOS<|endoftext|>\r漢ß<|endoftext|>👍🏽٣٤٥٦㋿", "tokens": 53, "pieces": ["ß", ",ᵃs", "\r\n", "
B", "<", "9", "'ll", " \n", " <", "EOT", ">🙂", "é", "9", "Z", "
", ">i", "OS", "<|", "endoftext", "|>\r", "漢ß", "<|", "endoftext", "|>👍🏽", "٣٤٥", "٦", "㋿"]} +{"text": "camelCase'Re,t'rem'ſꟲ🙂\r\nſ　👍🏽fi\r/iOSZ(", "tokens": 31, "pieces": ["camel", "Case'Re", ",t're", "m'ſ", "ꟲ", "🙂<", "META", "_START", ">\r\n", "ſ", "　", "👍🏽", "fi", "\r", "/i", "OSZ", "("]} +{"text": "'re'VE'/\r\n#$%Ⅳ\r\n'DZfiعEOT'Re
t　字😀🏽<|endoftext|>", "tokens": 31, "pieces": ["'re'VE", "'/\r\n", "#$%", "Ⅳ", "\r\n", "'DZfiع", "EOT'Re", "
t", "　字", "😀🏽<|", "endoftext", "|>"]} +{"text": "éHTTPServerᵃ👍🏽\r\n\r\n<Ⅳ're\"0\u000b9👍🏽​ع! ­'T\r\n\r\n ३'VE<|fim_prefix|>​\n/‍\n0\u000b٣٤٥٦Džungla", "tokens": 57, "pieces": ["é", "HTTPServerᵃ", "👍🏽\r\n\r\n", "<", "Ⅳ", "'re", "\"", "0", "\u000b", "9", "👍🏽​", "ع", "!", " ", "­'", "T", "\r\n\r\n", " ", "३", "'VE", "<|", "fim", "_prefix", "|>​\n/", "‍\n", "0", "\u000b", "٣٤٥", "٦", "Džungla"]} +{"text": "'　..!0eſB \n ㋿'ſ's'T'漢\r\nZ\n/Abaé dAbEOT<|endoftext|>́👍🏽", "tokens": 40, "pieces": ["'", "　", "..!", "0", "eſ", "B", " \n", " ㋿'", "ſ's", "'T", "'漢", "\r\n", "Z", "\n", "/Abaé", " ", " d", "Ab", "EOT", "<|", "endoftext", "|>́👍🏽"]} +{"text": "'M/\r\nZHTTPServer　<ᵃ \n e\n/fi.d9fi㋿'M字camelCasemd/👍🏽Ⅳ㍿字é😀🏽\r\r\t'T'll", "tokens": 46, "pieces": ["'M", "/\r\n", "ZHTTPServer", "　", "<ᵃ", " \n", " e", "\n", "/fi", ".d", "9", "fi", "㋿'", "M字camel", "Casemd", "/👍🏽", "Ⅳ", "㍿字é", "😀🏽\r\r", "\t", "'T'll"]} +{"text": "🙂ᵃ9ḍ̇‍AbⅣ३߅ſ\"", "tokens": 41, "pieces": ["🙂", "ᵃ", "9", "ḍ̇", "‍Ab", "Ⅳ३", "ß", "…ſ", "\""]} +{"text": "­𐞁EOTDžHTTPServerEOTعa\r\n\r\n", "tokens": 16, "pieces": ["­𐞁EOTDžHTTPServer", "EOTعa", "\r\n\r\n"]} +{"text": "éع!", "tokens": 3, "pieces": ["éع", "!"]} +{"text": "åaDžunglaacamelCase \n'D㋿/\r\n'sfí\n(\r!!EOTfi#$%iOS's'👍🏽(eEOT漢camelCase'S<\"㋿\r\n\u000b'T ", "tokens": 56, "pieces": ["åa", "Džungla", "acamel", "Case", " \n", "'D", "㋿/\r\n", "'sfí", "\n", "(\r", "!!", "EOTfi", "#$%", "i", "OS's", "'👍🏽(", "e", "EOT漢camel", "Case'S", "<\"㋿\r\n", "\u000b", "'T", " "]} +{"text": "\n\r\na/bAb\r
字­ſEOT漢0Ａ", "tokens": 14, "pieces": ["\n\r\n", "a", "/b", "Ab", "\r", "
字", "­ſ", "EOT漢", "0", "Ａ"]} +{"text": "ꟲs​ſ\t ́'ReaBaDž/'ReéⅣAbficamelCase𐞁aå\r\n\r\n<|endoftext|>İ\t \n m\u000b", "tokens": 42, "pieces": ["ꟲs", "​ſ", "\t", " ́'Re", "a", "Ba", "Dž", "/'", "Reé", "Ⅳ", "Abficamel", "Case𐞁aå", "\r\n\r\n", "<|", "endoftext", "|>", "İ", "\t \n", " m", "\u000b"]} +{"text": "㍿(iOS\n/ᵃ㋿ #$%aİétaB 👍🏽Ａ!!('ll#$%!!HTTPServerᵃ́　HTTPServer‍'re", "tokens": 44, "pieces": ["㍿(", "i", "OS", "\n", "/ᵃ", "㋿", " ", "#$%", "a", "İéta", "B", " ", " 👍🏽", "Ａ", "!!('", "ll", "#$%!!", "HTTPServerᵃ́", "　HTTPServer", "‍'", "re"]} +{"text": "३Ⅳ!!camelCase'reaAbå​\r👍🏽 😀🏽́ḍ̇ⅣⅣ", "tokens": 28, "pieces": ["३Ⅳ", "!!", "camel", "Case're", "a", "Abå", "​\r", "👍🏽", " ", "😀🏽́", "ḍ̇", "ⅣⅣ"]} +{"text": "İ9mＡ㍿　\u000b\r\n\r\n<|endoftext|>Z'M'Reİ \n mZ🙂m12345678Ⅳ😀🏽٣٤٥٦\"12345678EOT(EOTB0\n𐞁😀🏽t'llå", "tokens": 64, "pieces": ["İ", "9", "m", "Ａ", "㍿", "　\u000b\r\n\r\n", "<|", "endoftext", "|>", "Z'M", "'Re", "İ", " \n", " m", "Z", "🙂m", "123", "456", "78Ⅳ", "😀🏽<", "EOT", ">", "٣٤٥", "٦", "\"", "123", "456", "78", "EOT", "(EOTB", "0", "\n", "𐞁", "😀🏽", "t'll", "å"]} +{"text": "'re'res!'s'll'D'Reé漢''ſ 'Re \n aB\r", "tokens": 18, "pieces": ["'re're", "s", "!'", "s'll", "'D'Re", "é漢", "''", "ſ", " '", "Re", " \n", " a", "B", "\r"]} +{"text": "> \ń­\r\n\r\n12345678/…HTTPServerᵃm\n 'T…,ſ\n/ꟲ/\r\n \nå㋿.DžunglaHTTPServerᵃ\t", "tokens": 55, "pieces": [">", " \n", "́", "­\r\n\r\n", "123", "456", "78", "/", "…HTTPServerᵃm", "\n", " ", " '", "T", "…", ",ſ", "\n", "/ꟲ", "/\r\n", " \n", "å", "㋿.", "Džungla", "HTTPServerᵃ", "\t", ""]} +{"text": ">å,camelCasesＡ字Z<|fim_prefix|>s🙂fi're'iOS\n/𐞁­m'VE'D'ſ🙂½", "tokens": 35, "pieces": [">å", ",camel", "Cases", "Ａ字", "Z", "<|", "fim", "_prefix", "|>", "s", "🙂fi're", "'i", "OS", "\n", "/𐞁", "­m'VE", "'D'ſ", "🙂", "½"]} +{"text": "\r\n\r\n\"!!　fi'S🙂0İ0\u000b👍🏽('S\"EOT", "tokens": 18, "pieces": ["\r\n\r\n", "\"!!", "　fi'S", "🙂", "0", "İ", "0", "\u000b", "👍🏽('", "S", "\"EOT"]} +{"text": "ꟲ🙂/\r\nå912345678㍿३ABCſع\n'Tİ'T/\r\n/", "tokens": 26, "pieces": ["ꟲ", "🙂/\r\n", "å", "912", "345", "678", "㍿", "३", "ABCſع", "\n", "'Tİ'T", "/\r\n/", ""]} +{"text": "(tB(<|fim_prefix|>'D\"'re㍿\rDž\r\n'reB.'ſ'M", "tokens": 23, "pieces": ["(t", "B", "(<|", "fim", "_prefix", "|>'", "D", "\"'", "re", "㍿\r", "Dž", "\r\n", "'re", "B", ".'", "ſ'M"]} +{"text": "'漢iOS\n/ \néd漢,é\r\"m\rſDžunglaaBéß(İ漢!!ſع\r, 'SDžungla camelCase", "tokens": 42, "pieces": ["'漢i", "OS", "\n", "/", " \n", "éd漢", ",é", "\r", "\"m", "\r", "ſ", "Džunglaa", "Béß", "(İ漢", "!!", "ſع", "\r", ",", " ", " '", "SDžungla", " camel", "Case"]} +{"text": "t(12345678dEOT👍🏽", "tokens": 11, "pieces": ["t", "(", "123", "456", "78", "d", "EOT", "👍🏽"]} +{"text": "ꟲ字🙂s", "tokens": 6, "pieces": ["ꟲ字", "🙂s"]} +{"text": "Abå\r\n\r\nDžungla<|endoftext|>Ａ're'DⅣéaB!aBZiOS,\rDžungla#$%Ａ0e\n ́Ⅳ\r\n\n/🙂.㋿iOS", "tokens": 54, "pieces": ["Abå", "\r\n\r\n", "Džungla", "<|", "endoftext", "|>", "Ａ're", "'D", "Ⅳ", "éa", "B", "!a", "BZi", "OS", ",\r", "Džungla", "#$%", "Ａ", "0", "e", "\n", " ́", "Ⅳ", "\r\n\n", "/🙂.㋿", "i", "OS"]} +{"text": "'T½漢>éḍ̇'re#$%a/bdB'S", "tokens": 20, "pieces": ["'T", "½", "漢", ">éḍ̇'re", "#$%<", "EOT", ">a", "/bd", "B'S"]} +{"text": "0'\r\n\r\n", "tokens": 2, "pieces": ["0", "'\r\n\r\n"]} +{"text": "'T٣٤٥٦12345678/ ,㍿\"'TBⅣ㍿", "tokens": 21, "pieces": ["'T", "٣٤٥", "٦12", "345", "678", "/", " ", " ,㍿\"'", "TB", "Ⅳ", "㍿"]} +{"text": "m!<|endoftext|>.é🙂> 'ſ \n !!å", "tokens": 19, "pieces": ["m", "!<|", "endoftext", "|>.", "é", "🙂>", " '", "ſ", " \n", " !!", "å"]} +{"text": "t-\r\n\r\nDž‍", "tokens": 6, "pieces": ["t", "-\r\n\r\n", "Dž", "‍"]} +{"text": "<|endoftext|>\r\n\r\nſ　'D(​Džungla‍EOTİ 9B. \n camelCase!!t…>…'ll'Mḍ̇­EOT.½\t\u000b", "tokens": 51, "pieces": ["<|", "endoftext", "|>\r\n\r\n", "ſ", "　", "'D", "(​", "Džungla", "‍EOT", "İ", " ", " ", "9", "B", ".", " \n", " camel", "Case", "!!", "t", "…", ">", "…", "'ll'M", "ḍ̇", "­EOT", ".", "½", "\t\u000b"]} +{"text": "iOS٣٤٥٦\tt/\r\nᵃ<|fim_prefix|>.Ⅳ👍🏽🙂12345678\u000bmDžB/\r\n", "tokens": 35, "pieces": ["i", "OS", "٣٤٥", "٦", "\tt", "/\r\n", "ᵃ", "<|", "fim", "_prefix", "|>.", "Ⅳ", "👍🏽🙂", "123", "456", "78", "\u000bm", "DžB", "/\r\n"]} +{"text": ".\r\nＡ<́9d's　'll😀🏽é\r\n\r\n aB\rZcamelCase'sm>0<|endoftext|>s'reABC'Reḍ̇‍\r\n\r\n字Džungla\t\u000ba/bAb", "tokens": 50, "pieces": [".\r\n", "Ａ", "<́", "9", "d's", "　", "'ll", "😀🏽", "é", "\r\n\r\n", " a", "B", "\r", "Zcamel", "Case's", "m", ">", "0", "<|", "endoftext", "|>", "s're", "ABC'Re", "ḍ̇", "‍\r\n\r\n", "字Džungla", "\t", "\u000ba", "/b", "Ab"]} +{"text": " !漢fiEOT㍿a/bm'ree\r\n\r\n!!<|endoftext|>\nع…", "tokens": 25, "pieces": [" !", "漢fi", "EOT", "㍿a", "/bm're", "e", "\r\n\r\n", "!!<|", "endoftext", "|>\n", "ع", "…"]} +{"text": "s​\n/å,漢\u000b", "tokens": 8, "pieces": ["s", "​\n/", "å", ",漢", "\u000b"]} +{"text": "字٣٤٥٦'ſßſ/\r\na/bꟲ!!'T<|endoftext|>🙂å\r\n\r\naå", "tokens": 31, "pieces": ["字", "٣٤٥", "٦", "'ſßſ", "/\r\n", "a", "/bꟲ", "!!'", "T", "<|", "endoftext", "|>🙂", "å", "\r\n\r\n", "aå"]} +{"text": "ZaB >٣٤٥٦<|endoftext|>'llB\rZ(<|endoftext|>HTTPServerEOT>0😀🏽'D'T(HTTPServer,ß \n　a/bⅣ\u000baB B\r\n/'llße", "tokens": 57, "pieces": ["Za", "B", " ", " >", "٣٤٥", "٦", "<|", "endoftext", "|>'", "ll", "B", "\r", "Z", "(<|", "endoftext", "|>", "HTTPServer", "EOT", ">", "0", "😀🏽'", "D'T", "(HTTPServer", ",ß", " \n", "　a", "/b", "Ⅳ", "\u000ba", "B", " B", "\r\n", "/'", "llße"]} +{"text": "s'", "tokens": 2, "pieces": ["s", "'"]} +{"text": "<|endoftext|>e\n­'llé!12345678ſ'S're-𐞁٣٤٥٦\r\n\n ('sᵃ", "tokens": 35, "pieces": ["<|", "endoftext", "|>", "e", "\n", "­'", "llé", "!", "123", "456", "78", "ſ'S", "'re", "-𐞁", "٣٤٥", "٦", "\r\n\n", " ('", "sᵃ"]} +{"text": "DžunglaDžungla(- a/b👍🏽\n .👍🏽0ع\n!!'ReéiOS ­ \naB　", "tokens": 37, "pieces": ["Džungla", "Džungla", "(-", " ", " a", "/b", "👍🏽\n", " ", ".👍🏽", "0", "ع", "\n", "!!'", "Reéi", "OS", " ", " ­", " \n", "a", "B", "　"]} +{"text": "å#$%a ḍ̇å'VE'S‍<漢‍<😀🏽\r\n'S\t. \tiOSAb \ns's٣٤٥٦'ll/\r\nع.ḍ̇'Re 9a/bAbABC'S", "tokens": 52, "pieces": ["å", "#$%", "a", " ḍ̇å'VE", "'S", "‍<", "漢", "‍<😀🏽\r\n", "'S", "\t", ".", " ", "\ti", "OSAb", " \n", "s's", "٣٤٥", "٦", "'ll", "/\r\n", "ع", ".ḍ̇'Re", " ", "9", "a", "/b", "Ab", "ABC'S"]} +{"text": "t!'Refis\t'ſ#$%\r\na Dž㍿0\r\n\r\nİ12345678ABCß字㋿HTTPServer𐞁Dž!!(", "tokens": 40, "pieces": ["t", "!'", "Refis", "\t", "'ſ", "#$%\r\n", "a", " ", " Dž", "㍿", "0", "\r\n\r\n", "İ", "123", "456", "78", "ABCß字", "㋿HTTPServer𐞁", "Dž", "!!("]} +{"text": "٣٤٥٦३", "tokens": 5, "pieces": ["٣٤٥", "٦३"]} +{"text": "ß Ⅳ\n/'VE\r\nZ-#$%㋿🙂 \n ㍿iOS \n 'T㋿ع'ReAb!́é", "tokens": 38, "pieces": ["ß", " ", " ", "Ⅳ", "\n", "/'", "VE", "\r\n", "Z", "-#$%㋿🙂", " \n", " ", " ㍿", "i", "OS", " \n", " '", "T", "㋿ع'Re", "Ab", "!́é"]} +{"text": "'ſ-½\n \niOSé　　>0/\r\n /'sa'Re'\t\"!! B­عDž \n9\té \n", "tokens": 33, "pieces": ["'ſ", "-", "½", "\n \n", "i", "OSé", "　", "　", ">", "0", "/\r\n", " ", " /'", "sa'Re", "'", "\t", "\"!!", " B", "­ع", "Dž", " \n", "9", "\té", " \n"]} +{"text": "0'TåAb", "tokens": 5, "pieces": ["0", "'Tå", "Ab"]} +{"text": "漢é\r­Ab'ReᵃiOS!!ᵃſ<|fim_prefix|>𐞁 éß
\r", "­Ab'Re", "ᵃi", "OS", "!!", "ᵃſ", "<|", "fim", "_prefix", "|>", "𐞁", " ", " éß", "
", ".\r0'VE㍿'T­<|endoftext|>/!!", "tokens": 43, "pieces": [" '", "S", "!!", " ", " '", "M", ",𐞁ß", "
", "'VEＡi", "OS", "‍", "0", "𐞁", ">.\r", "0", "'VE", "㍿'", "T", "­<|", "endoftext", "|>/!!"]} +{"text": "HTTPServer!!a/bDž😀🏽DžⅣ<\tét\n/ \n <|fim_prefix|>AbaBé٣٤٥٦iOS'M𐞁३9! \n /
s", "tokens": 48, "pieces": ["HTTPServer", "!!", "a", "/b", "Dž", "😀🏽", "Dž", "Ⅳ", "<", "\tét", "\n", "/", " \n", " <|", "fim", "_prefix", "|>", "Aba", "Bé", "٣٤٥", "٦", "i", "OS'M", "𐞁", "३9", "!", " \n", " /", "
s"]} +{"text": "HTTPServerＡ>'reḍ̇　\"'S\r\n\r\n漢aBＡ< \n B/​­iOS­漢-a/baB​㍿㋿'ſ'S'VEß", "tokens": 46, "pieces": ["HTTPServer", "Ａ", ">'", "reḍ̇", "　", "\"'", "S", "\r\n\r\n", "漢a", "BＡ", "<", " \n", " <", "META", "_START", ">B", "/​­", "i", "OS", "­漢", "-a", "/ba", "B", "​㍿㋿'", "ſ'S", "'VEß"]} +{"text": "‍9ꟲ \ńᵃfiİ㍿ \nfi'VEå​Ａᵃ0漢camelCase'reſAbEOT", "tokens": 35, "pieces": ["‍", "9", "ꟲ", " \n", "́ᵃfi", "İ", "㍿", " \n", "fi'VE", "å", "​Ａᵃ", "0", "漢camel", "Case're", "ſ", "Ab", "EOT"]} +{"text": "a/b'Tꟲ/\r\nABC0字", "tokens": 14, "pieces": ["a", "/b'T", "ꟲ", "/\r\n", "ABC", "", "0", "字"]} +{"text": "'VE'VEᵃ𐞁'ſcamelCase‍<|endoftext|>så½İ", "tokens": 30, "pieces": ["'VE'VE", "ᵃ𐞁'ſ", "camel", "Case", "‍<|", "endoftext", "|>", "s", "å", "½", "İ"]} +{"text": "Dž'ſ!!ABC'lladé\r\n\r\n㍿Ab're", "tokens": 15, "pieces": ["Dž'ſ", "!!", "ABC'll", "adé", "\r\n\r\n", "㍿Ab're"]} +{"text": "ḍ̇", "tokens": 3, "pieces": ["ḍ̇"]} +{"text": "HTTPServers\nꟲ🙂>a㍿", "tokens": 15, "pieces": ["HTTPServers", "\n", "ꟲ", "🙂>", "a", "㍿"]} +{"text": "😀🏽Ab/\r\n!ß㍿ 'ſ \n/.", " '", "ſ", " \n", "/.<", "m", " HTTPServer", "😀🏽", " "]} +{"text": "'ſ'llaBᵃ\n/", "tokens": 10, "pieces": ["'ſ'll", "a", "Bᵃ", "\n", "/"]} +{"text": "½ \n ​-0字å \n.'ſ٣٤٥٦Dž9 \n'ſ'Rea😀🏽 \t…t漢#$%Ab🙂ḍ̇", "tokens": 39, "pieces": ["½", " \n", " ​-", "0", "字å", " \n", ".'", "ſ", "٣٤٥", "٦", "Dž", "9", " \n", "'ſ'Re", "a", "😀🏽", " \t", "…t漢", "#$%", "Ab", "🙂ḍ̇"]} +{"text": "EOTⅣ9", "tokens": 8, "pieces": ["EOT", "Ⅳ9"]} +{"text": "EOT,\r<|endoftext|>😀🏽३!Bİ‍ᵃiOSعAb　ᵃ'ſ/<|endoftext|>　ḍ̇İ'ſᵃ'M/ꟲ😀🏽ع\r'red!<|endoftext|>㍿aB", "tokens": 77, "pieces": ["EOT", ",\r", "<|", "endoftext", "|>😀🏽", "३", "!Bİ", "‍ᵃi", "OSعAb", "　ᵃ'ſ", "/<|", "endoftext", "|>", "　ḍ̇", "İ'ſ", "ᵃ'M", "/ꟲ", "😀🏽", "ع", "\r", "'red", "!<|", "endoftext", "|><", "EOT", ">㍿", "a", "B"]} +{"text": " éſd,<|endoftext|>å12345678Džt!!\nعa😀🏽EOT­\"'s½…'Ⅳ­'D \n ,'ſ\n-fi0", "tokens": 49, "pieces": [" éſd", ",<|", "endoftext", "|>", "å", "123", "456", "78", "Džt", "!!\n", "عa", "😀🏽", "EOT", "­\"'", "s", "½", "…", "'", "Ⅳ", "­'", "D", " \n", " ,'", "ſ", "\n", "-fi", "0"]} +{"text": "<|fim_prefix|>", "tokens": 6, "pieces": ["<|", "fim", "_prefix", "|>"]} +{"text": "/mEOT​ ABC", "tokens": 9, "pieces": ["/m", "EOT", "​", " ", " ABC"]} +{"text": "\r\n\r\nfi ½a/b㋿\" \u000bZt's漢३é", "tokens": 20, "pieces": ["\r\n\r\n", "fi", " ", " ", "½", "a", "/b", "㋿\"", " ", "\u000bZt's", "漢", "३", "é"]} +{"text": "t…٣٤٥٦Z'Reİ", "tokens": 10, "pieces": ["t", "…", "٣٤٥", "٦", "Z'Re", "İ"]} +{"text": " 'Re㍿😀🏽\" \n㋿iOS(12345678İＡ½,‍🙂‍'S's.　…a/b-'sİ­<|fim_prefix|>İＡ", "tokens": 48, "pieces": [" ", " '", "Re", "㍿😀🏽\"", " \n", "㋿i", "OS", "(", "123", "456", "78", "İＡ", "½", ",‍🙂‍'", "S's", ".", "　", "…a", "/b", "-'", "s", "İ", "­<|", "fim", "_prefix", "|>", "İＡ"]} +{"text": "a\r\n\r\nＡ​<|endoftext|>'s's'T३'M.sＡABCDž👍🏽Džungla\r\n𐞁Džunglaع\u000bEOT​'ll\r
HTTPServer🙂½", "tokens": 50, "pieces": ["a", "\r\n\r\n", "Ａ", "​<|", "endoftext", "|>'", "s's", "'T", "३", "'M", ".s", "ＡABCDž", "👍🏽", "Džungla", "\r\n", "𐞁Džunglaع", "\u000bEOT", "​'", "ll", "\r", "
HTTPServer", "🙂", "½"]} +{"text": "Ⅳ٣٤٥٦'VE0#$%🙂9-#$%é!EOT𐞁ḍ̇e　fiAb", "tokens": 31, "pieces": ["Ⅳ٣٤", "٥٦", "'VE", "0", "#$%🙂", "9", "-#$%", "é", "!EOT𐞁ḍ̇e", "　fi", "Ab"]} +{"text": "👍🏽字/\r\nBDž \n#$%\"eßß½\"", "tokens": 16, "pieces": ["👍🏽", "字", "/\r\n", "BDž", " \n", "#$%\"", "eßß", "½", "\""]} +{"text": "'T-'Re𐞁", "tokens": 7, "pieces": ["'T", "-'", "Re𐞁"]} +{"text": "!!\r\n'Ret \n\r\ns#$%Ⅳ", "tokens": 14, "pieces": ["!!\r\n", "'Ret", " \n\r\n", "s", "#$%", "Ⅳ", ""]} +{"text": "!!ſ𐞁👍🏽ABC\nḍ̇a/b<", "tokens": 17, "pieces": ["!!", "ſ𐞁", "👍🏽", "ABC", "\n", "ḍ̇a", "/b", "<"]} +{"text": "å　DžunglaEOT!!ḍ̇-Ab>.'sdaB'reḍ̇\"d\r\n\r\n #$%Ａ\n/", "tokens": 31, "pieces": ["å", "　Džungla", "EOT", "!!", "ḍ̇", "-Ab", ">.'", "sda", "B're", "ḍ̇", "\"d", "\r\n\r\n", " #$%", "Ａ", "\n", "/"]} +{"text": "'M'ſABC'​👍🏽 ", "tokens": 10, "pieces": ["'M'ſ", "ABC", "'​👍🏽", " "]} +{"text": "\n'D'ſ/ \n'M­dḍ̇>㋿<|fim_prefix|>ᵃ漢\" \n\r\n0​ſſ​-", "tokens": 34, "pieces": ["\n", "'D'ſ", "/", " \n", "'M", "­dḍ̇", ">㋿<|", "fim", "_prefix", "|>", "ᵃ漢", "\"", " \n\r\n", "0", "​ſſ", "​-"]} +{"text": "Z'ſ'عᵃfi", "tokens": 9, "pieces": ["Z'ſ", "'عᵃfi"]} +{"text": "ſ!'ſ/\r\n \n𐞁eaB👍🏽́ da", "tokens": 17, "pieces": ["ſ", "!'", "ſ", "/\r\n", " \n", "𐞁ea", "B", "👍🏽́", " da"]} +{"text": "Ａ漢 12345678ḍ̇HTTPServer㍿\n/'Re\r\n\r\n‍B9ßaB'S!!👍🏽'M!!'llＡꟲ­👍🏽Dž9m½ع.字m\u000bＡ", "tokens": 53, "pieces": ["Ａ漢", " ", "123", "456", "78", "ḍ̇", "HTTPServer", "㍿\n/", "'Re", "\r\n\r\n", "‍B", "9", "ßa", "B'S", "!!👍🏽'", "M", "!!'", "ll", "Ａꟲ", "­👍🏽", "Dž", "9", "m", "½", "ع", ".字m", "\u000bＡ"]} +{"text": "camelCasee \n ٣٤٥٦\u000b　ABC'T", "tokens": 13, "pieces": ["camel", "Casee", " \n", " ", "٣٤٥", "٦", "\u000b", "　ABC'T"]} +{"text": "'s'Må\u000b ́Ab'T(‍", "tokens": 11, "pieces": ["'s'M", "å", "\u000b", " ́Ab'T", "(‍"]} +{"text": " \n\n/#$%ḍ̇'Re'Ta/b㋿>ḍ̇ \u000bع'll'ſ
 \t!३éＡ \n camelCasea/bå'ſB0B…< \n ,,a", "tokens": 56, "pieces": [" \n\n", "/#$%", "ḍ̇'Re", "'", "Ta", "/b", "㋿>", "ḍ̇", " ", "\u000bع'll", "'ſ", "
 ", "\t", "!", "३", "é", "Ａ", " \n", " camel", "Casea", "/bå'ſ", "B", "0", "B", "…", "<", " \n", " ,,", "a"]} +{"text": " \n …<|fim_prefix|>\"𐞁!㍿'ss", "tokens": 20, "pieces": [" \n", " ", "…", "<|", "fim", "_prefix", "|>\"", "𐞁", "!㍿'", "ss"]} +{"text": "ᵃ😀🏽 é,…fi", "tokens": 13, "pieces": ["ᵃ", "😀🏽", " é", ",", "…fi"]} +{"text": "é're-\r\n", "tokens": 3, "pieces": ["é're", "-\r\n"]} +{"text": "12345678e½'M\rABC's㍿<\"𐞁éſ㋿👍🏽Ab　iOS'T\rⅣ('ll'llABCABCBiOS­>Ⅳ", "tokens": 46, "pieces": ["123", "456", "78", "e", "½", "'M", "\r", "ABC's", "㍿<\"", "𐞁éſ", "㋿👍🏽", "Ab", "　i", "OS'T", "\r", "Ⅳ", "('", "ll'll", "ABCABCBi", "OS", "­>", "Ⅳ"]} +{"text": "\r<٣٤٥٦\"İ́", "tokens": 9, "pieces": ["\r", "<", "٣٤٥", "٦", "\"İ́"]} +{"text": "<|fim_prefix|>.aBＡ𐞁é‍mAbABCåé'reꟲaaBåDž字‍9!é'llＡ!\rDžᵃ'Ret", "tokens": 53, "pieces": ["<|", "fim", "_prefix", "|>.", "a", "BＡ𐞁é", "‍m", "Ab", "ABCå", "é're", "ꟲaa", "Bå", "Dž字", "‍", "9", "!é'll", "Ａ", "!\r", "Džᵃ'Re", "t"]} +{"text": "fi0\n/🙂camelCase<ᵃ𐞁", "tokens": 21, "pieces": ["fi", "0", "\n", "/<", "META", "_START", ">🙂", "camel", "Case", "<ᵃ𐞁", ""]} +{"text": "​ß\r\nå漢å\n/>< \n 'T३EOT'٣٤٥٦ABC漢t9s !!३", "tokens": 33, "pieces": ["​ß", "\r\n", "å漢å", "\n/", "><", " \n", " '", "T", "३", "EOT", "'", "٣٤٥", "٦", "ABC漢t", "9", "s", " ", "!!", "३"]} +{"text": "(d㍿　e.'s‍fiåİ\nB/\r\n \n­　(ß", "tokens": 24, "pieces": ["(d", "㍿", "　e", ".'", "s", "‍fi", "å", "İ", "\n", "B", "/\r\n", " \n", "­", "　", "(ß"]} +{"text": "'VE\r\n!!🙂\r\nⅣB​Ab12345678#$%'TdDžſABC", "tokens": 21, "pieces": ["'VE", "\r\n", "!!🙂\r\n", "Ⅳ", "B", "​Ab", "123", "456", "78", "#$%'", "Td", "Džſ", "ABC"]} +{"text": "𐞁're'sDžungladᵃ9\r­t-…/𐞁ḍ̇<\n(𐞁aB字, \nå\u000b<|fim_prefix|><|endoftext|>'​‍ꟲ", "tokens": 61, "pieces": ["𐞁're", "'s", "Džungladᵃ", "9", "\r", "­t", "-", "…", "/𐞁ḍ̇", "<\n", "(<", "META", "_START", ">𐞁a", "B字", ",", " \n", "å", "\u000b", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'​‍", "ꟲ"]} +{"text": "İ'VE", "tokens": 6, "pieces": ["İ", "'", "VE"]} +{"text": "Ab …ꟲ٣٤٥٦­HTTPServer…<|fim_prefix|>ſ", "tokens": 23, "pieces": ["Ab", " ", "…ꟲ", "٣٤٥", "٦", "­HTTPServer", "…", "<|", "fim", "_prefix", "|>", "ſ"]} +{"text": ">å\u000bＡABC,HTTPServerǻ-(½EOTZ'D🙂Džungla字ᵃİ👍🏽\t\n३🙂👍🏽m 'll'S\u000bſ/\r\ncamelCase
", "tokens": 50, "pieces": [">å", "\u000bＡABC", ",HTTPServerǻ", "-(", "½", "EOTZ'D", "🙂Džungla字ᵃ", "İ", "👍🏽", "\t\n", "३", "🙂👍🏽", "m", " ", "'ll'S", "\u000bſ", "/\r\n", "camel", "Case", "
"]} +{"text": " 'T\"'T<|endoftext|>9Dž/\r\nABC😀🏽éDž字>!!!HTTPServer ㍿\r\n\r\nHTTPServer٣٤٥٦'Re12345678Džungla½", "tokens": 53, "pieces": [" '", "T", "\"'", "T", "<|", "endoftext", "|>", "9", "Dž", "/\r\n", "ABC", "😀🏽", "é", "Dž字", ">!!!", "HTTPServer", " ", " ㍿\r\n\r\n", "HTTPServer", "٣٤٥", "٦", "'Re", "123", "456", "78", "Džungla", "½"]} +{"text": "<|endoftext|>عAbⅣ\r\n\r\nméßⅣ\n/ABC𐞁ßſcamelCase'M𐞁(ficamelCaseᵃiOSB\r\n\r\nſ½é'Saa", "tokens": 51, "pieces": ["<|", "endoftext", "|>", "عAb", "Ⅳ", "\r\n\r\n", "méß", "Ⅳ", "\n", "/ABC𐞁ßſ", "camel", "Case'M", "𐞁", "(ficamel", "Caseᵃi", "OSB", "\r\n\r\n", "ſ", "½", "é'S", "aa"]} +{"text": "a\r🙂'ree字<|endoftext|>aB­./漢", "tokens": 20, "pieces": ["a", "\r", "🙂'", "ree字", "<|", "endoftext", "|><", "EOT", ">a", "B", "­./", "漢"]} +{"text": "Ａ٣٤٥٦'DZ'VE㋿mB/\r\n, \n👍🏽Z㍿ßfi'T३\n/DžunglaDž('M…\n/9…\r", "tokens": 46, "pieces": ["Ａ", "٣٤٥", "٦", "'DZ'VE", "㋿m", "B", "/\r\n", ",", " \n", "👍🏽", "Z", "㍿ßfi'T", "३", "\n", "/Džungla", "Dž", "('", "M", "…\n", "/", "9", "…\r"]} +{"text": "'Se½fi٣٤٥٦३İ😀🏽 ,-́fia\n!<|fim_prefix|>fiAb!/Z.écamelCaseB½'D㋿HTTPServera'VEå漢", "tokens": 47, "pieces": ["'Se", "½", "fi", "٣٤٥", "٦३", "İ", "😀🏽", " ,-́", "fia", "\n", "!<|", "fim", "_prefix", "|>", "fi", "Ab", "!/", "Z", ".écamel", "Case", "B", "½", "'D", "㋿HTTPServera'VE", "å漢"]} +{"text": "e👍🏽'­
\rsḍ̇ \r\n\r\n/\r\n/DžAb<|endoftext|>\r\nعⅣ
iOSfi\r\n👍🏽d३ Dž\r9tEOT½e#$%a/bDžungla", "tokens": 56, "pieces": ["e", "👍🏽'­", "
\r", "sḍ̇", " \r\n\r\n", "/\r\n/", "DžAb", "<|", "endoftext", "|>\r\n", "ع", "Ⅳ", "
i", "OSfi", "\r\n", "👍🏽", "d", "३", " Dž", "\r", "9", "t", "EOT", "½", "e", "#$%", "a", "/b", "Džungla"]} +{"text": "́​'Re'D", "tokens": 5, "pieces": ["́", "​'", "Re'D"]} +{"text": " \n …d㍿
<ꟲé.ع're're", "tokens": 19, "pieces": [" \n", " ", "…d", "㍿", "
", "<ꟲé", ".ع're", "'re"]} +{"text": "å'DABC's'D𐞁<|endoftext|>́😀🏽,​\r\n\r\n<|endoftext|>ḍ̇e​­👍🏽aBéعaB'Reꟲ㍿", "tokens": 60, "pieces": ["å'D", "ABC's", "'D𐞁", "<|", "endoftext", "|>́😀🏽,​\r\n\r\n", "<|", "endoftext", "|>", "ḍ̇e", "​­<", "META", "_START", ">👍🏽", "a", "Béعa", "B'Re", "ꟲ", "㍿"]} +{"text": "9 ३́/\r\n\t", "tokens": 6, "pieces": ["9", " ", "३", "́", "/\r\n", "\t"]} +{"text": "'re'字!é'ſaB漢٣٤٥٦'Re٣٤٥٦é", "tokens": 21, "pieces": ["'re", "'字", "!é'ſ", "a", "B漢", "٣٤٥", "٦", "'Re", "٣٤٥", "٦", "é"]} +{"text": "́a/b\u000b\r\n\r\n!!é 12345678'll\t,Ab'Sſſm-.\n/a/bAbⅣ<|fim_prefix|>/\r\n(>mcamelCase#$%a/b\t'HTTPServer>漢\t", "tokens": 50, "pieces": ["́a", "/b", "\u000b\r\n\r\n", "!!", "é", " ", "123", "456", "78", "'ll", "\t", ",Ab'S", "ſſm", "-.\n/", "a", "/b", "Ab", "Ⅳ", "<|", "fim", "_prefix", "|>/\r\n", "(>", "mcamel", "Case", "#$%", "a", "/b", "\t", "'HTTPServer", ">漢", "\t"]} +{"text": "عdＡBs'dcamelCase/\"DžunglaaBt𐞁>'ſ12345678aBZ.#$%éꟲ½\u000bs<", "tokens": 46, "pieces": ["عd", "ＡBs'd", "camel", "Case", "/\"", "Džunglaa", "Bt𐞁", ">'", "ſ", "123", "456", "78", "a", "BZ", ".<", "META", "_START", ">#$%", "éꟲ", "½", "\u000bs", "<<", "EOT", ">"]} +{"text": "9t<|fim_prefix|>́\n/-12345678dꟲ<|endoftext|>ſa'M½٣٤٥٦ \n \n३d>d", "tokens": 38, "pieces": ["9", "t", "<|", "fim", "_prefix", "|>́\n/", "-", "123", "456", "78", "dꟲ", "<|", "endoftext", "|>", "ſa'M", "½٣٤", "٥٦", " \n \n", "३", "d", ">d"]} +{"text": "\ta/bİ٣٤٥٦<|fim_prefix|>é漢>'Sſ<\t'D\u000bAb<|endoftext|>tḍ̇'MéDž \nḍ̇漢ᵃDžunglá", "tokens": 51, "pieces": ["\ta", "/b", "İ", "٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "é漢", ">'", "Sſ", "<", "\t", "'D", "\u000bAb", "<|", "endoftext", "|>", "tḍ̇'M", "é", "Dž", " \n", "ḍ̇漢ᵃ", "Džunglá"]} +{"text": "ſ'T/!aBaEOT३aB'Re/\"a/bſ", "tokens": 16, "pieces": ["ſ'T", "/!", "a", "Ba", "EOT", "३", "a", "B'Re", "/\"", "a", "/bſ"]} +{"text": "'T.!Z
t漢ᵃ\r\n\r\n9👍🏽/\r\na/bع\réſZZ'Re,", "tokens": 24, "pieces": ["'T", ".!", "Z", "
t漢ᵃ", "\r\n\r\n", "9", "👍🏽/\r\n", "a", "/bع", "\r", "éſ", "ZZ'Re", ","]} +{"text": "ABCEOT>/\r\n'D're", "tokens": 9, "pieces": ["ABC", "EOT", ">/\r\n", "'D're"]} +{"text": "dſ9HTTPServerEOTꟲ/", "tokens": 13, "pieces": ["dſ", "9", "HTTPServer", "EOTꟲ", "/"]} +{"text": "00ådDžungla'👍🏽…Dž're<\u000bZiOS𐞁'ſ'VEé9s'DDž Ａ<|fim_prefix|>'T𐞁\n/ß s'M('ſ", "tokens": 59, "pieces": ["00", "åd", "Džungla", "'👍🏽", "…Dž're", "<", "\u000bZi", "OS𐞁'ſ", "'VEé", "9", "s", "'", "DDž", " ", " Ａ", "<|", "fim", "_prefix", "|>'", "T𐞁", "\n", "/ß", " s'M", "('", "ſ"]} +{"text": "　ḍ̇", "tokens": 4, "pieces": ["　ḍ̇"]} +{"text": "'s㍿　0aB\tHTTPServerEOTfi🙂 dé́ 'D!!\r\n\r\nt'M\r\nåDž.\n/HTTPServer", "tokens": 33, "pieces": ["'s", "㍿", "　", "0", "a", "B", "\tHTTPServer", "EOTfi", "🙂", " ", " dé́", " ", " '", "D", "!!\r\n\r\n", "t'M", "\r\n", "å", "Dž", ".\n/", "HTTPServer"]} +{"text": ">'S\n/Ａ'StEOT㍿३(å👍🏽\r/\r\n'll'M'/\r\n\r\n\r\n\r\n(.'D​ 　😀🏽ᵃ🙂!!Ａſ \nB　", "tokens": 51, "pieces": [">'", "S", "\n", "/Ａ'S", "t", "EOT", "㍿", "३", "(å", "👍🏽\r/\r\n", "'ll'M", "'/\r\n\r\n\r\n\r\n", "(.'", "D", "​", " ", " <", "EOT", ">", "　", "😀🏽", "ᵃ", "🙂<", "META", "_START", ">!!", "Ａſ", " \n", "B", "　"]} +{"text": "­é", "tokens": 2, "pieces": ["­é"]} +{"text": "<|fim_prefix|>\t𐞁字fi
", "tokens": 14, "pieces": ["<|", "fim", "_prefix", "|>", "\t𐞁字fi", "
"]} +{"text": ",ᵃ字aBa/bZABC字'漢­👍🏽字0,字 <|fim_prefix|>eABC㍿aB
", "tokens": 36, "pieces": [",ᵃ字a", "Ba", "/b", "ZABC字", "'漢", "­👍🏽", "字", "0", ",字", " ", "<|", "fim", "_prefix", "|>", "e", "ABC", "㍿a", "B", "
"]} +{"text": "㋿३'S \n'siOS\n/٣٤٥٦camelCase'ſ­Ab'SＡaBsé", "tokens": 26, "pieces": ["㋿", "३", "'S", " \n", "'si", "OS", "\n", "/", "٣٤٥", "٦", "camel", "Case'ſ", "­Ab'S", "Ａa", "Bsé"]} +{"text": "漢!>'scamelCasé", "tokens": 7, "pieces": ["漢", "!>'", "scamel", "Casé"]} +{"text": "-9Džunglaåm/e'M-Ab\u000b \nＡᵃ\u000b", "tokens": 35, "pieces": ["-", "9", "Džunglaåm", "/e'M", "-Ab", "", "\u000b \n", "Ａᵃ", "\u000b"]} +{"text": " (camelCase　-#$%'Ta99!é#$%\n/'Dé㋿Džungla
\r\n\r\nᵃ<|fim_prefix|>aB㍿-　　\r\n/​\n\r\n\r\neABCa/b
", "tokens": 55, "pieces": [" (", "camel", "Case", "　", "-#$%'", "Ta", "99", "!é", "#$%\n/", "'Dé", "㋿Džungla", "
\r\n\r\n", "ᵃ", "<|", "fim", "_prefix", "|>", "a", "B", "㍿-", "　　\r\n", "/<", "META", "_START", ">​\n\r\n\r\n", "e", "ABCa", "/b", "
"]} +{"text": "​é😀🏽ḍ̇Dž.\r\n\r\n's½\u000b'T́", "tokens": 16, "pieces": ["​é", "😀🏽", "ḍ̇", "Dž", ".\r\n\r\n", "'s", "½", "\u000b", "'T́"]} +{"text": ">å(
<㋿'s٣٤٥٦-(a/b#$%İ'll#$%'T'S\"", "tokens": 26, "pieces": [">å", "(", "
", "<㋿'", "s", "٣٤٥", "٦", "-(", "a", "/b", "#$%", "İ'll", "#$%'", "T'S", "\""]} +{"text": "\r\nḍ̇\r\n12345678 \n (Ⅳd<|endoftext|>ABC'll…éDžZ'll‍­'T\u000b0­ \n ٣٤٥٦(éＡEOTeå!́𐞁ḍ̇…m<|endoftext|>", "tokens": 68, "pieces": ["\r\n", "ḍ̇", "\r\n", "123", "456", "78", " \n", " (", "Ⅳ", "d", "<|", "endoftext", "|>", "ABC'll", "…é", "DžZ'll", "‍­'", "T", "\u000b", "0", "­", " \n", " ", "٣٤٥", "٦", "(é", "ＡEOTeå", "!́𐞁ḍ̇", "…m", "<|", "endoftext", "|>"]} +{"text": "é 'ſ㋿Z'VE<|fim_prefix|>\n/12345678'Re
s<㋿eᵃ\t漢\r\n\r\n漢ᵃ\r\n12345678ᵃ­åsZⅣ", "tokens": 52, "pieces": ["é", " ", " '", "ſ", "㋿Z'VE", "<|", "fim", "_prefix", "|>\n/", "123", "456", "78", "'Re", "
s", "<㋿", "eᵃ", "\t漢", "\r\n\r\n", "漢ᵃ", "\r\n", "123", "456", "78", "ᵃ", "­ås", "Z", "Ⅳ"]} +{"text": "𐞁𐞁½'s", "tokens": 10, "pieces": ["𐞁𐞁", "½", "'s"]} +{"text": "/\r\n ", "tokens": 2, "pieces": ["/\r\n", " "]} +{"text": "漢>0're", "tokens": 4, "pieces": ["漢", ">", "0", "'re"]} +{"text": "'SEOT\n/ᵃßEOTa/bZ㍿ \nع,عHTTPServer‍  ‍m'DＡ'Reعع", "tokens": 33, "pieces": ["'SEOT", "\n", "/ᵃß", "EOTa", "/b", "Z", "㍿", " \n", "ع", ",عHTTPServer", "‍", " ", " ", "‍m'D", "Ａ'Re", "عع"]} +{"text": "!ßⅣḍ̇ꟲ'ſ㍿", "tokens": 15, "pieces": ["!ß", "Ⅳ", "ḍ̇ꟲ'ſ", "㍿"]} +{"text": "ꟲ12345678\rſ<|fim_prefix|>٣٤٥٦​Džunglaé𐞁Dž#$%012345678\r\n\r\n٣٤٥٦!Džungla😀🏽!é\n/½12345678'D ㍿\r\nd\r\n\r\nİ
Ab-'😀🏽fi", "tokens": 78, "pieces": ["ꟲ", "123", "456", "78", "\r", "ſ", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "​Džunglaé𐞁", "Dž", "#$%", "012", "345", "678", "\r\n\r\n", "٣٤٥", "٦", "!Džungla", "😀🏽!", "é", "\n", "/", "½12", "345", "678", "'D", " ", "㍿\r\n", "d", "\r\n\r\n", "İ", "", "
Ab", "-'😀🏽", "fi"]} +{"text": "𐞁'Re\n/Ab𐞁ABC३ß㍿ \n३ABC'ſ字字m're 'ReDžungla　s👍🏽ḍ̇DžunglasDž9<|endoftext|>\t\n/३\r\n(𐞁ḍ̇'M'VEABC", "tokens": 71, "pieces": ["𐞁'Re", "\n", "/Ab𐞁", "ABC", "३", "ß", "㍿", " \n", "३", "ABC'ſ", "字字m're", " ", "'Re", "Džungla", "　s", "👍🏽", "ḍ̇", "Džunglas", "Dž", "9", "<|", "endoftext", "|>", "\t\n", "/", "३", "\r\n", "(𐞁ḍ̇'M", "'VEABC"]} +{"text": "iOS(ABCfiDžunglae🙂EOT­Džungla㍿#$%12345678camelCase'ſ#$%#$%'MaB٣٤٥٦'Reé'llſſ", "tokens": 45, "pieces": ["i", "OS", "(ABCfi", "Džunglae", "🙂EOT", "­Džungla", "㍿#$%", "123", "456", "78", "camel", "Case'ſ", "#$%#$%'", "Ma", "B", "٣٤٥", "٦", "'Reé'll", "ſſ"]} +{"text": "٣٤٥٦㋿a\r\naa('llcamelCaseDžunglaDž\n/a ZiOS🙂३ABCa/b> \n0'ſ\n/-#$%𐞁'll\r\n\u000b'D>㍿\r\n\r\n", "tokens": 59, "pieces": ["٣٤٥", "٦", "㋿a", "\r\n", "aa", "('", "llcamel", "Case", "Džungla", "Dž", "\n", "/a", " ", " <", "META", "_START", ">Zi", "OS", "🙂", "३", "ABCa", "/b", ">", " \n", "0", "'ſ", "\n", "/-#$%", "𐞁'll", "\r\n", "\u000b", "'D", ">㍿\r\n\r\n"]} +{"text": "😀🏽Džꟲ0Dž'rea👍🏽 \nꟲ'D́Ａ­Z 9Ａs,#$% DžHTTPServer.\"Ⅳ", "tokens": 41, "pieces": ["😀🏽", "Džꟲ", "0", "Dž're", "a", "👍🏽", " \n", "ꟲ'D", "́", "Ａ", "­Z", " ", " ", "9", "Ａs", ",#$%", " DžHTTPServer", ".\"", "Ⅳ"]} +{"text": "३ſ<|endoftext|>ⅣBB! \n\t'Dm…🙂-/\r\nⅣaB 're👍🏽camelCase😀🏽ḍ̇aBéßEOT👍🏽😀🏽½#$%'sع'Då", "tokens": 63, "pieces": ["३", "ſ", "<|", "endoftext", "|>", "Ⅳ", "BB", "!", " \n", "\t", "'Dm", "…", "🙂-/\r\n", "Ⅳ", "a", "B", " '", "re", "👍🏽", "camel", "Case", "😀🏽", "ḍ̇a", "Béß", "EOT", "👍🏽😀🏽<", "EOT", ">", "½", "#$%'", "sع'D", "å"]} +{"text": "9< \n!m12345678a\r\n٣٤٥٦iOS \n
عé'ſ­,ꟲcamelCaseＡ३.\"'Re!\u000b,ᵃ٣٤٥٦'ſ'M\t", "tokens": 52, "pieces": ["9", "<", " \n", "!m", "123", "456", "78", "a", "\r\n", "٣٤٥", "٦", "i", "OS", "", " \n", "
عé'ſ", "­,", "ꟲcamel", "Case", "Ａ", "३", ".\"'", "Re", "!", "\u000b", ",ᵃ", "٣٤٥", "٦", "'ſ'M", "\t"]} +{"text": "m\n…½\r३　Džungla!/ ع,<> 'S<|fim_prefix|>aB'Ma/bsaBİ.iOS", "tokens": 35, "pieces": ["m", "\n", "…", "½", "\r", "३", "　Džungla", "!/", " ع", ",<>", " '", "S", "<|", "fim", "_prefix", "|>", "a", "B'M", "a", "/bsa", "Bİ", ".i", "OS"]} +{"text": ">>å'sZ(", "tokens": 6, "pieces": [">>", "å's", "Z", "("]} +{"text": "0㋿🙂!!- \naBa/b \n/,
9ꟲ-a<|fim_prefix|>Dž　9fiＡiOS…'VE'D<|fim_prefix|>'T12345678 \n \n é12345678t\r ", "tokens": 57, "pieces": ["0", "㋿🙂!!-", " \n", "a", "Ba", "/b", " \n", "/,", "
", "9", "ꟲ", "-a", "<|", "fim", "_prefix", "|>", "Dž", "　", "9", "fi", "Ａi", "OS", "…", "'VE'D", "<|", "fim", "_prefix", "|>'", "T", "123", "456", "78", " \n \n", " é", "123", "456", "78", "t", "\r", " "]} +{"text": "'S \n عd'VE३(0-㍿", "tokens": 13, "pieces": ["'S", " \n", " عd'VE", "३", "(", "0", "-㍿"]} +{"text": "('Re's👍🏽12345678'VE‍EOTع.12345678!!漢 \n'ſＡ漢,a/b\r\n\r\n 
…'S\"Ａ\r\n…ᵃsEOTＡ'", "tokens": 51, "pieces": ["('", "Re's", "👍🏽", "123", "456", "78", "'VE", "‍EOTع", ".", "123", "456", "78", "!!", "漢", " \n", "'ſ", "Ａ漢", ",a", "/b", "\r\n\r\n", " ", "
", "", "…", "'S", "\"Ａ", "\r\n", "…ᵃs", "EOTＡ", "'"]} +{"text": "­ e­'s\r\n\r\n", "tokens": 7, "pieces": ["­", " ", " e", "­'", "s", "\r\n\r\n"]} +{"text": "!\r\n\r\nåßDžunglaEOT漢HTTPServer漢camelCase<|endoftext|>\t \n ABC\r\n🙂𐞁\u000b<|fim_prefix|>\"'T'Re'sta　'DaBBé(‍\t👍🏽", "tokens": 62, "pieces": ["!\r\n\r\n", "åß", "Džungla", "EOT漢HTTPServer漢camel", "Case", "<|", "endoftext", "|>", "\t \n", " ABC", "\r\n", "🙂𐞁", "\u000b", "<|", "fim", "_prefix", "|>\"'", "T'Re", "'sta", "　", "'", "Da", "BB", "é", "(‍", "\t", "👍🏽"]} +{"text": " \n \r\n\r\n", "tokens": 2, "pieces": [" \n \r\n\r\n"]} +{"text": "٣٤٥٦aB́B\n/'ſ'sa\n/👍🏽9é🙂e!!𐞁३HTTPServer,Ａ", "tokens": 37, "pieces": ["٣٤٥", "٦", "a", "B́", "B", "\n", "/'", "ſ's", "a", "\n", "/👍🏽", "9", "é", "🙂e", "!!", "𐞁", "३", "HTTPServer", ",Ａ"]} +{"text": "ع<|endoftext|>­iOS漢", "tokens": 12, "pieces": ["ع", "<|", "endoftext", "|>­", "i", "OS漢"]} +{"text": "'VE \n　'S
s\n/'T㍿.aB9camelCase12345678Ａ", "tokens": 23, "pieces": ["'VE", " \n", "　", "'S", "
s", "\n", "/'", "T", "㍿.", "a", "B", "9", "camel", "Case", "123", "456", "78", "Ａ"]} +{"text": "ᵃİ,㍿e9!!㍿\r!!", "tokens": 19, "pieces": ["ᵃ", "İ", ",㍿", "e", "9", "!!㍿\r", "!!"]} +{"text": "e<|endoftext|>t12345678ꟲé㍿fi'MＡ\na/b 'ABC(́å", "tokens": 33, "pieces": ["e", "<|", "endoftext", "|>", "t", "123", "456", "78", "ꟲé", "㍿fi'M", "Ａ", "\n", "a", "/b", " ", "'ABC", "(́å"]} +{"text": "عHTTPServer\n/", "tokens": 5, "pieces": ["عHTTPServer", "\n", "/"]} +{"text": "ⅣiOS㋿'sABC'M<|endoftext|>.iOS >sßa/bAb\rḍ̇३!!­٣٤٥٦'DZ<𐞁/\r\n(#$%,½<'re𐞁", "tokens": 58, "pieces": ["Ⅳ", "i", "OS", "㋿'", "s", "ABC'M", "<|", "endoftext", "|>.", "i", "OS", " >", "sßa", "/b", "Ab", "\r", "ḍ̇", "३", "!!­", "٣٤٥", "٦", "'DZ", "<𐞁", "/\r\n", "(#$%,", "½", "<'", "re𐞁"]} +{"text": "-字ḍ̇s-aABCDžaB12345678d<|endoftext|>HTTPServerＡḍ̇\r\nZ0\nB\"ᵃeé\"", "tokens": 41, "pieces": ["-字ḍ̇s", "-a", "ABCDža", "B", "123", "456", "78", "d", "<|", "endoftext", "|>", "HTTPServer", "Ａḍ̇", "\r\n", "Z", "0", "\n", "B", "\"ᵃeé", "\""]} +{"text": "édḍ̇ABC\u000bḍ̇ᵃ9<|fim_prefix|>\r㍿'reHTTPServerét\"İå🙂d ßaB㋿‍٣٤٥٦.aB'D
\r\n\r\n", "tokens": 54, "pieces": ["édḍ̇", "ABC", "\u000bḍ̇ᵃ", "9", "<|", "fim", "_prefix", "|>\r", "㍿'", "re", "HTTPServerét", "\"İå", "🙂d", " ", " ßa", "B", "㋿‍", "٣٤٥", "٦", ".a", "B'D", "
\r\n\r\n"]} +{"text": "🙂㋿Zꟲ½,‍", "tokens": 11, "pieces": ["🙂㋿", "Zꟲ", "½", ",‍"]} +{"text": " \n
३'ſꟲ", "tokens": 8, "pieces": [" \n", "
", "३", "'ſꟲ"]} +{"text": " 12345678Z\tİ㍿\n/'VE#$%…!.'Džunglas\r\n\r\n-s'Re‍'MHTTPServerfi<0عZ३㋿EOTＡ𐞁", "tokens": 58, "pieces": [" ", "123", "456", "78", "Z", "\tİ", "㍿\n/", "'VE", "#$%", "…", "!.'", "Džunglas", "<", "META", "_START", ">\r\n\r\n", "-s", "'", "Re", "‍'", "MHTTPServerfi", "<", "0", "ع", "Z", "३", "㋿EOTＡ𐞁"]} +{"text": "'reaBſ<|endoftext|>😀🏽0tiOSＡſHTTPServer", "tokens": 25, "pieces": ["'rea", "Bſ", "<|", "endoftext", "|>😀🏽", "0", "ti", "OS", "Ａſ", "HTTPServer"]} +{"text": "a/३camelCaseＡZ'VE", "tokens": 9, "pieces": ["a", "/", "३", "camel", "Case", "ＡZ'VE"]} +{"text": "🙂<|endoftext|>EOT", "tokens": 10, "pieces": ["🙂<|", "endoftext", "|>", "EOT"]} +{"text": "a're𐞁'llEOTＡd​Dž'T<|fim_prefix|>/\r\n … /\r\n'Abꟲ३e#$%'Rea/b-", "tokens": 44, "pieces": ["a're", "𐞁'll", "EOTＡd", "​Dž'T", "<|", "fim", "_prefix", "|>/\r\n", " ", "…", "", " ", "/\r\n", "'Abꟲ", "३", "e", "#$%'", "Rea", "/b", "-"]} +{"text": "İ\t🙂𐞁#$%dm‍m,Ab漢\r\n\t👍🏽 \n 'sm", "tokens": 23, "pieces": ["İ", "\t", "🙂𐞁", "#$%", "dm", "‍m", ",Ab漢", "\r\n", "\t", "👍🏽", " \n", " '", "sm"]} +{"text": "\" \n 're\t-'D\nd \n 'SᵃZ\r\n\r\nع'ſ \n😀🏽'Då\n/　'T''D", "tokens": 33, "pieces": ["\"", " \n", " '", "re", "\t", "-'", "D", "\n", "d", " \n", " '", "Sᵃ", "Z", "\r\n\r\n", "ع'ſ", " \n", "😀🏽'", "Då", "\n", "/", "　", "'T", "''", "D"]} +{"text": "!!'ſ…\n#$%…\tİ𐞁½( ⅣHTTPServer12345678iOSZ \r\n\r\nt!!Ⅳ
9\n're", "tokens": 39, "pieces": ["!!'", "ſ", "…\n", "#$%", "…", "\tİ𐞁", "½", "(", " ", "Ⅳ", "HTTPServer", "123", "456", "78", "i", "OSZ", " \r\n\r\n", "t", "!!", "Ⅳ", "
", "9", "\n", "'re"]} +{"text": "\tcamelCase'S/\r\nABC'ReiOS㋿'Res \r\n12345678…\r \n  -iOS漢🙂٣٤٥٦é…㍿　\r\n\r\nＡ's!!-camelCase\rABC<|fim_prefix|>>/ḍ̇!!", "tokens": 60, "pieces": ["\tcamel", "Case'S", "/\r\n", "ABC'Re", "i", "OS", "㋿'", "Res", " \r\n", "123", "456", "78", "…\r \n", " ", " ", "-i", "OS漢", "🙂", "٣٤٥", "٦", "é", "…", "㍿", "　\r\n\r\n", "Ａ's", "!!-", "camel", "Case", "\r", "ABC", "<|", "fim", "_prefix", "|>>/", "ḍ̇", "!!"]} +{"text": "'T å'VE \n \n 'T\nꟲſ ßé'M‍\rAb😀🏽/\r\nＡ३/\r\nع !!é㍿", "tokens": 37, "pieces": ["'T", " å'VE", " \n \n", " '", "T", "\n", "ꟲſ", " ", " ßé'M", "‍\r", "Ab", "😀🏽/\r\n", "Ａ", "३", "/\r\n", "ع", " ", "!!", "é", "㍿"]} +{"text": "s㍿>٣٤٥٦12345678\r\n\r\n ​\r\n's字s字😀🏽's𐞁Džungla㋿ABC'ſDžungla🙂t'VE\n/eDž \n'SeiOS<字'SAbع", "tokens": 66, "pieces": ["s", "㍿>", "٣٤٥", "٦12", "345", "678", "\r\n\r\n", " ​\r\n", "'s字s字", "😀🏽'", "s𐞁", "Džungla", "㋿ABC'ſ", "Džungla", "🙂t'VE", "\n", "/e", "Dž", " \n", "'Sei", "OS", "<<", "META", "_START", ">字'S", "Abع"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "‍! \n're​.", "tokens": 5, "pieces": ["‍!", " \n", "'re", "​."]} +{"text": "㍿'TiOSiOS!'re'S >ſ　ꟲ'T٣٤٥٦'Reſ漢'ReᵃiOS/\r\n-\r\n‍ع٣٤٥٦́0😀🏽ABC \n/\r\n\r\n", "tokens": 48, "pieces": ["㍿'", "Ti", "OSi", "OS", "!'", "re'S", " ", ">ſ", "　ꟲ'T", "٣٤٥", "٦", "'Reſ漢'Re", "ᵃi", "OS", "/\r\n", "-\r\n", "‍ع", "٣٤٥", "٦", "́", "0", "😀🏽", "ABC", " \n", "/\r\n\r\n"]} +{"text": "\"!\"½字½'reZéfiDžungla", "tokens": 13, "pieces": ["\"!\"", "½", "字", "½", "'re", "Zéfi", "Džungla"]} +{"text": "afi\t#$%<|endoftext|>Ab/\r\n EOT \n ꟲ'll\t👍🏽…aBaB'Ret㍿#$%ḍ̇", "tokens": 40, "pieces": ["afi", "\t", "#$%<|", "endoftext", "|>", "Ab", "/\r\n", " EOT", " \n", " ꟲ'll", "\t", "👍🏽", "…a", "Ba", "B'Re", "t", "㍿#$%", "ḍ̇"]} +{"text": "é-\r\n\r\n\u000b字('re'D \n ٣٤٥٦३\r\n\r\n!عB!iOSé-edABC\n/ \ncamelCase.漢mcamelCasedé", "tokens": 42, "pieces": ["é", "-\r\n\r\n", "\u000b字", "('", "re'D", " \n", " ", "٣٤٥", "٦", "", "३", "\r\n\r\n", "!ع", "B", "!i", "OSé", "-ed", "ABC", "\n", "/", " \n", "camel", "Case", ".漢mcamel", "Casedé"]} +{"text": "́sDžunglaat \n 漢DžHTTPServer㋿ \n ABCꟲm", "tokens": 21, "pieces": ["́s", "Džunglaat", " \n", " 漢DžHTTPServer", "㋿", " \n", " ABCꟲm"]} +{"text": "½'M!!!", "tokens": 3, "pieces": ["½", "'M", "!!!"]} +{"text": "👍🏽　'll<|endoftext|>'D 𐞁0Dž­ſßZiOS👍🏽\"३å.ḍ̇­ ­-\r \ns'M", "tokens": 48, "pieces": ["👍🏽", "　", "'ll", "<|", "endoftext", "|>'", "D", " <", "EOT", ">𐞁", "0", "Dž", "­ſß", "Zi", "OS", "👍🏽\"", "३", "å", ".ḍ̇", "­", " ", " ­-\r", " \n", "s'M"]} +{"text": "<|fim_prefix|>>字'M9a/béiOS12345678B ㋿å'll'Mt!!ḍ̇ḍ̇́<|endoftext|>!", "tokens": 44, "pieces": ["<|", "fim", "_prefix", "|>>", "字'M", "9", "a", "/béi", "OS", "123", "456", "78", "B", " ", " ㋿", "å'll", "'Mt", "!!", "ḍ̇ḍ̇́", "<|", "endoftext", "|>!"]} +{"text": "ع\u000b/ \n #$%", "tokens": 10, "pieces": ["ع", "\u000b", "/", " \n", " <", "META", "_START", ">#$%"]} +{"text": "'réécamelCase/\r\n\"/\r\na/b'reABCfi<ع#$%aB\t", "tokens": 20, "pieces": ["'réécamel", "Case", "/\r\n", "\"/\r\n", "a", "/b're", "ABCfi", "<ع", "#$%", "a", "B", "\t"]} +{"text": "٣٤٥٦<|fim_prefix|>٣٤٥٦iOSḍ̇Dž'T\n/…Džungla…é", "tokens": 35, "pieces": ["٣٤٥", "٦", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "i", "OSḍ̇", "Dž'T", "\n", "/", "…Džungla", "…é"]} +{"text": "ABCcamelCase'DcamelCase 😀🏽字12345678ꟲ🙂 漢", "tokens": 20, "pieces": ["ABCcamel", "Case'D", "camel", "Case", " ", " 😀🏽", "字", "123", "456", "78", "ꟲ", "🙂", " 漢"]} +{"text": "ſ漢/iOS٣٤٥٦>\r㋿Z12345678'MDžungla\t'TDžungla\"'VEⅣEOTZ\n/½ta/b\r\n a'D… <|endoftext|>é'St\u000b", "tokens": 63, "pieces": ["ſ漢", "/i", "OS", "٣٤٥", "٦", ">\r", "㋿Z", "123", "456", "78", "'MDžungla", "\t", "'TDžungla", "\"'", "VE", "Ⅳ", "EOTZ", "\n", "/", "½", "ta", "/b", "\r\n", " a'D", "…", " ", "<|", "endoftext", "|>", "é'S", "t", "\u000b"]} +{"text": "\"-", "tokens": 1, "pieces": ["\"-"]} +{"text": "/", "tokens": 1, "pieces": ["/"]} +{"text": "e's 'll'ſZ\u000bDž\u000b<|endoftext|>'M \n 's/\r\nås'VEm,\r\n\r\nåḍ̇/😀🏽<|endoftext|>,s", "tokens": 50, "pieces": ["e's", " ", "'ll'ſ", "Z", "\u000bDž", "\u000b", "<|", "endoftext", "|>'", "M", " \n", " '", "s", "/\r\n", "ås'VE", "m", ",\r\n\r\n", "å", "ḍ̇", "/😀🏽<|", "endoftext", "|>,", "s"]} +{"text": "😀🏽", "tokens": 3, "pieces": ["😀🏽"]} +{"text": "/Ⅳḍ̇(ABC　\r.­½​iOS…\r\n\r\n\"\n <12345678ß.HTTPServerZ ᵃ-'S\ta#$%", "tokens": 42, "pieces": ["/", "Ⅳ", "ḍ̇", "(ABC", "　\r", ".­", "½", "​i", "OS", "…\r\n\r\n", "\"\n", " ", " <", "123", "456", "78", "ß", ".HTTPServer", "Z", " ᵃ", "-<", "META", "_START", ">'", "S", "\ta", "#$%"]} +{"text": "aⅣİAb", "tokens": 5, "pieces": ["a", "Ⅳ", "İAb"]} +{"text": " (t'M​ꟲ \n(́ǻEOT \n e'sEOT \n　å<|fim_prefix|>camelCase　㍿\t\r\n\r\n\r\n", "tokens": 38, "pieces": [" ", "(t'M", "​ꟲ", " \n", "(́ǻ", "EOT", " \n", " e's", "EOT", " \n", "　å", "<|", "fim", "_prefix", "|>", "camel", "Case", "　", "㍿", "\t\r\n\r\n\r\n"]} +{"text": "ⅣEOTt\t/\r\n­'VE𐞁\t<३-,'s\n/㋿'VE'ſ\n<|fim_prefix|>9३'Dع'M 'S
Ⅳ0aB३ABC\r\n​s🙂", "tokens": 62, "pieces": ["Ⅳ", "EOTt", "\t", "/\r\n", "­'", "VE𐞁", "\t", "<", "३", "-,'", "s", "\n", "/㋿'", "VE'ſ", "\n", "<|", "fim", "_prefix", "|>", "9३", "'Dع'M", " ", "'", "S", "", "
", "Ⅳ0", "a", "B", "३", "ABC", "\r\n", "​s", "🙂"]} +{"text": "å­𐞁<|fim_prefix|>'M٣٤٥٦ddsdcamelCase,iOSꟲé…\n/㋿écamelCase'Re'T'", "tokens": 42, "pieces": ["å", "­𐞁", "<|", "fim", "_prefix", "|>'", "M", "٣٤٥", "٦", "ddsdcamel", "Case", ",i", "OSꟲé", "…\n", "/㋿", "écamel", "Case'Re", "'T", "'"]} +{"text": "‍ſ'reé\u000bḍ̇e🙂…ſ're….㋿é'T𐞁‍#$%٣٤٥٦　
-é", "tokens": 43, "pieces": ["‍ſ're", "é", "\u000bḍ̇e", "🙂", "…ſ're", "…", ".㋿", "é'T", "𐞁", "‍#$%", "٣٤٥", "٦", "　", "
", "-é"]} +{"text": "<|fim_prefix|>漢-\r\nZ㍿!!'ſDžungla\rDžungla'Mt…Ａḍ̇​iOS
é'T𐞁(DžéeABCé", "tokens": 50, "pieces": ["<|", "fim", "_prefix", "|>", "漢", "-\r\n", "Z", "㍿!!'", "ſ", "Džungla", "\r", "Džungla'M", "t", "…Ａḍ̇", "​i", "OS", "
é'T", "𐞁", "(Džée", "ABCé"]} +{"text": ".漢", "tokens": 2, "pieces": [".漢"]} +{"text": "ᵃm", "tokens": 4, "pieces": ["ᵃm"]} +{"text": "́😀🏽DžHTTPServer.\n/İ< /\r\nḍ̇/\r\n're🙂'T'ſⅣ字a/b\u000b", "tokens": 29, "pieces": ["́", "😀🏽", "DžHTTPServer", ".\n/", "İ", "<", " /\r\n", "ḍ̇", "/\r\n", "'re", "🙂'", "T'ſ", "Ⅳ", "字a", "/b", "\u000b"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "㍿½٣٤٥٦a/bå'Reå'T", "tokens": 16, "pieces": ["㍿", "½٣٤", "٥٦", "a", "/bå'Re", "å'T"]} +{"text": "ꟲ,
 \nmß漢\u000b\na/bB́a/b/\r\n\r\nİfiİ", "tokens": 21, "pieces": ["ꟲ", ",", "
 \n", "mß漢", "\u000b\n", "a", "/b", "B́a", "/b", "/\r\n\r\n", "İfi", "İ"]} +{"text": " \n ३'ſ \n", "tokens": 6, "pieces": [" \n", " ", "३", "'ſ", " \n"]} +{"text": "9\n/DžunglaHTTPServer/漢ᵃ'TiOSHTTPServer߅'S", "tokens": 27, "pieces": ["", "9", "\n", "/Džungla", "HTTPServer", "/漢ᵃ'T", "i", "OSHTTPServerß", "…", "'S"]} +{"text": "fi's", "tokens": 2, "pieces": ["fi's"]} +{"text": "\r'Re ́/'MaBB😀🏽#$%<|endoftext|>\"d🙂aB", "tokens": 23, "pieces": ["\r", "'Re", " ́", "/'", "Ma", "BB", "😀🏽#$%<|", "endoftext", "|>\"", "d", "🙂a", "B"]} +{"text": "\tDžungla🙂漢字½ \n/\"'Mß\r\n\r\n\r\n/\r\n㋿å\"é.", "tokens": 25, "pieces": ["\tDžungla", "🙂漢字", "½", " \n", "/\"'", "Mß", "\r\n\r\n\r\n", "/\r\n", "㋿å", "\"é", "."]} +{"text": "iOS㍿a", "tokens": 6, "pieces": ["i", "OS", "㍿a"]} +{"text": "ᵃB\"HTTPServerᵃ, \r\n\r\né<|endoftext|>İ­EOT\r\nåḍ̇'DⅣ,'Re…٣٤٥٦\"a/b(.'re-'D
'Tåd　😀🏽\r'", "tokens": 60, "pieces": ["ᵃ", "B", "\"HTTPServerᵃ", ",", " \r\n\r\n", "é", "<|", "endoftext", "|>", "İ", "­EOT", "\r\n", "åḍ̇'D", "Ⅳ", ",'", "Re", "…", "٣٤٥", "٦", "\"a", "/b", "(.'", "re", "-'", "D", "
", "'Tåd", "　", "😀🏽\r", "'"]} +{"text": "'re!!EOT'llAbs'VEå🙂…<',12345678a/b ​㍿\n/
 HTTPServer(\n'M​", "tokens": 33, "pieces": ["'re", "!!", "EOT'll", "Abs'VE", "å", "🙂", "…", "<',", "123", "456", "78", "a", "/b", " ​㍿\n/", "
", " HTTPServer", "(\n", "'M", "​"]} +{"text": "eEOTfi' \n\"　'M", "tokens": 9, "pieces": ["e", "EOTfi", "'", " \n", "\"", "　", "'M"]} +{"text": "‍'SAbt㋿ !<㍿HTTPServer　'T/éåＡ/\r\n㍿३/\r\nع>!!<|fim_prefix|>\rt'TaB.9", "tokens": 51, "pieces": ["‍'", "SAbt", "㋿", " <", "META", "_START", ">!<㍿", "HTTPServer", "", "　", "'T", "/éå", "Ａ", "/\r\n", "㍿", "३", "/\r\n", "ع", ">!!<|", "fim", "_prefix", "|>\r", "t'T", "a", "B", ".", "9"]} +{"text": "ḍ̇३9HTTPServer ,camelCaseåḍ̇İ\"'ſDžungla12345678 \n <|endoftext|><|fim_prefix|>éſ,Ⅳ𐞁åHTTPServerꟲ½fi'Reع #$%å ", "tokens": 67, "pieces": ["ḍ̇", "३9", "HTTPServer", " ", ",camel", "Caseåḍ̇", "İ", "\"'", "ſ", "Džungla", "123", "456", "78", " \n", " <|", "endoftext", "|><|", "fim", "_prefix", "|>", "éſ", ",", "Ⅳ", "𐞁å", "HTTPServerꟲ", "½", "fi'Re", "ع", " ", " #$%", "å", " "]} +{"text": "< \n㍿'M😀🏽<㍿'Re\n/ḍ̇(\r\n\r\n<|endoftext|>ſ \n,'ſ", "tokens": 37, "pieces": ["<", " \n", "㍿<", "EOT", ">'", "M", "😀🏽<㍿'", "Re", "\n", "/ḍ̇", "(\r\n\r\n", "<|", "endoftext", "|>", "ſ", " \n", ",'", "ſ"]} +{"text": "ᵃ/ \né'll­iOSḍ̇HTTPServera½字漢👍🏽iOSé's<|fim_prefix|>iOS(…éHTTPServer‍!!0,ſa/b/\r\n
", "tokens": 52, "pieces": ["ᵃ", "/", " \n", "é'll", "­i", "OS", "ḍ̇", "HTTPServera", "½", "字漢", "👍🏽", "i", "OSé's", "<|", "fim", "_prefix", "|>", "i", "OS", "(", "…é", "HTTPServer", "‍!!", "0", ",ſa", "/b", "/\r\n", "
"]} +{"text": "­Ab㍿'ll字㋿>'re\t'reZ'漢'T\r\n\r\nfi'S…\n/'VE<|fim_prefix|>9s'Sé٣٤٥٦", "tokens": 42, "pieces": ["­Ab", "㍿'", "ll字", "㋿>'", "re", "\t", "'re", "Z", "'漢'T", "\r\n\r\n", "fi'S", "…\n", "/'", "VE", "<|", "fim", "_prefix", "|>", "9", "s'S", "é", "٣٤٥", "٦"]} +{"text": "é'ſZ -ß,\r\n\r\ncamelCase/\r\n\" !!𐞁\r\n\u000b\r\nZ \n<|endoftext|>#$%٣٤٥٦​𐞁 \n", "tokens": 42, "pieces": ["é'ſ", "Z", " ", "-ß", ",\r\n\r\n", "camel", "Case", "/\r\n", "\"", " ", " !!", "𐞁", "\r\n\u000b\r\n", "Z", " \n", "<|", "endoftext", "|>#$%", "٣٤٥", "٦", "​𐞁", " \n"]} +{"text": "👍🏽iOS'SABC", "tokens": 7, "pieces": ["👍🏽", "i", "OS'S", "ABC"]} +{"text": "<漢ḍ̇/\r\n㋿fi字\u000b-", "tokens": 13, "pieces": ["<漢ḍ̇", "/\r\n", "㋿fi字", "\u000b", "-"]} +{"text": " \nåḍ̇ſ\"", "tokens": 14, "pieces": [" \n", "/,'", "S", "!<", "EOT", ">ḍ̇ſ", "\""]} +{"text": "d\u000b字ABC٣٤٥٦👍🏽", "tokens": 11, "pieces": ["d", "\u000b字", "ABC", "٣٤٥", "٦", "👍🏽"]} +{"text": "㋿\t", "tokens": 4, "pieces": ["㋿", "\t"]} +{"text": "ᵃ", "tokens": 6, "pieces": ["ᵃ"]} +{"text": "字'll\r\n /d \nDž　Džungla.12345678​ !!Dž'VEḍ̇/\r\n­'ſＡåfi'VE", "tokens": 46, "pieces": ["字'll", "\r\n", " /", "d", " \n", "Dž", "　Džungla", ".", "123", "456", "78", "​", " <", "EOT", ">!!", "Dž", "'", "VEḍ̇", "/\r\n", "­'", "ſ", "Ａåfi'VE", ""]} +{"text": "'Må‍३'VE<|fim_prefix|>
Z'é<\r\n\r\n'S!fié‍", "tokens": 23, "pieces": ["'Må", "‍", "३", "'VE", "<|", "fim", "_prefix", "|>", "
Z", "'é", "<\r\n\r\n", "'S", "!fié", "‍"]} +{"text": "å /\r\n/!Ⅳ \n !٣٤٥٦", "tokens": 14, "pieces": ["å", " /\r\n/", "!", "Ⅳ", " \n", " !", "٣٤٥", "٦"]} +{"text": "'T 漢 \n 'Dḍ̇mDž<|fim_prefix|> ​
…ſ<३　 \u000b>Z'D 👍🏽ea­iOS0漢Džungla 'ſⅣZ.", "tokens": 51, "pieces": ["'T", " ", " 漢", " \n", " '", "Dḍ̇m", "Dž", "<|", "fim", "_prefix", "|>", " ", " ​", "
", "…ſ", "<", "३", "　 ", "\u000b", ">Z'D", " ", "👍🏽", "ea", "­i", "OS", "0", "漢Džungla", " '", "ſ", "Ⅳ", "Z", "."]} +{"text": "HTTPServerß>camelCase12345678e İDžungla'Re\t", "tokens": 20, "pieces": ["HTTPServerß", ">", "camel", "Case", "123", "456", "78", "e", " İDžungla'Re", "\t"]} +{"text": ",\r\n\r\n<|endoftext|>å ABC字'S \n'VE㍿Dž(HTTPServer \n'S🙂é​'S'ſeaé /\r\n \r\n'll字", "tokens": 50, "pieces": [",\r\n\r\n", "<|", "endoftext", "|>", "å", " ", " ABC字", "'", "S", " \n", "'VE", "㍿Dž", "(HTTPServer", " \n", "'S", "🙂é", "​'", "S'ſ", "eaé", " ", " /\r\n", " ", " <", "META", "_START", ">\r\n", "'ll字"]} +{"text": "㍿iOSEOT<|fim_prefix|>a/b'T/\r\nABC'll'll'VEt'Sſet'SAb0-Džunglaſ'ſ'S\n字字>\r\ndé🙂å㋿👍🏽𐞁0Dž", "tokens": 61, "pieces": ["㍿i", "OSEOT", "<|", "fim", "_prefix", "|>", "a", "/b'T", "/\r\n", "ABC", "'", "ll'll", "'VEt'S", "ſet'S", "Ab", "0", "-Džunglaſ'ſ", "'S", "\n", "字字", ">\r\n", "dé", "🙂å", "㋿👍🏽", "𐞁", "0", "Dž"]} +{"text": "'M½'ſ'll
\u000b iOSDž!!\t/\r\n字Z㍿ḍ̇<|fim_prefix|>ع㍿!", "tokens": 33, "pieces": ["'M", "½", "'ſ'll", "
\u000b", " i", "OSDž", "!!", "\t", "/\r\n", "字", "Z", "㍿ḍ̇", "<|", "fim", "_prefix", "|>", "ع", "㍿!"]} +{"text": ">😀🏽ᵃ<|fim_prefix|>ꟲABC/\r\n \n ½\r\n> \n\r🙂's३ \n 漢ꟲ\n/'ſ(\niOS\r\n\t, \n\r\n", "tokens": 47, "pieces": [">😀🏽", "ᵃ", "<|", "fim", "_prefix", "|>", "ꟲ", "ABC", "/\r\n", " \n", " ", " ", "½", "\r\n", ">", " \n\r", "🙂'", "s", "३", " \n", " 漢ꟲ", "\n", "/'", "ſ", "(\n", "i", "OS", "\r\n", "\t", ",", " \n\r\n"]} +{"text": "㍿'S\n/-㋿
fi‍e'ſ'se‍é", "tokens": 24, "pieces": ["㍿'", "S", "\n", "/-㋿", "
fi", "‍e'ſ", "'", "se", "‍é"]} +{"text": "🙂Džungla \n éABCABC \n/\r\n漢'T㋿👍🏽Ab😀🏽३\r\n'll\"'ſBaB<㋿㋿å\nd-iOS#$%漢aBHTTPServer", "tokens": 50, "pieces": ["🙂Džungla", " \n", " é", "ABCABC", " \n", "/\r\n", "漢'T", "㋿👍🏽", "Ab", "😀🏽", "३", "\r\n", "'ll", "\"'", "ſ", "Ba", "B", "<㋿㋿", "å", "\n", "d", "-i", "OS", "#$%", "漢a", "BHTTPServer"]} +{"text": "/\r\n \n'VE/ \ne!\n/½'ſ/\r\n \n fi٣٤٥٦12345678
9-\ncamelCase\n/'sＡ\n!", "tokens": 19, "pieces": ["e'M", "", "123", "456", "78", "
", "9", "-\n", "camel", "Case", "\n", "/'", "s", "Ａ", "\n", "!"]} +{"text": "ꟲEOT👍🏽字>dABC\t字's<'ſfi'S­㋿'Mß'D \r", "tokens": 28, "pieces": ["ꟲ", "EOT", "👍🏽", "字", ">d", "ABC", "\t字's", "<'", "ſfi'S", "­㋿'", "Mß'D", " \r"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": " 'S'Re(", "tokens": 4, "pieces": [" ", "'S'Re", "("]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "​ \n \n'VE Ⅳ(a/bEOT<|endoftext|>!!#$%𐞁🙂३\r字/\r\n\"ß🙂'sḍ̇́", "tokens": 51, "pieces": ["​", " \n \n", "'VE", " ", " ", "Ⅳ", "(a", "/b", "EOT", "<|", "endoftext", "|>!!<", "META", "_START", ">#$%", "𐞁", "🙂", "३", "\r", "字", "/\r\n", "\"ß", "🙂'", "sḍ̇́"]} +{"text": "㋿å's", "tokens": 6, "pieces": ["㋿å's"]} +{"text": " ", "tokens": 1, "pieces": [" "]} +{"text": "'ś३'re\r\r\n\r\n㋿.ꟲé㍿'M \n ſ<|endoftext|>å9Z0Z㍿>ABCAbⅣ'SaB'll'reB", "tokens": 49, "pieces": ["'ś", "३", "'re", "\r\r\n\r\n", "㋿.", "ꟲé", "㍿'", "M", " \n", " ſ", "<|", "endoftext", "|>", "å", "9", "Z", "0", "Z", "㍿>", "ABCAb", "Ⅳ", "'Sa", "B'll", "'re", "B"]} +{"text": "ABCⅣ-ᵃ \n 'THTTPServeréHTTPServer٣٤٥٦ \n<字'ſfi m𐞁é'llå1234567812345678", "tokens": 42, "pieces": ["ABC", "Ⅳ", "-ᵃ", " \n", " '", "THTTPServeré", "HTTPServer", "٣٤٥", "٦", " \n", "<字'ſ", "fi", " ", " m𐞁é'll", "å", "123", "456", "781", "234", "567", "8"]} +{"text": "#$%mİع㍿a/bfi𐞁fi३\r\n\r\n>'llHTTPServer'ſé<|fim_prefix|>…ſeZt0!! 漢\u000b\r\n½ \n.İ㋿fi漢 ", "tokens": 62, "pieces": ["#$%", "m", "İع", "㍿a", "/bfi𐞁fi", "३", "\r\n\r\n", ">'", "ll", "HTTPServer'ſ", "é", "<|", "fim", "_prefix", "|>", "…ſe", "Zt", "0", "!!", " ", " 漢", "\u000b\r\n", "½", "", " \n", ".İ", "㋿fi漢", " "]} +{"text": "　(.", "tokens": 2, "pieces": ["　", "(."]} +{"text": "㋿'ll…३😀🏽ABCEOT½'Re,'M'VE'reZAb　'ſ'VE('ll.ꟲ\t/'Dé-é\"!", "tokens": 43, "pieces": ["㋿'", "ll", "…", "३", "😀🏽", "ABCEOT", "½", "'Re", ",'", "M'VE", "'re", "ZAb", "　", "'ſ'VE", "('", "ll", ".ꟲ", "\t", "/'", "Dé", "-", "é", "\"!"]} +{"text": "'VEDžunglaＡ'ſ'S'Meé/\r\n.<|endoftext|>Ⅳ>Ａs٣٤٥٦12345678½　 'ſå12345678' sſ", "tokens": 50, "pieces": ["'VEDžungla", "Ａ'ſ", "'S'M", "eé", "/\r\n", ".<", "META", "_START", "><|", "endoftext", "|>", "Ⅳ", ">Ａs", "٣٤٥", "٦12", "345", "678", "½", "　", " ", "'ſå", "123", "456", "78", "'", " ", " sſ"]} +{"text": "'s.'ll🙂ſ/!aB", "tokens": 9, "pieces": ["'s", ".'", "ll", "🙂ſ", "/!", "a", "B"]} +{"text": ">(𐞁EOT>aa ! !Ａ漢9å \n 'Dᵃå \n ", "tokens": 33, "pieces": [">(", "𐞁", "EOT", ">aa", " <", "META", "_START", ">!", " !", "Ａ漢", "9", "å", " \n", " '", "Dᵃå", " \n", " <", "META", "_START", ">"]} +{"text": "aiOS \n ‍‍DžacamelCaseecamelCase
camelCaseḍ̇㋿​a/b\rt \n  camelCase‍t<|fim_prefix|>", "tokens": 40, "pieces": ["ai", "OS", " \n", " ‍‍", "Džacamel", "Caseecamel", "Case", "
camel", "Caseḍ̇", "㋿​", "a", "/b", "\r", "t", " \n", " ", " camel", "Case", "‍t", "<|", "fim", "_prefix", "|>"]} +{"text": "́'sfiع<|fim_prefix|>(!!٣٤٥٦Ⅳ>ḍ̇Z", "tokens": 26, "pieces": ["́'s", "fiع", "<|", "fim", "_prefix", "|>(!!", "٣٤٥", "٦Ⅳ", ">ḍ̇", "Z"]} +{"text": "\r\n\r\néHTTPServer👍🏽\r\n\r\nacamelCase‍­,(0\rHTTPServer👍🏽\n/å0,ß \n Dž🙂s \n ㍿", "tokens": 40, "pieces": ["\r\n\r\n", "é", "HTTPServer", "👍🏽\r\n\r\n", "acamel", "Case", "‍­,(", "0", "\r", "HTTPServer", "👍🏽\n/", "å", "0", ",ß", " \n", " Dž", "🙂s", " \n", " ㍿"]} +{"text": "­
m…'T٣٤٥٦Ⅳ!'S!", "tokens": 15, "pieces": ["­", "
m", "…", "'T", "٣٤٥", "٦Ⅳ", "!'", "S", "!"]} +{"text": "<|fim_prefix|>.\u000baB \n/Abß٣٤٥٦a/b
　　㍿å\tع>ß\r\niOS'll㍿EOT", "tokens": 41, "pieces": ["<|", "fim", "_prefix", "|>.", "\u000ba", "B", " \n", "/Abß", "٣٤٥", "٦", "a", "/b", "
　", "　", "㍿å", "\tع", ">ß", "\r\n", "i", "OS'll", "㍿EOT"]} +{"text": "9.Z<|fim_prefix|>😀🏽ᵃt'D", "tokens": 20, "pieces": ["9", ".Z", "<|", "fim", "_prefix", "|>😀🏽", "ᵃt'D", ""]} +{"text": "'D३(字'S9½/\r\n
\ré'T", "tokens": 12, "pieces": ["'D", "३", "(字'S", "9½", "/\r\n", "
\r", "é'T"]} +{"text": "'VE-fiAb𐞁,😀🏽 \nHTTPServerİᵃ/(B", "tokens": 22, "pieces": ["'VE", "-fi", "Ab𐞁", ",😀🏽", " \n", "HTTPServer", "İᵃ", "/(", "B"]} +{"text": "'Reé½('ABCa'D/\r\t👍🏽 \nİfi<​\">ßEOTꟲ9ßsd🙂/'re\r\n\r\n<12345678", "tokens": 41, "pieces": ["'Reé", "½", "('", "ABCa'D", "/\r", "\t", "👍🏽", " \n", "İfi", "<​\">", "ß", "EOTꟲ", "9", "ßsd", "🙂/'", "re", "\r\n\r\n", "<", "123", "456", "78", ""]} +{"text": "é\r\r\n\r0'ſ'#$%<|fim_prefix|>a", "tokens": 17, "pieces": ["é", "\r\r\n\r", "0", "'ſ", "'#$%<|", "fim", "_prefix", "|>", "a"]} +{"text": "/\r\n\r\nDžungla'M'ReAb\n/'ll㋿
३٣٤٥٦ 'SaBZé\tEOT12345678ع\nEOTſ#$%éiOS-tsa/bDž(㍿!!0", "tokens": 56, "pieces": ["/\r\n\r\n", "Džungla'M", "'Re", "Ab", "\n", "/'", "ll", "㋿", "
", "३٣٤", "٥٦", " ", "'Sa", "BZé", "\tEOT", "", "123", "456", "78", "ع", "\n", "EOTſ", "#$%", "éi", "OS", "-tsa", "/b", "Dž", "(㍿!!", "0"]} +{"text": "fi𐞁é‍t.", "tokens": 9, "pieces": ["fi𐞁é", "‍t", "."]} +{"text": "-…#$%.'<Džungla'StEOTEOT9'T½(\u000b!!\n/e#$%iOS \n", "tokens": 29, "pieces": ["-", "…", "#$%.'<", "Džungla'S", "t", "EOTEOT", "9", "'T", "½", "(", "\u000b", "!!\n/", "e", "#$%", "i", "OS", " \n"]} +{"text": "\r\n!!HTTPServer\t\"­sd\"½'VE'D're<|fim_prefix|>!!-ع", "tokens": 26, "pieces": ["\r\n", "!!", "HTTPServer", "\t", "\"­", "sd", "\"", "½", "'VE'D", "'re", "<|", "fim", "_prefix", "|>!!-", "ع"]} +{"text": "😀🏽​­'ſ's.　\r\n\r\néßm 're", "tokens": 17, "pieces": ["😀🏽​­'", "ſ's", ".", "　\r\n\r\n", "éßm", " ", "'re"]} +{"text": "Z0‍e", "tokens": 11, "pieces": ["Z", "", "0", "‍", "e"]} +{"text": "\n𐞁ꟲ'sa/bＡ >a/be12345678 \nＡ\r<|endoftext|>a\tDž.…d㋿", "tokens": 43, "pieces": ["\n", "𐞁ꟲ's", "a", "/b", "Ａ", " >", "a", "/be", "123", "456", "78", " \n", "Ａ", "\r", "<|", "endoftext", "|>", "a", "\tDž", ".", "…d", "㋿"]} +{"text": "🙂 \n iOS\"camelCaseＡ", "tokens": 8, "pieces": ["🙂", " \n", " i", "OS", "\"camel", "Case", "Ａ"]} +{"text": "camelCaseDž'Re \nAb字 a \n!!éZé!
 \n 'Ａ,/😀🏽!! 's٣٤٥٦ſ/ ٣٤٥٦", "tokens": 46, "pieces": ["camel", "Case", "Dž'Re", " \n", "Ab字", " a", " \n", "!!", "é", "Zé", "!", "
 \n", " '", "Ａ", ",/😀🏽!!", " '", "s", "٣٤٥", "٦", "ſ", "/", " ", " ", "٣٤٥", "٦"]} +{"text": "/İ>‍漢 \nfi​ /\r\nß(‍👍🏽'DⅣعᵃ \n aDžungla　 \n s", "tokens": 33, "pieces": ["/İ", ">‍", "漢", " \n", "fi", "​", " ", "/\r\n", "ß", "(‍👍🏽'", "D", "Ⅳ", "عᵃ", " \n", " a", "Džungla", "　 \n", " s"]} +{"text": " /\r\nعHTTPServer३./\r\n9३'ſ 'VEAb<|fim_prefix|>Ａé '३\r\n३㋿\nB9\r\na \n​\" ,عfiEOT-​", "tokens": 48, "pieces": [" ", "/\r\n", "عHTTPServer", "३", "./\r\n", "9३", "'ſ", " ", "'VEAb", "<|", "fim", "_prefix", "|>", "Ａé", " ", " '", "३", "\r\n", "३", "㋿\n", "B", "9", "\r\n", "a", " \n", "​\"", " ", " ,", "عfi", "EOT", "-​"]} +{"text": "👍🏽\r\n٣٤٥٦'S<|endoftext|>fiABCᵃ𐞁,sa/b\u000b\r\n.Džéꟲᵃ🙂", "tokens": 44, "pieces": ["👍🏽\r\n", "٣٤٥", "٦", "'S", "<|", "endoftext", "|>", "fi", "ABCᵃ𐞁", ",sa", "/b", "\u000b", "\r\n", ".Džéꟲᵃ", "🙂"]} +{"text": "ABC'ReaB \nfi !.'ll  -ꟲ\r\n\r\n
é\u000b//dZ👍🏽ḍ̇'Mßs　​t🙂 \na/b'SB-‍ ", "tokens": 44, "pieces": ["ABC'Re", "a", "B", " \n", "fi", " !.'", "ll", " ", " -", "ꟲ", "\r\n\r\n", "
é", "\u000b", "//", "d", "Z", "👍🏽", "ḍ̇'M", "ßs", "　", "​t", "🙂", " \n", "a", "/b", "'", "SB", "-‍", " "]} +{"text": "a/béZ\rDžunglaᵃ😀🏽 \n\u000b're\rABC", "tokens": 20, "pieces": ["a", "/bé", "Z", "\r", "Džunglaᵃ", "😀🏽", " \n", "\u000b", "'re", "\r", "ABC"]} +{"text": "\u000bé0㍿fiB'S\t­'ll'camelCase \n'M", "tokens": 18, "pieces": ["\u000bé", "0", "㍿fi", "B'S", "\t", "­'", "ll", "'camel", "Case", " \n", "'M"]} +{"text": "\n-", "tokens": 2, "pieces": ["\n", "-"]} +{"text": "字é😀🏽ABC'ſ \n \n…'T‍Dž‍a/b३m/!'T'T🙂HTTPServer½㋿", " \n", "…", "'T", "‍Dž", "‍a", "/b", "३", "m", "/!'", "T'T", "🙂HTTPServer", "½", "㋿<", "Abꟲi", "OS", "/\r\n", "e're", " ", "㍿HTTPServers", "३", "/\r\n\r\n"]} +{"text": "<|endoftext|> 漢'T३/\r\nع('St字\r\n\r\nع㍿ꟲ٣٤٥٦é#$%漢'VEcamelCase", "tokens": 37, "pieces": ["<|", "endoftext", "|>", " 漢'T", "३", "/\r\n", "ع", "('", "St字", "\r\n\r\n", "ع", "㍿ꟲ", "٣٤٥", "٦", "é", "#$%", "漢'VE", "camel", "Case"]} +{"text": " \n'Mfi字　㍿'a/ba/ḍ̇camelCasem\r \n!!'ſ!eå\r\n('ſå\t\r\n\r\n'D👍🏽\r\n\r\n\r\n\r\nAb", "tokens": 43, "pieces": [" \n", "'Mfi字", "　", "㍿'", "a", "/ba", "/ḍ̇camel", "Casem", "\r \n", "!!'", "ſ", "!eå", "\r\n", "('", "ſå", "\t\r\n\r\n", "'D", "👍🏽\r\n\r\n\r\n\r\n", "Ab"]} +{"text": "\r\n\r\nİZ🙂'İİa/b", "tokens": 11, "pieces": ["\r\n\r\n", "İZ", "🙂'", "İİ", "a", "/b"]} +{"text": "ß9.½'Re字//\r\n🙂a/baBعfi½camelCase/\r\n", "tokens": 21, "pieces": ["ß", "9", ".", "½", "'Re字", "//\r\n", "🙂a", "/ba", "Bعfi", "½", "camel", "Case", "/\r\n"]} +{"text": "\u000bZİ", "tokens": 5, "pieces": ["\u000b", "Zİ"]} +{"text": ">HTTPServerDžungla-HTTPServer aBAbm!!", "tokens": 15, "pieces": [">HTTPServer", "Džungla", "-HTTPServer", " a", "BAbm", "!!"]} +{"text": "iOS\"Bḍ̇ttᵃᵃ12345678ḍ̇9İ३🙂İ‍/.ſ'VE'ſ'SiOS'T'ꟲ'‍mᵃ. s", "tokens": 52, "pieces": ["i", "OS", "\"Bḍ̇ttᵃᵃ", "123", "456", "78", "ḍ̇", "9", "İ", "३", "🙂İ", "‍/.", "ſ'VE", "'ſ'S", "i", "OS'T", "'ꟲ", "'‍", "mᵃ", ".", " s"]} +{"text": "字<|fim_prefix|>", "tokens": 7, "pieces": ["字", "<|", "fim", "_prefix", "|>"]} +{"text": "🙂 're İ'S­ſ\r\r\n\r\nś'ſ‍0 \nſ'Re ½(aB're'Reé-ꟲ'ReDžungla‍EOT… (", "tokens": 42, "pieces": ["🙂", " '", "re", " İ'S", "­ſ", "\r\r\n\r\n", "ś'ſ", "‍", "0", " \n", "ſ'Re", " ", "½", "(a", "B're", "'Reé", "-ꟲ'Re", "Džungla", "‍EOT", "…", " ", "("]} +{"text": "BcamelCase'HTTPServer", "tokens": 9, "pieces": ["Bcamel", "Case", "'<", "META", "_START", ">HTTPServer"]} +{"text": "'ll'EOT\r٣٤٥٦ꟲa/b'VE\r\n<|endoftext|>'ſḍ̇<|fim_prefix|>'S''llAb　.'re٣٤٥٦ḍ̇", "tokens": 53, "pieces": ["'ll", "'EOT", "\r", "٣٤٥", "٦", "ꟲa", "/b'VE", "\r\n", "<|", "endoftext", "|>'", "ſḍ̇", "<|", "fim", "_prefix", "|><", "META", "_START", ">'", "S", "''", "ll", "Ab", "　", ".'", "re", "٣٤٥", "٦", "ḍ̇"]} +{"text": "#$% \n ३'reꟲ", "tokens": 9, "pieces": ["#$%", " \n", " ", "३", "'reꟲ"]} +{"text": " \n's'Re\n/漢३​\t \n…", "tokens": 11, "pieces": [" \n", "'s'Re", "\n", "/漢", "३", "​", "\t \n", "…"]} +{"text": "Ⅳ㋿漢-İꟲᵃ\"aB
字#$%<|fim_prefix|>\r\n\r\n sİ/\r\n'll>'DEOT<­aDžunglaB,٣٤٥٦aAb(/å\t", "tokens": 53, "pieces": ["Ⅳ", "㋿漢", "-İꟲᵃ", "\"a", "B", "
字", "#$%<|", "fim", "_prefix", "|>\r\n\r\n", " s", "İ", "/\r\n", "'ll", ">'", "DEOT", "<­", "a", "Džungla", "B", ",", "٣٤٥", "٦", "a", "Ab", "(/", "å", "\t"]} +{"text": "EOT㍿\ré\"ſ9<|endoftext|>Džungla's'S'll٣٤٥٦fi<|fim_prefix|>Ⅳ\n­ 0camelCaseé12345678e ' \naB<|fim_prefix|> ＡåDžungla0㋿‍", "tokens": 73, "pieces": ["EOT", "㍿\r", "é", "\"ſ", "9", "<|", "endoftext", "|>", "Džungla's", "'S'll", "٣٤٥", "٦", "fi", "<|", "fim", "_prefix", "|>", "Ⅳ", "\n", "­", " ", "0", "camel", "Caseé", "123", "456", "78", "e", " ", " '", " \n", "a", "B", "<|", "fim", "_prefix", "|>", " Ａå", "Džungla", "0", "㋿‍"]} +{"text": "👍🏽㍿\r\n\r\n\u000bé12345678㋿camelCaseⅣa/bZ<\r\n\r\n漢fi", "tokens": 57, "pieces": ["é", "Dž", "\"", "123", "456", "78", "/\r\n", "　", "'M", "<|", "fim", "_prefix", "|>㋿", "camel", "Case", "Ⅳ", "a", "/b", "Z", "<\r\n\r\n", "漢fi", ""]} +{"text": "㋿字'VE!!Ⅳ'T­#$%㍿漢camelCase", "tokens": 16, "pieces": ["'re", "\r\n", " 𐞁t", "/\r\n", ">㍿", "漢camel", "Case"]} +{"text": " \n 0a<​EOT٣٤٥٦‍( \n'reiOS'Reꟲ(字'MAb\n/'re­,#$%tEOTé😀🏽 ㍿a/b", "tokens": 48, "pieces": [" \n", " ", " ", "0", "a", "<​", "EOT", "٣٤٥", "٦", "‍(", " \n", "'rei", "OS'Re", "ꟲ", "(字'M", "Ab", "\n", "/'", "re", "­,#$%", "t", "EOTé", "😀🏽", " ", "㍿a", "/b"]} +{"text": "HTTPServerſ'T0", "tokens": 5, "pieces": ["HTTPServerſ'T", "0"]} +{"text": "'M😀🏽٣٤٥٦<́é'D 'ſ㋿ ½a/bᵃ'Re…Ⅳ-ZſaB,​‍'\r\naBEOT‍ᵃ0,'ſ", "tokens": 49, "pieces": ["'M", "😀🏽", "٣٤٥", "٦", "<́é'D", " '", "ſ", "㋿", " ", " ", "½", "a", "/bᵃ'Re", "…", "Ⅳ", "-Zſa", "B", ",​‍'\r\n", "a", "BEOT", "‍ᵃ", "0", ",'", "ſ"]} +{"text": "B\n'Re🙂 ABC-B", "tokens": 7, "pieces": ["B", "\n", "'Re", "🙂", " ", " ABC", "-B"]} +{"text": "-camelCaseeⅣé.३字\"deع'll😀🏽漢३'aB'Da/b\"camelCase(عſm漢
", "tokens": 33, "pieces": ["-camel", "Casee", "Ⅳ", "é", ".", "३", "字", "\"deع'll", "😀🏽", "漢", "३", "'a", "B'D", "a", "/b", "\"camel", "Case", "(عſm漢", "
"]} +{"text": "eAbé३'T12345678 \n", "tokens": 10, "pieces": ["e", "Abé", "३", "'T", "123", "456", "78", " \n"]} +{"text": "Z12345678é\n/0 m İ​EOT>!\r\n\n/'ll‍-'s", "tokens": 23, "pieces": ["Z", "123", "456", "78", "é", "\n", "/", "0", " ", " m", " İ", "​EOT", ">!\r\n\n/", "'ll", "‍-'", "s"]} +{"text": "'å㋿/\r\nＡḍ̇dⅣ
d👍🏽!\tå<​३'İ ­,'re", "tokens": 36, "pieces": ["'å", "㋿/\r\n", "Ａḍ̇d", "Ⅳ", "
d", "👍🏽!", "\tå", "<<", "META", "_START", "><", "EOT", ">​", "३", "'İ", " ", " ­,'", "re"]} +{"text": "Ⅳ Džungla,'VEꟲ㍿", "tokens": 22, "pieces": ["Ⅳ", " Džungla", ",'", "VEꟲ", "㍿"]} +{"text": "'ſ३㍿\u000b'Re​'reꟲᵃ😀🏽0٣٤٥٦\r\nmm\nAb/\r\n'T½'Re<|fim_prefix|>عéعcamelCase.'ſ \r\n\r\n12345678ＡAb‍<|endoftext|>\nfi", "tokens": 68, "pieces": ["'", "ſ", "३", "㍿", "\u000b", "'Re", "​'", "reꟲᵃ", "😀🏽", "0٣٤", "٥٦", "\r\n", "mm", "\n", "Ab", "/\r\n", "'T", "½", "'Re", "<|", "fim", "_prefix", "|>", "عéعcamel", "Case", ".<", "EOT", ">'", "ſ", " \r\n\r\n", "123", "456", "78", "ＡAb", "‍<|", "endoftext", "|>\n", "fi"]} +{"text": "\raBcamelCasetḍ̇aficamelCaseع'ſßfiع­٣٤٥٦t/　'Re
'VEaB'll'ſ \n EOT'aBdd \n Ａعa", "tokens": 47, "pieces": ["\r", "a", "Bcamel", "Casetḍ̇aficamel", "Caseع'ſ", "ßfiع", "­", "٣٤٥", "٦", "t", "/", "　", "'Re", "
", "'VEa", "B'll", "'ſ", " \n", " EOT", "'a", "Bdd", " \n", " Ａعa"]} +{"text": "'s9\n/éſ🙂\n/漢عHTTPServer漢㍿iOSع<ß'ZdiOS.\r\n\r\n're　ḍ̇ḍ̇\tß\ns<|endoftext|>\n/'VEſ", "tokens": 50, "pieces": ["'s", "9", "\n", "/éſ", "🙂\n/", "漢عHTTPServer漢", "㍿i", "OSع", "<ß", "'Zdi", "OS", ".\r\n\r\n", "'re", "　ḍ̇ḍ̇", "\tß", "\n", "s", "<|", "endoftext", "|>\n/", "'VEſ"]} +{"text": "'sm'S'reع
\n\r­ᵃm٣٤٥٦३'ll>漢'T'llHTTPServeråſ!!　漢𐞁Ab's", "tokens": 37, "pieces": ["'sm'S", "'reع", "
\n\r", "­ᵃm", "٣٤٥", "٦३", "'ll", ">漢'T", "'ll", "HTTPServeråſ", "!!", "　漢𐞁Ab's"]} +{"text": "9a/b​½12345678 \n 123456780'M<|endoftext|>㋿'TDžungla\t🙂a/bDž \n\neDž<|endoftext|>ḍ̇!HTTPServers\r\nDžunglaع9
ꟲ/ \nDžungla'Mé", "tokens": 76, "pieces": ["9", "a", "/b", "​", "½12", "345", "678", " \n", " ", "123", "456", "780", "'M", "<|", "endoftext", "|>㋿'", "TDžungla", "\t", "🙂a", "/b", "Dž", " \n\n", "e", "Dž", "<|", "endoftext", "|>", "ḍ̇", "!HTTPServers", "\r\n", "Džunglaع", "9", "
ꟲ", "/", " \n", "Džungla'M", "é"]} +{"text": "ع㍿́å'Ⅳ👍🏽aBms\r🙂Z​!<'VE9t㋿<", "tokens": 31, "pieces": ["ع", "㍿́", "å", "'", "Ⅳ", "👍🏽", "a", "Bms", "\r", "🙂Z", "​!<'", "VE", "9", "t", "㋿<"]} +{"text": "m/d <|endoftext|>'M<|endoftext|>", "tokens": 17, "pieces": ["m", "/d", " <|", "endoftext", "|>'", "M", "<|", "endoftext", "|>"]} +{"text": "'iOS\tDžungla!\r\n\"-!'VE\r\n\r\nEOT \n/\r\n\r\n>tع99漢\t'ſDžᵃ'aé's…'٣٤٥٦\r\n\r\n-fi0", "tokens": 45, "pieces": ["'i", "OS", "\tDžungla", "!\r\n", "\"-!'", "VE", "\r\n\r\n", "EOT", " \n", "/\r\n\r\n", ">tع", "99", "漢", "\t", "'ſ", "Džᵃ", "'aé's", "…", "'", "٣٤٥", "٦", "\r\n\r\n", "-fi", "0"]} +{"text": "m　'Re12345678a/bＡma0", "tokens": 11, "pieces": ["m", "　", "'Re", "123", "456", "78", "a", "/b", "Ａma", "0"]} +{"text": " \nå<㋿aaB\n/ABCiOSaB-😀🏽'T\r<\n/fifi-३åḍ̇<|endoftext|>\r\nd 
'Re٣٤٥٦'re", "tokens": 57, "pieces": [" \n", "å", "<㋿<", "META", "_START", ">aa", "B", "\n", "/ABCi", "OSa", "B", "-😀🏽'", "T", "\r", "<\n/", "fifi", "-", "३", "åḍ̇", "<|", "endoftext", "|>\r\n", "d", "", " ", "
", "'Re", "٣٤٥", "٦", "'re"]} +{"text": " \n ", "tokens": 2, "pieces": [" \n", " "]} +{"text": " \n ३a/b\r\nZiOSé字!!Dž", "tokens": 14, "pieces": [" \n", " ", "३", "a", "/b", "\r\n", "Zi", "OSé字", "!!", "Dž"]} +{"text": "'M\"ＡſⅣİⅣ'ss12345678ſ'TaBaB0", "tokens": 20, "pieces": ["'M", "\"Ａſ", "Ⅳ", "İ", "Ⅳ", "'ss", "123", "456", "78", "ſ'T", "a", "Ba", "B", "0"]} +{"text": "mع'll\nDž😀🏽 \n ..<|endoftext|>\t٣٤٥٦ \r\n\r\niOS's
!'TEOTEOTꟲ'> 'M😀🏽 /'Re​", "tokens": 56, "pieces": ["mع", "'", "ll", "\n", "Dž", "😀🏽", " \n", " <", "META", "_START", ">..<|", "endoftext", "|>", "\t", "٣٤٥", "٦", " ", " <", "META", "_START", ">\r\n\r\n", "i", "OS's", "
", "!'", "TEOTEOTꟲ", "'>", " ", "'M", "😀🏽", " ", " /'", "Re", "​"]} +{"text": "漢.٣٤٥٦a/baBDž>३ß<|fim_prefix|>漢Džungla- ><|fim_prefix|>9", "३", "ß", "<|", "fim", "_prefix", "|>", "漢Džungla", "-", " ", " ><|", "fim", "_prefix", "|>", "9", "fiå' \n fi/\r\nDž siOSḍ̇'VEé½'ll'ſ \n é'T\n \n'🙂ßé字'T字HTTPServerᵃ", "tokens": 58, "pieces": ["!!'", "Tع", "\n", "/'", "s", "👍🏽\r\n", "<|", "endoftext", "|>", "fiå", "'", " \n", " fi", "/\r\n", "Dž", " si", "OSḍ̇'VE", "é", "½", "'ll'ſ", " \n", " é'T", "\n \n", "'🙂", "ßé字'T", "字HTTPServerᵃ"]} +{"text": "'VEaB\n<​a/bHTTPServer!e\n/fiſ́🙂d", "tokens": 23, "pieces": ["'VEa", "B", "\n", "<​", "a", "/b", "HTTPServer", "!e", "\n", "/fiſ", "́", "🙂d"]} +{"text": "
AbHTTPServerAbＡ", "tokens": 6, "pieces": ["
Ab", "HTTPServer", "Ab", "Ａ"]} +{"text": "‍٣٤٥٦‍<|fim_prefix|>漢'S.ſᵃ!Bt,B½
''VEB   'T0å\nAbcamelCaseꟲſ \n", "tokens": 42, "pieces": ["‍", "٣٤٥", "٦", "‍<|", "fim", "_prefix", "|>", "漢'S", ".ſᵃ", "!Bt", ",B", "½", "
", "''", "VEB", "  ", " ", "'T", "0", "å", "\n", "Abcamel", "Caseꟲſ", " \n"]} +{"text": "½fiB\n!!#$%㋿ssEOT/‍Dž", "tokens": 17, "pieces": ["½", "fi", "B", "\n", "!!#$%㋿", "ss", "EOT", "/‍", "Dž"]} +{"text": "aB (iOS\r\n㋿,\n<İ 👍🏽㋿Džع\r\ns㍿ع'ſ'reİßⅣ­", "tokens": 39, "pieces": ["a", "B", " (", "i", "OS", "\r\n", "㋿,\n", "<İ", " 👍🏽㋿", "Dž", "ع", "\r\n", "s", "㍿ع'ſ", "'re", "İß", "Ⅳ", "­"]} +{"text": "'\"éaaB٣٤٥٦fi-…>\u000b½/\r\n'S\r\n\r\n/\r\n!!Ⅳ'TmiOSＡ　'VE ꟲ\"ع", "tokens": 39, "pieces": ["'\"", "éaa", "B", "٣٤٥", "٦", "fi", "-", "…", ">", "\u000b", "½", "/\r\n", "'S", "\r\n\r\n", "/\r\n", "!!", "Ⅳ", "'Tmi", "OSＡ", "　", "'VE", " ", " ꟲ", "\"ع"]} +{"text": "​.ⅣHTTPServer　fi…­\nḍ̇'M<|endoftext|>٣٤٥٦'ll'M//㍿㋿ḍ̇Ⅳa\n/  \rß٣٤٥٦B!!𐞁0fi​😀🏽३", "tokens": 62, "pieces": ["​.", "Ⅳ", "HTTPServer", "　fi", "…", "­\n", "ḍ̇'M", "<|", "endoftext", "|>", "٣٤٥", "٦", "'ll'M", "//㍿㋿", "ḍ̇", "Ⅳ", "a", "\n", "/", "  \r", "ß", "٣٤٥", "٦", "B", "!!", "𐞁", "0", "fi", "​😀🏽", "३"]} +{"text": " e<|endoftext|>>🙂\r's'llAb åDžungla0", "tokens": 22, "pieces": [" ", " e", "<|", "endoftext", "|>>🙂\r", "'s'll", "Ab", " ", " å", "Džungla", "0"]} +{"text": "'ſ‍👍🏽\r\n\r\nmHTTPServer\r\nAb", "tokens": 12, "pieces": ["'ſ", "‍👍🏽\r\n\r\n", "m", "HTTPServer", "\r\n", "Ab"]} +{"text": "　‍\"d'D
'M　å𐞁's.ḍ̇/\r\n३'½/ꟲB­fi/'Re'Re-\u000b/\r\n", "tokens": 36, "pieces": ["　", "‍\"", "d'D", "
", "'M", "　å𐞁's", ".ḍ̇", "/\r\n", "३", "'", "½", "/ꟲ", "B", "­fi", "/'", "Re'Re", "-", "\u000b", "/\r\n"]} +{"text": "EOT​é ́\r \n Dž,🙂\néꟲ　iOS'Re", "tokens": 24, "pieces": ["EOT", "​é", " ", " ́", "\r \n", " Dž", ",🙂\n", "éꟲ", "　i", "OS'Re"]} +{"text": "İＡ'٣٤٥٦字\taBḍ̇𐞁‍\u000b'M\n/\r\n'½(#$%ß", "tokens": 28, "pieces": ["İＡ", "'", "٣٤٥", "٦", "字", "\ta", "Bḍ̇𐞁", "‍", "\u000b", "'M", "\n", "/\r\n", "'", "½", "(#$%", "ß"]} +{"text": " \n B#$%́\r\n a/b­ \nm\u000b　fi", "tokens": 15, "pieces": [" \n", " B", "#$%́\r\n", " a", "/b", "­", " \n", "m", "\u000b", "　fi"]} +{"text": "Dž👍🏽
're'Rea/bBᵃ­Dž​m𐞁-0--aABC😀🏽mta/bfi \n ḍ̇ ½३éd ", "tokens": 50, "pieces": ["Dž", "👍🏽", "
", "'re'Re", "a", "/b", "Bᵃ", "­Dž", "​m𐞁", "-", "0", "--", "a", "ABC", "😀🏽", "mta", "/bfi", " \n", " ḍ̇", " ", "½", "", "३", "éd", " "]} +{"text": "㍿s9iOSſ- B!s🙂e/0'\rع́éa/bEOT 字Džungla😀🏽'S", "tokens": 38, "pieces": ["㍿s", "9", "i", "OSſ", "-", " ", " B", "!s", "🙂e", "/", "0", "'\r", "ع́éa", "/b", "EOT", " 字Džungla", "😀🏽'", "S"]} +{"text": "'iOS0 \r!!<|fim_prefix|>　12345678\r<|fim_prefix|>éEOT!!'M/\r\n'Dfi\r\n\r\nHTTPServer\n're", "tokens": 38, "pieces": ["'i", "OS", "0", " \r", "!!<|", "fim", "_prefix", "|>", "　", "123", "456", "78", "\r", "<|", "fim", "_prefix", "|>", "é", "EOT", "!!'", "M", "/\r\n", "'Dfi", "\r\n\r\n", "HTTPServer", "\n", "'re"]} +{"text": "😀🏽>aB>İ/'S12345678 <|endoftext|>
a/bＡa\t'VE<|fim_prefix|>'VE字fi३́'llé9ß<|endoftext|>Ⅳ<|endoftext|>\r\n\r\nß🙂'MEOT'M", "tokens": 65, "pieces": ["😀🏽>", "a", "B", ">İ", "/'", "S", "123", "456", "78", " <|", "endoftext", "|>", "
a", "/b", "Ａa", "\t", "'VE", "<|", "fim", "_prefix", "|>'", "VE字fi", "३", "́'ll", "é", "9", "ß", "<|", "endoftext", "|>", "Ⅳ", "<|", "endoftext", "|>\r\n\r\n", "ß", "🙂'", "MEOT'M"]} +{"text": "'smDž'VEİé", "tokens": 9, "pieces": ["'sm", "Dž'VE", "İé"]} +{"text": ">-aDžＡZ>漢\u000bع👍🏽'Mꟲ'Mté", "tokens": 21, "pieces": [">-", "a", "DžＡZ", ">漢", "\u000bع", "👍🏽'", "Mꟲ'M", "té"]} +{"text": "㋿३éABC(!'T½\"👍🏽9/\r\n'VE's　#$%\ra/b
camelCase's​​ iOS🙂‍", "tokens": 40, "pieces": ["㋿", "३", "é", "ABC", "(!'", "T", "½", "\"👍🏽", "9", "/\r\n", "'VE's", "　", "#$%\r", "a", "/b", "
camel", "Case's", "​​", " i", "OS", "🙂‍"]} +{"text": "ⅣAb ​'s٣٤٥٦ \u000b½½ꟲ/\r\n>", "tokens": 20, "pieces": ["Ⅳ", "Ab", " ", "​'", "s", "٣٤٥", "٦", " ", "\u000b", "½½", "ꟲ", "/\r\n", ">"]} +{"text": "🙂BABC㍿Ab'llZＡZ\t…>9字​", "tokens": 18, "pieces": ["🙂BABC", "㍿Ab'll", "ZＡZ", "\t", "…", ">", "9", "字", "​"]} +{"text": " \n\r'Dſ'S", "tokens": 5, "pieces": [" \n\r", "'Dſ'S"]} +{"text": "a/b/\r\n🙂٣٤٥٦!.‍<|fim_prefix|>>  \nſDž\n/camelCaseå𐞁 ", "tokens": 36, "pieces": ["a", "/b", "/\r\n", "🙂<", "EOT", ">", "٣٤٥", "٦", "!.‍<|", "fim", "_prefix", "|>>", "  \n", "ſ", "Dž", "\n", "/camel", "Caseå𐞁", " "]} +{"text": "9Džungla\u000b\u000b12345678İB'VE-'M漢'VE\r​😀🏽é'M٣٤٥٦d<|endoftext|>'S><|fim_prefix|>9 \n AbfiEOTDž
! \n 'M'T", "tokens": 63, "pieces": ["9", "Džungla", "\u000b", "\u000b", "123", "456", "78", "İB'VE", "-'", "M漢'VE", "\r", "​😀🏽", "é'M", "٣٤٥", "٦", "d", "<|", "endoftext", "|>'", "S", "><|", "fim", "_prefix", "|>", "9", " \n", " Abfi", "EOTDž", "
", "!", " \n", " '", "M'T", ""]} +{"text": "å­٣٤٥٦å.\"　ᵃa/b\"\n/½a/b😀🏽 \u000b \n's<|endoftext|>'s \n\u000b 9½‍­İ'㋿m
字🙂ts", "tokens": 55, "pieces": ["å", "­", "٣٤٥", "٦", "å", ".\"", "　ᵃa", "/b", "\"\n/", "½", "a", "/b", "😀🏽", " \u000b \n", "'s", "<|", "endoftext", "|>'", "s", " \n", "\u000b", " ", "9½", "‍­", "İ", "'㋿", "m", "", "
字", "🙂ts"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": ", ('M(\n/'re \n ٣٤٥٦\r\n<​字👍🏽\n \n 😀🏽a/bᵃ>'ſ<|fim_prefix|>漢́", "tokens": 44, "pieces": [",", " ", " ('", "M", "(\n/", "'re", " \n", " ", "٣٤٥", "٦", "\r\n", "<​", "字", "👍🏽\n", "", " \n", " 😀🏽", "a", "/bᵃ", ">'", "ſ", "<|", "fim", "_prefix", "|>", "漢́"]} +{"text": "<|endoftext|>'Sع½<३-Džungla'ScamelCase<|endoftext|>­\r<|fim_prefix|><|fim_prefix|>Ⅳ0-ABCHTTPServer('ll‍½-\n…\r\n\r\n字,\r\n\r\n<<|fim_prefix|>́0", "tokens": 71, "pieces": ["<|", "endoftext", "|>'", "S", "ع", "½", "<", "३", "-Džungla'S", "camel", "Case", "<|", "endoftext", "|>­\r", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>", "Ⅳ0", "-ABCHTTPServer", "('", "ll", "‍", "½", "-\n", "…\r\n\r\n", "字", ",\r\n\r\n", "<<|", "fim", "_prefix", "|>́", "0"]} +{"text": "<|endoftext|>…👍🏽\"'Re字Za/b ㍿　#$%<|fim_prefix|>ß👍🏽fi're.ſⅣ\n/३EOT,漢aع\r åſ‍👍🏽camelCase漢d\n!", "tokens": 63, "pieces": ["<|", "endoftext", "|>", "…", "👍🏽\"'", "Re字", "Za", "/b", " ", "㍿", "　", "#$%<|", "fim", "_prefix", "|>", "ß", "👍🏽", "fi're", ".ſ", "Ⅳ", "\n", "/", "३", "EOT", ",漢aع", "\r", " åſ", "‍👍🏽", "camel", "Case漢d", "\n", "!"]} +{"text": " <|endoftext|>\r\n'T12345678a/bß'Re\"½\u000b're/́dᵃ'S­😀🏽 漢'M\ncamelCase
ᵃ­ \n ٣٤٥٦ \n", "tokens": 49, "pieces": [" ", " <|", "endoftext", "|>\r\n", "'T", "123", "456", "78", "a", "/bß'Re", "\"", "½", "\u000b", "'re", "/́dᵃ'S", "­😀🏽", " 漢'M", "\n", "camel", "Case", "
ᵃ", "­", " \n", " ", "٣٤٥", "٦", " \n"]} +{"text": "Abå.!\n/Ab/\">\n/\r\nDžungla　!HTTPServerABC­!!<|fim_prefix|>é", "tokens": 32, "pieces": ["Abå", ".!\n/", "Ab", "/\">\n/\r\n", "Džungla", "　", "!HTTPServer", "ABC", "­!!<|", "fim", "_prefix", "|>", "é"]} +{"text": "/\r\n'T' 12345678İDžunglaDž'ſBᵃ😀🏽!'TBBß\r\n\r\n#$%EOT<́漢字 ", "tokens": 37, "pieces": ["/\r\n", "'T", "'", " ", "123", "456", "78", "İDžungla", "Dž'ſ", "Bᵃ", "😀🏽!'", "TBBß", "\r\n\r\n", "#$%", "EOT", "<́漢字", " "]} +{"text": "-…12345678㋿'", "tokens": 10, "pieces": ["-", "…", "123", "456", "78", "㋿'"]} +{"text": "Džᵃ\".EOTABC'll'rea/b12345678\n<|fim_prefix|>'ll㍿Džungla字​-e…ᵃعiOSعḍ̇­'ll٣٤٥٦'s😀🏽Ab…", "tokens": 61, "pieces": ["Džᵃ", "\".", "EOTABC'll", "'rea", "/b", "123", "456", "78", "\n", "<|", "fim", "_prefix", "|>'", "ll", "㍿Džungla字", "​-", "e", "…ᵃعi", "OSعḍ̇", "­'", "ll", "٣٤٥", "٦", "'s", "😀🏽", "Ab", "…"]} +{"text": " 9a‍ İ　👍🏽camelCaseİ'!ꟲᵃ-,tt'VEås\"'Re 'reᵃ…'s'S>", "tokens": 45, "pieces": [" ", " ", "9", "a", "‍", " İ", "　", "👍🏽", "camel", "Case", "İ", "'!", "ꟲᵃ", "-,", "tt'VE", "ås", "\"<", "META", "_START", ">'", "Re", " ", " '", "reᵃ", "…", "'s'S", ">"]} +{"text": "Džunglaſ😀🏽>ᵃ'TcamelCaseⅣ \nⅣ", "tokens": 20, "pieces": ["Džunglaſ", "😀🏽>", "ᵃ'T", "camel", "Case", "Ⅳ", " \n", "Ⅳ"]} +{"text": "åt​ſⅣiOS　 \r\niOSEOTiOS'Re-\t'ſ<|endoftext|>\r(㍿ \n iOS'Re\n'ſ漢'T ᵃ \n ", "tokens": 49, "pieces": ["åt", "​ſ", "Ⅳ", "i", "OS", "　 \r\n", "i", "OSEOTi", "OS'Re", "-", "\t", "'ſ", "<|", "endoftext", "|>\r", "(㍿", " \n", " i", "OS'Re", "\n", "'ſ漢'T", " ᵃ", " \n", " "]} +{"text": "́३३Ab<🙂", "tokens": 9, "pieces": ["́", "३३", "Ab", "<🙂"]} +{"text": "'s'lls😀🏽/ \n 0å\u000b", "tokens": 13, "pieces": ["'s'll", "s", "😀🏽/", " \n", " ", "0", "å", "\u000b"]} +{"text": " \u000bs<|endoftext|>'ll>́\nt,'M,\u000b​\r", "tokens": 21, "pieces": [" ", "\u000bs", "<|", "endoftext", "|>'", "ll", ">́", "\n", "t", ",'", "M", ",", "\u000b", "​\r"]} +{"text": " ́!!Z 'DHTTPServerm㍿9'D½/", "tokens": 17, "pieces": [" ", " ́", "!!", "Z", " ", "'DHTTPServerm", "㍿", "9", "'D", "½", "/"]} +{"text": "a/bᵃ½.'D/\r\n\n", "tokens": 10, "pieces": ["a", "/bᵃ", "½", ".'", "D", "/\r\n\n"]} +{"text": "'ſ½\"ḍ̇漢/.-t\t'DBe\"\u000bß \n ३ßᵃ\tſ9ꟲ́Ⅳ­(ß's\u000b𐞁́
#$%ᵃ字B", "tokens": 51, "pieces": ["'ſ", "½", "\"ḍ̇漢", "/.-", "t", "\t", "'DBe", "\"", "\u000bß", " \n", " ", "३", "ßᵃ", "\tſ", "9", "ꟲ́", "Ⅳ", "­(", "ß's", "\u000b𐞁́", "
", "#$%", "ᵃ字", "B"]} +{"text": "'Ａß 'T…'Refit('llaB \naB́ \n /३\r
,\r\n\r\n're​camelCase", "tokens": 27, "pieces": ["'Ａß", " ", "'T", "…", "'Refit", "('", "lla", "B", " \n", "a", "B́", " \n", " /", "३", "\r", "
", ",\r\n\r\n", "'re", "​camel", "Case"]} +{"text": "!!\n/漢İ३\"('VEfiꟲd㋿9iOS,'Ree>", "tokens": 23, "pieces": ["!!\n/", "漢", "İ", "३", "\"('", "VEfiꟲd", "㋿", "9", "i", "OS", ",'", "Ree", ">"]} +{"text": "\r\nZ's'D\rſ'VE<|fim_prefix|>'D \n >", "tokens": 17, "pieces": ["\r\n", "Z's", "'D", "\r", "ſ'VE", "<|", "fim", "_prefix", "|>'", "D", " \n", " >"]} +{"text": "'re \n a'reİe's𐞁𐞁", "tokens": 22, "pieces": ["'re", " \n", " a're", "İ", "e's", "𐞁𐞁", ""]} +{"text": "'ſ mfié0
éᵃé\n/­12345678t‍mEOT#$%ꟲ\r३३३aaBZ-‍👍🏽ßB", "tokens": 46, "pieces": ["'ſ", " mfié", "0", "
éᵃé", "\n", "/­", "123", "456", "78", "t", "‍m", "EOT", "#$%", "ꟲ", "\r", "३३३", "aa", "BZ", "-<", "META", "_START", ">‍👍🏽", "ß", "B"]} +{"text": "'ſ'Msß's🙂!/㍿'sEOTåfi'saBsaḍ̇ＡİꟲåDžungla0(‍HTTPServer", "tokens": 43, "pieces": ["'ſ", "'", "Msß's", "🙂!/㍿'", "s", "EOTåfi's", "a", "Bsaḍ̇", "Ａİꟲå", "Džungla", "0", "(‍", "HTTPServer"]} +{"text": " \n !́ſAbå>-\rḍ̇👍🏽BHTTPServercamelCaseḍ̇'T,\tt-字", "tokens": 30, "pieces": [" \n", " !́", "ſ", "Ab", "å", ">-\r", "ḍ̇", "👍🏽", "BHTTPServercamel", "Caseḍ̇'T", ",", "\tt", "-字"]} +{"text": "EOTعiOŚDžungla㋿👍🏽字🙂½t0'ſßⅣB!!
<|endoftext|>Dž👍🏽🙂Džḍ̇ .", "tokens": 51, "pieces": ["EOTعi", "OŚDžungla", "㋿👍🏽", "字", "🙂", "½", "t", "0", "'", "ſß", "Ⅳ", "B", "!!", "
", "<|", "endoftext", "|>", "Dž", "👍🏽🙂", "Džḍ̇", " ", " ."]} +{"text": "Ａ<|fim_prefix|>B'ree㍿字ABC/\r\ncamelCase0a/\r\n漢\ta٣٤٥٦३", "tokens": 28, "pieces": ["Ａ", "<|", "fim", "_prefix", "|>", "B're", "e", "㍿字", "ABC", "/\r\n", "camel", "Case", "0", "a", "/\r\n", "漢", "\ta", "٣٤٥", "٦३"]} +{"text": " aHTTPServer\r\n\n,ZaB'll ㋿\r㍿'VE Ａ's'VE12345678're😀🏽'S( \n ABC‍ſ\t", "tokens": 47, "pieces": [" a", "HTTPServer", "\r\n\n", ",Za", "B'll", "", " ", " ㋿\r", "㍿'", "VE", " <", "META", "_START", ">Ａ's", "'VE", "123", "456", "78", "'re", "😀🏽'", "S", "(", " \n", " ABC", "‍ſ", "\t"]} +{"text": "-a/b/\r\n- \n !é!\n/­>.é​\" \n \n -漢'S\r'Re'D!Džungla.'M३", "tokens": 31, "pieces": ["-a", "/b", "/\r\n", "-", " \n", " !", "é", "!\n/", "­>.", "é", "​\"", " \n \n", " -", "漢'S", "\r", "'Re'D", "!Džungla", ".'", "M", "३"]} +{"text": "\r\n🙂-/İDžunglaꟲ​ \n're🙂\u000bＡ'M'éé३\naé0𐞁fi12345678#$%HTTPServer\n/'re½\"ß", "tokens": 49, "pieces": ["\r\n", "🙂<", "EOT", ">-/", "İDžunglaꟲ", "​", " \n", "'re", "🙂", "\u000bＡ'M", "'éé", "३", "\n", "aé", "0", "𐞁fi", "123", "456", "78", "#$%", "HTTPServer", "\n", "/'", "re", "½", "\"ß"]} +{"text": "e-,𐞁t'ſⅣ​a/b>'ll'ſ<|fim_prefix|>!e12345678\n٣٤٥٦\tⅣt字ABC'reⅣ'reée'\r\n'
EOTHTTPServer'iOS<|fim_prefix|><|endoftext|>B", "tokens": 67, "pieces": ["e", "-,", "𐞁t'ſ", "Ⅳ", "​a", "/b", ">'", "ll'ſ", "<|", "fim", "_prefix", "|>!", "e", "123", "456", "78", "\n", "٣٤٥", "٦", "\t", "Ⅳ", "t字", "ABC're", "Ⅳ", "'reée", "'\r\n", "'", "
EOTHTTPServer", "'i", "OS", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "B"]} +{"text": "-\n/ABC \n 👍🏽́́ \n \naB/\r\n👍🏽'Dſ'ſ३camelCase0٣٤٥٦‍siOSDž#$%'😀🏽<\rꟲ<|endoftext|>'VE \n'ſa'T0'Re", "tokens": 64, "pieces": ["-\n/", "ABC", " \n", " ", " 👍🏽́́", " \n \n", "a", "B", "/\r\n", "👍🏽'", "Dſ'ſ", "", "३", "camel", "Case", "0٣٤", "٥٦", "‍si", "OSDž", "#$%'😀🏽<\r", "ꟲ", "<|", "endoftext", "|>'", "VE", " \n", "'ſa'T", "0", "'Re"]} +{"text": "'reeé٣٤٥٦>٣٤٥٦ABĆ'ſ--'VEt0<|fim_prefix|>\r\n\r\n​
\tDž<­fi字''s", "tokens": 38, "pieces": ["'reeé", "٣٤٥", "٦", ">", "٣٤٥", "٦", "ABĆ'ſ", "--'", "VEt", "0", "<|", "fim", "_prefix", "|>\r\n\r\n", "​", "
", "\tDž", "<­", "fi字", "''", "s"]} +{"text": "Džungla'llABC㍿ 9. 𐞁ᵃmſ𐞁Z", "tokens": 31, "pieces": ["Džungla", "'", "ll", "ABC", "㍿", " ", "9", ".", " 𐞁ᵃmſ𐞁", "Z"]} +{"text": "Dž12345678\u000b\"/ß\r\n𐞁'reé½Ab\n/0're'ſ>tAbß­a/a/b/AbHTTPServer\u000b'S", "tokens": 40, "pieces": ["Dž", "", "123", "456", "78", "\u000b", "\"/", "ß", "\r\n", "𐞁're", "é", "½", "Ab", "\n", "/", "0", "'re'ſ", ">t", "Abß", "­a", "/a", "/b", "/Ab", "HTTPServer", "\u000b", "'S"]} +{"text": "Ⅳ \n#$%İＡ<|fim_prefix|>sDžt!​'re\nfi", "tokens": 23, "pieces": ["Ⅳ", " \n", "#$%", "İＡ", "<|", "fim", "_prefix", "|>", "s", "Džt", "!​'", "re", "\n", "fi"]} +{"text": "''Tfi㋿!!", "tokens": 25, "pieces": ["''", "Tfi", "㋿!!"]} +{"text": "9 \n𐞁 \u000b  é-ḍ̇ \n aB'S'lla'T\u000b👍🏽🙂ABC漢åİ<|fim_prefix|>.漢'Ms", "tokens": 41, "pieces": ["9", " \n", "𐞁", " \u000b  ", " é", "-ḍ̇", " \n", " a", "B'S", "'lla'T", "\u000b", "👍🏽🙂", "ABC漢å", "İ", "<|", "fim", "_prefix", "|>.", "漢'M", "s"]} +{"text": "camelCase'M‍ݽ‍fi<|endoftext|>　㋿>aBs㍿\r\r\nع", "tokens": 27, "pieces": ["camel", "Case'M", "‍İ", "½", "‍fi", "<|", "endoftext", "|>", "　", "㋿>", "a", "Bs", "㍿\r\r\n", "ع"]} +{"text": "ḍ̇a/b#$%👍🏽
ᵃteåＡ½émⅣ", "tokens": 44, "pieces": ["", "ḍ̇a", "/b", "#$%👍🏽", "
ᵃteå", "Ａ", "½", "ém", "Ⅳ"]} +{"text": "'ſ漢​\r\n!d", "tokens": 7, "pieces": ["'ſ漢", "​\r\n", "!d"]} +{"text": "\t…👍🏽ḍ̇!!", "tokens": 17, "pieces": ["\t", "", "…", "👍🏽", "ḍ̇", "!!"]} +{"text": "\rDž \n s👍🏽's字-mع🙂👍🏽/‍!å12345678🙂\r\n\r\n…/'Reḍ̇ <|endoftext|>👍🏽<|fim_prefix|><|fim_prefix|>​,Bå", "tokens": 67, "pieces": ["\r", "Dž", " \n", " s", "👍🏽'", "s字", "-mع", "🙂👍🏽/‍!", "å", "123", "456", "78", "🙂\r\n\r\n", "…", "/'", "Reḍ̇", " ", " <|", "endoftext", "|>👍🏽<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>​,", "Bå"]} +{"text": "ſ<|endoftext|>a'S😀🏽'EOTe(😀🏽İEOT9éDž", "tokens": 28, "pieces": ["ſ", "<|", "endoftext", "|>", "a'S", "😀🏽'", "EOTe", "(😀🏽", "İEOT", "9", "é", "Dž"]} +{"text": "\r\n\r\n\tḍ̇aB\r\u000b‍\u000b  ­a/b''S'Me12345678३ABC​…< te're\n/　", "tokens": 34, "pieces": ["\r\n\r\n", "\tḍ̇a", "B", "\r", "\u000b", "‍", "\u000b ", " ", "­a", "/b", "''", "S'M", "e", "123", "456", "78३", "ABC", "​", "…", "<", " ", " te're", "\n", "/", "　"]} +{"text": "'re!!/ \n", "tokens": 4, "pieces": ["'re", "!!/", " \n"]} +{"text": "‍𐞁漢'ſ😀🏽\r\n\r\nḍ̇sß Džungla'M\n!/\r\n", "tokens": 26, "pieces": ["‍𐞁漢'ſ", "😀🏽\r\n\r\n", "ḍ̇sß", " Džungla'M", "\n", "!/\r\n"]} +{"text": "٣٤٥٦'D're'\t \n 'fi'TⅣ३EOT!#$%", "tokens": 19, "pieces": ["٣٤٥", "٦", "'D're", "'", "\t \n", " '", "fi'T", "Ⅳ३", "EOT", "!#$%"]} +{"text": "‍e'T ३‍/\r\n", "tokens": 8, "pieces": ["‍e'T", " ", " ", "३", "‍/\r\n"]} +{"text": "🙂'㋿.Z'­>é𐞁​", "tokens": 17, "pieces": ["🙂'㋿.", "Z", "'­>", "é𐞁", "​"]} +{"text": "iOS㍿", "tokens": 5, "pieces": ["i", "OS", "㍿"]} +{"text": "EOT‍Ⅳé'M0
عiOSé漢dcamelCase0", "tokens": 18, "pieces": ["EOT", "‍", "Ⅳ", "é'M", "0", "
عi", "OSé漢dcamel", "Case", "0"]} +{"text": "/🙂\t<|endoftext|><|endoftext|>‍Ab#$%​
ABC-ḍ̇𐞁HTTPServer
́\ré'Re\u000b½'ll#$%‍'VE　½!!<|fim_prefix|>!!é're<|endoftext|>漢'll", "tokens": 68, "pieces": ["/🙂", "\t", "<|", "endoftext", "|><|", "endoftext", "|>‍", "Ab", "#$%​", "
ABC", "-ḍ̇𐞁", "HTTPServer", "
́", "\r", "é'Re", "\u000b", "½", "'ll", "#$%‍'", "VE", "　", "½", "!!<|", "fim", "_prefix", "|>!!", "é're", "<|", "endoftext", "|>", "漢'll"]} +{"text": "'VE ḍ̇é‍ḍ̇ /\r\n-\rDž字,ḍ̇'S٣٤٥٦\t<|endoftext|>­.d'RemHTTPServer \n ½ßfiåع　'Re<|fim_prefix|>ss'Sd𐞁ABC", "tokens": 68, "pieces": ["'VE", " ", " ḍ̇é", "‍ḍ̇", " /\r\n", "-\r", "Dž字", ",ḍ̇'S", "٣٤٥", "٦", "\t", "<|", "endoftext", "|>­.", "d'Re", "m", "HTTPServer", " \n", " ", "½", "ßfiåع", "　", "'Re", "<|", "fim", "_prefix", "|>", "ss'S", "d𐞁", "ABC"]} +{"text": "😀🏽s'Re's'D>'reta/bꟲHTTPServerm,!​ \n", "tokens": 19, "pieces": ["😀🏽", "s'Re", "'s'D", ">'", "reta", "/bꟲ", "HTTPServerm", ",!​", " \n"]} +{"text": "camelCase<|fim_prefix|><|endoftext|>'s٣٤٥٦EOT//HTTPServer'ReBDž ABCßİⅣDž", "tokens": 35, "pieces": ["camel", "Case", "<|", "fim", "_prefix", "|><|", "endoftext", "|>'", "s", "٣٤٥", "٦", "EOT", "//", "HTTPServer'Re", "BDž", " ABCß", "İ", "Ⅳ", "Dž"]} +{"text": "éع‍e\r\ne٣٤٥٦'re३ßaéꟲ漢'D.‍Džſ𐞁­-12345678👍🏽ꟲḍ̇(Ⅳ/\r\n,/fi ß½
", "tokens": 53, "pieces": ["éع", "‍e", "\r\n", "e", "٣٤٥", "٦", "'re", "३", "ßaéꟲ漢'D", ".‍", "Džſ𐞁", "­-", "123", "456", "78", "👍🏽", "ꟲḍ̇", "(", "Ⅳ", "/\r\n", ",/", "fi", " ß", "½", "
"]} +{"text": ",\r\n\r\nDžAb­\r\n\u000b12345678漢 \n .㍿́Džéᵃ's👍🏽,dé<|endoftext|>/
ßḍ̇<|fim_prefix|><|endoftext|>\nm'ſ \n ", "tokens": 64, "pieces": [",\r\n\r\n", "DžAb", "­\r\n", "\u000b", "123", "456", "78", "漢", " \n", " .㍿́", "Džé", "ᵃ's", "👍🏽,", "dé", "<|", "endoftext", "|>/", "
ßḍ̇", "<|", "fim", "_prefix", "|><|", "endoftext", "|>\n", "m'ſ", " \n", " "]} +{"text": "\n/漢-İ9ꟲ're'Re<|endoftext|>#$%-㍿<|endoftext|>ABCfi.'llABCDžungla/\r\n'ſ'VEꟲ,漢,漢½Dž\n𐞁m…　<|endoftext|>ꟲ", "tokens": 76, "pieces": ["\n", "/漢", "-İ", "9", "ꟲ're", "'Re", "<|", "endoftext", "|>#$%-㍿<|", "endoftext", "|>", "ABCfi", ".'", "ll", "ABCDžungla", "/\r\n", "'ſ'VE", "ꟲ", ",", "漢", ",漢", "½", "Dž", "\n", "𐞁m", "…", "　", "<|", "endoftext", "|>", "ꟲ"]} +{"text": "🙂\"'ſßcamelCase
-'s!\"", "tokens": 13, "pieces": ["🙂\"'", "ſßcamel", "Case", "", "
", "-'", "s", "!\""]} +{"text": "İ́Ⅳ​iOS're'٣٤٥٦㍿ \u000b\r\n\r\n‍'VE ", "tokens": 33, "pieces": ["Ⅳ", "'Da字", "<|", "endoftext", "|>", "Ⅳ", "​i", "OS're", "'", "٣٤٥", "٦", "㍿", " \u000b\r\n\r\n", "‍'", "VE", " "]} +{"text": "ꟲa/b<|fim_prefix|>B\r字0<'s/\r\n!!𐞁\u000b\n/", "tokens": 26, "pieces": ["ꟲa", "/b", "<|", "fim", "_prefix", "|>", "B", "\r", "字", "0", "<'", "s", "/\r\n", "!!", "𐞁", "\u000b\n", "/"]} +{"text": "Ⅳ́\u000bEOTe'T s‍ \n'VE𐞁<|fim_prefix|>٣٤٥٦ABC0 \t/\r\n!/\r\n‍/ ㍿'s'sDž !!😀🏽", "tokens": 57, "pieces": ["å'ſ", "!", "Ⅳ३", ">", "\u000bEOTe'T", " s", "‍", " \n", "'VE𐞁", "<|", "fim", "_prefix", "|>", "٣٤٥", "٦", "ABC", "0", " ", "\t", "/\r\n", "!/\r\n", "‍/", " ", "㍿'", "s's", "Dž", " ", "!!😀🏽"]} +{"text": "B'T0fié \né<|fim_prefix|>/‍iOSDž
", "tokens": 25, "pieces": ["B'T", "0", "fié", " \n", "é", "<|", "fim", "_prefix", "|>/‍", "i", "OSDž", "
"]} +{"text": "9字'VEe", "tokens": 9, "pieces": ["9", "字'VE", "e", ""]} +{"text": "\ra/b/'S'sfiİ'D's<|endoftext|>0​Ź३B ('M漢㍿ABC𐞁'Mع
'DＡſ'>/å㋿ᵃ", "tokens": 50, "pieces": ["\r", "a", "/b", "/'", "S's", "fi", "İ'D", "'s", "<|", "endoftext", "|>", "0", "​Ź", "३", "B", " ('", "M漢", "㍿ABC𐞁'M", "ع", "
", "'DＡſ", "'>/", "å", "㋿ᵃ"]} +{"text": "'s'Td9 🙂åꟲéé३Ⅳ३12345678'Re\u000b\r\n \nZ\n/!!'llḍ̇camelCase", "tokens": 37, "pieces": ["'s'T", "d", "9", " ", " 🙂", "åꟲéé", "३Ⅳ३", "123", "456", "78", "'Re", "\u000b\r\n \n", "Z", "\n", "/!!'", "llḍ̇camel", "Case"]} +{"text": "fiZDžungla­<'re, !aB(Ａ\r\n\t ٣٤٥٦😀🏽‍", "tokens": 29, "pieces": ["fi", "ZDžungla", "­<'", "re", ",", " !", "a", "B", "(Ａ", "\r\n", "\t", "", " ", "٣٤٥", "٦", "😀🏽‍"]} +{"text": "(a/bmßa/baB'aBé<|fim_prefix|>Džungla / (.'sDž(>iOS…𐞁½ABC.\r\nfia/b!!iOS‍9 \r\n\r\nḍ̇　", "tokens": 58, "pieces": ["(a", "/bmßa", "/b", "a", "B", "'a", "Bé", "<|", "fim", "_prefix", "|>", "Džungla", " ", " /", " ", " (.'", "s", "Dž", "(>", "i", "OS", "…𐞁", "½", "ABC", ".\r\n", "fia", "/b", "!!", "i", "OS", "‍", "9", " \r\n\r\n", "ḍ̇", "　"]} +{"text": "​𐞁\n/ \n's\r\n\r\nmḍ̇'s\tＡİß\"Ab#$%'ll👍🏽's㋿㋿\n!!　d/عfi\n\"'re\rḍ̇
0", "tokens": 61, "pieces": ["​𐞁", "\n", "/<", "EOT", ">", " \n", "'s", "\r\n\r\n", "mḍ̇'s", "\t", "Ａİß", "\"Ab", "#$%'", "ll", "👍🏽'", "s", "㋿㋿\n", "!!", "　d", "/عfi", "\n", "\"'", "re", "\r", "ḍ̇", "
", "0"]} +{"text": "ſſEOT漢३t٣٤٥٦Ab-ſ́<|fim_prefix|>'ſiOS\"'ll\n\r\n\r\nſعaB'DiOS㍿🙂", "tokens": 42, "pieces": ["ſſ", "EOT漢", "३", "t", "٣٤٥", "٦", "Ab", "-ſ́", "<|", "fim", "_prefix", "|>'", "ſi", "OS", "\"'", "ll", "\n\r\n\r\n", "ſعa", "B'D", "i", "OS", "㍿🙂"]} +{"text": "३!!a!Džunglaꟲ\r\nDžungla'…e½é漢ſé'ſEOTéİ'VE>Abꟲ \n\r\n\r\ńＡiOS㋿👍🏽t'ſ", "tokens": 56, "pieces": ["३", "!!", "a", "!Džunglaꟲ", "\r\n", "Džungla", "'", "…e", "½", "é", "漢ſé'ſ", "EOTé", "İ'VE", ">Abꟲ", " \n\r\n\r\n", "́Ａi", "OS", "㋿👍🏽", "t'ſ"]} +{"text": "'re🙂\"\naB!!EOT\"åaBiOSḍ̇<-字ع/\r\nDž​é'ſꟲع\nꟲEOTBt ", "tokens": 38, "pieces": ["'re", "🙂\"\n", "a", "B", "!!", "EOT", "\"åa", "Bi", "OSḍ̇", "<-", "字ع", "/\r\n", "Dž", "​é'ſ", "ꟲع", "\n", "ꟲEOTBt", " "]} +{"text": "​é 字३é9é \n é\n/-'D", "tokens": 21, "pieces": ["​é", " <", "META", "_START", ">", " ", " 字", "३", "é", "9", "é", " \n", " é", "\n", "/-'", "D"]} +{"text": "\r\n٣٤٥٦>

\rEOTme Z<|endoftext|>/ḍ̇㍿字 \n'T", "tokens": 29, "pieces": ["\r\n", "٣٤٥", "٦", ">", "

\r", "EOTme", " Z", "<|", "endoftext", "|>/", "ḍ̇", "㍿字", " \n", "'T"]} +{"text": " İ\r\n\r\nHTTPServerAb\n/-/'ll㍿ß😀🏽EOT ́㋿'M㋿㋿m<|endoftext|>sé½\r\n…", "tokens": 46, "pieces": [" ", " İ", "\r\n\r\n", "HTTPServer", "Ab", "\n", "/-/'", "ll", "㍿ß", "😀🏽", "EOT", " ", " ́", "㋿'", "M", "㋿㋿", "m", "<|", "endoftext", "|>", "sé", "½", "\r\n", "…"]} +{"text": "'M!Dž㋿\n\n/İ ‍iOS'MᵃDžungla!!tß\na\u000b\n\nß -'VE\n/fiaBé9å<|fim_prefix|>12345678'ſé", "tokens": 56, "pieces": ["'M", "!Dž", "㋿\n\n/", "İ", " ", "‍i", "OS'M", "ᵃDžungla", "!!", "tß", "\n", "a", "\u000b\n\n", "ß", " ", " -'", "VE", "\n", "/fia", "Bé", "9", "å", "<|", "fim", "_prefix", "|>", "123", "456", "78", "'ſé"]} +{"text": ",ᵃcamelCase", "tokens": 6, "pieces": [",ᵃcamel", "Case"]} +{"text": "Ab
-ꟲ'D'\u000b>­#$%t ㍿iOS'M iOS\r\n\"🙂iOS👍🏽-ßa/b0　𐞁#$%\t", "tokens": 50, "pieces": ["Ab", "
", "-ꟲ'D", "'", "\u000b", "><", "META", "_START", ">­#$%", "t", " ㍿", "i", "OS'M", " i", "OS", "\r\n", "\"🙂", "i", "OS", "👍🏽-", "ßa", "/b", "0", "　", "𐞁", "#$%", "\t"]} +{"text": "ᵃ\r\n­३ZHTTPServerABC\n/<|endoftext|>", "tokens": 18, "pieces": ["ᵃ", "\r\n", "­", "३", "ZHTTPServer", "ABC", "\n", "/<|", "endoftext", "|>"]} +{"text": "d३>'Re-\u000b", "tokens": 6, "pieces": ["d", "३", ">'", "Re", "-", "\u000b"]} +{"text": "9　㋿", "tokens": 5, "pieces": ["9", "　", "㋿"]} +{"text": "\u000bB\t㋿/s", "tokens": 8, "pieces": ["\u000bB", "\t", "㋿/", "s"]} +{"text": "ꟲ😀🏽Dž/ABCaBＡßiOS'Rea", "tokens": 18, "pieces": ["ꟲ", "😀🏽", "Dž", "/ABCa", "BＡßi", "OS'Re", "a"]} +{"text": "éḍ̇<٣٤٥٦HTTPServer,, ", "tokens": 13, "pieces": ["éḍ̇", "<", "٣٤٥", "٦", "HTTPServer", ",,", " "]} +{"text": "'T
İ'VEⅣß", "tokens": 8, "pieces": ["'T", "
İ'VE", "Ⅳ", "ß"]} +{"text": "12345678DžAb\u000be(.\tABCiOSß​ꟲå漢'll­<|fim_prefix|>­é㋿ \n ", "tokens": 36, "pieces": ["123", "456", "78", "DžAb", "\u000be", "(.", "\tABCi", "OSß", "​ꟲå漢'll", "­<|", "fim", "_prefix", "|>­", "é", "㋿", " \n", " "]} +{"text": "ꟲ ع\n<|endoftext|> \n 'S'VE#$%'T👍🏽/\r\né", "tokens": 33, "pieces": ["ꟲ", " ", " ع", "\n", "<|", "endoftext", "|>", " \n", " '", "S'VE", "#$%'", "T", "👍🏽/\r\n", "<", "EOT", ">é"]} +{"text": "\n/Z \n ", "tokens": 4, "pieces": ["\n", "/Z", " \n", " "]} +{"text": "!\n/'Reå😀🏽\r\n\n/Ⅳ'Me字fi'VEſ漢 \n!a/bDž­🙂.𐞁-", "tokens": 34, "pieces": ["!\n/", "'Reå", "😀🏽\r\n\n/", "Ⅳ", "'Me字fi'VE", "ſ漢", " \n", "!a", "/b", "Dž", "­🙂.", "𐞁", "-"]} +{"text": "\rEOT​Z<|endoftext|>́#$%/", "tokens": 16, "pieces": ["\r", "EOT", "​Z", "<|", "endoftext", "|>́#$%/"]} +{"text": "-字‍\r‍!­'re३ß½a/b'٣٤٥٦0iOSＡ🙂ḍ̇ꟲ㍿camelCase/🙂ḍ̇‍'ll\tſḍ̇>'re३-a　/\r\n", "tokens": 60, "pieces": ["-字", "‍\r", "‍!­'", "re", "३", "ß", "½", "a", "/b", "'", "٣٤٥", "٦0", "i", "OSＡ", "🙂ḍ̇ꟲ", "㍿camel", "Case", "/🙂", "ḍ̇", "‍'", "ll", "\tſḍ̇", ">'", "re", "३", "-<", "META", "_START", ">a", "　", "/\r\n"]} +{"text": "😀🏽㋿­.éHTTPServer/\r\n9eABC!iOSİ'S12345678é'S-
  \n 's/​/-", "tokens": 34, "pieces": ["😀🏽㋿­.", "é", "HTTPServer", "/\r\n", "9", "e", "ABC", "!i", "OSİ'S", "123", "456", "78", "é'S", "-", "
  \n", " '", "s", "/​/-"]} +{"text": "éAb\r\n/>BEOT!!sDžå0're\n'S\r\n\r\n\n/😀🏽 \n <|endoftext|>\u000b<'T'Re'TaBß \n<|endoftext|>/ \n㍿\u000b", "tokens": 56, "pieces": ["é", "Ab", "\r\n", "/>", "BEOT", "!!", "s", "Džå", "0", "'re", "\n", "'S", "\r\n\r\n\n", "/😀🏽", " \n", " <|", "endoftext", "|>", "\u000b", "<'", "T'Re", "'Ta", "Bß", " \n", "<|", "endoftext", "|>/", " \n", "㍿", "\u000b"]} +{"text": " 0,é'sé\r\n\r\nABC's\r\n\rⅣte!Ａ ㋿", "tokens": 22, "pieces": [" ", " ", "0", ",é's", "é", "\r\n\r\n", "ABC's", "\r\n\r", "Ⅳ", "te", "!Ａ", " ", "㋿"]} +{"text": "ma/b字Džungla😀🏽B/\r\n🙂　td𐞁 \"#$%ABC'ssB\"'T\r\n'M", "tokens": 30, "pieces": ["ma", "/b字", "Džungla", "😀🏽", "B", "/\r\n", "🙂", "　td𐞁", " \"#$%", "ABC's", "s", "B", "\"'", "T", "\r\n", "'M"]} +{"text": "٣٤٥٦0㍿ \n", "tokens": 9, "pieces": ["٣٤٥", "٦0", "㍿", " \n"]} +{"text": "'T\"å \n aBDžZ ́İ
'Re\r\n'S' 𐞁½\nå 漢s\t0\r\n<‍'ll'll٣٤٥٦㍿tHTTPServerfi", "tokens": 52, "pieces": ["'T", "\"å", " \n", " a", "BDžZ", " ́", "İ", "
", "'Re", "\r\n", "'S", "'", " ", " 𐞁", "½", "\n", "å", " 漢s", "\t", "0", "\r\n", "<<", "EOT", ">‍'", "ll'll", "٣٤٥", "٦", "㍿t", "HTTPServerfi"]} +{"text": "İå㋿", "tokens": 6, "pieces": ["İå", "㋿"]} +{"text": "'VE're'ſſEOT\r\n\r\n", "tokens": 12, "pieces": ["'VE're", "'", "ſſ", "EOT", "\r\n\r\n"]} +{"text": "Bs ع>ſ\r\nd字漢", "tokens": 9, "pieces": ["Bs", " ع", ">ſ", "\r\n", "d字漢"]} +{"text": "Abḍ̇½mfi'retDž\r\ńss\r\n🙂", "tokens": 16, "pieces": ["Abḍ̇", "½", "mfi're", "t", "Dž", "\r\n", "́ss", "\r\n", "🙂"]} +{"text": "𐞁 \n ㋿😀🏽 \n aB0 \n EOTEOTꟲ áſſ٣٤٥٦​́! 'M\u000b　㍿", "tokens": 43, "pieces": ["𐞁", " \n", " ㋿😀🏽", " \n", " a", "B", "0", " \n", " EOTEOTꟲ", " áſſ", "٣٤٥", "٦", "​́", "!", " ", " '", "M", "\u000b", "　", "㍿"]} +{"text": "㍿('T\r<|endoftext|>,é'T'Std<…", "tokens": 20, "pieces": ["㍿('", "T", "\r", "<|", "endoftext", "|>,", "é'T", "'Std", "<", "…"]} +{"text": "(.'re're٣٤٥٦ \n 'sEOTⅣ'll㍿aBⅣ!'Re\n\t .", "tokens": 28, "pieces": ["(.'", "re're", "٣٤٥", "٦", " \n", " '", "s", "EOT", "Ⅳ", "'ll", "㍿a", "B", "Ⅳ", "!'", "Re", "\n", "\t", " ."]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ZaB/\r\nZmm​' 's'M!́'s'ſ\r\n👍🏽İAbfiع👍🏽Z'ReéHTTPServer( Z字ᵃ9½!!aaBAb\n/'re", "tokens": 48, "pieces": ["Za", "B", "/\r\n", "Zmm", "​'", " ", " '", "s'M", "!́'s", "'ſ", "\r\n", "👍🏽", "İAbfiع", "👍🏽", "Z'Re", "é", "HTTPServer", "(", " ", " Z字ᵃ", "9½", "!!", "aa", "BAb", "\n", "/'", "re"]} +{"text": ">ß'S12345678>👍🏽Abᵃ!fi👍🏽<|endoftext|>\r12345678­a/b🙂\"sHTTPServer\u000b#$%/　ß\t", "tokens": 44, "pieces": [">ß'S", "123", "456", "78", ">👍🏽", "Abᵃ", "!fi", "👍🏽<|", "endoftext", "|>\r", "123", "456", "78", "­a", "/b", "🙂\"", "s", "HTTPServer", "\u000b", "#$%/", "　ß", "\t"]} +{"text": "t'!!Džungladß12345678", "tokens": 11, "pieces": ["t", "'!!", "Džungladß", "123", "456", "78"]} +{"text": " 'Re👍🏽 \n 'Dḍ̇ 'ſ'll \r\n\r\n\"🙂́'!!!", "tokens": 22, "pieces": [" ", " '", "Re", "👍🏽", " \n", " '", "Dḍ̇", " ", "'ſ'll", " \r\n\r\n", "\"🙂́'!!!"]} +{"text": "fi<|endoftext|>ſ'så\r\n<|endoftext|>𐞁𐞁m½e𐞁d́½Ab/<|fim_prefix|>'Td'MaB0​(㍿'ſ'D'S'Re'Re­ſ́", "tokens": 70, "pieces": ["fi", "<|", "endoftext", "|>", "ſ's", "å", "\r\n", "<|", "endoftext", "|>", "𐞁𐞁m", "½", "e𐞁", "d́", "½", "Ab", "/<|", "fim", "_prefix", "|>'", "Td'M", "a", "B", "0", "​(㍿'", "ſ'D", "'S'Re", "'Re", "­ſ́"]} +{"text": "aB\r\n\r\n㍿İ\r\n\r\n‍EOT \n d　Z\r\n\r\n… \n \n's'Re0\tİ\n/Ab'll字ß\n/'D", "tokens": 33, "pieces": ["a", "B", "\r\n\r\n", "㍿İ", "\r\n\r\n", "‍EOT", " \n", " d", "　Z", "\r\n\r\n… \n \n", "'s'Re", "0", "\tİ", "\n", "/Ab'll", "字ß", "\n", "/'", "D"]} +{"text": "'D!12345678𐞁\r\n\r\na'DＡ'VE😀🏽'S(­
🙂mß\"٣٤٥٦ Džungla\u000b\"", "tokens": 38, "pieces": ["'D", "!", "123", "456", "78", "𐞁", "\r\n\r\n", "a'D", "Ａ'VE", "😀🏽'", "S", "(­", "
", "🙂mß", "\"", "٣٤٥", "٦", " Džungla", "\u000b", "\""]} +{"text": "ea/b're👍🏽عåtaB<|endoftext|>Ａs½㋿-Ab🙂,Zfi\n\r\n\r\n‍㍿'re<|fim_prefix|>­0\n/#$%İ're", "Ａs", "½", "㋿-", "Ab", "🙂,", "Zfi", "\n\r\n\r\n", "‍㍿'", "re", "<|", "fim", "_prefix", "|>­", "0", "\n", "/#$%", "İ're", ",e're're", "tokens": 18, "pieces": [" ", " Abéᵃꟲ", "<|", "fim", "_prefix", "|>,", "e're", "'re"]} +{"text": "　'llＡ<|endoftext|>Ab'll'sDž 'M#$%'D9#$%mᵃ/‍ſ'Re", "tokens": 34, "pieces": ["　", "'ll", "Ａ", "<|", "endoftext", "|>", "Ab'll", "'s", "Dž", "", " ", "'M", "#$%'", "D", "9", "#$%", "mᵃ", "/‍", "ſ'Re"]} +{"text": "ꟲ\n9\r\n\r\né!'DiOS👍🏽//s\r'reᵃḍ̇'Ree(aAb'DᵃB\r 𐞁🙂!!", "tokens": 40, "pieces": ["ꟲ", "\n", "9", "\r\n\r\n", "é", "!'", "Di", "OS", "👍🏽//", "s", "\r", "'reᵃḍ̇'Re", "e", "(a", "Ab'D", "ᵃ", "B", "\r", " 𐞁", "🙂!!"]} +{"text": "‍AbſcamelCase𐞁0\r\n\r\n12345678ḍ̇e!ḍ̇Bᵃ'Rea/b", "tokens": 29, "pieces": ["‍Abſcamel", "Case𐞁", "0", "\r\n\r\n", "123", "456", "78", "ḍ̇e", "!ḍ̇", "Bᵃ'Re", "a", "/b"]} +{"text": "'re…㋿​", "tokens": 7, "pieces": ["'re", "…", "㋿​"]} +{"text": " EOT're's'D'VEİ
s\t\u000b.'llDž,😀🏽", "tokens": 20, "pieces": [" EOT're", "'s'D", "'VEİ", "
s", "\t", "\u000b", ".'", "ll", "Dž", ",😀🏽"]} +{"text": "👍🏽ḍ̇'D", "tokens": 7, "pieces": ["👍🏽", "ḍ̇'D"]} +{"text": "/Ａa/b 'M🙂é字😀🏽'ſ\t漢 字­😀🏽m३'D\u000bm​ 字'M\"ꟲ'M<|endoftext|> 'D<|endoftext|>å", "tokens": 59, "pieces": ["/Ａa", "/b", " ", "'M", "🙂é字", "😀🏽'", "ſ", "\t漢", " ", " 字", "­😀🏽", "m", "३", "'", "D", "\u000bm", "​", " 字'M", "\"ꟲ'M", "<|", "endoftext", "|>", " '", "D", "<|", "endoftext", "|>", "å"]} +{"text": "é½\r\n\r\n\r\n<|fim_prefix|>́Dž३'TcamelCase", "tokens": 17, "pieces": ["é", "½", "\r\n\r\n\r\n", "<|", "fim", "_prefix", "|>́", "Dž", "३", "'Tcamel", "Case"]} +{"text": "d٣٤٥٦'ſ\u000b​>३
'M", "tokens": 16, "pieces": ["d", "٣٤٥", "٦", "'ſ", "\u000b", "​>", "३", "
", "'M", ""]} +{"text": "\n'reع字BſéficamelCase👍🏽'ABC,9aB \n𐞁
\n/fi're", "tokens": 36, "pieces": ["\n", "'reع字", "Bſéficamel", "Case", "👍🏽<", "META", "_START", ">'", "ABC", ",", "9", "a", "B", " \n", "𐞁", "
\n", "/fi're"]} +{"text": "/\r\n's<|endoftext|>'s½…fia/b!!t'ſ9'\r\n é'SaBm\u000bZ'0é
<|fim_prefix|>d\rＡ㋿", "tokens": 49, "pieces": ["/\r\n", "'s", "<|", "endoftext", "|>'", "s", "½", "…fia", "/b", "!!", "t", "'", "ſ", "9", "'\r\n", " ", " é'S", "a", "Bm", "\u000bZ", "'", "0", "é", "
", "<|", "fim", "_prefix", "|>", "d", "\r", "Ａ", "㋿"]} +{"text": "ḍ̇漢fiḍ̇0--ꟲ\r​12345678ßİ　å<|fim_prefix|>DžunglaHTTPServer'Re\u000bAb👍🏽('e'T'T\n!Ａ㍿㍿­", "tokens": 55, "pieces": ["ḍ̇漢fiḍ̇", "0", "--", "ꟲ", "\r", "​", "123", "456", "78", "ß", "İ", "　å", "<|", "fim", "_prefix", "|>", "Džungla", "HTTPServer'Re", "\u000bAb", "👍🏽('", "e'T", "'T", "\n", "!Ａ", "㍿㍿­"]} +{"text": "a/b-　𐞁\n/ t.fi e", "tokens": 15, "pieces": ["a", "/b", "-", "　𐞁", "\n", "/", " t", ".fi", " e"]} +{"text": "ᵃ!!'Re--. Z", "tokens": 10, "pieces": ["ᵃ", "!!'", "Re", "--.", " Z"]} +{"text": "İZDžungla㋿(å㍿iOS🙂aß३", "tokens": 21, "pieces": ["İZDžungla", "㋿(", "å", "㍿i", "OS", "🙂aß", "३"]} +{"text": "३å'M<|fim_prefix|>EOT'S
!!👍🏽é!　", "tokens": 21, "pieces": ["३", "å'M", "<|", "fim", "_prefix", "|>", "EOT'S", "
", "!!👍🏽", "é", "!", "　"]} +{"text": "­‍\"\n/EOT'll'D'llⅣ", "tokens": 10, "pieces": ["­‍\"\n/", "EOT'll", "'D'll", "Ⅳ"]} +{"text": "'\n‍'ll\n/t👍🏽 \n HTTPServer٣٤٥٦'T字🙂9\n", "tokens": 29, "pieces": ["'\n", "‍'", "ll", "\n", "/t", "👍🏽<", "EOT", ">", " \n", " <", "META", "_START", ">HTTPServer", "٣٤٥", "٦", "'T字", "🙂", "9", "\n"]} +{"text": "a/bİ\tİta/b'DcamelCase'll'ſ \n \r\n\r\n", "tokens": 15, "pieces": ["a", "/b", "İ", "\tİta", "/b'D", "camel", "Case'll", "'ſ", " \n \r\n\r\n"]} +{"text": "\"́/ᵃ㋿​ >٣٤٥٦", "tokens": 19, "pieces": ["\"́", "/ᵃ", "㋿<", "EOT", ">​", " >", "٣٤٥", "٦"]} +{"text": "'VEABC's(e#$%<|endoftext|>𐞁'Da", "tokens": 20, "pieces": ["'VEABC's", "(e", "#$%<|", "endoftext", "|>", "𐞁'D", "a"]} +{"text": "t\r!!'sEOT'llع'T🙂😀🏽字​, ‍\r\n\r\nḍ̇\r\n\r\nB/'re<'ſm", "tokens": 40, "pieces": ["t", "\r", "!!'", "s", "EOT'll", "ع", "'", "T", "🙂😀🏽", "字", "​,", " ", "‍<", "EOT", ">\r\n\r\n", "ḍ̇", "\r\n\r\n", "B", "/'", "re", "<'", "ſm"]} +{"text": "ع٣٤٥٦\r漢Ab<|fim_prefix|><|fim_prefix|>><|endoftext|> \n 
👍🏽d½㍿<'sZ'M'll./\r\n're é", "tokens": 50, "pieces": ["ع", "٣٤٥", "٦", "\r", "漢Ab", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>><|", "endoftext", "|>", " \n", " ", "
", "👍🏽", "d", "", "½", "㍿<'", "s", "Z'M", "'ll", "./\r\n", "'re", " ", " é"]} +{"text": "Dž'MAb", "tokens": 4, "pieces": ["Dž'M", "Ab"]} +{"text": "<\rꟲ٣٤٥٦a/bå'refiDžunglaᵃ0camelCaseḍ̇''ll'SⅣ'sⅣé<|endoftext|>/\r\n", "tokens": 45, "pieces": ["<\r", "ꟲ", "٣٤٥", "٦", "a", "/bå're", "fi", "Džunglaᵃ", "0", "camel", "Caseḍ̇", "''", "ll'S", "Ⅳ", "'s", "Ⅳ", "é", "<|", "endoftext", "|>/\r\n"]} +{"text": "/\r\n  -\u000b(<|fim_prefix|>emHTTPServer#$%ḿ½​#$%", "tokens": 22, "pieces": ["/\r\n", " ", " ", "-", "\u000b", "(<|", "fim", "_prefix", "|>", "em", "HTTPServer", "#$%", "ḿ", "½", "​#$%"]} +{"text": "é/0/\r\n!!a'll \nꟲ(Ab/٣٤٥٦'reå🙂<|fim_prefix|>'Re EOT'M\r\n\r\n/\r\n­ABC#$%ßHTTPServer'Deſ", "tokens": 46, "pieces": ["é", "/", "0", "/\r\n", "!!", "a'll", " \n", "ꟲ", "(Ab", "/", "٣٤٥", "٦", "'reå", "🙂<|", "fim", "_prefix", "|>'", "Re", " EOT'M", "\r\n\r\n", "/\r\n", "­ABC", "#$%", "ß", "HTTPServer'D", "eſ"]} +{"text": "'reABC'reHTTPServerHTTPServerfi
'M 'Tİ\"
'M…Ａ'S\ré٣٤٥٦\r\n\r\nfi­\r0Zᵃ́å!!'ſcamelCase.camelCase#$%>'Så", "tokens": 51, "pieces": ["'re", "ABC're", "HTTPServer", "HTTPServerfi", "
", "'M", " ", "'Tİ", "\"", "
", "'M", "…Ａ'S", "\r", "é", "٣٤٥", "٦", "\r\n\r\n", "fi", "­\r", "0", "Zᵃ́å", "!!'", "ſcamel", "Case", ".camel", "Case", "#$%>'", "Så"]} +{"text": "'t \n 'ſع'D½ع漢12345678­عå😀🏽😀🏽'", "tokens": 33, "pieces": ["'t", " \n", " '", "ſع'D", "½", "ع漢", "123", "456", "78", "­ع", "å", "😀🏽😀🏽'"]} +{"text": ".٣٤٥٦DžABC <😀🏽‍,B9\"Ａe, \n ", "tokens": 22, "pieces": [".", "٣٤٥", "٦", "DžABC", " ", " <😀🏽‍,", "B", "9", "\"Ａe", ",", " \n", " "]} +{"text": "\r\n\rße\u000bⅣ字-\r\n\r\n'Mḍ̇#$%12345678.३',ᵃ👍🏽då'ᵃ<-\r\n\r\nꟲ-'ſ's👍🏽(", "tokens": 45, "pieces": ["\r\n\r", "ße", "\u000b", "Ⅳ", "字", "-\r\n\r\n", "'Mḍ̇", "#$%", "123", "456", "78", ".", "३", "',", "ᵃ", "👍🏽", "då", "'ᵃ", "<-\r\n\r\n", "ꟲ", "-'", "ſ's", "👍🏽("]} +{"text": "Båß\r!!ꟲ  ᵃ😀🏽😀🏽\"0-　'M", "tokens": 24, "pieces": ["Båß", "\r", "!!", "ꟲ", " ", " ᵃ", "😀🏽😀🏽\"", "0", "-", "　", "'M"]} +{"text": "'ReABCå'DＡa İ12345678/\r\n9", "tokens": 13, "pieces": ["'Re", "ABCå'D", "Ａa", " İ", "123", "456", "78", "/\r\n", "9"]} +{"text": "'VEİßꟲ((​😀🏽<|fim_prefix|>ＡZ'Dß<|fim_prefix|>'ſ,'re#$%", "tokens": 36, "pieces": ["'VEİßꟲ", "((​😀🏽<|", "fim", "_prefix", "|>", "ＡZ'D", "ß", "<|", "fim", "_prefix", "|>'", "ſ", ",'", "re", "#$%"]} +{"text": ",\n \ndſ'D😀🏽#$%/\r\nss/e>字fia/b/Džungla\n/　🙂", "tokens": 27, "pieces": [",\n", " \n", "dſ'D", "😀🏽#$%/\r\n", "ss", "/e", ">字fia", "/b", "/Džungla", "\n", "/", "　", "🙂"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "\r​!!'resＡ…İ٣٤٥٦Ⅳꟲ''s½\tZ\"9३३\t\"İ\r\n\r\n'll(\n", "tokens": 32, "pieces": ["\r", "​!!'", "res", "Ａ", "…İ", "٣٤٥", "٦Ⅳ", "ꟲ", "''", "s", "½", "\tZ", "\"", "9३३", "\t", "\"İ", "\r\n\r\n", "'ll", "(\n"]} +{"text": "så  \n ſⅣ/\r\n\u000b\"🙂're ㋿ \n - \n ZＡ Ab
d,🙂🙂'Re'Ma/b字<|fim_prefix|>'VEⅣ😀🏽\r\n\r\n\r\n<|fim_prefix|>İ\u000b<12345678, st", "tokens": 54, "pieces": ["'𐞁", "Z", "Ａ", " ", " Ab", "
d", ",🙂🙂'", "Re'M", "a", "/b字", "<|", "fim", "_prefix", "|>'", "VE", "Ⅳ", "😀🏽\r\n\r\n\r\n", "<|", "fim", "_prefix", "|>", "İ", "\u000b", "<", "123", "456", "78", ",", " ", " st"]} +{"text": "camelCasedåABCDž𐞁'VÉ'Mع😀🏽", "tokens": 20, "pieces": ["camel", "Casedå", "ABCDž𐞁'VE", "́'M", "ع", "😀🏽"]} +{"text": "Z\r\n😀🏽ABC'S .'Sddḍ̇'Res½'SiOS\nABC<|endoftext|>a/b12345678-İ \r\n\r\n👍🏽　ꟲ'T\"EOTé", "tokens": 52, "pieces": ["Z", "\r\n", "😀🏽", "ABC'S", " ", " .'", "Sddḍ̇'Re", "s", "", "½", "'Si", "OS", "\n", "ABC", "<|", "endoftext", "|>", "a", "/b", "123", "456", "78", "-İ", " \r\n\r\n", "👍🏽", "　ꟲ'T", "\"EOTé"]} +{"text": "\tİB'ſ😀🏽<|endoftext|>İHTTPServer\r\nt٣٤٥٦'VE's㍿
🙂\r\n​'Re\n", "İHTTPServer", "\r\n", "t", "٣٤٥", "٦", "'VE's", "㍿", "
", "🙂\r\n", "​'", "Re", "\n", "३字Džungla'S㋿'Re/…㍿tABCå'S<|fim_prefix|>३,́.aB'VE \n‍e​　𐞁", "tokens": 51, "pieces": ["", "३", "字Džungla'S", "㋿'", "Re", "/", "…", "㍿t", "ABC", "å'S", "<|", "fim", "_prefix", "|>", "३", ",́", ".a", "B'VE", " \n", "‍e", "​", "　𐞁"]} +{"text": " \tZaBcamelCaset'ſ,ſ(mßå/12345678㍿½
Z'red", "tokens": 30, "pieces": [" ", "\t", "Za", "Bcamel", "Caset'ſ", ",ſ", "(mßå", "/", "123", "456", "78", "㍿", "½", "
Z're", "d"]} +{"text": "'🙂camelCase<|fim_prefix|>", "tokens": 10, "pieces": ["'🙂", "camel", "Case", "<|", "fim", "_prefix", "|>"]} +{"text": "camelCaseḍ̇-ſ/\r\nåaB'ſ\u000b㍿㋿camelCase
'‍'.Z/\r\n-iOS#$%\t'S're㍿ḍ̇ fi‍\"", "tokens": 57, "pieces": ["camel", "Caseḍ̇", "-ſ", "/\r\n", "å", "a", "B'ſ", "\u000b", "㍿㋿", "camel", "Case", "
", "'‍'.", "Z", "/\r\n", "-<", "META", "_START", ">i", "OS", "#$%", "\t", "'S're", "㍿ḍ̇", " ", " fi", "‍\""]} +{"text": "aB's''  Ａ'/\r\nＡ'.", "tokens": 11, "pieces": ["a", "B's", "''", " ", " Ａ", "'/\r\n", "Ａ", "'."]} +{"text": "'ſ  'llå‍ſABC'S😀🏽12345678-㋿Bm /\r\n…AbB㋿\r\n\r\n​é　'Ś‍\rDžungla字å", "tokens": 48, "pieces": ["'ſ", " ", " ", "'llå", "‍ſ", "ABC'S", "😀🏽", "123", "456", "78", "-㋿", "Bm", " ", " /\r\n", "…Ab", "B", "㋿\r\n\r\n", "​é", "　", "'Ś", "‍\r", "Džungla字å"]} +{"text": "'S\r\n​‍ 'så<|endoftext|>٣٤٥٦å!!\t/\r\n
Ａ!ḍ̇scamelCase(éꟲé½/\r\nꟲß<|endoftext|>å🙂\n/#$%ß", "tokens": 60, "pieces": ["'S", "\r\n", "​‍", " ", " '", "så", "<|", "endoftext", "|>", "٣٤٥", "٦", "å", "!!", "\t", "/\r\n", "
Ａ", "!ḍ̇scamel", "Case", "(éꟲé", "½", "/\r\n", "ꟲß", "<|", "endoftext", "|>", "å", "🙂\n/", "#$%", "ß"]} +{"text": " \u000b/\r\n'll‍ſsß,३eéaB'ré'Sm㋿ /\r\n\n/Z😀🏽ſ\"iOS'㋿ᵃcamelCase𐞁 \n Ａa/bEOTB!", "tokens": 59, "pieces": [" ", "\u000b", "/\r\n", "'ll", "‍ſsß", ",", "३", "eéa", "B're", "́'S", "m", "㋿", " ", "/\r\n\n/", "Z", "😀🏽", "ſ", "\"i", "OS", "'㋿", "ᵃcamel", "Case𐞁", " \n", " Ａa", "/b", "EOTB", "!"]} +{"text": "\n\taB,s9d Ⅳ\nع\n🙂…ABC…é/\r\nm㍿tt'ſ9é­#$%a's<|fim_prefix|>åABCعꟲḍ̇𐞁", "tokens": 55, "pieces": ["\n", "\ta", "B", ",s", "9", "d", " ", "Ⅳ", "\n", "ع", "\n", "🙂", "…ABC", "…é", "/\r\n", "m", "㍿tt'ſ", "9", "é", "­#$%", "a's", "<|", "fim", "_prefix", "|>", "å", "ABCعꟲḍ̇𐞁"]} +{"text": "(iOS\r\n fi/'T字", "tokens": 15, "pieces": ["(i", "OS", "\r\n", "", " fi", "/'", "T字"]} +{"text": "𐞁'D'ſ🙂/­", "tokens": 17, "pieces": ["𐞁", "'", "D'ſ", "🙂<", "EOT", ">/­"]} +{"text": " åEOT camelCase­Ａ\r\nm\r\nd", "tokens": 14, "pieces": [" ", " å", "EOT", " ", " camel", "Case", "­Ａ", "\r\n", "m", "\r\n", "d"]} +{"text": "12345678'T'S,", "tokens": 6, "pieces": ["123", "456", "78", "'T'S", ","]} +{"text": "camelCase‍!'Re<|endoftext|>0Be('", "Re", "<|", "endoftext", "|>", "0", "Be", "(<", "m"]} +{"text": "İ'T \n/ABCm٣٤٥٦🙂ꟲ", "tokens": 14, "pieces": ["İ'T", " \n", "/ABCm", "٣٤٥", "٦", "🙂ꟲ"]} +{"text": "!0DžcamelCase٣٤٥٦a/bm 字a/b
ſ\r\n.'Ⅳ\r\n\r\n \na/b!İ\nＡ's'll<|fim_prefix|><|fim_prefix|>'Re'sꟲ<ꟲABCꟲEOT'D", "tokens": 64, "pieces": ["!", "0", "Džcamel", "Case", "٣٤٥", "٦", "a", "/bm", " ", " 字a", "/b", "", "
ſ", "\r\n", ".'", "Ⅳ", "\r\n\r\n \n", "a", "/b", "!İ", "\n", "Ａ's", "'ll", "<|", "fim", "_prefix", "|><|", "fim", "_prefix", "|>'", "Re's", "ꟲ", "<ꟲABCꟲ", "EOT'D"]} +{"text": " \n🙂!!", "tokens": 3, "pieces": [" \n", "🙂!!"]} +{"text": "m'VE 'll
.'Re<'ll m", "tokens": 16, "pieces": ["m'VE", " ", " '", "ll", "
", ".'", "Re", "<'", "ll", "", " ", " m"]} +{"text": "9d#$%Ⅳ!\t12345678𐞁\n/\r😀🏽12345678ع\ta,a/b'D!!漢#$%\"ſ", "tokens": 34, "pieces": ["9", "d", "#$%", "Ⅳ", "!", "\t", "123", "456", "78", "𐞁", "\n", "/\r", "😀🏽", "123", "456", "78", "ع", "\ta", ",a", "/b'D", "!!", "漢", "#$%\"", "ſ"]} +{"text": "e'M! \n\r\nſḍ̇'DDž́ABCABC Ａ'Ret'Sa's<ꟲݽs", "tokens": 29, "pieces": ["e'M", "!", " \n\r\n", "ſḍ̇'D", "Dž́", "ABCABC", " Ａ'Re", "t'S", "a's", "<ꟲ", "İ", "½", "s"]} +{"text": "<|endoftext|>　/-🙂<", "tokens": 11, "pieces": ["<|", "endoftext", "|>", "　", "/-🙂<"]} +{"text": "s\u000b", "tokens": 2, "pieces": ["s", "\u000b"]} +{"text": " \n 'D'reİ𐞁a/b <|fim_prefix|>½́dm<|endoftext|>字\n/é0fi字\u000bHTTPServer漢'T ­🙂/9å ᵃ\u000b", "tokens": 56, "pieces": [" \n", " '", "D're", "İ𐞁a", "/b", " ", "<|", "fim", "_prefix", "|>", "½", "́dm", "<|", "endoftext", "|>", "字", "\n", "/é", "0", "fi字", "\u000bHTTPServer漢'T", " ", " <", "EOT", ">­🙂/", "9", "å", " ᵃ", "\u000b"]} +{"text": "‍­", "tokens": 2, "pieces": ["‍­"]} +{"text": "iOS٣٤٥٦\r\n\r\n\r,\r \u000b's٣٤٥٦ßß­ \nABC漢😀🏽🙂'VE字0३a'sḍ̇\u000bHTTPServera३字\r\n\r\n<|endoftext|>
😀🏽٣٤٥٦'D㍿", "tokens": 63, "pieces": ["i", "OS", "٣٤٥", "٦", "\r\n\r\n\r", ",\r", " ", "\u000b", "'s", "٣٤٥", "٦", "ßß", "­", " \n", "ABC漢", "😀🏽🙂'", "VE字", "0३", "a's", "ḍ̇", "\u000bHTTPServera", "३", "字", "\r\n\r\n", "<|", "endoftext", "|>", "
", "😀🏽", "٣٤٥", "٦", "'D", "㍿"]} +{"text": "\n/<|fim_prefix|>EOT 'M'D٣٤٥٦EOT.Ａ'DⅣ\t!!< \n/d<0,👍🏽<|endoftext|>t-BaBZABC", "tokens": 47, "pieces": ["\n", "/<|", "fim", "_prefix", "|>", "EOT", " '", "M'D", "٣٤٥", "٦", "EOT", ".Ａ'D", "Ⅳ", "\t", "!!<", " \n", "/d", "<", "0", ",👍🏽<|", "endoftext", "|>", "t", "-Ba", "BZABC"]} +{"text": "EOTdsa/b🙂\r \n\r\n\r\n0'M", "tokens": 11, "pieces": ["EOTdsa", "/b", "🙂\r", " \n\r\n\r\n", "0", "'M"]} +{"text": "é​𐞁/\r\n-,fiＡ٣٤٥٦<|endoftext|>m​é\r>-d'Re'ſ>aBDž'\nABC", "tokens": 37, "pieces": ["é", "​𐞁", "/\r\n", "-,", "fi", "Ａ", "٣٤٥", "٦", "<|", "endoftext", "|>", "m", "​é", "\r", ">-", "d'Re", "'ſ", ">a", "BDž", "'\n", "ABC"]} +{"text": "ſ>'sḿ'rea👍🏽\"iOS12345678", "tokens": 15, "pieces": ["ſ", ">'", "sḿ're", "a", "👍🏽\"", "i", "OS", "123", "456", "78"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "ＡcamelCase", "tokens": 3, "pieces": ["Ａcamel", "Case"]} +{"text": "Zs😀🏽\r\nd-09 !\r0EOT'McamelCase\u000b­३\r\n'Re \n <|fim_prefix|>👍🏽é", "tokens": 37, "pieces": ["Zs", "😀🏽\r\n", "d", "-", "09", "", " ", " !\r", "0", "EOT'M", "camel", "Case", "\u000b", "­", "३", "\r\n", "'Re", " \n", " <|", "fim", "_prefix", "|>👍🏽", "é"]} +{"text": "Z9Ａfi٣٤٥٦\r…å३", "tokens": 14, "pieces": ["Z", "9", "Ａfi", "٣٤٥", "٦", "\r", "…å", "३"]} +{"text": "#$%　12345678'll ३'s<|fim_prefix|><|endoftext|>𐞁
'RefiHTTPServerſ \n Bİ​ ‍åſa'T12345678eå>'re
\u000b\n/ /\r\n", "tokens": 57, "pieces": ["#$%", "　", "123", "456", "78", "'ll", " ", "३", "'s", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "𐞁", "
", "'Refi", "HTTPServerſ", " \n", " Bİ", "​", " ", " ‍", "åſa'T", "123", "456", "78", "eå", ">'", "re", "
\u000b\n", "/", " ", " /\r\n"]} +{"text": "㍿ 'VE😀🏽½/>'ſ字m'T字Dž㋿ḍ̇'ſ!'ſ½İ　ḍ̇‍fi🙂tss\t👍🏽\"'!!", "tokens": 50, "pieces": ["㍿", " ", "'VE", "😀🏽", "½", "/>'", "ſ字m'T", "字", "Dž", "㋿ḍ̇'ſ", "!'", "ſ", "½", "İ", "　ḍ̇", "‍fi", "🙂tss", "\t", "👍🏽\"'!!"]} +{"text": "/a/b \"𐞁Bd''VE0Ⅳᵃᵃ'Mé'Sعm#$%'VE's👍🏽…m\n!ᵃ<|fim_prefix|>aB½/\r\n 'T,
", "tokens": 56, "pieces": ["/a", "/b", " \"", "𐞁Bd", "''", "VE", "0Ⅳ", "ᵃᵃ'M", "é'S", "عm", "#$%'", "VE's", "👍🏽", "…m", "\n", "!ᵃ", "<|", "fim", "_prefix", "|><", "EOT", ">a", "B", "½", "/\r\n", " '", "T", ",", "
"]} +{"text": "<|fim_prefix|>aa/b<😀🏽'M½/\r\n­\r'ſABC​é‍!!­\r\nHTTPServeŕ.Z漢s/ \n ", "tokens": 37, "pieces": ["<|", "fim", "_prefix", "|>", "aa", "/b", "<😀🏽'", "M", "½", "/\r\n", "­\r", "'ſ", "ABC", "​é", "‍!!­\r\n", "HTTPServeŕ", ".Z漢s", "/", " \n", " "]} +{"text": "0­٣٤٥٦Džungla𐞁'ſＡeABC're'VEع.(t'rea/b'ſ(…", "tokens": 33, "pieces": ["0", "­", "٣٤٥", "٦", "Džungla𐞁'ſ", "Ａe", "ABC're", "'VEع", ".(", "t're", "a", "/b'ſ", "(", "…"]} +{"text": "m iOS'TＡ12345678\u000b
'Re\r\n\r\ncamelCaseé٣٤٥٦DžDž0,́!!👍🏽å'D ½s
9s㋿ABCDž'ſAb(­", "tokens": 61, "pieces": ["m", " i", "OS'T", "Ａ", "123", "456", "78", "\u000b", "
", "'Re", "\r\n\r\n", "camel", "Caseé", "٣٤٥", "٦", "DžDž", "0", ",́", "!!👍🏽", "å", "<", "META", "_START", ">'", "D", " ", " ", "½", "s", "
", "9", "s", "㋿", "ABCDž'ſ", "Ab", "(­"]} +{"text": "é'­
#$%é\n/٣٤٥٦a dDžungla", "tokens": 21, "pieces": ["é", "'­", "
", "#$%", "é", "\n", "/", "٣٤٥", "٦", "a", " ", " d", "Džungla"]} +{"text": "‍#$%👍🏽 \nİ/ſ㋿<\r\n\r\n\r\nİ,éaBＡDžungla\r\n\r\nＡßABCaB\r\n('S'McamelCase…㋿😀🏽\t(9", "tokens": 48, "pieces": ["‍#$%👍🏽", " \n", "İ", "/ſ", "㋿<\r\n\r\n\r\n", "İ", ",éa", "BＡDžungla", "\r\n\r\n", "Ａß", "ABCa", "B", "\r\n", "('", "S'M", "camel", "Case", "…", "㋿😀🏽", "\t", "(", "9"]} +{"text": "\ré\r\nＡ /\r'TEOT!0\"'VEꟲḍ̇㍿​‍㋿fi'T😀🏽.HTTPServer", "tokens": 38, "pieces": ["\r", "é", "\r\n", "Ａ", " /\r", "'TEOT", "!", "0", "\"'", "VEꟲḍ̇", "㍿​‍㋿", "fi'T", "😀🏽.", "HTTPServer"]} +{"text": "'Re", "tokens": 1, "pieces": ["'Re"]} +{"text": "DžunglafiABC漢　''M
\r\n\r\nfia/bᵃé\"'Re…٣٤٥٦/\r\nDž​'s!!m,B", "tokens": 36, "pieces": ["Džunglafi", "ABC漢", "　", "''", "M", "
\r\n\r\n", "fia", "/bᵃé", "\"'", "Re", "…", "٣٤٥", "٦", "/\r\n", "Dž", "​'", "s", "!!", "m", ",B"]} +{"text": "漢éßtiOS½\n/t/Ａ<|endoftext|>​é.\t \n Dž㋿.>'S㋿
> Ａ\tåaB>'Dfi9", "tokens": 46, "pieces": ["漢éßti", "OS", "½", "\n", "/t", "/Ａ", "<|", "endoftext", "|>​", "é", ".", "\t \n", " Dž", "㋿.>'", "S", "㋿", "
", ">", " Ａ", "\tåa", "B", ">'", "Dfi", "9"]} +{"text": "iOS're(Ａ<|endoftext|>sEOT　 \u000b😀🏽.'Reſ \n­㍿#$%Ⅳİꟲ!<|endoftext|>½ß漢HTTPServer!!<|endoftext|>12345678é\u000b/<|endoftext|>", "tokens": 73, "pieces": ["i", "OS're", "(Ａ", "<|", "endoftext", "|>", "s", "EOT", "　 ", "\u000b", "😀🏽.'", "Reſ", " \n", "­㍿#$%", "Ⅳ", "İꟲ", "!<|", "endoftext", "|>", "½", "ß漢", "HTTPServer", "!!<|", "endoftext", "|>", "123", "456", "78", "é", "\u000b", "/<|", "endoftext", "|>"]} +{"text": "ABCDžungla३…\n", "tokens": 9, "pieces": ["ABCDžungla", "३", "…\n"]} +{"text": "t\u000b'll!camelCase…EOT,m'S/\r\nＡ \n ABC'TeⅣ/​漢s'Mİ-👍🏽‍'TcamelCase", "tokens": 42, "pieces": ["t", "\u000b", "'ll", "!camel", "Case", "…EOT", ",m'S", "/\r\n", "Ａ", " \n", " ABC'T", "e", "Ⅳ", "/​<", "META", "_START", "><", "META", "_START", ">漢s'M", "İ", "-👍🏽‍'", "Tcamel", "Case"]} +{"text": "…㍿Ⅳ,Džİ \ncamelCase'!!0́½ß \né\r<　\n/", "tokens": 27, "pieces": ["…", "㍿", "Ⅳ", ",Džİ", " \n", "camel", "Case", "'!!", "0", "́", "½", "ß", " \n", "é", "\r", "<", "　\n", "/"]} +{"text": "12345678ſ!ABC0aBsdEOTHTTPServerＡ　Ab-漢½/må", "tokens": 23, "pieces": ["123", "456", "78", "ſ", "!ABC", "0", "a", "Bsd", "EOTHTTPServer", "Ａ", "　Ab", "-漢", "½", "/må"]} +{"text": "e", "tokens": 1, "pieces": ["e"]} +{"text": "", "tokens": 0, "pieces": []} +{"text": "aB", "tokens": 2, "pieces": ["a", "B"]} +{"text": "漢 \n \"
Ａ½ḍ̇<|fim_prefix|><|endoftext|>Džع9", "tokens": 28, "pieces": ["漢", "", " \n", " \"", "
Ａ", "½", "ḍ̇", "<|", "fim", "_prefix", "|><|", "endoftext", "|>", "Džع", "9"]} +{"text": "ꟲ", "tokens": 3, "pieces": ["ꟲ"]} +{"text": "ḍ̇", "tokens": 3, "pieces": ["ḍ̇"]} +{"text": "e३'s'Re,é,at🙂/\r\n½iOS'VEfi-🙂aB\r\nſDžungla \n㋿((", "tokens": 34, "pieces": ["e", "३", "'s'Re", ",é", ",at", "🙂/\r\n", "½", "i", "OS'VE", "fi", "-🙂", "a", "B", "\r\n", "ſ", "Džungla", " \n", "㋿<", "EOT", ">(("]} +{"text": "ع​/\r\n \n İ", "tokens": 5, "pieces": ["ع", "​/\r\n", " \n", " İ"]} +{"text": " 𐞁😀🏽​", "tokens": 10, "pieces": [" ", " 𐞁", "😀🏽​"]} +{"text": " ‍‍#$%\tétaB''M", "tokens": 10, "pieces": [" ", " ‍‍#$%", "\téta", "B", "''", "M"]} +{"text": "9㍿es", "tokens": 5, "pieces": ["9", "㍿es"]} +{"text": "\u000b9'Re­\t''ſ'llḍ̇m\u000bDžunglaḍ̇ \n a", "tokens": 23, "pieces": ["\u000b", "9", "'Re", "­", "\t", "''", "ſ'll", "ḍ̇m", "\u000bDžunglaḍ̇", " \n", " ", " a"]} +{"text": "camelCase9Dž12345678camelCaseAb'sAbABCcamelCase'/\r\nDžungla!!>Dž­e…Ａ", "tokens": 31, "pieces": ["camel", "Case", "9", "Dž", "123", "456", "78", "camel", "Case", "Ab's", "Ab", "ABCcamel", "Case", "'/\r\n", "Džungla", "!!>", "Dž", "­e", "…Ａ"]} +{"text": "İ'VE,", "tokens": 4, "pieces": ["İ'VE", ","]} +{"text": "aB🙂…ABCع'ſ'DHTTPServer😀🏽as#$%\r'VE'a/b٣٤٥٦३d <|endoftext|>㍿#$%'Re's 😀🏽", "tokens": 47, "pieces": ["a", "B", "🙂", "…ABCع'ſ", "'DHTTPServer", "😀🏽", "as", "#$%\r", "'VE", "'a", "/b", "٣٤٥", "٦३", "d", " <|", "endoftext", "|>㍿#$%'", "Re's", " ", " 😀🏽"]} +{"text": "aB ­ ㍿ét,#$%㍿s's\r\nḍ̇ß\r\n!9ᵃ", "tokens": 27, "pieces": ["a", "B", " ­", " ", "㍿ét", ",#$%㍿", "s's", "\r\n", "ḍ̇ß", "\r\n", "!", "9", "ᵃ"]} +{"text": "\n/åéHTTPServer", "tokens": 6, "pieces": ["\n", "/åé", "HTTPServer"]} +{"text": "\ré字\n", "tokens": 4, "pieces": ["\r", "é字", "\n"]} +{"text": "EOTİ(́HTTPServer'Re>　a
(\r\nfi 
/ iOS'D", "tokens": 21, "pieces": ["EOTİ", "(́HTTPServer'Re", ">", "　a", "
", "(\r\n", "fi", " ", "
", "/", " i", "OS'D"]} +{"text": "EOT- \r\n\r\n\nt'MB​ſ'Reaé-
㍿Džungla. 0㍿é", "tokens": 31, "pieces": ["EOT", "-", " \r\n\r\n\n", "t'M", "B", "​ſ'Re", "aé", "-", "
", "㍿Džungla", ".", " ", " ", "0", "㍿é"]} +{"text": "a12345678<|endoftext|>!!édDž\rAb \nB#$%　𐞁🙂𐞁#$%'ll0'ſ'D🙂'Re123456780👍🏽're \n", "tokens": 50, "pieces": ["a", "123", "456", "78", "<|", "endoftext", "|>!!", "éd", "Dž", "\r", "Ab", " \n", "B", "#$%", "　𐞁", "🙂𐞁", "#$%'", "ll", "0", "'ſ'D", "🙂'", "Re", "123", "456", "780", "👍🏽'", "re", " \n"]} +{"text": "\r>ABC٣٤٥٦\"٣٤٥٦½　'sⅣEOT0­­ \n a(/\r\n🙂漢.-　DžunglaB
 \n عß👍🏽ع/'s", "tokens": 50, "pieces": ["\r", "><", "META", "_START", ">ABC", "٣٤٥", "٦", "\"", "٣٤٥", "٦½", "　", "'s", "Ⅳ", "EOT", "0", "­­", " \n", " a", "(/\r\n", "🙂漢", ".-", "　Džungla", "B", "
 \n", " عß", "👍🏽", "ع", "/'", "s"]} +{"text": " 👍🏽
‍'ll.Džungla'ReABC12345678𐞁­Z\n/‍​'Re𐞁'M,", "tokens": 39, "pieces": [" ", " 👍🏽", "
", "‍'", "ll", ".Džungla'Re", "ABC", "123", "456", "78", "𐞁", "­Z", "\n", "/‍​<", "META", "_START", ">'", "Re𐞁'M", ","]} +{"text": "éⅣ0<|fim_prefix|>'fitdＡtfi'reİ🙂a/bfiſ㋿\r\n\r\n😀🏽ß
字 ß<'M", "tokens": 39, "pieces": ["é", "Ⅳ0", "<|", "fim", "_prefix", "|>'", "fitd", "Ａtfi're", "İ", "🙂a", "/bfiſ", "㋿\r\n\r\n", "😀🏽", "ß", "
字", " <", "META", "_START", ">ß", "<'", "M"]} +{"text": "字　\ndſ,　­(½\n<|fim_prefix|>ABC\"\r 'VE३'re'TiOS/½\"'ll३,'(>", "tokens": 38, "pieces": ["字", "　\n", "dſ", ",", "　", "­(", "½", "\n", "<|", "fim", "_prefix", "|>", "ABC", "\"\r", " '", "VE", "३", "'re'T", "i", "OS", "/", "½", "\"'", "ll", "३", ",'(>"]} +{"text": "\naZᵃ12345678'D9< \n aBᵃⅣiOSḍ̇", "tokens": 25, "pieces": ["\n", "a", "Zᵃ", "123", "456", "78", "'D", "9", "<", " \n", " a", "Bᵃ", "Ⅳ", "i", "OSḍ̇"]} +{"text": "Ab \n ́‍/\reß/\r\ne 'Re/½\r'TⅣEOT", "tokens": 21, "pieces": ["Ab", " \n", " ́", "‍/\r", "eß", "/\r\n", "e", " '", "Re", "/", "½", "\r", "'T", "Ⅳ", "EOT"]} +{"text": "ſ/ HTTPServerꟲ", "tokens": 8, "pieces": ["ſ", "/", " HTTPServerꟲ"]} +{"text": "ABCB'Re\n(ᵃ㋿\r\n 🙂/\r\n.!!'s'ſ́éiOSZ\n0", "tokens": 27, "pieces": ["ABCB'Re", "\n", "(ᵃ", "㋿\r\n", " ", " 🙂/\r\n", ".!!'", "s'ſ", "́éi", "OSZ", "\n", "0"]} diff --git a/litellm-rust/crates/token-counter/tests/token_counter.rs b/litellm-rust/crates/token-counter/tests/token_counter.rs index 0e71a5c03cf..12c59768952 100644 --- a/litellm-rust/crates/token-counter/tests/token_counter.rs +++ b/litellm-rust/crates/token-counter/tests/token_counter.rs @@ -1,4 +1,5 @@ use rstest::rstest; +use serde::Deserialize; use litellm_token_counter::{CountableRequest, Error, InputTokenCount, TokenCounter}; @@ -174,3 +175,147 @@ fn tool_choice_and_system_discount_change_the_count() { fn loading_a_bad_tokenizer_is_a_load_error() { assert!(matches!(TokenCounter::from_json("{}"), Err(Error::Load(_)))); } + +/// A tiktoken encoding: its fixture directory, the vendored rank file Python +/// loads, the constructor, and the model `generate.py` counted the requests for. +#[derive(Clone, Copy)] +struct TiktokenEncoding { + fixtures: &'static str, + rank_file: &'static str, + load: fn(&str) -> Result, + model: &'static str, +} + +const CL100K: TiktokenEncoding = TiktokenEncoding { + fixtures: "cl100k", + rank_file: "9b5ad71b2ce5302211f9c61530b329a4922fc6a4", + load: TokenCounter::from_cl100k_ranks, + model: "gpt-4", +}; + +const O200K: TiktokenEncoding = TiktokenEncoding { + fixtures: "o200k", + rank_file: "fb374d419588a4632f3f557e76b4b70aebbca790", + load: TokenCounter::from_o200k_ranks, + model: "gpt-4o", +}; + +fn tiktoken_counter(encoding: TiktokenEncoding) -> TokenCounter { + let path = format!( + "{}/../../../litellm/litellm_core_utils/tokenizers/{}", + env!("CARGO_MANIFEST_DIR"), + encoding.rank_file + ); + let ranks = std::fs::read_to_string(&path).expect("rank file is in the repo"); + (encoding.load)(&ranks).expect("ranks load") +} + +fn tiktoken_fixture(encoding: TiktokenEncoding, name: &str) -> String { + let path = format!( + "{}/tests/fixtures/{}/{name}", + env!("CARGO_MANIFEST_DIR"), + encoding.fixtures + ); + std::fs::read_to_string(&path).expect("fixture generated by tests/fixtures/generate.py") +} + +#[derive(Deserialize)] +struct TextFixture { + text: String, + tokens: usize, +} + +#[derive(Deserialize)] +struct RequestFixture { + body: String, + input_tokens: usize, +} + +/// Reference counts come from `tiktoken.get_encoding(name)`; see +/// `tests/fixtures/generate.py`. +#[rstest] +#[case::cl100k(CL100K)] +#[case::o200k(O200K)] +fn tiktoken_text_counts_match_tiktoken(#[case] encoding: TiktokenEncoding) { + let counter = tiktoken_counter(encoding); + let fixtures: Vec = tiktoken_fixture(encoding, "texts.jsonl") + .lines() + .map(|line| serde_json::from_str(line).expect("fixture line is json")) + .collect(); + assert!(fixtures.len() > 3000); + let mismatches: Vec<_> = fixtures + .iter() + .filter_map(|fixture| { + let count = counter.count_text(&fixture.text).expect("text counts"); + (count != fixture.tokens).then(|| (fixture.text.clone(), fixture.tokens, count)) + }) + .collect(); + assert!( + mismatches.is_empty(), + "(text, tiktoken, rust): {mismatches:?}" + ); +} + +/// Reference counts come from the proxy's admission counter +/// (`_count_input_tokens(body, model)`), so this pins the shared message, +/// tool and reply-priming accounting on the tiktoken paths as well. +#[rstest] +#[case::cl100k(CL100K)] +#[case::o200k(O200K)] +fn tiktoken_request_counts_match_python_admission_counter(#[case] encoding: TiktokenEncoding) { + let counter = tiktoken_counter(encoding); + let fixtures: Vec = tiktoken_fixture(encoding, "requests.jsonl") + .lines() + .map(|line| serde_json::from_str(line).expect("fixture line is json")) + .collect(); + let counts: Vec = fixtures + .iter() + .map(|fixture| { + let request = CountableRequest::parse(fixture.body.as_bytes()).expect("fixture parses"); + let count = counter.count_request(&request).expect("fixture counts"); + assert_eq!(count.model.as_deref(), Some(encoding.model)); + assert_eq!(count.input_tokens, fixture.input_tokens, "{}", fixture.body); + count.input_tokens + }) + .collect(); + assert!(counts.last().is_some_and(|tokens| *tokens >= 50_000)); +} + +#[rstest] +#[case::cl100k(CL100K)] +#[case::o200k(O200K)] +fn tiktoken_shares_the_message_accounting_with_the_anthropic_path( + #[case] encoding: TiktokenEncoding, +) { + let counter = tiktoken_counter(encoding); + let count = |body: &str| { + counter + .count_request(&CountableRequest::parse(body.as_bytes()).expect("parses")) + .expect("counts") + .input_tokens + }; + let text = |text: &str| counter.count_text(text).expect("counts"); + let base = count(r#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#); + assert_eq!(base, 3 + text("user") + text("hi") + 3); + assert_eq!( + count(r#"{"model":"m","messages":[{"role":"user","name":"al","content":"hi"}]}"#), + base + text("al") + 1 + ); + assert_eq!( + count(r#"{"model":"m","messages":[{"role":"user","content":"hi"}],"tool_choice":"none"}"#), + base + 1 + ); +} + +#[rstest] +#[case::empty("")] +#[case::not_base64("!!!! 0")] +#[case::missing_rank("YQ==")] +#[case::rank_not_a_number("YQ== x")] +#[case::single_byte_tokens_missing("YWI= 0")] +fn loading_a_bad_rank_file_is_a_load_error( + #[case] rank_file: &str, + #[values(CL100K, O200K)] encoding: TiktokenEncoding, +) { + assert!(matches!((encoding.load)(rank_file), Err(Error::Ranks(_)))); +} diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 4058a3d72dd..56c9147e066 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -13,6 +13,7 @@ import json import sys import threading import time +from collections.abc import Callable from typing import TYPE_CHECKING, Any, Final if TYPE_CHECKING: @@ -34,6 +35,7 @@ class InMemoryCache(BaseCache): default_ttl: int | None = 600, # default ttl is 10 minutes. At maximum litellm rate limiting logic requires objects to be in memory for 1 minute max_size_per_item: int | None = 1024, # 1MB = 1024KB + clock: Callable[[], float] | None = None, ): """ max_size_in_memory [int]: Maximum number of items in cache. done to prevent memory leaks. Use 200 items as a default @@ -49,6 +51,7 @@ class InMemoryCache(BaseCache): self.ttl_dict: dict = {} self.expiration_heap: list[tuple[float, str]] = [] self._increment_lock = threading.Lock() + self._clock = clock if clock is not None else lambda: time.time() def check_value_size(self, value: Any): """ @@ -91,7 +94,7 @@ class InMemoryCache(BaseCache): """ Check if a specific key is expired """ - return key in self.ttl_dict and time.time() > self.ttl_dict[key] + return key in self.ttl_dict and self._clock() > self.ttl_dict[key] def _remove_key(self, key: str) -> None: """ @@ -113,7 +116,7 @@ class InMemoryCache(BaseCache): - 3. the size of in-memory cache is bounded """ - current_time: Final = time.time() + current_time: Final = self._clock() # Step 1: Remove expired or outdated items while self.expiration_heap: @@ -147,7 +150,7 @@ class InMemoryCache(BaseCache): Check if ttl is set for a key """ ttl_time: Final = self.ttl_dict.get(key) - if ttl_time is None or float(ttl_time) < time.time(): # if ttl is not set, allow override + if ttl_time is None or float(ttl_time) < self._clock(): # if ttl is not set, allow override return True else: return False @@ -167,10 +170,10 @@ class InMemoryCache(BaseCache): self.cache_dict[key] = value if self.allow_ttl_override(key): # if ttl is not set, set it to default ttl if "ttl" in kwargs and kwargs["ttl"] is not None: - self.ttl_dict[key] = time.time() + float(kwargs["ttl"]) + self.ttl_dict[key] = self._clock() + float(kwargs["ttl"]) heapq.heappush(self.expiration_heap, (self.ttl_dict[key], key)) else: - self.ttl_dict[key] = time.time() + self.default_ttl + self.ttl_dict[key] = self._clock() + self.default_ttl heapq.heappush(self.expiration_heap, (self.ttl_dict[key], key)) async def async_set_cache(self, key, value, **kwargs): diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 6cecc1e6157..ee01a53ecb3 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -4,6 +4,8 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers. import asyncio import base64 +import hashlib +import json import os from collections.abc import Awaitable, Callable, Generator from contextlib import AbstractAsyncContextManager @@ -343,6 +345,22 @@ class MCPClient: if auth_value: self.update_auth_value(auth_value) + async def discovery_auth_fingerprint(self) -> str: + request: Final = httpx.Request("POST", self.server_url or "http://localhost/", headers=self._get_auth_headers()) + if self._resolved_auth is None: + return self._hash_discovery_auth(request) + flow: Final = self._resolved_auth.async_auth_flow(request) + try: + authenticated: Final = await flow.__anext__() + return self._hash_discovery_auth(authenticated) + finally: + await flow.aclose() + + @staticmethod + def _hash_discovery_auth(request: httpx.Request) -> str: + material: Final = json.dumps((str(request.url), tuple(sorted(request.headers.multi_items())))) + return hashlib.sha256(material.encode()).hexdigest() + def _create_transport_context( self, ) -> tuple[_TransportContext, httpx.AsyncClient | None]: @@ -781,7 +799,7 @@ class MCPClient: # Return a default error result instead of raising return self.error_tool_result(e) - async def list_prompts(self) -> list[Prompt]: + async def list_prompts(self, *, raise_on_error: bool = False) -> list[Prompt]: """List available prompts from the server.""" verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") @@ -811,6 +829,8 @@ class MCPClient: verbose_logger.warning("MCP client list_prompts was cancelled") raise except Exception as e: + if raise_on_error: + raise error_type: Final = type(e).__name__ verbose_logger.error( "MCP client list_prompts failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", @@ -869,7 +889,7 @@ class MCPClient: ) raise - async def list_resources(self) -> list[Resource]: + async def list_resources(self, *, raise_on_error: bool = False) -> list[Resource]: """List available resources from the server.""" verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio") @@ -899,6 +919,8 @@ class MCPClient: verbose_logger.warning("MCP client list_resources was cancelled") raise except Exception as e: + if raise_on_error: + raise error_type: Final = type(e).__name__ verbose_logger.error( "MCP client list_resources failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", @@ -916,7 +938,7 @@ class MCPClient: # Return empty list instead of raising to allow graceful degradation return [] - async def list_resource_templates(self) -> list[ResourceTemplate]: + async def list_resource_templates(self, *, raise_on_error: bool = False) -> list[ResourceTemplate]: """List available resource templates from the server.""" verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio") @@ -949,6 +971,8 @@ class MCPClient: verbose_logger.warning("MCP client list_resource_templates was cancelled") raise except Exception as e: + if raise_on_error: + raise error_type: Final = type(e).__name__ verbose_logger.error( "MCP client list_resource_templates failed - Error Type: %s, Error: %s, Server: %s, Transport: %s", diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index dc41c7dadc8..caac8e888fd 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -673,7 +673,7 @@ class SlackAlerting(CustomBatchLogger): Create a standard message for a budget alert """ _all_fields_as_dict: Final[dict[str, object]] = user_info.model_dump(exclude_none=True) - _all_fields_as_dict.pop("token") + _all_fields_as_dict.pop("token", None) msg = "" for k, v in _all_fields_as_dict.items(): if isinstance(v, Litellm_EntityType): diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 77bf4820a1a..8a976a966a6 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1130,7 +1130,7 @@ class CustomGuardrail(CustomLogger): def add_standard_logging_guardrail_information_to_request_data( self, - guardrail_json_response: Exception | str | dict | list[dict], + guardrail_json_response: object, request_data: dict, guardrail_status: GuardrailStatus, start_time: float | None = None, @@ -1217,6 +1217,7 @@ class CustomGuardrail(CustomLogger): _, metadata_bucket = get_or_create_metadata_bucket(request_data) _append_guardrail_info(metadata_bucket) + _sync_guardrail_info_to_logging_obj(request_data, request_data.get("litellm_logging_obj")) _guardrail_self_recorded.set(True) @@ -1275,17 +1276,10 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ - # Convert None to empty dict to satisfy type requirements - guardrail_response: dict[str, object] | str = {} if response is None else response - - # For apply_guardrail functions in custom_code_guardrail scenario, - # simplify the logged response to "allow", "deny", or "mask" - if original_inputs is not None and isinstance(response, dict): - # Check if inputs were modified by comparing them - if self._inputs_were_modified(original_inputs, response): - guardrail_response = "mask" - else: - guardrail_response = "allow" + guardrail_response: Final = self._summarize_guardrail_response( + response=response, + original_inputs=original_inputs, + ) verbose_logger.debug("Guardrail response: %s", response) @@ -1300,6 +1294,27 @@ class CustomGuardrail(CustomLogger): ) return response + def _summarize_guardrail_response( + self, + response: object, + original_inputs: Mapping[str, object] | None, + ) -> object: + """Reduce a hook's return value to what is safe to log as ``guardrail_response``. + + ``apply_guardrail`` returns the (possibly masked) inputs and ``async_pre_call_hook`` + returns the (possibly modified) request payload. Neither is a provider verdict, and + logging them verbatim ships the user's prompt to every logging sink (OTEL spans, + Datadog, spend logs), so both collapse to ``"allow"`` / ``"mask"`` by comparing + against ``original_inputs``, a copy taken before the hook ran. A string result is the + hook's own rejection message (the proxy turns it into a 400), not user input, so it is + logged as is. + """ + if response is None: + return {} + if original_inputs is None or not isinstance(response, Mapping): + return response + return "mask" if self._inputs_were_modified(original_inputs, response) else "allow" + @staticmethod def _is_guardrail_intervention(e: Exception) -> bool: """Retained spelling for existing callers; prefer ``is_guardrail_intervention``.""" @@ -1339,24 +1354,9 @@ class CustomGuardrail(CustomLogger): ) raise e - def _inputs_were_modified(self, original_inputs: dict, response: dict) -> bool: - """ - Compare original inputs with response to determine if content was modified. - - Returns True if the inputs were modified (mask scenario), False otherwise (allow scenario). - """ - # Get all keys from both dictionaries - all_keys: Final = set(original_inputs.keys()) | set(response.keys()) - - # Compare each key's value - for key in all_keys: - original_value = original_inputs.get(key) - response_value = response.get(key) - if original_value != response_value: - return True - - # No modifications detected - return False + def _inputs_were_modified(self, original_inputs: Mapping[str, object], response: Mapping[str, object]) -> bool: + """True when any baseline key's value differs in ``response`` (mask), False otherwise (allow).""" + return any(response.get(key) != value for key, value in original_inputs.items()) def mask_content_in_string( self, @@ -1463,6 +1463,31 @@ def _sync_guardrail_info_to_logging_obj(request_data: dict, logging_obj: object) _append_slg_to_litellm_params(mcd.get("litellm_params"), entries) +_PRE_CALL_CONTENT_KEYS: Final = frozenset( + {"messages", "input", "prompt", "system", "instructions", "tools", "functions", "function_call", "tool_choice"} +) + + +def _original_inputs_for( + func_name: str, + kwargs: Mapping[str, object], + request_data: Mapping[str, object], + event_type: GuardrailEventHooks | None, +) -> dict | None: # mutable-ok: matches _process_response(original_inputs=) signature + """Baseline the hook's return value is compared against to decide "allow" vs "mask". + + ``apply_guardrail`` masks a fresh ``inputs`` dict, so that dict is the baseline. Pre-call + hooks edit the request in place and return it, so the baseline is a deep copy of the + prompt-bearing keys taken before the hook runs. + """ + if func_name == "apply_guardrail": + inputs: Final = kwargs.get("inputs") + return inputs if isinstance(inputs, dict) else None + if event_type != GuardrailEventHooks.pre_call: + return None + return {key: copy.deepcopy(value) for key, value in request_data.items() if key in _PRE_CALL_CONTENT_KEYS} + + def log_guardrail_information(func): """ Decorator to add standard logging guardrail information to any function @@ -1521,9 +1546,7 @@ def log_guardrail_information(func): event_type: Final = _infer_event_type_from_function_name(func.__name__) # Store original inputs for comparison (for apply_guardrail functions) - original_inputs = None - if func.__name__ == "apply_guardrail" and "inputs" in kwargs: - original_inputs = kwargs.get("inputs") + original_inputs: Final = _original_inputs_for(func.__name__, kwargs, request_data, event_type) logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj") self_recorded_token: Final = _guardrail_self_recorded.set(False) @@ -1563,9 +1586,7 @@ def log_guardrail_information(func): event_type: Final = _infer_event_type_from_function_name(func.__name__) # Store original inputs for comparison (for apply_guardrail functions) - original_inputs = None - if func.__name__ == "apply_guardrail" and "inputs" in kwargs: - original_inputs = kwargs.get("inputs") + original_inputs: Final = _original_inputs_for(func.__name__, kwargs, request_data, event_type) logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj") self_recorded_token: Final = _guardrail_self_recorded.set(False) diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index ed47533e700..f8fd417392f 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -3,7 +3,12 @@ from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger from litellm.integrations.otel.logger import OpenTelemetryV2 -from litellm.integrations.otel.mappers.langfuse import LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT +from litellm.integrations.otel.mappers.langfuse import ( + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_TRACE_NAME, +) +from litellm.integrations.otel.model.metadata import caller_trace_name from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output from litellm.integrations.otel.plumbing.context import request_root_span @@ -13,6 +18,18 @@ if TYPE_CHECKING: class LangfuseOpenTelemetryV2(OpenTelemetryV2): + """Names the trace from the request. Langfuse reads ``langfuse.trace.name`` off the root observation, + and the proxy's root span is still recording when the LLM call starts.""" + + def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: + root: Final = request_root_span() + name: Final = caller_trace_name(kwargs) + if root is not None and root.is_recording() and name is not None: + root.set_attribute(LANGFUSE_TRACE_NAME, name) + super().log_pre_api_call(model, messages, kwargs) + + +class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): """Stamps the request's input and output on the root observation while it is still recording. Langfuse shows a trace's input and output from its root observation. The proxy's root span ends diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 630aa313dc9..9ac748b231c 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -554,6 +554,7 @@ class OpenTelemetryV2(CustomLogger): capture_content=self.config.capture_span_content, time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, request_route=request_root_http_route(), + trace_name=call.trace_name, ) end_time_ns: Final = to_ns(end_time) if carrier is not None and carrier.span is not None: @@ -984,8 +985,8 @@ def build_otel_v2_logger( def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: - if "langfuse" not in config.mapper_names or not config.capture_span_content: + if "langfuse" not in config.mapper_names: return OpenTelemetryV2 - from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2, LangfuseOpenTelemetryV2 - return LangfuseOpenTelemetryV2 + return LangfuseContentOpenTelemetryV2 if config.capture_span_content else LangfuseOpenTelemetryV2 diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 01063d85355..98ff0f155a1 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -28,6 +28,7 @@ from litellm.integrations.otel.model.payloads import ( LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" +LANGFUSE_TRACE_NAME: Final = "langfuse.trace.name" class LangfuseMapper: @@ -36,6 +37,7 @@ class LangfuseMapper: "langfuse.observation.model.name": lambda d: d.request_model or None, "langfuse.observation.metadata.provider": lambda d: d.provider or None, "langfuse.observation.id": lambda d: d.identity.call_id or None, + LANGFUSE_TRACE_NAME: lambda d: d.trace_name or None, "langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None, "langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None, } diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index ee116aca46b..cc81b689708 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -48,6 +48,8 @@ from litellm.integrations.otel.model.utils import as_str, to_seconds if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload +LANGFUSE_TRACE_NAME_HEADER: Final = "langfuse_trace_name" + @dataclass(frozen=True) class RequestIdentity: @@ -215,6 +217,7 @@ class LLMCallEvent: # needs to be reasonable for a span that never gets closed (a leak). provisional_span_name: str time_to_first_chunk_seconds: float | None + trace_name: str | None @classmethod def from_dict(cls, kwargs: Mapping[str, Any]) -> LLMCallEvent: @@ -231,9 +234,30 @@ class LLMCallEvent: upstream_started=kwargs.get("api_call_start_time") is not None, provisional_span_name=f"{operation.value} {model}".strip(), time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs), + trace_name=caller_trace_name(kwargs), ) +def caller_trace_name(kwargs: Mapping[str, object]) -> str | None: + request: Final = _as_str_mapping(kwargs.get("litellm_params")) + if request is None: + return None + proxy_request: Final = _as_str_mapping(request.get("proxy_server_request")) + headers: Final = _as_str_mapping(proxy_request.get("headers")) if proxy_request is not None else None + from_header: Final = as_str(headers.get(LANGFUSE_TRACE_NAME_HEADER)) if headers is not None else None + if from_header: + return from_header + return next( + ( + name + for key in ("metadata", "litellm_metadata") + if (metadata := _as_str_mapping(request.get(key))) is not None + and (name := as_str(metadata.get("trace_name"))) + ), + None, + ) + + def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: """Seconds from the upstream request being issued (``api_call_start_time``) to the first streamed chunk (``completion_start_time``); ``None`` for diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d0959a6c2e9..c11c4a7a27d 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -387,6 +387,7 @@ class LLMCallSpanData: output_type: GenAIOutputType | None = None call_type: str | None = None request_route: str | None = None + trace_name: str | None = None @classmethod def from_standard_logging_payload( @@ -395,6 +396,7 @@ class LLMCallSpanData: capture_content: bool = False, time_to_first_chunk_seconds: float | None = None, request_route: str | None = None, + trace_name: str | None = None, ) -> LLMCallSpanData: params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {}) # The single parse of the request's metadata — the request-vs-provider @@ -436,6 +438,7 @@ class LLMCallSpanData: output_type=resolve_output_type(call_type), call_type=call_type or None, request_route=request_route or context.identity.request_route, + trace_name=trace_name, ) diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 54b116639e7..cdc108a6b4e 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -27,6 +27,7 @@ from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.websearch_interception.tools import is_web_search_tool_responses from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata from litellm.litellm_core_utils.llm_judge import ( @@ -335,6 +336,16 @@ def _forwards_nothing(value: object) -> bool: return value is None or (isinstance(value, list) and len(value) == 0) +def _request_has_hosted_web_search(request: Mapping[str, object]) -> bool: + if request.get("web_search_options") is not None: + return True + tools: Final = request.get("tools") + return isinstance(tools, Sequence) and any( + isinstance(tool, Mapping) and tool.get("type") != "function" and is_web_search_tool_responses(tool) + for tool in tools + ) + + def _judgeable_sample( ops: _SurfaceOps, kwargs: Mapping[str, object], @@ -343,9 +354,14 @@ def _judgeable_sample( ) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object], str] | None: """The normalized chat conversation, the forwardable generation params, and the judgeable final text; None when this request's shapes cannot be sampled (no text and no - tool call to serialize, or a shape the owner transformations reject).""" + tool call to serialize, hosted web search the shadow cannot replay comparably, + or a shape the owner transformations reject).""" + if _request_has_hosted_web_search(_proxy_wire_body(kwargs) if ops.wire_params else model_parameters): + return None try: request: Final = ops.chat_request(kwargs, model_parameters) + if _request_has_hosted_web_search(request): + return None items: Final = _MESSAGE_ITEMS_ADAPTER.validate_python(request.get("messages")) messages: Final = _CHAT_MESSAGES_ADAPTER.validate_python( tuple(m.model_dump(exclude_none=True) if isinstance(m, BaseModel) else m for m in items) diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index c3b6a008411..71b30614d8d 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import Final import litellm @@ -14,6 +15,20 @@ except (ImportError, AttributeError): filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") +CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4" +O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790" + + +def cl100k_base_rank_file() -> str: + """The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines).""" + return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii") + + +def o200k_base_rank_file() -> str: + """The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines).""" + return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") + + # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR. # This keeps tiktoken fully offline-capable by default (see #1071). diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 75046f2cf87..4923bdda305 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -90,12 +90,12 @@ class _ResponseDoneBody(TypedDict, total=False): output: ReadOnly[Sequence[Mapping[str, object]]] -class _ScopedWebSocket(Protocol): +class ScopedWebSocket(Protocol): @property def scope(self) -> _ASGIScope: ... -class _ClientWebSocket(_ScopedWebSocket, Protocol): +class _ClientWebSocket(ScopedWebSocket, Protocol): async def send_text(self, data: str) -> None: ... async def receive_text(self) -> str: ... async def close(self, code: int = 1000, reason: str | None = None) -> None: ... @@ -1149,7 +1149,7 @@ class RealTimeStreaming: ) @staticmethod - def _detect_beta_header(websocket: _ScopedWebSocket) -> bool: + def _detect_beta_header(websocket: ScopedWebSocket) -> bool: """Return True if the client sent 'OpenAI-Beta: realtime=v1'. Checks the raw ASGI scope headers so it works for both FastAPI WebSocket @@ -1584,6 +1584,6 @@ class RealTimeStreaming: verbose_logger.debug("Could not relay the upstream close to the client: %s", e) -def client_sent_openai_beta_realtime_header(websocket: _ScopedWebSocket) -> bool: +def client_sent_openai_beta_realtime_header(websocket: ScopedWebSocket) -> bool: """True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``.""" return RealTimeStreaming._detect_beta_header(websocket) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 1de2533d514..3b128899f45 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -381,7 +381,7 @@ class _MessageCountParams: from litellm.utils import print_verbose actual_model: Final = _fix_model_name(model) - if actual_model == "gpt-3.5-turbo-0301": + if uses_legacy_message_accounting(model): self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n self.tokens_per_name = -1 # if there's a name, the role is omitted elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms: @@ -615,7 +615,7 @@ def _get_exact_count_function( ) -> TokenCounterFunction: """ Get the function to count tokens based on the model and custom tokenizer.""" - from litellm.utils import _select_tokenizer, print_verbose + from litellm.utils import _select_tokenizer if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) @@ -627,15 +627,7 @@ def _get_exact_count_function( return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": - model_to_use: Final = _fix_model_name(model) - try: - if "gpt-4o" in model_to_use: - encoding = tiktoken.get_encoding("o200k_base") - else: - encoding = tiktoken.encoding_for_model(model_to_use) - except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - encoding = tiktoken.get_encoding("cl100k_base") + encoding: Final = openai_tokenizer_encoding(model) def encode_length(text: str) -> int: return len(encoding.encode(text, disallowed_special=())) @@ -651,6 +643,25 @@ def _get_exact_count_function( return _get_tiktoken_count_function(encode_length) +def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: + """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" + from litellm.utils import print_verbose + + model_to_use: Final = _fix_model_name(model) + if "gpt-4o" in model_to_use: + return tiktoken.get_encoding("o200k_base") + try: + return tiktoken.encoding_for_model(model_to_use) + except KeyError: + print_verbose("Warning: model not found. Using cl100k_base encoding.") + return tiktoken.get_encoding("cl100k_base") + + +def uses_legacy_message_accounting(model: str) -> bool: + """Whether `token_counter` prices messages with the `gpt-3.5-turbo-0301` constants (4 per message, -1 per name).""" + return _fix_model_name(model) == "gpt-3.5-turbo-0301" + + def _fix_model_name(model: str) -> str: """We normalize some model names to others""" if model in litellm.azure_llms: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 9b3a57cc422..8ff9f2e0679 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -400,7 +400,7 @@ class LiteLLMAnthropicMessagesAdapter: Anthropic web search tools have: - type starting with "web_search" (e.g., "web_search_20260209") - - name = "web_search" + - legacy name = "web_search" without a client input_schema Args: tool: Tool definition dict @@ -410,7 +410,9 @@ class LiteLLMAnthropicMessagesAdapter: """ tool_type: Final = tool.get("type", "") tool_name: Final = tool.get("name", "") - return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search" + return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or ( + tool_name == "web_search" and "input_schema" not in tool + ) def translate_anthropic_messages_to_openai( self, diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index e9913f0108d..146915dd6fd 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -13,7 +13,11 @@ from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from ....litellm_core_utils.realtime_streaming import RealTimeStreaming +from ....litellm_core_utils.realtime_streaming import ( + RealTimeStreaming, + ScopedWebSocket, + client_sent_openai_beta_realtime_header, +) from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..azure import AzureChatCompletion @@ -31,6 +35,19 @@ async def forward_messages(client_ws: Any, backend_ws: Any): pass +def azure_realtime_protocol_for_client( + configured_protocol: object, + *, + query_params: RealtimeQueryParams | None, + websocket: ScopedWebSocket, +) -> str: + if isinstance(configured_protocol, str) and configured_protocol: + return configured_protocol + if (query_params or {}).get("intent") == "transcription": + return "GA" + return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" + + class _ProxyClientWebSocket(Protocol): """Client-facing websocket handle: this path only closes it after a failed handshake.""" diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index bbbda4d14b6..53a3e634adf 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -56,6 +56,7 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset( ) _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"}) +_BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"}) _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" @@ -187,6 +188,43 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI ) return {key: value for key, value in params.items() if key != "service_tier"} + def _handle_unsupported_reasoning_summary( + self, params: dict[str, object], model: str, drop_params: bool + ) -> dict[str, object]: + reasoning: Final = params.get("reasoning") + if not self.use_openai_path or not isinstance(reasoning, dict): + return params + summary: Final = reasoning.get("summary") + if summary is None or ( + isinstance(summary, str) and summary in _BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES + ): + return params + if not drop_params: + raise litellm.utils.UnsupportedParamsError( + status_code=400, + message=( + f"bedrock_mantle does not support reasoning.summary={summary!r} for {model!r}; the Bedrock Mantle " + "OpenAI Responses path only accepts 'auto'. Set `drop_params: true` (litellm_settings or this " + 'deployment\'s litellm_params) to have LiteLLM drop it, or set `model_reasoning_summary = "auto"` ' + "in the client (Codex CLI: ~/.codex/config.toml)." + ), + ) + verbose_logger.warning( + "Bedrock Mantle Responses API: dropping unsupported reasoning.summary %r (supported: %s).", + summary, + sorted(_BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES), + ) + stripped: Final = { # mutable-ok: map_openai_params contract returns a plain dict + key: value for key, value in reasoning.items() if key != "summary" + } + return ( + {**params, "reasoning": stripped} # mutable-ok: map_openai_params contract returns a plain dict + if stripped + else { # mutable-ok: map_openai_params contract returns a plain dict + key: value for key, value in params.items() if key != "reasoning" + } + ) + def transform_responses_api_request( self, model: str, @@ -343,12 +381,16 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI model: str, drop_params: bool, ) -> dict: - params: Final = self._handle_unsupported_service_tier( - super().map_openai_params( - response_api_optional_params=response_api_optional_params, - model=model, + params: Final = self._handle_unsupported_reasoning_summary( + self._handle_unsupported_service_tier( + super().map_openai_params( + response_api_optional_params=response_api_optional_params, + model=model, + drop_params=drop_params, + ), drop_params=drop_params, ), + model=model, drop_params=drop_params, ) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 26ad9a02a79..b4e1856f499 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -511,7 +511,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): m = cast(dict, message) m.pop("provider_specific_fields", None) m.pop("thinking_blocks", None) - m.pop("reasoning_content", None) return messages diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 33e0a0a923f..2fe11d9f7bd 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -497,9 +497,14 @@ class OpenAIResponsesHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts") or () data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data # rebind-ok: data is an out-param else: + rewritten_texts: Final = guardrailed_inputs.get("texts") or () + if len(rewritten_texts) != len(extracted.task_mappings): + from litellm.proxy.policy_engine.pipeline_executor import UnappliableRequestRewrite + + raise UnappliableRequestRewrite(guardrail_to_apply.guardrail_name or "unknown") await self._apply_guardrail_responses_to_input( messages=input_data, - responses=guardrailed_inputs.get("texts") or (), + responses=rewritten_texts, task_mappings=extracted.task_mappings, ) verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input")) @@ -635,10 +640,12 @@ class OpenAIResponsesHandler(BaseTranslation): """ Apply guardrail responses back to input messages. + ``responses`` pairs positionally with ``task_mappings``; the caller rejects + the request when the two disagree, so this never has to guess an alignment. + Override this method to customize how responses are applied. """ - for task_idx, guardrail_response in enumerate(responses): - mapping = task_mappings[task_idx] + for guardrail_response, mapping in zip(responses, task_mappings): msg_idx = cast(int, mapping[0]) content_idx_optional = cast(int | None, mapping[1]) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 69fe5678de9..d113b2b4f6b 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -23,6 +23,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO, ) +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( _encode_tool_call_id_with_signature, @@ -108,6 +109,21 @@ else: StreamingChoices = Any +SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "none", "disable") + + +def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=( + f"Invalid `reasoning_effort`: {reasoning_effort!r}. " + f"Must be one of: {', '.join(repr(effort) for effort in SUPPORTED_REASONING_EFFORTS)}. " + "To drop this param, set `litellm.drop_params = True` or pass in `(.., drop_params=True)` " + "in the request - https://docs.litellm.ai/docs/completion/drop_params" + ), + status_code=400, + ) + + class VertexAIBaseConfig: def get_mapped_special_auth_params(self) -> dict: """ @@ -842,7 +858,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "includeThoughts": False, } else: - raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") + raise _unsupported_reasoning_effort(reasoning_effort) @staticmethod def _map_reasoning_effort_to_thinking_level( @@ -890,7 +906,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: return {"thinkingLevel": "low", "includeThoughts": False} else: - raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") + raise _unsupported_reasoning_effort(reasoning_effort) @staticmethod def _is_thinking_budget_zero(thinking_budget: int | None) -> bool: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aadd9bd3028..7fa09951eae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57695,6 +57695,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://fireworks.ai/models/deepseek-ai/deepseek-v4p1-flash", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-vision-exp": { "cache_read_input_token_cost": 7e-09, "input_cost_per_token": 2.2e-07, @@ -57745,6 +57762,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://fireworks.ai/models/deepseek-ai/deepseek-v4p1-flash", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/deepseek-v4-flash-vision-exp": { "cache_read_input_token_cost": 7e-09, "input_cost_per_token": 2.2e-07, diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index df3f9d2096b..56bfd98895d 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -10,6 +10,7 @@ import re from collections.abc import Callable, Coroutine, Mapping from dataclasses import dataclass from io import IOBase +from types import MappingProxyType from typing import Any, Final, cast import httpx @@ -19,6 +20,7 @@ from litellm._logging import verbose_logger from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure_ai.ocr.common_utils import ( + is_azure_cohere_parse_model, is_azure_document_intelligence_model, ) from litellm.llms.base_llm.ocr.transformation import ( @@ -52,21 +54,32 @@ class _PreparedOCRRequest: litellm_params: dict[str, object] effective_timeout: float | httpx.Timeout litellm_logging_obj: LiteLLMLoggingObj + caller_supplied_api_key: bool = True + caller_supplied_api_base: bool = True -@dataclass -class _PreparedRustOCRCall: - api_key: str | None - api_base: str | None - headers: dict[str, object] - optional_params: dict[str, object] - - -_RUST_OCR_PROVIDERS: Final = { - "mistral", - "azure_ai", - "vertex_ai", -} +_RUST_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "vertex_ai"}) +_RUST_OCR_CONFIG_FIELDS: Final = frozenset( + { + "azure_ad_token", + "tenant_id", + "client_id", + "client_secret", + "azure_scope", + "azure_authority_host", + "azure_credential", + "azure_federated_token_file", + "vertex_credentials", + "vertex_ai_credentials", + "vertex_project", + "vertex_ai_project", + "vertex_location", + "vertex_ai_location", + } +) +_RUST_OCR_SECRET_FIELDS: Final = frozenset( + {"azure_ad_token", "client_secret", "azure_federated_token_file", "vertex_credentials", "vertex_ai_credentials"} +) def _prepare_ocr_request( @@ -94,6 +107,7 @@ def _prepare_ocr_request( if doc_type not in ["document_url", "image_url"]: raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'") + caller_supplied_api_key: Final = api_key is not None caller_supplied_api_base: Final = api_base is not None ( @@ -187,182 +201,256 @@ def _prepare_ocr_request( litellm_params=dict(litellm_params), effective_timeout=effective_timeout, litellm_logging_obj=litellm_logging_obj, + caller_supplied_api_key=caller_supplied_api_key, + caller_supplied_api_base=caller_supplied_api_base, ) -def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: - if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native": - return False - if not prepared_request.provider_config.supports_rust_bridge(): - return False - return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS - - -def _rust_bridge_optional_params( - prepared_request: _PreparedOCRRequest, - resolve_secret: Callable[[str], str | None], -) -> dict[str, object]: - optional_params: Final = dict(prepared_request.optional_params) - if prepared_request.custom_llm_provider == "vertex_ai": - vertex_project: Final = ( - prepared_request.litellm_params.get("vertex_project") - or prepared_request.litellm_params.get("vertex_ai_project") - or litellm.vertex_project - or resolve_secret("VERTEXAI_PROJECT") - ) - vertex_location: Final = ( - prepared_request.litellm_params.get("vertex_location") - or prepared_request.litellm_params.get("vertex_ai_location") - or litellm.vertex_location - or resolve_secret("VERTEXAI_LOCATION") - or resolve_secret("VERTEX_LOCATION") - ) - if vertex_project is not None: - optional_params["vertex_project"] = vertex_project - if vertex_location is not None: - optional_params["vertex_location"] = vertex_location - return optional_params - - -def _rust_bridge_api_base( - prepared_request: _PreparedOCRRequest, - resolve_secret: Callable[[str], str | None], -) -> str | None: - if prepared_request.api_base is not None: - return prepared_request.api_base - if prepared_request.custom_llm_provider == "azure_ai": - if is_azure_document_intelligence_model(prepared_request.model): - return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") - return resolve_secret("AZURE_AI_API_BASE") +def _rust_ocr_provider(request: rust_ocr_bridge.LiteLLMOcrRequest) -> str | None: + if request.custom_llm_provider is not None: + return request.custom_llm_provider + prefix: Final = request.model.partition("/")[0] + if prefix in _RUST_OCR_PROVIDERS: + return prefix + if request.model.startswith("mistral-ocr"): + return "mistral" return None -def _prepare_rust_ocr_call( - prepared_request: _PreparedOCRRequest, - resolve_api_key: Callable[[str], str | None], -) -> _PreparedRustOCRCall: - provider_config: Final = prepared_request.provider_config - api_key_env_var: Final = provider_config.get_api_key_env_var() - resolved_api_key: Final = prepared_request.api_key or ( - resolve_api_key(api_key_env_var) if api_key_env_var is not None else None +def _rust_ocr_supported(request: rust_ocr_bridge.LiteLLMOcrRequest) -> bool: + provider: Final = _rust_ocr_provider(request) + if provider not in _RUST_OCR_PROVIDERS or request.kwargs.get(OCR_REQUEST_FORMAT_PARAM) == "native": + return False + if provider == "azure_ai": + return ( + not is_azure_cohere_parse_model(request.model) + and not callable(request.kwargs.get("azure_ad_token_provider")) + and request.kwargs.get("azure_username") is None + and request.kwargs.get("azure_password") is None + ) + return True + + +def _rust_bridge_optional_params( + request: rust_ocr_bridge.LiteLLMOcrRequest, + resolve_secret: Callable[[str], str | None], +) -> Mapping[str, object]: + optional_params: Final = MappingProxyType( + { + name: value + for name, value in request.kwargs.items() + if (name not in GenericLiteLLMParams.model_fields or name in _RUST_OCR_CONFIG_FIELDS) + and name not in {"litellm_logging_obj", "aocr", "litellm_call_id", "proxy_server_request"} + } ) - resolved_headers: Final = provider_config.validate_environment( - headers=prepared_request.extra_headers or {}, - model=prepared_request.model, - api_key=resolved_api_key, - api_base=prepared_request.api_base, - litellm_params=prepared_request.litellm_params, + provider: Final = _rust_ocr_provider(request) + if provider == "azure_ai" and litellm.enable_azure_ad_token_refresh is True: + return MappingProxyType({**optional_params, "enable_azure_ad_token_refresh": True}) + if provider != "vertex_ai": + return optional_params + project: Final = ( + request.kwargs.get("vertex_project") + or request.kwargs.get("vertex_ai_project") + or litellm.vertex_project + or resolve_secret("VERTEXAI_PROJECT") ) - resolved_complete_url: Final = provider_config.get_complete_url( - api_base=prepared_request.api_base, - model=prepared_request.model, - optional_params=prepared_request.optional_params, - litellm_params=prepared_request.litellm_params, + location: Final = ( + request.kwargs.get("vertex_location") + or request.kwargs.get("vertex_ai_location") + or litellm.vertex_location + or resolve_secret("VERTEXAI_LOCATION") + or resolve_secret("VERTEX_LOCATION") ) - rust_api_base: Final = _rust_bridge_api_base(prepared_request, resolve_api_key) - rust_optional_params: Final = _rust_bridge_optional_params(prepared_request, resolve_api_key) - prepared_request.litellm_logging_obj.pre_call( + credentials: Final = ( + request.kwargs.get("vertex_credentials") + or request.kwargs.get("vertex_ai_credentials") + or resolve_secret("VERTEXAI_CREDENTIALS") + ) + vertex_params: Final = MappingProxyType( + { + name: value + for name, value in ( + ("vertex_project", project), + ("vertex_location", location), + ("vertex_credentials", credentials), + ) + if value is not None + } + ) + return MappingProxyType({**optional_params, **vertex_params}) + + +def _rust_bridge_input_sources( + request: rust_ocr_bridge.LiteLLMOcrRequest, + optional_params: Mapping[str, object], +) -> Mapping[str, str]: + proxy_request: Final = request.kwargs.get("proxy_server_request") + if not isinstance(proxy_request, Mapping): + return MappingProxyType({}) + proxy_request_mapping: Final = cast( # cast-ok: runtime Mapping check loses generic key and value types + Mapping[object, object], proxy_request + ) + body_value: Final = proxy_request_mapping.get("body") + if not isinstance(body_value, Mapping): + return MappingProxyType({}) + body: Final = cast( # cast-ok: runtime Mapping check loses generic key and value types + Mapping[object, object], body_value + ) + credential_fields_value: Final = proxy_request_mapping.get("credential_fields", ()) + credential_fields: Final = ( + frozenset(name for name in credential_fields_value if isinstance(name, str)) + if isinstance(credential_fields_value, (list, tuple, set, frozenset)) + else frozenset() + ) + names: Final = frozenset(optional_params) | frozenset({"api_key", "api_base", "extra_headers"}) + request_sources: Final = MappingProxyType( + {name: "request" for name in names if name in body or name in credential_fields} + ) + if litellm.enable_azure_ad_token_refresh is True and "enable_azure_ad_token_refresh" in optional_params: + return MappingProxyType({**request_sources, "enable_azure_ad_token_refresh": "deployment"}) + return request_sources + + +def _marshal_rust_ocr_request( + request: rust_ocr_bridge.LiteLLMOcrRequest, + resolve_secret: Callable[[str], str | None], +) -> rust_ocr_bridge.LiteLLMOcrRequest: + if not isinstance(request.document, dict): + raise TypeError(f"document must be a dict with 'type' and URL/file field, got {type(request.document)}") + document: Final = ( + convert_file_document_to_url_document(request.document) + if request.document.get("type") == "file" + else request.document + ) + provider: Final = _rust_ocr_provider(request) + api_key: Final = request.api_key or resolve_secret("MISTRAL_API_KEY") if provider == "mistral" else request.api_key + optional_params: Final = _rust_bridge_optional_params(request, resolve_secret) + input_sources: Final = _rust_bridge_input_sources(request, optional_params) + logged_optional_params: Final = MappingProxyType( + {name: "****" if name in _RUST_OCR_SECRET_FIELDS else value for name, value in optional_params.items()} + ) + logged_kwargs: Final = MappingProxyType( + { + name: "****" if name in _RUST_OCR_SECRET_FIELDS else value + for name, value in request.kwargs.items() + if name != "proxy_server_request" + } + ) + logging_obj: Final = cast( # cast-ok: bridge kwargs carry the prepared logging object + LiteLLMLoggingObj, request.kwargs["litellm_logging_obj"] + ) + logging_obj.update_from_kwargs( + kwargs=dict(logged_kwargs), # mutable-ok: logging API requires an owned dict + model=request.model, + optional_params=dict(logged_optional_params), # mutable-ok: logging API requires an owned dict + litellm_params={ + "litellm_call_id": request.kwargs.get("litellm_call_id"), + "api_base": request.api_base, + }, # mutable-ok: legacy logging requires a concrete params dict + custom_llm_provider=provider, + ) + logging_obj.pre_call( input="OCR document processing", - api_key=resolved_api_key, - additional_args={ + api_key=api_key, + additional_args={ # mutable-ok: pre_call mutates the additional_args dict "complete_input_dict": { - "model": prepared_request.model, - "document": prepared_request.document, - **rust_optional_params, - }, - "api_base": resolved_complete_url, - "headers": resolved_headers, + "model": request.model, + "document": document, + **logged_optional_params, + }, # mutable-ok: callbacks consume a JSON-serializable request dict + "api_base": request.api_base or "", + "headers": request.extra_headers or {}, # mutable-ok: logging callbacks consume a concrete headers dict }, ) - return _PreparedRustOCRCall( - api_key=resolved_api_key, - api_base=rust_api_base, - headers=cast(dict[str, object], resolved_headers), - optional_params=rust_optional_params, + return rust_ocr_bridge.LiteLLMOcrRequest( + model=request.model, + document=document, + api_key=api_key, + api_base=request.api_base, + timeout=request.timeout if request.timeout is not None else request_timeout, + custom_llm_provider=request.custom_llm_provider, + extra_headers=request.extra_headers, + kwargs=optional_params, + input_sources=input_sources, ) def _map_rust_ocr_error( error: Exception, - prepared_request: _PreparedOCRRequest, + request: rust_ocr_bridge.LiteLLMOcrRequest, exception_types: tuple[type[BaseException], type[BaseException]] | None, ) -> Exception: - if exception_types is None: + if exception_types is None or not isinstance(error, exception_types[1]): return error - _, upstream_error = exception_types - if not isinstance(error, upstream_error): + provider: Final = _rust_ocr_provider(request) + if provider is None: return error - error_args: Final = cast( # cast-ok: BaseException.args is typed with Any in the standard library stubs + provider_config: Final = ProviderConfigManager.get_provider_ocr_config( + model=request.model.removeprefix(f"{provider}/"), provider=litellm.LlmProviders(provider) + ) + if provider_config is None: + return error + error_args: Final = cast( # cast-ok: Python exceptions expose positional args as a tuple tuple[object, ...], error.args ) - status_value: Final = error_args[0] if error_args else 0 - message_value: Final = error_args[1] if len(error_args) > 1 else str(error) - status: Final = status_value if isinstance(status_value, int) else 0 - message: Final = message_value if isinstance(message_value, str) else str(message_value) - error_factory: Final = cast( # cast-ok: the legacy provider interface leaves callable parameters untyped - Callable[..., Exception], prepared_request.provider_config.get_error_class + status: Final = error_args[0] if error_args and isinstance(error_args[0], int) else 500 + message: Final = str(error_args[1]) if len(error_args) > 1 else str(error) + error_factory: Final = cast( # cast-ok: provider configs expose heterogeneous exception factories + Callable[..., Exception], provider_config.get_error_class ) return error_factory( - error_message=message, - status_code=status or 500, - headers={}, # mutable-ok: provider error factories require a concrete header dict - ) + error_message=message, status_code=status or 500, headers={} + ) # mutable-ok: provider error factories require a concrete headers dict def _run_rust_ocr( - prepared_request: _PreparedOCRRequest, + request: rust_ocr_bridge.LiteLLMOcrRequest, resolve_api_key: Callable[[str], str | None], ) -> OCRResponse | None: if rust_ocr_bridge.load_rust_ocr() is None: return None - prepared: Final = _prepare_rust_ocr_call( - prepared_request=prepared_request, - resolve_api_key=resolve_api_key, - ) + marshalled: Final = _marshal_rust_ocr_request(request, resolve_api_key) + input_sources: Final = marshalled.input_sources try: - rust_response: Final = rust_ocr_bridge.ocr( - model=prepared_request.model, - document=prepared_request.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared.headers, - optional_params=prepared.optional_params, - timeout=prepared_request.effective_timeout, + response: Final = rust_ocr_bridge.ocr( + model=marshalled.model, + document=dict(marshalled.document), # mutable-ok: PyO3 OCR binding requires a concrete dict + api_key=marshalled.api_key, + api_base=marshalled.api_base, + custom_llm_provider=marshalled.custom_llm_provider, + extra_headers=marshalled.extra_headers, + optional_params=dict(marshalled.kwargs), # mutable-ok: PyO3 OCR binding requires a concrete dict + input_sources=input_sources, + timeout=marshalled.timeout, ) except Exception as error: - raise _map_rust_ocr_error(error, prepared_request, native_exception_types()) from error - if rust_response is None: - return None - return OCRResponse.model_validate(rust_response) + raise _map_rust_ocr_error(error, request, native_exception_types()) from error + return OCRResponse.model_validate(response) if response is not None else None async def _run_rust_aocr( - prepared_request: _PreparedOCRRequest, + request: rust_ocr_bridge.LiteLLMOcrRequest, resolve_api_key: Callable[[str], str | None], ) -> OCRResponse | None: if rust_ocr_bridge.load_rust_aocr() is None: return None - prepared: Final = _prepare_rust_ocr_call( - prepared_request=prepared_request, - resolve_api_key=resolve_api_key, - ) + marshalled: Final = _marshal_rust_ocr_request(request, resolve_api_key) + input_sources: Final = marshalled.input_sources try: - rust_response: Final = await rust_ocr_bridge.aocr( - model=prepared_request.model, - document=prepared_request.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared.headers, - optional_params=prepared.optional_params, - timeout=prepared_request.effective_timeout, + response: Final = await rust_ocr_bridge.aocr( + model=marshalled.model, + document=dict(marshalled.document), # mutable-ok: PyO3 OCR binding requires a concrete dict + api_key=marshalled.api_key, + api_base=marshalled.api_base, + custom_llm_provider=marshalled.custom_llm_provider, + extra_headers=marshalled.extra_headers, + optional_params=dict(marshalled.kwargs), # mutable-ok: PyO3 OCR binding requires a concrete dict + input_sources=input_sources, + timeout=marshalled.timeout, ) except Exception as error: - raise _map_rust_ocr_error(error, prepared_request, native_exception_types()) from error - if rust_response is None: - return None - return OCRResponse.model_validate(rust_response) + raise _map_rust_ocr_error(error, request, native_exception_types()) from error + return OCRResponse.model_validate(response) if response is not None else None @client @@ -444,7 +532,29 @@ async def aocr( "extra_headers": extra_headers, "kwargs": kwargs, } + request: Final = rust_ocr_bridge.LiteLLMOcrRequest( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) try: + if rust_enabled() and _rust_ocr_supported(request): + from litellm.secret_managers.main import get_secret_str + + rust_response: Final = await _run_rust_aocr( + request=request, + resolve_api_key=get_secret_str, + ) + if rust_response is None: + verbose_logger.debug("Async Rust OCR bridge unavailable; falling back to Python path") + else: + return rust_response + prepared: Final = _prepare_ocr_request( model=model, document=document, @@ -459,18 +569,6 @@ async def aocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_enabled(): - from litellm.secret_managers.main import get_secret_str - - rust_response: Final = await _run_rust_aocr( - prepared_request=prepared, - resolve_api_key=get_secret_str, - ) - if rust_response is None: - verbose_logger.debug("Async Rust OCR bridge unavailable; falling back to Python path") - else: - return rust_response - response = base_llm_http_handler.ocr( model=prepared.model, document=prepared.document, @@ -494,9 +592,11 @@ async def aocr( return response except Exception as e: + error_provider: Final = custom_llm_provider or _rust_ocr_provider(request) + error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( - model=model, - custom_llm_provider=custom_llm_provider, + model=error_model, + custom_llm_provider=error_provider, original_exception=e, completion_kwargs=completion_kwargs, extra_kwargs=kwargs, @@ -714,9 +814,31 @@ def ocr( "extra_headers": extra_headers, "kwargs": kwargs, } + request: Final = rust_ocr_bridge.LiteLLMOcrRequest( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) try: _is_async: Final = kwargs.pop("aocr", False) is True completion_kwargs["aocr"] = _is_async + if rust_enabled() and _rust_ocr_supported(request): + from litellm.secret_managers.main import get_secret_str + + rust_response: Final = _run_rust_ocr( + request=request, + resolve_api_key=get_secret_str, + ) + if rust_response is None: + verbose_logger.debug("Rust OCR bridge unavailable; falling back to Python path") + else: + return rust_response + prepared: Final = _prepare_ocr_request( model=model, document=document, @@ -731,18 +853,6 @@ def ocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_enabled(): - from litellm.secret_managers.main import get_secret_str - - rust_response: Final = _run_rust_ocr( - prepared_request=prepared, - resolve_api_key=get_secret_str, - ) - if rust_response is None: - verbose_logger.debug("Rust OCR bridge unavailable; falling back to Python path") - else: - return rust_response - response: Final = base_llm_http_handler.ocr( model=prepared.model, document=prepared.document, @@ -760,9 +870,11 @@ def ocr( return response except Exception as e: + error_provider: Final = custom_llm_provider or _rust_ocr_provider(request) + error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( - model=model, - custom_llm_provider=custom_llm_provider, + model=error_model, + custom_llm_provider=error_provider, original_exception=e, completion_kwargs=completion_kwargs, extra_kwargs=kwargs, diff --git a/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py b/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py index 28f3ec6521a..0ab76588b1f 100644 --- a/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py @@ -7,7 +7,6 @@ just a form that asks the user for their API key — not a full identity-provide Endpoints implemented here: GET /.well-known/oauth-authorization-server — OAuth authorization server metadata - GET /.well-known/oauth-protected-resource — OAuth protected resource metadata GET /v1/mcp/oauth/authorize — Shows HTML form to collect the API key POST /v1/mcp/oauth/authorize — Stores temp auth code and redirects POST /v1/mcp/oauth/token — Exchanges code for a bearer JWT token @@ -19,7 +18,7 @@ import html as _html_module import time import uuid from typing import Final, cast -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse import jwt from fastapi import APIRouter, Depends, Form, HTTPException, Request @@ -27,14 +26,15 @@ from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from litellm._logging import verbose_proxy_logger from litellm.proxy._experimental.mcp_server.db import store_user_credential -from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - get_request_base_url, -) from litellm.proxy._experimental.mcp_server.oauth_utils import ( + BYOK_RESOURCE_METADATA_PATH, TOKEN_NO_CACHE_HEADERS, + get_request_base_url, validate_loopback_redirect_uri, + well_known_root_suffix, ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.middleware.per_request_root_path_middleware import get_server_root_paths # --------------------------------------------------------------------------- # In-memory store for pending authorization codes. @@ -596,13 +596,10 @@ def _build_authorize_html( # --------------------------------------------------------------------------- -@router.get("/.well-known/oauth-authorization-server", include_in_schema=False) -async def oauth_authorization_server_metadata(request: Request) -> JSONResponse: - """RFC 8414 Authorization Server Metadata for the BYOK OAuth flow.""" - base_url: Final = get_request_base_url(request) +def _byok_authorization_server_response(base_url: str, issuer: str) -> JSONResponse: return JSONResponse( { - "issuer": base_url, + "issuer": issuer, "authorization_endpoint": f"{base_url}/v1/mcp/oauth/authorize", "token_endpoint": f"{base_url}/v1/mcp/oauth/token", "response_types_supported": ["code"], @@ -612,14 +609,36 @@ async def oauth_authorization_server_metadata(request: Request) -> JSONResponse: ) -@router.get("/.well-known/oauth-protected-resource", include_in_schema=False) -async def oauth_protected_resource_metadata(request: Request) -> JSONResponse: - """RFC 9728 Protected Resource Metadata pointing back at this server.""" +@router.get("/.well-known/oauth-authorization-server", include_in_schema=False) +async def oauth_authorization_server_metadata(request: Request) -> JSONResponse: base_url: Final = get_request_base_url(request) + return _byok_authorization_server_response(base_url, base_url) + + +@router.get(f"/.well-known/oauth-authorization-server{well_known_root_suffix()}/v1/mcp/oauth", include_in_schema=False) +async def byok_authorization_server_metadata(request: Request) -> JSONResponse: + base_url: Final = get_request_base_url(request) + return _byok_authorization_server_response(base_url, f"{base_url}/v1/mcp/oauth") + + +@router.get("/.well-known/oauth-authorization-server/{root_path:path}/v1/mcp/oauth", include_in_schema=False) +async def byok_prefixed_authorization_server_metadata(request: Request, root_path: str) -> JSONResponse: + prefix: Final = f"/{root_path}" + if prefix not in get_server_root_paths(): + raise HTTPException(status_code=404, detail="Unknown proxy root path") + parsed: Final = urlparse(get_request_base_url(request)) + base_url: Final = f"{parsed.scheme}://{parsed.netloc}{prefix}" + return _byok_authorization_server_response(base_url, f"{base_url}/v1/mcp/oauth") + + +@router.get(BYOK_RESOURCE_METADATA_PATH, include_in_schema=False) +async def byok_protected_resource_metadata(request: Request) -> JSONResponse: + base_url: Final = get_request_base_url(request) + parsed: Final = urlparse(base_url) return JSONResponse( { - "resource": base_url, - "authorization_servers": [base_url], + "resource": f"{parsed.scheme}://{parsed.netloc}", + "authorization_servers": (f"{base_url}/v1/mcp/oauth",), } ) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 42fbe82531c..bafe33d0a6b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1842,6 +1842,30 @@ async def register_client_with_server( return JSONResponse(token_response) +@router.get("/authorize/mcp-session") +async def authorize_mcp_session( + request: Request, + redirect_uri: str, + client_id: str, + state: str = "", + code_challenge: str | None = None, + code_challenge_method: str | None = None, + response_type: str | None = None, + resource: str | None = None, +) -> Response: + return aggregate_authorize( + request=request, + client_id=client_id, + redirect_uri=redirect_uri, + state=state, + code_challenge=code_challenge, + code_challenge_method=code_challenge_method, + response_type=response_type, + session_user_id=_session_cookie_user_id(request), + resource=resource, + ) + + @router.get("/{mcp_server_name}/authorize") @router.get("/authorize") async def authorize( @@ -2393,8 +2417,7 @@ async def _build_oauth_protected_resource_response( per-server URL completes the same sign-in flow the aggregate ``/mcp`` endpoint supports and is admitted with a gateway session bearer. The per-server relay authorize/token endpoints stay registered for the keyed interactive flow (which - is challenged with an explicit ``authorization_uri``), and the root-resolved - (unnamed) legacy shape keeps the relay authorization server. + is challenged with an explicit ``authorization_uri``). Args: request: FastAPI Request object @@ -2405,15 +2428,11 @@ async def _build_oauth_protected_resource_response( Returns: OAuth protected resource metadata dict """ + if mcp_server_name is None: + return oauth_protected_resource_root(request) + request_base_url: Final = get_request_base_url(request) client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) - explicitly_named: Final = mcp_server_name is not None - - # When no server name provided, try to resolve the single OAuth2 server - if mcp_server_name is None: - resolved: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) - if resolved: - mcp_server_name = resolved.server_name or resolved.name mcp_server: MCPServer | None = None if mcp_server_name: @@ -2478,7 +2497,7 @@ async def _build_oauth_protected_resource_response( if obo_response is not None: return obo_response - if explicitly_named and mcp_server is not None and mcp_server.advertises_gateway_authorization_server: + if mcp_server is not None and mcp_server.advertises_gateway_authorization_server: return { "authorization_servers": [f"{request_base_url}/mcp"], "resource": resource_url, @@ -2542,6 +2561,17 @@ def _jwt_auth_issuers() -> list: return issuers +@router.get("/.well-known/oauth-protected-resource") +def oauth_protected_resource_root(request: Request) -> dict[str, str | tuple[str, ...]]: + request_base_url: Final = get_request_base_url(request) + parsed: Final = urlparse(request_base_url) + return { + "resource": f"{parsed.scheme}://{parsed.netloc}", + "authorization_servers": (f"{request_base_url}/mcp",), + "scopes_supported": (), + } + + def _build_aggregate_protected_resource_response(request: Request) -> dict: """RFC 9728 metadata for the aggregate /mcp resource: the gateway itself is the authorization server. No per-server names or scopes leak here; access @@ -2568,14 +2598,14 @@ def _build_aggregate_authorization_server_response(request: Request) -> dict: The issuer is ``{base}/mcp`` and must stay equal to the value the aggregate protected-resource document advertises: spec clients verify the issuer in the metadata matches the one that derived the well-known URL. - Advertises the root /authorize, /token, and /register endpoints and + Advertises the MCP session authorize endpoint, root /token and /register endpoints, and ``token_endpoint_auth_methods_supported: ["none", ...]`` because DCR clients (Claude Desktop, MCP Inspector) register as public clients; PKCE S256 is mandatory in the gateway's authorize flow.""" request_base_url: Final = get_request_base_url(request) return { "issuer": f"{request_base_url}/mcp", - "authorization_endpoint": f"{request_base_url}/authorize", + "authorization_endpoint": f"{request_base_url}/authorize/mcp-session", "token_endpoint": f"{request_base_url}/token", "introspection_endpoint": f"{request_base_url}/introspect", "registration_endpoint": f"{request_base_url}/register", @@ -2645,7 +2675,6 @@ async def oauth_protected_resource_mcp_standard(request: Request, mcp_server_nam # LiteLLM legacy pattern: /.well-known/oauth-protected-resource/{server_name}/mcp # Kept for backward compatibility with existing deployments @router.get(f"/.well-known/oauth-protected-resource{well_known_root_suffix()}/{{mcp_server_name}}/mcp") -@router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp(request: Request, mcp_server_name: str | None = None): """ OAuth protected resource discovery endpoint using LiteLLM legacy URL pattern. @@ -2666,6 +2695,8 @@ async def oauth_protected_resource_mcp(request: Request, mcp_server_name: str | def _build_oauth_authorization_server_response( request: Request, mcp_server_name: str | None, + *, + issuer_path: str | None = None, ) -> dict: """Build OAuth authorization server metadata response (gateway-as-AS shape). @@ -2694,7 +2725,13 @@ def _build_oauth_authorization_server_response( _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server") - issuer: Final = f"{request_base_url}/{mcp_server_name}" if explicitly_named else request_base_url + issuer: Final = ( + f"{request_base_url}/{issuer_path}" + if issuer_path is not None + else f"{request_base_url}/{mcp_server_name}" + if explicitly_named + else request_base_url + ) return { "issuer": issuer, @@ -2724,6 +2761,7 @@ async def oauth_authorization_server_mcp_standard(request: Request, mcp_server_n return _build_oauth_authorization_server_response( request=request, mcp_server_name=mcp_server_name, + issuer_path=f"mcp/{mcp_server_name}", ) @@ -2802,7 +2840,7 @@ async def jwks_json(request: Request): # Additional legacy pattern support -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}/mcp") +@router.get(f"/.well-known/oauth-authorization-server{well_known_root_suffix()}/{{mcp_server_name}}/mcp") async def oauth_authorization_server_legacy(request: Request, mcp_server_name: str): """ OAuth authorization server discovery for legacy /{server_name}/mcp pattern. @@ -2810,6 +2848,7 @@ async def oauth_authorization_server_legacy(request: Request, mcp_server_name: s return _build_oauth_authorization_server_response( request=request, mcp_server_name=mcp_server_name, + issuer_path=f"{mcp_server_name}/mcp", ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index cc7ea0c1ea5..3291d5effc2 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -10,6 +10,7 @@ import asyncio import datetime import hashlib import json +import math import os import re import time @@ -26,8 +27,9 @@ from collections.abc import ( from contextlib import asynccontextmanager from dataclasses import dataclass, replace from functools import lru_cache +from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypedDict, TypeVar, cast from urllib.parse import ParseResult, urlparse import anyio @@ -44,11 +46,12 @@ from mcp.types import ( ResourceTemplate, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl, BaseModel +from pydantic import AnyUrl, BaseModel, TypeAdapter from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ( MCP_CLIENT_TIMEOUT, MCP_HEALTH_CHECK_TIMEOUT, @@ -91,6 +94,7 @@ from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, canonicalize_url_identity, + get_byok_www_authenticate, ) from litellm.proxy._experimental.mcp_server.outbound_credentials import ( Error, @@ -193,7 +197,6 @@ if TYPE_CHECKING: from mcp.shared.context import RequestContext from mcp.types import CreateMessageRequestParams - from litellm.caching.caching import InMemoryCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.mcp_server.mcp_toolset import MCPToolset @@ -1232,7 +1235,7 @@ async def _resolve_byok_mcp_auth_header( "Complete the OAuth authorization flow to provide your API key." ), }, - headers={"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'}, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, ) return byok_cred @@ -1677,6 +1680,105 @@ def _record_mcp_guardrail_evaluations( verbose_logger.warning("Failed to record MCP guardrail evaluation for logging: %s", e) +_DiscoveryItem = TypeVar("_DiscoveryItem", bound=BaseModel) +_DiscoveryKey: TypeAlias = tuple[str, str | None] +_DISCOVERY_CACHE_LIMIT: Final = 1024 + + +class _DiscoveryCache(Generic[_DiscoveryItem]): + def __init__( + self, ttl: float, clock: Callable[[], float], adapter: TypeAdapter[tuple[_DiscoveryItem, ...]] + ) -> None: + self._ttl = ttl + self._adapter = adapter + self._entries = InMemoryCache(max_size_in_memory=_DISCOVERY_CACHE_LIMIT, max_size_per_item=64, clock=clock) + self._pending: dict[ + _DiscoveryKey, asyncio.Task[list[_DiscoveryItem]] + ] = {} # mutable-ok: constant-time fetch registration + self._waiters: dict[asyncio.Task[list[_DiscoveryItem]], int] = {} # mutable-ok: constant-time waiter accounting + + def invalidate(self, server_id: str) -> None: + prefix: Final = f"[{json.dumps(server_id)}," + keys: Final = cast( # cast-ok: private cache contains only JSON string keys + "tuple[str, ...]", tuple(self._entries.cache_dict) + ) + for entry_key in keys: + if entry_key.startswith(prefix): + self._entries.delete_cache(entry_key) + for key in tuple(self._pending): + if key[0] == server_id: + self._pending.pop(key) + + @staticmethod + def _observe_completion(task: asyncio.Task[list[_DiscoveryItem]]) -> None: + if not task.cancelled(): + task.exception() + + async def get( + self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[list[_DiscoveryItem]]] + ) -> tuple[_DiscoveryItem, ...]: + if self._ttl <= 0: + return tuple(await fetch()) + entry: Final[object] = self._entries.get_cache(json.dumps(key)) + if entry is not None: + return self._adapter.validate_python(entry) + pending: Final = self._pending.get(key) + if pending is not None: + return await self._await_fetch(key, pending) + if len(self._pending) >= _DISCOVERY_CACHE_LIMIT: + return tuple(await fetch()) + task: Final = asyncio.create_task(self._fetch(key, fetch)) + self._pending[key] = task + task.add_done_callback(self._observe_completion) + return await self._await_fetch(key, task) + + async def _await_fetch( + self, key: _DiscoveryKey, task: asyncio.Task[list[_DiscoveryItem]] + ) -> tuple[_DiscoveryItem, ...]: + self._waiters[task] = self._waiters.get(task, 0) + 1 + try: + return tuple(item.model_copy(deep=True) for item in await asyncio.shield(task)) + finally: + remaining: Final = self._waiters[task] - 1 + if remaining: + self._waiters[task] = remaining + else: + self._waiters.pop(task) + if self._pending.get(key) is task: + self._pending.pop(key) + if not task.done(): + task.cancel() + + async def _fetch( + self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[list[_DiscoveryItem]]] + ) -> list[_DiscoveryItem]: + try: + items: Final = await fetch() + if self._pending.get(key) is asyncio.current_task(): + self._entries.set_cache( + json.dumps(key), + self._adapter.dump_json(tuple(items)), + ttl=self._ttl, + ) + return items + finally: + if self._pending.get(key) is asyncio.current_task(): + self._pending.pop(key) + + +def _mcp_discovery_cache_ttl() -> float: + raw: Final = os.environ.get("LITELLM_MCP_DISCOVERY_CACHE_TTL", "60") + try: + ttl: Final = float(raw) + except ValueError: + verbose_logger.warning("Invalid LITELLM_MCP_DISCOVERY_CACHE_TTL; using 60 seconds") + return 60.0 + if not math.isfinite(ttl) or ttl < 0: + verbose_logger.warning("Invalid LITELLM_MCP_DISCOVERY_CACHE_TTL; using 60 seconds") + return 60.0 + return ttl + + class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") @@ -1793,6 +1895,7 @@ class MCPServerManager: cred_provider: UpstreamCredentialProvider | None = None, per_user_oauth_token_store: InvalidatableOAuthTokenStore | None = None, per_user_token_cache: MCPPerUserTokenCache | None = None, + discovery_clock: Callable[[], float] = time.monotonic, ): self._per_user_oauth_token_store = per_user_oauth_token_store or LazyPerUserOAuthTokenStore( self.get_mcp_server_by_id @@ -1802,6 +1905,16 @@ class MCPServerManager: oauth_token_store=self._per_user_oauth_token_store, token_exchanger=build_token_exchanger(), ) + discovery_ttl: Final = _mcp_discovery_cache_ttl() + self._prompt_discovery_cache = _DiscoveryCache[Prompt]( + discovery_ttl, discovery_clock, TypeAdapter(tuple[Prompt, ...]) + ) + self._resource_discovery_cache = _DiscoveryCache[Resource]( + discovery_ttl, discovery_clock, TypeAdapter(tuple[Resource, ...]) + ) + self._template_discovery_cache = _DiscoveryCache[ResourceTemplate]( + discovery_ttl, discovery_clock, TypeAdapter(tuple[ResourceTemplate, ...]) + ) self.registry: dict[str, MCPServer] = {} self._openapi_health_probes: Callable[[str], _OpenAPIHealthProbe] = lru_cache(maxsize=128)(_OpenAPIHealthProbe) self.config_mcp_servers: dict[str, MCPServer] = {} @@ -2529,6 +2642,7 @@ class MCPServerManager: self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") _warn_config_id_jag_server_outruns_sso(new_server) + self._invalidate_discovery_lists(server_id) self.config_mcp_servers[server_id] = new_server self._set_oauth_discovery_deferred( server_id, @@ -2730,6 +2844,7 @@ class MCPServerManager: global_mcp_tool_registry, ) + self._invalidate_discovery_lists(server.server_id) prefix_root: Final = normalize_server_name(get_server_prefix(server)) if server.spec_path and prefix_root: openapi_key_prefix: Final = prefix_root + MCP_TOOL_PREFIX_SEPARATOR @@ -3106,6 +3221,7 @@ class MCPServerManager: # env_vars_are_encrypted=False. new_server: Final = await self.build_mcp_server_from_table(mcp_server, env_vars_are_encrypted=False) self._assign_unique_short_prefix(new_server) + self._invalidate_discovery_lists(mcp_server.server_id) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) self.prime_oauth_metadata_discovery(new_server) @@ -3142,6 +3258,7 @@ class MCPServerManager: previous_server=self.registry[mcp_server.server_id], ) self._assign_unique_short_prefix(new_server) + self._invalidate_discovery_lists(mcp_server.server_id) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) self.prime_oauth_metadata_discovery(new_server) @@ -4375,6 +4492,41 @@ class MCPServerManager: ) raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge) + def _invalidate_discovery_lists(self, server_id: str) -> None: + self._prompt_discovery_cache.invalidate(server_id) + self._resource_discovery_cache.invalidate(server_id) + self._template_discovery_cache.invalidate(server_id) + + def _discovery_key( + self, + server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | dict[str, str] | None, + extra_headers: dict[str, str] | None, + stdio_env: dict[str, str] | None, + subject_token: str | None, + credential_fingerprint: str | None = None, + ) -> _DiscoveryKey: + per_user: Final = ( + server.requires_per_user_auth + or self._references_per_user_env_var(server) + or server.delegate_auth_to_upstream + or server.auth_type in (MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag) + ) + if not (per_user or mcp_auth_header or extra_headers or stdio_env or subject_token): + return server.server_id, None + identity: Final = ( + (user_api_key_auth.user_id, user_api_key_auth.api_key) + if per_user and user_api_key_auth is not None + else None + ) + material: Final = json.dumps( + (identity, mcp_auth_header, extra_headers, stdio_env, subject_token, credential_fingerprint), + sort_keys=True, + separators=(",", ":"), + ) + return server.server_id, hashlib.sha256(material.encode()).hexdigest() + async def get_prompts_from_server( self, server: MCPServer, @@ -4384,47 +4536,38 @@ class MCPServerManager: add_prefix: bool = True, raw_headers: dict[str, str] | None = None, ) -> list[Prompt]: - """ - Helper method to get prompts from a single MCP server with prefixed names. - - Args: - server (MCPServer): The server to query prompts from - mcp_auth_header: Optional auth header for MCP server - - Returns: - List[Prompt]: List of prompts available on the server with prefixed names - """ - - verbose_logger.debug("Connecting to url: %s", server.url) - verbose_logger.info("get_prompts_from_server for %s...", server.name) - - client = None - try: - if server.static_headers: - if extra_headers is None: - extra_headers = {} - extra_headers.update(server.static_headers) - + headers: Final = ( + dict( + chain( + extra_headers.items() if extra_headers else (), + server.static_headers.items() if server.static_headers else (), + ) + ) + or None + ) stdio_env: Final = self._build_stdio_env(server, raw_headers) subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) - - client = await self._create_mcp_client( + client: Final = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, - extra_headers=extra_headers, + extra_headers=headers, stdio_env=stdio_env, subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + ) + credential_fingerprint: Final = await client.discovery_auth_fingerprint() + key: Final = self._discovery_key( + server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint ) - prompts: Final = await client.list_prompts() + async def fetch() -> list[Prompt]: + return await client.list_prompts(raise_on_error=True) - prefixed_or_original_prompts: Final = self._create_prefixed_prompts(prompts, server, add_prefix=add_prefix) - - return prefixed_or_original_prompts - - except Exception as e: - verbose_logger.warning("Failed to get prompts from server %s: %s", server.name, e) + items: Final = await self._prompt_discovery_cache.get(key, fetch) + return self._create_prefixed_prompts(items, server, add_prefix=add_prefix) + except Exception as error: + verbose_logger.warning("Failed to get prompts from server %s: %s", server.name, error) return [] async def get_resources_from_server( @@ -4436,38 +4579,38 @@ class MCPServerManager: add_prefix: bool = True, raw_headers: dict[str, str] | None = None, ) -> list[Resource]: - """Fetch available resources from a single MCP server.""" - - verbose_logger.debug("Connecting to url: %s", server.url) - verbose_logger.info("get_resources_from_server for %s...", server.name) - - client = None - try: - if server.static_headers: - if extra_headers is None: - extra_headers = {} - extra_headers.update(server.static_headers) - + headers: Final = ( + dict( + chain( + extra_headers.items() if extra_headers else (), + server.static_headers.items() if server.static_headers else (), + ) + ) + or None + ) stdio_env: Final = self._build_stdio_env(server, raw_headers) subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) - - client = await self._create_mcp_client( + client: Final = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, - extra_headers=extra_headers, + extra_headers=headers, stdio_env=stdio_env, subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + ) + credential_fingerprint: Final = await client.discovery_auth_fingerprint() + key: Final = self._discovery_key( + server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint ) - resources: Final = await client.list_resources() + async def fetch() -> list[Resource]: + return await client.list_resources(raise_on_error=True) - prefixed_resources: Final = self._create_prefixed_resources(resources, server, add_prefix=add_prefix) - - return prefixed_resources - - except Exception as e: - verbose_logger.warning("Failed to get resources from server %s: %s", server.name, e) + items: Final = await self._resource_discovery_cache.get(key, fetch) + return self._create_prefixed_resources(items, server, add_prefix=add_prefix) + except Exception as error: + verbose_logger.warning("Failed to get resources from server %s: %s", server.name, error) return [] async def get_resource_templates_from_server( @@ -4479,40 +4622,38 @@ class MCPServerManager: add_prefix: bool = True, raw_headers: dict[str, str] | None = None, ) -> list[ResourceTemplate]: - """Fetch available resource templates from a single MCP server.""" - - verbose_logger.debug("Connecting to url: %s", server.url) - verbose_logger.info("get_resource_templates_from_server for %s...", server.name) - - client = None - try: - if server.static_headers: - if extra_headers is None: - extra_headers = {} - extra_headers.update(server.static_headers) - + headers: Final = ( + dict( + chain( + extra_headers.items() if extra_headers else (), + server.static_headers.items() if server.static_headers else (), + ) + ) + or None + ) stdio_env: Final = self._build_stdio_env(server, raw_headers) subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) - - client = await self._create_mcp_client( + client: Final = await self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, - extra_headers=extra_headers, + extra_headers=headers, stdio_env=stdio_env, subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + ) + credential_fingerprint: Final = await client.discovery_auth_fingerprint() + key: Final = self._discovery_key( + server, user_api_key_auth, mcp_auth_header, headers, stdio_env, subject_token, credential_fingerprint ) - resource_templates: Final = await client.list_resource_templates() + async def fetch() -> list[ResourceTemplate]: + return await client.list_resource_templates(raise_on_error=True) - prefixed_templates: Final = self._create_prefixed_resource_templates( - resource_templates, server, add_prefix=add_prefix - ) - - return prefixed_templates - - except Exception as e: - verbose_logger.warning("Failed to get resource templates from server %s: %s", server.name, e) + items: Final = await self._template_discovery_cache.get(key, fetch) + return self._create_prefixed_resource_templates(items, server, add_prefix=add_prefix) + except Exception as error: + verbose_logger.warning("Failed to get resource_templates from server %s: %s", server.name, error) return [] async def read_resource_from_server( @@ -5220,7 +5361,7 @@ class MCPServerManager: return prefixed_tools def _create_prefixed_prompts( - self, prompts: list[Prompt], server: MCPServer, add_prefix: bool = True + self, prompts: Sequence[Prompt], server: MCPServer, add_prefix: bool = True ) -> list[Prompt]: """ Create prefixed prompts and update prompt mapping. @@ -5247,7 +5388,7 @@ class MCPServerManager: return prefixed_prompts def _create_prefixed_resources( - self, resources: list[Resource], server: MCPServer, add_prefix: bool = True + self, resources: Sequence[Resource], server: MCPServer, add_prefix: bool = True ) -> list[Resource]: """Prefix resource names and track origin server for read requests.""" @@ -5264,7 +5405,7 @@ class MCPServerManager: def _create_prefixed_resource_templates( self, - resource_templates: list[ResourceTemplate], + resource_templates: Sequence[ResourceTemplate], server: MCPServer, add_prefix: bool = True, ) -> list[ResourceTemplate]: @@ -6001,6 +6142,7 @@ class MCPServerManager: failure is logged, never raised, because the DB write already succeeded and the TTL remains the backstop. """ + self._invalidate_discovery_lists(server_id) try: await self._per_user_oauth_token_store.invalidate(user_id, server_id) except Exception as exc: # noqa: BLE001 - cache drop is best-effort; TTL is the backstop @@ -6466,6 +6608,9 @@ class MCPServerManager: for registry_key in dropped_registry_keys: self._invalidate_oauth_discovery_state(previous_registry[registry_key].server_id) + for server_id in previous_registry.keys() | registered_registry.keys(): + if previous_registry.get(server_id) != registered_registry.get(server_id): + self._invalidate_discovery_lists(server_id) self.registry = registered_registry # A discovery task may have published into ``previous_registry`` while # this replacement was being staged. Reconcile every published entry diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index 1ca2ffc703d..39865a35ec6 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -16,6 +16,7 @@ from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( normalize_token_endpoint_auth_method, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.middleware.per_request_root_path_middleware import get_request_root_path if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -126,6 +127,14 @@ def _resolve_proxy_base_url_env() -> str | None: return None +BYOK_RESOURCE_METADATA_PATH: Final = "/v1/mcp/oauth/protected-resource" + + +def get_byok_www_authenticate() -> str: + base_url: Final = _resolve_proxy_base_url_env() or get_request_root_path().rstrip("/") + return f'Bearer resource_metadata="{base_url}{BYOK_RESOURCE_METADATA_PATH}"' + + def get_request_base_url(request: Request) -> str: """ Get the base URL for the request, considering X-Forwarded-* headers. diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 43a7f894db8..fc87db69e16 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -56,6 +56,7 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import ( ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, + get_byok_www_authenticate, get_passthrough_www_authenticate, get_route_relative_request_path, well_known_root_suffix, @@ -2852,7 +2853,7 @@ if MCP_AVAILABLE: "server_name": mcp_server.server_name or mcp_server.name, "message": "User identity is required for BYOK servers", }, - headers={"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'}, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, ) # Check shared credential cache before hitting the DB. @@ -2873,9 +2874,7 @@ if MCP_AVAILABLE: "Complete the OAuth authorization flow to provide your API key." ), }, - headers={ - "WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"' - }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, ) return @@ -2914,7 +2913,7 @@ if MCP_AVAILABLE: "Complete the OAuth authorization flow to provide your API key." ), }, - headers={"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'}, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, ) async def execute_mcp_tool( @@ -3068,9 +3067,7 @@ if MCP_AVAILABLE: "Complete the OAuth authorization flow to provide your API key." ), }, - headers={ - "WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"' - }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, ) mcp_auth_header = byok_cred elif mcp_server.is_byok: diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 53af85baac6..cb7a18cd107 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -22415,47 +22415,34 @@ }, "/.well-known/oauth-protected-resource": { "get": { - "description": "OAuth protected resource discovery endpoint using LiteLLM legacy URL pattern.\n\nLegacy pattern: /{server_name}/mcp\nDiscovery path: /.well-known/oauth-protected-resource/{server_name}/mcp\n\nThis endpoint is kept for backward compatibility. New integrations should\nuse the standard MCP pattern (/mcp/{server_name}) instead.", - "operationId": "oauth_protected_resource_mcp__well_known_oauth_protected_resource_get", - "parameters": [ - { - "in": "query", - "name": "mcp_server_name", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Mcp Server Name" - } - } - ], + "operationId": "oauth_protected_resource_root__well_known_oauth_protected_resource_get", "responses": { "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "422": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/HTTPValidationError" + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "title": "Response Oauth Protected Resource Root Well Known Oauth Protected Resource Get", + "type": "object" } } }, - "description": "Validation Error" + "description": "Successful Response" } }, - "summary": "Oauth Protected Resource Mcp", + "summary": "Oauth Protected Resource Root", "tags": [ "mcp_byok_oauth" ] @@ -24636,47 +24623,34 @@ }, "/.well-known/oauth-protected-resource": { "get": { - "description": "OAuth protected resource discovery endpoint using LiteLLM legacy URL pattern.\n\nLegacy pattern: /{server_name}/mcp\nDiscovery path: /.well-known/oauth-protected-resource/{server_name}/mcp\n\nThis endpoint is kept for backward compatibility. New integrations should\nuse the standard MCP pattern (/mcp/{server_name}) instead.", - "operationId": "oauth_protected_resource_mcp__well_known_oauth_protected_resource_get_2", - "parameters": [ - { - "in": "query", - "name": "mcp_server_name", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Mcp Server Name" - } - } - ], + "operationId": "oauth_protected_resource_root__well_known_oauth_protected_resource_get_2", "responses": { "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "422": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/HTTPValidationError" + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "title": "Response Oauth Protected Resource Root Well Known Oauth Protected Resource Get", + "type": "object" } } }, - "description": "Validation Error" + "description": "Successful Response" } }, - "summary": "Oauth Protected Resource Mcp", + "summary": "Oauth Protected Resource Root", "tags": [ "mcp_discoverable" ] @@ -25052,6 +25026,129 @@ ] } }, + "/authorize/mcp-session": { + "get": { + "operationId": "authorize_mcp_session_authorize_mcp_session_get", + "parameters": [ + { + "in": "query", + "name": "redirect_uri", + "required": true, + "schema": { + "title": "Redirect Uri", + "type": "string" + } + }, + { + "in": "query", + "name": "client_id", + "required": true, + "schema": { + "title": "Client Id", + "type": "string" + } + }, + { + "in": "query", + "name": "state", + "required": false, + "schema": { + "default": "", + "title": "State", + "type": "string" + } + }, + { + "in": "query", + "name": "code_challenge", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Challenge" + } + }, + { + "in": "query", + "name": "code_challenge_method", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Challenge Method" + } + }, + { + "in": "query", + "name": "response_type", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Response Type" + } + }, + { + "in": "query", + "name": "resource", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Resource" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Authorize Mcp Session", + "tags": [ + "mcp_discoverable" + ] + } + }, "/callback": { "get": { "description": "OAuth 2.0 authorization response handler for MCP loopback clients.\n\nAccepts either:\n\n- A successful authorization response (``code`` + ``state``), which is\n forwarded back to the validated client ``redirect_uri`` with the\n original (un-wrapped) ``state``.\n- An error response (``error``[+``error_description``/``error_uri``]), per\n RFC 6749 \u00a74.1.2.1. When ``state`` is present and decodes to a trusted\n ``redirect_uri``, the error params are propagated back to the client so\n its OAuth library can surface them. Otherwise we render an HTML error\n page so the user is not left on an opaque 422 / blank screen.", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7b0e92d81ae..3d22923c0a8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4388,7 +4388,12 @@ class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase): agent_ids: tuple[str, ...] = () +class TeamInfoMember(Member): + user_alias: str | None = None + + class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable): + members_with_roles: tuple[TeamInfoMember, ...] = () team_member_budget_table: LiteLLM_BudgetTableFull | None = None # Resources inherited from access groups (separate from direct assignments) access_group_models: list[str] | None = None diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 1efc9611fe6..18095aaafb4 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1022,6 +1022,19 @@ async def common_checks( fallback_spend=user_object.spend or 0.0, max_budget=user_budget, ) + call_info: Final = CallInfo( + spend=user_spend, + max_budget=user_budget, + user_id=user_object.user_id, + user_email=user_object.user_email, + event_group=Litellm_EntityType.USER, + ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="user_budget", + user_info=call_info, + ) + ) if math.isfinite(user_budget) and user_spend >= user_budget: raise litellm.BudgetExceededError( current_cost=user_spend, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e3a2b892721..9f58aaf24f1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2437,6 +2437,7 @@ class ProxyBaseLLMRequestProcessing: if self._is_streaming_request( data=self.data, is_streaming_request=is_streaming_request ) or self._is_streaming_response(response): # use generate_responses to stream responses + selected_data_generator: AsyncGenerator[str, None] | None = None # Call response headers hook for streaming success stream_callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook( data=self.data, @@ -2561,14 +2562,9 @@ class ProxyBaseLLMRequestProcessing: None if _should_return_raw_model_name(self.data) else requested_model_from_client ), ) - return await create_response( - generator=wrap_sse_stream_with_keepalive_pings( - stream=selected_data_generator, - ping_interval_seconds=litellm.anthropic_sse_ping_interval_seconds, - ), - media_type="text/event-stream", - headers=custom_headers, - request=request, + selected_data_generator = wrap_sse_stream_with_keepalive_pings( + stream=selected_data_generator, + ping_interval_seconds=litellm.anthropic_sse_ping_interval_seconds, ) # Non-streaming response - fall through to normal response handling elif select_data_generator: @@ -2595,6 +2591,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, ) ) + if selected_data_generator is not None: return await create_response( generator=selected_data_generator, media_type="text/event-stream", diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 1d4024bb84e..dc329e55e31 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -3,6 +3,7 @@ import asyncio import gc import json import os +import socket import sys import tracemalloc from collections import Counter @@ -237,6 +238,80 @@ def _process_memory_usage(process: _ProcessHandle) -> _ProcessMemoryUsage: ) +PROC_STATM_PATH: Final = "/proc/self/statm" +PROC_MEMINFO_PATH: Final = "/proc/meminfo" +PSUTIL_MISSING_ERROR: Final = "Install psutil for memory monitoring: pip install psutil" + + +class _ProcMemoryInfo(NamedTuple): + rss: int + vms: int + + +class _ProcFilesystemProcess: + """Memory of the running process read from the Linux proc filesystem, for images without psutil.""" + + def __init__( + self, + statm_path: str = PROC_STATM_PATH, + meminfo_path: str = PROC_MEMINFO_PATH, + page_size: int | None = None, + ) -> None: + self._statm_path: Final = statm_path + self._meminfo_path: Final = meminfo_path + self._page_size: Final = os.sysconf("SC_PAGE_SIZE") if page_size is None else page_size + + def memory_info(self) -> _ProcMemoryInfo: + with open(self._statm_path, encoding="ascii") as statm: + size_pages, resident_pages = statm.read().split()[:2] + return _ProcMemoryInfo(rss=int(resident_pages) * self._page_size, vms=int(size_pages) * self._page_size) + + def memory_percent(self) -> float: + with open(self._meminfo_path, encoding="ascii") as meminfo: + total_kilobytes: Final = next(int(line.split()[1]) for line in meminfo if line.startswith("MemTotal:")) + return self.memory_info().rss / (total_kilobytes * 1024) * 100 + + +def _process_handle() -> _ProcessHandle | None: + try: + import psutil + except ImportError: + return _ProcFilesystemProcess() if os.path.exists(PROC_STATM_PATH) else None + return psutil.Process() + + +def _health_status(memory_percent: float) -> str: + if memory_percent > 80: + return "critical" + if memory_percent > 60: + return "warning" + return "healthy" + + +class _SummaryProcessMemory(TypedDict, total=False): + summary: ReadOnly[str] + ram_usage_mb: ReadOnly[float] + system_memory_percent: ReadOnly[float] + error: ReadOnly[str] + + +def _summary_process_memory(process: _ProcessHandle | None) -> tuple[_SummaryProcessMemory, str]: + if process is None: + missing: Final[_SummaryProcessMemory] = {"error": PSUTIL_MISSING_ERROR} + return missing, "healthy" + try: + usage: Final = _process_memory_usage(process) + except Exception as e: + unreadable: Final[_SummaryProcessMemory] = {"error": str(e)} + return unreadable, "healthy" + memory: Final[_SummaryProcessMemory] = { + "summary": f"{usage.resident_megabytes:.1f} MB ({usage.percent:.1f}% of system memory)", + "ram_usage_mb": round(usage.resident_megabytes, 2), + "system_memory_percent": round(usage.percent, 2), + } + return memory, _health_status(usage.percent) + + @router.get("/debug/memory/summary", include_in_schema=False) async def get_memory_summary( _: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -246,6 +321,7 @@ async def get_memory_summary( Returns: - worker_pid: Process ID + - hostname: Host (the pod on Kubernetes) the worker runs on - status: Overall health based on memory usage - memory: Process memory usage and RAM info - caches: Cache item counts and descriptions @@ -263,35 +339,7 @@ async def get_memory_summary( user_api_key_cache, ) - # Get process memory info - process_memory = {} - health_status = "healthy" - - try: - import psutil - - usage: Final = _process_memory_usage(psutil.Process()) - memory_mb: Final = usage.resident_megabytes - memory_percent: Final = usage.percent - - process_memory = { - "summary": f"{memory_mb:.1f} MB ({memory_percent:.1f}% of system memory)", - "ram_usage_mb": round(memory_mb, 2), - "system_memory_percent": round(memory_percent, 2), - } - - # Check memory health status - if memory_percent > 80: - health_status = "critical" - elif memory_percent > 60: - health_status = "warning" - else: - health_status = "healthy" - - except ImportError: - process_memory["error"] = "Install psutil for memory monitoring: pip install psutil" - except Exception as e: - process_memory["error"] = str(e) + process_memory, health_status = _summary_process_memory(_process_handle()) # Get cache information caches: Final[dict[str, object]] = {} @@ -347,6 +395,7 @@ async def get_memory_summary( return { "worker_pid": os.getpid(), + "hostname": socket.gethostname(), "status": health_status, "memory": process_memory, "caches": { diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index e93bc96da69..54021e68980 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -74,6 +74,8 @@ DisablePreparedStatementsFlag = Annotated[ bool, BeforeValidator(partial(token_auth_flag_enabled, env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR)) ] MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR: Final = "DATABASE_MAX_IDLE_CONNECTION_LIFETIME" +DATABASE_SSLMODE_ENV_VAR: Final = "DATABASE_SSLMODE" +DATABASE_SSLROOTCERT_ENV_VAR: Final = "DATABASE_SSLROOTCERT" # schema.prisma pins `provider = "postgresql"`, so these are the only schemes # Prisma can actually connect with. @@ -135,6 +137,7 @@ def add_missing_query_params(url: str, params: Mapping[str, str | int | float]) LIBPQ_VERIFY_SSLMODES: Final[frozenset[str]] = frozenset({"verify-ca", "verify-full"}) +PRISMA_TLS_PARAM_KEYS: Final[frozenset[str]] = frozenset({"sslmode", "sslcert", "sslaccept"}) PEM_CERT_HEADER: Final = b"-----BEGIN CERTIFICATE-----" PG_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103) TLS_PROBE_TIMEOUT_SECONDS: Final = 10.0 @@ -263,6 +266,19 @@ def connection_params_from_url(url: str) -> Mapping[str, str | int | float]: ) +def token_refresh_params_from_url(url: str) -> Mapping[str, str | int | float]: + """Return the params a re-minted token URL carries over from the URL it replaces. + + The pool and timeout params plus Prisma's TLS params (already translated from + libpq spelling), so a refreshed URL keeps verifying the server the way the + first one did. + """ + kept: Final = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS + return MappingProxyType( + {key: value for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query) if key in kept} + ) + + def unsupported_db_scheme(database_url: str) -> str | None: """Return the connection URL scheme when it is not PostgreSQL, else None. @@ -312,6 +328,9 @@ class DatabaseURLSettings(BaseSettings): default=None, validation_alias=MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR ) + database_sslmode: str | None = Field(default=None, validation_alias=DATABASE_SSLMODE_ENV_VAR) + database_sslrootcert: str | None = Field(default=None, validation_alias=DATABASE_SSLROOTCERT_ENV_VAR) + # Writer database_url: str | None = Field(default=None, validation_alias="DATABASE_URL") direct_url: str | None = Field(default=None, validation_alias="DIRECT_URL") @@ -353,6 +372,25 @@ class DatabaseURLSettings(BaseSettings): azure_postgresql_auth=self.azure_postgresql_auth, ) + def tls_params(self) -> Mapping[str, str]: + """``sslmode`` / ``sslrootcert`` query params for every URL assembled from the discrete vars. + + A root cert on its own means ``verify-full``: under libpq's default + ``prefer`` the CA would never be consulted, and PgBouncer would dial + Postgres unverified with the bundle loaded. + """ + sslmode: Final = self.database_sslmode or ("verify-full" if self.database_sslrootcert else None) + return MappingProxyType( + { + key: value + for key, value in ( + ("sslmode", sslmode), + ("sslrootcert", self.database_sslrootcert), + ) + if value + } + ) + def build_writer_url(self) -> str | None: """Return the writer URL to set, or ``None`` to leave it as-is. @@ -362,6 +400,12 @@ class DatabaseURLSettings(BaseSettings): A ``DATABASE_URL`` the supervisor pointed at the in-container PgBouncer is kept even under token auth: the pooler renews the token upstream. """ + assembled: Final = self._assemble_writer_url() + if assembled is None: + return None + return add_missing_query_params(assembled, self.tls_params()) + + def _assemble_writer_url(self) -> str | None: auth: Final = self.token_auth() if auth is not None and database_url_is_pooled(): return None @@ -411,6 +455,12 @@ class DatabaseURLSettings(BaseSettings): pre-existing ``DATABASE_URL_READ_REPLICA``. Reader fields fall back to the writer's values. """ + assembled: Final = self._assemble_reader_url() + if assembled is None: + return None + return add_missing_query_params(assembled, self.tls_params()) + + def _assemble_reader_url(self) -> str | None: if not self.database_host_read_replica: return None # reader is opt-in if self.database_url_read_replica: diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 21b73f27a80..acd01b0e99e 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta from typing import Any, Final, Protocol from litellm._logging import verbose_proxy_logger -from litellm.proxy.db.db_url_settings import add_missing_query_params, connection_params_from_url +from litellm.proxy.db.db_url_settings import add_missing_query_params, token_refresh_params_from_url from litellm.proxy.db.token_auth import ( DEFAULT_POSTGRES_PORT, DatabaseTokenAuth, @@ -441,7 +441,7 @@ class PrismaWrapper: endpoint: Final = self._iam_endpoint if self._iam_endpoint is not None else self._endpoint_from_env() db_url: Final = add_missing_query_params( endpoint.build_url(mint_database_token(auth, endpoint)), - connection_params_from_url(os.environ.get(self._db_url_env_var, "")), + token_refresh_params_from_url(os.environ.get(self._db_url_env_var, "")), ) os.environ[self._db_url_env_var] = db_url return db_url diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index e35b1c8c82b..0131a67db8b 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -104,6 +104,15 @@ class SpendCounterReseed: SpendCounterReseed._locks.popitem(last=False) return lock + @staticmethod + async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None: + """Apply local deltas after an in-flight reseed establishes the spend balance.""" + lock: Final = await SpendCounterReseed._get_lock(counter_key) + async with lock: + return await spend_counter_cache.async_increment_cache( + key=counter_key, value=increment, local_only=True, refresh_ttl=True + ) + @staticmethod async def from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None: """ @@ -245,7 +254,10 @@ class SpendCounterReseed: value=current_value, ) else: - await spend_counter_cache.async_increment_cache(key=counter_key, value=db_spend, refresh_ttl=True) + cached_spend: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) + seeded_spend: Final = max(db_spend, float(cached_spend)) if cached_spend is not None else db_spend + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=seeded_spend) + return seeded_spend except Exception: verbose_proxy_logger.exception( "SpendCounterReseed.coalesced: failed to warm counter %s", @@ -438,11 +450,16 @@ class SpendCounterReseed: value=current_value, ) else: - await spend_counter_cache.async_increment_cache(key=counter_key, value=window_spend) + cached_spend: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) + seeded_spend: Final = ( + max(window_spend, float(cached_spend)) if cached_spend is not None else window_spend + ) + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=seeded_spend) + return seeded_spend except Exception: verbose_proxy_logger.exception( "SpendCounterReseed.coalesced_window: failed to warm counter %s", counter_key, ) raise - return window_spend + return current_value diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 6e29d44662e..de9618a44a1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -338,10 +338,9 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai estimated cost) and the ``azure`` provider label to the recorded guardrail information. Follows the OpenAI moderation override pattern (openai/moderations.py).""" - guardrail_response: Final[dict | str] = ( # mutable-ok: mirrors CustomGuardrail._process_response - ("mask" if self._inputs_were_modified(original_inputs, response) else "allow") - if original_inputs is not None and isinstance(response, dict) - else ({} if response is None else response) # mutable-ok: empty placeholder, never mutated + guardrail_response: Final = self._summarize_guardrail_response( + response=response, + original_inputs=original_inputs, ) self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=guardrail_response, diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index f7f500b1adc..8fed1f906e5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -18,6 +18,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_tool_message_for_guardrail, ) from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -261,6 +262,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): fail_on_error: bool | None = True, streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, + async_handler: AsyncHTTPHandler | None = None, **kwargs, ) -> None: """ @@ -273,9 +275,13 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): streaming_end_of_stream_only (bool | None): Scan streamed output once at end of stream instead of every streaming_sampling_rate chunks. Defaults to False. streaming_sampling_rate (int | None): Scan the accumulated streamed output every Nth chunk. Defaults to 5. + async_handler (AsyncHTTPHandler | None): HTTP client to call AI Guard with. Defaults to the shared + guardrail-callback client. **kwargs: Additional arguments passed to the CustomGuardrail base class. """ - self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) self.fail_on_error = True if fail_on_error is None else fail_on_error self._set_streaming_params( CrowdStrikeAIDRGuardrailConfigModelOptionalParams( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 219f6f270ed..b1e4f6fd9c3 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -32,32 +32,36 @@ from litellm.router_utils.auto_router_model_naming import ( strategy_router_dependencies, ) -ILLEGAL_DISPLAY_PARAMS: Final = [ - "messages", - "api_key", - "prompt", - "input", - "client_secret", - "azure_ad_token", - "azure_username", - "azure_password", - "vertex_credentials", - "vertex_ai_credentials", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "aws_web_identity_token", - "extra_headers", - "headers", - "exception", # internal; not JSON-serializable, never for display - "litellm_metadata", # internal tracking metadata with auth objects; not for display -] # Provider routing fields. Allowed for proxy admins so they can see which # region/version a deployment is checking; gated at the endpoint layer for # non-admin callers (see _strip_admin_only_fields_from_health_result). -ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version") +ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version", "aws_bedrock_runtime_endpoint") -MINIMAL_DISPLAY_PARAMS: Final = ["model", "mode_error"] +MINIMAL_DISPLAY_PARAMS: Final = frozenset({"model", "mode_error"}) + +HEALTH_DISPLAY_PARAMS: Final = ( + MINIMAL_DISPLAY_PARAMS + | frozenset(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS) + | frozenset( + { + "custom_llm_provider", + "mode", + "base_model", + "aws_region_name", + "region_name", + "watsonx_region_name", + "vertex_project", + "vertex_location", + "tpm", + "rpm", + "error", + "raw_request_typed_dict", + "x-ratelimit-remaining-requests", + "x-ratelimit-remaining-tokens", + "x-ms-region", + } + ) +) # Modes whose health-check probe is a chat-style completion call and # therefore accept `max_tokens`. Other modes (embedding, image_generation, @@ -143,14 +147,10 @@ def _get_random_llm_message(): def _clean_endpoint_data(endpoint_data: dict, details: bool | None = True): """ - Clean the endpoint data for display to users. + Keep only the explicitly approved, JSON-safe diagnostic fields for display to users. """ - endpoint_data.pop("litellm_logging_obj", None) - return ( - {k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS} - if details is not False - else {k: v for k, v in endpoint_data.items() if k in MINIMAL_DISPLAY_PARAMS} - ) + displayed: Final = HEALTH_DISPLAY_PARAMS if details is not False else MINIMAL_DISPLAY_PARAMS + return {k: v for k, v in endpoint_data.items() if k in displayed} def health_check_filter_kwargs_from_general_settings( @@ -258,8 +258,52 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None: return params.get("model") if isinstance(params, Mapping) else None +def _owner_team_id(deployment: Mapping[str, object]) -> str | None: + info: Final = deployment.get("model_info") + owner: Final = info.get("team_id") if isinstance(info, Mapping) else None + return owner if isinstance(owner, str) else None + + +def _team_public_model_name(deployment: Mapping[str, object]) -> str | None: + info: Final = deployment.get("model_info") + name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None + return name if isinstance(name, str) else None + + +def _deployments_routed_by_name( + model_list: Sequence[Mapping[str, object]], model_name: str, team_id: str | None +) -> tuple[Mapping[str, object], ...]: + """The deployments a request for ``model_name`` from this caller routes to. + + A team's own copies published under that name win, then deployments carrying it as + ``model_name``. A caller with no team reaches a public name only when nothing carries + it as ``model_name``, and only an admin still has another team's deployment in a + scoped ``model_list`` by then. + """ + own_copies: Final = tuple( + x + for x in model_list + if team_id is not None and _owner_team_id(x) == team_id and _team_public_model_name(x) == model_name + ) + if own_copies: + return own_copies + by_name: Final = tuple(x for x in model_list if x.get("model_name") == model_name) + if by_name or team_id is not None: + return by_name + return tuple(x for x in model_list if _team_public_model_name(x) == model_name) + + +def deployments_targeted_by_name( + model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None +) -> tuple[Mapping[str, object], ...]: + """``model`` targets deployments the way a request for it routes, else by ``litellm_params.model``.""" + return _deployments_routed_by_name(model_list, model, team_id) or tuple( + x for x in model_list if _deployment_model(x) == model + ) + + def _narrow_to_target( - model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None + model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None, team_id: str | None ) -> tuple[Mapping[str, object], ...]: """Narrow to the requested deployment. An id matching nothing keeps the whole list.""" if model_id is not None: @@ -267,8 +311,7 @@ def _narrow_to_target( return by_id or tuple(model_list) if model is None: return tuple(model_list) - by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) - return by_param or tuple(x for x in model_list if x.get("model_name") == model) + return deployments_targeted_by_name(model_list, model, team_id) def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool: @@ -813,13 +856,18 @@ async def perform_health_check( instrumentation_context: dict | None = None, health_check_skip_disabled_background_models: bool = False, router: "Router | None" = None, + team_id: str | None = None, ): """ Perform a health check on the system. When model_id is provided, only the deployment with that id is checked (so models that share the same name but have different ids are checked separately). - When model (name) is provided, all deployments matching that name are checked. + When model (name) is provided, the deployments a request for that name from the + caller (``team_id``) would route to are checked: the caller's team copies published + under that name, else the deployments named that way, else a public name that only + another team's deployment carries, else the deployments whose ``litellm_params.model`` + is that string. When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments @@ -850,7 +898,7 @@ async def perform_health_check( cycle_start_time: Final = time.monotonic() requested_model_count: Final = len(model_list) skip_disabled: Final = health_check_skip_disabled_background_models - narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id), skip_disabled) + narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled) if not narrowed: if instrumentation_enabled: logger.debug( diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 475780cb005..2ca502bf3b4 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -36,6 +36,9 @@ from litellm.proxy._types import ( UserAPIKeyAuth, WebhookEvent, ) +from litellm.proxy.auth.auth_checks import ( + _resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request +) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check ) @@ -44,6 +47,7 @@ from litellm.proxy.auth.master_key_policy import ( alternative_auth_enabled, insecure_master_key_reason, ) +from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.health_check_latest import LatestHealthCheckRow @@ -52,6 +56,7 @@ from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, _clean_endpoint_data, _update_litellm_params_for_health_check, + deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, perform_health_check, run_with_timeout, @@ -63,6 +68,7 @@ from litellm.proxy.middleware.in_flight_requests_middleware import ( get_in_flight_requests, ) from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager +from litellm.router import Router from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, @@ -922,7 +928,7 @@ def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: def _strip_admin_only_fields_from_health_result(result: dict) -> dict: """ Return a copy of the /health response with provider routing fields - (``api_base``, ``api_version``) removed from each healthy/unhealthy + (``ADMIN_ONLY_HEALTH_DISPLAY_PARAMS``) removed from each healthy/unhealthy endpoint entry. Used to hide those fields from non-admin callers while still showing them which deployments they own and whether each one is healthy. Proxy admins receive the unmodified result. @@ -936,41 +942,68 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict: return out -def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None: +def _health_accessible_model_names( + user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None +) -> frozenset[str] | None: + """Model names the caller may health-check, or None when the key is unrestricted.""" + granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict) + if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models: + return None + if llm_router is None: + return frozenset(granted_models) + return frozenset( + get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=llm_router.get_model_names(team_id=user_api_key_dict.team_id), + model_access_groups=llm_router.get_model_access_groups(), + ) + ) + + +def _caller_may_probe_deployment( + deployment: Mapping[str, object], + allowed_models: frozenset[str] | None, + llm_router: Router | None, + team_id: str | None, + caller_is_admin: bool, +) -> bool: + """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" + if not caller_is_admin and not Router._deployment_usable_by_team(deployment, team_id): + return False + if allowed_models is None: + return True + if llm_router is None: + return deployment.get("model_name") in allowed_models + model: Final = dict(deployment) + return any( + llm_router.should_include_deployment(model_name=name, model=model, team_id=team_id) for name in allowed_models + ) + + +def _resolve_targeted_model_ids( + model_list: list, model: str | None, model_id: str | None, team_id: str | None +) -> set | None: """ Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of - deployment IDs the response should be scoped to. + deployment IDs the response should be scoped to, mirroring the live-path + narrowing in ``perform_health_check()``: ``model_id`` wins when given and + matches ``model_info.id`` only; ``model`` targets the deployments a request + for that name from the caller would route to, else those whose + ``litellm_params.model`` provider string is that value (``deployments_targeted_by_name``). - Mirrors the live-path semantics in ``perform_health_check()``: ``model`` - matches either the deployment's ``model_name`` alias or its - ``litellm_params.model`` provider string. ``model_id`` matches - ``model_info.id``. - - Both query params are validated against the supplied ``model_list``. - Callers pass an already-scoped list (filtered to the caller's allowed - models for non-admins, full list for admins), so a ``model_id`` that - isn't present resolves to an empty set rather than a single-element - set — preventing a non-admin from reading another deployment's cached - health entry by guessing its ID. - - Returns ``None`` when no targeting is requested — callers should treat - that as "no filter." + Callers pass an already-scoped list, so a ``model_id`` outside the + caller's scope resolves to an empty set and never to the unvalidated id. + Returns ``None`` when no targeting is requested. """ - if not model and not model_id: + if model_id: + return {i for m in model_list if (i := (m.get("model_info") or {}).get("id")) == model_id} + if not model: return None - target_ids: Final[set] = set() - for m in model_list: - deployment_id = (m.get("model_info") or {}).get("id") - if not deployment_id: - continue - if model_id and deployment_id == model_id: - target_ids.add(deployment_id) - continue - if model: - litellm_model = (m.get("litellm_params") or {}).get("model") - if m.get("model_name") == model or litellm_model == model: - target_ids.add(deployment_id) - return target_ids + return { + i + for m in deployments_targeted_by_name(model_list, model, team_id) + if (i := (m.get("model_info") or {}).get("id")) + } def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: set) -> dict: @@ -1051,8 +1084,12 @@ def _health_endpoint_resolve_target_model_name( model_id: str | None, llm_router, ) -> str | None: - """Map ``model_id`` (without ``model``) to ``model_name`` for live health checks.""" - if not model_id or model: + """Map ``model_id`` to its deployment's ``model_name`` for live health checks. + + ``model_id`` wins over ``model``, so an id no deployment carries is a 404 even + when it is paired with a known name. + """ + if not model_id: return model if llm_router is None: raise HTTPException( @@ -1138,7 +1175,9 @@ async def health_endpoint( response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE if is_admin: return result - response.headers["Litellm-Health-Field-Notice"] = "api_base and api_version are admin-only on this endpoint" + response.headers["Litellm-Health-Field-Notice"] = ( + f"{', '.join(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)} are admin-only on this endpoint" + ) return _strip_admin_only_fields_from_health_result(result) try: @@ -1162,32 +1201,24 @@ async def health_endpoint( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "Model list not initialized"}, ) - _llm_model_list = copy.deepcopy(llm_model_list) - ### FILTER MODELS FOR ONLY THOSE USER HAS ACCESS TO ### - # Live path: scope by model_name (every deployment has one). - # Cache path: scope by model_id (the cache is keyed on model_id). - # Consequence: a deployment whose model_name the caller can access - # but which lacks model_info.id will appear in the live /health - # response but NOT in the background-cache /health response. This is - # surfaced via the "warnings" field below so operators can fix the - # missing model_info.id rather than guess at the discrepancy. - # Keys granted SpecialModelNames.all_proxy_models carry the literal - # "all-proxy-models" entry, which matches no real model_name; treat - # them as unrestricted instead of filtering the list down to nothing. - # Keys granted SpecialModelNames.all_team_models inherit the parent - # team's allowlist (same semantics as get_key_models in - # model_checks.py). Without a team_id the sentinel cannot resolve and - # stays in the list, matching nothing; denied rather than - # unrestricted, mirroring _resolve_key_models_for_auth_check. - accessible_models = list(user_api_key_dict.models) - if SpecialModelNames.all_team_models.value in accessible_models and user_api_key_dict.team_id is not None: - accessible_models = list(user_api_key_dict.team_models) - restrict_to_allowed_models: Final = ( - len(accessible_models) > 0 and SpecialModelNames.all_proxy_models.value not in accessible_models - ) - if restrict_to_allowed_models: - allowed_models: Final = set(accessible_models) - _llm_model_list = [m for m in _llm_model_list if m.get("model_name") in allowed_models] + allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) + restrict_to_allowed_models: Final = not is_admin or allowed_models is not None + _llm_model_list: Final = [ + m + for m in copy.deepcopy(llm_model_list) + if not restrict_to_allowed_models + or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin) + ] + targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id) + if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": f"key not allowed to health-check model_id {model_id}" + if model_id + else f"key not allowed to health-check model {model}" + }, + ) if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the @@ -1195,7 +1226,6 @@ async def health_endpoint( # healthy_count, otherwise an unhealthy "foo" combined with any # other healthy model would still report healthy_count > 0 and # the targeted-503 path would never fire. - targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id) if restrict_to_allowed_models: allowed_model_ids: Final = { (m.get("model_info") or {}).get("id") @@ -1207,7 +1237,7 @@ async def health_endpoint( # intersection of "targeted" and "allowed." filter_ids: Final = targeted_ids if targeted_ids is not None else allowed_model_ids filtered: Final = _filter_health_check_results_by_model_ids(health_check_results, filter_ids) - if targeted_ids is None and not allowed_model_ids: + if targeted_ids is None and _llm_model_list and not allowed_model_ids: # Caller has accessible model_names but none of the # matching deployments expose a model_info.id, so the # cache filter (which keys on model_id) drops every @@ -1246,6 +1276,7 @@ async def health_endpoint( model_id=model_id, max_concurrency=health_check_concurrency, router=llm_router, + team_id=user_api_key_dict.team_id, **_hc_filter, ) return _post_process(router_result) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 1e946cc2e23..64da2ad00f6 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -180,7 +180,7 @@ class _ProxyDBLogger(CustomLogger): # here because the input above is constructed non-None. _error_information = cast( StandardLoggingPayloadErrorInformation, - _sanitize_error_information_for_spend_logs(_error_information), + _sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), ) _metadata["error_information"] = _error_information diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index c2805d00c2e..8f7b515c22a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2016,6 +2016,7 @@ async def add_litellm_data_to_request( "method": request.method, "headers": _logging_safe_headers, "body": None, # filled in post-strip; see below + "credential_fields": tuple(sorted(name for name in _TRANSPORT_ONLY_CREDENTIAL_KEYS if name in data)), "arrival_time": arrival_time, # Track when request arrived at proxy } diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c050368b3fe..e9e37540dd8 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -61,6 +61,7 @@ from litellm.proxy._types import ( SpecialProxyStrings, TeamAccessGroupModelGrant, TeamAddMemberResponse, + TeamInfoMember, TeamInfoResponseObject, TeamInfoResponseObjectTeamTable, TeamListResponseObject, @@ -4292,37 +4293,35 @@ async def _add_team_member_budget_table( return team_info_response_object -async def _hydrate_member_emails( +async def _hydrate_member_user_details( prisma_client: PrismaClient, members: Sequence[Member], -) -> tuple[Member, ...]: - """Fill in ``user_email`` for roster entries that were stored without one. - - ``members_with_roles`` is a denormalized snapshot written at add-time, so an entry - stored with ``user_email=None`` keeps that null even once the user row has an email. - Look the missing ones up in ``LiteLLM_UserTable`` (one indexed query) and fill them - in. A stored email is never overwritten - the snapshot stays the source of truth - wherever it has a value. - """ - missing_user_ids: Final = frozenset(m.user_id for m in members if not m.user_email and m.user_id is not None) - if not missing_user_ids: - return tuple(members) - - user_rows: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await _user_db(prisma_client).find_many( - where={ # mutable-ok: Prisma query filters are dict-shaped - "user_id": { # mutable-ok: Prisma query filters are dict-shaped - "in": sorted(missing_user_ids) +) -> tuple[TeamInfoMember, ...]: + """Attach ``user_alias`` and fill in a missing ``user_email`` from ``LiteLLM_UserTable`` in one query.""" + user_ids: Final = frozenset(m.user_id for m in members if m.user_id is not None) + user_rows: Final[Sequence[prisma_models.LiteLLM_UserTable]] = ( + await _user_db(prisma_client).find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_id": { # mutable-ok: Prisma query filters are dict-shaped + "in": sorted(user_ids) + } } - } + ) + if user_ids + else () ) - email_by_user_id: Final = MappingProxyType({u.user_id: u.user_email for u in user_rows if u.user_email}) + user_by_id: Final = MappingProxyType({u.user_id: u for u in user_rows}) - return tuple( - m.model_copy(update={"user_email": email_by_user_id[m.user_id]}) # mutable-ok: pydantic update payload - if not m.user_email and m.user_id is not None and m.user_id in email_by_user_id - else m - for m in members - ) + def hydrate(m: Member) -> TeamInfoMember: + user_row: Final = user_by_id.get(m.user_id) if m.user_id is not None else None + return TeamInfoMember( + role=m.role, + user_id=m.user_id, + user_email=m.user_email or (user_row.user_email if user_row is not None else None), + user_alias=user_row.user_alias if user_row is not None else None, + ) + + return tuple(hydrate(m) for m in members) async def _resolve_team_access_group_resources( @@ -4462,17 +4461,12 @@ async def team_info( # Resolve resources inherited from access groups resolved_team_info: Final = await _resolve_team_access_group_resources(_team_info) - # Fill in emails the add-time roster snapshot never captured - hydrated_members: Final = await _hydrate_member_emails( + hydrated_members: Final = await _hydrate_member_user_details( prisma_client=prisma_client, members=resolved_team_info.members_with_roles, ) hydrated_team_info: Final = resolved_team_info.model_copy( - update={ # mutable-ok: pydantic update payload - # list(), not the tuple: model_copy skips validation, so the field has - # to be handed the list[Member] the response model declares. - "members_with_roles": list(hydrated_members) # mutable-ok: declared list[Member] - } + update={"members_with_roles": hydrated_members} # mutable-ok: pydantic update payload ) response_object: Final = TeamInfoResponseObject( diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index ed193c7f434..ad45781d5d2 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -58,6 +58,15 @@ class UndeliverableStreamRewrite(Exception): self.guardrail_name: Final = guardrail_name +class UnappliableRequestRewrite(Exception): + def __init__(self, guardrail_name: str) -> None: + super().__init__( + f"Guardrail '{guardrail_name}' rewrote the request in a way this endpoint cannot apply, " + "so the request was rejected rather than sent unrewritten" + ) + self.guardrail_name: Final = guardrail_name + + def _tool_call_shape(tool_call: object) -> tuple[object, object]: plain: Final = tool_call.model_dump() if isinstance(tool_call, BaseModel) else tool_call function: Final = plain.get("function") if isinstance(plain, Mapping) else None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b16e09b8948..3c70f51da2d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3377,10 +3377,8 @@ async def _increment_spend_counter_cache(counter_key: str, increment: float): ) return current_value - return await spend_counter_cache.async_increment_cache( - key=counter_key, - value=increment, - refresh_ttl=True, + return await SpendCounterReseed.increment_in_memory( + spend_counter_cache=spend_counter_cache, counter_key=counter_key, increment=increment ) @@ -3403,10 +3401,8 @@ async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncreme redis_cache: Final = spend_counter_cache.redis_cache if redis_cache is None: for item in pending: - await spend_counter_cache.async_increment_cache( - key=item.counter_key, - value=item.increment, - refresh_ttl=True, + await SpendCounterReseed.increment_in_memory( + spend_counter_cache=spend_counter_cache, counter_key=item.counter_key, increment=item.increment ) return ttl: Final = redis_cache.get_ttl() diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 3d0bd5e61c9..20b4708c193 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -145,12 +145,14 @@ ROUTE_ENDPOINT_MAPPING: Final = { } +_AVAILABLE_MODELS_HINT: Final = "Call `/v1/models` to view available models for your key." + + class ProxyModelNotFoundError(HTTPException): def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True): self.retryable_with_model_read_through: Final = retryable_with_model_read_through - detail: Final = { - "error": f"{route}: Invalid model name passed in model={model_name}. Call `/v1/models` to view available models for your key." - } + self.spend_log_error_message: Final = f"{route}: Invalid model name passed in. {_AVAILABLE_MODELS_HINT}" + detail: Final = {"error": f"{route}: Invalid model name passed in model={model_name}. {_AVAILABLE_MODELS_HINT}"} super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ef21551ca93..eedec0619db 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -34,7 +34,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router -from litellm.rust_bridge.token_counter import count_anthropic_input_tokens, uses_anthropic_tokenizer +from litellm.rust_bridge.token_counter import RustTokenizer, count_input_tokens, rust_tokenizer from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -1365,8 +1365,9 @@ async def count_request_input_tokens( Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so counting a large prompt inline stalls every other request on the worker. - Models on the Anthropic tokenizer are counted from the raw body by the Rust - bridge when it is enabled, which parses and tokenizes with the GIL released. + Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base + and o200k_base) are counted from the raw body by the bridge when it is enabled, once per + distinct tokenizer, which parses and tokenizes with the GIL released. Everything it declines is counted in Python, large prompts in a worker thread. The counts are reused by both the max-cost and the input-cost estimate. @@ -1374,23 +1375,31 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - rust_count: Final = ( - await count_anthropic_input_tokens(raw_body) - if raw_body is not None and any(uses_anthropic_tokenizer(model) for model in models) - else None + tokenizers: Final[Mapping[str, RustTokenizer | None]] = MappingProxyType( + {model: rust_tokenizer(model) for model in models} + ) + distinct_tokenizers: Final[tuple[RustTokenizer, ...]] = tuple( + dict.fromkeys(tokenizer for tokenizer in tokenizers.values() if tokenizer is not None) + ) + rust_counts_by_tokenizer: Final[Mapping[RustTokenizer, int]] = MappingProxyType( + { + tokenizer: count.input_tokens + for tokenizer in distinct_tokenizers + if raw_body is not None and (count := await count_input_tokens(raw_body, tokenizer)) is not None + } ) rust_counts: Final = MappingProxyType( { - model: rust_count.input_tokens - for model in models - if rust_count is not None and uses_anthropic_tokenizer(model) + model: rust_counts_by_tokenizer[tokenizer] + for model, tokenizer in tokenizers.items() + if tokenizer is not None and tokenizer in rust_counts_by_tokenizer } ) python_models: Final = tuple(model for model in models if model not in rust_counts) - if not python_models: - return rust_counts python_counts: Final = ( - _count_input_tokens_for_models(request_body=request_body, models=python_models) + MappingProxyType({}) + if not python_models + else _count_input_tokens_for_models(request_body=request_body, models=python_models) if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS else await asyncio.to_thread( _count_input_tokens_for_models, @@ -1398,6 +1407,7 @@ async def count_request_input_tokens( models=python_models, ) ) + verbose_proxy_logger.debug("input token counts: rust=%s python=%s", dict(rust_counts), dict(python_counts)) return MappingProxyType({**rust_counts, **python_counts}) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index da79328fa59..8cfb6354dd0 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2925,6 +2925,32 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] # mutable-ok: rows are enriched in place +async def _count_grouped_sessions( + prisma_client: "PrismaClient", + where_clause: str, + sql_params: Sequence[object], + next_param_index: int, +) -> tuple[int, bool]: + """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" + count_query: Final = f""" + SELECT COUNT(*) AS total_count + FROM ( + SELECT 1 + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + GROUP BY {_SESSION_GROUP_KEY_SQL} + LIMIT ${next_param_index} + ) AS bounded_sessions + """ + count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( + prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + ) + raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 + return ( + (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) + ) + + async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], @@ -2944,11 +2970,19 @@ async def _ui_session_grouped_spend_logs( next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, api_key)``, resumed from the ``session_cursor`` keyset ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. Each session is represented + page depth does not degrade the query plan. A request for ``page > 1`` + without a cursor (the UI jumping straight to the last page, or back to a + page it never walked through) falls back to ``OFFSET (page - 1) * + page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` + window the capped ``total`` promises, so a page never runs past that total + and one starting at or past it returns no rows without a query. Each session is represented by its newest non-MCP row, enriched by ``_build_ui_spend_logs_response`` exactly like the flat listing, and the response carries ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). + (capped like the flat total). A page that runs out of sessions while still + holding some is itself the end of the list, so its ``total`` is + ``offset + len(page)`` and the grouped count query is skipped; a page that + starts past the end says nothing about the total, so that one is counted. """ where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" cmp_op: Final = "<" if sort_desc else ">" @@ -2963,6 +2997,10 @@ async def _ui_session_grouped_spend_logs( ) cursor_params: Final[tuple[object, ...]] = cursor if cursor else () limit_index: Final = next_param_index + len(cursor_params) + offset: Final = (page - 1) * page_size if cursor is None else 0 + page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) + offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () + offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" page_query: Final = f""" SELECT {_SESSION_KEY_EXPR} AS session_key, @@ -2973,36 +3011,29 @@ async def _ui_session_grouped_spend_logs( GROUP BY {_SESSION_GROUP_KEY_SQL} {having_clause} ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} + LIMIT ${limit_index} {offset_clause} """ - page_rows: Final[Sequence[_SessionPageRow]] = await _query_raw( - prisma_client, page_query, *sql_params, *cursor_params, page_size + 1 + page_rows: Final[Sequence[_SessionPageRow]] = ( + () + if page_limit <= 0 + else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) ) - has_more: Final = len(page_rows) > page_size - visible_rows: Final = page_rows[:page_size] + has_more: Final = len(page_rows) > page_limit + visible_rows: Final = page_rows[:page_limit] next_cursor: Final = ( f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" if has_more and visible_rows else None ) - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 + page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list + total_records, total_is_capped = ( + (offset + len(page_rows), False) + if page_ends_the_list + else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - total_is_capped: Final = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP - total_records: Final = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 01398d38687..3fd20bb7e81 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1167,6 +1167,7 @@ def _redact_prompt_fields_in_guardrail_entry( def _sanitize_error_information_for_spend_logs( error_information: StandardLoggingPayloadErrorInformation | None, + original_exception: BaseException | None = None, ) -> StandardLoggingPayloadErrorInformation | None: """ Sanitize ``error_information`` before it lands in ``LiteLLM_SpendLogs.metadata``. @@ -1188,7 +1189,12 @@ def _sanitize_error_information_for_spend_logs( if error_information is None: return None - sanitized = cast(dict, {**error_information}) + persisted: Final = ( + {**error_information, "error_message": original_exception.spend_log_error_message} + if isinstance(original_exception, ProxyModelNotFoundError) + else error_information + ) + sanitized = cast(dict, {**persisted}) if not should_store_prompts_and_responses_in_spend_logs(): for field in ("error_message", "traceback"): diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8e6142090a4..89625021e37 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -121,6 +121,11 @@ from litellm.proxy.db.create_views import ( should_create_missing_views, ) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.db_url_settings import ( + DatabaseURLSettings, + add_missing_query_params, + token_refresh_params_from_url, +) from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, @@ -4054,7 +4059,10 @@ class PrismaClient: # loop and times out after 30s. if token_auth is not None and reader_iam_endpoint is not None: reader_token: Final = mint_database_token(token_auth, reader_iam_endpoint) - read_replica_url = reader_iam_endpoint.build_url(reader_token) + read_replica_url = add_missing_query_params( + reader_iam_endpoint.build_url(reader_token), + token_refresh_params_from_url(read_replica_url), + ) os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url reader_kwargs: Final[dict[str, Any]] = {"datasource": {"url": read_replica_url}} if http_client is not None: @@ -7807,7 +7815,7 @@ def construct_database_url_from_env_vars() -> str | None: if database_schema: database_url += f"?schema={database_schema}" - return database_url + return add_missing_query_params(database_url, DatabaseURLSettings.from_env().tls_params()) return None diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index b824a5928c6..44c47af57f4 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -33,7 +33,7 @@ from litellm.utils import ProviderConfigManager from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ..llms.azure.common_utils import get_azure_ad_token -from ..llms.azure.realtime.handler import AzureOpenAIRealtime +from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..llms.openai.realtime.handler import OpenAIRealtime @@ -413,14 +413,14 @@ async def _arealtime( api_version = api_version or litellm_params.api_version or "2024-10-01-preview" - realtime_protocol = ( + configured_realtime_protocol: Final = ( kwargs.get("realtime_protocol") or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - if realtime_protocol is None and (query_params or {}).get("intent") == "transcription": - realtime_protocol = "GA" - realtime_protocol = realtime_protocol or "beta" + realtime_protocol: Final = azure_realtime_protocol_for_client( + configured_realtime_protocol, query_params=query_params, websocket=websocket + ) resolved_azure_ad_token: Final = ( None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) ) @@ -586,9 +586,7 @@ def _azure_realtime_health_protocol( configured: Final = configured_raw if isinstance(configured_raw, str) else None if configured is not None: return configured, query_params - if query_params is not None: - return "GA", query_params - return "beta", None + return "GA", query_params def _realtime_health_check_auth_headers( @@ -621,8 +619,8 @@ async def _realtime_health_check( api_key: str - api key custom_llm_provider: str - custom llm provider realtime_protocol: Optional[str] - protocol version ("GA"/"v1" for GA path, "beta" for beta path); - None resolves it for Azure from model_params/env, with transcription-only models probing GA - plus intent=transcription the way real calls do + None resolves it for Azure from model_params/env and otherwise probes GA, the upstream a client + without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: bool - True if connection is successful, False otherwise diff --git a/litellm/router.py b/litellm/router.py index 597bcfaa20f..8865543badd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -124,9 +124,11 @@ from litellm.router_utils.add_retry_fallback_headers import ( add_retry_headers_to_response, apply_quality_router_decision_headers, apply_remaining_usage_headers, + complexity_router_decision_headers, ensure_response_additional_headers, get_hidden_params_dict, prepare_response_for_header_attachment, + replace_complexity_router_headers, response_in_flight_token_count, ) from litellm.router_utils.auto_router_model_naming import ( @@ -555,6 +557,7 @@ class FallbackAwareAnthropicMessagesStream: def __init__(self, async_generator: AsyncGenerator[bytes, None], source_iterator: object) -> None: self._async_generator = async_generator self._source_iterator = source_iterator + self.fallback_headers_adopted = False self._hidden_params = dict( # mutable-ok: mutated in place by merge_fallback_hidden_params getattr(source_iterator, "_hidden_params", None) or {} ) @@ -565,6 +568,7 @@ class FallbackAwareAnthropicMessagesStream: def adopt_fallback_source(self, fallback_response: object) -> None: self._source_iterator = fallback_response + self.fallback_headers_adopted = True def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": return self @@ -593,7 +597,9 @@ class FallbackAwareAnthropicMessagesStream: self._hidden_params = { # mutable-ok: matches _hidden_params' existing dict[str, object] shape **self._hidden_params, **fallback_hidden_params, - "additional_headers": {**existing_headers, **fallback_headers}, # mutable-ok: same shape + "additional_headers": dict( # mutable-ok: hidden params expect a writable header bag + replace_complexity_router_headers(existing_headers, fallback_headers) + ), } @@ -3128,6 +3134,8 @@ class Router: async generator. """ + fallback_headers_adopted: bool = False + def __init__(self, async_generator: AsyncGenerator): import time from datetime import datetime @@ -3179,6 +3187,12 @@ class Router: # api_base, additional_headers) keep flowing. self._hidden_params = dict(getattr(source_iterator, "_hidden_params", None) or {}) + def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]: + prepared: Final = Router._prepare_fallback_hidden_params(fallback_response) + self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} # mutable-ok: stream metadata + self.fallback_headers_adopted = True + return prepared + def __aiter__(self): return self @@ -3269,8 +3283,8 @@ class Router: include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True, ) + prepared_fallback_hidden_params = wrapper.adopt_fallback_headers(fallback_response) if hasattr(fallback_response, "__aiter__"): - prepared_fallback_hidden_params = Router._prepare_fallback_hidden_params(fallback_response) async for fallback_item in fallback_response: Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) if partial_usage is not None: @@ -3305,7 +3319,8 @@ class Router: exc, ) - return FallbackResponsesStreamWrapper(stream_with_fallbacks()) + wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks()) + return wrapper def _completion_streaming_iterator( self, @@ -11108,7 +11123,7 @@ class Router: self, response: object, model_group: str | None = None, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, ) -> Any: """ Add the most accurate rate limit headers for a given model response. @@ -11124,6 +11139,7 @@ class Router: additional_headers: Final = ensure_response_additional_headers(response) additional_headers["x-litellm-model-group"] = model_group apply_quality_router_decision_headers(additional_headers, request_kwargs) + additional_headers.update(complexity_router_decision_headers(request_kwargs)) if model_group is not None: remaining_usage: Final = await self.get_remaining_model_group_usage(model_group) diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index 3251ea457cf..bc88feef7d2 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -1,7 +1,10 @@ import json +import math +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final, Protocol, TypedDict, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError class FallbackErrorInfo(TypedDict): @@ -15,6 +18,68 @@ class _HiddenParamsHost(Protocol): _hidden_params: dict[str, object] +_EMPTY_OBJECT_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) +_ROUTING_HEADER_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_COMPLEXITY_ROUTER_HEADER_PREFIX: Final = "x-litellm-complexity-router-" + + +def _routing_header_mapping(value: object) -> Mapping[str, object]: + try: + mapping: Final[Mapping[str, object]] = _ROUTING_HEADER_MAPPING.validate_python(value, strict=True) + return mapping + except ValidationError: + return _EMPTY_OBJECT_MAPPING + + +def _header_string(value: object) -> str | None: + if not isinstance(value, str): + return None + normalized: Final = value.strip() + return normalized if normalized and all(" " <= character <= "~" for character in normalized) else None + + +def complexity_router_decision_headers(request_kwargs: object) -> Mapping[str, str]: + data: Final = _routing_header_mapping(request_kwargs) + metadata_key: Final = "litellm_metadata" if "litellm_metadata" in data else "metadata" + decision: Final = _routing_header_mapping(_routing_header_mapping(data.get(metadata_key)).get("routing_decision")) + if decision.get("router_type") != "complexity": + return MappingProxyType({}) + score: Final = decision.get("score") + values: Final = ( + ("tier", decision.get("tier")), + ("cause", decision.get("cause")), + ( + "score", + str(score) + if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score) + else None, + ), + ( + "reasoning-effort", + _routing_header_mapping(decision.get("tier_litellm_params")).get("reasoning_effort"), + ), + ) + return MappingProxyType( + { + f"{_COMPLEXITY_ROUTER_HEADER_PREFIX}{key}": header_value + for key, value in values + if (header_value := _header_string(value)) is not None + } + ) + + +def replace_complexity_router_headers( + existing_headers: Mapping[str, object], new_headers: Mapping[str, object] +) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in (*existing_headers.items(), *new_headers.items()) + if key in new_headers or not key.startswith(_COMPLEXITY_ROUTER_HEADER_PREFIX) + } + ) + + class HiddenParamsAsyncIteratorWrapper: """ Wraps a bare async generator/iterator (e.g. a provider's raw SSE diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index b7fdb5a98ef..89eab71ccba 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -2,13 +2,57 @@ from __future__ import annotations -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import Final, Protocol, cast # noqa: TID251 # native extension exposes dynamically typed callables import httpx -from litellm.rust_bridge.bindings import NativeBinding +import litellm +from litellm.constants import request_timeout +from litellm.llms.azure_ai.ocr.common_utils import is_azure_cohere_parse_model +from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse +from litellm.rust_bridge.bindings import NativeBinding, native_exception_types from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import ProviderConfigManager + +_RUST_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "vertex_ai"}) +_RUST_OCR_CONFIG_FIELDS: Final = frozenset( + { + "azure_ad_token", + "tenant_id", + "client_id", + "client_secret", + "azure_scope", + "azure_authority_host", + "azure_credential", + "azure_federated_token_file", + "vertex_credentials", + "vertex_ai_credentials", + "vertex_project", + "vertex_ai_project", + "vertex_location", + "vertex_ai_location", + } +) +_RUST_OCR_SECRET_FIELDS: Final = frozenset( + {"azure_ad_token", "client_secret", "azure_federated_token_file", "vertex_credentials", "vertex_ai_credentials"} +) + + +@dataclass(frozen=True, slots=True) +class LiteLLMOcrRequest: + model: str + document: Mapping[str, object] + api_key: str | None + api_base: str | None + timeout: float | httpx.Timeout | None + custom_llm_provider: str | None + extra_headers: dict[str, object] | None + kwargs: Mapping[str, object] + input_sources: Mapping[str, str] | None = None class RustOcr(Protocol): @@ -21,6 +65,7 @@ class RustOcr(Protocol): custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> dict[str, object]: raise NotImplementedError @@ -36,11 +81,32 @@ class RustAocr(Protocol): custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> Awaitable[dict[str, object]]: raise NotImplementedError +class _OCRLogging(Protocol): + def update_from_kwargs( + self, + *, + kwargs: dict[str, object], + model: str, + optional_params: dict[str, object], + litellm_params: dict[str, object], + custom_llm_provider: str | None, + ) -> None: ... + + def pre_call( + self, + *, + input: str, + api_key: str | None, + additional_args: dict[str, object], + ) -> None: ... + + def _as_ocr(value: object) -> RustOcr | None: return cast(RustOcr, value) if callable(value) else None @@ -61,6 +127,264 @@ def load_rust_aocr() -> RustAocr | None: return _AOCR.load() +def provider(request: LiteLLMOcrRequest) -> str | None: + if request.custom_llm_provider is not None: + return request.custom_llm_provider + prefix: Final = request.model.partition("/")[0] + if prefix in _RUST_OCR_PROVIDERS: + return prefix + if request.model.startswith("mistral-ocr"): + return "mistral" + return None + + +def supported(request: LiteLLMOcrRequest) -> bool: + request_provider: Final = provider(request) + if request_provider not in _RUST_OCR_PROVIDERS: + return False + if request_provider == "azure_ai": + return ( + not is_azure_cohere_parse_model(request.model) + and not callable(request.kwargs.get("azure_ad_token_provider")) + and request.kwargs.get("azure_username") is None + and request.kwargs.get("azure_password") is None + ) + return True + + +def _optional_params(request: LiteLLMOcrRequest, resolve_secret: Callable[[str], str | None]) -> Mapping[str, object]: + optional_params: Final = MappingProxyType( + { + name: value + for name, value in request.kwargs.items() + if (name not in GenericLiteLLMParams.model_fields or name in _RUST_OCR_CONFIG_FIELDS) + and name not in ("litellm_logging_obj", "aocr", "litellm_call_id", "proxy_server_request") + } + ) + request_provider: Final = provider(request) + if request_provider == "azure_ai" and litellm.enable_azure_ad_token_refresh is True: + return MappingProxyType({**optional_params, "enable_azure_ad_token_refresh": True}) + if request_provider != "vertex_ai": + return optional_params + project: Final = ( + request.kwargs.get("vertex_project") + or request.kwargs.get("vertex_ai_project") + or litellm.vertex_project + or resolve_secret("VERTEXAI_PROJECT") + ) + location: Final = ( + request.kwargs.get("vertex_location") + or request.kwargs.get("vertex_ai_location") + or litellm.vertex_location + or resolve_secret("VERTEXAI_LOCATION") + or resolve_secret("VERTEX_LOCATION") + ) + credentials: Final = ( + request.kwargs.get("vertex_credentials") + or request.kwargs.get("vertex_ai_credentials") + or resolve_secret("VERTEXAI_CREDENTIALS") + ) + vertex_params: Final = MappingProxyType( + { + name: value + for name, value in ( + ("vertex_project", project), + ("vertex_location", location), + ("vertex_credentials", credentials), + ) + if value is not None + } + ) + return MappingProxyType({**optional_params, **vertex_params}) + + +def _input_sources(request: LiteLLMOcrRequest, optional_params: Mapping[str, object]) -> Mapping[str, str]: + proxy_request_value: Final = request.kwargs.get("proxy_server_request") + if not isinstance(proxy_request_value, Mapping): + return MappingProxyType({}) + proxy_request: Final = cast( # cast-ok: runtime Mapping check narrows metadata with unknown key and value types + Mapping[object, object], proxy_request_value + ) + credential_fields_value: Final = proxy_request.get("credential_fields", ()) + credential_fields: Final = ( + frozenset(name for name in credential_fields_value if isinstance(name, str)) + if isinstance(credential_fields_value, (list, tuple, set, frozenset)) + else frozenset() + ) + request_fields_value: Final = proxy_request.get("body_fields") + request_fields: Sequence[object] + if isinstance(request_fields_value, Sequence) and not isinstance(request_fields_value, (str, bytes)): + request_fields = cast( # cast-ok: runtime Sequence check excludes scalar strings and bytes + Sequence[object], request_fields_value + ) + else: + body_value: Final = proxy_request.get("body") + request_fields = ( + tuple(cast(Mapping[object, object], body_value)) # cast-ok: runtime Mapping check establishes iterable keys + if isinstance(body_value, Mapping) + else () + ) + names: Final = frozenset(optional_params) | frozenset({"api_key", "api_base", "extra_headers"}) + request_sources: Final = MappingProxyType( + {name: "request" for name in names if name in request_fields or name in credential_fields} + ) + if litellm.enable_azure_ad_token_refresh is True and "enable_azure_ad_token_refresh" in optional_params: + return MappingProxyType({**request_sources, "enable_azure_ad_token_refresh": "deployment"}) + return request_sources + + +def _marshal( + request: LiteLLMOcrRequest, + resolve_secret: Callable[[str], str | None], + convert_file_document: Callable[[dict[str, object]], dict[str, str]], +) -> LiteLLMOcrRequest: + if not isinstance(request.document, dict): + raise TypeError(f"document must be a dict with 'type' and URL/file field, got {type(request.document)}") + document: Final = ( + convert_file_document(request.document) if request.document.get("type") == "file" else request.document + ) + request_provider: Final = provider(request) + api_key: Final = ( + request.api_key or resolve_secret("MISTRAL_API_KEY") if request_provider == "mistral" else request.api_key + ) + optional_params: Final = _optional_params(request, resolve_secret) + input_sources: Final = _input_sources(request, optional_params) + logged_optional_params: Final = MappingProxyType( + {name: "****" if name in _RUST_OCR_SECRET_FIELDS else value for name, value in optional_params.items()} + ) + logged_kwargs: Final = MappingProxyType( + { + name: "****" if name in _RUST_OCR_SECRET_FIELDS else value + for name, value in request.kwargs.items() + if name != "proxy_server_request" + } + ) + logging_obj: Final = cast( # cast-ok: client decorator injects the logging object through untyped kwargs + _OCRLogging, request.kwargs["litellm_logging_obj"] + ) + logging_obj.update_from_kwargs( + kwargs=dict(logged_kwargs), # mutable-ok: legacy logging mutates its kwargs copy + model=request.model, + optional_params=dict(logged_optional_params), # mutable-ok: legacy logging requires concrete dict params + litellm_params={ # mutable-ok: legacy logging requires a concrete params dict + "litellm_call_id": request.kwargs.get("litellm_call_id"), + "api_base": request.api_base, + }, + custom_llm_provider=request_provider, + ) + logging_obj.pre_call( + input="OCR document processing", + api_key=api_key, + additional_args={ # mutable-ok: pre_call mutates the additional_args dict + "complete_input_dict": { # mutable-ok: callbacks consume a JSON-serializable request dict + "model": request.model, + "document": document, + **logged_optional_params, + }, + "api_base": request.api_base or "", + "headers": request.extra_headers or {}, # mutable-ok: logging callbacks consume a concrete headers dict + }, + ) + return LiteLLMOcrRequest( + model=request.model, + document=document, + api_key=api_key, + api_base=request.api_base, + timeout=request.timeout if request.timeout is not None else request_timeout, + custom_llm_provider=request.custom_llm_provider, + extra_headers=request.extra_headers, + kwargs=optional_params, + input_sources=input_sources, + ) + + +def _map_error(error: Exception, request: LiteLLMOcrRequest) -> Exception: + exception_types: Final = native_exception_types() + if exception_types is None or not isinstance(error, exception_types[1]): + return error + request_provider: Final = provider(request) + if request_provider is None: + return error + provider_config: Final = ProviderConfigManager.get_provider_ocr_config( + model=request.model.removeprefix(f"{request_provider}/"), provider=litellm.LlmProviders(request_provider) + ) + if provider_config is None: + return error + error_args: Final = cast( # cast-ok: BaseException.args exposes Any while native errors carry scalar args + tuple[object, ...], error.args + ) + status: Final = error_args[0] if error_args and isinstance(error_args[0], int) else 500 + message: Final = str(error_args[1]) if len(error_args) > 1 else str(error) + error_factory: Final = cast( # cast-ok: legacy provider error factories have untyped callable parameters + Callable[..., Exception], provider_config.get_error_class + ) + return error_factory( + error_message=message, + status_code=status or 500, + headers={}, # mutable-ok: provider error factories require a concrete headers dict + ) + + +def _response(response: Mapping[str, object]) -> OCRResponse: + provider_native_response: Final = response.get(PROVIDER_NATIVE_RESPONSE_KEY) + normalized: Final = OCRResponse.model_validate( + MappingProxyType({key: value for key, value in response.items() if key != PROVIDER_NATIVE_RESPONSE_KEY}) + ) + if isinstance(provider_native_response, Mapping): + normalized.set_provider_native_response(provider_native_response) + return normalized + + +def run( + request: LiteLLMOcrRequest, + resolve_secret: Callable[[str], str | None], + convert_file_document: Callable[[dict[str, object]], dict[str, str]], +) -> OCRResponse | None: + if load_rust_ocr() is None: + return None + marshalled: Final = _marshal(request, resolve_secret, convert_file_document) + try: + response: Final = ocr( + model=marshalled.model, + document=dict(marshalled.document), # mutable-ok: PyO3 OCR binding requires a concrete dict + api_key=marshalled.api_key, + api_base=marshalled.api_base, + custom_llm_provider=marshalled.custom_llm_provider, + extra_headers=marshalled.extra_headers, + optional_params=dict(marshalled.kwargs), # mutable-ok: PyO3 OCR binding requires a concrete dict + input_sources=marshalled.input_sources, + timeout=marshalled.timeout, + ) + except Exception as error: + raise _map_error(error, request) from error + return _response(response) if response is not None else None + + +async def arun( + request: LiteLLMOcrRequest, + resolve_secret: Callable[[str], str | None], + convert_file_document: Callable[[dict[str, object]], dict[str, str]], +) -> OCRResponse | None: + if load_rust_aocr() is None: + return None + marshalled: Final = _marshal(request, resolve_secret, convert_file_document) + try: + response: Final = await aocr( + model=marshalled.model, + document=dict(marshalled.document), # mutable-ok: PyO3 OCR binding requires a concrete dict + api_key=marshalled.api_key, + api_base=marshalled.api_base, + custom_llm_provider=marshalled.custom_llm_provider, + extra_headers=marshalled.extra_headers, + optional_params=dict(marshalled.kwargs), # mutable-ok: PyO3 OCR binding requires a concrete dict + input_sources=marshalled.input_sources, + timeout=marshalled.timeout, + ) + except Exception as error: + raise _map_error(error, request) from error + return _response(response) if response is not None else None + + def ocr( *, model: str, @@ -71,6 +395,7 @@ def ocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + input_sources: Mapping[str, str] | None = None, ) -> dict[str, object] | None: rust_ocr: Final = load_rust_ocr() if rust_ocr is None: @@ -83,6 +408,7 @@ def ocr( custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, optional_params=optional_params, + input_sources=dict(input_sources or {}), # mutable-ok: native boundary requires a concrete dict timeout_seconds=_timeout_to_seconds(timeout), ) @@ -97,6 +423,7 @@ async def aocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + input_sources: Mapping[str, str] | None = None, ) -> dict[str, object] | None: rust_aocr: Final = load_rust_aocr() if rust_aocr is None: @@ -109,5 +436,6 @@ async def aocr( custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, optional_params=optional_params, + input_sources=dict(input_sources or {}), # mutable-ok: native boundary requires a concrete dict timeout_seconds=_timeout_to_seconds(timeout), ) diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index ee755b317d8..d36234f56c1 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -5,17 +5,20 @@ from __future__ import annotations from collections.abc import Awaitable from dataclasses import dataclass from functools import lru_cache -from typing import Final, Protocol, cast # noqa: TID251 # native extension exposes dynamically typed callables +from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.default_encoding import cl100k_base_rank_file, o200k_base_rank_file +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding, uses_legacy_message_accounting from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.configuration import rust_enabled from litellm.rust_bridge.runtime import BridgeErrorContext, RustHandled, aattempt -from litellm.utils import claude_json_str -from litellm.utils import uses_anthropic_tokenizer as _python_uses_anthropic_tokenizer +from litellm.utils import claude_json_str, huggingface_tokenizer_kind + +RustTokenizer = Literal["anthropic", "cl100k_base", "o200k_base"] class RustTokenCounter(Protocol): @@ -27,6 +30,12 @@ class RustTokenCounterFactory(Protocol): def __call__(self, tokenizer_json: str) -> RustTokenCounter: raise NotImplementedError + def from_cl100k_ranks(self, rank_file: str) -> RustTokenCounter: + raise NotImplementedError + + def from_o200k_ranks(self, rank_file: str) -> RustTokenCounter: + raise NotImplementedError + @dataclass(frozen=True, slots=True) class InputTokenCount: @@ -50,18 +59,41 @@ def _as_factory(value: object) -> RustTokenCounterFactory | None: TOKEN_COUNTER: Final = NativeBinding("TokenCounter", validate=_as_factory) -def uses_anthropic_tokenizer(model: str) -> bool: - if litellm.disable_token_counter is True or litellm.disable_hf_tokenizer_download is True: - return False - return _python_uses_anthropic_tokenizer(model) +def rust_tokenizer(model: str) -> RustTokenizer | None: + """The Rust counter for the tokenizer `litellm.token_counter` selects for `model`, `None` when Python must count. + + Mirrors `_select_tokenizer_helper`: the Anthropic tokenizer has a Rust port, the other HuggingFace + downloads do not, and of the tiktoken encodings `cl100k_base` and `o200k_base` do (p50k/r50k do not). Rust + prices every message with the default constants, so the legacy `gpt-3.5-turbo-0301` accounting stays in + Python.""" + if litellm.disable_token_counter is True: + return None + kind: Final = None if litellm.disable_hf_tokenizer_download is True else huggingface_tokenizer_kind(model) + if kind == "anthropic": + return "anthropic" + if kind is not None or uses_legacy_message_accounting(model): + return None + match openai_tokenizer_encoding(model).name: + case "cl100k_base": + return "cl100k_base" + case "o200k_base": + return "o200k_base" + case _: + return None @lru_cache(maxsize=4) -def _anthropic_counter(factory: RustTokenCounterFactory) -> RustTokenCounter: - return factory(claude_json_str) +def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> RustTokenCounter: + match tokenizer: + case "anthropic": + return factory(claude_json_str) + case "cl100k_base": + return factory.from_cl100k_ranks(cl100k_base_rank_file()) + case "o200k_base": + return factory.from_o200k_ranks(o200k_base_rank_file()) -async def count_anthropic_input_tokens(body: bytes) -> InputTokenCount | None: +async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: if not rust_enabled(): return None factory: Final = TOKEN_COUNTER.load() @@ -69,11 +101,14 @@ async def count_anthropic_input_tokens(body: bytes) -> InputTokenCount | None: return None try: attempt: Final = await aattempt( - native_call=lambda: _anthropic_counter(factory).acount_request(body), + native_call=lambda: _counter(factory, tokenizer).acount_request(body), adapt=_INPUT_TOKEN_COUNT.validate_python, - context=BridgeErrorContext(route="token_counter", provider="anthropic", model=""), + context=BridgeErrorContext(route="token_counter", provider=tokenizer, model=""), ) except (RuntimeError, ValueError) as error: - verbose_logger.debug("Rust token counter failed, counting in Python: %s", error) + verbose_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) return None - return attempt.value if isinstance(attempt, RustHandled) else None + if not isinstance(attempt, RustHandled): + return None + verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) + return attempt.value diff --git a/litellm/utils.py b/litellm/utils.py index a765e1b1246..1a77655a5a4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2227,25 +2227,39 @@ def uses_anthropic_tokenizer(model: str) -> bool: return model in litellm.anthropic_models and "claude-3" not in model -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: +HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "llama2", "llama3"] + + +def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: + """Which HuggingFace tokenizer `token_counter` selects for a model; `None` means tiktoken.""" if model in litellm.cohere_models and "command-r" in model: - # cohere - cohere_tokenizer: Final = Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") - return {"type": "huggingface_tokenizer", "tokenizer": cohere_tokenizer} - # anthropic - elif uses_anthropic_tokenizer(model): - claude_tokenizer: Final = Tokenizer.from_str(claude_json_str) - return {"type": "huggingface_tokenizer", "tokenizer": claude_tokenizer} - # llama2 - elif "llama-2" in model.lower() or "replicate" in model.lower(): - tokenizer = Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} - # llama3 - elif "llama-3" in model.lower(): - tokenizer = Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} - else: + return "cohere" + if uses_anthropic_tokenizer(model): + return "anthropic" + if "llama-2" in model.lower() or "replicate" in model.lower(): + return "llama2" + if "llama-3" in model.lower(): + return "llama3" + return None + + +def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: + kind: Final = huggingface_tokenizer_kind(model) + if kind is None: return None + return {"type": "huggingface_tokenizer", "tokenizer": _load_huggingface_tokenizer(kind)} + + +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> Tokenizer: + match kind: + case "cohere": + return Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") + case "anthropic": + return Tokenizer.from_str(claude_json_str) + case "llama2": + return Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + case "llama3": + return Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -9400,11 +9414,9 @@ class ProviderConfigManager: ReductoParseV3Config, ) - if model == "parse-v3": - return ReductoParseV3Config() if model == "parse-legacy": return ReductoParseLegacyConfig() - return None + return ReductoParseV3Config() MistralOCRConfig: Final = litellm_utils.MistralOCRConfig PROVIDER_TO_CONFIG_MAP: Final = { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aadd9bd3028..7fa09951eae 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57695,6 +57695,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://fireworks.ai/models/deepseek-ai/deepseek-v4p1-flash", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-vision-exp": { "cache_read_input_token_cost": 7e-09, "input_cost_per_token": 2.2e-07, @@ -57745,6 +57762,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://fireworks.ai/models/deepseek-ai/deepseek-v4p1-flash", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/deepseek-v4-flash-vision-exp": { "cache_read_input_token_cost": 7e-09, "input_cost_per_token": 2.2e-07, diff --git a/pyproject.toml b/pyproject.toml index d33d693f794..448451f7f93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,7 @@ e2e-dev = [ "playwright==1.61.0", "websockets>=15.0.1,<16.0", "locust==2.45.0", + "psutil==7.2.2", "mcp>=1.28.1,<2.0", ] proxy-dev = [ diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index d2b842364a6..588402e3996 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -144,6 +144,8 @@ def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( "tests/e2e/pytest.ini", "tests/e2e/gateway/stage_mirror_ci_config.yml", ".github/e2e-stack/up.sh", + ".github/e2e-stack/start-idp.sh", + "tests/e2e/idp_realm.json", ".github/workflows/test-e2e-changed.yml", ), ) diff --git a/tests/code_coverage_tests/test_e2e_idp_stack.py b/tests/code_coverage_tests/test_e2e_idp_stack.py new file mode 100644 index 00000000000..93596af915b --- /dev/null +++ b/tests/code_coverage_tests/test_e2e_idp_stack.py @@ -0,0 +1,115 @@ +import json +import os +import subprocess +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +START_IDP = ROOT / ".github/e2e-stack/start-idp.sh" + + +def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = "", port: int = 8181, real_curl: bool = False): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "docker.jsonl" + programs = { + "docker": """import json, os, sys +with open(os.environ['DOCKER_LOG'], 'a') as out: + out.write(json.dumps(sys.argv[1:]) + '\\n') +if os.environ['FAILURE'] == 'schema' and 'psql' in sys.argv: + sys.exit(17) +if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv: + sys.exit(18) +""", + "curl": "import os, sys; sys.exit(1 if os.environ['FAILURE'] == 'readiness' else 0)\n", + "uname": "import os; print(os.environ['PLATFORM'])\n", + } + if real_curl: + del programs["curl"] + for name, source in programs.items(): + program = bin_dir / name + program.write_text(f"#!{sys.executable}\n{source}") + program.chmod(0o755) + result = subprocess.run( + ["bash", str(START_IDP)], + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "DOCKER_LOG": str(calls), + "PLATFORM": platform, + "FAILURE": failure, + "DATABASE_HOST": "127.0.0.1", + "DATABASE_PORT": "5544", + "DATABASE_USER": "fixture_user", + "DATABASE_PASSWORD": "fixture_password", + "DATABASE_NAME": "fixture_db", + "E2E_KEYCLOAK_PORT": str(port), + "E2E_KEYCLOAK_STARTUP_TIMEOUT": "0", + }, + capture_output=True, + text=True, + timeout=10, + ) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + +@pytest.mark.parametrize("platform", ("Linux", "Darwin")) +def test_idp_uses_existing_database_and_imports_runner_realm(tmp_path: Path, platform: str) -> None: + result, calls = run_start(tmp_path, platform=platform) + + assert result.returncode == 0, result.stderr + schema, _, launch = calls + host = "127.0.0.1" if platform == "Linux" else "host.docker.internal" + assert schema[schema.index("-h") + 1] == host + assert schema[schema.index("-p") + 1] == "5544" + assert "ON_ERROR_STOP=1" in schema + assert "CREATE SCHEMA IF NOT EXISTS keycloak" in schema + assert f"KC_DB_URL_HOST={host}" in launch + assert "KC_DB_URL_PORT=5544" in launch + assert "KC_DB_SCHEMA=keycloak" in launch + assert "KC_DB_POOL_MAX_SIZE=10" in launch + assert f"{ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" in launch + assert "KC_HTTP_PORT=8181" in launch + if platform == "Linux": + assert launch[launch.index("--network") + 1] == "host" + else: + assert launch[launch.index("-p") + 1] == "127.0.0.1:8181:8181" + assert "Keycloak realm is up" in result.stdout + + +@pytest.mark.parametrize(("failure", "code"), (("schema", 17), ("launch", 18), ("readiness", 1))) +def test_idp_failure_stops_stack_startup(tmp_path: Path, failure: str, code: int) -> None: + result, calls = run_start(tmp_path, failure=failure) + + assert result.returncode == code + assert "Keycloak realm is up" not in result.stdout + if failure == "schema": + assert len(calls) == 1, "do not replace an IdP when its database is unavailable" + + +def test_readiness_requires_the_imported_realm_on_the_configured_port(tmp_path: Path) -> None: + expected_path = "/realms/litellm-e2e/.well-known/openid-configuration" + observed_paths: list[str] = [] + + class Discovery(BaseHTTPRequestHandler): + def do_GET(self) -> None: + observed_paths.append(self.path) + self.send_response(200 if self.path == expected_path else 404) + self.end_headers() + + with ThreadingHTTPServer(("127.0.0.1", 0), Discovery) as server: + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + result, _ = run_start(tmp_path, port=server.server_port, real_curl=True) + finally: + server.shutdown() + worker.join(timeout=5) + + assert result.returncode == 0, result.stderr + assert observed_paths == [expected_path] + assert "Keycloak realm is up" in result.stdout diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 34fbe9d9247..a58c13d6a1c 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,9 +17,9 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic -- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite +- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` @@ -163,7 +163,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput | session_anomaly (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly | memory (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 1183096b81e..44564a51e26 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -27,14 +27,50 @@ The suites run against a live proxy, so bring one up first by running the litell 2. Bring up a Postgres and a Redis for the proxy to use. The repo-root `docker-compose.yml` already defines a Postgres on `5432`; a `docker run -p 6379:6379 redis:7` covers Redis. Point `DATABASE_URL` / `REDIS_HOST` / `REDIS_PORT` at whatever you run. Tests that read Redis directly default to the deployed shape (TLS + cluster mode) whenever `REDIS_HOST` is set, so for a local standalone Redis also set `REDIS_CLUSTER=false` and `REDIS_SSL=false` (plus `REDIS_PASSWORD` when your Redis requires auth) -3. Start the litellm proxy locally against your config and confirm it is live: +3. Start the identity provider the JWT API tests authenticate against, then the litellm proxy against your config, and confirm both are live. It is a real Keycloak, running the realm in `tests/e2e/idp_realm.json`, and the proxy trusts it because `JWT_PUBLIC_KEY_URL` points at that realm's JWKS. The proxy caches the JWKS for `public_key_ttl` (600s) and does not refetch on an unknown `kid`, so keep its data volume across restarts; restart the proxy if you deliberately replace that volume: ```bash - set -a && source .env && set +a + docker run -d --name litellm-e2e-idp -p 8480:8080 \ + -e KC_BOOTSTRAP_ADMIN_USERNAME=admin -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + -v "$PWD/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + -v litellm-e2e-idp-data:/opt/keycloak/data \ + quay.io/keycloak/keycloak:26.7.3 start-dev --import-realm + curl -fs --retry 30 --retry-delay 2 --retry-all-errors http://127.0.0.1:8480/realms/litellm-e2e/.well-known/openid-configuration + export JWT_ISSUER=http://127.0.0.1:8480/realms/litellm-e2e + export JWT_AUDIENCE=litellm-e2e + export JWT_PUBLIC_KEY_URL="$JWT_ISSUER/protocol/openid-connect/certs" litellm --config .yml --port 4000 curl -fs http://localhost:4000/health/liveliness ``` + The tests reach Keycloak at `E2E_KEYCLOAK_URL` (default `http://127.0.0.1:8480`) and provision their identities through its admin API, so they also need `E2E_KEYCLOAK_ADMIN_USER` and `E2E_KEYCLOAK_ADMIN_PASSWORD` (`admin` / `admin` for the throwaway container above; the deployed stacks take theirs from a secret). JWT auth is an enterprise feature, so the proxy needs `LITELLM_LICENSE` in its environment, and its config needs the JWT block below. `enable_jwt_auth` only routes bearer tokens with three dot-separated segments into the JWT path, so `sk-` virtual keys and the master key keep working for every other suite. `proxy_batch_write_at` is lowered so the JWT spend-attribution test sees its row well inside the poll deadline: + + ```yaml + general_settings: + proxy_batch_write_at: 5 + enable_jwt_auth: true + litellm_jwtauth: + user_id_jwt_field: sub + user_email_jwt_field: email + team_ids_jwt_field: groups + user_id_upsert: true + ``` + + Set `JWT_ISSUER` to the exact realm URL used by the test runner and `JWT_AUDIENCE=litellm-e2e`. The realm explicitly maps this audience, `sub`, `email`, and `groups`; the proxy fetches real signing keys from its JWKS endpoint. The rejection tests obtain signed tokens with a different audience or issuer and verify the corresponding rejection reason. The issuer test uses a different HTTP Host when requesting a token from the isolated, dynamically named test IdP. + + Keycloak's password grant is a test-only provisioning shortcut, not a production login recommendation. The `litellm-e2e-admin` client adds the proxy's admin scope; the normal client does not. Never reuse this permissive realm outside an isolated test stack. + + Management tests can use the shared `idp` and `jwt_identity` fixtures. Each test gets a unique Keycloak group/user and a matching proxy user/team. Setup and fallback cleanup use the master key; the operations and read-backs being tested must explicitly use `caller_key=idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID)` (or a member token). See `management/test_jwt_management_e2e.py` for create/read/update/clear/delete and tenant-denial examples. A group claim alone is not database team membership: permission tests explicitly add the member and prove an allowed read before asserting the denied write. + + Every successful IdP create immediately registers cleanup, including partial setup failures. Cleanup failures emit warnings. Tokens are minted on demand, and the expiration test waits relative to the token's actual `exp` with a bounded clock-drift check. To check first-attempt behavior locally, run both files with `--reruns 0`: + + ```bash + E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=admin \ + uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v + ``` + + Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The GitHub Actions changed-test stack starts the same digest-pinned Keycloak through `.github/e2e-stack/start-idp.sh`, imports the checked-out realm, and exports the IdP URL and credentials in `stack.env`. Both runners configure issuer/audience validation and store the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. Both wait for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them + 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): ```bash @@ -65,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e1b987cbfd9..36569896125 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,23 +17,52 @@ import functools import os from collections.abc import Generator, Iterator from datetime import datetime, timezone +from typing import Final import pytest import requests - -from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL +from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL, unique_marker from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup +from e2e_http import unwrap from fixture_mode import fixture_mode_collection_error, fixture_report_lines -from provider_edge import replay_leftover_error +from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from models import TeamNewBody, UserNewBody, UserNewResponse +from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client - _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +@pytest.fixture(scope="session") +def idp() -> Keycloak: + return keycloak_from_env() + + +@pytest.fixture +def jwt_identity(idp: Keycloak, resources: ResourceManager, proxy: ProxyClient) -> Identity: + marker: Final = unique_marker() + identity: Final = idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) + resources.defer(lambda: proxy.delete_user(identity.user_id)) + # Seed the canonical user before any JWT call populates the auth cache. + # Group claims grant team access; management membership is added by the test. + unwrap( + proxy.transport.post( + "/user/new", + headers=proxy.transport.master, + json=UserNewBody( + user_id=identity.user_id, user_email=f"{identity.username}@example.com", user_role="internal_user" + ), + response_type=UserNewResponse, + ) + ) + team_id: Final = proxy.create_team(TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=identity.group)) + resources.defer(lambda: proxy.delete_team(team_id)) + return identity + + def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", @@ -60,6 +89,11 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set", ) + config.addinivalue_line( + "markers", + "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " + "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index c8d7037d2fd..d1227fe7c0c 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,6 +76,10 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} + +- {id: mgmt.key.jwt.lifecycle, module: mgmt, tier: P0, surface: api, assertions: [lifecycle], source: "management_endpoints/key_management_endpoints.py", rationale: "An IdP-issued admin JWT creates, reads, updates, clears and deletes a key; omitted fields survive updates"} +- {id: mgmt.key.jwt.member_denied, module: mgmt, tier: P0, surface: api, assertions: [member_denied], source: "auth/handle_jwt.py", rationale: "A valid member JWT cannot update an admin-managed key and denial leaves it unchanged"} +- {id: mgmt.key.jwt.other_team_denied, module: mgmt, tier: P0, surface: api, assertions: [other_team_denied], source: "auth/handle_jwt.py", rationale: "A valid JWT for another existing team cannot read the key"} - {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} - {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} - {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 814ebae2e0b..9292e5f07db 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -9,9 +9,12 @@ - {id: other.auth.llm_chat.not_bearer_scheme_denied, module: other, tier: P0, area: auth, assertions: [not_bearer_scheme_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "NotBearer scheme on chat is 401/403"} - {id: other.auth.realtime.missing_header_denied, module: other, tier: P1, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §9.19 / LIT-4778", rationale: "Realtime client-secret and calls routes reject requests without Authorization"} - {id: other.config.responses.metadata_redis_ttl_bounded, module: other, tier: P0, area: config, assertions: [ttl_bounded], source: "responses + redis cache", rationale: "Responses store+metadata must not leave TTL-unbounded Redis entries (LIT-1201)"} -- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:77-150", rationale: "Valid JWT with correct issuer + claims grants access"} -- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:125-135", rationale: "Expired JWT rejected even with valid signature"} -- {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:145-150", rationale: "Bad/missing signature fails verification"} +- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:1217-1256 auth_jwt / user_api_key_auth.py:1365-1377", rationale: "An access token issued by the configured IdP whose groups claim names an existing team is accepted on /chat/completions"} +- {id: other.auth.jwt.spend_attributed_to_claims, module: other, tier: P0, area: auth, assertions: [spend_attributed_to_claims], source: "handle_jwt.py:2224 auth_builder / user_api_key_auth.py:1438-1474", rationale: "The spend log row for a JWT-authenticated call carries the team_id from the groups claim and the user_id from sub, which for a real IdP is an opaque uuid, not a virtual key's identity. Single-group claim only: the proxy picks the team from a set, so attribution over several groups is unordered"} +- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:1244-1250", rationale: "A token the IdP issued with a one-second lifespan is rejected 401 (Token Expired) once it lapses, even though its signature still verifies; leeway is 0"} +- {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:1158-1166 _decode_jwt_with_public_key", rationale: "A genuine token whose signature bytes were altered fails verification with 401"} +- {id: other.auth.jwt.unknown_team_denied, module: other, tier: P0, area: auth, assertions: [unknown_team_denied], source: "handle_jwt.py:1549-1632 find_team_with_model_access", rationale: "A verified JWT whose groups claim resolves to no existing team is denied with 403 naming the unresolved team, never silently admitted without a team. The proxy words it as a model-access denial, the same body an existing team without model access gets"} +- {id: other.auth.jwt.virtual_key_unaffected, module: other, tier: P0, area: auth, assertions: [virtual_key_unaffected], source: "handle_jwt.py:213 is_jwt / user_api_key_auth.py:1332-1333", rationale: "enable_jwt_auth only routes three-segment bearer tokens into the JWT branch, so sk- virtual keys keep working on the same proxy"} - {id: other.auth.model_access_group.wildcard_bare_name_allowed, module: other, tier: P0, area: auth, assertions: [wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "A grant of a group holding a wildcard deployment covers the bare model names callers actually send, not only the provider-prefixed spelling"} - {id: other.auth.model_access_group.member_allowed, module: other, tier: P0, area: auth, assertions: [member_allowed], source: "auth_checks.py:3232", rationale: "A key whose allow-list is a model access group can call the deployments in that group"} - {id: other.auth.model_access_group.non_member_denied, module: other, tier: P0, area: auth, assertions: [non_member_denied], source: "auth_checks.py:3232", rationale: "That same grant reaches nothing outside the group, including provider models the group's wildcard does not cover"} @@ -48,3 +51,6 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} + +- {id: other.auth.jwt.wrong_issuer_denied, module: other, tier: P0, area: auth, assertions: [wrong_issuer_denied], source: "auth/handle_jwt.py", rationale: "A signed token with the correct audience and an unexpected issuer is rejected"} +- {id: other.auth.jwt.wrong_audience_denied, module: other, tier: P0, area: auth, assertions: [wrong_audience_denied], source: "auth/handle_jwt.py", rationale: "A signed token from the trusted issuer intended for another app is rejected"} diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..e95d27bab84 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,9 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, messages], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "Under locust load split round robin over /chat/completions and /v1/messages with every request retrying through failing mock deployments, holding Redis in CLIENT PAUSE ALL for the phase trips the breaker and every request still succeeds, with latency, RSS, and CPU reported as p50/p90/p99 against the pre-pause baseline; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} +- {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"} - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 691335ffdd5..e15a0cd0f8f 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,7 +13,6 @@ from pathlib import Path from typing import Final from dotenv import load_dotenv - from fixture_mode import deterministic_marker, parse_fixture_mode from provider_edge import provider_edge_api_base @@ -144,6 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" +REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) @@ -160,6 +160,14 @@ ANOMALY_MAX_KEY_SPEND_USD = float( ANOMALY_SPEND_SETTLE_SECONDS = float( os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75") ) +MEMORY_REQUESTS_PER_PHASE = int(os.environ.get("E2E_MEMORY_REQUESTS_PER_PHASE", "300")) +MEMORY_RETRIES_PER_REQUEST = int(os.environ.get("E2E_MEMORY_RETRIES_PER_REQUEST", "2")) +MEMORY_TRANSCRIPT_TURNS = int(os.environ.get("E2E_MEMORY_TRANSCRIPT_TURNS", "40")) +MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4")) +MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15")) +MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) +MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48")) +MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) def ws_base_url() -> str: @@ -181,8 +189,7 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: site = ( os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com" ).strip().removeprefix("https://").removeprefix("http://").rstrip("/") - if site.startswith("app."): - site = site[len("app.") :] + site = site.removeprefix("app.") host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}" base = f"https://{host}/v1/mcp" return f"{base}?toolsets={toolsets}" if toolsets else base diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 415c72bbb3c..ce069720c6e 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -130,6 +130,20 @@ class ProbeResult(BaseModel): return 200 <= self.status_code < 500 and self.status_code != 404 +class ExternalWrite(BaseModel): + """Outcome of a write to a non-proxy API (an identity provider's admin API) + that answers with a status and, on create, a Location header naming the new + resource rather than a JSON body.""" + + status_code: int + location: str = "" + body: str = "" + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 300 + + class StreamingResponse(BaseModel): """Raw outcome for calls whose body is provider-native or streamed: status, the x-litellm-call-id header, the x-litellm-response-cost header (StandardLogging @@ -257,23 +271,23 @@ def assert_auth_denied(result: StreamingResponse, context: str) -> None: f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" ) - def wire_body(json: BaseModel) -> dict[str, object]: if isinstance(json, PartialBody): return json.model_dump(by_alias=True, exclude_unset=True) return json.model_dump(by_alias=True, exclude_none=True) -def _headers(headers: BaseModel) -> dict[str, str]: - dumped: dict[str, object] = headers.model_dump(by_alias=True, exclude_none=True) +def _flat(model: BaseModel) -> dict[str, str]: + dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) return {key: str(value) for key, value in dumped.items()} +def _headers(headers: BaseModel) -> dict[str, str]: + return _flat(headers) + + def _params(params: BaseModel | None) -> dict[str, str]: - if params is None: - return {} - dumped: dict[str, object] = params.model_dump(by_alias=True, exclude_none=True) - return {key: str(value) for key, value in dumped.items()} + return _flat(params) if params is not None else {} TRANSIENT_STATUSES: frozenset[int] = frozenset({529}) @@ -416,6 +430,62 @@ def get_external[R: BaseModel]( return classify(resp, response_type) +def post_form_external[R: BaseModel]( + url: str, + *, + form: BaseModel, + response_type: type[R], + headers: BaseModel | None = None, + timeout: float = 30.0, +) -> Result[R]: + """POST an absolute URL outside the proxy as `application/x-www-form-urlencoded`, + the encoding OAuth 2 token endpoints take. Like get_external: no proxy base url, + no proxy auth, and the same tagged-union classification as every other call.""" + try: + resp = requests.post( + url, + data=_flat(form), + headers=_headers(headers) if headers is not None else None, + timeout=timeout, + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return classify(resp, response_type) + + +def post_json_external( + url: str, + *, + headers: BaseModel, + json: BaseModel, + timeout: float = 30.0, +) -> ExternalWrite: + """POST an absolute URL outside the proxy under its own bearer, for an API that + answers a create with a status and a Location header rather than a JSON body.""" + try: + resp = requests.post( + url, + headers=_headers(headers), + json=json.model_dump(by_alias=True, exclude_none=True), + timeout=timeout, + ) + except requests.RequestException as exc: + return ExternalWrite(status_code=-1, body=str(exc)) + return ExternalWrite( + status_code=resp.status_code, + location=resp.headers.get("Location", ""), + body=resp.text, + ) + + +def delete_external(url: str, *, headers: BaseModel, timeout: float = 30.0) -> ExternalWrite: + try: + resp = requests.delete(url, headers=_headers(headers), timeout=timeout) + except requests.RequestException as exc: + return ExternalWrite(status_code=-1, body=str(exc)) + return ExternalWrite(status_code=resp.status_code, body=resp.text) + + def delete[R: BaseModel]( url: URL, *, diff --git a/tests/e2e/gateway/redis_chaos_ci_config.yml b/tests/e2e/gateway/redis_chaos_ci_config.yml new file mode 100644 index 00000000000..f7a71c50a71 --- /dev/null +++ b/tests/e2e/gateway/redis_chaos_ci_config.yml @@ -0,0 +1,19 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + use_redis_transaction_buffer: true + +litellm_settings: + callbacks: ["prometheus"] + require_auth_for_metrics_endpoint: false + enable_redis_auth_cache: true + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 + socket_timeout: 0.1 + +router_settings: + num_retries: 2 + disable_cooldowns: true diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 229e8514dee..8c8e64443cb 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -1,4 +1,11 @@ general_settings: + proxy_batch_write_at: 5 + enable_jwt_auth: true + litellm_jwtauth: + user_id_jwt_field: sub + user_email_jwt_field: email + team_ids_jwt_field: groups + user_id_upsert: true proxy_config_reload_interval_seconds: 7 store_prompts_in_spend_logs: true database_connection_pool_limit: 10 diff --git a/tests/e2e/idp.py b/tests/e2e/idp.py new file mode 100644 index 00000000000..6d2fc84eb27 --- /dev/null +++ b/tests/e2e/idp.py @@ -0,0 +1,228 @@ +"""Provision isolated identities and obtain signed tokens from the test Keycloak realm.""" + +from __future__ import annotations + +import os +import secrets +import warnings +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Final, Literal + +import pytest +from e2e_http import ( + AuthHeaders, + ExternalWrite, + NetworkError, + Result, + Success, + delete_external, + post_form_external, + post_json_external, +) +from pydantic import BaseModel, Field + +KEYCLOAK_URL_ENV: Final = "E2E_KEYCLOAK_URL" +KEYCLOAK_REALM_ENV: Final = "E2E_KEYCLOAK_REALM" +KEYCLOAK_ADMIN_USER_ENV: Final = "E2E_KEYCLOAK_ADMIN_USER" +KEYCLOAK_ADMIN_PASSWORD_ENV: Final = "E2E_KEYCLOAK_ADMIN_PASSWORD" + +DEFAULT_KEYCLOAK_URL: Final = "http://127.0.0.1:8480" +DEFAULT_REALM: Final = "litellm-e2e" +TESTS_CLIENT_ID: Final = "litellm-e2e-tests" +SHORT_LIVED_CLIENT_ID: Final = "litellm-e2e-shortlived" +ADMIN_CLIENT_ID: Final = "litellm-e2e-admin" +WRONG_AUDIENCE_CLIENT_ID: Final = "litellm-e2e-other-app" + +_START_HINT: Final = ( + "Start it with the `docker run ... quay.io/keycloak/keycloak` command in tests/e2e/CONTRIBUTING.md, " + f"and point {KEYCLOAK_URL_ENV} / {KEYCLOAK_ADMIN_USER_ENV} / {KEYCLOAK_ADMIN_PASSWORD_ENV} at it" +) + + +class TokenGrantForm(BaseModel): + """The direct-access (password) grant an OAuth 2 token endpoint takes, form encoded.""" + + grant_type: Literal["password"] = "password" + client_id: str + username: str + password: str + + +class TokenResponse(BaseModel): + access_token: str = Field(repr=False) + + +class TokenRequestHeaders(BaseModel): + host: str | None = None + + +class GroupCreateBody(BaseModel): + name: str + + +class PasswordCredential(BaseModel): + type: Literal["password"] = "password" + value: str + temporary: bool = False + + +class UserCreateBody(BaseModel): + """Keycloak's admin representation of a new user. `firstName` / `lastName` and + an empty `requiredActions` matter: a realm's default VERIFY_PROFILE action + otherwise leaves the account "not fully set up" and every grant fails.""" + + username: str + email: str + email_verified: bool = Field(default=True, alias="emailVerified") + first_name: str = Field(default="E2E", alias="firstName") + last_name: str = Field(default="Tester", alias="lastName") + enabled: bool = True + groups: tuple[str, ...] + credentials: tuple[PasswordCredential, ...] + required_actions: tuple[str, ...] = Field(default=(), alias="requiredActions") + + +def created_id(write: ExternalWrite, context: str) -> str: + """The new resource's id, which Keycloak returns only as the last segment of + the Location header on a 201.""" + if write.status_code != 201: + pytest.fail(f"Keycloak refused to create {context}: HTTP {write.status_code} {write.body[:300]}") + if not write.location or write.location.endswith("/"): + pytest.fail(f"Keycloak created {context} without a resource id in its Location header") + return write.location.rsplit("/", 1)[-1] + + +@dataclass(frozen=True, slots=True) +class Identity: + """One provisioned IdP user: the `sub` the proxy will see, the credential the + test signs in with, and the group whose name the litellm team carries.""" + + user_id: str + username: str + password: str = field(repr=False) + group: str + group_id: str + + +@dataclass(frozen=True, slots=True) +class Keycloak: + base_url: str + realm: str + admin_username: str + admin_password: str = field(repr=False) + + @property + def issuer(self) -> str: + return f"{self.base_url}/realms/{self.realm}" + + @property + def jwks_url(self) -> str: + return f"{self.issuer}/protocol/openid-connect/certs" + + def token_url(self, realm: str) -> str: + return f"{self.base_url}/realms/{realm}/protocol/openid-connect/token" + + def _admin_url(self, path: str) -> str: + return f"{self.base_url}/admin/realms/{self.realm}{path}" + + def _admin_headers(self) -> AuthHeaders: + """A fresh admin token per call: the master realm's tokens are short lived, + and a cached one would expire in the middle of a slow test.""" + form: Final = TokenGrantForm(client_id="admin-cli", username=self.admin_username, password=self.admin_password) + result: Final = post_form_external(self.token_url("master"), form=form, response_type=TokenResponse) + return AuthHeaders(authorization=f"Bearer {self._token(result, 'the Keycloak admin credential')}") + + def _token(self, result: Result[TokenResponse], context: str) -> str: + match result: + case Success(data=granted): + return granted.access_token + case NetworkError(message=message): + return pytest.fail(f"No live Keycloak at {self.base_url} for {context}: {message}. {_START_HINT}") + case _: + return pytest.fail(f"Keycloak refused {context}: {result}") + + def create_group(self, name: str) -> str: + return created_id( + post_json_external( + self._admin_url("/groups"), headers=self._admin_headers(), json=GroupCreateBody(name=name) + ), + f"group {name}", + ) + + def create_user(self, *, username: str, email: str, password: str, group: str) -> str: + return created_id( + post_json_external( + self._admin_url("/users"), + headers=self._admin_headers(), + json=UserCreateBody( + username=username, + email=email, + groups=(group,), + credentials=(PasswordCredential(value=password),), + ), + ), + f"user {username}", + ) + + def delete_user(self, user_id: str) -> None: + self._delete(f"/users/{user_id}") + + def delete_group(self, group_id: str) -> None: + self._delete(f"/groups/{group_id}") + + def _delete(self, path: str) -> None: + try: + headers: Final = self._admin_headers() + except pytest.fail.Exception as exc: + warnings.warn(f"Keycloak cleanup could not authenticate for {path}: {exc}", RuntimeWarning, stacklevel=2) + return + result: Final = delete_external(self._admin_url(path), headers=headers) + if result.status_code not in (204, 404): + warnings.warn( + f"Keycloak cleanup failed for {path}: HTTP {result.status_code} {result.body[:300]}", + RuntimeWarning, + stacklevel=2, + ) + + def provision(self, *, marker: str, group: str, defer: Callable[[Callable[[], object]], None]) -> Identity: + """Create `group` and a user in it, credentialed with a password generated + for this test alone, and hand back the identity a token can be minted for.""" + group_id: Final = self.create_group(group) + defer(lambda: self.delete_group(group_id)) + username: Final = f"e2e-jwt-user-{marker}" + password: Final = secrets.token_urlsafe(24) + user_id: Final = self.create_user( + username=username, email=f"{username}@example.com", password=password, group=group + ) + defer(lambda: self.delete_user(user_id)) + return Identity(user_id=user_id, username=username, password=password, group=group, group_id=group_id) + + def access_token( + self, identity: Identity, *, client_id: str = TESTS_CLIENT_ID, issuer_host: str | None = None + ) -> str: + """Sign `identity` in through the direct-access grant and hand back the + access token Keycloak signed, exactly as it came off the wire.""" + result: Final = post_form_external( + self.token_url(self.realm), + form=TokenGrantForm(client_id=client_id, username=identity.username, password=identity.password), + response_type=TokenResponse, + headers=TokenRequestHeaders(host=issuer_host), + ) + return self._token(result, f"a token for {identity.username}") + + +def keycloak_from_env() -> Keycloak: + admin_username: Final = os.environ.get(KEYCLOAK_ADMIN_USER_ENV, "").strip() + admin_password: Final = os.environ.get(KEYCLOAK_ADMIN_PASSWORD_ENV, "").strip() + if not admin_username or not admin_password: + pytest.fail( + f"The JWT suite needs {KEYCLOAK_ADMIN_USER_ENV} and {KEYCLOAK_ADMIN_PASSWORD_ENV} to provision " + f"identities in its Keycloak realm, and neither may be empty. {_START_HINT}" + ) + return Keycloak( + base_url=os.environ.get(KEYCLOAK_URL_ENV, DEFAULT_KEYCLOAK_URL).rstrip("/"), + realm=os.environ.get(KEYCLOAK_REALM_ENV, "").strip() or DEFAULT_REALM, + admin_username=admin_username, + admin_password=admin_password, + ) diff --git a/tests/e2e/idp_realm.json b/tests/e2e/idp_realm.json new file mode 100644 index 00000000000..3b747e7a5dd --- /dev/null +++ b/tests/e2e/idp_realm.json @@ -0,0 +1,210 @@ +{ + "realm": "litellm-e2e", + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "accessTokenLifespan": 300, + "clients": [ + { + "clientId": "litellm-e2e-tests", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ], + "defaultClientScopes": [ + "email", + "basic" + ] + }, + { + "clientId": "litellm-e2e-shortlived", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "attributes": { + "access.token.lifespan": "1" + }, + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ], + "defaultClientScopes": [ + "email", + "basic" + ] + }, + { + "clientId": "litellm-e2e-admin", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "defaultClientScopes": [ + "email", + "litellm_proxy_admin", + "basic" + ], + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ] + }, + { + "clientId": "litellm-e2e-other-app", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "defaultClientScopes": [ + "email", + "basic" + ], + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e-other-app", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ] + } + ], + "clientScopes": [ + { + "name": "litellm_proxy_admin", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + } + }, + { + "name": "email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true" + }, + "protocolMappers": [ + { + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "config": { + "user.attribute": "email", + "claim.name": "email", + "jsonType.label": "String", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "name": "basic", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false" + }, + "protocolMappers": [ + { + "name": "sub", + "protocol": "openid-connect", + "protocolMapper": "oidc-sub-mapper", + "config": { + "access.token.claim": "true" + } + } + ] + } + ] +} diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index 3a926ef2a61..fa608d157cc 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -3,26 +3,23 @@ from __future__ import annotations import os import pytest - -from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV +from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV from load_client import LoadClient, build_client from proxy_client import ProxyClient +_OPT_IN_MARKERS = ( + ("weekly", WEEKLY_ANOMALY_OPT_IN_ENV), + ("redis_chaos", REDIS_CHAOS_OPT_IN_ENV), +) -def pytest_collection_modifyitems( - config: pytest.Config, items: list[pytest.Item] -) -> None: - if os.environ.get(WEEKLY_ANOMALY_OPT_IN_ENV): - return - deselected = [ - item for item in items if item.get_closest_marker("weekly") is not None - ] + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + opted_out = {marker for marker, opt_in_env in _OPT_IN_MARKERS if not os.environ.get(opt_in_env)} + deselected = [item for item in items if any(item.get_closest_marker(marker) is not None for marker in opted_out)] if not deselected: return config.hook.pytest_deselected(items=deselected) - items[:] = [ - item for item in items if item.get_closest_marker("weekly") is None - ] + items[:] = [item for item in items if item not in deselected] @pytest.fixture(scope="session") diff --git a/tests/e2e/load/locust_load.py b/tests/e2e/load/locust_load.py index e0da8ba70c2..40f9f333db5 100644 --- a/tests/e2e/load/locust_load.py +++ b/tests/e2e/load/locust_load.py @@ -1,17 +1,26 @@ from __future__ import annotations import csv +import os +import subprocess +import sys +import tempfile +from collections.abc import Sequence from dataclasses import dataclass from itertools import accumulate from pathlib import Path +from typing import Final from pydantic import BaseModel, TypeAdapter +_LOCUSTFILE = Path(__file__).with_name("locustfile.py") +_CSV_PREFIX = "locust" _GENERATOR_SATURATION_MARKER = "CPU usage above" _MAX_REPORTED_ERRORS = 5 class LocustStatEntry(BaseModel): + name: str num_requests: int num_failures: int start_time: float @@ -29,12 +38,25 @@ class LoadError: occurrences: int +@dataclass(frozen=True, slots=True) +class EndpointLoad: + """One route's share of a phase, so a run that silently drove only one of them is visible.""" + + name: str + requests: int + failures: int + p50_seconds: float + + @dataclass(frozen=True, slots=True) class LoadResult: requests: int failures: int requests_per_second: float - median_response_seconds: float + p50_seconds: float + p90_seconds: float + p99_seconds: float + endpoints: tuple[EndpointLoad, ...] errors: tuple[LoadError, ...] generator_warnings: tuple[str, ...] @@ -53,33 +75,65 @@ class LoadResult: lines.append("locust recorded no error breakdown") return "; ".join((*lines, *self.generator_warnings)) + def latency_summary(self) -> str: + return f"p50 {self.p50_seconds:.3f}s, p90 {self.p90_seconds:.3f}s, p99 {self.p99_seconds:.3f}s" -def median_seconds(entries: list[LocustStatEntry]) -> float: - samples = sorted( - (milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items() - ) + def endpoint_summary(self) -> str: + return ", ".join( + f"{endpoint.name} {endpoint.requests} requests, {endpoint.failures} failures, " + f"p50 {endpoint.p50_seconds:.3f}s" + for endpoint in self.endpoints + ) + + +def percentile_seconds(entries: Sequence[LocustStatEntry], fraction: float) -> float: + """The response time at `fraction` of the merged histograms, in seconds. + + Locust buckets response times by millisecond, so this reads the first bucket whose + running count reaches the rank, the same lower-sample convention locust's own + percentiles use. + """ + samples = sorted((milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items()) total = sum(count for _, count in samples) if total == 0: return 0.0 running = accumulate(count for _, count in samples) - return next( - milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= total / 2 - ) / 1000.0 + rank: Final = total * fraction + return next(milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= rank) / 1000.0 + + +def per_endpoint(entries: Sequence[LocustStatEntry]) -> tuple[EndpointLoad, ...]: + """Each locust request name's own totals, in the order the names first appear.""" + names: Final = tuple(dict.fromkeys(entry.name for entry in entries)) + grouped: Final = ((name, tuple(entry for entry in entries if entry.name == name)) for name in names) + return tuple( + EndpointLoad( + name=name, + requests=sum(entry.num_requests for entry in group), + failures=sum(entry.num_failures for entry in group), + p50_seconds=percentile_seconds(group, 0.5), + ) + for name, group in grouped + ) def aggregate_stats( - entries: list[LocustStatEntry], + entries: Sequence[LocustStatEntry], errors: tuple[LoadError, ...], generator_warnings: tuple[str, ...], ) -> LoadResult: requests = sum(entry.num_requests for entry in entries) failures = sum(entry.num_failures for entry in entries) + endpoints = per_endpoint(entries) if not entries or requests == 0: return LoadResult( requests=requests, failures=failures, requests_per_second=0.0, - median_response_seconds=0.0, + p50_seconds=0.0, + p90_seconds=0.0, + p99_seconds=0.0, + endpoints=endpoints, errors=errors, generator_warnings=generator_warnings, ) @@ -88,7 +142,10 @@ def aggregate_stats( requests=requests, failures=failures, requests_per_second=requests / elapsed if elapsed > 0 else 0.0, - median_response_seconds=median_seconds(entries), + p50_seconds=percentile_seconds(entries, 0.5), + p90_seconds=percentile_seconds(entries, 0.9), + p99_seconds=percentile_seconds(entries, 0.99), + endpoints=endpoints, errors=errors, generator_warnings=generator_warnings, ) @@ -121,3 +178,74 @@ def read_generator_warnings(stderr: str) -> tuple[str, ...]: if _GENERATOR_SATURATION_MARKER in line ) return tuple(dict.fromkeys(saturated)) + + +def run_gateway_load( + *, + base_url: str, + api_keys: tuple[str, ...], + model: str, + endpoints: tuple[str, ...], + users: int, + spawn_rate: float, + duration_seconds: float, +) -> LoadResult: + """Drive `endpoints` from headless locust and aggregate what it reported. + + Each simulated user picks one of `api_keys`, so auth and budget lookups spread over a + pool of virtual keys instead of keeping one key's cache entry permanently warm, and one + of `endpoints` round robin, so the run covers every route the caller asked for. + """ + with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir: + csv_prefix = Path(report_dir) / _CSV_PREFIX + completed = subprocess.run( + [ + sys.executable, + "-m", + "locust", + "--headless", + "--json", + "--csv", + str(csv_prefix), + "--locustfile", + str(_LOCUSTFILE), + "--host", + base_url, + "--users", + str(users), + "--spawn-rate", + str(spawn_rate), + "--run-time", + f"{int(duration_seconds)}s", + "--exit-code-on-error", + "0", + ], + env={ + **os.environ, + "LOAD_API_KEYS": ",".join(api_keys), + "LOAD_MODEL": model, + "LOAD_ENDPOINTS": ",".join(endpoints), + }, + capture_output=True, + text=True, + timeout=duration_seconds + 120, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"locust exited {completed.returncode} before it could report throughput " + f"(a startup failure, not request failures, which are folded into the JSON summary via " + f"--exit-code-on-error 0):\n{completed.stderr}" + ) + try: + entries = _STATS_ADAPTER.validate_json(completed.stdout) + except ValueError as exc: + raise RuntimeError( + f"locust exited 0 but did not print a parseable --json throughput summary on stdout; " + f"got stdout={completed.stdout!r}, stderr={completed.stderr!r}" + ) from exc + return aggregate_stats( + entries, + read_errors(csv_prefix.with_name(f"{_CSV_PREFIX}_failures.csv")), + read_generator_warnings(completed.stderr), + ) diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py new file mode 100644 index 00000000000..9b7bdf2ee1e --- /dev/null +++ b/tests/e2e/load/locustfile.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os +import random +import uuid +from itertools import cycle +from typing import Final + +from locust import FastHttpUser, constant, task + +_MODEL: Final = os.environ["LOAD_MODEL"] +_API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(",")) +_NEXT_ENDPOINT: Final = cycle(os.environ["LOAD_ENDPOINTS"].split(",")) +_FILLER: Final = "x" * 40_000 + + +def _payload() -> dict[str, object]: + """A prompt no other request sent, so the response cache never answers for the deployment. + + Both endpoints take the same body: /v1/messages requires max_tokens, which /chat/completions + also accepts, so one payload serves the whole round robin. Padded to tens of KB so a + per-request bookkeeping cost that scales with body size (string formatting, hashing) shows + up in the CPU and log-size budgets instead of hiding behind a 40-byte prompt. + """ + return { + "model": _MODEL, + "messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex} {_FILLER}"}], + "max_tokens": 16, + } + + +class GatewayUser(FastHttpUser): + """One simulated user, pinned to one endpoint for its lifetime. + + Endpoints are handed out round robin as users spawn, so a run spreads evenly over them + while each user's traffic stays on a single route, the way a real client behaves. + """ + + wait_time = constant(0) + + def on_start(self) -> None: + self.headers = {"Authorization": f"Bearer {random.choice(_API_KEYS)}"} + self.endpoint = next(_NEXT_ENDPOINT) + + @task + def call(self) -> None: + self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any + self.endpoint, + json=_payload(), + headers=self.headers, + name=self.endpoint, + ) diff --git a/tests/e2e/load/phase_budget.py b/tests/e2e/load/phase_budget.py new file mode 100644 index 00000000000..066e2579da8 --- /dev/null +++ b/tests/e2e/load/phase_budget.py @@ -0,0 +1,81 @@ +"""Comparing one load phase against another, for tests that degrade a dependency mid-run. + +Two shapes of ceiling, because the metrics divide into two kinds. RSS and CPU are +machine-shaped: RSS scales with worker count and CPU with core count, so an absolute number +calibrated on one runner means nothing on the next, and what travels is the ratio against a +healthy phase measured on the same machine in the same run. Latency and log volume are not: +a ratio there is actively misleading, because a dependency that fails fast once its breaker +opens can make the degraded phase look cheaper than the healthy one while still being far +slower or noisier than a user should ever see. Those get a flat ceiling, which is the promise +the test is actually making. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, TypeAlias + + +def _rendered(value: float, unit: str, decimals: int) -> str: + return f"{value:.{decimals}f}{unit}" + + +@dataclass(frozen=True, slots=True) +class RatioBudget: + """One metric's healthy value, its degraded value, and how much growth is allowed.""" + + name: str + baseline: float + degraded: float + ratio_ceiling: float + unit: str + decimals: int = 1 + + @property + def ratio(self) -> float | None: + """How many times the baseline the degraded value is, or None if there is no baseline.""" + return self.degraded / self.baseline if self.baseline > 0 else None + + def violation(self) -> str | None: + """Why this metric fails its budget, or None if it passes.""" + ratio: Final = self.ratio + if ratio is None: + return ( + f"{self.name} measured {_rendered(self.baseline, self.unit, self.decimals)} in the healthy phase, " + f"so there is nothing to compare the degraded phase against; the measurement did not happen" + ) + if ratio > self.ratio_ceiling: + return ( + f"{self.name} went from {_rendered(self.baseline, self.unit, self.decimals)} healthy to " + f"{_rendered(self.degraded, self.unit, self.decimals)} degraded, {ratio:.1f}x the baseline and past " + f"the {self.ratio_ceiling:.1f}x allowed" + ) + return None + + +@dataclass(frozen=True, slots=True) +class AbsoluteBudget: + """One metric's degraded value against a flat ceiling, for metrics a ratio cannot bound.""" + + name: str + measured: float + ceiling: float + unit: str + decimals: int = 1 + + def violation(self) -> str | None: + """Why this metric fails its budget, or None if it passes.""" + if self.measured > self.ceiling: + return ( + f"{self.name} measured {_rendered(self.measured, self.unit, self.decimals)} in the degraded phase, " + f"past the {_rendered(self.ceiling, self.unit, self.decimals)} allowed" + ) + return None + + +Budget: TypeAlias = RatioBudget | AbsoluteBudget + + +def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]: + """Every budget the run blew, so one failure reports all of them instead of the first.""" + return tuple(violation for budget in budgets if (violation := budget.violation()) is not None) diff --git a/tests/e2e/load/proxy_usage.py b/tests/e2e/load/proxy_usage.py new file mode 100644 index 00000000000..83463c078b8 --- /dev/null +++ b/tests/e2e/load/proxy_usage.py @@ -0,0 +1,164 @@ +"""Resident memory and CPU of the proxy process tree, sampled on a background thread. + +The proxy under load runs several worker processes, and `/metrics` cannot report their +memory: litellm sets PROMETHEUS_MULTIPROC_DIR when num_workers > 1, and the multiprocess +collector drops the process collector's `process_resident_memory_bytes` / +`process_cpu_seconds_total` entirely. So the test measures the tree itself through psutil, +which needs the proxy to run on the same host as the test. +""" + +from __future__ import annotations + +import math +import threading +import time +from dataclasses import dataclass +from typing import Final + +import psutil +from pydantic import BaseModel, ConfigDict + + +class _MemoryInfo(BaseModel): + model_config = ConfigDict(from_attributes=True) + + rss: int + + +@dataclass(frozen=True, slots=True) +class UsageSample: + elapsed_seconds: float + rss_bytes: int + cpu_seconds: float + + +@dataclass(frozen=True, slots=True) +class UsageWindow: + """The samples taken across one phase, plus what they say about that phase.""" + + samples: tuple[UsageSample, ...] + + def rss_percentile(self, fraction: float) -> int: + if not self.samples: + return 0 + ordered: Final = sorted(sample.rss_bytes for sample in self.samples) + return ordered[_rank(len(ordered), fraction)] + + def cpu_seconds_consumed(self) -> float: + """CPU seconds the tree burned across the window, from its monotonic counter.""" + if len(self.samples) < 2: + return 0.0 + return self.samples[-1].cpu_seconds - self.samples[0].cpu_seconds + + def cpu_seconds_per_request(self, requests: int) -> float: + """CPU seconds the tree spent per request served. + + The portable cost figure: cores-busy saturates at the worker count under enough load, + so it reads the same whether a request costs 10 ms of CPU or 40 ms. This does not. + """ + return self.cpu_seconds_consumed() / requests if requests else 0.0 + + def cpu_utilization_percentiles(self) -> tuple[float, float, float]: + """Per-interval CPU utilization (cores busy) at p50, p90 and p99. + + Derived from consecutive samples of the cumulative counter rather than + psutil's own cpu_percent, so it covers every process in the tree including + workers that came and went between samples. + """ + rates: Final = sorted( + (later.cpu_seconds - earlier.cpu_seconds) / (later.elapsed_seconds - earlier.elapsed_seconds) + for earlier, later in zip(self.samples, self.samples[1:]) + if later.elapsed_seconds > earlier.elapsed_seconds + ) + if not rates: + return 0.0, 0.0, 0.0 + return ( + rates[_rank(len(rates), 0.5)], + rates[_rank(len(rates), 0.9)], + rates[_rank(len(rates), 0.99)], + ) + + def summary(self) -> str: + p50_cpu, p90_cpu, p99_cpu = self.cpu_utilization_percentiles() + return ( + f"RSS p50 {self.rss_percentile(0.5) / 2**20:.0f} MB, " + f"p90 {self.rss_percentile(0.9) / 2**20:.0f} MB, " + f"p99 {self.rss_percentile(0.99) / 2**20:.0f} MB; " + f"CPU cores busy p50 {p50_cpu:.2f}, p90 {p90_cpu:.2f}, p99 {p99_cpu:.2f}; " + f"{self.cpu_seconds_consumed():.1f} CPU seconds consumed" + ) + + +def _rank(count: int, fraction: float) -> int: + """Index of the sample at `fraction`, the same lower-sample convention as locust's percentiles.""" + return min(count - 1, max(0, math.ceil(count * fraction) - 1)) + + +def _read_process(process: psutil.Process) -> tuple[int, float] | None: + try: + with process.oneshot(): + memory: Final = _MemoryInfo.model_validate(process.memory_info()) + times: Final = process.cpu_times() + return memory.rss, times.user + times.system + except (psutil.NoSuchProcess, psutil.AccessDenied): + return None + + +class ProxyUsageSampler: + """Samples the proxy process tree every `interval_seconds` until stopped. + + `split()` returns the samples taken so far and starts a new window, so one sampler + covers a baseline phase and a chaos phase without a gap between them. + """ + + def __init__(self, pid: int, interval_seconds: float = 1.0) -> None: + self._process: Final = psutil.Process(pid) + self._interval: Final = interval_seconds + self._stop: Final = threading.Event() + self._lock: Final = threading.Lock() + self._samples: list[UsageSample] = [] # mutable-ok: a sampling buffer the reader drains under a lock + self._started: Final = time.monotonic() + self._thread: Final = threading.Thread(target=self._run, name="proxy-usage-sampler", daemon=True) + + def __enter__(self) -> ProxyUsageSampler: + self._thread.start() + return self + + def __exit__(self, *_: object) -> None: + self._stop.set() + self._thread.join(timeout=self._interval * 5) + + def _tree(self) -> tuple[psutil.Process, ...]: + try: + return (self._process, *self._process.children(recursive=True)) + except psutil.NoSuchProcess: + return () + + def _sample(self) -> UsageSample | None: + readings: Final = tuple(reading for process in self._tree() if (reading := _read_process(process)) is not None) + if not readings: + return None + return UsageSample( + elapsed_seconds=time.monotonic() - self._started, + rss_bytes=sum(rss for rss, _ in readings), + cpu_seconds=sum(cpu for _, cpu in readings), + ) + + def _run(self) -> None: + while not self._stop.is_set(): + sample = self._sample() + if sample is not None: + with self._lock: + self._samples.append(sample) + self._stop.wait(self._interval) + + def split(self) -> UsageWindow: + """The window that ends now; the next one starts from this window's last sample. + + The boundary sample is carried into the next window so its CPU counter has a + starting point, which is what makes the two windows' utilization comparable. + """ + with self._lock: + taken = tuple(self._samples) + self._samples = [taken[-1]] if taken else [] # rebind-ok: drains the buffer under the lock + return UsageWindow(samples=taken) diff --git a/tests/e2e/load/test_locust_load.py b/tests/e2e/load/test_locust_load.py index e3cc6f3efd5..af3e1483099 100644 --- a/tests/e2e/load/test_locust_load.py +++ b/tests/e2e/load/test_locust_load.py @@ -1,13 +1,14 @@ from __future__ import annotations from pathlib import Path +from typing import Final from locust_load import ( LoadError, LoadResult, LocustStatEntry, aggregate_stats, - median_seconds, + percentile_seconds, read_errors, read_generator_warnings, ) @@ -18,12 +19,14 @@ _FAILURES_HEADER = "Method,Name,Error,Occurrences,First Seen,Last Seen\n" def _entry( *, num_requests: int, + name: str = "/chat/completions", num_failures: int = 0, start_time: float = 1000.0, last_request_timestamp: float = 1010.0, response_times: dict[int, int] | None = None, ) -> LocustStatEntry: return LocustStatEntry( + name=name, num_requests=num_requests, num_failures=num_failures, start_time=start_time, @@ -41,35 +44,48 @@ def _result( requests=10, failures=10, requests_per_second=1.0, - median_response_seconds=0.05, + p50_seconds=0.05, + p90_seconds=0.08, + p99_seconds=0.1, + endpoints=(), errors=errors, generator_warnings=generator_warnings, ) -class TestSerialLatency: +class TestPercentiles: def test_median_is_the_middle_sample_not_the_mean_a_slow_tail_would_drag(self) -> None: # Nine fast requests and one very slow one: the mean is 1.99s, the median is 20ms. entry = _entry(num_requests=10, response_times={20: 9, 20000: 1}) - assert median_seconds([entry]) == 0.02 + assert percentile_seconds([entry], 0.5) == 0.02 - def test_median_merges_the_histograms_of_every_stats_entry(self) -> None: + def test_the_tail_percentiles_reach_the_slow_samples_the_median_hides(self) -> None: + # 100 samples: 89 fast, 10 slow, 1 very slow. p50 sits in the fast bucket, p90 in the + # slow one, and p99 lands on the single very slow sample. + entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 20000: 1}) + + assert percentile_seconds([entry], 0.5) == 0.02 + assert percentile_seconds([entry], 0.9) == 0.5 + assert percentile_seconds([entry], 0.99) == 0.5 + assert percentile_seconds([entry], 1.0) == 20.0 + + def test_percentiles_merge_the_histograms_of_every_stats_entry(self) -> None: # Per entry the median would be 10ms and 90ms; merged, the middle of the five samples is 90ms. entries = [ _entry(num_requests=2, response_times={10: 2}), _entry(num_requests=3, response_times={90: 3}), ] - assert median_seconds(entries) == 0.09 + assert percentile_seconds(entries, 0.5) == 0.09 def test_an_even_split_takes_the_lower_middle_sample_as_locust_itself_does(self) -> None: entry = _entry(num_requests=4, response_times={10: 2, 90: 2}) - assert median_seconds([entry]) == 0.01 + assert percentile_seconds([entry], 0.5) == 0.01 def test_no_samples_reports_zero_rather_than_dividing_by_an_empty_histogram(self) -> None: - assert median_seconds([]) == 0.0 + assert percentile_seconds([], 0.5) == 0.0 class TestAggregate: @@ -84,9 +100,20 @@ class TestAggregate: result = aggregate_stats([entry], (), ()) assert result.requests_per_second == 3.0 - assert result.median_response_seconds == 0.057 + assert result.p50_seconds == 0.057 + assert result.p99_seconds == 0.057 assert result.failure_ratio == 0.0 + def test_tail_percentiles_come_from_the_slow_end_of_the_histogram(self) -> None: + entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 3000: 1}) + + result = aggregate_stats([entry], (), ()) + + assert result.p50_seconds == 0.02 + assert result.p90_seconds == 0.5 + assert result.p99_seconds == 0.5 + assert result.latency_summary() == "p50 0.020s, p90 0.500s, p99 0.500s" + def test_throughput_spans_from_the_earliest_start_when_locust_reports_several_entries(self) -> None: entries = [ _entry(num_requests=60, start_time=1000.0, last_request_timestamp=1030.0), @@ -103,6 +130,49 @@ class TestAggregate: assert result.requests == 0 assert result.requests_per_second == 0.0 assert result.failure_ratio == 1.0 + assert result.endpoints == () + + +class TestPerEndpoint: + def test_each_route_keeps_its_own_requests_failures_and_median(self) -> None: + entries: Final = ( + _entry(name="/chat/completions", num_requests=100, response_times={20: 100}), + _entry(name="/v1/messages", num_requests=40, num_failures=3, response_times={900: 40}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert tuple((one.name, one.requests, one.failures, one.p50_seconds) for one in result.endpoints) == ( + ("/chat/completions", 100, 0, 0.02), + ("/v1/messages", 40, 3, 0.9), + ) + + def test_several_stats_entries_for_one_route_fold_into_a_single_row(self) -> None: + entries: Final = ( + _entry(name="/v1/messages", num_requests=10, response_times={30: 10}), + _entry(name="/v1/messages", num_requests=30, num_failures=1, response_times={30: 30}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert tuple((one.name, one.requests, one.failures) for one in result.endpoints) == (("/v1/messages", 40, 1),) + + def test_a_route_that_never_ran_is_absent_so_a_one_sided_run_cannot_pass_unnoticed(self) -> None: + result: Final = aggregate_stats((_entry(name="/chat/completions", num_requests=10),), (), ()) + + assert tuple(one.name for one in result.endpoints) == ("/chat/completions",) + + def test_the_summary_names_every_route_with_its_counts(self) -> None: + entries: Final = ( + _entry(name="/chat/completions", num_requests=2, response_times={20: 2}), + _entry(name="/v1/messages", num_requests=1, num_failures=1, response_times={500: 1}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert result.endpoint_summary() == ( + "/chat/completions 2 requests, 0 failures, p50 0.020s, /v1/messages 1 requests, 1 failures, p50 0.500s" + ) class TestErrorBreakdown: @@ -133,8 +203,7 @@ class TestErrorBreakdown: def test_diagnosis_caps_the_list_and_says_how_many_it_left_out(self) -> None: result = _result( errors=tuple( - LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index) - for index in range(1, 9) + LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index) for index in range(1, 9) ) ) diff --git a/tests/e2e/load/test_phase_budget.py b/tests/e2e/load/test_phase_budget.py new file mode 100644 index 00000000000..ea9e56afb0d --- /dev/null +++ b/tests/e2e/load/test_phase_budget.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from typing import Final + +from phase_budget import AbsoluteBudget, RatioBudget, violations + + +def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> RatioBudget: + return RatioBudget( + name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0 + ) + + +class TestRatioBudget: + def test_growth_within_the_ceiling_is_not_a_violation(self) -> None: + assert _budget(baseline=100, degraded=199).violation() is None + + def test_growth_exactly_at_the_ceiling_is_allowed(self) -> None: + assert _budget(baseline=100, degraded=200).violation() is None + + def test_growth_past_the_ceiling_reports_both_values_and_the_ratio(self) -> None: + violation: Final = _budget(baseline=100, degraded=250).violation() + + assert violation is not None + assert "100 MB" in violation + assert "250 MB" in violation + assert "2.5x" in violation + assert "2.0x allowed" in violation + + def test_shrinking_is_never_a_violation(self) -> None: + assert _budget(baseline=100, degraded=10).violation() is None + + def test_a_missing_baseline_is_a_violation_rather_than_a_silent_pass(self) -> None: + # The trap this guards: 0 as a baseline would make every ratio a division by zero, and + # treating it as "no growth" would pass a run that measured nothing at all. + violation: Final = _budget(baseline=0, degraded=4000).violation() + + assert violation is not None + assert "nothing to compare" in violation + + def test_the_unit_and_decimals_carry_into_the_message(self) -> None: + violation: Final = RatioBudget( + name="p99 latency", baseline=0.16, degraded=9.5, ratio_ceiling=8.0, unit="s", decimals=3 + ).violation() + + assert violation is not None + assert "0.160s" in violation + assert "9.500s" in violation + + +class TestAbsoluteBudget: + def test_a_value_under_the_ceiling_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=1.2, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_exactly_at_the_ceiling_is_allowed(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=5.0, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_past_the_ceiling_reports_the_measurement_and_the_ceiling(self) -> None: + violation: Final = AbsoluteBudget( + name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3 + ).violation() + + assert violation is not None + assert "9.500s" in violation + assert "5.000s allowed" in violation + + def test_a_flat_ceiling_fails_a_degraded_phase_that_is_cheaper_than_its_baseline(self) -> None: + # The whole reason this shape exists: once the breaker opens, requests skip Redis instead + # of waiting on its socket timeout, so the chaos phase can measure faster than the healthy + # one. A ratio against that baseline passes; the user still waited 9.5s. + assert _budget(baseline=20.0, degraded=9.5, ceiling=2.0).violation() is None + assert AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s").violation() is not None + + def test_a_zero_measurement_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="log bytes per request", measured=0, ceiling=12_000, unit=" B").violation() is None + + +class TestViolations: + def test_every_blown_budget_is_reported_not_just_the_first(self) -> None: + blown: Final = violations( + ( + _budget(baseline=100, degraded=500), + _budget(baseline=100, degraded=120), + RatioBudget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), + ) + ) + + assert len(blown) == 2 + assert blown[0].startswith("p99 RSS") + assert blown[1].startswith("CPU per request") + + def test_both_budget_shapes_report_together(self) -> None: + blown: Final = violations( + ( + _budget(baseline=100, degraded=500), + AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3), + ) + ) + + assert len(blown) == 2 + assert blown[0].startswith("p99 RSS") + assert blown[1].startswith("p99 latency") + + def test_a_run_inside_every_budget_reports_nothing(self) -> None: + assert violations((_budget(baseline=100, degraded=150),)) == () diff --git a/tests/e2e/load/test_proxy_usage.py b/tests/e2e/load/test_proxy_usage.py new file mode 100644 index 00000000000..915c564de50 --- /dev/null +++ b/tests/e2e/load/test_proxy_usage.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import Final + +from proxy_usage import UsageSample, UsageWindow + +_MB: Final = 2**20 + + +def _window(*points: tuple[float, int, float]) -> UsageWindow: + return UsageWindow( + samples=tuple( + UsageSample(elapsed_seconds=elapsed, rss_bytes=rss, cpu_seconds=cpu) for elapsed, rss, cpu in points + ) + ) + + +class TestRssPercentiles: + def test_the_tail_percentiles_reach_the_peak_the_median_hides(self) -> None: + # 100 one-second samples: 89 flat, 10 elevated, 1 spike. The median stays flat, p90 sees the + # elevated plateau, and only the max reaches the spike. + window: Final = _window( + *((float(i), 100 * _MB, float(i)) for i in range(89)), + *((float(89 + i), 300 * _MB, float(89 + i)) for i in range(10)), + (99.0, 900 * _MB, 99.0), + ) + + assert window.rss_percentile(0.5) == 100 * _MB + assert window.rss_percentile(0.9) == 300 * _MB + assert window.rss_percentile(0.99) == 300 * _MB + assert window.rss_percentile(1.0) == 900 * _MB + + def test_an_empty_window_reports_zero_rather_than_indexing_nothing(self) -> None: + assert _window().rss_percentile(0.5) == 0 + + +class TestCpuUtilization: + def test_utilization_is_the_counter_delta_over_the_interval_not_the_counter_itself(self) -> None: + # The counter climbs 0.5 CPU seconds per second, then 4.0 per second: half a core, then four. + window: Final = _window((0.0, _MB, 0.0), (1.0, _MB, 0.5), (2.0, _MB, 1.0), (3.0, _MB, 5.0)) + + p50, p90, p99 = window.cpu_utilization_percentiles() + + assert (p50, p90, p99) == (0.5, 4.0, 4.0) + assert window.cpu_seconds_consumed() == 5.0 + + def test_a_single_sample_has_no_interval_and_reports_zero(self) -> None: + window: Final = _window((0.0, _MB, 3.0)) + + assert window.cpu_utilization_percentiles() == (0.0, 0.0, 0.0) + assert window.cpu_seconds_consumed() == 0.0 + + def test_cost_per_request_separates_runs_that_cores_busy_reports_identically(self) -> None: + # Both windows pin 4 cores for 10 seconds, so utilization cannot tell them apart. The + # second one served a tenth of the traffic for the same CPU, which is the regression shape. + window: Final = _window(*((float(i), _MB, 4.0 * i) for i in range(11))) + + assert window.cpu_utilization_percentiles()[0] == 4.0 + assert window.cpu_seconds_per_request(4000) == 0.01 + assert window.cpu_seconds_per_request(400) == 0.1 + + def test_no_requests_reports_zero_cost_rather_than_dividing_by_zero(self) -> None: + assert _window((0.0, _MB, 0.0), (1.0, _MB, 1.0)).cpu_seconds_per_request(0) == 0.0 + + def test_summary_reports_every_percentile_in_human_units(self) -> None: + window: Final = _window((0.0, 200 * _MB, 0.0), (1.0, 200 * _MB, 1.5), (2.0, 200 * _MB, 3.0)) + + assert window.summary() == ( + "RSS p50 200 MB, p90 200 MB, p99 200 MB; " + "CPU cores busy p50 1.50, p90 1.50, p99 1.50; 3.0 CPU seconds consumed" + ) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py new file mode 100644 index 00000000000..9a9e8082e5b --- /dev/null +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -0,0 +1,454 @@ +"""Live e2e: the proxy under load keeps serving every request while Redis is down entirely. + +Runs against a proxy booted from tests/e2e/gateway/redis_chaos_ci_config.yml, which points +cache_params at a real Redis with litellm's default socket_timeout. That one client backs all +three Redis touchpoints on the request path: the virtual-key auth cache, the response cache, +and the cross-pod spend counter the cost-tracking callback awaits. + +The load runs in two phases against one model group of three mock deployments. The two at +order 1 raise InternalServerError and the one at order 2 serves, so every request burns its +retries on the failing pair (a 500 is retryable, so retries keep re-picking inside the lowest +order) and the router's order-based fallback then re-targets order 2. Every request is expected +to succeed, and each one carries retry breadcrumbs into cost tracking. + +Traffic is split round robin between /chat/completions and /v1/messages, one endpoint per +simulated user: the Redis touchpoints and the cost-tracking callback are shared by both, but +the Anthropic Messages route reaches them through its own request path, so a regression that +only shows up there would not surface from chat completions alone. + +Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE ALL for the +length of the phase, simulating Redis being down outright rather than merely slow to write. +Every touchpoint times out: the auth cache read falls back to Postgres, the response cache +read and write both fail, and the spend counter increment times out and the callback +stringifies the request metadata, breadcrumbs included, into a failed-tracking alert. On +v1.100.0 that string doubled per request until the worker hung (LIT-6780), which is what the +per-phase RSS, CPU, and log-bytes budgets are here to catch. + +Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: +a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops +the process collector's memory and CPU series. Log bytes are read from the file the proxy's +stdout/stderr was redirected to, so the same host requirement covers that too. Deselected +unless E2E_REDIS_CHAOS is set. +""" + +from __future__ import annotations + +import os +import re +import time +from collections.abc import Iterator +from dataclasses import dataclass +from itertools import pairwise +from pathlib import Path +from typing import Final + +import pytest +import redis +from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_http import NoBody +from lifecycle import ResourceManager +from load_client import LoadClient +from locust_load import LoadResult, run_gateway_load +from models import KeyGenerateBody, LiteLLMParamsBody +from phase_budget import AbsoluteBudget, Budget, RatioBudget, violations +from proxy_client import ProxyClient +from proxy_usage import ProxyUsageSampler, UsageWindow + +pytestmark: Final = pytest.mark.e2e + +MODEL_GROUP: Final = f"redis-chaos-fable-{unique_marker()}" +MOCK_MODEL: Final = "anthropic/claude-fable-5-1" +FAILING_DEPLOYMENTS: Final = 2 +SERVING_DEPLOYMENTS: Final = 1 +FAILING_ORDER: Final = 1 +SERVING_ORDER: Final = 2 +KEY_POOL_SIZE: Final = 8 +LOAD_ENDPOINTS: Final = ("/chat/completions", "/v1/messages") +LOCUST_USERS: Final = 50 +LOCUST_SPAWN_RATE: Final = 50.0 +BASELINE_SECONDS: Final = 60.0 +CHAOS_SECONDS: Final = 90.0 +REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) + +# RSS and CPU are budgeted as a multiple of the same metric in the baseline phase, because both +# are machine-shaped: RSS scales with worker count and CPU with core count, so a number +# calibrated on one runner means nothing on another. RSS moved 0.91x-1.40x across three otherwise +# identical local runs, so it stays loose; CPU per request held steady at 1.33x-1.36x across the +# same runs, so it sits close to what is actually measured. That makes CPU the likeliest of these +# to flake first on a runner whose core count shifts how much of baseline CPU is fixed per-request +# work: loosen it rather than widening the others if a CI run trips it without a real cause. +CHAOS_RSS_RATIO_CEILING: Final = 2.0 +CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0 + +# Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once +# the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos +# phase can come in faster than baseline (local runs measured p90 at 0.61x) and a ratio passes on a +# phase that was never slow. What a user actually cares about is the wall-clock number, which these +# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.19s, p90 0.23s, p99 +# 0.69s and 3.5 KB of log per request, with several times that left as slack for a shared CI runner. +CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 1.0 +CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 2.0 +CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 3.0 +CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 10_000.0 + +DRAIN_TIMEOUT_SECONDS: Final = 30.0 +DRAIN_POLL_SECONDS: Final = 1.0 + +TIMEOUT_FAILURES_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M +) +# The state gauge carries a pid label under the multiprocess collector, one series per worker, +# so this matches any label order rather than a bare {state="open"} that never appears. +BREAKER_OPEN_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_state\{[^}]*state="open"[^}]*\} ([0-9.e+]+)$', re.M +) +BREAKER_TRANSITIONS_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M +) + + +def _deployment_metric_re(name: str, model_ids: tuple[str, ...]) -> re.Pattern[str]: + """A per-deployment counter, narrowed to the deployments one run registered, so traffic + anything else sends the same proxy during the run cannot pad the retry count.""" + ids: Final = "|".join(re.escape(model_id) for model_id in model_ids) + return re.compile(rf'^litellm_{name}\{{[^}}]*model_id="(?:{ids})"[^}}]*\}} ([0-9.e+]+)$', re.M) + + +@dataclass(frozen=True, slots=True) +class Phase: + """One load phase's traffic and what the proxy's process tree did during it.""" + + name: str + load: LoadResult + usage: UsageWindow + redis_timeouts: float + log_bytes: int + + @property + def timeouts_per_request(self) -> float: + return self.redis_timeouts / self.load.requests if self.load.requests else 0.0 + + @property + def cpu_seconds_per_request(self) -> float: + return self.usage.cpu_seconds_per_request(self.load.requests) + + @property + def log_bytes_per_request(self) -> float: + return self.log_bytes / self.load.requests if self.load.requests else 0.0 + + def report(self) -> str: + return ( + f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, " + f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}; " + f"{self.cpu_seconds_per_request * 1000:.1f} ms CPU per request; " + f"{self.log_bytes_per_request:.0f} log bytes per request; " + f"{self.timeouts_per_request:.2f} Redis timeouts per request; " + f"by endpoint: {self.load.endpoint_summary()}" + ) + + +def _failing_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=MOCK_MODEL, + api_key="sk-redis-chaos-not-used", + mock_response="litellm.InternalServerError", + order=FAILING_ORDER, + ) + + +def _serving_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=MOCK_MODEL, + api_key="sk-redis-chaos-not-used", + mock_response="redis chaos ok", + order=SERVING_ORDER, + ) + + +@pytest.fixture +def proxy_pid() -> int: + """The proxy's PID, which the workflow exports after starting it. + + Required rather than discovered: picking a process out of the table by name would be + ambiguous on a developer machine running more than one proxy. + """ + pid: Final = os.environ.get("E2E_PROXY_PID") + assert pid and pid.isdigit(), ( + "E2E_PROXY_PID must hold the PID of the proxy under test; RSS and CPU are read from " + "its process tree because a multi-worker proxy does not report them on /metrics" + ) + return int(pid) + + +@pytest.fixture +def proxy_log() -> Path: + """Path to the proxy's stdout/stderr log, which the workflow captures to a file. + + Required rather than discovered for the same reason as proxy_pid: a developer machine may + have more than one proxy log around. + """ + path: Final = os.environ.get("E2E_PROXY_LOG") + assert path, "E2E_PROXY_LOG must hold the path the proxy's stdout/stderr was redirected to" + return Path(path) + + +def _log_bytes(path: Path) -> int: + return path.stat().st_size + + +@pytest.fixture +def redis_control() -> Iterator[redis.Redis[bytes]]: + """A control connection to the proxy's Redis, which unpauses it in teardown as a safety net. + + CLIENT PAUSE ALL freezes every connection including this one, so REDIS_PAUSE_MS is sized + to the chaos phase: by the time teardown runs, the pause has + already lapsed on its own and CLIENT UNPAUSE here returns immediately. It only actually + waits out a lapsed pause if the chaos phase itself overran that duration. + """ + host: Final = os.environ.get("REDIS_HOST") + port: Final = os.environ.get("REDIS_PORT") + assert host and port, "REDIS_HOST and REDIS_PORT must name the Redis the proxy under test uses" + control: Final = redis.Redis(host=host, port=int(port), socket_timeout=5) + try: + yield control + finally: + control.client_unpause() # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + control.close() + + +def _scrape(proxy: ProxyClient) -> str: + """One /metrics body, read once per checkpoint so every counter comes from the same instant.""" + scrape: Final = proxy.probe("/metrics", params=NoBody()) + assert scrape.status_code == 200, ( + f"/metrics did not answer ({scrape.status_code}: {scrape.body[:200]}), so no counter can be read; " + f"a silent 0 here would turn every before-and-after difference negative" + ) + return scrape.body + + +def _metric(scrape: str, pattern: re.Pattern[str]) -> float: + return sum(float(match.group(1)) for match in pattern.finditer(scrape)) + + +def _scrape_after_drain(proxy: ProxyClient, pattern: re.Pattern[str]) -> str: + """A /metrics body taken once `pattern`'s count has stopped moving. + + `set_llm_deployment_failure_metrics` runs from the async logging callback queue, so a load + generator that just stopped sending traffic can still have thousands of failure increments + in flight, and a scrape taken the instant load stops undercounts them. Settling on the + counter rather than sleeping a fixed duration keeps the wait proportional to how backed up + the queue actually is. + """ + deadline: Final = time.monotonic() + DRAIN_TIMEOUT_SECONDS + + def scrapes() -> Iterator[str]: + yield _scrape(proxy) + while time.monotonic() < deadline: + time.sleep(DRAIN_POLL_SECONDS) + yield _scrape(proxy) + + settled: Final = next( + (later for earlier, later in pairwise(scrapes()) if _metric(earlier, pattern) == _metric(later, pattern)), + None, + ) + return settled if settled is not None else _scrape(proxy) + + +def _register_deployments(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]: + """The model ids this run registered, which scope its per-deployment metric reads.""" + params: Final = ( + *(_failing_params() for _ in range(FAILING_DEPLOYMENTS)), + *(_serving_params() for _ in range(SERVING_DEPLOYMENTS)), + ) + model_ids: Final = tuple(proxy.create_model(MODEL_GROUP, one) for one in params) + for model_id in model_ids: + resources.defer(lambda doomed=model_id: proxy.delete_model(doomed)) + return model_ids + + +def _generate_key_pool(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]: + """A pool of virtual keys so auth and budget lookups are not one permanently warm + cache entry; each locust user picks one, so Redis auth reads actually happen.""" + keys: Final = tuple( + proxy.generate_key( + KeyGenerateBody(models=[MODEL_GROUP], key_alias=f"e2e-redis-chaos-{unique_marker()}-{index}") + ) + for index in range(KEY_POOL_SIZE) + ) + for key in keys: + resources.defer(lambda doomed=key: proxy.delete_key(doomed)) + return keys + + +def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: + return run_gateway_load( + base_url=PROXY_BASE_URL, + api_keys=keys, + model=MODEL_GROUP, + endpoints=LOAD_ENDPOINTS, + users=LOCUST_USERS, + spawn_rate=LOCUST_SPAWN_RATE, + duration_seconds=seconds, + ) + + +def _latency_budget(percentile: str, measured: float, ceiling: float) -> Budget: + return AbsoluteBudget(name=f"{percentile} latency", measured=measured, ceiling=ceiling, unit="s", decimals=3) + + +def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, fraction: float) -> Budget: + return RatioBudget( + name=f"{percentile} RSS", + baseline=baseline.rss_percentile(fraction) / 2**20, + degraded=degraded.rss_percentile(fraction) / 2**20, + ratio_ceiling=CHAOS_RSS_RATIO_CEILING, + unit=" MB", + decimals=0, + ) + + +def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: + """What a Redis outage is allowed to cost. + + Every request still succeeding is the headline assertion, but a proxy can answer every + request while leaking: the v1.100.0 regression (LIT-6780) served traffic the whole way up + to a 61 GB worker. These bound the cost of serving it. RSS and CPU are bounded against the + same run's healthy phase, latency and log bytes against a flat ceiling; see phase_budget + for why the two kinds of metric cannot share one shape. + + Latency and RSS are budgeted at p50, p90 and p99 so a regression that only shows up in the + tail (or only in the median) cannot hide behind the other. RSS gets the tightest bound: the + failure path has no business allocating more per request. CPU and log bytes are each budgeted + once, as an amount per request rather than per percentile: cores-busy saturates at the worker + count under load, so its percentiles read the same whether a request costs 10 ms of CPU or + 40, and cannot budget anything; per-request is the figure that actually moves. Log bytes + isolates the cost of the failed-tracking alert's own noisy error handling from the CPU it + burns doing useful retry work, since the two would otherwise be indistinguishable in one + CPU number. + """ + return ( + _latency_budget("p50", chaos.load.p50_seconds, CHAOS_P50_LATENCY_CEILING_SECONDS), + _latency_budget("p90", chaos.load.p90_seconds, CHAOS_P90_LATENCY_CEILING_SECONDS), + _latency_budget("p99", chaos.load.p99_seconds, CHAOS_P99_LATENCY_CEILING_SECONDS), + _rss_budget("p50", baseline.usage, chaos.usage, 0.5), + _rss_budget("p90", baseline.usage, chaos.usage, 0.9), + _rss_budget("p99", baseline.usage, chaos.usage, 0.99), + RatioBudget( + name="CPU per request", + baseline=baseline.cpu_seconds_per_request * 1000, + degraded=chaos.cpu_seconds_per_request * 1000, + ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING, + unit=" ms", + ), + AbsoluteBudget( + name="log bytes per request", + measured=chaos.log_bytes_per_request, + ceiling=CHAOS_LOG_BYTES_PER_REQUEST_CEILING, + unit=" B", + decimals=0, + ), + ) + + +@pytest.mark.redis_chaos +class TestRedisChaos: + @pytest.mark.covers( + "reliability.circuit_breaker.redis_timeout.stays_responsive", + exercised_on=("chat_completions", "messages"), + ) + def test_load_survives_redis_being_down( + self, + client: LoadClient, + resources: ResourceManager, + proxy_pid: int, + proxy_log: Path, + redis_control: redis.Redis[bytes], + ) -> None: + proxy: Final = client.proxy + model_ids: Final = _register_deployments(proxy, resources) + keys: Final = _generate_key_pool(proxy, resources) + + retries_re: Final = _deployment_metric_re("deployment_failure_responses_total", model_ids) + cooldown_re: Final = _deployment_metric_re("deployment_cooled_down_total", model_ids) + + at_start: Final = _scrape(proxy) + log_at_start: Final = _log_bytes(proxy_log) + + with ProxyUsageSampler(proxy_pid) as sampler: + baseline_load: Final = _drive(keys, BASELINE_SECONDS) + baseline_usage: Final = sampler.split() + after_baseline: Final = _scrape(proxy) + log_after_baseline: Final = _log_bytes(proxy_log) + + redis_control.client_pause(REDIS_PAUSE_MS, all=True) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + chaos_load: Final = _drive(keys, CHAOS_SECONDS) + chaos_usage: Final = sampler.split() + at_end: Final = _scrape_after_drain(proxy, retries_re) + log_at_end: Final = _log_bytes(proxy_log) + + baseline: Final = Phase( + name="baseline", + load=baseline_load, + usage=baseline_usage, + redis_timeouts=_metric(after_baseline, TIMEOUT_FAILURES_RE) - _metric(at_start, TIMEOUT_FAILURES_RE), + log_bytes=log_after_baseline - log_at_start, + ) + chaos: Final = Phase( + name="chaos", + load=chaos_load, + usage=chaos_usage, + redis_timeouts=_metric(at_end, TIMEOUT_FAILURES_RE) - _metric(after_baseline, TIMEOUT_FAILURES_RE), + log_bytes=log_at_end - log_after_baseline, + ) + report: Final = f"{baseline.report()} | {chaos.report()}" + + for phase in (baseline, chaos): + assert phase.load.requests > 0, ( + f"{phase.name} drove no traffic at all, so it proved nothing: {phase.load.diagnosis()}. {report}" + ) + assert frozenset(endpoint.name for endpoint in phase.load.endpoints) == frozenset(LOAD_ENDPOINTS), ( + f"{phase.name} drove {tuple(endpoint.name for endpoint in phase.load.endpoints)} rather than every " + f"endpoint in {LOAD_ENDPOINTS}; the round robin hands one endpoint to each simulated user, so a " + f"missing one means a route never ran and its request path was never exercised. {report}" + ) + assert phase.load.failures == 0, ( + f"{phase.name} had {phase.load.failures} of {phase.load.requests} requests fail. Every request " + f"must succeed: the failing deployments sit at order {FAILING_ORDER} and the serving one at order " + f"{SERVING_ORDER}, so once the retries on order {FAILING_ORDER} are spent the order-based fallback " + f"lands on the serving deployment. Failures mean it was cooled down, the fallback did not run, or " + f"a Redis failure reached the response path. {phase.load.diagnosis()}. {report}" + ) + + cooldowns: Final = _metric(at_end, cooldown_re) - _metric(at_start, cooldown_re) + assert cooldowns == 0, ( + f"{cooldowns:.0f} deployments were cooled down during the run; the failing deployments are supposed " + f"to stay in rotation so every request keeps exercising the retry path. {report}" + ) + + retries: Final = _metric(at_end, retries_re) - _metric(at_start, retries_re) + assert retries >= baseline.load.requests + chaos.load.requests, ( + f"only {retries:.0f} deployment failures were counted across " + f"{baseline.load.requests + chaos.load.requests} requests; the mock deployments did not fail, so no " + f"request carried retry breadcrumbs into cost tracking and the regression path was never entered. " + f"{report}" + ) + + transitions: Final = _metric(at_end, BREAKER_TRANSITIONS_RE) - _metric(after_baseline, BREAKER_TRANSITIONS_RE) + breaker_open: Final = _metric(at_end, BREAKER_OPEN_RE) >= 1 + assert transitions >= 1 or breaker_open, ( + f"pausing Redis produced no circuit breaker state transitions and it ended closed; nothing on the " + f"request path ever saw Redis fail, so this run proved nothing. {report}" + ) + + blown: Final = violations(_chaos_budgets(baseline, chaos)) + assert not blown, ( + f"pausing Redis cost the proxy more than a Redis outage is allowed to: {'; '.join(blown)}. {report}" + ) + + rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1) + assert rows, ( + f"no spend rows landed for the first key in the pool; a Redis outage must not cost the proxy its " + f"spend logs, which are written to Postgres through a queue rather than through Redis. {report}" + ) + + print(f"\nredis chaos load: {report}") # noqa: T201 # the numbers this test exists to report, read off the CI log diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 1ef0d89a8f9..e17b92a13ed 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -10,9 +10,7 @@ import time from dataclasses import dataclass import jwt - from e2e_config import MASTER_KEY -from proxy_client import ProxyClient from e2e_http import ( AuthHeaders, NetworkError, @@ -37,6 +35,8 @@ from models import ( KeyDeleteBody, KeyGenerateBody, KeyGenerateResponse, + KeyInfoParams, + KeyInfoResponse, KeyListParams, KeyListResponse, KeyRegenerateBody, @@ -81,6 +81,7 @@ from models import ( UserNewResponse, UserUpdateBody, ) +from proxy_client import ProxyClient MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied" ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route" @@ -152,13 +153,21 @@ class ManagementClient: def update_key_models(self, key: str, models: list[str]) -> None: _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) - def delete_key_strict(self, key: str) -> None: + def key_info_as(self, key: str, *, caller_key: str) -> Result[KeyInfoResponse]: + return self.proxy.transport.get( + "/key/info", + headers=self.proxy.transport.bearer(caller_key), + params=KeyInfoParams(key=key), + response_type=KeyInfoResponse, + ) + + def delete_key_strict(self, key: str, *, caller_key: str | None = None) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only ProxyClient.delete_key used at teardown.""" _ = unwrap( self.proxy.transport.post( "/key/delete", - headers=self.proxy.transport.master, + headers=self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key), json=KeyDeleteBody(keys=[key]), response_type=NoBody, ) diff --git a/tests/e2e/management/test_jwt_management_e2e.py b/tests/e2e/management/test_jwt_management_e2e.py new file mode 100644 index 00000000000..22306da8eb8 --- /dev/null +++ b/tests/e2e/management/test_jwt_management_e2e.py @@ -0,0 +1,91 @@ +"""Management writes and tenant isolation under credentials issued by Keycloak.""" + +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from idp import ADMIN_CLIENT_ID, Identity, Keycloak +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyGenerateBody, KeyUpdateBody, TeamNewBody, UserNewBody + +pytestmark = pytest.mark.e2e + + +class TestJwtManagement: + @pytest.mark.covers("mgmt.key.jwt.lifecycle") + def test_admin_creates_reads_updates_clears_and_deletes_a_key( + self, client: ManagementClient, idp: Keycloak, jwt_identity: Identity, resources: ResourceManager + ) -> None: + admin: Final = idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID) + alias: Final = f"e2e-jwt-key-{unique_marker()}" + created: Final = unwrap( + client.generate_key( + KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group, models=[CHEAP_OPENAI_MODEL]), + caller_key=admin, + ) + ) + resources.defer(lambda: client.proxy.delete_key(created.key)) + + original: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert original.key_alias == alias and original.team_id == jwt_identity.group + assert original.models == [CHEAP_OPENAI_MODEL] + + updated_alias: Final = f"{alias}-updated" + unwrap( + client.update_key(KeyUpdateBody(key=created.key, key_alias=updated_alias, rpm_limit=120), caller_key=admin) + ) + updated: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert updated.key_alias == updated_alias and updated.rpm_limit == 120 + assert updated.models == [CHEAP_OPENAI_MODEL], "omitted models must preserve the restriction" + + unwrap(client.update_key(KeyUpdateBody(key=created.key, models=[]), caller_key=admin)) + cleared: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert cleared.models == [] and cleared.rpm_limit == 120 + + assert unwrap(client.key_list(updated_alias, caller_key=admin)).total_count == 1 + client.delete_key_strict(created.key, caller_key=admin) + assert unwrap(client.key_list(updated_alias, caller_key=admin)).total_count == 0 + + @pytest.mark.covers("mgmt.key.jwt.member_denied", "mgmt.key.jwt.other_team_denied") + def test_member_cannot_write_and_another_team_cannot_read_the_key( + self, client: ManagementClient, idp: Keycloak, jwt_identity: Identity, resources: ResourceManager + ) -> None: + admin: Final = idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID) + member: Final = idp.access_token(jwt_identity) + alias: Final = f"e2e-jwt-owned-{unique_marker()}" + created: Final = unwrap( + client.generate_key(KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group), caller_key=admin) + ) + resources.defer(lambda: client.proxy.delete_key(created.key)) + + client.add_team_member(jwt_identity.group, jwt_identity.user_id) + assert unwrap(client.key_info_as(created.key, caller_key=member)).info.key_alias == alias + + refused: Final = client.update_key(KeyUpdateBody(key=created.key, key_alias="forbidden"), caller_key=member) + assert isinstance(refused, UnauthorizedError), f"member write was accepted: {refused}" + assert "does not have permissions for endpoint" in refused.body.lower(), ( + f"expected a permission denial: {refused}" + ) + assert unwrap(client.key_info_as(created.key, caller_key=admin)).info.key_alias == alias + + marker: Final = unique_marker() + outsider: Final = idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) + resources.defer(lambda: client.proxy.delete_user(outsider.user_id)) + client.create_user( + UserNewBody( + user_id=outsider.user_id, user_email=f"{outsider.username}@example.com", user_role="internal_user" + ) + ) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=marker, team_id=outsider.group)) + resources.defer(lambda: client.proxy.delete_team(team_id)) + client.add_team_member(outsider.group, outsider.user_id) + outsider_token: Final = idp.access_token(outsider) + hidden: Final = client.key_info_as(created.key, caller_key=outsider_token) + assert isinstance(hidden, UnknownApiError) and hidden.status_code == 403, ( + f"another team must not read this key: {hidden}" + ) + assert unwrap(client.key_info_as(created.key, caller_key=admin)).info.team_id == jwt_identity.group diff --git a/tests/e2e/models.py b/tests/e2e/models.py index faf8557498b..f362d4cc6e5 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -11,7 +11,16 @@ from datetime import datetime from typing import Final, Literal from e2e_http import PartialBody -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, RootModel, model_serializer, model_validator +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + JsonValue, + RootModel, + model_serializer, + model_validator, +) # ---------- keys ---------- @@ -695,6 +704,7 @@ class SpendLogRow(BaseModel): total_tokens: int | None = None request_tags: list[str] | None = None metadata: SpendLogMetadata | None = None + proxy_server_request: JsonValue = None class SpendLogs(RootModel[list[SpendLogRow]]): @@ -922,10 +932,12 @@ class LiteLLMParamsBody(BaseModel): auto_router_default_model: str | None = None auto_router_embedding_model: str | None = None tags: list[str] | None = None - mock_response: str | None = None + mock_response: str | list[float] | None = None timeout: float | None = None tpm: int | None = None weight: int | None = None + cooldown_time: float | None = None + order: int | None = None ModelMode = Literal["batch", "realtime", "image_generation"] @@ -1260,6 +1272,19 @@ class TagListResponse(RootModel[list[TagListEntry]]): # ---------- health / lifecycle ---------- +class ProcessMemory(BaseModel): + ram_usage_mb: float | None = None + system_memory_percent: float | None = None + error: str | None = None + + +class MemorySummaryResponse(BaseModel): + worker_pid: int + hostname: str | None = None + status: str + memory: ProcessMemory + + class ReadinessResponse(BaseModel): """GET /health/readiness (public probe). The low-detail payload a load balancer sees: `status` plus the resolved DB state (`connected`, diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index 1aa83ac42c7..4313bbe4068 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -1,11 +1,14 @@ """Client for the `other` holding-pen suite: the auth gate (master key vs an -invalid key on an admin route) and the process-lifecycle health probes -(liveness, public readiness, authenticated readiness diagnostics). +invalid key on an admin route), JWT auth against the suite's Keycloak realm +(idp.py), and the process-lifecycle health probes (liveness, public readiness, +authenticated readiness diagnostics). Holds the shared ProxyClient so `resources` / `scoped_key` still clean up, and adds only the routes these behaviors need. The health probes deliberately send no auth header (public routes), so they go through the transport with an empty -headers model rather than a bearer. +headers model rather than a bearer. JWT tests reach the identity provider +through `idp`, which provisions identities and mints tokens through Keycloak's +own endpoints, so no test ever holds a signing key. """ from __future__ import annotations @@ -13,6 +16,7 @@ from __future__ import annotations from dataclasses import dataclass from e2e_http import NoBody, ProbeResult, Result +from idp import Keycloak, keycloak_from_env from models import ( ReadinessDetailsResponse, ReadinessResponse, @@ -26,6 +30,11 @@ from proxy_client import ProxyClient class OtherClient: proxy: ProxyClient + @property + def idp(self) -> Keycloak: + """Resolved per use, so the suite's non-JWT tests never need the IdP env.""" + return keycloak_from_env() + def liveness(self) -> ProbeResult: """GET /health/liveliness. Unauthenticated; the probe returns status + raw body so the test can assert the worker reports itself alive.""" diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py new file mode 100644 index 00000000000..2bed40f4d69 --- /dev/null +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -0,0 +1,157 @@ +"""Real Keycloak tokens exercise verification, attribution and virtual-key coexistence.""" + +from __future__ import annotations + +import base64 +import time +from typing import Final + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from idp import SHORT_LIVED_CLIENT_ID, WRONG_AUDIENCE_CLIENT_ID, Identity +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, TeamNewBody +from other_client import OtherClient +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + + +class IssuedClaims(BaseModel): + """Read the IdP's signed payload only to check the test precondition.""" + + exp: int + sub: str + iss: str + aud: str | list[str] + + +def _claims(token: str) -> IssuedClaims: + payload: Final = token.split(".")[1] + return IssuedClaims.model_validate_json(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + + +def _provision(client: OtherClient, resources: ResourceManager, *, marker: str) -> Identity: + """A Keycloak group and a user in it, torn down with the test. The group name + is what the token's `groups` claim carries, which is what the proxy resolves + as a litellm team id.""" + identity: Final = client.idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) + resources.defer(lambda: client.proxy.delete_user(identity.user_id)) + return identity + + +@pytest.fixture +def identity(client: OtherClient, resources: ResourceManager) -> Identity: + """An IdP identity whose group is also a real litellm team, so anything the + proxy rejects is about the token and never about an unresolvable team.""" + marker: Final = unique_marker() + provisioned: Final = _provision(client, resources, marker=marker) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=provisioned.group)) + resources.defer(lambda: client.proxy.delete_team(team_id)) + return provisioned + + +def _ping() -> ChatBody: + return ChatBody( + model=CHEAP_OPENAI_MODEL, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=16, + ) + + +def _corrupt_signature(token: str) -> str: + header, payload, signature = token.split(".") + flipped: Final = "A" if signature[10] != "A" else "B" + return f"{header}.{payload}.{signature[:10]}{flipped}{signature[11:]}" + + +class TestJwtAuth: + @pytest.mark.covers("other.auth.jwt.valid_token_allows", "other.auth.jwt.spend_attributed_to_claims") + def test_valid_token_for_an_existing_team_is_accepted_and_attributed( + self, client: OtherClient, identity: Identity + ) -> None: + token: Final = client.idp.access_token(identity) + + assert _claims(token).sub == identity.user_id, "IdP must emit the provisioned user as sub" + response: Final = unwrap(client.proxy.chat(token, _ping())) + assert response.id is not None and response.choices, ( + f"chat under a valid JWT returned no completion: {response}" + ) + + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert rows, f"no spend log row for request {response.id} within the poll deadline" + row: Final = rows[0] + assert row.team_id == identity.group, ( + f"spend row must carry the team from the JWT groups claim {identity.group!r}, got {row.team_id!r}" + ) + assert row.user == identity.user_id, ( + f"spend row must carry the user from the JWT sub claim {identity.user_id!r}, got {row.user!r}" + ) + + @pytest.mark.covers("other.auth.jwt.invalid_signature_denied") + def test_tampered_signature_is_rejected(self, client: OtherClient, identity: Identity) -> None: + tampered: Final = _corrupt_signature(client.idp.access_token(identity)) + + result: Final = client.proxy.chat(tampered, _ping()) + assert isinstance(result, UnauthorizedError), ( + f"a JWT whose signature does not verify must be rejected with 401, got {result}" + ) + assert "signature verification failed" in result.body.lower(), ( + f"the 401 must come from signature verification, not another auth failure, got {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.jwt.expired_denied") + def test_expired_token_is_rejected(self, client: OtherClient, identity: Identity) -> None: + expiring: Final = client.idp.access_token(identity, client_id=SHORT_LIVED_CLIENT_ID) + delay: Final = _claims(expiring).exp - time.time() + 1 + assert delay <= 5, f"short-lived client expiry or IdP clock drifted: wait would be {delay}s" + time.sleep(max(0, delay)) + + result: Final = client.proxy.chat(expiring, _ping()) + assert isinstance(result, UnauthorizedError), ( + f"an expired JWT must be rejected with 401 even though its signature verifies, got {result}" + ) + assert "expired" in result.body.lower(), f"the 401 must say the token expired, got {result.body[:300]}" + + @pytest.mark.covers("other.auth.jwt.wrong_issuer_denied") + def test_signed_token_from_the_wrong_issuer_is_rejected(self, client: OtherClient, identity: Identity) -> None: + token: Final = client.idp.access_token(identity, issuer_host="unexpected-issuer.invalid") + claims: Final = _claims(token) + assert claims.iss != client.idp.issuer and "litellm-e2e" in claims.aud + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnauthorizedError), f"wrong issuer must be rejected: {result}" + assert "issuer" in result.body.lower(), f"expected issuer validation to reject the token: {result}" + + @pytest.mark.covers("other.auth.jwt.wrong_audience_denied") + def test_signed_token_for_another_application_is_rejected(self, client: OtherClient, identity: Identity) -> None: + token: Final = client.idp.access_token(identity, client_id=WRONG_AUDIENCE_CLIENT_ID) + claims: Final = _claims(token) + assert claims.iss == client.idp.issuer and "litellm-e2e" not in ( + [claims.aud] if isinstance(claims.aud, str) else claims.aud + ) + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnauthorizedError), f"wrong audience must be rejected: {result}" + assert "audience" in result.body.lower(), f"expected audience validation to reject the token: {result}" + + @pytest.mark.covers("other.auth.jwt.unknown_team_denied") + def test_token_naming_a_team_that_does_not_exist_is_rejected( + self, client: OtherClient, resources: ResourceManager + ) -> None: + stranger: Final = _provision(client, resources, marker=unique_marker()) + token: Final = client.idp.access_token(stranger) + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnknownApiError) and result.status_code == 403, ( + f"a valid JWT whose groups name no existing team must be rejected with 403, got {result}" + ) + assert stranger.group in result.body, ( + f"the 403 must name the team it could not resolve ({stranger.group}), got {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.jwt.virtual_key_unaffected") + def test_plain_virtual_key_still_works_with_jwt_auth_enabled(self, client: OtherClient, scoped_key: str) -> None: + response: Final = unwrap(client.proxy.chat(scoped_key, _ping())) + assert response.choices, f"an sk- key must keep working on a proxy with enable_jwt_auth, got {response}" diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 1bac5116a9d..1fe2ec905ef 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -63,6 +63,7 @@ from models import ( ModelNewBody, ModelNewResponse, ModelsListParams, + MemorySummaryResponse, ModelsListResponse, ModelUpdateBody, OcrBody, @@ -72,6 +73,11 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, + TeamDeleteBody, + TeamNewBody, + TeamNewResponse, + UserDeleteBody, + UserDeleteResponse, ToolsetCreateBody, ToolsetRow, ToolsetUpdateBody, @@ -467,6 +473,17 @@ class ProxyClient: ) ).info + def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: + return { + url: transport.get( + "/debug/memory/summary", + headers=transport.master, + params=NoBody(), + response_type=MemorySummaryResponse, + ) + for url, transport in self.replicas.items() + } + def read_back_everywhere[R: BaseModel]( self, path: str, @@ -794,6 +811,41 @@ class ProxyClient: if not is_ok(result): warnings.warn(f"delete_credential({credential_name!r}) failed: {result}", stacklevel=2) + def create_team(self, body: TeamNewBody) -> str: + return unwrap( + self.transport.post( + "/team/new", + headers=self.transport.master, + json=body, + response_type=TeamNewResponse, + ) + ).team_id + + def delete_team(self, team_id: str) -> None: + result = self.transport.post( + "/team/delete", + headers=self.transport.master, + json=TeamDeleteBody(team_ids=[team_id]), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_team({team_id!r}) failed: {result}", stacklevel=2) + + def delete_user(self, user_id: str) -> None: + """Best-effort teardown; a 404 is not a leak, since JWT tests defer this for + a user the proxy only upserts after a successful auth.""" + result = self.transport.post( + "/user/delete", + headers=self.transport.master, + json=UserDeleteBody(user_ids=[user_id]), + response_type=UserDeleteResponse, + ) + match result: + case Success() | UnknownApiError(status_code=404): + return + case _: + warnings.warn(f"delete_user({user_id!r}) failed: {result}", stacklevel=2) + # ---- LLM calls ------------------------------------------------------ def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c3f8865f218..774d9644497 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,3 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set + redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index 98501f9bd7c..2d5e546dc6c 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -13,10 +13,7 @@ from __future__ import annotations from collections.abc import Iterator import pytest -from requests import RequestException - from complexity_router_client import ComplexityRouterClient, build_client -from proxy_client import ProxyClient from e2e_http import NoBody, Success from lifecycle import ResourceManager from models import ( @@ -26,6 +23,8 @@ from models import ( LiteLLMParamsBody, ModelsListResponse, ) +from proxy_client import ProxyClient +from requests import RequestException ROUTER_MODEL = "complexity-smart-router" ROUTER_PARAMS = LiteLLMParamsBody( @@ -120,8 +119,6 @@ def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # @pytest.fixture def complexity_key(resources: ResourceManager, client: ComplexityRouterClient) -> str: """Per-test key allowed to call the complexity router and its tier backends.""" - key = client.proxy.generate_key( - KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router") - ) + key = client.proxy.generate_key(KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router")) resources.defer(lambda: client.proxy.delete_key(key)) return key diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 1efcb1a045b..df2ff03aa4a 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -11,6 +11,8 @@ body, so a single long-lived proxy serves every reliability behavior. from __future__ import annotations +from collections.abc import Sequence + from pydantic import ValidationError from proxy_client import ProxyClient @@ -49,6 +51,13 @@ def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: ) +def create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> str: + return proxy.create_model( + name, + LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1", cooldown_time=0), + ) + + def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: """Register a deployment with a 1ms deadline the real backend always exceeds.""" return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001)) @@ -109,6 +118,7 @@ def chat_override( override: RouterSettingsOverride | None = None, stream: bool = False, cache: dict[str, bool] | None = {"no-cache": True}, + history: Sequence[ChatMessage] = (), ) -> StreamingResponse: """POST /chat/completions with an optional per-request router_settings_override, returning the raw outcome so tests read status, body, and reliability headers.""" @@ -117,7 +127,7 @@ def chat_override( headers=proxy.transport.bearer(key), json=ReliabilityChatBody( model=model, - messages=[ChatMessage(role="user", content=content)], + messages=[*history, ChatMessage(role="user", content=content)], max_tokens=512, stream=stream, router_settings_override=override, diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py new file mode 100644 index 00000000000..17e3e1a1996 --- /dev/null +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -0,0 +1,263 @@ +"""Live e2e: a few hundred requests that fail before any provider answers must not +grow the proxy's resident memory past a fixed budget once the proxy is warm. + +The regression this guards shipped in v1.100.0: every retry breadcrumb copied the +whole request and the copies nested into one router-global list, so a proxy under +retry-heavy failing traffic grew until it was OOM-killed. The traffic here has that +shape: a model group whose deployments refuse at the socket (an unreachable base +URL) with cooldown_time 0 so the router keeps retrying them, per-request retries, +and a fallback group that refuses the same way, each request carrying a long chat +transcript so every whole-request copy costs hundreds of containers instead of a +handful. Under the stack's cooldown policy a +deployment that fails a handful of times in a row is benched (a bad-credential 401 +included), the router answers "No deployments available" without retrying, and the +retry loop that leaks stops running; cooldown_time 0 keeps it running. + +Two identical phases run back to back. The first is the warmup that grows the +proxy's caches and allocator arenas to their steady state, the second is the one +the budget applies to, so a healthy proxy shows the second phase adding roughly +nothing while a leaking one adds a fixed amount per request. RSS is read through +/debug/memory/summary on every configured replica; a burst of failing calls leaves +a transient bulge of garbage that gc reclaims within seconds, so each checkpoint +samples until no new worker has answered for a settle window and keeps the lowest +reading per worker. The growth is judged per worker (by replica address, hostname +and pid, since pods in their own pid namespaces report the same pids) so each +worker is compared with itself, and the two checkpoints must see the same workers: +a single load-balanced address reaches the workers behind it one answer at a time, +and a worker that answered only one checkpoint would otherwise drop out of the +comparison, which is where a leaking worker could hide. + +RSS alone is a coarse gauge: on the release stack (spend logs storing prompts, +json logs, prometheus and otel callbacks) the same v1.100.0 breadcrumbs grew RSS +by only about 15 MB per 300 failing requests, while every failing request's stored +request snapshot carried a copy of the request per failed attempt, over 100 KB on +the first call and a couple of MB once the copies nested, against tens of KB with +the fix. So the first check sends one failing request before the phases, reads its +spend log back through /spend/logs, and holds the stored request body to a fixed +size budget: the deterministic catch for a breadcrumb that copies the whole +request. It runs before the phases because the leaking writer drops its own rows +under the phases' traffic (a queue budget hit, a recursion limit on the nested +copies), which would turn the size check into a missing-row check. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import pytest + +from complexity_router_client import ComplexityRouterClient +from e2e_config import ( + MEMORY_CONCURRENCY, + MEMORY_REQUESTS_PER_PHASE, + MEMORY_RETRIES_PER_REQUEST, + MEMORY_RSS_BUDGET_MB, + MEMORY_RSS_SAMPLE_INTERVAL_SECONDS, + MEMORY_RSS_SETTLE_SAMPLES, + MEMORY_STORED_REQUEST_BUDGET_KB, + MEMORY_TRANSCRIPT_TURNS, + unique_marker, +) +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ChatMessage, RouterSettingsOverride, SpendLogRow +from proxy_client import ProxyClient +from reliability_support import chat_override, create_never_benched_refusing_deployment + +pytestmark = pytest.mark.e2e + +DEPLOYMENTS_PER_GROUP: Final = 2 +RSS_SAMPLE_CAP: Final = 4 * MEMORY_RSS_SETTLE_SAMPLES + + +@dataclass(frozen=True, slots=True) +class FailedCall: + status_code: int + seconds: float + body_head: str + call_id: str | None + + +WorkerKey = tuple[str, str | None, int] + + +@dataclass(frozen=True, slots=True) +class RssReading: + replica: str + hostname: str | None + worker_pid: int + ram_usage_mb: float + + @property + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) + + +@dataclass(frozen=True, slots=True) +class WorkerGrowth: + warm: RssReading + after: RssReading + + @property + def growth_mb(self) -> float: + return self.after.ram_usage_mb - self.warm.ram_usage_mb + + +def _register_refusing_group(proxy: ProxyClient, resources: ResourceManager, name: str) -> None: + for model_id in tuple(create_never_benched_refusing_deployment(proxy, name) for _ in range(DEPLOYMENTS_PER_GROUP)): + resources.defer(lambda model_id=model_id: proxy.delete_model(model_id)) + + +def _transcript(turns: int) -> tuple[ChatMessage, ...]: + return tuple( + ChatMessage(role=role, content=f"turn {turn} {role}") + for turn in range(turns) + for role in ("user", "assistant") + ) + + +TRANSCRIPT: Final = _transcript(MEMORY_TRANSCRIPT_TURNS) + + +def _fail_once(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> FailedCall: + started: Final = time.perf_counter() + resp: Final = chat_override( + proxy, key, model, f"memory regression {unique_marker()}", override=override, history=TRANSCRIPT + ) + return FailedCall(resp.status_code, time.perf_counter() - started, resp.body[:300], resp.call_id) + + +def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> tuple[FailedCall, ...]: + with ThreadPoolExecutor(max_workers=MEMORY_CONCURRENCY) as pool: + futures: Final = tuple( + pool.submit(_fail_once, proxy, key, model, override) for _ in range(MEMORY_REQUESTS_PER_PHASE) + ) + return tuple(future.result() for future in futures) + + +def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: + time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) + return tuple( + RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) + for replica, result in proxy.memory_summary_everywhere().items() + for body in (unwrap(result),) + if body.memory.ram_usage_mb is not None + ) + + +def _readings_until_no_new_worker( + proxy: ProxyClient, readings: tuple[RssReading, ...], samples: int, samples_since_new_worker: int +) -> tuple[RssReading, ...]: + if samples >= RSS_SAMPLE_CAP or samples_since_new_worker >= MEMORY_RSS_SETTLE_SAMPLES: + return readings + sample: Final = _read_rss_everywhere_after_pause(proxy) + known: Final = frozenset(reading.worker for reading in readings) + new_worker_answered: Final = any(reading.worker not in known for reading in sample) + return _readings_until_no_new_worker( + proxy, readings + sample, samples + 1, 0 if new_worker_answered else samples_since_new_worker + 1 + ) + + +def _settled_rss_per_worker(proxy: ProxyClient) -> Mapping[WorkerKey, RssReading]: + readings: Final = _readings_until_no_new_worker(proxy, (), 0, 0) + assert readings, "no /debug/memory/summary read carried ram_usage_mb, so the proxy cannot report its RSS" + return MappingProxyType( + { + worker: min((reading for reading in readings if reading.worker == worker), key=lambda r: r.ram_usage_mb) + for worker in {reading.worker for reading in readings} + } + ) + + +def _heaviest_worker_growth( + warm: Mapping[WorkerKey, RssReading], after: Mapping[WorkerKey, RssReading] +) -> WorkerGrowth: + assert warm.keys() == after.keys(), ( + f"the workers answering /debug/memory/summary changed between the checkpoints, so not every worker can " + f"be compared with itself: gone after the measured batch {sorted(warm.keys() - after.keys())} (a worker " + f"that died or was restarted under failing traffic, which is what an OOM kill looks like), first seen " + f"after it {sorted(after.keys() - warm.keys())} (the warm window never reached them, so they have no " + f"baseline; raise E2E_MEMORY_RSS_SETTLE_SAMPLES if the stack has more workers than the window covers)" + ) + return max((WorkerGrowth(warm[worker], after[worker]) for worker in warm), key=lambda growth: growth.growth_mb) + + +def _assert_every_call_failed_through_fallback(calls: Sequence[FailedCall], fallback: str) -> None: + served: Final = tuple(call for call in calls if call.status_code == 200) + assert not served, ( + f"{len(served)} of {len(calls)} calls came back 200, so they reached a provider and never " + f"exercised the retry loop: {served[0].body_head}" + ) + without_fallback: Final = tuple(call for call in calls if fallback not in call.body_head) + assert not without_fallback, ( + f"{len(without_fallback)} of {len(calls)} failures never named the fallback group {fallback}, " + f"so the request did not run through retries into the fallback: {without_fallback[0].body_head}" + ) + + +def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: + assert call.call_id, ( + f"the failing call carried no x-litellm-call-id header, so its spend log cannot be read back: {call.body_head}" + ) + rows: Final[Sequence[SpendLogRow]] = proxy.poll_logs_for_request_id(call.call_id) + assert rows, ( + f"no spend log row appeared for failing call {call.call_id} within the poll window: either the stack " + "writes no spend logs or its writer dropped the row, which the v1.100.0 one did once the stored " + "request outgrew the writer's queue budget" + ) + snapshot: Final = rows[0].proxy_server_request + assert snapshot, ( + f"spend log {call.call_id} stored no request body, so the stack is not running with " + "general_settings.store_prompts_in_spend_logs and the stored-request check would pass vacuously" + ) + return len(json.dumps(snapshot).encode()) / 1024 + + +class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.memory.under_slo") + def test_failing_requests_do_not_grow_rss_or_stored_request( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + marker: Final = unique_marker() + primary: Final = f"reliability-memory-{marker}" + fallback: Final = f"reliability-memory-fb-{marker}" + _register_refusing_group(client.proxy, resources, primary) + _register_refusing_group(client.proxy, resources, fallback) + override: Final = RouterSettingsOverride( + num_retries=MEMORY_RETRIES_PER_REQUEST, fallbacks=[{primary: [fallback]}] + ) + + probe: Final = _fail_once(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback((probe,), fallback) + stored_kb: Final = _stored_request_kb(client.proxy, probe) + assert stored_kb <= MEMORY_STORED_REQUEST_BUDGET_KB, ( + f"the spend log of one failing request stored a {stored_kb:.0f} KB request body, past the " + f"{MEMORY_STORED_REQUEST_BUDGET_KB:.0f} KB budget for a {len(TRANSCRIPT)}-message transcript with " + f"{MEMORY_RETRIES_PER_REQUEST} retries and a fallback; the retry breadcrumbs are copying the whole " + f"request into the stored snapshot the way the v1.100.0 ones did" + ) + + warmup: Final = _fail_many(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback(warmup, fallback) + warm: Final = _settled_rss_per_worker(client.proxy) + + measured: Final = _fail_many(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback(measured, fallback) + after: Final = _settled_rss_per_worker(client.proxy) + + heaviest: Final = _heaviest_worker_growth(warm, after) + assert heaviest.growth_mb <= MEMORY_RSS_BUDGET_MB, ( + f"proxy RSS grew {heaviest.growth_mb:.1f} MB over a second batch of {MEMORY_REQUESTS_PER_PHASE} failing " + f"requests ({MEMORY_RETRIES_PER_REQUEST} retries each plus a fallback) after an identical warmup batch, " + f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} on " + f"{heaviest.warm.hostname or 'an unnamed host'} behind {heaviest.warm.replica} settled at " + f"{heaviest.warm.ram_usage_mb:.1f} MB warm and " + f"{heaviest.after.ram_usage_mb:.1f} MB after; failing requests are leaking memory the way the " + f"v1.100.0 retry breadcrumbs did" + ) diff --git a/tests/e2e/test_idp.py b/tests/e2e/test_idp.py new file mode 100644 index 00000000000..33a09a0f13a --- /dev/null +++ b/tests/e2e/test_idp.py @@ -0,0 +1,179 @@ +"""Harness coverage for idp.py: the pure parts of the Keycloak client, which are +the ones a wrong value in silently mistargets. No proxy and no IdP needed, so +these carry no `e2e` marker and run everywhere.""" + +from __future__ import annotations + +from collections.abc import Callable, Generator +from contextlib import ExitStack, contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import SimpleQueue +from threading import Thread +from typing import Final + +import pytest +from e2e_http import ExternalWrite +from idp import ( + KEYCLOAK_ADMIN_PASSWORD_ENV, + KEYCLOAK_ADMIN_USER_ENV, + KEYCLOAK_REALM_ENV, + KEYCLOAK_URL_ENV, + Keycloak, + PasswordCredential, + UserCreateBody, + created_id, + keycloak_from_env, +) + +_REALM: Final = Keycloak( + base_url="http://keycloak:8080", realm="litellm-e2e", admin_username="admin", admin_password="pw" +) + + +def test_realm_urls_match_keycloaks_own_layout() -> None: + assert _REALM.issuer == "http://keycloak:8080/realms/litellm-e2e" + assert _REALM.jwks_url == "http://keycloak:8080/realms/litellm-e2e/protocol/openid-connect/certs" + assert _REALM.token_url("master") == "http://keycloak:8080/realms/master/protocol/openid-connect/token" + + +def test_created_id_is_the_last_segment_of_the_location_header() -> None: + created: Final = ExternalWrite( + status_code=201, location="http://keycloak:8080/admin/realms/litellm-e2e/groups/abc-123" + ) + assert created_id(created, "a group") == "abc-123" + + +def test_a_refused_create_fails_the_test_with_the_idps_own_words() -> None: + with pytest.raises(BaseException, match=r"409.*already exists"): + created_id(ExternalWrite(status_code=409, body="Group already exists"), "a group") + + +@pytest.mark.parametrize("location", ["", "http://keycloak/groups/"]) +def test_create_without_a_resource_id_fails(location: str) -> None: + with pytest.raises(pytest.fail.Exception, match="resource id"): + created_id(ExternalWrite(status_code=201, location=location), "a group") + + +@contextmanager +def _idp_server( + *, user_status: int = 201, delete_status: int = 204, admin_status: int = 200 +) -> Generator[tuple[Keycloak, SimpleQueue[str]]]: + """Exercise provisioning failures through the same HTTP transport as live tests.""" + deletions: SimpleQueue[str] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + pass + + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path.endswith("/token"): + self.send_response(admin_status) + self.end_headers() + self.wfile.write(b'{"access_token":"synthetic-harness-token"}') + else: + self.send_response(user_status if self.path.endswith("/users") else 201) + self.send_header("Location", f"{self.path}/resource-1") + self.end_headers() + if user_status != 201 and self.path.endswith("/users"): + self.wfile.write(b"injected create failure") + + def do_DELETE(self) -> None: + deletions.put(self.path) + self.send_response(delete_status) + self.end_headers() + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield ( + Keycloak( + base_url=f"http://127.0.0.1:{server.server_port}", + realm="test", + admin_username="admin", + admin_password="pw", + ), + deletions, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_partial_provisioning_removes_the_group_when_user_creation_fails() -> None: + with _idp_server(user_status=500) as (idp, deletions): + with ExitStack() as cleanup: + + def defer(callback: Callable[[], object]) -> None: + cleanup.callback(callback) + + with pytest.raises(pytest.fail.Exception, match="injected create failure"): + idp.provision(marker="partial", group="team", defer=defer) + assert deletions.get_nowait() == "/admin/realms/test/groups/resource-1" + assert deletions.empty() + + +def test_successful_provisioning_cleans_up_user_before_group() -> None: + with _idp_server() as (idp, deletions): + with ExitStack() as cleanup: + + def defer(callback: Callable[[], object]) -> None: + cleanup.callback(callback) + + idp.provision(marker="complete", group="team", defer=defer) + assert deletions.get_nowait() == "/admin/realms/test/users/resource-1" + assert deletions.get_nowait() == "/admin/realms/test/groups/resource-1" + assert deletions.empty() + + +def test_cleanup_failure_is_visible() -> None: + with _idp_server(delete_status=500) as (idp, _): + with pytest.warns(RuntimeWarning, match="cleanup failed.*HTTP 500"): + idp.delete_group("group") + + +def test_expired_admin_credentials_do_not_abort_remaining_cleanups() -> None: + with _idp_server(admin_status=401) as (idp, _): + cleanup: Final = ExitStack() + cleanup.callback(idp.delete_group, "group") + cleanup.callback(idp.delete_user, "user") + with pytest.warns(RuntimeWarning, match="cleanup could not authenticate") as warnings: + cleanup.close() + assert len(warnings) == 2 + + +def test_new_users_are_born_fully_set_up() -> None: + """A user without a profile or with a pending required action authenticates + nowhere: Keycloak answers every grant with "Account is not fully set up".""" + body: Final = UserCreateBody( + username="e2e", email="e2e@example.com", groups=("team",), credentials=(PasswordCredential(value="pw"),) + ).model_dump(by_alias=True) + + assert body["requiredActions"] == () + assert body["firstName"] and body["lastName"] and body["emailVerified"] is True + assert body["credentials"][0]["temporary"] is False + + +def test_connection_details_come_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(KEYCLOAK_URL_ENV, "http://keycloak.litellm.svc.cluster.local:8080/") + monkeypatch.setenv(KEYCLOAK_REALM_ENV, "other-realm") + monkeypatch.setenv(KEYCLOAK_ADMIN_USER_ENV, "admin") + monkeypatch.setenv(KEYCLOAK_ADMIN_PASSWORD_ENV, "pw") + + resolved: Final = keycloak_from_env() + + assert resolved.issuer == "http://keycloak.litellm.svc.cluster.local:8080/realms/other-realm" + assert resolved.admin_username == "admin" and resolved.admin_password == "pw" + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_a_missing_admin_credential_fails_loudly_instead_of_skipping( + monkeypatch: pytest.MonkeyPatch, blank: str +) -> None: + monkeypatch.setenv(KEYCLOAK_ADMIN_USER_ENV, "admin") + monkeypatch.setenv(KEYCLOAK_ADMIN_PASSWORD_ENV, blank) + + with pytest.raises(BaseException, match=KEYCLOAK_ADMIN_PASSWORD_ENV): + keycloak_from_env() diff --git a/tests/store_model_in_db_tests/test_openai_error_handling.py b/tests/store_model_in_db_tests/test_openai_error_handling.py index 5de7c427d19..22707d522bc 100644 --- a/tests/store_model_in_db_tests/test_openai_error_handling.py +++ b/tests/store_model_in_db_tests/test_openai_error_handling.py @@ -210,7 +210,8 @@ async def test_chat_completion_bad_model_with_spend_logs(): assert "traceback" in error_info assert error_info["error_code"] == "400" assert error_info["error_class"] in ("ProxyModelNotFoundError", "BadRequestError") - assert "non-existent-model" in error_info["error_message"] + assert "non-existent-model" not in error_info["error_message"] + assert "/chat/completions: Invalid model name passed in" in error_info["error_message"] # Verify request details assert log_entry["cache_hit"] == "False" diff --git a/tests/test_litellm/caching/test_in_memory_cache.py b/tests/test_litellm/caching/test_in_memory_cache.py index 85e8308ae91..40ad4f0c6f0 100644 --- a/tests/test_litellm/caching/test_in_memory_cache.py +++ b/tests/test_litellm/caching/test_in_memory_cache.py @@ -250,3 +250,27 @@ def test_in_memory_cache_prunes_expired_heap_entries_below_capacity(): assert len(in_memory_cache.cache_dict) == 5 assert len(in_memory_cache.ttl_dict) == 5 assert len(in_memory_cache.expiration_heap) == 5 + + +def test_in_memory_cache_injected_clock_controls_expiry_and_eviction() -> None: + class Clock: + now = 0.0 + + def __call__(self) -> float: + return self.now + + clock = Clock() + cache = InMemoryCache(max_size_in_memory=2, default_ttl=60, clock=clock) + cache.set_cache("first", "original", ttl=10) + clock.now = 9.0 + cache.set_cache("second", "survivor") + assert cache.get_cache("first") == "original" + clock.now = 10.001 + assert cache.get_cache("first") is None + cache.set_cache("third", "replacement") + assert cache.get_cache("second") == "survivor" + clock.now = 69.001 + cache.set_cache("fourth", "new") + assert cache.get_cache("second") is None + assert cache.get_cache("third") == "replacement" + assert cache.get_cache("fourth") == "new" diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py index c1c3569c0e2..bf9eeeb9968 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py @@ -10,12 +10,15 @@ Covers the three defects from the ticket: handling live only on the native path). """ +import tempfile import time import pytest from litellm_enterprise.enterprise_callbacks.secret_detection import ( _ENTERPRISE_SecretDetection, + _default_detect_secrets_config, + _masked_entity_count, ) from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -29,6 +32,13 @@ URL_ENCODED_KEY = "Bearer%20sk-Ab3dEf6Gh7Ij8Kl9Mn0Pq2Rs3Tu4Vw5X" AWS_KEYS = [f"AKIAIOSFODNN7EXAMPL{suffix}" for suffix in "FEDCBA"] +@pytest.fixture(autouse=True) +def _isolate_masked_entity_count(): + token = _masked_entity_count.set(None) + yield + _masked_entity_count.reset(token) + + def _guardrail() -> _ENTERPRISE_SecretDetection: return _ENTERPRISE_SecretDetection(guardrail_name="hide-secrets", event_hook="pre_call", default_on=True) @@ -58,6 +68,561 @@ def test_scan_message_preserves_quoted_benign_identifiers(): assert guardrail.redact_text(content) == content +@pytest.mark.parametrize( + "content,secret", + [ + ("REDIS_PASSWORD=aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + ("SESSION_SECRET=Kp7Nq2Wz9Bt4Xr6Vm1Ls", "Kp7Nq2Wz9Bt4Xr6Vm1Ls"), + ('{"db_password": "Tq8Zm2XpLv9KdNbRcYw3"}', "Tq8Zm2XpLv9KdNbRcYw3"), + ("api_secret: Zx4Kp9Lm2Qr7Ns3Vt", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("password = hunter2brahms9x", "hunter2brahms9x"), + ("client_secret=Hq7Zm3XkLp9Wd2Nb", "Hq7Zm3XkLp9Wd2Nb"), + ('apiKey: "aB3dE6gH9jK2mN5p"', "aB3dE6gH9jK2mN5p"), + ('{"clientSecret": "Kp7Nq2Wz9Bt4Xr6Vm1Ls"}', "Kp7Nq2Wz9Bt4Xr6Vm1Ls"), + ('dbPassword = "Zx4Kp9Lm2Qr7Ns3Vt"', "Zx4Kp9Lm2Qr7Ns3Vt"), + ("MY_APP_DB_PASSWORD=Kp7Nq2Wz9Bt4Xr6Vm1Ls", "Kp7Nq2Wz9Bt4Xr6Vm1Ls"), + ("x_api_key: 8f3Kd9Lm2Qr7Ns3Vt", "8f3Kd9Lm2Qr7Ns3Vt"), + ("password: Zm9vYmFyYmF6+abc/def123=", "Zm9vYmFyYmF6+abc/def123="), + ("REDIS_PASSWORD=correcthorsebattery", "correcthorsebattery"), + ('SECRET_KEY = "django-insecure-9v2xk4qw8z"', "django-insecure-9v2xk4qw8z"), + ( + "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ), + ("password=aB3dE6gH9jK2", "aB3dE6gH9jK2"), + ("api_key: hunter2!brahms", "hunter2!brahms"), + ('db_password: "p@ssw0rd!2026"', "p@ssw0rd!2026"), + ( + 'url: "postgresql://user:s3cr3t@db-host:5432/app"', + "postgresql://user:s3cr3t@db-host:5432/app", + ), + ( + 'db_password: "postgresql://user:s3cr3t@db-host:5432/app"', + "postgresql://user:s3cr3t@db-host:5432/app", + ), + ( + 'signing_secret_url: "https://example.com/cb?sig=Zx4Kp9Lm2Qr7Ns3Vt"', + "https://example.com/cb?sig=Zx4Kp9Lm2Qr7Ns3Vt", + ), + ( + 'redis_secret_url: "redis://:Zx4Kp9Lm2Qr7Ns3Vt@cache-host:6379/0"', + "Zx4Kp9Lm2Qr7Ns3Vt", + ), + ("password=2026-09-08T17:38:40Zbrahms", "2026-09-08T17:38:40Zbrahms"), + ( + '{"password": "YOUR_API_KEY_HERE", "client_secret": "correcthorsebattery"}', + "correcthorsebattery", + ), + ("docker run -e REDIS_PASSWORD=aB3dE6gH9jK2mN5p \\\n -e REDIS_PORT=6379 redis", "aB3dE6gH9jK2mN5p"), + ("DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt && echo done", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("password = Zx4Kp9Lm2Qr7Ns3Vt # rotate me", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("my db password: Zx4Kp9Lm2Qr7Ns3Vt.", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("export DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt DB_HOST=db.internal", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt; systemctl restart app", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt | tee creds.txt", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt > setup.log", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("docker run -e DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt --name app postgres", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("password=correcthorsebattery please", "correcthorsebattery"), + ("DB_PASSWORD = Zx4Kp9Lm2Qr7Ns3Vt; systemctl restart app", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD = Zx4Kp9Lm2Qr7Ns3Vt \\", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD = Zx4Kp9Lm2Qr7Ns3Vt DB_HOST=db.internal", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD = Zx4Kp9Lm2Qr7Ns3Vt --db-host=db.internal", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("DB_PASSWORD = Zx4Kp9Lm2Qr7Ns3Vt DEBUG=", "Zx4Kp9Lm2Qr7Ns3Vt"), + ], + ids=[ + "env-password", + "env-secret", + "json-field", + "yaml-field", + "bare-assignment", + "client-secret", + "camel-case-key", + "camel-case-secret", + "camel-case-password", + "namespaced-env", + "underscored-header", + "base64-padding", + "digit-free-value", + "django-secret-key", + "slashed-aws-secret", + "shortest-accepted-value", + "punctuation-bearing-password", + "symbol-heavy-password", + "connection-string-under-a-url-key", + "connection-string-under-a-credential-key", + "signed-url-under-a-credential-key", + "password-only-url-under-a-credential-key", + "timestamp-prefixed-password", + "credential-after-a-rejected-placeholder", + "docker-flag-with-a-line-continuation", + "shell-command-after-the-value", + "inline-comment-after-the-value", + "sentence-ending-in-the-value", + "second-assignment-after-the-value", + "semicolon-after-the-value", + "pipe-after-the-value", + "redirect-after-the-value", + "docker-flag-after-the-value", + "prose-after-a-shell-assignment", + "spaced-assignment-then-a-shell-command", + "spaced-assignment-then-a-line-continuation", + "spaced-assignment-then-a-second-assignment", + "spaced-assignment-then-a-dashed-flag", + "spaced-assignment-then-an-empty-assignment", + ], +) +def test_scan_message_redacts_credentials_assigned_to_credential_keys(content, secret): + guardrail = _guardrail() + + assert secret not in guardrail.redact_text(content) + + +def test_scan_message_redacts_only_the_first_token_of_a_shell_assignment(): + guardrail = _guardrail() + content = "docker run -e REDIS_PASSWORD=aB3dE6gH9jK2mN5p \\\n -e REDIS_PORT=6379 redis && echo done" + + assert ( + guardrail.redact_text(content) + == "docker run -e REDIS_PASSWORD=[REDACTED] \\\n -e REDIS_PORT=6379 redis && echo done" + ) + + +@pytest.mark.parametrize("operator", [";", "&&", "|"]) +def test_scan_message_keeps_a_shell_operator_glued_to_the_value(operator): + guardrail = _guardrail() + + assert ( + guardrail.redact_text(f"DB_PASSWORD=Zx4Kp9Lm2Qr7Ns3Vt{operator} systemctl restart app") + == f"DB_PASSWORD=[REDACTED]{operator} systemctl restart app" + ) + + +def test_scan_message_closes_a_yaml_block_at_the_next_unindented_line(): + guardrail = _guardrail() + content = "api_key: >\n aB3dE6gH9jK2mN5p\nSteps\n Rotate-Before-Friday please" + + assert guardrail.redact_text(content) == "api_key: >\n [REDACTED]\nSteps\n Rotate-Before-Friday please" + + +def test_scan_message_redacts_every_credential_on_one_line(): + guardrail = _guardrail() + content = '{"db_password": "Tq8Zm2XpLv9KdNbRcYw3", "client_secret": "correcthorsebattery"}' + + assert guardrail.redact_text(content) == '{"db_password": "[REDACTED]", "client_secret": "[REDACTED]"}' + + +@pytest.mark.parametrize( + "content", + [ + "The user forgot their password and asked for a reset link", + "Rotate the client secret every 90 days", + "The secret: keep it quiet", + "My password: correct horse battery staple", + "secretary: Maria Gonzalez", + "password_reset_email: Please click the link below to reset", + 'config = {"api_key": "YOUR_API_KEY_HERE"}', + "api_key: ", + '{"max_tokens": 4096, "model": "gpt-4o-mini"}', + 'def get_api_key():\n return os.environ["OPENAI_API_KEY"]', + ' valid_token = UserAPIKeyAuth(user_id="u1")', + 'password = get_password(user, "prod")', + "monkey=aB3dE6gH9jK2mN5p", + "idempotency_key: req_2026090712000000", + 'cache_key = "u1_user_api_key_user_id"', + "the key: 2026-09-07T12:00:00Z", + "api_key: os.environ/E2B_API_KEY", + "langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET", + "api_key = OPENAI_API_KEY", + "password = pwd12345678", + "model_key: gpt-4o-mini-2024-07-18", + "openrouter/anthropic/claude-3-5-sonnet-20240620", + '{"content-type": "application/json"}', + "passwordless_login: enabled-for-all-users", + 'password: "I forgot mine, can you reset it"', + "secret_sauce: tomatoes-basil-garlic-oregano", + "user_secret_question: what-was-your-first-pet", + "password_reset_url: example.com/reset-password/flow", + "private_key_path: keys/prod/server-cert.pem", + "litellm.completion(model=model, api_key=openai_api_key)", + "params['aws_secret_access_key'] = aws_secret_access_key", + 'api_key = "OPENAI_API_KEY"', + "model_list:\n - litellm_params:\n api_key: 'PERPLEXITY_API_KEY'", + 'config = build(_provider("ve_missing", api_key_env="VE_MISSING_KEY"))', + "api_key = get_api_key_from_env()", + "api_key = get_secret_str(MISTRAL_OCR_API_KEY_ENV_VAR)", + "secret_manager = MagicMock(spec=BaseSecretManager)", + "api_key = self.resolve_server_api_key(", + "api_key = sys.argv[1]", + "password = credentials[environment]", + 'api_key_created_at: "2026-09-08T17:38:40Z"', + 'api_key_expires_at: "2026-09-08T17:38:40.123456+05:30"', + 'password_reset_url: "https://example.com/reset-password/flow"', + 'secret_docs_url: "https://example.com/reset-password/flow#step-2"', + '{"api_key_created_at": "2026-09-08T17:38:40Z", "password_reset_url": "https://example.com/reset/flow"}', + "secret_sauce: tomatoes-basil-garlic-oregano.", + "secret_docs_url: https://example.com/docs/keys, then rotate", + "api_key_created_at: 2026-09-08T17:38:40Z; api_key_env: OPENAI_API_KEY!", + "api_key: $OPENAI_API_KEY", + 'api_key: "${OPENAI_API_KEY}"', + "private_key_path: /keys/prod/server-cert.pem", + "password_hint: your usual one followed by Ticket-LIT7049-Suffix", + "Translate this recipe note into French:\nsecret_sauce: Worcestershire sauce", + "api_key = Massachusetts (the state, not a key)", + "secret_sauce:Worcestershire sauce", + "password: correctHorseBattery != anotherValue", + ], + ids=[ + "prose-password", + "prose-secret", + "colon-prose-secret", + "colon-prose-password", + "secretary", + "sentence-after-keyword", + "uppercase-placeholder", + "templated-placeholder", + "max-tokens", + "code-paste", + "constructor-call", + "indirect-reference", + "word-ending-in-key", + "idempotency-key", + "cache-key", + "timestamp-after-key", + "env-reference", + "env-reference-nested", + "env-variable-name", + "below-minimum-length", + "model-name", + "namespaced-model-name", + "media-type", + "hyphenated-english", + "quoted-sentence-under-a-credential-key", + "hyphenated-phrase", + "hyphenated-question", + "url-under-credential-key", + "path-under-credential-key", + "snake-case-argument", + "snake-case-assignment", + "quoted-env-variable-name", + "quoted-env-name-in-a-config", + "quoted-env-name-in-a-code-paste", + "bare-call", + "call-with-an-argument", + "keyword-argument-call", + "unclosed-call", + "positional-subscript", + "keyed-subscript", + "timestamp-under-a-credential-key", + "offset-timestamp-under-a-credential-key", + "url-under-a-credential-key", + "fragment-url-under-a-credential-key", + "metadata-object-under-credential-keys", + "hyphenated-english-ending-a-sentence", + "url-followed-by-a-clause", + "timestamp-and-env-name-with-trailing-punctuation", + "shell-variable-reference", + "quoted-braced-shell-variable-reference", + "absolute-path-under-a-credential-key", + "sentence-holding-a-later-mixed-case-token", + "capitalized-word-starting-a-phrase", + "capitalized-word-before-a-parenthetical", + "yaml-scalar-without-a-space-after-the-colon", + "comparison-operator-after-the-value", + ], +) +def test_scan_message_keeps_benign_values(content): + guardrail = _guardrail() + + assert guardrail.scan_message_for_secrets(content) == [] + assert guardrail.redact_text(content) == content + + +@pytest.mark.parametrize( + "value,redacted", + [("aB3dE6gH9jK2", True), ("aB3dE6gH9jK", False)], + ids=["at-minimum-length", "below-minimum-length"], +) +def test_credential_keyword_detector_honours_its_minimum_length(value, redacted): + guardrail = _guardrail() + + assert (value not in guardrail.redact_text(f"password={value}")) is redacted + + +@pytest.mark.parametrize( + "value,redacted", + [("aB3dE6gH9jK2", True), ("aB3dE6gH9jK", False)], + ids=["at-default-minimum-length", "below-default-minimum-length"], +) +def test_credential_keyword_detector_defaults_its_minimum_length(value, redacted): + guardrail = _ENTERPRISE_SecretDetection( + guardrail_name="hide-secrets", + event_hook="pre_call", + default_on=True, + detect_secrets_config={ + "plugins_used": [ + {key: setting for key, setting in plugin.items() if key != "minimum_length"} + for plugin in _default_detect_secrets_config["plugins_used"] + ] + }, + ) + + assert (value not in guardrail.redact_text(f"password={value}")) is redacted + + +def test_credential_keyword_detector_honours_keyword_exclude(): + guardrail = _ENTERPRISE_SecretDetection( + guardrail_name="hide-secrets", + event_hook="pre_call", + default_on=True, + detect_secrets_config={ + "plugins_used": [ + {**plugin, "keyword_exclude": "fixture_"} if plugin["name"] == "CredentialKeywordDetector" else plugin + for plugin in _default_detect_secrets_config["plugins_used"] + ] + }, + ) + content = "fixture_password=aB3dE6gH9jK2mN5p\npassword=Kp7Nq2Wz9Bt4Xr6Vm1Ls" + + assert guardrail.redact_text(content) == "fixture_password=aB3dE6gH9jK2mN5p\npassword=[REDACTED]" + + +@pytest.mark.parametrize("minimum_length", ["12", 0, -1, 1.5], ids=["string", "zero", "negative", "float"]) +def test_credential_keyword_detector_rejects_an_unusable_minimum_length(minimum_length): + guardrail = _ENTERPRISE_SecretDetection( + guardrail_name="hide-secrets", + event_hook="pre_call", + default_on=True, + detect_secrets_config={ + "plugins_used": [ + {**plugin, "minimum_length": minimum_length} + if plugin["name"] == "CredentialKeywordDetector" + else plugin + for plugin in _default_detect_secrets_config["plugins_used"] + ] + }, + ) + + with pytest.raises(ValueError, match="minimum_length"): + guardrail.scan_message_for_secrets("password=aB3dE6gH9jK2mN5p") + + +@pytest.mark.parametrize( + "content", + [ + "[db\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "[\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "[note] have a look\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "]\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "[]\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + ], + ids=["unclosed", "bare-bracket", "bracketed-prose", "stray-close", "empty-header"], +) +def test_scan_message_reads_a_config_with_a_broken_section_header(content): + guardrail = _guardrail() + + assert "Zx4Kp9Lm2Qr7Ns3Vt" not in guardrail.redact_text(content) + + +@pytest.mark.parametrize( + "content", + [ + "=orphan\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + " indented before any key\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "greeting = %(name)s\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + "token = a\x00b\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n", + ], + ids=["empty-key", "leading-continuation", "interpolation", "nul-byte"], +) +def test_scan_message_reads_lines_that_a_stock_ini_parser_rejects(content): + guardrail = _guardrail() + + assert "Zx4Kp9Lm2Qr7Ns3Vt" not in guardrail.redact_text(content) + + +def test_scan_message_reads_a_config_that_repeats_a_section(): + guardrail = _guardrail() + content = "[db]\nhost = localhost\n[db]\npassword = Zx4Kp9Lm2Qr7Ns3Vt\n" + + assert "Zx4Kp9Lm2Qr7Ns3Vt" not in guardrail.redact_text(content) + + +def test_scan_message_keeps_every_value_when_a_config_repeats_a_key(): + guardrail = _guardrail() + content = ( + "model_list:\n" + " - model_name: gpt-4o\n litellm_params:\n api_key: aB3dE6gH9jK2mN5p\n" + " - model_name: claude\n litellm_params:\n api_key: Kp7Nq2Wz9Bt4Xr6Vm1Ls\n" + ) + + redacted = guardrail.redact_text(content) + + assert "aB3dE6gH9jK2mN5p" not in redacted + assert "Kp7Nq2Wz9Bt4Xr6Vm1Ls" not in redacted + + +@pytest.mark.parametrize( + "content,secret", + [ + ( + f"api_key: {OPENAI_KEY}\nREDIS_PASSWORD=aB3dE6gH9jK2mN5p", + "aB3dE6gH9jK2mN5p", + ), + ( + f"OPENAI_API_KEY={OPENAI_KEY}\nDB_PASSWORD=Kp7Nq2Wz9Bt4Xr6Vm1Ls", + "Kp7Nq2Wz9Bt4Xr6Vm1Ls", + ), + ( + f"api_key: {OPENAI_KEY}\npassword =\n Zx4Kp9Lm2Qr7Ns3Vt", + "Zx4Kp9Lm2Qr7Ns3Vt", + ), + ( + "Here is my config, can you review it?\nREDIS_PASSWORD=aB3dE6gH9jK2mN5p", + "aB3dE6gH9jK2mN5p", + ), + ( + "REDIS_PASSWORD=aB3dE6gH9jK2mN5p\nCan you tell me what is wrong with it?", + "aB3dE6gH9jK2mN5p", + ), + ( + "Hi team\nplease rotate this before Friday\ndb_password=Zx4Kp9Lm2Qr7Ns3Vt\nthanks!", + "Zx4Kp9Lm2Qr7Ns3Vt", + ), + ( + "model_list:\n - model_name: gpt-4o\n litellm_params:\n api_key: aB3dE6gH9jK2mN5p\n", + "aB3dE6gH9jK2mN5p", + ), + ("api_key: >\n aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + ("api_key: |-\n aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + ("secret= \\\n aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + ("password =\n# rotate me\n Zx4Kp9Lm2Qr7Ns3Vt", "Zx4Kp9Lm2Qr7Ns3Vt"), + ("api_key =\n; rotate me\n aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + (" # pasted from the vault\napi_key=aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + (" [db]\napi_key=aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + (" pasted with a leading indent\napi_key=aB3dE6gH9jK2mN5p", "aB3dE6gH9jK2mN5p"), + ], + ids=[ + "flat-assignment", + "env-file", + "continuation-line", + "prose-before", + "prose-after", + "prose-both-sides", + "indented-config", + "yaml-folded-block", + "yaml-literal-block", + "backslash-continuation", + "comment-inside-a-value", + "semicolon-comment-inside-a-value", + "indented-comment-above", + "indented-section-header-above", + "indented-prose-above", + ], +) +def test_scan_message_still_sees_assignments_sharing_a_message_with_a_vendor_key(content, secret): + guardrail = _guardrail() + + redacted = guardrail.redact_text(content) + assert secret not in redacted + assert OPENAI_KEY not in redacted + + +def test_environment_reference_filter_only_drops_the_whole_value(): + guardrail = _guardrail() + + for reference in ("os.environ/OPENAI_API_KEY", "os.environ/e2b_api_key"): + assert guardrail.redact_text(f"password={reference}") == f"password={reference}" + assert guardrail.redact_text("password=notos.environ/OPENAI_API_KEY") == ("password=[REDACTED]") + + +def test_environment_variable_names_are_dropped_only_for_the_keyword_plugin(): + guardrail = _guardrail() + + assert guardrail.redact_text("password=REDIS_PASSWORD") == "password=REDIS_PASSWORD" + assert guardrail.scan_message_for_secrets('k = "ABCD1234_EFGH5678_IJKLMN"') == [ + {"type": "Base64 High Entropy String", "value": "ABCD1234_EFGH5678_IJKLMN"} + ] + + +def test_masked_entity_count_keeps_the_vendor_type_beside_the_entropy_type(): + guardrail = _guardrail() + _masked_entity_count.set({}) + + guardrail.redact_text('k = "ghp_abcdefghijklmnopqrstuvwxyzABCDEF1234"') + + assert _masked_entity_count.get() == { + "Base64 High Entropy String": 1, + "GitHub Token": 1, + } + + +@pytest.mark.parametrize( + "content", + [ + f"api_key: '{OPENAI_KEY}'\n" + + "a: &a [" + + ", ".join(['"x"'] * 9) + + "]\n" + + "".join(f"{chr(98 + i)}: &{chr(98 + i)} [" + ", ".join([f"*{chr(97 + i)}"] * 9) + "]\n" for i in range(7)), + f"api_key: '{OPENAI_KEY}'\ndeep: " + "[" * 400 + "]" * 400, + f"api_key: '{OPENAI_KEY}'\nbroken: [unclosed", + ], + ids=["anchor-expansion", "deep-nesting", "unparseable"], +) +def test_scan_message_contains_hostile_config_text(content, monkeypatch, tmp_path): + guardrail = _guardrail() + monkeypatch.setenv("TMPDIR", str(tmp_path)) + monkeypatch.setattr(tempfile, "tempdir", None) + + started = time.perf_counter() + found = guardrail.scan_message_for_secrets(content) + + assert time.perf_counter() - started < 10.0 + assert OPENAI_KEY in [secret["value"] for secret in found] + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize( + "content", + [ + f"api_key = '{OPENAI_KEY}'\nbase = abcdefghijkl\npassword = x\n %(base)sZZZZQQQQ\n", + "base = abcdefghijkl\npassword = x\n %(base)sZZZZQQQQ\n", + f"api_key = '{OPENAI_KEY}'\nbase = Kp7Nq2Wz9Bt4\npassword = x\n" + " %(base)s-primary\nnote = Kp7Nq2Wz9Bt4-primary is the hostname\n", + 'base = "abcdefghijkl"\npassword = "%(base)sZZZZQQQQ"\n', + ], + ids=[ + "vendor-key-present", + "no-vendor-key", + "value-echoed-elsewhere", + "quoted-interpolation", + ], +) +def test_scan_message_never_reports_a_value_the_message_does_not_hold(content): + guardrail = _guardrail() + + for secret in guardrail.scan_message_for_secrets(content): + assert secret["value"] in content + + +def test_scan_message_leaves_unrelated_text_alone_when_a_value_is_echoed(): + guardrail = _guardrail() + content = ( + f"api_key = '{OPENAI_KEY}'\nbase = Kp7Nq2Wz9Bt4\npassword = x\n" + " %(base)s-primary\nnote = Kp7Nq2Wz9Bt4-primary is the hostname\n" + ) + + assert "note = Kp7Nq2Wz9Bt4-primary is the hostname" in guardrail.redact_text(content) + + +def test_masked_entity_count_counts_each_secret_once(): + guardrail = _guardrail() + _masked_entity_count.set({}) + + guardrail.redact_text(f"first {OPENAI_KEY} second {OPENAI_KEY}") + + assert _masked_entity_count.get() == {"Strict OpenAI API Key": 1} + + def test_scan_message_redacts_every_openai_key_occurrence(): guardrail = _guardrail() content = f"first {OPENAI_KEY}, second {OPENAI_KEY}" @@ -81,9 +646,7 @@ def test_scan_message_requires_ascii_digits_for_openai_like_values(): def test_scan_message_redacts_openai_key_after_separator(): guardrail = _guardrail() - assert guardrail.redact_text(f"openai_{OPENAI_KEY} key-{OPENAI_KEY}") == ( - "openai_[REDACTED] key-[REDACTED]" - ) + assert guardrail.redact_text(f"openai_{OPENAI_KEY} key-{OPENAI_KEY}") == ("openai_[REDACTED] key-[REDACTED]") assert guardrail.redact_text(URL_ENCODED_KEY) == "Bearer%20[REDACTED]" @@ -102,6 +665,31 @@ def test_scan_message_stays_linear_on_repeated_sk_separators(): assert time.perf_counter() - started < 2.0 +@pytest.mark.parametrize( + "content", + [ + f"api_key: '{OPENAI_KEY}'\npassword=" + "a-" * 10_000 + "!", + f"api_key: '{OPENAI_KEY}'\npassword:" + '"' * 20_000, + f"api_key: '{OPENAI_KEY}'\n" + "api_key:" * 10_000, + f"api_key: '{OPENAI_KEY}'\nsecret=" + "aB3dE6gH9jK2mN5p " * 2_000, + f"api_key: '{OPENAI_KEY}'\n" + "\n".join(f"password{i}=aB3dE6gH9jK2mN5p{i}" for i in range(3_000)), + ], + ids=[ + "value-run", + "quote-run", + "keyword-run", + "value-repeat", + "assignment-flood", + ], +) +def test_scan_message_stays_linear_on_adversarial_credential_lines(content): + guardrail = _guardrail() + + started = time.perf_counter() + guardrail.redact_text(content) + assert time.perf_counter() - started < 10.0 + + def test_scan_message_redacts_whole_stripe_live_key(): guardrail = _guardrail() @@ -119,8 +707,8 @@ def test_scan_message_replaces_longest_overlapping_match_first(): guardrail = _guardrail() content = f'token = "{OPENAI_KEY}/extra"' - detected = guardrail.scan_message_for_secrets(content) - assert [secret["value"] for secret in detected] == [f"{OPENAI_KEY}/extra", OPENAI_KEY] + values = [secret["value"] for secret in guardrail.scan_message_for_secrets(content)] + assert values == [f"{OPENAI_KEY}/extra", OPENAI_KEY] assert guardrail.redact_text(content) == 'token = "[REDACTED]"' diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 3283af26cf7..f72316f5d5e 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -1704,8 +1704,9 @@ async def test_empty_http_event_stream_uses_the_existing_request_deadline() -> N "initialize_not_found", ), ) +@pytest.mark.parametrize("raise_on_error", (False, True)) async def test_optional_discovery_capabilities_and_errors( - method: str, outcome: str, caplog: pytest.LogCaptureFixture + method: str, outcome: str, caplog: pytest.LogCaptureFixture, raise_on_error: bool ) -> None: import logging from unittest.mock import Mock @@ -1783,7 +1784,11 @@ async def test_optional_discovery_capabilities_and_errors( "resources/list": client.list_resources, "resources/templates/list": client.list_resource_templates, }[method] - result: Final = await operation() + if raise_on_error and outcome in ("internal_error", "unauthorized", "timeout", "initialize_not_found"): + with pytest.raises((McpError, httpx.HTTPError)): + await operation(raise_on_error=True) + return + result: Final = await operation(raise_on_error=raise_on_error) requests: Final = tuple( JSONRPCMessage.model_validate_json(call.args[0].content).root @@ -1907,3 +1912,25 @@ def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "MCPServerManager" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("resolved", (False, True)) +async def test_discovery_auth_fingerprint_tracks_effective_credentials(resolved: bool) -> None: + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + + def client(token: str) -> MCPClient: + return MCPClient( + server_url="https://example.com/mcp", + auth_type=MCPAuth.api_key, + auth_value=None if resolved else token, + resolved_auth=StaticHeaderAuth(token) if resolved else None, + ) + + original: Final = await client("private-original-credential").discovery_auth_fingerprint() + repeated: Final = await client("private-original-credential").discovery_auth_fingerprint() + replaced: Final = await client("private-replaced-credential").discovery_auth_fingerprint() + assert original == repeated + assert original != replaced + assert len(original) == 64 + assert "private-original-credential" not in original diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index 55e2dcdc270..44dda57dd27 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -45,6 +45,21 @@ class TestSlackAlerting(unittest.TestCase): result = self.slack_alerting._get_percent_of_max_budget_left(user_info) self.assertEqual(result, -0.2) + def test_get_user_info_str_omits_absent_token_for_user_alert(self): + user_info = CallInfo( + spend=85.0, + max_budget=100.0, + user_id="user-1", + user_email="person@example.com", + event_group=Litellm_EntityType.USER, + ) + + result = self.slack_alerting._get_user_info_str(user_info) + + self.assertIn("*user_id:* `user-1`", result) + self.assertIn("*user_email:* `person@example.com`", result) + self.assertNotIn("*token:*", result) + def test_get_event_and_event_message_max_budget(self): # Initial setup with no event event = None diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py index 8f93a9a564f..8db84b090a0 100644 --- a/tests/test_litellm/integrations/otel/test_langfuse_logger.py +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -1,9 +1,9 @@ -"""Tests for ``LangfuseOpenTelemetryV2``: the root observation's input and output are stamped from the -request-task hooks, while the root span is still recording, so Langfuse can show them on the trace.""" +"""Tests for the Langfuse OTel v2 loggers: the trace name and the root observation's input and output are +stamped from the request task while the root span is still recording, so Langfuse can show them on the trace.""" import asyncio import json -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence from typing import Final import pytest @@ -14,7 +14,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanE import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 -from litellm.integrations.otel.logger import build_otel_v2_logger # noqa: E402 +from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 @@ -41,6 +41,7 @@ from litellm.types.utils import ( # noqa: E402 INPUT_ATTR: Final = "langfuse.observation.input" OUTPUT_ATTR: Final = "langfuse.observation.output" +TRACE_NAME_ATTR: Final = "langfuse.trace.name" CHAT_DATA: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "ping"}]} @@ -306,6 +307,73 @@ def test_unrenderable_output_never_raises_into_the_request(): assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs +def _run_named_request( + logger: OpenTelemetryV2, exporter: InMemorySpanExporter, litellm_params: Mapping[str, object] +) -> tuple[Mapping[str, object], Mapping[str, object]]: + response: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root: Final = _start_root(logger) + logger.log_pre_api_call( + model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "call_1", "litellm_params": litellm_params} + ) + root.end() + payload: Final = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-mini", + "messages": CHAT_DATA["messages"], + "response": response.model_dump(), + "status": "success", + "litellm_call_id": "call_1", + "metadata": {}, + "hidden_params": {}, + } + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": payload, "litellm_params": litellm_params}, response, None, None + ) + ) + generation: Final = next( + span for span in exporter.get_finished_spans() if span.name != LITELLM_PROXY_REQUEST_SPAN_NAME + ) + return _root_attrs(exporter), dict(generation.attributes or {}) + + +@pytest.mark.parametrize("capture", ["span_only", "no_content"]) +def test_langfuse_trace_name_header_names_the_root_and_the_generation_over_body_metadata(capture): + logger, exporter = _logger(capture=capture) + + root_attrs, generation_attrs = _run_named_request( + logger, + exporter, + { + "metadata": {"trace_name": "from-body"}, + "proxy_server_request": {"headers": {"langfuse_trace_name": "from-header"}}, + }, + ) + + assert root_attrs[TRACE_NAME_ATTR] == "from-header" + assert generation_attrs[TRACE_NAME_ATTR] == "from-header" + + +def test_body_metadata_trace_name_names_the_root_and_the_generation(): + logger, exporter = _logger() + + root_attrs, generation_attrs = _run_named_request( + logger, exporter, {"metadata": {"trace_name": "from-body"}, "proxy_server_request": {"headers": {}}} + ) + + assert root_attrs[TRACE_NAME_ATTR] == "from-body" + assert generation_attrs[TRACE_NAME_ATTR] == "from-body" + + +def test_unnamed_request_leaves_the_trace_name_off_both_spans(): + logger, exporter = _logger() + + root_attrs, generation_attrs = _run_named_request(logger, exporter, {"proxy_server_request": {"headers": {}}}) + + assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs + + @pytest.mark.parametrize( ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index addadf8e598..8baf9310538 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -28,6 +28,7 @@ from litellm.integrations.otel import ( ) from litellm.integrations.otel.mappers.genai import GenAIMapper from litellm.integrations.otel.model import spans as spans_mod +from litellm.integrations.otel.model.metadata import LLMCallEvent, caller_trace_name from litellm.integrations.otel.model.payloads import ( LLMCallSpanData, RequestIdentity, @@ -722,6 +723,37 @@ def test_request_identity_falls_back_to_legacy_team_keys(): assert ident.team_alias == "legacy" +@pytest.mark.parametrize( + ("request_data", "expected"), + [ + ({"proxy_server_request": {"headers": {"langfuse_trace_name": "from-header"}}}, "from-header"), + ({"metadata": {"trace_name": "from-body"}}, "from-body"), + ({"litellm_metadata": {"trace_name": "from-anthropic-body"}}, "from-anthropic-body"), + ( + { + "proxy_server_request": {"headers": {"langfuse_trace_name": "from-header"}}, + "metadata": {"trace_name": "from-body"}, + }, + "from-header", + ), + ({"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, "body"), + ({"proxy_server_request": {"headers": {}}, "metadata": {"user_api_key_team_id": "t1"}}, None), + ({}, None), + ], + ids=["header", "body", "anthropic-body", "header-beats-body", "blank-header-falls-through", "neither", "empty"], +) +def test_caller_trace_name_prefers_the_langfuse_header_over_body_metadata(request_data, expected): + assert caller_trace_name({"litellm_params": request_data}) == expected + assert LLMCallEvent.from_dict({"litellm_params": request_data}).trace_name == expected + + +def test_llm_span_data_carries_the_caller_trace_name(): + data: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload(), trace_name="nightly-eval") + + assert data.trace_name == "nightly-eval" + assert LLMCallSpanData.from_standard_logging_payload(_sample_payload()).trace_name is None + + def test_llm_span_carries_proxy_request_route(): """The LLM span records the proxy route the request arrived on, so it can be filtered by endpoint (``/v1/responses`` vs ``/v1/chat/completions``) without diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 94cb79f53b8..bcdda93383a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -134,6 +134,11 @@ def test_langfuse_mapper_observation_attrs(): assert attrs["langfuse.trace.metadata.team_id"] == "t1" +def test_langfuse_mapper_names_the_trace_from_the_caller(): + assert LangfuseMapper().map(_llm_call(trace_name="nightly-eval"))["langfuse.trace.name"] == "nightly-eval" + assert "langfuse.trace.name" not in LangfuseMapper().map(_llm_call(trace_name=None)) + + def test_langfuse_mapper_skips_when_no_messages(): data = _llm_call(messages_in=(), choices_out=()) attrs = LangfuseMapper().map(data) diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index ddc8439a83a..bb4822eae57 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -1,4 +1,5 @@ import asyncio +import datetime as dt from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional from unittest.mock import AsyncMock @@ -9,9 +10,16 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy._types import CallTypes, UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks, Mode -from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailTracingDetail +from litellm.types.utils import ( + Choices, + GenericGuardrailAPIInputs, + GuardrailTracingDetail, + Message, + ModelResponse, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -2345,6 +2353,57 @@ class TestUndecoratedApplyGuardrailIsLogged: assert _Labelled.seen_label == "docs-style" + @pytest.mark.asyncio + async def test_post_call_recorded_outside_decorator_reaches_standard_logging_object(self): + """LIT-7608 regression: the auto-wrapped pre_call apply_guardrail copies the request bucket + into logging_obj.litellm_params["metadata"]. A post_call entry recorded later without the + decorator (the Bedrock streaming hook) must not be shadowed by that stale copy.""" + messages: Final = [{"role": "user", "content": "hello there"}] + litellm_metadata: Final[dict] = {"user_api_key_user_id": "u1"} + logging_obj: Final = Logging( + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + messages=messages, + stream=True, + call_type=CallTypes.acompletion.value, + start_time=dt.datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + ) + logging_obj.update_environment_variables( + litellm_params={"litellm_metadata": litellm_metadata}, + optional_params={}, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + custom_llm_provider="bedrock", + ) + request_data: Final = { + "model": "bedrock-haiku", + "messages": messages, + "litellm_metadata": litellm_metadata, + "litellm_logging_obj": logging_obj, + } + guardrail: Final = _UndecoratedGuardrail(guardrail_name="bedrock-pre", event_hook=GuardrailEventHooks.pre_call) + + await guardrail.apply_guardrail( + inputs=GenericGuardrailAPIInputs(texts=["hello there"]), + request_data=request_data, + input_type="request", + logging_obj=logging_obj, + ) + guardrail.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response={"action": "NONE"}, + request_data=request_data, + guardrail_status="success", + event_type=GuardrailEventHooks.post_call, + ) + await logging_obj.async_success_handler( + result=ModelResponse(choices=[Choices(message=Message(role="assistant", content="general kenobi"))]), + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + ) + + entries: Final = logging_obj.model_call_details["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_mode"] for e in entries] == ["pre_call", "post_call"] + class _ApplyOnlyObserver(CustomGuardrail): """Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook.""" @@ -2829,3 +2888,131 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook: assert response.choices[0].message.content == "filtered response" assert "guardrail_to_apply" not in request_data assert len(_guardrail_entries(request_data)) == 1 + + +class TestPreCallHookResponseIsNotLoggedVerbatim: + """Regression for LIT-6935: a pre_call hook returning the request payload leaked the prompt + into ``guardrail_response`` and from there onto OTEL guardrail spans.""" + + @staticmethod + def _logged_response(request_data: dict[str, object]) -> object: + metadata = request_data["litellm_metadata"] + assert isinstance(metadata, dict) + entries = metadata["standard_logging_guardrail_information"] + assert len(entries) == 1 + return entries[0]["guardrail_response"] + + @staticmethod + def _request() -> dict[str, object]: + return { + "model": "gpt-4.1-mini", + "input": "SECRET_PROMPT", + "messages": [{"role": "user", "content": "SECRET_PROMPT"}], + "litellm_metadata": {}, + } + + @pytest.mark.asyncio + async def test_pre_call_hook_returning_request_logs_allow(self): + class PassthroughGuardrail(CustomGuardrail): + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: object, + data: dict[str, object], + call_type: str, + ) -> dict[str, object]: + return data + + data = self._request() + await PassthroughGuardrail(guardrail_name="g").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), cache=None, data=data, call_type="aresponses" + ) + + assert self._logged_response(data) == "allow" + + @pytest.mark.asyncio + async def test_pre_call_hook_returning_modified_copy_logs_mask(self): + class MaskingGuardrail(CustomGuardrail): + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: object, + data: dict[str, object], + call_type: str, + ) -> dict[str, object]: + return {**data, "input": "[MASKED]"} + + data = self._request() + await MaskingGuardrail(guardrail_name="g").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), cache=None, data=data, call_type="aresponses" + ) + + assert self._logged_response(data) == "mask" + + @pytest.mark.asyncio + async def test_pre_call_hook_mutating_request_in_place_logs_mask(self): + class InPlaceMaskingGuardrail(CustomGuardrail): + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: object, + data: dict[str, object], + call_type: str, + ) -> dict[str, object]: + messages = data["messages"] + assert isinstance(messages, list) + messages[0]["content"] = "[MASKED]" + return data + + data = self._request() + await InPlaceMaskingGuardrail(guardrail_name="g").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), cache=None, data=data, call_type="acompletion" + ) + + assert self._logged_response(data) == "mask" + + @pytest.mark.asyncio + async def test_pre_call_hook_returning_rejection_string_logs_that_string(self): + class RejectingGuardrail(CustomGuardrail): + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: object, + data: dict[str, object], + call_type: str, + ) -> str: + return "Blocked by policy" + + data = self._request() + result = await RejectingGuardrail(guardrail_name="g").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), cache=None, data=data, call_type="acompletion" + ) + + assert result == "Blocked by policy" + assert self._logged_response(data) == "Blocked by policy" + + @pytest.mark.asyncio + async def test_pre_call_hook_removing_legacy_functions_in_place_logs_mask(self): + class FunctionStrippingGuardrail(CustomGuardrail): + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: object, + data: dict[str, object], + call_type: str, + ) -> dict[str, object]: + data["functions"] = [] + data["function_call"] = "none" + return data + + data = {**self._request(), "functions": [{"name": "delete_db"}], "function_call": "auto"} + await FunctionStrippingGuardrail(guardrail_name="g").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), cache=None, data=data, call_type="acompletion" + ) + + assert self._logged_response(data) == "mask" diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index eecd876219e..76efe9c8576 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -2,6 +2,7 @@ the detached pipeline's single attempt-row write, and the cache-first job lookup.""" import asyncio +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -421,6 +422,183 @@ class TestSurfaceNormalization: assert "previous_response_id" not in shadow_call assert "instructions" not in shadow_call + @pytest.mark.parametrize( + "call_type,search_params,model", + [ + ("completion", {"web_search_options": {}}, "anthropic/claude-fable-5"), + ( + "acompletion", + {"web_search_options": {"search_context_size": "high"}}, + "anthropic/claude-fable-5", + ), + ( + "acompletion", + {"tools": [{"type": "web_search_20260209", "name": "web_search"}]}, + "anthropic/claude-fable-5", + ), + ( + "anthropic_messages", + {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, + "anthropic/claude-fable-5", + ), + ( + "anthropic_messages", + {"tools": [{"type": "web_search_20260209", "name": "web_search"}]}, + "anthropic/claude-fable-5", + ), + ( + "anthropic_messages", + {"tools": [{"name": "web_search"}]}, + "anthropic/claude-fable-5", + ), + ("aresponses", {"tools": [{"type": "web_search"}]}, "anthropic/claude-fable-5"), + ("responses", {"tools": [{"type": "web_search_preview"}]}, "anthropic/claude-fable-5"), + ("aresponses", {"tools": [{"type": "web_search_2025_08_26"}]}, "anthropic/claude-fable-5"), + ( + "responses", + {"tools": [{"type": "web_search_preview_2025_03_11"}]}, + "anthropic/claude-fable-5", + ), + ("aresponses", {"tools": [{"type": "web_search"}]}, "bedrock/us.anthropic.claude-fable-5"), + ( + "responses", + {"tools": [{"type": "web_search_preview"}]}, + "bedrock/us.anthropic.claude-fable-5", + ), + ( + "acompletion", + { + "tools": [ + {"type": "function", "function": {"name": "WebSearch", "parameters": {"type": "object"}}}, + {"type": "web_search_20260209", "name": "web_search"}, + ] + }, + "anthropic/claude-fable-5", + ), + ], + ids=[ + "chat-empty-options", + "chat-configured-options", + "chat-provider-transformed-tools", + "messages-native-search", + "messages-dated-search", + "messages-legacy-search-normalized", + "responses-search", + "responses-preview", + "responses-dated-search", + "responses-dated-preview", + "responses-bedrock-erases-search", + "responses-bedrock-erases-preview", + "chat-mixed-client-and-hosted-tools", + ], + ) + async def test_hosted_web_search_skips_shadow_calls_and_spend( + self, call_type: str, search_params: Mapping[str, object], model: str + ) -> None: + base_kwargs: Final = _success_kwargs(call_type=call_type, model=model) + is_chat: Final = call_type in ("completion", "acompletion") + is_responses: Final = call_type in ("responses", "aresponses") + hook_kwargs: Final = { + **base_kwargs, + "model": model, + "messages": "what is new" if is_responses else base_kwargs["messages"], + "standard_logging_object": { + **base_kwargs["standard_logging_object"], + "model_parameters": search_params if is_chat else {}, + }, + "litellm_params": { + **base_kwargs["litellm_params"], + "proxy_server_request": {"body": {} if is_chat else search_params}, + }, + } + prisma: Final = _prisma() + router: Final = _router() + counter: Final = {"spend:shadow_eval:job-1": 0.1, "spend:shadow_eval:job-2": 0.1} + logger: Final = _logger( + router=router, + prisma=prisma, + jobs=(_job(max_budget=0.2), _job(id="job-2", max_budget=0.2)), + counter_store=counter, + ) + + await logger.async_log_success_event( + hook_kwargs, RESPONSES_API_RESPONSE if is_responses else RESPONSE, None, None + ) + await _drain(logger) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + assert logger._test_funnel == [("job-1", "unjudgeable"), ("job-2", "unjudgeable")] + assert logger._job_starts == {} + assert logger._test_counter == {"spend:shadow_eval:job-1": 0.1, "spend:shadow_eval:job-2": 0.1} + + @pytest.mark.parametrize( + "call_type,tool_name", + [ + (call_type, tool_name) + for call_type in ("completion", "acompletion", "anthropic_messages", "responses", "aresponses") + for tool_name in ("WebSearch", "litellm_web_search", "web_search") + ], + ) + async def test_client_web_search_tools_remain_sampled(self, call_type: str, tool_name: str) -> None: + is_chat: Final = call_type in ("completion", "acompletion") + is_responses: Final = call_type in ("responses", "aresponses") + tool: Final = ( + {"type": "function", "function": {"name": tool_name, "parameters": {"type": "object"}}} + if is_chat + else {"type": "function", "name": tool_name, "parameters": {"type": "object"}} + if is_responses + else {"name": tool_name, "input_schema": {"type": "object", "properties": {}}} + ) + source: Final = {"tools": [tool], "web_search_options": None} + base_kwargs: Final = _success_kwargs(call_type=call_type) + hook_kwargs: Final = { + **base_kwargs, + "messages": "search for current news" if is_responses else base_kwargs["messages"], + "standard_logging_object": { + **base_kwargs["standard_logging_object"], + "model_parameters": source if is_chat else {}, + }, + "litellm_params": { + **base_kwargs["litellm_params"], + "proxy_server_request": {"body": {} if is_chat else source}, + }, + } + + prisma, router = await self._drive(hook_kwargs, RESPONSES_API_RESPONSE if is_responses else RESPONSE) + + assert router.acompletion.call_count == 2 + shadow_call: Final = router.acompletion.call_args_list[0].kwargs + assert shadow_call["tools"][0]["function"]["name"] == tool_name + assert "web_search_options" not in shadow_call + prisma.db.litellm_shadowevalattempt.create.assert_called_once() + + @pytest.mark.parametrize("call_type", ["completion", "acompletion"]) + async def test_chat_search_removed_by_guardrail_still_samples(self, call_type: str) -> None: + base_kwargs: Final = _success_kwargs( + call_type=call_type, + request_metadata={ + "standard_logging_guardrail_information": [{"guardrail_name": "g", "guardrail_mode": "pre_call"}] + }, + ) + hook_kwargs: Final = { + **base_kwargs, + "litellm_params": { + **base_kwargs["litellm_params"], + "proxy_server_request": { + "body": {"web_search_options": {}, "tools": [{"type": "web_search_20260209"}]} + }, + }, + } + + prisma, router = await self._drive(hook_kwargs, RESPONSE) + + shadow_call: Final = router.acompletion.call_args_list[0].kwargs + assert "web_search_options" not in shadow_call + assert "tools" not in shadow_call + assert router.acompletion.call_count == 2 + prisma.db.litellm_shadowevalattempt.create.assert_called_once() + @pytest.mark.parametrize("payload_shape", ["typed", "dict"]) @pytest.mark.parametrize("call_type", ["aresponses", "responses"]) async def test_responses_arms_normalize_bare_string_input_and_instructions(self, call_type, payload_shape): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index f6ee1cd71c0..03b9840b1c3 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3358,6 +3358,27 @@ def test_is_web_search_tool(): assert adapter._is_web_search_tool(regular_tool) is False +@pytest.mark.parametrize("schema", [{}, {"type": "object", "properties": {"query": {"type": "string"}}}]) +def test_translate_anthropic_client_web_search_preserves_schema_and_choice(schema: dict[str, object]) -> None: + from litellm.types.llms.anthropic import AnthropicMessagesRequest + + request: Final = AnthropicMessagesRequest( + model="gpt-5.4-mini", + max_tokens=128, + messages=[{"role": "user", "content": "Search for current news"}], + tools=[{"name": "web_search", "input_schema": schema}], + tool_choice={"type": "tool", "name": "web_search"}, + ) + + translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(request) + + assert "web_search_options" not in translated + assert translated["tools"] == [ + {"type": "function", "function": {"name": "web_search", "parameters": schema}} + ] + assert translated["tool_choice"] == {"type": "function", "function": {"name": "web_search"}} + + def test_translate_anthropic_to_openai_with_web_search_tool(): """ Test that Anthropic web search tools are converted to web_search_options parameter. diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 9457d5faaff..40566261c84 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -539,6 +539,90 @@ class TestBedrockMantleServiceTier: assert "priority" in str(mock_warning.call_args) +class TestBedrockMantleReasoningSummary: + @pytest.mark.parametrize("summary", ["concise", "detailed"]) + def test_unsupported_reasoning_summary_dropped_when_drop_params_true(self, summary): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": summary}}, + model="openai.gpt-5.6-sol", + drop_params=True, + ) + assert params["reasoning"] == {"effort": "medium"} + + def test_reasoning_summary_only_field_drops_reasoning(self): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"summary": "detailed"}}, + model="openai.gpt-5.6-sol", + drop_params=True, + ) + assert "reasoning" not in params + + @pytest.mark.parametrize("summary", ["concise", "detailed"]) + def test_unsupported_reasoning_summary_raises_when_drop_params_false(self, summary): + cfg = BedrockMantleResponsesAPIConfig() + with pytest.raises(litellm.UnsupportedParamsError) as excinfo: + cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": summary}}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + assert summary in str(excinfo.value) + assert "reasoning.summary" in str(excinfo.value) + assert "drop_params" in str(excinfo.value) + + def test_unhashable_reasoning_summary_raises_unsupported_params_error(self): + cfg = BedrockMantleResponsesAPIConfig() + with pytest.raises(litellm.UnsupportedParamsError) as excinfo: + cfg.map_openai_params( + response_api_optional_params={"reasoning": {"summary": ["detailed"]}}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + assert "reasoning.summary" in str(excinfo.value) + + @pytest.mark.parametrize("drop_params", [True, False]) + def test_supported_reasoning_summary_kept(self, drop_params): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": "auto"}}, + model="openai.gpt-5.6-sol", + drop_params=drop_params, + ) + assert params["reasoning"] == {"effort": "medium", "summary": "auto"} + + def test_reasoning_summary_kept_on_standard_path(self): + cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False) + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": "detailed"}}, + model="openai.gpt-oss-120b", + drop_params=False, + ) + assert params["reasoning"] == {"effort": "medium", "summary": "detailed"} + + def test_absent_reasoning_untouched(self): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"stream": True}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + assert params == {"stream": True} + + def test_drop_logged_at_warning_level(self, caplog): + cfg = BedrockMantleResponsesAPIConfig() + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": "detailed"}}, + model="openai.gpt-5.6-sol", + drop_params=True, + ) + warnings = [record for record in caplog.records if "dropping unsupported reasoning.summary" in record.getMessage()] + assert len(warnings) == 1 + assert "detailed" in warnings[0].getMessage() + + class TestBedrockMantleCodexRequestEndToEnd: def test_codex_priority_tier_request_becomes_mantle_acceptable(self): cfg = BedrockMantleResponsesAPIConfig() diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index d4ef4282b27..8479397efa7 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -501,8 +501,8 @@ def test_unmapped_model_fallback_function_calling(): assert info["supports_function_calling"] is True -def test_transform_messages_helper_strips_thinking_blocks(): - """thinking_blocks must not be forwarded to Fireworks chat completions.""" +def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_content(): + """Fireworks rejects thinking_blocks but requires reasoning_content to be replayed for reasoning_history.""" config = FireworksAIConfig() messages = [ {"role": "user", "content": "Translate a poem."}, @@ -519,7 +519,7 @@ def test_transform_messages_helper_strips_thinking_blocks(): messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} ) assert "thinking_blocks" not in out[1] - assert "reasoning_content" not in out[1] + assert out[1]["reasoning_content"] == "internal" assert out[1]["content"] == "I can help." diff --git a/tests/test_litellm/llms/reducto/test_parse_v3.py b/tests/test_litellm/llms/reducto/test_parse_v3.py index bacd12db58a..1d0c826ef8b 100644 --- a/tests/test_litellm/llms/reducto/test_parse_v3.py +++ b/tests/test_litellm/llms/reducto/test_parse_v3.py @@ -143,3 +143,28 @@ async def test_parse_v3_reducto_id_passthrough_skips_upload(disable_aiohttp_tran assert parse_request_body["input"] == "reducto://already-uploaded.pdf" assert parse_request_body["retrieval"]["chunk_mode"] == "section" assert response.pages[0].markdown.startswith("Page 1 block A") + + +@pytest.mark.asyncio +async def test_unknown_model_uses_current_protocol_without_local_rejection( + disable_aiohttp_transport, respx_mock +): + parse_route = respx_mock.post("https://platform.reducto.ai/parse").respond( + json=_reducto_parse_response() + ) + + response = await litellm.aocr( + model="reducto/future-parse-model", + document={ + "type": "document_url", + "document_url": "reducto://already-uploaded.pdf", + }, + api_key="test-key", + api_base="https://platform.reducto.ai", + ) + + assert parse_route.called + assert json.loads(parse_route.calls[0].request.read()) == { + "input": "reducto://already-uploaded.pdf" + } + assert response.model == "future-parse-model" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index d2788408e09..101f6e6fa5d 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -5769,3 +5769,70 @@ def test_calculate_web_search_requests_counts_unique_queries(): assert VertexGeminiConfig._calculate_web_search_requests([]) is None assert VertexGeminiConfig._calculate_web_search_requests([{"webSearchQueries": ["", ""]}]) is None + + +@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai"]) +@pytest.mark.parametrize( + "model", + ["gemini-2.5-flash", "gemini-3-pro-preview"], + ids=["thinking_budget_mapper", "thinking_level_mapper"], +) +@pytest.mark.parametrize("reasoning_effort", ["banana", "xhigh"]) +def test_invalid_reasoning_effort_is_a_400_not_a_500(custom_llm_provider, model, reasoning_effort): + """Regression for #40474. + + Both reasoning_effort mappers used to end their if/elif chain in a bare `ValueError`, which + `exception_type()` has no branch for, so it fell through to `APIConnectionError` and the proxy + answered a malformed client request with a retryable HTTP 500. `xhigh` is covered alongside the + nonsense value because it is a member of litellm's own `REASONING_EFFORT` literal, so callers + bridging from OpenAI-shaped code reach it without typing anything wrong. + """ + from litellm.utils import get_optional_params + + with pytest.raises(litellm.BadRequestError) as exc_info: + get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + reasoning_effort=reasoning_effort, + drop_params=True, + ) + + assert exc_info.value.status_code == 400 + message: Final = str(exc_info.value) + assert reasoning_effort in message + for supported in ("minimal", "low", "medium", "high", "none", "disable"): + assert supported in message + + +@pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai"]) +def test_invalid_reasoning_effort_surfaces_as_400_through_completion(custom_llm_provider): + """The same request through `completion()` must not come back as a retryable 500. + + Needs no provider credentials: param mapping runs before any network call. + """ + with pytest.raises(litellm.BadRequestError) as exc_info: + completion( + model=f"{custom_llm_provider}/gemini-3-pro-preview", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort="banana", + ) + + assert exc_info.value.status_code == 400 + assert not isinstance(exc_info.value, litellm.APIConnectionError) + + +@pytest.mark.parametrize("model", ["gemini-2.5-flash", "gemini-3-pro-preview"]) +def test_supported_reasoning_efforts_still_map(model): + """Guards the fix against over-rejecting: every advertised value must still produce a config.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + SUPPORTED_REASONING_EFFORTS, + ) + + for effort in SUPPORTED_REASONING_EFFORTS: + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, + optional_params={}, + model=model, + drop_params=False, + ) + assert "thinkingConfig" in result diff --git a/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py b/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py index 0c8b1cc2836..460aff3e8d1 100644 --- a/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py +++ b/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py @@ -1,20 +1,18 @@ """ -Regression tests for Azure Document Intelligence api_base resolution in OCR. +Regression tests for Azure Document Intelligence api_base ownership in OCR. `azure_ai` exposes two OCR services on one provider; the `doc-intelligence` -sub-route must resolve to `AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT`, not to the -generic `AZURE_AI_API_BASE` fallback that `get_llm_provider` injects. These tests -pin that routing and guard the backwards-compatibility contract that an explicitly -supplied api_base is always honoured. +sub-route must defer environment resolution to Rust, not accept the generic +`AZURE_AI_API_BASE` fallback that `get_llm_provider` injects. An explicitly +supplied api_base is still always honoured. """ from litellm.llms.azure_ai.ocr.common_utils import ( is_azure_document_intelligence_model, ) -from litellm.ocr.main import _prepare_ocr_request, _rust_bridge_api_base +from litellm.ocr.main import _prepare_ocr_request _DOC = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} -_DOC_INTELLIGENCE_ENDPOINT = "https://di.cognitiveservices.azure.com" _AZURE_AI_API_BASE = "https://generic-azure-ai.example.com" @@ -23,13 +21,6 @@ class _FakeLogging: return None -def _resolve_secret(name: str) -> str | None: - return { - "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": _DOC_INTELLIGENCE_ENDPOINT, - "AZURE_AI_API_BASE": _AZURE_AI_API_BASE, - }.get(name) - - def _prepare(model: str, api_base: str | None): return _prepare_ocr_request( model=model, @@ -56,15 +47,13 @@ class TestIsAzureDocumentIntelligenceModel: class TestDocIntelligenceApiBaseResolution: def test_generic_azure_ai_base_does_not_hijack_doc_intelligence(self, monkeypatch): - """Without an explicit api_base, the AZURE_AI_API_BASE fallback must not - overwrite the endpoint, so it resolves to the Document Intelligence one.""" + """The generic Azure base must not overwrite Rust-owned DI resolution.""" monkeypatch.setenv("AZURE_AI_API_BASE", _AZURE_AI_API_BASE) monkeypatch.delenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT", raising=False) prepared = _prepare("azure_ai/doc-intelligence/prebuilt-layout", None) assert prepared.api_base is None - assert _rust_bridge_api_base(prepared, _resolve_secret) == _DOC_INTELLIGENCE_ENDPOINT def test_explicit_api_base_is_honoured_for_doc_intelligence(self, monkeypatch): """A caller-supplied api_base must always win, even for doc-intelligence.""" @@ -74,7 +63,6 @@ class TestDocIntelligenceApiBaseResolution: prepared = _prepare("azure_ai/doc-intelligence/prebuilt-layout", custom) assert prepared.api_base == custom - assert _rust_bridge_api_base(prepared, _resolve_secret) == custom def test_generic_azure_ai_base_still_applies_to_mistral_ocr(self, monkeypatch): """Non doc-intelligence azure_ai models keep using AZURE_AI_API_BASE.""" diff --git a/tests/test_litellm/ocr/test_ocr_native_format.py b/tests/test_litellm/ocr/test_ocr_native_format.py index 249fbda713e..46e9a4d3729 100644 --- a/tests/test_litellm/ocr/test_ocr_native_format.py +++ b/tests/test_litellm/ocr/test_ocr_native_format.py @@ -1,52 +1,60 @@ """ -Tests for the OCR `req_format` option in the SDK request path: -providers that don't support a native response must reject it, and the Rust -bridge (which only returns the normalized shape) must not serve native requests. +Tests for the OCR `req_format` option in the SDK request path. """ -import dataclasses -from unittest.mock import MagicMock - import pytest import litellm -from litellm.llms.azure_ai.ocr.cohere_parse_transformation import AzureAICohereParseConfig -from litellm.llms.cohere.ocr.transformation import CohereParseConfig -from litellm.ocr.main import _PreparedOCRRequest, _rust_ocr_supported +from litellm.rust_bridge import ocr as rust_ocr_bridge +from litellm.rust_bridge.ocr import LiteLLMOcrRequest DOCUMENT = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} -def _prepared(optional_params: dict[str, object]) -> _PreparedOCRRequest: - return _PreparedOCRRequest( - model="doc-intelligence/prebuilt-layout", - document=dict(DOCUMENT), +def _request( + optional_params: dict[str, object], model: str = "azure_ai/doc-intelligence/prebuilt-layout" +) -> LiteLLMOcrRequest: + return LiteLLMOcrRequest( + model=model, + document=DOCUMENT, api_key="fake-key", - api_base="https://example.cognitiveservices.azure.com", - custom_llm_provider="azure_ai", + api_base=None, + custom_llm_provider=None, extra_headers=None, - provider_config=MagicMock(), - optional_params=optional_params, - litellm_params={}, - effective_timeout=60.0, - litellm_logging_obj=MagicMock(), + timeout=60.0, + kwargs=optional_params, ) @pytest.mark.parametrize("optional_params", [{}, {"req_format": "litellm"}]) def test_rust_ocr_serves_default_format(optional_params): - assert _rust_ocr_supported(_prepared(optional_params)) is True + assert rust_ocr_bridge.supported(_request(optional_params)) is True -def test_rust_ocr_skipped_for_native_format(): - assert _rust_ocr_supported(_prepared({"req_format": "native"})) is False +def test_rust_ocr_serves_native_format_for_document_intelligence(): + assert rust_ocr_bridge.supported(_request({"req_format": "native"})) is True -@pytest.mark.parametrize("provider_config", [CohereParseConfig(), AzureAICohereParseConfig()]) -def test_rust_ocr_skipped_for_configs_without_bridge_support(provider_config): - prepared = dataclasses.replace(_prepared({}), provider_config=provider_config) +def test_rust_ocr_response_retains_provider_native_response(): + provider_response = {"status": "succeeded", "analyzeResult": {"content": "native"}} + response = rust_ocr_bridge._response( + { + "pages": [], + "model": "prebuilt-layout", + "document_annotation": None, + "usage_info": {"pages_processed": 0}, + "object": "ocr", + "provider_native_response": provider_response, + } + ) - assert _rust_ocr_supported(prepared) is False + assert response.get_provider_native_response() == provider_response + assert response.model_dump().get("provider_native_response") is None + + +@pytest.mark.parametrize("model", ["cohere/cohere-parse", "azure_ai/cohere-parse"]) +def test_rust_ocr_skipped_for_unsupported_models(model): + assert rust_ocr_bridge.supported(_request({}, model)) is False @pytest.mark.asyncio diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index c34833221cc..dbb4f822d0b 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -3,7 +3,6 @@ import builtins import importlib import types -from typing import Any import httpx import pytest @@ -59,6 +58,7 @@ class RecordingBridge: custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> dict[str, object]: self.calls.append( @@ -70,6 +70,7 @@ class RecordingBridge: "custom_llm_provider": custom_llm_provider, "extra_headers": extra_headers, "optional_params": optional_params, + "input_sources": input_sources, "timeout_seconds": timeout_seconds, } ) @@ -91,6 +92,7 @@ class RecordingAsyncBridge: custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> dict[str, object]: self.calls.append( @@ -102,6 +104,7 @@ class RecordingAsyncBridge: "custom_llm_provider": custom_llm_provider, "extra_headers": extra_headers, "optional_params": optional_params, + "input_sources": input_sources, "timeout_seconds": timeout_seconds, } ) @@ -118,6 +121,7 @@ class RaisingBridge: custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> dict[str, object]: raise RuntimeError("bridge failed") @@ -133,6 +137,7 @@ class RaisingAsyncBridge: custom_llm_provider: str | None, extra_headers: dict[str, object] | None, optional_params: dict[str, object], + input_sources: dict[str, str], timeout_seconds: float | None, ) -> dict[str, object]: raise RuntimeError("bridge failed") @@ -144,6 +149,9 @@ class RecordingLogging: def __init__(self) -> None: self.pre_call_kwargs: dict[str, object] | None = None + def update_from_kwargs(self, **kwargs: object) -> None: + self.update_kwargs = kwargs + def pre_call( self, *, @@ -158,66 +166,32 @@ class RecordingLogging: } -class FakeOCRConfig: - """A stand-in ``BaseOCRConfig`` that echoes the request it would build.""" - - def __init__(self, api_key_env_var: str = "MISTRAL_API_KEY") -> None: - self.api_key_env_var = api_key_env_var - - def get_api_key_env_var(self) -> str: - return self.api_key_env_var - - def validate_environment( - self, - *, - headers: dict[str, object], - model: str, - api_key: str | None, - api_base: str | None, - litellm_params: dict[str, object], - ) -> dict[str, object]: - return {"Authorization": f"Bearer {api_key}", **headers} - - def get_complete_url( - self, - *, - api_base: str | None, - model: str, - optional_params: dict[str, object], - litellm_params: dict[str, object], - ) -> str: - return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" - - def get_error_class(self, error_message: str, status_code: int, headers: dict[str, str]) -> BaseLLMException: - return BaseLLMException(status_code=status_code, message=error_message, headers=headers) - - -def build_prepared_request( +def build_request( *, logging_obj: RecordingLogging | None = None, - provider_config: FakeOCRConfig | None = None, model: str = "mistral-ocr-latest", document: dict[str, object] = DOCUMENT, api_key: str | None = "sk-test", api_base: str | None = None, - custom_llm_provider: str = "mistral", + custom_llm_provider: str | None = "mistral", extra_headers: dict[str, object] | None = None, optional_params: dict[str, object] | None = None, litellm_params: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = 12.5, -) -> Any: - return ocr_main._PreparedOCRRequest( +) -> rust_bridge.LiteLLMOcrRequest: + return rust_bridge.LiteLLMOcrRequest( model=model, document=document, api_key=api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, - provider_config=provider_config or FakeOCRConfig(), - optional_params=optional_params or {}, - litellm_params=litellm_params or {}, - effective_timeout=timeout, - litellm_logging_obj=logging_obj or RecordingLogging(), + timeout=timeout, + kwargs={ + **(optional_params or {}), + **(litellm_params or {}), + "litellm_logging_obj": logging_obj or RecordingLogging(), + }, ) @@ -425,6 +399,7 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): "x-trace-id": "trace-1", }, "optional_params": {"include_image_base64": True, "pages": [0]}, + "input_sources": {}, "timeout_seconds": 12.5, } @@ -456,6 +431,7 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): "custom_llm_provider": "vertex_ai", "extra_headers": None, "optional_params": {"vertex_project": "project-1"}, + "input_sources": {}, "timeout_seconds": 42.0, } @@ -467,7 +443,7 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): rust_bridge._OCR.override(bridge) response = ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( logging_obj=logging_obj, api_base="https://proxy.internal", extra_headers={"x-trace-id": "trace-1"}, @@ -486,10 +462,10 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): "api_base": "https://proxy.internal", "custom_llm_provider": "mistral", "extra_headers": { - "Authorization": "Bearer sk-test", "x-trace-id": "trace-1", }, "optional_params": {"include_image_base64": True}, + "input_sources": {}, "timeout_seconds": 12.5, } @@ -499,7 +475,7 @@ def test_rust_upstream_error_uses_ocr_provider_error_mapping(): mapped = ocr_main._map_rust_ocr_error( error, - build_prepared_request(), + build_request(), (RuntimeError, RustUpstreamError), ) @@ -514,7 +490,7 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request(api_key=None, timeout=None), + request=build_request(api_key=None, timeout=None), resolve_api_key=lambda name: "sk-from-vault" if name == "MISTRAL_API_KEY" else None, ) @@ -530,7 +506,7 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): raise AssertionError(f"resolver should not be called for {name}") ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( api_key="sk-explicit", timeout=None, ), @@ -540,7 +516,7 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): assert bridge.calls[0]["api_key"] == "sk-explicit" -def test_run_rust_ocr_uses_provider_api_key_env_var(): +def test_run_rust_ocr_uses_mistral_secret_manager_without_provider_config(): bridge = RecordingBridge() resolver_calls = [] litellm.rust(True) @@ -551,16 +527,15 @@ def test_run_rust_ocr_uses_provider_api_key_env_var(): return "sk-provider-env" ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( - provider_config=FakeOCRConfig(api_key_env_var="PROVIDER_OCR_API_KEY"), - model="provider-ocr-model", + request=build_request( + model="mistral-ocr-latest", api_key=None, timeout=None, ), resolve_api_key=_resolver, ) - assert resolver_calls == ["PROVIDER_OCR_API_KEY"] + assert resolver_calls == ["MISTRAL_API_KEY"] assert bridge.calls[0]["api_key"] == "sk-provider-env" @@ -570,7 +545,7 @@ def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata(): rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( custom_llm_provider="vertex_ai", model="mistral-ocr-maas", litellm_params={ @@ -588,6 +563,7 @@ def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata(): "include_image_base64": True, "vertex_project": "project-1", "vertex_location": "us-central1", + "vertex_credentials": "redacted", } @@ -600,10 +576,11 @@ def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_mana return { "VERTEXAI_PROJECT": "project-from-secret", "VERTEXAI_LOCATION": "us-east5", + "VERTEXAI_CREDENTIALS": "credentials-from-secret", }.get(name) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( custom_llm_provider="vertex_ai", model="mistral-ocr-maas", timeout=None, @@ -613,44 +590,210 @@ def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_mana assert bridge.calls[0]["optional_params"]["vertex_project"] == "project-from-secret" assert bridge.calls[0]["optional_params"]["vertex_location"] == "us-east5" + assert bridge.calls[0]["optional_params"]["vertex_credentials"] == "credentials-from-secret" -def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): +def test_prepare_rust_ocr_call_defers_azure_environment_resolution_to_rust(): bridge = RecordingBridge() litellm.rust(True) rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( custom_llm_provider="azure_ai", model="pixtral-12b-2409", + api_key=None, api_base=None, timeout=None, ), - resolve_api_key=lambda name: "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None, + resolve_api_key=lambda name: pytest.fail(f"Python resolved Azure secret {name}"), ) - assert bridge.calls[0]["api_base"] == "https://azure.example.com" + assert bridge.calls[0]["api_base"] is None + assert bridge.calls[0]["api_key"] is None + assert bridge.calls[0]["extra_headers"] is None -def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): +def test_prepare_rust_ocr_call_defers_document_intelligence_environment_to_rust(): bridge = RecordingBridge() litellm.rust(True) rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( custom_llm_provider="azure_ai", model="doc-intelligence/prebuilt-layout", api_base=None, timeout=None, ), - resolve_api_key=lambda name: ( - "https://document-intelligence.example.com" if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" else None - ), + resolve_api_key=lambda name: pytest.fail(f"Python resolved Azure secret {name}"), ) - assert bridge.calls[0]["api_base"] == "https://document-intelligence.example.com" + assert bridge.calls[0]["api_base"] is None + + +def test_prepare_rust_ocr_call_forwards_raw_azure_auth_inputs(): + bridge = RecordingBridge() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + + ocr_main._run_rust_ocr( + request=build_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key=None, + api_base="https://azure.example.com", + extra_headers={"x-trace-id": "trace-1"}, + litellm_params={ + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "azure_scope": "scope", + "azure_authority_host": "https://login.example.com", + "azure_credential": "ClientSecretCredential", + "azure_federated_token_file": "/token", + }, + timeout=None, + ), + resolve_api_key=lambda name: pytest.fail(f"Python resolved Azure secret {name}"), + ) + + call = bridge.calls[0] + assert call["api_key"] is None + assert call["api_base"] == "https://azure.example.com" + assert call["extra_headers"] == {"x-trace-id": "trace-1"} + assert call["optional_params"] == { + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "azure_scope": "scope", + "azure_authority_host": "https://login.example.com", + "azure_credential": "ClientSecretCredential", + "azure_federated_token_file": "/token", + } + assert call["input_sources"] == {} + + +def test_prepare_rust_ocr_call_preserves_proxy_input_sources(): + bridge = RecordingBridge() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + request_values = { + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "azure_authority_host": "https://login.example.com", + "api_base": "https://azure.example.com", + } + + ocr_main._run_rust_ocr( + request=build_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key="request-key", + api_base="https://azure.example.com", + litellm_params={ + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "azure_authority_host": "https://login.example.com", + "proxy_server_request": {"body": request_values, "credential_fields": ("api_key",)}, + }, + ), + resolve_api_key=lambda _name: None, + ) + + assert bridge.calls[0]["input_sources"] == { + **{name: "request" for name in request_values}, + "api_key": "request", + } + + marshaled = rust_bridge._marshal( + build_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key="request-key", + api_base="https://azure.example.com", + litellm_params={ + "proxy_server_request": { + "body": {"api_base": "https://azure.example.com"}, + "credential_fields": ("api_key",), + } + }, + ), + lambda _name: None, + lambda document: document, + ) + assert marshaled.input_sources == {"api_base": "request", "api_key": "request"} + + +def test_rust_ocr_logging_redacts_azure_credentials(): + bridge = RecordingBridge() + logging_obj = RecordingLogging() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + + ocr_main._run_rust_ocr( + request=build_request( + logging_obj=logging_obj, + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key=None, + litellm_params={"azure_ad_token": "token", "client_secret": "secret"}, + ), + resolve_api_key=lambda _name: None, + ) + + assert logging_obj.update_kwargs["optional_params"] == { + "azure_ad_token": "****", + "client_secret": "****", + } + assert logging_obj.pre_call_kwargs is not None + additional_args = logging_obj.pre_call_kwargs["additional_args"] + assert isinstance(additional_args, dict) + complete_input = additional_args["complete_input_dict"] + assert isinstance(complete_input, dict) + assert complete_input["azure_ad_token"] == "****" + assert complete_input["client_secret"] == "****" + + +def test_rust_eligibility_rejects_python_only_azure_auth_modes(): + for params in ( + {"azure_ad_token_provider": lambda: "token"}, + {"azure_username": "user"}, + {"azure_password": "password"}, + ): + assert not ocr_main._rust_ocr_supported( + build_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + litellm_params=params, + ) + ) + + +def test_prepare_rust_ocr_call_forwards_global_azure_refresh(monkeypatch: pytest.MonkeyPatch): + bridge = RecordingBridge() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True) + + ocr_main._run_rust_ocr( + request=build_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key=None, + api_base="https://azure.example.com", + litellm_params={"proxy_server_request": {"body": {"enable_azure_ad_token_refresh": True}}}, + timeout=None, + ), + resolve_api_key=lambda _name: None, + ) + + assert bridge.calls[0]["optional_params"] == {"enable_azure_ad_token_refresh": True} + assert bridge.calls[0]["input_sources"] == {"enable_azure_ad_token_refresh": "deployment"} def test_run_rust_ocr_runs_pre_call_logging(): @@ -660,7 +803,7 @@ def test_run_rust_ocr_runs_pre_call_logging(): rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( + request=build_request( logging_obj=logging_obj, api_base="https://api.mistral.ai/v1", extra_headers={"x-trace-id": "trace-1"}, @@ -676,9 +819,8 @@ def test_run_rust_ocr_runs_pre_call_logging(): complete_input = additional_args["complete_input_dict"] assert complete_input["document"] == DOCUMENT assert complete_input["include_image_base64"] is True - assert additional_args["api_base"] == "https://api.mistral.ai/v1/ocr" + assert additional_args["api_base"] == "https://api.mistral.ai/v1" assert additional_args["headers"] == { - "Authorization": "Bearer sk-test", "x-trace-id": "trace-1", } @@ -696,12 +838,11 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert response.pages[0].markdown == "hello world" assert len(fake_bridge.calls) == 1 call = fake_bridge.calls[0] - assert call["model"] == "mistral-ocr-latest" + assert call["model"] == MODEL assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" - assert call["custom_llm_provider"] == "mistral" + assert call["custom_llm_provider"] is None assert call["extra_headers"] == { - "Authorization": "Bearer sk-test", "x-trace-id": "trace-1", } assert call["optional_params"].get("include_image_base64") is True @@ -717,8 +858,29 @@ def test_ocr_routes_azure_ai_to_rust_when_enabled(fake_bridge): assert isinstance(response, OCRResponse) assert len(fake_bridge.calls) == 1 - assert fake_bridge.calls[0]["model"] == "pixtral-12b-2409" - assert fake_bridge.calls[0]["custom_llm_provider"] == "azure_ai" + assert fake_bridge.calls[0]["model"] == "azure_ai/pixtral-12b-2409" + assert fake_bridge.calls[0]["custom_llm_provider"] is None + assert fake_bridge.calls[0]["extra_headers"] is None + + +def test_ocr_routes_azure_entra_inputs_to_rust_without_python_auth(fake_bridge): + response = litellm.ocr( + model="azure_ai/pixtral-12b-2409", + document=DOCUMENT, + api_base="https://example.services.ai.azure.com", + azure_ad_token="entra-token", + tenant_id="tenant", + client_id="client", + ) + + assert isinstance(response, OCRResponse) + assert fake_bridge.calls[0]["api_key"] is None + assert fake_bridge.calls[0]["extra_headers"] is None + assert fake_bridge.calls[0]["optional_params"] == { + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + } def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge): @@ -768,12 +930,11 @@ async def test_aocr_routes_to_async_rust_when_enabled(fake_async_bridge): assert response.pages[0].markdown == "hello world" assert len(fake_async_bridge.calls) == 1 call = fake_async_bridge.calls[0] - assert call["model"] == "mistral-ocr-latest" + assert call["model"] == MODEL assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" - assert call["custom_llm_provider"] == "mistral" + assert call["custom_llm_provider"] is None assert call["extra_headers"] == { - "Authorization": "Bearer sk-test", "x-trace-id": "trace-1", } assert call["optional_params"].get("include_image_base64") is True @@ -864,3 +1025,137 @@ def test_ocr_provider_configs_expose_api_key_env_vars(): assert AzureDocumentIntelligenceOCRConfig().get_api_key_env_var() == "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" assert VertexAIOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" assert VertexAIDeepSeekOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_rust_receives_unmapped_azure_options(asynchronous, fake_bridge, fake_async_bridge): + from typing import Final + + arguments: Final = { + "model": "azure_ai/doc-intelligence/prebuilt-layout", + "document": DOCUMENT, + "api_key": "test-key", + "pages": [0, 2], + "features": ["languages", "style"], + "provider_extension": {"enabled": True}, + } + if asynchronous: + await litellm.aocr(**arguments) + else: + litellm.ocr(**arguments) + call: Final = (fake_async_bridge if asynchronous else fake_bridge).calls[0] + assert call["model"] == arguments["model"] + assert call["custom_llm_provider"] is None + assert call["extra_headers"] is None + assert call["optional_params"] == { + "pages": [0, 2], + "features": ["languages", "style"], + "provider_extension": {"enabled": True}, + } + + +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.asyncio +async def test_python_fallback_maps_original_options_once(enabled, monkeypatch): + from io import BytesIO + from typing import Final + + class PythonHandler: + def __init__(self): + self.calls = [] + + def ocr(self, **kwargs): + self.calls.append(kwargs) + return OCRResponse(pages=[], model=kwargs["model"]) + + handler: Final = PythonHandler() + monkeypatch.setattr(ocr_main, "base_llm_http_handler", handler) + litellm.rust(enabled) + rust_bridge._OCR.override(None) + rust_bridge._AOCR.override(None) + for asynchronous in (False, True): + file: Final = BytesIO(b"test document") + arguments: Final = { + "model": "azure_ai/doc-intelligence/prebuilt-layout", + "document": {"type": "file", "file": file}, + "api_key": "test-key", + "pages": [0, 2], + } + if asynchronous: + await litellm.aocr(**arguments) + else: + litellm.ocr(**arguments) + assert handler.calls[-1]["optional_params"]["pages"] == "1,3" + assert handler.calls[-1]["document"]["document_url"].endswith("dGVzdCBkb2N1bWVudA==") + assert len(handler.calls) == 2 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("model", ["mistral/mistral-ocr-latest", "azure_ai/doc-intelligence/prebuilt-read"]) +@pytest.mark.asyncio +async def test_native_public_ocr_matches_python(model, asynchronous): + import json + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from threading import Thread + from typing import Final + from urllib.parse import parse_qsl, urlsplit + + native: Final = rust_bridge_loader.get_native_bridge() + if native is None: + pytest.skip("requires the compiled Rust extension") + calls: Final = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body: Final = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + target: Final = urlsplit(self.path) + calls.append( + ( + target.path, + parse_qsl(target.query), + self.headers.get("Authorization"), + self.headers.get("Ocp-Apim-Subscription-Key"), + body, + ) + ) + payload: Final = ( + {"status": "succeeded", "analyzeResult": {"pages": []}} + if "doc-intelligence" in model + else {"pages": [{"index": 0, "markdown": "hello"}]} + ) + encoded: Final = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *_args): + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = Thread(target=server.serve_forever, daemon=True) + thread.start() + responses: Final = [] + try: + for enabled in (False, True): + litellm.rust(enabled) + arguments: Final = { + "model": model, + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "api_key": "test-key", + "api_base": f"http://127.0.0.1:{server.server_port}", + "pages": [0, 2], + "timeout": 3.0, + } + response: Final = await litellm.aocr(**arguments) if asynchronous else litellm.ocr(**arguments) + responses.append(response.model_dump()) + assert len(calls) == 2 + assert calls[0] == calls[1] + for key in ("model", "pages", "object"): + assert responses[0][key] == responses[1][key] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py index 9f2feddb0e3..55accfb169d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py @@ -97,6 +97,88 @@ def unauthenticated_client(): # --------------------------------------------------------------------------- +@pytest.mark.parametrize("byok_first", [True, False]) +def test_byok_challenge_discovers_api_key_flow(monkeypatch, byok_first): + from litellm.proxy._experimental.mcp_server import byok_oauth_endpoints, discoverable_endpoints + from litellm.proxy._experimental.mcp_server.oauth_utils import get_byok_www_authenticate + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + app = FastAPI() + routers = (byok_oauth_endpoints.router, discoverable_endpoints.router) + for item in routers if byok_first else reversed(routers): + app.include_router(item) + with TestClient(app) as session: + challenge = get_byok_www_authenticate() + assert challenge == 'Bearer resource_metadata="/v1/mcp/oauth/protected-resource"' + response = session.get(challenge.split('"')[1]) + assert response.status_code == 200 + assert response.json() == { + "resource": "http://testserver", + "authorization_servers": ["http://testserver/v1/mcp/oauth"], + } + authorization = session.get("/.well-known/oauth-authorization-server/v1/mcp/oauth") + assert authorization.status_code == 200 + metadata = authorization.json() + assert metadata["issuer"] == response.json()["authorization_servers"][0] + assert metadata["authorization_endpoint"] == "http://testserver/v1/mcp/oauth/authorize" + assert metadata["token_endpoint"] == "http://testserver/v1/mcp/oauth/token" + assert metadata["code_challenge_methods_supported"] == ["S256"] + + +@pytest.mark.parametrize( + ("base_url", "root_path", "expected"), + [ + ("", "", "/v1/mcp/oauth/protected-resource"), + ("", "/proxy", "/proxy/v1/mcp/oauth/protected-resource"), + ("https://gateway.example.com/proxy", "/proxy", "https://gateway.example.com/proxy/v1/mcp/oauth/protected-resource"), + ], +) +def test_byok_challenge_preserves_external_base(monkeypatch, base_url, root_path, expected): + from litellm.proxy._experimental.mcp_server.oauth_utils import get_byok_www_authenticate + + monkeypatch.setenv("PROXY_BASE_URL", base_url) + monkeypatch.setenv("SERVER_ROOT_PATH", root_path) + assert get_byok_www_authenticate() == f'Bearer resource_metadata="{expected}"' + + +def test_byok_discovery_preserves_per_request_prefixes(monkeypatch): + from fastapi import FastAPI + + from litellm.proxy._experimental.mcp_server.server import _check_byok_credential + from litellm.proxy.middleware.per_request_root_path_middleware import PerRequestRootPathMiddleware + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setenv("SERVER_ROOT_PATHS", "/tenant-a,/tenant-b") + app = FastAPI() + app.include_router(router) + app.add_middleware(PerRequestRootPathMiddleware, root_paths=("/tenant-a", "/tenant-b")) + server = MCPServer(server_id="byok-prefix", name="byok-prefix", transport=MCPTransport.http, is_byok=True) + + @app.get("/challenge") + async def challenge(): + await _check_byok_credential(server, None) + + with TestClient(app) as client: + for prefix in ("/tenant-a", "/tenant-b", ""): + challenge_response = client.get(f"{prefix}/challenge") + assert challenge_response.status_code == 401 + metadata_path = f"{prefix}/v1/mcp/oauth/protected-resource" + assert challenge_response.headers["www-authenticate"] == f'Bearer resource_metadata="{metadata_path}"' + prm = client.get(metadata_path) + assert prm.status_code == 200 + issuer = f"http://testserver{prefix}/v1/mcp/oauth" + assert prm.json()["authorization_servers"] == [issuer] + asm = client.get(f"/.well-known/oauth-authorization-server{prefix}/v1/mcp/oauth") + assert asm.status_code == 200 + assert asm.json()["issuer"] == issuer + assert asm.json()["authorization_endpoint"] == f"{issuer}/authorize" + assert asm.json()["token_endpoint"] == f"{issuer}/token" + assert client.get("/.well-known/oauth-authorization-server/unknown/v1/mcp/oauth").status_code == 404 + + def test_oauth_authorization_server_metadata(client): resp = client.get("/.well-known/oauth-authorization-server") assert resp.status_code == 200 @@ -107,15 +189,6 @@ def test_oauth_authorization_server_metadata(client): assert "S256" in data["code_challenge_methods_supported"] -def test_oauth_protected_resource_metadata(client): - resp = client.get("/.well-known/oauth-protected-resource") - assert resp.status_code == 200 - data = resp.json() - assert "resource" in data - assert "authorization_servers" in data - assert len(data["authorization_servers"]) == 1 - - # --------------------------------------------------------------------------- # Authorization GET endpoint # --------------------------------------------------------------------------- @@ -501,7 +574,7 @@ async def test_check_byok_credential_no_user_id(): @pytest.mark.asyncio -async def test_check_byok_credential_missing_credential(): +async def test_check_byok_credential_missing_credential(monkeypatch): """BYOK server with a known user but no stored credential → 401.""" from litellm.proxy._experimental.mcp_server.server import _check_byok_credential from litellm.proxy._types import UserAPIKeyAuth @@ -515,6 +588,11 @@ async def test_check_byok_credential_missing_credential(): ) user_auth = UserAPIKeyAuth(user_id="user-99", api_key="sk-test") + from litellm.proxy._experimental.mcp_server import server as server_module + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(server_module, "_byok_cred_cache", {}) mock_prisma = MagicMock() with ( @@ -526,6 +604,10 @@ async def test_check_byok_credential_missing_credential(): ): with pytest.raises(HTTPException) as exc_info: await _check_byok_credential(server, user_auth) + with pytest.raises(HTTPException) as cached_exc: + await _check_byok_credential(server, user_auth) + assert cached_exc.value.status_code == 401 + assert cached_exc.value.headers == exc_info.value.headers assert exc_info.value.status_code == 401 detail: Any = exc_info.value.detail @@ -533,7 +615,38 @@ async def test_check_byok_credential_missing_credential(): assert detail["server_id"] == "byok-2" headers = exc_info.value.headers or {} assert "WWW-Authenticate" in headers # type: ignore[operator] - assert "oauth-protected-resource" in headers["WWW-Authenticate"] # type: ignore[index] + assert headers["WWW-Authenticate"] == 'Bearer resource_metadata="/v1/mcp/oauth/protected-resource"' + + +@pytest.mark.asyncio +async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monkeypatch): + from datetime import datetime, timezone + + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") + monkeypatch.setattr(mcp_module, "_byok_cred_cache", {}) + server = MCPServer(server_id="byok-discovery", name="byok-discovery", transport=MCPTransport.http, is_byok=True) + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + with pytest.raises(HTTPException) as exc_info: + await mcp_module.execute_mcp_tool( + name="list_regions", + arguments={}, + allowed_mcp_servers=[server], + requested_server_id=server.server_id, + start_time=datetime.now(timezone.utc), + user_api_key_auth=UserAPIKeyAuth(user_id="byok-discovery-user"), + ) + assert exc_info.value.status_code == 401 + assert exc_info.value.detail["server_id"] == server.server_id + assert exc_info.value.headers == { + "WWW-Authenticate": 'Bearer resource_metadata="https://gateway.example.com/proxy/v1/mcp/oauth/protected-resource"' + } @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 1b7e3d6a1c3..9ea870d3210 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -3433,51 +3433,84 @@ async def test_oauth_protected_resource_gateway_managed_oauth2_advertises_gatewa global_mcp_server_manager.registry.clear() +@pytest.mark.parametrize("server_count", [0, 1, 2]) +@pytest.mark.parametrize("byok_first", [True, False]) +@pytest.mark.parametrize( + ("base_url", "origin"), + [ + ("https://gateway.example.com", "https://gateway.example.com"), + ("https://gateway.example.com/proxy", "https://gateway.example.com"), + ("http://[::1]:4000/proxy", "http://[::1]:4000"), + ], +) +def test_root_protected_resource_discovers_gateway(monkeypatch, server_count, byok_first, base_url, origin): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from litellm.proxy._experimental.mcp_server import byok_oauth_endpoints, discoverable_endpoints + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("PROXY_BASE_URL", base_url) + monkeypatch.setattr( + global_mcp_server_manager, + "registry", + { + f"oauth_{index}": MCPServer( + server_id=f"oauth_{index}", + name=f"oauth_{index}", + server_name=f"oauth_{index}", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + ) + for index in range(server_count) + }, + ) + app = FastAPI() + routers = (byok_oauth_endpoints.router, discoverable_endpoints.router) + for router in routers if byok_first else reversed(routers): + app.include_router(router) + with TestClient(app) as client: + response = client.get("/.well-known/oauth-protected-resource", params={"mcp_server_name": "oauth_0"}) + assert response.status_code == 200 + assert response.json() == { + "resource": origin, + "authorization_servers": [f"{base_url}/mcp"], + "scopes_supported": [], + } + authorization = client.get("/.well-known/oauth-authorization-server/mcp") + assert authorization.status_code == 200 + metadata = authorization.json() + assert metadata["issuer"] == response.json()["authorization_servers"][0] + assert metadata["authorization_endpoint"] == f"{base_url}/authorize/mcp-session" + assert metadata["token_endpoint"] == f"{base_url}/token" + assert metadata["registration_endpoint"] == f"{base_url}/register" + aggregate = client.get("/.well-known/oauth-protected-resource/mcp") + assert aggregate.status_code == 200 + assert aggregate.json()["resource"] == f"{base_url}/mcp" + + @pytest.mark.asyncio -async def test_oauth_protected_resource_root_resolved_single_server_keeps_relay_as(): - """The unnamed (bare-root) legacy shape resolves the single configured oauth2 server and - must keep advertising the per-server relay authorization server: only an EXPLICITLY - named request opts into the gateway-as-AS flow (LIT-4864), so pre-existing single-server - deployments discovering through the root document are byte-identical.""" - try: - from fastapi import Request +async def test_unnamed_protected_resource_builder_uses_gateway_origin(monkeypatch): + from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - _build_oauth_protected_resource_response, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.proxy._types import MCPTransport - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - only_server = MCPServer( - server_id="solo_mcp", - name="solo_mcp", - server_name="solo_mcp", - alias="solo_mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - authorization_url="https://idp.example.com/authorize", - token_url="https://idp.example.com/oauth/token", + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, ) - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm.example.com/" - mock_request.headers = {} - - global_mcp_server_manager.registry.clear() - try: - global_mcp_server_manager.registry[only_server.server_id] = only_server - response = await _build_oauth_protected_resource_response( - request=mock_request, mcp_server_name=None, use_standard_pattern=False - ) - assert response["authorization_servers"] == ["https://litellm.example.com/solo_mcp"] - finally: - global_mcp_server_manager.registry.clear() + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = Request( + {"type": "http", "scheme": "https", "server": ("gateway.example.com", 443), "path": "/", "headers": []} + ) + response = await _build_oauth_protected_resource_response(request, None, False) + assert response == { + "resource": "https://gateway.example.com", + "authorization_servers": ("https://gateway.example.com/mcp",), + "scopes_supported": (), + } @pytest.mark.asyncio @@ -4137,8 +4170,9 @@ async def test_discovery_root_does_not_expose_private_server_for_external_client assert "/test_oauth/" not in authorization_response["authorization_endpoint"] assert "/test_oauth/" not in authorization_response["token_endpoint"] assert authorization_response["scopes_supported"] == [] - assert resource_response["authorization_servers"] == ["https://llm.example.com"] - assert resource_response["scopes_supported"] == [] + assert tuple(resource_response["authorization_servers"]) == ("https://llm.example.com/mcp",) + assert resource_response["resource"] == "https://llm.example.com" + assert not resource_response["scopes_supported"] finally: global_mcp_server_manager.registry.clear() @@ -9014,7 +9048,7 @@ def test_aggregate_wellknown_routes_serve_gateway_metadata(): assert asm.status_code == 200 assert asm.json()["issuer"] == "http://testserver/mcp" - assert asm.json()["authorization_endpoint"] == "http://testserver/authorize" + assert asm.json()["authorization_endpoint"] == "http://testserver/authorize/mcp-session" assert "none" in asm.json()["token_endpoint_auth_methods_supported"] @@ -9077,11 +9111,7 @@ def test_well_known_root_suffix_reflects_server_root_path(): @pytest.mark.asyncio -async def test_bare_origin_discovery_resolves_single_server_not_aggregate(): - """The always-on aggregate front door must not change bare-origin discovery: with one - oauth2 server configured, the no-suffix /.well-known/oauth-{authorization-server, - protected-resource} still resolves THAT server, so an existing single-server deployment's - discovery is unchanged. The aggregate document lives only at the /mcp-suffixed routes.""" +async def test_root_resource_uses_gateway_without_changing_authorization_relay(): from fastapi import Request from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( @@ -9105,10 +9135,10 @@ async def test_bare_origin_discovery_resolves_single_server_not_aggregate(): resource_response = await _build_oauth_protected_resource_response( request=mock_request, mcp_server_name=None, use_standard_pattern=True ) - # per-server, not aggregate: the single server's name is in the endpoints assert "/test_oauth/authorize" in authorization_response["authorization_endpoint"] assert authorization_response["issuer"] == "https://llm.example.com" - assert resource_response["authorization_servers"] == ["https://llm.example.com/test_oauth"] + assert tuple(resource_response["authorization_servers"]) == ("https://llm.example.com/mcp",) + assert resource_response["resource"] == "https://llm.example.com" finally: global_mcp_server_manager.registry.clear() @@ -10640,7 +10670,7 @@ class TestPerRequestRootPathDiscovery: assert asm.status_code == 200 assert asm.json()["issuer"] == "http://testserver/tenant-a/mcp" - assert asm.json()["authorization_endpoint"] == "http://testserver/tenant-a/authorize" + assert asm.json()["authorization_endpoint"] == "http://testserver/tenant-a/authorize/mcp-session" # The prefixed authorize URL routes to the real handler (not 404): # under per-request root_path the whole app is reachable per-prefix, @@ -10799,6 +10829,68 @@ def _consent_flow_handle(page: str) -> str: return match.group(1) +@pytest.mark.parametrize("redirect_uri", ["http://127.0.0.1:51234/callback", "https://client.example.com/callback"]) +@pytest.mark.parametrize("signed_in", [True, False]) +def test_root_discovery_origin_authorizes_mcp_session(monkeypatch, redirect_uri, signed_in): + from urllib.parse import parse_qs, urlparse + + client, session_cookie, minted = _native_client_app(monkeypatch) + root = client.get("/.well-known/oauth-protected-resource") + assert root.status_code == 200 + assert root.json()["resource"] == "http://testserver" + authorization = client.get("/.well-known/oauth-authorization-server/mcp") + assert authorization.status_code == 200 + metadata = authorization.json() + registered = client.post(metadata["registration_endpoint"], json={"redirect_uris": [redirect_uri]}) + assert registered.status_code == 201 + if signed_in: + client.cookies.set("token", session_cookie) + response = client.get( + metadata["authorization_endpoint"], + params={ + "response_type": "code", + "client_id": registered.json()["client_id"], + "redirect_uri": redirect_uri, + "state": "mcp-state", + "code_challenge": _s256("v" * 43), + "code_challenge_method": "S256", + "resource": root.json()["resource"], + }, + follow_redirects=False, + ) + assert response.status_code == 303 + target = urlparse(response.headers["location"]) + assert target.path == ("/ui/connect" if signed_in else "/sso/key/generate") + if signed_in: + flow = parse_qs(target.query)["connect_flow"][0] + described = client.get("/authorize/flow", params={"flow": flow}) + assert described.status_code == 200 + assert described.json()["state"] == "unscoped" + assert minted == [] + + +@pytest.mark.parametrize("valid_client", [True, False]) +def test_mcp_session_authorize_rejects_invalid_registration_or_pkce(monkeypatch, valid_client): + client, session_cookie, minted = _native_client_app(monkeypatch) + redirect_uri = "https://client.example.com/callback" + registered = client.post("/register", json={"redirect_uris": [redirect_uri]}) + assert registered.status_code == 201 + client.cookies.set("token", session_cookie) + response = client.get( + "/authorize/mcp-session", + params={ + "client_id": registered.json()["client_id"] if valid_client else "unknown-client", + "redirect_uri": redirect_uri, + "response_type": "code", + }, + follow_redirects=False, + ) + assert response.status_code == 400 + assert response.json()["error"] == ("invalid_request" if valid_client else "invalid_client") + assert "location" not in response.headers + assert minted == [] + + def test_native_client_login_walks_discovery_consent_token_refresh_and_revoke(monkeypatch): """The whole ``lite login --pkce`` server side over the real router: a Go CLI reads the versioned discovery document, registers a loopback public client, the signed-in user consents to a team, @@ -11186,3 +11278,99 @@ async def test_dcr_refusal_is_actionable_without_upstream_body( assert f"HTTP {upstream_status}" in str(exc.value.detail) assert "pre-registered OAuth client" in str(exc.value.detail) assert "private upstream details" not in str(exc.value.detail) + + +@pytest.mark.parametrize("prefix", ["", "/tenant-a", "/tenant-b"]) +@pytest.mark.parametrize( + ("server_name", "pattern"), + [ + ("issuer_test", "mcp/{server}"), + ("issuer_test", "{server}/mcp"), + ("issuer_test", "{server}"), + ("mcp", "mcp/{server}"), + ("mcp", "{server}/mcp"), + ], +) +def test_per_server_authorization_metadata_issuer_matches_discovery_path( + _no_proxy_base_url, _isolated_mcp_registry, prefix, server_name, pattern +): + server = _create_oauth2_server(server_id=server_name, name=server_name, server_name=server_name, alias=server_name) + _isolated_mcp_registry[server.server_id] = server + client = _prefixed_discovery_client(["/tenant-a", "/tenant-b"]) + path = pattern.format(server=server_name) + response = client.get(f"{prefix}/.well-known/oauth-authorization-server/{path}") + assert response.status_code == 200 + metadata = response.json() + assert metadata["issuer"] == f"http://testserver{prefix}/{path}" + assert metadata["authorization_endpoint"] == f"http://testserver{prefix}/{server_name}/authorize" + assert metadata["token_endpoint"] == f"http://testserver{prefix}/{server_name}/token" + assert metadata["registration_endpoint"] == f"http://testserver{prefix}/{server_name}/register" + + +@pytest.mark.parametrize("prefix", ["", "/tenant-a"]) +@pytest.mark.parametrize("relay", [False, True]) +@pytest.mark.parametrize("pattern", ["mcp/{server}", "{server}/mcp"]) +def test_named_resource_discovery_follows_matching_authorization_issuer( + _no_proxy_base_url, _isolated_mcp_registry, prefix, relay, pattern +): + server = _create_oauth2_server().model_copy(update={"per_server_oauth_discovery": relay}) + _isolated_mcp_registry[server.server_id] = server + client = _prefixed_discovery_client(["/tenant-a"]) + path = pattern.format(server=server.server_name) + response = client.get(f"{prefix}/.well-known/oauth-protected-resource/{path}") + assert response.status_code == 200 + resource = response.json() + issuer_path = server.server_name if relay else "mcp" + assert resource["resource"] == f"http://testserver{prefix}/{path}" + assert resource["authorization_servers"] == [f"http://testserver{prefix}/{issuer_path}"] + authorization = client.get(f"{prefix}/.well-known/oauth-authorization-server/{issuer_path}") + assert authorization.status_code == 200 + assert authorization.json()["issuer"] == resource["authorization_servers"][0] + + +def test_static_root_path_authorization_discovery_preserves_issuer(monkeypatch, tmp_path): + import subprocess + import sys + + monkeypatch.setenv("SERVER_ROOT_PATH", "/gateway") + monkeypatch.setenv("PROXY_BASE_URL", "http://testserver/gateway") + monkeypatch.setenv("LITELLM_UI_PATH", str(tmp_path / "ui")) + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import json +from fastapi import FastAPI +from fastapi.testclient import TestClient +from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router +from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager +from litellm.types.mcp import MCPAuth, MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer + +global_mcp_server_manager.registry['example'] = MCPServer( + server_id='example', name='example', server_name='example', alias='example', + transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + authorization_url='https://idp.example.com/authorize', token_url='https://idp.example.com/token', +) +app = FastAPI(root_path='/gateway') +app.include_router(router) +with TestClient(app) as client: + responses = { + path: client.get('/.well-known/oauth-authorization-server/gateway/' + path) + for path in ('mcp/example', 'example/mcp', 'example', 'mcp') + } + print(json.dumps({path: {'status': response.status_code, 'body': response.json()} + for path, response in responses.items()})) +""", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ) + responses = json.loads(result.stdout) + for path in ("mcp/example", "example/mcp", "example", "mcp"): + assert responses[path]["status"] == 200, responses[path] + assert responses[path]["body"]["issuer"] == f"http://testserver/gateway/{path}" + assert responses["example/mcp"]["body"]["token_endpoint"] == "http://testserver/gateway/example/token" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py index 56851d31241..67948375403 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py @@ -1233,13 +1233,14 @@ class TestResolveByokMcpAuthHeader: assert result == "stored-cred" @pytest.mark.asyncio - async def test_byok_server_raises_401_when_no_credential_stored(self): + async def test_byok_server_raises_401_when_no_credential_stored(self, monkeypatch): from fastapi import HTTPException from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( _resolve_byok_mcp_auth_header, ) + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") server = self._server(is_byok=True) user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard") @@ -1252,6 +1253,9 @@ class TestResolveByokMcpAuthHeader: assert exc_info.value.status_code == 401 assert exc_info.value.detail["error"] == "byok_auth_required" + assert exc_info.value.headers == { + "WWW-Authenticate": 'Bearer resource_metadata="https://gateway.example.com/proxy/v1/mcp/oauth/protected-resource"' + } @pytest.mark.asyncio async def test_byok_server_checks_credential_and_keeps_caller_header_when_supplied(self): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index b637586d7ff..d2987c5112e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -30,7 +30,7 @@ from mcp.types import ( TextResourceContents, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl +from pydantic import AnyUrl, TypeAdapter from litellm.constants import MCP_METADATA_TIMEOUT from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( @@ -3708,6 +3708,7 @@ class TestMCPServerManager: mock_prompt = Prompt(name="hello", description="Say hi") mock_client = AsyncMock() mock_client.list_prompts = AsyncMock(return_value=[mock_prompt]) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") with patch.object( manager, @@ -3779,6 +3780,7 @@ class TestMCPServerManager: mock_client = AsyncMock() mock_resources = [Resource(name="file", uri="https://example.com/file")] mock_client.list_resources = AsyncMock(return_value=mock_resources) + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") prefixed_resources = [Resource(name="alias-server-file", uri="https://example.com/file")] with ( @@ -3788,11 +3790,6 @@ class TestMCPServerManager: new_callable=AsyncMock, return_value=mock_client, ) as mock_create_client, - patch.object( - manager, - "_create_prefixed_resources", - return_value=prefixed_resources, - ) as mock_prefix, ): result = await manager.get_resources_from_server( server=server, @@ -3808,7 +3805,6 @@ class TestMCPServerManager: assert called_kwargs["mcp_auth_header"] == "auth" assert called_kwargs["extra_headers"] == {"X-Test": "1", "X-Static": "static"} mock_client.list_resources.assert_awaited_once() - mock_prefix.assert_called_once_with(mock_resources, server, add_prefix=True) assert result == prefixed_resources @pytest.mark.asyncio @@ -3832,9 +3828,10 @@ class TestMCPServerManager: ) ] mock_client.list_resource_templates = AsyncMock(return_value=mock_templates) - prefixed_templates = [ + mock_client.discovery_auth_fingerprint = AsyncMock(return_value="test-credential-hash") + expected_templates = [ ResourceTemplate( - name="alias-server-template", + name="template", uriTemplate="https://example.com/{id}", ) ] @@ -3846,11 +3843,6 @@ class TestMCPServerManager: new_callable=AsyncMock, return_value=mock_client, ) as mock_create_client, - patch.object( - manager, - "_create_prefixed_resource_templates", - return_value=prefixed_templates, - ) as mock_prefix, ): result = await manager.get_resource_templates_from_server( server=server, @@ -3866,10 +3858,10 @@ class TestMCPServerManager: extra_headers=None, stdio_env=None, subject_token=None, + user_api_key_auth=None, ) mock_client.list_resource_templates.assert_awaited_once() - mock_prefix.assert_called_once_with(mock_templates, server, add_prefix=False) - assert result == prefixed_templates + assert result == expected_templates @pytest.mark.asyncio async def test_read_resource_from_server_success(self): @@ -13020,3 +13012,436 @@ async def test_openapi_health_cancellation_does_not_poison_cache(respx_mock, mon assert cached.status == "healthy" assert len(attempts) == 2 assert route.call_count == 1 + + +class _DiscoveryClock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +class _DiscoveryUpstream: + def __init__(self) -> None: + self.requests: tuple[tuple[str, str], ...] = () + self.outcome = "supported" + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.release.set() + + async def respond(self, request: httpx.Request) -> httpx.Response: + from mcp.types import JSONRPCMessage, JSONRPCRequest + + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = JSONRPCMessage.model_validate_json(request.content).root + if not isinstance(payload, JSONRPCRequest): + return httpx.Response(202) + self.requests = (*self.requests, (payload.method, request.headers.get("authorization", ""))) + if payload.method == "initialize": + return httpx.Response(200, json={ + "jsonrpc": "2.0", "id": payload.id, + "result": {"protocolVersion": "2025-03-26", "serverInfo": {"name": "discovery", "version": "1"}, + "capabilities": {} if self.outcome == "unsupported" else {"prompts": {}, "resources": {}}}, + }) + self.entered.set() + await self.release.wait() + if self.outcome == "failure": + return httpx.Response(503) + if self.outcome == "cancelled": + raise asyncio.CancelledError() + if self.outcome == "rejected": + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload.id, + "error": {"code": -32601, "message": "Unsupported"}}) + result: Final = { + "prompts/list": {"prompts": [{"name": "example", "description": "original"}]}, + "resources/list": {"resources": [{"name": "example", "uri": "test://example", "description": "original"}]}, + "resources/templates/list": {"resourceTemplates": [{"name": "example", "uriTemplate": "test://{name}", "description": "original"}]}, + "tools/list": {"tools": []}, + }[payload.method] + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": result}) + + @property + def initializes(self) -> int: + return sum(method == "initialize" for method, _auth in self.requests) + + +def _discovery_server() -> MCPServer: + return MCPServer(server_id="discovery", name="discovery", url="https://discovery.example/mcp", transport=MCPTransport.http) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ("prompts", "resources", "templates")) +async def test_discovery_cache_reuses_raw_results_and_expires(kind: str) -> None: + import respx + + clock: Final = _DiscoveryClock() + manager: Final = MCPServerManager(discovery_clock=clock) + upstream: Final = _DiscoveryUpstream() + operation: Final = {"prompts": manager.get_prompts_from_server, "resources": manager.get_resources_from_server, + "templates": manager.get_resource_templates_from_server}[kind] + server: Final = _discovery_server() + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + first: Final = await operation(server, None) + assert len(first) == 1 + assert first[0].name == "discovery-example" + first[0].description = "caller changed it" + second: Final = await operation(server, None, add_prefix=False) + assert second[0].name == "example" + assert second[0].description == "original" + assert upstream.initializes == 1 + clock.now = 59.999 + assert (await operation(server, None))[0].name == "discovery-example" + assert upstream.initializes == 1 + clock.now = 60.001 + assert (await operation(server, None))[0].name == "discovery-example" + assert upstream.initializes == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ("prompts", "resources", "templates")) +@pytest.mark.parametrize("outcome", ("unsupported", "rejected", "failure")) +async def test_discovery_cache_empty_results_and_failures(kind: str, outcome: str) -> None: + import respx + + manager: Final = MCPServerManager() + upstream: Final = _DiscoveryUpstream() + upstream.outcome = outcome + operation: Final = {"prompts": manager.get_prompts_from_server, "resources": manager.get_resources_from_server, + "templates": manager.get_resource_templates_from_server}[kind] + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + assert await operation(_discovery_server(), None) == [] + assert await operation(_discovery_server(), None) == [] + assert upstream.initializes == (2 if outcome == "failure" else 1) + if outcome == "failure": + upstream.outcome = "supported" + assert (await operation(_discovery_server(), None))[0].name == "discovery-example" + assert upstream.initializes == 3 + + +@pytest.mark.asyncio +async def test_discovery_cache_isolates_forwarded_credentials_and_shares_static_auth() -> None: + import respx + + manager: Final = MCPServerManager() + upstream: Final = _DiscoveryUpstream() + server: Final = _discovery_server() + first_user: Final = UserAPIKeyAuth(user_id="first") + second_user: Final = UserAPIKeyAuth(user_id="second") + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + for user in (first_user, second_user): + assert len(await manager.get_prompts_from_server(server, user)) == 1 + assert upstream.initializes == 1 + for credential in ("first-secret", "second-secret", "first-secret"): + assert len(await manager.get_prompts_from_server(server, first_user, extra_headers={"Authorization": credential})) == 1 + assert upstream.initializes == 3 + assert {auth for method, auth in upstream.requests if method == "prompts/list"} == {"", "first-secret", "second-secret"} + + +@pytest.mark.asyncio +async def test_discovery_cache_coalesces_and_survives_waiter_cancellation() -> None: + import respx + + manager: Final = MCPServerManager() + upstream: Final = _DiscoveryUpstream() + upstream.release.clear() + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + tasks: Final = tuple(asyncio.create_task(manager.get_prompts_from_server(_discovery_server(), None)) for _ in range(10)) + await asyncio.wait_for(upstream.entered.wait(), timeout=5) + tasks[0].cancel() + with pytest.raises(asyncio.CancelledError): + await tasks[0] + upstream.release.set() + results: Final = await asyncio.wait_for(asyncio.gather(*tasks[1:]), timeout=5) + assert all(result[0].name == "discovery-example" for result in results) + assert upstream.initializes == 1 + assert results[0][0] is not results[1][0] + assert (await manager.get_prompts_from_server(_discovery_server(), None))[0].name == "discovery-example" + assert upstream.initializes == 1 + + +@pytest.mark.asyncio +async def test_discovery_cache_invalidation_during_fetch_does_not_repopulate_old_results() -> None: + import respx + + manager: Final = MCPServerManager() + upstream: Final = _DiscoveryUpstream() + upstream.release.clear() + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + task: Final = asyncio.create_task(manager.get_prompts_from_server(_discovery_server(), None)) + await asyncio.wait_for(upstream.entered.wait(), timeout=5) + manager._invalidate_discovery_lists("discovery") + upstream.release.set() + assert (await task)[0].name == "discovery-example" + assert len(await manager.get_prompts_from_server(_discovery_server(), None)) == 1 + assert upstream.initializes == 2 + manager._invalidate_discovery_lists("discovery") + assert len(await manager.get_prompts_from_server(_discovery_server(), None)) == 1 + assert upstream.initializes == 3 + + +@pytest.mark.asyncio +async def test_discovery_cache_can_be_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + import respx + + monkeypatch.setenv("LITELLM_MCP_DISCOVERY_CACHE_TTL", "0") + manager: Final = MCPServerManager() + upstream: Final = _DiscoveryUpstream() + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + assert len(await manager.get_prompts_from_server(_discovery_server(), None)) == 1 + assert len(await manager.get_prompts_from_server(_discovery_server(), None)) == 1 + assert upstream.initializes == 2 + + +@pytest.mark.parametrize("value,expected", (("invalid", 60.0), ("nan", 60.0), ("inf", 60.0), ("-1", 60.0), ("12.5", 12.5))) +def test_discovery_cache_ttl_validation(value: str, expected: float, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _mcp_discovery_cache_ttl + + monkeypatch.setenv("LITELLM_MCP_DISCOVERY_CACHE_TTL", value) + assert _mcp_discovery_cache_ttl() == expected + + +@pytest.mark.parametrize("auth_type", (MCPAuth.oauth2, MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag)) +def test_discovery_cache_keys_isolate_user_dependent_auth(auth_type: MCPAuth) -> None: + manager: Final = MCPServerManager() + server: Final = _discovery_server().model_copy(update={"auth_type": auth_type}) + first: Final = manager._discovery_key(server, UserAPIKeyAuth(user_id="first"), None, None, None, None) + second: Final = manager._discovery_key(server, UserAPIKeyAuth(user_id="second"), None, None, None, None) + anonymous: Final = manager._discovery_key(server, None, None, None, None, None) + assert len({first, second, anonymous}) == 3 + assert "first" not in str(first) + assert "second" not in str(second) + + +@pytest.mark.asyncio +async def test_discovery_cache_retries_cancelled_fetches() -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + + async def cancelled() -> list[Prompt]: + raise asyncio.CancelledError() + + async def supported() -> list[Prompt]: + return [Prompt(name="recovered")] + + with pytest.raises(asyncio.CancelledError): + await cache.get(("server", None), cancelled) + assert [item.name for item in await cache.get(("server", None), supported)] == ["recovered"] + + +@pytest.mark.asyncio +async def test_discovery_cache_cancels_fetch_when_last_waiter_leaves() -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + entered: Final = asyncio.Event() + stopped: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def fetch() -> list[Prompt]: + entered.set() + try: + await release.wait() + return [Prompt(name="result")] + finally: + stopped.set() + + tasks: Final = tuple(asyncio.create_task(cache.get(("server", None), fetch)) for _ in range(3)) + await asyncio.wait_for(entered.wait(), timeout=5) + for task in tasks: + task.cancel() + outcomes: Final = await asyncio.gather(*tasks, return_exceptions=True) + assert all(isinstance(outcome, asyncio.CancelledError) for outcome in outcomes) + try: + await asyncio.wait_for(stopped.wait(), timeout=1) + finally: + release.set() + + +@pytest.mark.asyncio +async def test_discovery_cache_bounds_detached_fetches_without_dropping_results() -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + entered: Final[asyncio.Queue[None]] = asyncio.Queue() + release: Final = asyncio.Event() + + async def blocked() -> list[Prompt]: + await entered.put(None) + await release.wait() + return [Prompt(name="blocked")] + + tasks: Final = tuple(asyncio.create_task(cache.get((str(index), None), blocked)) for index in range(1024)) + try: + for _ in tasks: + await asyncio.wait_for(entered.get(), timeout=5) + active_tasks: Final = frozenset(asyncio.all_tasks()) + + async def overflow() -> list[Prompt]: + assert frozenset(asyncio.all_tasks()) <= active_tasks + return [Prompt(name="overflow")] + + result: Final = await cache.get(("overflow", None), overflow) + assert [item.name for item in result] == ["overflow"] + finally: + release.set() + outcomes: Final = await asyncio.gather(*tasks) + assert all(result[0].name == "blocked" for result in outcomes) + + +@pytest.mark.asyncio +async def test_discovery_cache_tracks_resolved_credentials_across_workers() -> None: + import respx + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import UpstreamCredentialProvider + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result + from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject + + class CredentialSource(UpstreamCredentialProvider): + def __init__(self) -> None: + super().__init__() + self.token: str | None = "token-a" + + async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]: + if self.token is None: + return Error(CredError.of_unauthorized("Credential revoked")) + return Ok(StaticHeaderAuth("Bearer " + self.token)) + + source: Final = CredentialSource() + managers: Final = (MCPServerManager(cred_provider=source), MCPServerManager(cred_provider=source)) + server: Final = MCPServer( + server_id="discovery", name="discovery", url="https://discovery.example/mcp", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="discovery-client", + authorization_url="https://discovery.example/authorize", token_url="https://discovery.example/token", + ) + user: Final = UserAPIKeyAuth(user_id="same-user", api_key="same-key") + upstream: Final = _DiscoveryUpstream() + + async def respond(request: httpx.Request) -> httpx.Response: + response: Final = await upstream.respond(request) + if '"prompts/list"' not in request.content.decode(): + return response + from mcp.types import JSONRPCMessage, JSONRPCRequest + + payload: Final = JSONRPCMessage.model_validate_json(request.content).root + assert isinstance(payload, JSONRPCRequest) + name: Final = {"Bearer token-a": "account-a", "Bearer token-b": "account-b"}[request.headers["authorization"]] + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": {"prompts": [{"name": name}]}}) + + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=respond) + for manager in managers: + assert [item.name for item in await manager.get_prompts_from_server(server, user)] == ["discovery-account-a"] + assert upstream.initializes == 2 + source.token = "token-b" + for manager in managers: + assert [item.name for item in await manager.get_prompts_from_server(server, user)] == ["discovery-account-b"] + assert upstream.initializes == 4 + source.token = None + for manager in managers: + assert await manager.get_prompts_from_server(server, user) == [] + assert upstream.initializes == 4 + + +@pytest.mark.asyncio +async def test_discovery_resolves_stored_oauth_for_the_requesting_user() -> None: + import respx + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken + + class TokenStore: + def __init__(self) -> None: + self.calls: tuple[tuple[str, str], ...] = () + + async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + self.calls = (*self.calls, (user_id, server_id)) + return OAuthToken(access_token="stored-token") + + async def invalidate(self, user_id: str, server_id: str) -> None: + return None + + store: Final = TokenStore() + manager: Final = MCPServerManager(per_user_oauth_token_store=store) + server: Final = MCPServer( + server_id="discovery", name="discovery", url="https://discovery.example/mcp", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="discovery-client", + authorization_url="https://discovery.example/authorize", token_url="https://discovery.example/token", + ) + user: Final = UserAPIKeyAuth(user_id="requesting-user") + upstream: Final = _DiscoveryUpstream() + with respx.mock(base_url="https://discovery.example") as router: + router.route().mock(side_effect=upstream.respond) + assert len(await manager.get_prompts_from_server(server, user)) == 1 + assert len(await manager.get_prompts_from_server(server, user)) == 1 + assert store.calls == (("requesting-user", "discovery"), ("requesting-user", "discovery")) + assert upstream.initializes == 1 + assert ("prompts/list", "Bearer stored-token") in upstream.requests + + +@pytest.mark.asyncio +async def test_discovery_cache_evicts_results_at_capacity() -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + + async def original() -> list[Prompt]: + return [Prompt(name="original")] + + async def refetched() -> list[Prompt]: + return [Prompt(name="refetched")] + + for index in range(1025): + assert (await cache.get((f"server-{index:04}", None), original))[0].name == "original" + assert (await cache.get(("server-1024", None), refetched))[0].name == "original" + assert (await cache.get(("server-0000", None), refetched))[0].name == "refetched" + + +@pytest.mark.asyncio +async def test_discovery_cache_invalidation_preserves_other_servers_and_pending_fetches() -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + entered: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def original() -> list[Prompt]: + return [Prompt(name="original")] + + async def blocked() -> list[Prompt]: + entered.set() + await release.wait() + return [Prompt(name="pending")] + + async def refetched() -> list[Prompt]: + return [Prompt(name="refetched")] + + assert (await cache.get(("server", None), original))[0].name == "original" + assert (await cache.get(("server-extra", None), original))[0].name == "original" + task: Final = asyncio.create_task(cache.get(("other", None), blocked)) + await asyncio.wait_for(entered.wait(), timeout=5) + cache.invalidate("server") + release.set() + assert (await asyncio.wait_for(task, timeout=5))[0].name == "pending" + assert (await cache.get(("other", None), refetched))[0].name == "pending" + assert (await cache.get(("server-extra", None), refetched))[0].name == "original" + assert (await cache.get(("server", None), refetched))[0].name == "refetched" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("description", ("x" * 96_000, "é" * 40_000), ids=("ascii", "unicode")) +async def test_discovery_cache_returns_oversized_results_without_retaining_them(description: str) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _DiscoveryCache + + cache: Final = _DiscoveryCache[Prompt](60, _DiscoveryClock(), TypeAdapter(tuple[Prompt, ...])) + fetch: Final = AsyncMock(return_value=[Prompt(name="large", description=description)]) + for _ in range(2): + result: Final = await cache.get(("server", None), fetch) + assert result[0].description == description + assert fetch.await_count == 2 diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8777e24e209..dc614d18662 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -1,7 +1,7 @@ import asyncio import json from types import SimpleNamespace -from typing import TYPE_CHECKING, Final, Optional +from typing import TYPE_CHECKING, Final, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch if TYPE_CHECKING: @@ -29,6 +29,7 @@ from litellm.proxy._types import ( ProxyException, SSOUserDefinedValues, UserAPIKeyAuth, + WebhookEvent, ) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, @@ -53,6 +54,7 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.constants import ( DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, @@ -5431,6 +5433,9 @@ async def test_common_checks_personal_user_budget_blocks_in_gather(): async def _spend_by_counter(counter_key, fallback_spend, max_budget=None, **kwargs): return 999.0 if counter_key == "spend:user:u1" else 0.0 + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + with ( patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), @@ -5445,13 +5450,136 @@ async def test_common_checks_personal_user_budget_blocks_in_gather(): general_settings={}, route="/chat/completions", llm_router=None, - proxy_logging_obj=MagicMock(), + proxy_logging_obj=proxy_logging_obj, valid_token=token, request=MagicMock(spec=Request), ) + await asyncio.sleep(0) assert "User=u1" in str(over.value) +async def _run_internal_user_budget_alert( + *, + spend: float, +) -> tuple[AsyncMock, litellm.BudgetExceededError | None]: + from fastapi import Request + + from litellm.proxy.auth.auth_checks import common_checks + + user: Final = LiteLLM_UserTable( + user_id="user-1", + user_email="person@example.com", + spend=0.0, + max_budget=100.0, + ) + token: Final = UserAPIKeyAuth(token="hashed-key-1", user_id="user-1") + slack_alerting: Final = SlackAlerting(alerting=["webhook"]) + send_alert: Final = AsyncMock() + alert_finished: Final = asyncio.Event() + + async def _get_spend( + counter_key: str, + fallback_spend: float, + max_budget: float | None = None, + **kwargs: object, + ) -> float: + assert counter_key == "spend:user:user-1" + assert fallback_spend == 0.0 + assert max_budget == 100.0 + return spend + + async def _budget_alerts( + *, + type: Literal["user_budget"], + user_info: CallInfo, + ) -> None: + assert type == "user_budget" + try: + await slack_alerting.budget_alerts(type=type, user_info=user_info) + finally: + alert_finished.set() + + proxy_logging_obj: Final = MagicMock(budget_alerts=_budget_alerts) + + async def _check() -> bool: + return await common_checks( + request_body={"messages": [{"role": "user", "content": "hi"}]}, + team_object=None, + user_object=user, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=proxy_logging_obj, + valid_token=token, + request=MagicMock(spec=Request), + ) + + async def _check_for_error() -> litellm.BudgetExceededError | None: + if spend < 100.0: + assert await _check() is True + return None + + with pytest.raises(litellm.BudgetExceededError) as raised: + await _check() + return raised.value + + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), # test-quality-ok: common_checks has no database seam + patch("litellm.proxy.proxy_server.get_current_spend", _get_spend), # test-quality-ok: common_checks imports it locally + patch.object(slack_alerting, "send_alert", send_alert), + ): + error: Final = await _check_for_error() + await asyncio.wait_for(alert_finished.wait(), timeout=1.0) + + return send_alert, error + + +@pytest.mark.asyncio +async def test_common_checks_internal_user_budget_below_threshold_does_not_emit_alert(): + send_alert, error = await _run_internal_user_budget_alert(spend=84.0) + + assert error is None + send_alert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_common_checks_internal_user_budget_emits_user_threshold_event(): + send_alert, error = await _run_internal_user_budget_alert(spend=85.0) + + assert error is None + send_alert.assert_awaited_once() + event: Final = send_alert.await_args.kwargs["user_info"] + assert isinstance(event, WebhookEvent) + assert event.event == "threshold_crossed" + assert event.event_group == Litellm_EntityType.USER + assert event.user_id == "user-1" + assert event.user_email == "person@example.com" + assert event.spend == 85.0 + assert event.max_budget == 100.0 + assert event.token is None + assert event.key_alias is None + assert event.team_id is None + assert event.organization_id is None + + +@pytest.mark.asyncio +async def test_common_checks_internal_user_budget_emits_crossed_event_and_rejects(): + send_alert, error = await _run_internal_user_budget_alert(spend=100.0) + + assert error is not None + assert error.current_cost == 100.0 + assert error.max_budget == 100.0 + send_alert.assert_awaited_once() + event: Final = send_alert.await_args.kwargs["user_info"] + assert isinstance(event, WebhookEvent) + assert event.event == "budget_crossed" + assert event.event_group == Litellm_EntityType.USER + assert event.user_id == "user-1" + assert event.user_email == "person@example.com" + + @pytest.mark.asyncio async def test_common_checks_personal_user_budget_skipped_for_team_key(): """A user's personal max_budget does not apply to a team-scoped key. @@ -5475,6 +5603,9 @@ async def test_common_checks_personal_user_budget_skipped_for_team_key(): async def _no_membership(*args, **kwargs): return None + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + with ( patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), @@ -5489,11 +5620,12 @@ async def test_common_checks_personal_user_budget_skipped_for_team_key(): general_settings={}, route="/chat/completions", llm_router=None, - proxy_logging_obj=MagicMock(), + proxy_logging_obj=proxy_logging_obj, valid_token=token, request=MagicMock(spec=Request), ) assert result is True + proxy_logging_obj.budget_alerts.assert_not_awaited() @pytest.mark.asyncio @@ -5518,6 +5650,9 @@ async def test_common_checks_personal_user_budget_enforced_on_team_key_when_flag async def _no_membership(*args, **kwargs): return None + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + with ( patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), @@ -5533,10 +5668,11 @@ async def test_common_checks_personal_user_budget_enforced_on_team_key_when_flag general_settings={"apply_user_budget_to_team_keys": True}, route="/chat/completions", llm_router=None, - proxy_logging_obj=MagicMock(), + proxy_logging_obj=proxy_logging_obj, valid_token=token, request=MagicMock(spec=Request), ) + await asyncio.sleep(0) assert "ExceededBudget: User=u1" in str(exc_info.value) @@ -5553,6 +5689,9 @@ async def test_common_checks_personal_user_budget_still_enforced_on_personal_key async def _spend_by_counter(counter_key, fallback_spend, max_budget=None, **kwargs): return 999.0 if counter_key == "spend:user:u1" else 0.0 + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + with ( patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), @@ -5567,10 +5706,11 @@ async def test_common_checks_personal_user_budget_still_enforced_on_personal_key general_settings={"apply_user_budget_to_team_keys": True}, route="/chat/completions", llm_router=None, - proxy_logging_obj=MagicMock(), + proxy_logging_obj=proxy_logging_obj, valid_token=token, request=MagicMock(spec=Request), ) + await asyncio.sleep(0) @pytest.mark.parametrize( diff --git a/tests/test_litellm/proxy/common_utils/test_debug_utils.py b/tests/test_litellm/proxy/common_utils/test_debug_utils.py new file mode 100644 index 00000000000..163ea530be9 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_debug_utils.py @@ -0,0 +1,69 @@ +import os +import socket +from pathlib import Path + +import pytest + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.debug_utils import ( + PSUTIL_MISSING_ERROR, + _ProcFilesystemProcess, + _summary_process_memory, + get_memory_summary, +) + +PAGE_SIZE = 4096 +STATM_SIZE_PAGES = 100_000 +STATM_RESIDENT_PAGES = 30_000 +MEMINFO_TOTAL_KB = 1_000_000 + + +@pytest.fixture +def proc_process(tmp_path: Path) -> _ProcFilesystemProcess: + statm = tmp_path / "statm" + statm.write_text(f"{STATM_SIZE_PAGES} {STATM_RESIDENT_PAGES} 5000 1 0 20000 0\n") + meminfo = tmp_path / "meminfo" + meminfo.write_text( + f"MemTotal: {MEMINFO_TOTAL_KB} kB\nMemFree: 400000 kB\nMemAvailable: 600000 kB\n" + ) + return _ProcFilesystemProcess(statm_path=str(statm), meminfo_path=str(meminfo), page_size=PAGE_SIZE) + + +def test_proc_filesystem_process_reads_resident_and_virtual_bytes_from_statm( + proc_process: _ProcFilesystemProcess, +) -> None: + memory_info = proc_process.memory_info() + + assert memory_info.rss == STATM_RESIDENT_PAGES * PAGE_SIZE + assert memory_info.vms == STATM_SIZE_PAGES * PAGE_SIZE + + +def test_proc_filesystem_process_reports_share_of_meminfo_total(proc_process: _ProcFilesystemProcess) -> None: + expected_percent = STATM_RESIDENT_PAGES * PAGE_SIZE / (MEMINFO_TOTAL_KB * 1024) * 100 + + assert proc_process.memory_percent() == pytest.approx(expected_percent) + + +def test_summary_reports_rss_from_the_proc_filesystem(proc_process: _ProcFilesystemProcess) -> None: + memory, health_status = _summary_process_memory(proc_process) + + assert memory["ram_usage_mb"] == round(STATM_RESIDENT_PAGES * PAGE_SIZE / (1024 * 1024), 2) + assert memory["system_memory_percent"] == pytest.approx(12.0) + assert health_status == "healthy" + assert "error" not in memory + + +def test_summary_without_any_memory_source_names_psutil_and_reports_no_rss() -> None: + memory, health_status = _summary_process_memory(None) + + assert memory == {"error": PSUTIL_MISSING_ERROR} + assert health_status == "healthy" + + +@pytest.mark.asyncio +async def test_memory_summary_names_the_host_and_worker_that_answered() -> None: + summary = await get_memory_summary(UserAPIKeyAuth()) + + assert summary["hostname"] == socket.gethostname() + assert summary["worker_pid"] == os.getpid() + assert summary["memory"]["ram_usage_mb"] > 0 diff --git a/tests/test_litellm/proxy/db/test_db_url_settings.py b/tests/test_litellm/proxy/db/test_db_url_settings.py index 875dca4bee3..f5fb1bda0c1 100644 --- a/tests/test_litellm/proxy/db/test_db_url_settings.py +++ b/tests/test_litellm/proxy/db/test_db_url_settings.py @@ -34,10 +34,12 @@ from pydantic import ValidationError from litellm.proxy.db.db_url_settings import ( PG_SSL_REQUEST, DatabaseURLSettings, + token_refresh_params_from_url, translate_libpq_ssl_params, unsupported_db_scheme, unsupported_db_scheme_message, ) +from litellm.proxy.db.pgbouncer import PgBouncerPlan, PgBouncerSettings, plan_pgbouncer from litellm.proxy.db.token_auth import AzureEntraTokenAuth, RdsIamTokenAuth @@ -51,6 +53,8 @@ _MANAGED_DB_ENV_VARS = ( "AZURE_POSTGRESQL_AUTH", "DATABASE_DISABLE_PREPARED_STATEMENTS", "DATABASE_MAX_IDLE_CONNECTION_LIFETIME", + "DATABASE_SSLMODE", + "DATABASE_SSLROOTCERT", "DATABASE_URL", "DIRECT_URL", "DATABASE_URL_READ_REPLICA", @@ -781,6 +785,109 @@ def test_libpq_verify_full_and_sslrootcert_become_prisma_strict_sslcert(monkeypa } +def _tls_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATABASE_HOST", "writer.example.com") + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm_db") + monkeypatch.setenv("DATABASE_SSLMODE", "verify-full") + monkeypatch.setenv("DATABASE_SSLROOTCERT", "/certs/rds-bundle.pem") + + +def test_tls_env_vars_make_the_minted_iam_writer_url_verify_the_server(monkeypatch: pytest.MonkeyPatch): + """The supervisor starts PgBouncer from the URL assembled here, before any + config.yaml is read, so an IAM URL with no TLS params leaves PgBouncer on + ``prefer`` (no SNI, no verification) and the RDS handshake fails.""" + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + _tls_env(monkeypatch) + + with _stub_iam_token("WRITER_TOKEN"): + assert _apply() is True + + url: Final = os.environ["DATABASE_URL"] + assert url.startswith("postgresql://litellm:WRITER_TOKEN@writer.example.com:5432/litellm_db?") + assert _query(url) == { + "sslmode": ["require"], + "sslcert": ["/certs/rds-bundle.pem"], + "sslaccept": ["strict"], + "max_idle_connection_lifetime": ["60"], + } + + +def test_tls_env_vars_apply_to_the_password_writer_and_the_assembled_reader(monkeypatch: pytest.MonkeyPatch): + _tls_env(monkeypatch) + monkeypatch.setenv("DATABASE_PASSWORD", "s3cr3t") + monkeypatch.setenv("DATABASE_SCHEMA", "public") + monkeypatch.setenv("DATABASE_HOST_READ_REPLICA", "reader.example.com") + + assert _apply() is True + + expected: Final = { + "schema": ["public"], + "sslmode": ["require"], + "sslcert": ["/certs/rds-bundle.pem"], + "sslaccept": ["strict"], + "max_idle_connection_lifetime": ["60"], + } + assert os.environ["DATABASE_URL"].startswith("postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?") + assert _query(os.environ["DATABASE_URL"]) == expected + assert os.environ["DATABASE_URL_READ_REPLICA"].startswith( + "postgresql://litellm:s3cr3t@reader.example.com:5432/litellm_db?" + ) + assert _query(os.environ["DATABASE_URL_READ_REPLICA"]) == expected + + +def test_sslrootcert_env_var_alone_means_verify_full_for_prisma_and_pgbouncer(monkeypatch: pytest.MonkeyPatch): + """Under libpq's default ``prefer`` a root cert is never consulted, so a URL + carrying only ``sslrootcert`` would leave PgBouncer on ``prefer`` with the CA + loaded but unused. Supplying a CA and nothing else must verify.""" + _tls_env(monkeypatch) + monkeypatch.delenv("DATABASE_SSLMODE") + monkeypatch.setenv("DATABASE_PASSWORD", "s3cr3t") + + assert _apply() is True + + url: Final = os.environ["DATABASE_URL"] + assert _query(url) == { + "sslmode": ["require"], + "sslcert": ["/certs/rds-bundle.pem"], + "sslaccept": ["strict"], + "max_idle_connection_lifetime": ["60"], + } + plan: Final = plan_pgbouncer(url, PgBouncerSettings(enabled=True), Path("/run/pgb"), None) + assert isinstance(plan, PgBouncerPlan), plan + assert "server_tls_sslmode = verify-full" in plan.ini + assert "server_tls_ca_file = /run/pgb/server-ca.pem" in plan.ini + + +def test_tls_env_vars_never_override_a_pinned_database_url(monkeypatch: pytest.MonkeyPatch): + writer: Final = ( + "postgresql://pinned:url@db.example.com:5432/litellm_db?sslmode=disable&max_idle_connection_lifetime=60" + ) + reader: Final = "postgresql://pinned:url@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60" + monkeypatch.setenv("DATABASE_URL", writer) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", reader) + monkeypatch.setenv("DATABASE_HOST_READ_REPLICA", "reader.example.com") + _tls_env(monkeypatch) + + assert _apply() is False + + assert os.environ["DATABASE_URL"] == writer + assert os.environ["DATABASE_URL_READ_REPLICA"] == reader + + +def test_token_refresh_params_keep_the_prisma_tls_dialect_but_not_the_schema(): + kept: Final = token_refresh_params_from_url( + "postgresql://u:TOKEN@db.example.com:5432/litellm_db" + "?schema=tenant&connection_limit=5&sslmode=require&sslcert=/certs/root.pem&sslaccept=strict" + ) + assert dict(kept) == { + "connection_limit": "5", + "sslmode": "require", + "sslcert": "/certs/root.pem", + "sslaccept": "strict", + } + + def _issue_cert( subject: str, issuer: x509.Certificate | None, issuer_key: ec.EllipticCurvePrivateKey | None, ca: bool ) -> tuple[x509.Certificate, ec.EllipticCurvePrivateKey]: diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index 7b5a0bf10f9..bf7df3077ea 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -411,7 +411,7 @@ class TestPgBouncerProcess: port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1, - ready_timeout_seconds=0.3, + ready_timeout_seconds=2.0, ) assert pooler.start() is None first_pid: Final = pooler.pid @@ -421,8 +421,8 @@ class TestPgBouncerProcess: with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: _listening(wrong_port)) - assert _wait_until(lambda: any("did not start listening" in record.message for record in caplog.records)) port_file.write_text(str(port)) + assert _wait_until(lambda: any("did not start listening" in record.message for record in caplog.records)) assert _wait_until(lambda: _listening(port)) assert _wait_until(lambda: not _listening(wrong_port)) pooler.stop() diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py index 963f6a5640f..99e494fccd5 100644 --- a/tests/test_litellm/proxy/db/test_prisma_client.py +++ b/tests/test_litellm/proxy/db/test_prisma_client.py @@ -299,6 +299,10 @@ def test_azure_entra_mint_writes_an_encoded_url_into_the_db_url_env_var(azure_en "connection_limit=20&pgbouncer=true&max_idle_connection_lifetime=45", {"connection_limit": ["20"], "pgbouncer": ["true"], "max_idle_connection_lifetime": ["45"]}, ), + ( + "sslmode=require&sslcert=/certs/root.pem&sslaccept=strict&schema=tenant", + {"sslmode": ["require"], "sslcert": ["/certs/root.pem"], "sslaccept": ["strict"]}, + ), ], ) def test_token_refresh_keeps_the_connection_params_of_the_url_it_replaces( diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py index 966a638f6a4..3bc7e1f02f8 100644 --- a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -2,8 +2,9 @@ import asyncio import logging import os import sys -from typing import Any, Dict -from unittest.mock import AsyncMock, MagicMock, patch +from typing import Any, Dict, Final +from unittest.mock import AsyncMock, MagicMock, call, patch +from urllib.parse import parse_qs, urlsplit import pytest @@ -927,6 +928,47 @@ def test_prisma_client_init_falls_back_to_writer_when_reader_iam_token_fails( ) +def test_prisma_client_init_keeps_reader_tls_params_on_the_minted_iam_url( + monkeypatch: pytest.MonkeyPatch, +): + """The initial reader mint rebuilds the URL from host/port/user/db, so the + Prisma TLS dialect on DATABASE_URL_READ_REPLICA must be carried over or + a verify-only database rejects the reader and reads fall to the writer.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader_user@reader.aurora.local:5432/litellm" + "?schema=tenant&sslmode=require&sslcert=/certs/root.pem&sslaccept=strict", + ) + + prisma_factory: Final = MagicMock(name="Prisma") + fake_prisma_module: Final = MagicMock(Prisma=prisma_factory) + monkeypatch.setitem(sys.modules, "prisma", fake_prisma_module) + + fake_iam_module: Final = MagicMock(generate_iam_auth_token=MagicMock(return_value="READER-TOKEN")) + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_iam_module) + + from litellm.proxy.utils import PrismaClient + + client: Final = PrismaClient( + database_url="postgresql://writer@writer.aurora.local:5432/litellm", + proxy_logging_obj=MagicMock(), + ) + + assert isinstance(client.db, RoutingPrismaWrapper) + reader_url: Final = os.environ["DATABASE_URL_READ_REPLICA"] + assert reader_url.startswith("postgresql://reader_user:READER-TOKEN@reader.aurora.local:5432/litellm?") + assert parse_qs(urlsplit(reader_url).query) == { + "schema": ["tenant"], + "sslmode": ["require"], + "sslcert": ["/certs/root.pem"], + "sslaccept": ["strict"], + } + assert prisma_factory.call_args_list == [call(), call(datasource={"url": reader_url})] + + @pytest.mark.asyncio async def test_connect_degrades_writer_when_reader_available(): """A writer connect failure with a healthy reader must NOT abort proxy diff --git a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py index 0361d5cfe8f..69ec174903b 100644 --- a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py +++ b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py @@ -8,6 +8,7 @@ allowed to run: only when the row is missing or belongs to an older window. from __future__ import annotations import asyncio +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final @@ -15,6 +16,7 @@ from typing import Final import pytest from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import PROXY_DB_LOOKUP_MAX_CONCURRENCY from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed @@ -78,6 +80,39 @@ def _row(window_start: datetime, spend: float) -> SimpleNamespace: return SimpleNamespace(window_start=window_start, spend=spend) +class _PausedSpendTable: + def __init__(self, spend: float) -> None: + self.spend: Final = spend + self.read_started: Final = asyncio.Event() + self.resume_read: Final = asyncio.Event() + + async def find_unique(self, where: Mapping[str, object]) -> SimpleNamespace: + self.read_started.set() + await self.resume_read.wait() + return _row(WINDOW_START, self.spend) + + +async def _reseed_with_paused_table( + table: _PausedSpendTable, cache: DualCache, counter_key: str, window: bool +) -> float | None: + prisma: Final = SimpleNamespace(db=SimpleNamespace(litellm_usertable=table, litellm_budgetwindowspend=table)) + if window: + return await SpendCounterReseed.coalesced_window( + prisma_client=prisma, + spend_counter_cache=cache, + counter_key=counter_key, + entity_type="Team", + entity_id="team-1", + window_duration="1d", + window_start=WINDOW_START, + ) + return await SpendCounterReseed.coalesced( + prisma_client=prisma, + spend_counter_cache=cache, + counter_key=counter_key, + ) + + @pytest.mark.asyncio async def test_window_from_table_reads_row_by_primary_key(): """The lookup must use the table's own entity_type values ("key"), not the @@ -270,6 +305,63 @@ async def test_coalesced_window_seeds_a_cold_counter_from_the_row(): assert prisma.db.litellm_spendlogs.call_count == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("window", [False, True], ids=["primary", "window"]) +@pytest.mark.parametrize("concurrent_spend", [989.01459411, 995.0, 900.0]) +async def test_cold_reseed_does_not_add_database_spend_to_concurrent_cache( + window: bool, + concurrent_spend: float, +) -> None: + cache: Final = DualCache(in_memory_cache=InMemoryCache()) + counter_key: Final = "spend:team:team-1:window:1d" if window else "spend:user:user-1" + db_spend: Final = 989.01459411 + table: Final = _PausedSpendTable(db_spend) + reseed_task: Final = asyncio.create_task(_reseed_with_paused_table(table, cache, counter_key, window)) + + await asyncio.wait_for(table.read_started.wait(), timeout=5) + cache.in_memory_cache.set_cache(key=counter_key, value=concurrent_spend) + table.resume_read.set() + result: Final = await asyncio.wait_for(reseed_task, timeout=5) + + expected: Final = max(db_spend, concurrent_spend) + assert cache.in_memory_cache.get_cache(key=counter_key) == expected + assert result == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("window", [False, True], ids=["primary", "window"]) +@pytest.mark.parametrize("batch", [False, True], ids=["single_increment", "batch_increment"]) +@pytest.mark.parametrize("increment", [5.0, -5.0], ids=["charge", "refund"]) +async def test_cold_reseed_preserves_concurrent_local_increment( + monkeypatch: pytest.MonkeyPatch, window: bool, batch: bool, increment: float +) -> None: + from litellm.proxy import proxy_server + + cache: Final = DualCache(in_memory_cache=InMemoryCache()) + counter_key: Final = ( + f"spend:team:concurrent-{batch}-{increment}:window:1d" + if window + else f"spend:user:concurrent-{batch}-{increment}" + ) + table: Final = _PausedSpendTable(100.0) + monkeypatch.setattr(proxy_server, "spend_counter_cache", cache) + reseed_task: Final = asyncio.create_task(_reseed_with_paused_table(table, cache, counter_key, window)) + await asyncio.wait_for(table.read_started.wait(), timeout=5) + + increment_task: Final = asyncio.create_task( + proxy_server._apply_spend_counter_increments( + pending=(proxy_server._PendingSpendIncrement(counter_key=counter_key, increment=increment),) + ) + if batch + else proxy_server._increment_spend_counter_cache(counter_key=counter_key, increment=increment) + ) + await asyncio.sleep(0) + table.resume_read.set() + await asyncio.wait_for(asyncio.gather(reseed_task, increment_task), timeout=5) + + assert cache.in_memory_cache.get_cache(key=counter_key) == 100.0 + increment + + @pytest.mark.asyncio async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) @@ -304,8 +396,7 @@ async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the @pytest.mark.asyncio async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error(): assert ( - await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") - is None + await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") is None ) assert ( await SpendCounterReseed.end_user_from_db( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py index a07157396df..a9ca13a463d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py @@ -1,3 +1,7 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Final, cast +import json from unittest.mock import patch import httpx @@ -7,6 +11,9 @@ from pydantic import ValidationError import litellm from litellm.exceptions import Timeout +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr import initialize_guardrail from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr.crowdstrike_aidr import ( @@ -1719,3 +1726,165 @@ async def test_streaming_params_from_config_control_output_scan_cadence( handler = _initialize_from_config(mode="post_call", **configured) assert await _guard_calls_for_stream(handler, list("ABCDEFGHIJ")) == expected_calls + + +@asynccontextmanager +async def _guardrail_redacting(secret: str, replacement: str) -> AsyncIterator[CrowdStrikeAIDRHandler]: + def redacted(content: object) -> object: + if isinstance(content, str): + return content.replace(secret, replacement) + if isinstance(content, list): + return [ + {**part, "text": redacted(part["text"])} if isinstance(part, dict) and "text" in part else part + for part in content + ] + return content + + def respond(request: httpx.Request) -> httpx.Response: + sent: Final = json.loads(request.content)["guard_input"]["messages"] + return httpx.Response( + status_code=200, + json={ + "result": { + "blocked": False, + "transformed": True, + "guard_output": { + "messages": [{**message, "content": redacted(message.get("content"))} for message in sent] + }, + }, + }, + request=request, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler: Final = AsyncHTTPHandler() + handler.client = client + yield CrowdStrikeAIDRHandler( + mode="pre_call", + guardrail_name="crowdstrike-aidr-guard", + api_key="pts_crowdstrike_tokenid", + api_base="https://api.crowdstrike.com/aidr/aiguard", + async_handler=handler, + ) + + +class _MessageShapedGuardrail(CustomGuardrail): + """Returns one text per chat message and no ``structured_messages`` rewrite. + + Prompt Security and friends scan messages rather than Responses text parts, + which is the shape that outnumbers the endpoint's own bookkeeping. + """ + + def __init__(self, redacted: str) -> None: + super().__init__(guardrail_name="message-shaped") + self.redacted: Final = redacted + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: str, + logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + messages: Final = inputs.get("structured_messages") or () + return {"texts": [self.redacted for _ in messages]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("case", "instructions", "responses_input"), + [ + ( + "instructions add a system message", + "be terse", + [{"role": "user", "content": [{"type": "input_text", "text": "my ssn is 078-05-1120"}]}], + ), + ( + "tool items add messages that carry no text", + None, + [ + {"role": "user", "content": [{"type": "input_text", "text": "my ssn is 078-05-1120"}]}, + {"type": "function_call", "call_id": "c1", "name": "get_x", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": "42"}, + ], + ), + ], +) +async def test_unalignable_rewrite_is_rejected_never_sent_unredacted( + case: str, + instructions: str | None, + responses_input: list[dict[str, object]], +) -> None: + """An unalignable rewrite must fail the request, not forward the raw prompt. + + Skipping the write-back would hand the model the unredacted text, so a + guardrail could be bypassed by adding ``instructions`` or a tool call. + """ + from litellm.proxy.policy_engine.pipeline_executor import UnappliableRequestRewrite + + data: dict[str, object] = {"model": "gpt-4o", "input": responses_input} + if instructions is not None: + data["instructions"] = instructions + + with pytest.raises(UnappliableRequestRewrite): + await OpenAIResponsesHandler().process_input_messages( + data=data, + guardrail_to_apply=_MessageShapedGuardrail("my ssn is "), + ) + + assert "078-05-1120" in str(responses_input), case + + +@pytest.mark.asyncio +async def test_aligned_rewrite_is_written_back() -> None: + """Matching counts must still redact the input in place.""" + responses_input: list[dict[str, object]] = [ + {"role": "user", "content": [{"type": "input_text", "text": "my ssn is 078-05-1120"}]} + ] + + await OpenAIResponsesHandler().process_input_messages( + data={"model": "gpt-4o", "input": responses_input}, + guardrail_to_apply=_MessageShapedGuardrail("my ssn is "), + ) + + assert cast(list, responses_input[0]["content"])[0]["text"] == "my ssn is " + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("case", "responses_input", "redacted_input"), + [ + ( + "instructions add a system message", + [{"role": "user", "content": [{"type": "input_text", "text": "my ssn is 078-05-1120"}]}], + [{"role": "user", "content": [{"type": "input_text", "text": "my ssn is "}]}], + ), + ( + "tool items sit between two user turns", + [ + {"role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + {"type": "function_call", "call_id": "c1", "name": "get_x", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": "42"}, + {"role": "user", "content": [{"type": "input_text", "text": "my ssn is 078-05-1120"}]}, + ], + [ + {"role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + {"type": "function_call", "call_id": "c1", "name": "get_x", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": "42"}, + {"role": "user", "content": [{"type": "input_text", "text": "my ssn is "}]}, + ], + ), + ], +) +async def test_structured_rewrite_lands_on_shapes_the_flat_path_cannot_align( + case: str, + responses_input: list[dict[str, object]], + redacted_input: list[dict[str, object]], +) -> None: + data: dict[str, object] = {"model": "gpt-5.6", "instructions": "be terse", "input": responses_input} + + async with _guardrail_redacting("078-05-1120", "") as guardrail: + await OpenAIResponsesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert data["input"] == redacted_input, case + assert data["instructions"] == "be terse", case diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index c8e47c0f02a..40c69a2989b 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1,9 +1,12 @@ import asyncio +import copy import json import time -from typing import Final +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from datetime import datetime, timedelta from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -19,6 +22,7 @@ from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 from litellm.models.credentials import CredentialItem from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.router import Router from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, _show_no_redis_warning, @@ -1612,7 +1616,7 @@ async def test_health_endpoint_filters_model_list_by_user_access(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) assert "model_list" in captured, "health_endpoint did not call _perform_health_check_and_save" returned_names = {m["model_name"] for m in captured["model_list"]} @@ -1675,7 +1679,7 @@ async def test_health_endpoint_keeps_full_model_list_for_all_proxy_models(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) returned_names = {m["model_name"] for m in captured["model_list"]} assert returned_names == { @@ -1743,12 +1747,231 @@ async def test_health_endpoint_resolves_all_team_models_to_team_allowlist(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) returned_names = {m["model_name"] for m in captured["model_list"]} assert returned_names == {"model-b"}, f"all-team-models key should health-check the team's models: {returned_names}" +def _router_for(model_list: Sequence[Mapping[str, object]]) -> Router: + return Router(model_list=copy.deepcopy(list(model_list))) + + +_ACCESS_GROUP_MODEL_LIST = [ + { + "model_name": "bedrock-nova", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-bedrock", "access_groups": ["bedrock-group"]}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-openai"}, + }, +] +_ACCESS_GROUP_ROUTER = _router_for(_ACCESS_GROUP_MODEL_LIST) +_TEAM_MODEL_LIST = [ + _ACCESS_GROUP_MODEL_LIST[0], + { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": { + "id": "id-team-b", + "team_id": "team-b", + "team_public_model_name": "bedrock-nova", + "access_groups": ["bedrock-group"], + }, + }, +] +_TEAM_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-bedrock"}, + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} +_ACCESS_GROUP_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-bedrock"}, + {"model": "openai/gpt-5.4-mini", "model_id": "id-openai"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} + + +@contextmanager +def _proxy_health_globals( + llm_model_list: Sequence[Mapping[str, object]], + llm_router: object, + use_background_health_checks: bool = False, + health_check_results: Mapping[str, object] | None = None, +) -> Iterator[None]: + with ( + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_model_list", list(llm_model_list) + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_router", llm_router + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.use_background_health_checks", use_background_health_checks + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.user_model", None + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_results", dict(health_check_results or {}) + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_details", True + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_concurrency", 1 + ), + ): + yield + + +@pytest.mark.asyncio +async def test_health_endpoint_expands_access_group_on_live_path(): + """ + LIT-6907 / gh-28206: a key granted a model access group carries the group + name in user_api_key_dict.models. Matching it as a literal model_name + filtered every deployment out and /health answered 0/0 for a model the + same key could call. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(_ACCESS_GROUP_MODEL_LIST, _ACCESS_GROUP_ROUTER), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"]), + model=None, + model_id=None, + ) + + assert [m["model_name"] for m in captured["model_list"]] == ["bedrock-nova"] + + +@pytest.mark.asyncio +async def test_health_endpoint_expands_access_group_on_background_cache_path(): + """ + LIT-6907: the background-cache path scoped the cached entries through the + same literal model_name match, so an access-group key got an empty result + plus a warning blaming missing model_info.id. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _ACCESS_GROUP_MODEL_LIST, + _ACCESS_GROUP_ROUTER, + use_background_health_checks=True, + health_check_results=_ACCESS_GROUP_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"]), + model=None, + model_id=None, + ) + + assert [e["model_id"] for e in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + assert "warnings" not in result + + +@pytest.mark.asyncio +async def test_health_endpoint_treats_no_team_all_team_models_as_unrestricted(): + """ + A key granted "all-team-models" without a team resolves to an empty + allowlist in the auth layer, which means unrestricted. /health used to + keep the unresolved sentinel and filter every deployment out instead. + """ + from fastapi import Response + + from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(_ACCESS_GROUP_MODEL_LIST, None), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-test-key", models=[SpecialModelNames.all_team_models.value], team_id=None + ), + model=None, + model_id=None, + ) + + assert {m["model_name"] for m in captured["model_list"]} == {"bedrock-nova", "gpt-5.4-mini"} + + +@pytest.mark.asyncio +async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches(): + """ + The missing-model_info.id warning is only true when a matching deployment + exists without an id. A key whose grants match no deployment at all gets a + plain empty result, not advice to populate ids that are already there. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _ACCESS_GROUP_MODEL_LIST, + _ACCESS_GROUP_ROUTER, + use_background_health_checks=True, + health_check_results=_ACCESS_GROUP_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["no-such-model"]), + model=None, + model_id=None, + ) + + assert result["healthy_count"] == 0 + assert result["unhealthy_count"] == 0 + assert "warnings" not in result + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ @@ -1940,7 +2163,7 @@ async def test_health_endpoint_admin_sees_routing_fields_non_admin_does_not(): # withheld so clients that previously parsed them can detect the change. assert ( non_admin_response.headers.get("Litellm-Health-Field-Notice") - == "api_base and api_version are admin-only on this endpoint" + == "api_base, api_version, aws_bedrock_runtime_endpoint are admin-only on this endpoint" ) assert "Litellm-Health-Field-Notice" not in admin_response.headers @@ -2029,7 +2252,7 @@ async def test_health_endpoint_blocks_cross_scope_model_id_under_background_cach cache filter was driven by an unvalidated ID and the global cache leaked id-b's entry to the caller. """ - from fastapi import Response + from fastapi import HTTPException, Response from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.health_endpoints._health_endpoints import health_endpoint @@ -2080,21 +2303,18 @@ async def test_health_endpoint_blocks_cross_scope_model_id_under_background_cach ): # Calling with model="model-b" rather than model_id="id-b" because # the model_id branch raises 404 when llm_router is None. The bug - # being verified is the same: targeted resolver must drop entries - # not in the caller's scoped model_list. With the fix, the result - # has no leaked endpoints and the targeted-503 path fires. - result = await health_endpoint( - response=response, - user_api_key_dict=user_api_key_dict, - model="model-b", - model_id=None, - ) + # being verified is the same: a target outside the caller's scoped + # model_list is refused before the cache is read. + with pytest.raises(HTTPException) as refused: + await health_endpoint( + response=response, + user_api_key_dict=user_api_key_dict, + model="model-b", + model_id=None, + ) - leaked_ids = {ep.get("model_id") for ep in result.get("healthy_endpoints", [])} - leaked_ids |= {ep.get("model_id") for ep in result.get("unhealthy_endpoints", [])} - assert "id-b" not in leaked_ids, "background cache leaked an out-of-scope deployment to a scoped caller" - assert result["healthy_count"] == 0 - assert response.status_code == 503 + assert refused.value.status_code == 403 + assert "leaky-internal.test" not in str(refused.value.detail) @pytest.mark.asyncio @@ -2226,6 +2446,7 @@ async def test_health_endpoint_returns_503_when_requested_model_has_no_healthy_e response=response, user_api_key_dict=user_api_key_dict, model="model-a", + model_id=None, ) assert response.status_code == 503 @@ -2286,6 +2507,7 @@ async def test_health_endpoint_returns_200_when_requested_model_has_healthy_endp response=response, user_api_key_dict=user_api_key_dict, model="model-a", + model_id=None, ) assert response.status_code == 200 @@ -2670,6 +2892,691 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, assert canary not in str(cleaned) +async def _live_probed_model_ids( + model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, model: str | None = None +) -> set[str]: + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(model_list, _router_for(model_list)), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=model, model_id=None) + + return {m["model_info"]["id"] for m in captured["model_list"]} + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_another_teams_deployment_behind_a_shared_access_group(): + """ + Expanding an access group must not reach past the team boundary: a + team-a key holding the group name may not probe team-b's deployment even + though that deployment sits in the same group. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + ) + + assert probed == {"id-bedrock"} + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team(): + """ + Routing never serves a team-owned deployment to a caller without a team + (``filter_team_based_models``), so a team-less access-group key must not + probe team-b's deployment with team-b's credentials either. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + ) + + assert probed == {"id-bedrock"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "expected_ids"), + [(None, {"id-bedrock"}), ("team-a", {"id-bedrock"}), ("team-b", {"id-bedrock", "id-team-b"})], +) +async def test_health_endpoint_keeps_an_unrestricted_non_admin_key_to_its_own_team(team_id, expected_ids): + """ + A key with no model restriction is still bound by routing's team rule: + it may probe global deployments and its own team's, never another team's. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=[], team_id=team_id), + ) + + assert probed == expected_ids + + +@pytest.mark.asyncio +async def test_health_endpoint_lets_a_proxy_admin_probe_every_teams_deployment(): + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert probed == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_unrestricted_non_admin_key_to_its_own_team_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=[], team_id="team-a"), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + +@pytest.mark.asyncio +async def test_health_endpoint_shows_a_teams_own_deployment_by_its_public_name(): + """ + A team key names its team deployment by ``team_public_model_name``, while + the proxy model list carries the internal ``__`` + name; the deployment must still be probed for its own team. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + ) + + assert probed == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_another_teams_deployment_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_targeted_deployment_outside_the_callers_scope_on_live_path(): + """ + A scoped key asking for a deployment it may not see must get a 403 and no + probe at all: probing the rest of its scope instead would report another + deployment's health under the requested id and store it as such. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + fake_perform = AsyncMock() + + with ( + _proxy_health_globals(_TEAM_MODEL_LIST, _router_for(_TEAM_MODEL_LIST)), + patch( # test-quality-ok: the probe must never run; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + fake_perform, + ), + pytest.raises(HTTPException) as excinfo, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + model=None, + model_id="id-team-b", + ) + + assert excinfo.value.status_code == 403 + fake_perform.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_targeted_deployment_outside_the_callers_scope_on_background_cache_path(): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ), + pytest.raises(HTTPException) as excinfo, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + model="bedrock-nova_team-b_9f2c", + model_id=None, + ) + + assert excinfo.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + +_TEAM_ONLY_MODEL_LIST = [_TEAM_MODEL_LIST[1]] +_BARE_NAME_MODEL_LIST = [ + {"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}, "model_info": {"id": "id-nano"}}, + { + "model_name": "gpt-5.4-nano_team-b_7c3d", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"}, + }, +] +_BARE_NAME_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "gpt-5.4-nano", "model_id": "id-nano"}, + {"model": "gpt-5.4-nano", "model_id": "id-nano-team-b"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_a_team_only_deployment_by_its_public_name_on_live_path(): + """ + A team key targets its deployment by ``team_public_model_name``; when that + name resolves to nothing but the team deployment, the probe must run rather + than 403 as if the key were out of scope. + """ + probed = await _live_probed_model_ids( + _TEAM_ONLY_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + ) + + assert probed == {"id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_a_team_only_deployment_by_its_public_name_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_only_the_owning_teams_copy_behind_a_shared_public_name_on_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] + + +async def _live_narrowed_model_ids( + model_list: Sequence[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + model: str | None = None, + model_id: str | None = None, +) -> set[str]: + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + probed = [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list] + return probed, [], {} + + with ( + _proxy_health_globals(model_list, _router_for(model_list)), + patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=user_api_key_dict, model=model, model_id=model_id + ) + + return {ep["model_id"] for ep in result["healthy_endpoints"]} + + +_ADMIN_OUTSIDE_TEAM_B = UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_role=LitellmUserRoles.PROXY_ADMIN) + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies(): + """ + An admin outside team-b asks for ``bedrock-nova``. Team-b's copy answers to + that name only for team-b (routing keys public names by team), so probing + it too would spend team-b's credentials and let a healthy team copy mask a + down global deployment as 200. + """ + probed = await _live_narrowed_model_ids(_TEAM_MODEL_LIST, _ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova") + + assert probed == {"id-bedrock"} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_owning_teams_copy_behind_a_shared_public_name(): + """Team-b's requests for ``bedrock-nova`` route to its copy alone, so its health probe reaches only that copy.""" + probed = await _live_narrowed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + ) + + assert probed == {"id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_teams_copy_when_provider_model_equals_public_name(): + """A bare provider model equal to the public name must not pull the global copy into the team's probe.""" + probed = await _live_narrowed_model_ids( + _BARE_NAME_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"), + model="gpt-5.4-nano", + ) + + assert probed == {"id-nano-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_only_the_teams_copy_when_provider_model_equals_public_name_on_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _BARE_NAME_MODEL_LIST, + _router_for(_BARE_NAME_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_BARE_NAME_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"), + model="gpt-5.4-nano", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-nano-team-b"] + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_a_team_only_public_name_for_an_admin_on_live_path(): + """ + An admin's request for a public name only team-b's deployment carries routes + to that deployment, so the health probe for that name must reach it too + instead of answering an empty 503. + """ + probed = await _live_narrowed_model_ids(_TEAM_ONLY_MODEL_LIST, _ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova") + + assert probed == {"id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_a_team_only_public_name_for_an_admin_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_keeps_a_team_only_public_name_off_a_team_less_key(use_background_health_checks): + """ + A key with no team holds the name ``bedrock-nova`` but never sees team-b's + deployment, so the public-name fallback an admin gets must not open it up. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"]), + model="bedrock-nova", + model_id=None, + ) + + assert refused.value.status_code == 403 + assert "bedrock-nova" in str(refused.value.detail) + probe.assert_not_awaited() + + +def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): + resolve = _health_endpoints_module._resolve_targeted_model_ids + + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", "id-team-b", None) == {"id-team-b"} + assert resolve([_TEAM_MODEL_LIST[0]], "bedrock-nova", "id-team-b", None) == set() + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, None) == {"id-bedrock"} + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, "team-b") == {"id-team-b"} + assert resolve(_TEAM_ONLY_MODEL_LIST, "bedrock-nova", None, None) == {"id-team-b"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_rejects_an_in_scope_model_paired_with_a_foreign_model_id(use_background_health_checks): + """ + A key scoped to ``bedrock-nova`` pairs that name with another team's + deployment id. The in-scope name must not carry the foreign id past the + 403: the live path narrows by id first, so the caller's own deployment + would be probed and its result stored under the foreign id. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"]), + model="bedrock-nova", + model_id="id-team-b", + ) + + assert refused.value.status_code == 403 + assert "id-team-b" in str(refused.value.detail) + probe.assert_not_awaited() + + +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +@pytest.mark.asyncio +async def test_health_endpoint_returns_404_for_a_model_paired_with_an_unknown_model_id(use_background_health_checks): + """ + ``model_id`` wins over ``model``: pairing a known name with an id no + deployment carries gets the same 404 as the lone unknown id, before any + probe runs or a result is stored under the unknown id. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, + model="bedrock-nova", + model_id="id-nobody-has", + ) + + assert refused.value.status_code == 404 + assert "id-nobody-has" in str(refused.value.detail) + probe.assert_not_awaited() + + +def test_health_test_connection_keeps_error_and_raw_request_through_the_allowlist(monkeypatch): + """ + The dashboard's Test Connect button reads ``result.error`` and + ``result.raw_request_typed_dict`` from /health/test_connection, so the + allowlist must keep both while dropping the probe's own params. + """ + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + app = FastAPI() + app.include_router(_health_endpoints_module.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + client = TestClient(app) + + with ( + patch( # test-quality-ok: the endpoint reads the proxy-global DB client and 500s when it is None; it has no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + respx.mock(assert_all_called=True) as respx_mock, + ): + respx_mock.post(host="api.openai.com", path="/v1/chat/completions").respond( + status_code=401, json={"error": {"message": "Incorrect API key provided"}} + ) + response = client.post( + "/health/test_connection", + json={ + "mode": "chat", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test", "timeout": 7}, + }, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["status"] == "error" + assert "Incorrect API key provided" in body["result"]["error"] + assert "api.openai.com" in body["result"]["raw_request_typed_dict"]["raw_request_api_base"] + assert not {"api_key", "timeout", "exception"} & set(body["result"]) + + +def test_clean_endpoint_data_keeps_only_json_safe_diagnostics(): + """ + LIT-6907: _clean_endpoint_data used to copy every litellm_param not on a + deny list, so a nested mapping keyed by a tuple reached jsonable_encoder + and 500'd /health. Only the explicit allowlist survives now. + """ + from fastapi.encoders import jsonable_encoder + + from litellm.proxy.health_check import _clean_endpoint_data + + cleaned = _clean_endpoint_data( + { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "custom_llm_provider": "bedrock", + "aws_region_name": "us-east-1", + "metadata": {("us-east-1", "primary"): "canary-nested-mapping"}, + "allow_client_keepalive_override": False, + "api_key": "CANARY-API-KEY", + "x-ratelimit-remaining-requests": 99, + "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + "aws_bedrock_runtime_endpoint": "https://vpce-bedrock.example.test", + }, + details=True, + ) + + assert cleaned == { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "custom_llm_provider": "bedrock", + "aws_region_name": "us-east-1", + "x-ratelimit-remaining-requests": 99, + "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + "aws_bedrock_runtime_endpoint": "https://vpce-bedrock.example.test", + } + assert jsonable_encoder(cleaned) == cleaned + + +@pytest.mark.asyncio +async def test_health_endpoint_result_survives_non_json_safe_deployment_params(): + """ + LIT-6907: the full /health path with a deployment carrying a tuple-keyed + nested mapping must produce a response FastAPI can encode, with the + approved diagnostics intact and the offending param absent. + """ + from fastapi import Response + from fastapi.encoders import jsonable_encoder + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + { + "model_name": "bedrock-nova", + "litellm_params": { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "aws_region_name": "us-east-1", + "aws_access_key_id": "CANARY-ACCESS-KEY", + "metadata": {("us-east-1", "primary"): "canary-nested-mapping"}, + }, + "model_info": {"id": "id-bedrock"}, + } + ] + + with ( + _proxy_health_globals(model_list, None), + patch( # test-quality-ok: the provider probe is faked; the assertion is the response shaping after it + "litellm.ahealth_check", AsyncMock(return_value={"x-ratelimit-remaining-requests": 99}) + ), + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-admin-key", user_role=LitellmUserRoles.PROXY_ADMIN), + model=None, + model_id=None, + ) + + encoded = jsonable_encoder(result) + assert encoded["healthy_count"] == 1 + entry = encoded["healthy_endpoints"][0] + assert entry["model_id"] == "id-bedrock" + assert entry["aws_region_name"] == "us-east-1" + assert entry["x-ratelimit-remaining-requests"] == 99 + assert "metadata" not in entry + assert "CANARY" not in str(encoded) + + class TestConfigBaseForHealthCheck: """A request that sets its own connection fields gets a base without the configuration's credentials; anything it leaves unset still comes from diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index c96cfc3ee4a..fad137af66b 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,5 +1,7 @@ import asyncio +import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -7,6 +9,7 @@ import pytest from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( _get_budget_reservation_from_metadata, @@ -15,6 +18,7 @@ from litellm.proxy.hooks.proxy_track_cost_callback import ( _update_database_and_spend_counters, run_spend_event, ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload @@ -2347,3 +2351,27 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() await run_spend_event(b"garbage\n") mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): + raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = _ProxyDBLogger(spend_writer=lambda: writer) + + await logger.async_post_call_failure_hook( + request_data={"model": raw_model, "messages": [{"role": "user", "content": "hi"}]}, + original_exception=ProxyModelNotFoundError(route="/chat/completions", model_name=raw_model), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ + "error_information" + ] + assert "medical records" not in json.dumps(error_information) + assert ( + error_information["error_message"] + == "/chat/completions: Invalid model name passed in. Call `/v1/models` to view available models for your key." + ) + assert error_information["error_class"] == "ProxyModelNotFoundError" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 2f6561046b1..2fb496d6231 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -11220,13 +11220,14 @@ async def test_team_info_returns_model_aliases(): @pytest.mark.asyncio -async def test_team_info_hydrates_member_emails_from_the_user_table(): - """/team/info must fill in emails missing from the members_with_roles snapshot. +async def test_team_info_hydrates_member_names_and_emails_from_the_user_table(): + """/team/info must attach each member's display name and fill in emails missing + from the members_with_roles snapshot. - members_with_roles is written at add-time, so a member added by user_id alone - carries user_email=None forever. Without this join the Admin UI's member table - shows "-" for a user that has an email on their user row. A stored email is left - exactly as-is. + members_with_roles is written at add-time, so it never carries user_alias and a + member added by user_id alone carries user_email=None forever. Without this join + the Admin UI's member table can only show emails. A stored email is left exactly + as-is. """ from fastapi import Request @@ -11246,13 +11247,8 @@ async def test_team_info_hydrates_member_emails_from_the_user_table(): find_many = AsyncMock( return_value=[ - LiteLLM_UserTable( - user_id="no-email-on-roster", - user_email="real@example.com", - max_budget=None, - spend=0.0, - models=[], - ) + _user_row("no-email-on-roster", "real@example.com", "Real Person"), + _user_row("already-stored", "current@example.com", "Stored Person"), ] ) @@ -11270,12 +11266,12 @@ async def test_team_info_hydrates_member_emails_from_the_user_table(): ) members = response["team_info"].members_with_roles - assert [(m.user_id, m.user_email) for m in members] == [ - ("no-email-on-roster", "real@example.com"), - ("already-stored", "stored@example.com"), + assert [(m.user_id, m.user_email, m.user_alias) for m in members] == [ + ("no-email-on-roster", "real@example.com", "Real Person"), + ("already-stored", "stored@example.com", "Stored Person"), ] - # only the member actually missing an email is looked up - assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["no-email-on-roster"]}} + find_many.assert_awaited_once() + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["already-stored", "no-email-on-roster"]}} @pytest.mark.asyncio @@ -12472,89 +12468,93 @@ async def test_resolve_existing_member_user_ids_skips_the_query_when_no_user_ids repo.return_value.table.find_many.assert_not_awaited() -def _user_row(user_id: str, user_email: str | None) -> LiteLLM_UserTable: +def _user_row(user_id: str, user_email: str | None, user_alias: str | None = None) -> LiteLLM_UserTable: return LiteLLM_UserTable( - user_id=user_id, user_email=user_email, max_budget=None, spend=0.0, models=[] + user_id=user_id, user_email=user_email, user_alias=user_alias, max_budget=None, spend=0.0, models=[] ) @pytest.mark.asyncio -async def test_hydrate_member_emails_fills_in_emails_the_roster_snapshot_never_captured(): - """A member added by user_id alone has user_email=None on the stored roster entry. - - /team/info has to fill it in from the user row, or the UI renders "-" for a user - that plainly has an email. +async def test_hydrate_member_user_details_attaches_alias_and_fills_in_missing_email(): + """The stored roster never carries a display name, and a member added by user_id + alone has user_email=None. /team/info has to fill both in from the user row so the + UI can show and search by a human-readable name instead of only an email. """ - from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_user_details - find_many = AsyncMock(return_value=[_user_row("by-id", "found@example.com")]) + find_many = AsyncMock(return_value=[_user_row("by-id", "found@example.com", "Found Person")]) with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: repo.return_value.table.find_many = find_many - hydrated = await _hydrate_member_emails( + hydrated = await _hydrate_member_user_details( prisma_client=MagicMock(), members=[Member(user_id="by-id", role="admin")], ) - assert [(m.user_id, m.user_email, m.role) for m in hydrated] == [("by-id", "found@example.com", "admin")] + assert [(m.user_id, m.user_email, m.user_alias, m.role) for m in hydrated] == [ + ("by-id", "found@example.com", "Found Person", "admin") + ] find_many.assert_awaited_once() assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["by-id"]}} @pytest.mark.asyncio -async def test_hydrate_member_emails_never_overwrites_a_stored_email(): - """The snapshot wins wherever it has a value - hydration only fills blanks. - - Overwriting would be a real behavior change to /team/info; filling a null is not. - """ - from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails - - find_many = AsyncMock(return_value=[_user_row("has-email", "current@example.com")]) +async def test_hydrate_member_user_details_never_overwrites_a_stored_email(): + """The snapshot wins wherever it has a value - hydration only fills blanks.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_user_details with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: - repo.return_value.table.find_many = find_many + repo.return_value.table.find_many = AsyncMock( + return_value=[_user_row("has-email", "current@example.com", "Current Name")] + ) - hydrated = await _hydrate_member_emails( + hydrated = await _hydrate_member_user_details( prisma_client=MagicMock(), members=[Member(user_id="has-email", user_email="stored@example.com", role="user")], ) - assert hydrated[0].user_email == "stored@example.com" - # nothing was missing, so no round-trip either - find_many.assert_not_awaited() + assert (hydrated[0].user_email, hydrated[0].user_alias) == ("stored@example.com", "Current Name") @pytest.mark.asyncio -async def test_hydrate_member_emails_leaves_members_alone_when_the_user_row_has_no_email(): - """A user row with no email leaves the member as-is rather than inventing one.""" - from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails +async def test_hydrate_member_user_details_leaves_blanks_when_the_user_row_is_bare_or_missing(): + """A user row with no email or alias, or no user row at all, must not invent values.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_user_details with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: - repo.return_value.table.find_many = AsyncMock(return_value=[_user_row("no-email", None)]) + repo.return_value.table.find_many = AsyncMock(return_value=[_user_row("bare", None)]) - hydrated = await _hydrate_member_emails( + hydrated = await _hydrate_member_user_details( prisma_client=MagicMock(), - members=[Member(user_id="no-email", role="user"), Member(user_email="e@example.com", role="user")], + members=[ + Member(user_id="bare", role="user"), + Member(user_id="deleted", user_email="gone@example.com", role="user"), + Member(user_email="e@example.com", role="user"), + ], ) - assert [m.user_email for m in hydrated] == [None, "e@example.com"] + assert [(m.user_id, m.user_email, m.user_alias) for m in hydrated] == [ + ("bare", None, None), + ("deleted", "gone@example.com", None), + (None, "e@example.com", None), + ] @pytest.mark.asyncio -async def test_hydrate_member_emails_skips_the_query_when_every_member_has_one(): - """No blanks means /team/info pays for no extra query.""" - from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails +async def test_hydrate_member_user_details_skips_the_query_when_no_member_has_a_user_id(): + """Email-only roster entries give nothing to look up, so /team/info pays for no query.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_user_details with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: repo.return_value.table.find_many = AsyncMock() - hydrated = await _hydrate_member_emails( + hydrated = await _hydrate_member_user_details( prisma_client=MagicMock(), - members=[Member(user_id="a", user_email="a@example.com", role="user")], + members=[Member(user_email="a@example.com", role="user")], ) - assert hydrated[0].user_email == "a@example.com" + assert [(m.user_email, m.user_alias) for m in hydrated] == [("a@example.com", None)] repo.return_value.table.find_many.assert_not_awaited() diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index ee062b14b96..3e0acf917aa 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import json import math +from types import MappingProxyType from typing import Final import pytest @@ -207,8 +210,13 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: ANTHROPIC_TOKENIZER_MODEL: Final = "claude-sonnet-4-5-20250929" +CL100K_MODEL: Final = "gpt-4" +O200K_MODEL: Final = "gpt-4o" RUST_COUNTED_BODY: Final = {"model": ANTHROPIC_TOKENIZER_MODEL, "max_tokens": 16, "messages": ANTHROPIC_MESSAGES} RUST_INPUT_TOKENS: Final = 4_321 +RUST_INPUT_TOKENS_BY_TOKENIZER: Final = MappingProxyType( + {"anthropic": RUST_INPUT_TOKENS, "cl100k_base": 1_234, "o200k_base": 2_345} +) class _FakeDeclined(Exception): @@ -225,33 +233,57 @@ class _FakeNative: class _RecordingCounter: - bodies: Final[list[bytes]] = [] + """Stands in for one native counter; records `(tokenizer, body)` on the shared factory.""" - def __init__(self, tokenizer_json: str) -> None: - pass + def __init__(self, factory: _RecordingFactory, tokenizer: rust_token_counter.RustTokenizer) -> None: + self.factory = factory + self.tokenizer = tokenizer async def acount_request(self, body: bytes) -> object: - self.bodies.append(body) - return {"model": ANTHROPIC_TOKENIZER_MODEL, "input_tokens": RUST_INPUT_TOKENS} + self.factory.calls.append((self.tokenizer, body)) + return {"model": "", "input_tokens": RUST_INPUT_TOKENS_BY_TOKENIZER[self.tokenizer]} + + +class _RecordingFactory: + """Stands in for the native `TokenCounter` class: called with tokenizer JSON, or `from_*_ranks`.""" + + def __init__(self) -> None: + self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] + + def __call__(self, tokenizer_json: str) -> _RecordingCounter: + return _RecordingCounter(self, "anthropic") + + def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: + return _RecordingCounter(self, "cl100k_base") + + def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: + return _RecordingCounter(self, "o200k_base") class _DecliningCounter: - def __init__(self, tokenizer_json: str) -> None: - pass - async def acount_request(self, body: bytes) -> object: raise _FakeDeclined("unsupported content block") +class _DecliningFactory: + def __call__(self, tokenizer_json: str) -> _DecliningCounter: + return _DecliningCounter() + + def from_cl100k_ranks(self, rank_file: str) -> _DecliningCounter: + return _DecliningCounter() + + def from_o200k_ranks(self, rank_file: str) -> _DecliningCounter: + return _DecliningCounter() + + @pytest.fixture def rust_counter(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) - rust_token_counter._anthropic_counter.cache_clear() + rust_token_counter._counter.cache_clear() configuration.reset_rust_configuration() - _RecordingCounter.bodies.clear() yield rust_token_counter.TOKEN_COUNTER.reset() - rust_token_counter._anthropic_counter.cache_clear() + rust_token_counter._counter.cache_clear() configuration.reset_rust_configuration() @@ -270,8 +302,9 @@ def rust_counter(monkeypatch: pytest.MonkeyPatch): async def test_rust_count_replaces_python_tokenizing_on_every_llm_route( rust_counter: None, route: str, request_body: dict ) -> None: + factory: Final = _RecordingFactory() litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(_RecordingCounter) + rust_token_counter.TOKEN_COUNTER.override(factory) raw_body: Final = json.dumps(request_body).encode() counts: Final = await count_request_input_tokens( @@ -279,53 +312,136 @@ async def test_rust_count_replaces_python_tokenizing_on_every_llm_route( ) assert dict(counts) == {ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS} - assert _RecordingCounter.bodies == [raw_body] + assert factory.calls == [("anthropic", raw_body)] @pytest.mark.asyncio -async def test_rust_decline_falls_back_to_python_count(rust_counter: None) -> None: +@pytest.mark.parametrize("model", (CL100K_MODEL, "azure/gpt-35-turbo", "gemini/gemini-2.5-pro", "my-router-alias")) +async def test_tiktoken_cl100k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: + factory: Final = _RecordingFactory() litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(_DecliningCounter) + rust_token_counter.TOKEN_COUNTER.override(factory) + body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} + raw_body: Final = json.dumps(body).encode() + + counts: Final = await count_request_input_tokens( + request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body + ) + + assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"]} + assert factory.calls == [("cl100k_base", raw_body)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", (O200K_MODEL, "gpt-5", "o3", "gpt-4.1", "chatgpt-4o-latest")) +async def test_tiktoken_o200k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + rust_token_counter.TOKEN_COUNTER.override(factory) + body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} + raw_body: Final = json.dumps(body).encode() + + counts: Final = await count_request_input_tokens( + request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body + ) + + assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"]} + assert factory.calls == [("o200k_base", raw_body)] + + +@pytest.mark.asyncio +async def test_multi_model_request_counts_once_per_tokenizer_and_python_for_the_rest(rust_counter: None) -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + rust_token_counter.TOKEN_COUNTER.override(factory) + models: Final = ( + CL100K_MODEL, + ANTHROPIC_TOKENIZER_MODEL, + "gemini/gemini-2.5-pro", + O200K_MODEL, + "gpt-5", + "replicate/meta/llama-2-70b-chat", + ) + body: Final = {"model": list(models), "messages": ANTHROPIC_MESSAGES} + raw_body: Final = json.dumps(body).encode() python_counts: Final = await count_request_input_tokens( - request_body=RUST_COUNTED_BODY, route="/v1/messages", llm_router=None + request_body=body, route="/v1/chat/completions", llm_router=None ) counts: Final = await count_request_input_tokens( - request_body=RUST_COUNTED_BODY, + request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body + ) + + assert factory.calls == [("cl100k_base", raw_body), ("anthropic", raw_body), ("o200k_base", raw_body)] + assert dict(counts) == { + CL100K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], + "gemini/gemini-2.5-pro": RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], + ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS, + O200K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], + "gpt-5": RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], + "replicate/meta/llama-2-70b-chat": python_counts["replicate/meta/llama-2-70b-chat"], + } + assert counts["replicate/meta/llama-2-70b-chat"] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", (ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL)) +async def test_rust_decline_falls_back_to_python_count(rust_counter: None, model: str) -> None: + litellm.rust(True) + rust_token_counter.TOKEN_COUNTER.override(_DecliningFactory()) + body: Final = {**RUST_COUNTED_BODY, "model": model} + python_counts: Final = await count_request_input_tokens(request_body=body, route="/v1/messages", llm_router=None) + + counts: Final = await count_request_input_tokens( + request_body=body, route="/v1/messages", llm_router=None, - raw_body=json.dumps(RUST_COUNTED_BODY).encode(), + raw_body=json.dumps(body).encode(), ) assert dict(counts) == dict(python_counts) - assert counts[ANTHROPIC_TOKENIZER_MODEL] != RUST_INPUT_TOKENS + assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() @pytest.mark.asyncio async def test_disabled_rust_never_sees_the_raw_body(rust_counter: None) -> None: + factory: Final = _RecordingFactory() litellm.rust(False) - rust_token_counter.TOKEN_COUNTER.override(_RecordingCounter) + rust_token_counter.TOKEN_COUNTER.override(factory) + body: Final = {"model": [ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL], "messages": ANTHROPIC_MESSAGES} counts: Final = await count_request_input_tokens( - request_body=RUST_COUNTED_BODY, - route="/v1/messages", + request_body=body, + route="/v1/chat/completions", llm_router=None, - raw_body=json.dumps(RUST_COUNTED_BODY).encode(), + raw_body=json.dumps(body).encode(), ) - assert _RecordingCounter.bodies == [] - assert counts[ANTHROPIC_TOKENIZER_MODEL] != RUST_INPUT_TOKENS + assert factory.calls == [] + assert set(counts) == {ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL} + assert not set(counts.values()) & set(RUST_INPUT_TOKENS_BY_TOKENIZER.values()) @pytest.mark.asyncio -async def test_non_anthropic_tokenizer_models_stay_in_python(rust_counter: None) -> None: +@pytest.mark.parametrize("model", ("replicate/meta/llama-2-70b-chat", "meta-llama/Llama-3-8b", "text-davinci-003")) +async def test_models_without_a_rust_tokenizer_stay_in_python( + rust_counter: None, monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + monkeypatch.setattr( + litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"} + ) + factory: Final = _RecordingFactory() litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(_RecordingCounter) - body: Final = {"model": "gpt-4o", "messages": ANTHROPIC_MESSAGES} + rust_token_counter.TOKEN_COUNTER.override(factory) + body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} + python_counts: Final = await count_request_input_tokens( + request_body=body, route="/v1/chat/completions", llm_router=None + ) counts: Final = await count_request_input_tokens( request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=json.dumps(body).encode() ) - assert _RecordingCounter.bodies == [] - assert counts["gpt-4o"] != RUST_INPUT_TOKENS + assert factory.calls == [] + assert dict(counts) == dict(python_counts) + assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 671a8ae63fc..6e43ac4a12b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -6629,6 +6629,30 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} +def _session_grouped_paginating_prisma(sessions, counted_total=None): + """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" + + async def mock_query_raw(sql_query, *params): + if "COUNT(*) AS total_count" in sql_query: + return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] + if "DISTINCT ON" in sql_query: + return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] + if "COALESCE(SUM(spend)" in sql_query: + return [] + bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) + limit = params[int(bounds.group(1)) - 1] + offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 + return [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ] + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + return mock_prisma + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatch): """One row per (session, api_key), session-count total, and a keyset cursor for the next page.""" @@ -6742,6 +6766,203 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor(client, monkeypatch): + """page > 1 with no session_cursor (the UI's last-page jump) serves the sessions that page starts at.""" + sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(60)) + mock_prisma = _session_grouped_paginating_prisma(sessions) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + "group_by_session": "true", + "page": 3, + "page_size": 25, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["page"] == 3 + assert data["total"] == 60 + assert data["has_more"] is False + assert data["next_session_cursor"] is None + assert [row["request_id"] for row in data["data"]] == [f"req-sess-{index:02d}" for index in range(50, 60)] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): + """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) + mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + "group_by_session": "true", + "page": 1, + "page_size": 25, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total_is_capped"] is False + assert data["total_pages"] == 1 + assert len(data["data"]) == 10 + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_page_past_the_end_keeps_the_real_total(client, monkeypatch): + """An empty page past the last one says nothing about the total, so it is counted rather than inferred.""" + sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(100)) + mock_prisma = _session_grouped_paginating_prisma(sessions) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + "group_by_session": "true", + "page": 4, + "page_size": 50, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["data"] == [] + assert data["total"] == 100, "the empty page's offset is not a total" + assert data["total_pages"] == 2 + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty(client, monkeypatch): + """The last page inside the capped total still lists sessions; the page after it is empty and costs no query.""" + cap = spend_management_endpoints.SPEND_LOGS_PAGINATION_COUNT_CAP + sessions = tuple((f"sess-{index:06d}", "2026-08-29 10:00:00") for index in range(cap + 50)) + mock_prisma = _session_grouped_paginating_prisma(sessions) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + try: + start_date, end_date = _default_date_range() + params = { + "start_date": start_date, + "end_date": end_date, + "group_by_session": "true", + "page_size": 25, + } + last_page = client.get( + "/spend/logs/ui", + params={**params, "page": cap // 25}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert last_page.status_code == 200, last_page.text + last_page_data = last_page.json() + assert last_page_data["total"] == cap + assert last_page_data["total_is_capped"] is True + assert last_page_data["data"][0]["request_id"] == f"req-sess-{cap - 25:06d}" + assert len(last_page_data["data"]) == 25 + + mock_prisma.db.query_raw.reset_mock() + past_cap = client.get( + "/spend/logs/ui", + params={**params, "page": cap // 25 + 1}, + headers={"Authorization": "Bearer sk-test"}, + ) + assert past_cap.status_code == 200, past_cap.text + past_cap_data = past_cap.json() + assert past_cap_data["data"] == [] + assert past_cap_data["has_more"] is False + assert past_cap_data["total"] == cap + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_group_by_session_last_page_stops_at_the_capped_total(client, monkeypatch): + """A page size that does not divide the cap still ends the last page at the capped total it reports.""" + cap = spend_management_endpoints.SPEND_LOGS_PAGINATION_COUNT_CAP + sessions = tuple((f"sess-{index:06d}", "2026-08-29 10:00:00") for index in range(cap + 50)) + mock_prisma = _session_grouped_paginating_prisma(sessions) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + "group_by_session": "true", + "page": cap // 7 + 1, + "page_size": 7, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["total"] == cap + assert [row["request_id"] for row in data["data"]] == [ + f"req-sess-{index:06d}" for index in range(cap - cap % 7, cap) + ] + assert data["has_more"] is True + assert data["next_session_cursor"] is not None + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_offset_for_non_starttime_sort( client, monkeypatch diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index a7de3f1d8d6..54e5a6d5385 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -528,8 +528,8 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ - {"session_key": "req-1", "api_key": "k", "last_activity": "2026-02-16 10:00:00"}, - {"session_key": "req-2", "api_key": "k", "last_activity": "2026-02-16 09:00:00"}, + {"session_key": f"req-{index}", "api_key": "k", "last_activity": f"2026-02-16 10:{59 - index:02d}:00"} + for index in range(51) ] representative_rows = [ {"request_id": "req-1", "api_key": "k", "metadata": "{}", "session_id": None}, @@ -538,7 +538,7 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): async def mock_query_raw(sql_query, *params): if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 12}] + return [{"total_count": 60}] if "DISTINCT ON" in sql_query: return representative_rows return session_rows @@ -590,11 +590,11 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): assert "COUNT(*) OVER ()" not in rep_sql assert [row["request_id"] for row in response["data"]] == ["req-1", "req-2"] - assert response["total"] == 12 + assert response["total"] == 60 assert response["total_is_capped"] is False - assert response["total_pages"] == 1 - assert response["has_more"] is False - assert response["next_session_cursor"] is None + assert response["total_pages"] == 2 + assert response["has_more"] is True + assert response["next_session_cursor"] == "2026-02-16 10:10:00|k|req-49" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index dc8a87a97ad..e9bbaf0c96e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -20,6 +20,7 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup @@ -2755,6 +2756,31 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form( # ── _redact_logged_api_key unit tests ────────────────────────────────────── +@pytest.mark.parametrize( + ("original_exception", "expected_error_message"), + [ + ( + ProxyModelNotFoundError(route="/chat/completions", model_name=_RAW_MODEL_WITH_PROMPT), + "/chat/completions: Invalid model name passed in. Call `/v1/models` to view available models for your key.", + ), + (ValueError("provider timed out"), "provider timed out"), + ], +) +def test_sanitize_error_information_persists_no_raw_model_for_an_unknown_model_rejection( + original_exception: Exception, expected_error_message: str +): + error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=original_exception) + + sanitized: Final = _sanitize_error_information_for_spend_logs( + error_information, original_exception=original_exception + ) + + assert sanitized is not None + assert sanitized["error_message"] == expected_error_message + assert "medical records" not in json.dumps(sanitized) + assert sanitized["error_class"] == type(original_exception).__name__ + + def test_redact_logged_api_key_none_returns_none(): assert _redact_logged_api_key(None) is None diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 50b26577e5c..efbb5eedad4 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -8259,6 +8259,83 @@ class TestStreamingResponseHeadersFollowFallback: assert result.headers["x-callback-header"] == "kept" +class _MessagesFallbackStream: + def __init__(self) -> None: + self.fallback_headers_adopted = False + self._hidden_params: dict[str, object] = { + "additional_headers": { + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-reasoning-effort": "xhigh", + } + } + self._chunks = iter( + ( + b'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"OK"}}\n\n', + ) + ) + + def __aiter__(self) -> "_MessagesFallbackStream": + return self + + async def __anext__(self) -> bytes: + self._hidden_params = { + "model_id": "fallback-deployment", + "additional_headers": {"x-fallback-only": "yes"}, + } + self.fallback_headers_adopted = True + try: + return next(self._chunks) + except StopIteration: + raise StopAsyncIteration from None + + async def aclose(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_messages_http_headers_refresh_after_lazy_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.caching.caching import DualCache + + stream = _MessagesFallbackStream() + logging_obj = MagicMock() + logging_obj.litellm_call_id = "messages-fallback-headers" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + logging_obj.litellm_params = {} + processor = ProxyBaseLLMRequestProcessing( + data={"model": "auto-router", "stream": True, "litellm_logging_obj": logging_obj} + ) + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + monkeypatch.setattr(litellm, "callbacks", []) + + async def call() -> _MessagesFallbackStream: + return stream + + async def fake_route_request(**_kwargs: object) -> object: + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + response = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="anthropic_messages", + proxy_logging_obj=proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + is_streaming_request=True, + skip_pre_call_logic=True, + ) + + assert isinstance(response, StreamingResponse) + assert stream.fallback_headers_adopted is True + assert response.headers["x-litellm-model-id"] == "fallback-deployment" + assert response.headers["x-fallback-only"] == "yes" + assert "x-litellm-complexity-router-tier" not in response.headers + assert "x-litellm-complexity-router-reasoning-effort" not in response.headers + + class TestPassthroughHeadersAcceptImmutableMappings: """LIT-6767: the streaming branch now hands the passthrough helpers an immutable mapping.""" diff --git a/tests/test_litellm/proxy/test_component_allowlists.py b/tests/test_litellm/proxy/test_component_allowlists.py index 0fdb43d60da..3073908fa54 100644 --- a/tests/test_litellm/proxy/test_component_allowlists.py +++ b/tests/test_litellm/proxy/test_component_allowlists.py @@ -196,6 +196,20 @@ def test_gateway_drops_ui_and_swagger_mounts(): f"Mount {path} must not be served by the gateway" +def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): + """The gateway serves /debug/memory/summary, since the RSS that matters is the + serving worker's and the memory regression e2e test reads it on every gateway + replica; the heavier and mutating /debug/memory routes stay on the backend.""" + debug_memory_routes = { + getattr(r, "path"): r for r in app.router.routes if str(getattr(r, "path", "")).startswith("/debug/memory/") + } + assert {"/debug/memory/summary", "/debug/memory/details", "/debug/memory/gc/configure"} <= set(debug_memory_routes) + assert _is_gateway_route(debug_memory_routes["/debug/memory/summary"]), \ + "/debug/memory/summary must survive the gateway route trim" + for path in ("/debug/memory/details", "/debug/memory/gc/configure"): + assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" + + def test_every_app_mount_is_assigned_to_a_component(): """Every Mount on the proxy app must be consciously assigned to a component. diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index 5aa80213134..c0c853ae2c5 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -735,6 +735,121 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f assert call_kwargs["health_check_skip_disabled_background_models"] is True +@pytest.mark.asyncio +async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_name(): + """``/health?model=`` must probe the team deployment, not an empty list.""" + from litellm.proxy.health_check import perform_health_check + + team_deployment = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, + } + other_deployment = { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-openai"}, + } + probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {})) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check( + model_list=[team_deployment, other_deployment], model="bedrock-nova", team_id="team-b" + ) + + assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == ["id-team-b"] + assert [ep["model_id"] for ep in healthy] == ["id-team-b"] + assert unhealthy == [] + + +@pytest.mark.asyncio +async def test_perform_health_check_keeps_a_public_name_off_another_team(): + """A team's public model name is not a global alias: a caller from another team must not probe its deployment.""" + from litellm.proxy.health_check import perform_health_check + + team_deployment = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, + } + probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {})) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check( + model_list=[team_deployment], model="bedrock-nova", team_id="team-a" + ) + + probe.assert_not_awaited() + assert healthy == [] + assert unhealthy == [] + + +_GLOBAL_DEPLOYMENT = { + "model_name": "bedrock-nova", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-bedrock"}, +} +_TEAM_B_COPY = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, +} +_GLOBAL_BARE_NAME = { + "model_name": "gpt-5.4-nano", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano"}, +} +_TEAM_B_BARE_COPY = { + "model_name": "gpt-5.4-nano_team-b_7c3d", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "model", "model_list", "expected_ids"), + [ + (None, "bedrock-nova", [_TEAM_B_COPY], ["id-team-b"]), + (None, "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]), + ("team-b", "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]), + ("team-b", "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano-team-b"]), + (None, "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano"]), + (None, "bedrock/us.amazon.nova-2-lite-v1:0", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock", "id-team-b"]), + ], + ids=[ + "a team-less caller reaches a public name nothing else carries", + "model_name wins over a public name for a team-less caller", + "a team's own copy wins over the global model_name", + "a team's own copy wins over a litellm_params.model equal to the public name", + "model_name wins over a litellm_params.model equal to it for a team-less caller", + "a provider model string no name carries still matches litellm_params.model", + ], +) +async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes( + team_id, model, model_list, expected_ids +): + """``/health?model=`` probes the deployments a request for that name from the same caller would route to.""" + from litellm.proxy.health_check import perform_health_check + + probe = AsyncMock( + return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": i} for i in expected_ids], [], {}) + ) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model=model, team_id=team_id) + + assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids + assert [ep["model_id"] for ep in healthy] == expected_ids + assert unhealthy == [] + + def test_parse_background_health_check_model_groups_unset_returns_none(): from litellm.proxy.health_check import parse_background_health_check_model_groups diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 97e308d7c3c..dd3669644af 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -872,9 +872,9 @@ def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] - assert hc_module._narrow_to_target(deployments, None, "no-such-id") == tuple(deployments) - assert hc_module._narrow_to_target(deployments, None, "a-1") == tuple(deployments) - assert hc_module._narrow_to_target(deployments, "a", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, None, "no-such-id", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, None, "a-1", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, "a", None, None) == tuple(deployments) def _nested_router_fixture(parent_tier: str): diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ec9025a5220..37b983d709a 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -769,6 +769,7 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hello"}], + "api_key": "request-key", } user_api_key_dict = UserAPIKeyAuth( @@ -796,6 +797,8 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r assert "proxy_server_request" not in snapshot_body, ( "proxy_server_request must be excluded from its own body snapshot to prevent the body from self-referencing" ) + assert "api_key" not in snapshot_body + assert updated["proxy_server_request"]["credential_fields"] == ("api_key",) def test_refresh_proxy_server_request_body_snapshot_picks_up_guardrail_masking(): diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 41ba57c4615..f7021763a4d 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -3,6 +3,7 @@ import pytest +from typing import Final from unittest.mock import MagicMock from fastapi import HTTPException @@ -1297,3 +1298,13 @@ async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through( assert agents_find_unique.await_count == 2 assert model_table.find_many_wheres == [] + + +def test_proxy_model_not_found_error_keeps_the_raw_model_only_in_the_client_response(): + raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + + error: Final = ProxyModelNotFoundError(route="/chat/completions", model_name=raw_model) + + assert raw_model in error.detail["error"] + assert raw_model not in error.spend_log_error_message + assert error.spend_log_error_message.startswith("/chat/completions: Invalid model name passed in") diff --git a/tests/test_litellm/proxy/utils/helpers/test_misc_helpers.py b/tests/test_litellm/proxy/utils/helpers/test_misc_helpers.py index 7968fa40655..6267428e02e 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_misc_helpers.py +++ b/tests/test_litellm/proxy/utils/helpers/test_misc_helpers.py @@ -187,6 +187,22 @@ def test_construct_database_url_from_env_vars_with_schema(monkeypatch): } +def test_construct_database_url_from_env_vars_carries_tls_env(monkeypatch: pytest.MonkeyPatch): + """The CLI password path builds its URL here, so DATABASE_SSLMODE and + DATABASE_SSLROOTCERT must reach PgBouncer through it too.""" + monkeypatch.setenv("DATABASE_HOST", "db.example.com") + monkeypatch.setenv("DATABASE_USERNAME", "user") + monkeypatch.setenv("DATABASE_PASSWORD", "pass") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.setenv("DATABASE_SCHEMA", "public") + monkeypatch.setenv("DATABASE_SSLMODE", "verify-full") + monkeypatch.setenv("DATABASE_SSLROOTCERT", "/etc/ssl/certs/ca-certificates.crt") + assert construct_database_url_from_env_vars() == ( + "postgresql://user:pass@db.example.com/litellm" + "?schema=public&sslmode=verify-full&sslrootcert=%2Fetc%2Fssl%2Fcerts%2Fca-certificates.crt" + ) + + def test_construct_database_url_from_env_vars_error_path_missing_host(monkeypatch): monkeypatch.delenv("DATABASE_HOST", raising=False) monkeypatch.setenv("DATABASE_USERNAME", "user") diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 761e87ac764..0827bbcdc38 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -266,7 +266,8 @@ def test_transcription_only_detection_rejects_speech_model(local_model_cost_map) @pytest.mark.asyncio -async def test_azure_health_check_keeps_beta_path_for_speech_model(): +async def test_azure_health_check_probes_the_ga_upstream_for_an_unconfigured_speech_model(monkeypatch): + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) connect = _CapturingConnect() with patch("websockets.connect", connect): assert await realtime_main._realtime_health_check( @@ -276,14 +277,18 @@ async def test_azure_health_check_keeps_beta_path_for_speech_model(): api_base="https://my-endpoint.openai.azure.com", api_version="2024-10-01-preview", ) - assert connect.url == ( - "wss://my-endpoint.openai.azure.com/openai/realtime" - "?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" - ) + assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-4o-realtime-preview" + + +_AZURE_BETA_HEALTH_URL: Final = ( + "wss://my-endpoint.openai.azure.com/openai/realtime" + "?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" +) @pytest.mark.asyncio -async def test_azure_health_check_honors_deployment_realtime_protocol(): +async def test_azure_health_check_honors_deployment_realtime_protocol(monkeypatch): + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) connect = _CapturingConnect() with patch("websockets.connect", connect): assert await realtime_main._realtime_health_check( @@ -292,9 +297,24 @@ async def test_azure_health_check_honors_deployment_realtime_protocol(): api_key="fake-key", api_base="https://my-endpoint.openai.azure.com", api_version="2024-10-01-preview", - model_params={"realtime_protocol": "GA"}, + model_params={"realtime_protocol": "beta"}, ) - assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-4o-realtime-preview" + assert connect.url == _AZURE_BETA_HEALTH_URL + + +@pytest.mark.asyncio +async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): + monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", "beta") + connect = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-4o-realtime-preview", + custom_llm_provider="azure", + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="2024-10-01-preview", + ) + assert connect.url == _AZURE_BETA_HEALTH_URL class _ConnectThatStopsAfterCapturingTheUrl: @@ -327,7 +347,60 @@ async def test_arealtime_azure_ai_on_a_foundry_host_connects_to_the_azure_openai api_key="fake-key", litellm_logging_obj=FakeLogging(), ) - assert connect.url == ( - "wss://my-project.services.ai.azure.com/openai/realtime" - "?api-version=2024-10-01-preview&deployment=gpt-realtime-mini" + assert connect.url == "wss://my-project.services.ai.azure.com/openai/v1/realtime?model=gpt-realtime-mini" + + +class _ClientWebSocketWithHeaders: + def __init__(self, headers: tuple[tuple[bytes, bytes], ...]) -> None: + self.scope: Final = {"headers": headers} + + +_GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) +_BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) + + +async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: + connect: Final = _ConnectThatStopsAfterCapturingTheUrl() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="azure/gpt-realtime", + websocket=websocket, + api_base="https://my-endpoint.openai.azure.com", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + **kwargs, + ) + return connect.url + + +@pytest.mark.asyncio +async def test_arealtime_azure_ga_client_without_beta_header_dials_the_ga_upstream(monkeypatch): + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + assert ( + await _azure_backend_url_dialed_for(_GA_CLIENT) + == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-realtime" + ) + + +@pytest.mark.asyncio +async def test_arealtime_azure_beta_header_client_keeps_the_beta_upstream(monkeypatch): + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + assert await _azure_backend_url_dialed_for(_BETA_CLIENT) == ( + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + ) + + +@pytest.mark.asyncio +async def test_arealtime_azure_explicit_beta_protocol_wins_over_a_ga_client(monkeypatch): + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + assert await _azure_backend_url_dialed_for(_GA_CLIENT, realtime_protocol="beta") == ( + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + ) + + +@pytest.mark.asyncio +async def test_arealtime_azure_env_beta_protocol_wins_over_a_ga_client(monkeypatch): + monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", "beta") + assert await _azure_backend_url_dialed_for(_GA_CLIENT) == ( + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" ) diff --git a/tests/test_litellm/router_utils/test_add_retry_fallback_headers.py b/tests/test_litellm/router_utils/test_add_retry_fallback_headers.py index 3a0deeb13d8..eb7490d76f8 100644 --- a/tests/test_litellm/router_utils/test_add_retry_fallback_headers.py +++ b/tests/test_litellm/router_utils/test_add_retry_fallback_headers.py @@ -1,12 +1,17 @@ import json +from collections.abc import Mapping +from typing import Literal +import pytest from pydantic import BaseModel from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, add_retry_headers_to_response, + complexity_router_decision_headers, get_fallback_errors_from_headers, get_hidden_params_dict, + replace_complexity_router_headers, ) @@ -15,6 +20,122 @@ class StreamingWrapper: self._hidden_params = {"additional_headers": {"x-existing": "keep"}} +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_complexity_router_decision_headers_exposes_only_bounded_fields( + metadata_key: Literal["metadata", "litellm_metadata"], +) -> None: + headers = complexity_router_decision_headers( + { + metadata_key: { + "routing_decision": { + "router_type": "complexity", + "tier": " REASONING ", + "cause": "heuristic_scorer", + "score": 0.75, + "tier_litellm_params": {"reasoning_effort": "xhigh", "api_key": "secret"}, + "signals": ["private prompt"], + "matched_keyword": "private prompt", + } + } + } + ) + + assert dict(headers) == { + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-cause": "heuristic_scorer", + "x-litellm-complexity-router-score": "0.75", + "x-litellm-complexity-router-reasoning-effort": "xhigh", + } + + +@pytest.mark.parametrize( + "decision, expected", + [ + ( + {"router_type": "complexity", "tier": "SIMPLE", "cause": "heuristic_scorer", "score": 0}, + { + "x-litellm-complexity-router-tier": "SIMPLE", + "x-litellm-complexity-router-cause": "heuristic_scorer", + "x-litellm-complexity-router-score": "0", + }, + ), + ( + {"router_type": "complexity", "tier": "COMPLEX", "cause": "llm_classifier"}, + { + "x-litellm-complexity-router-tier": "COMPLEX", + "x-litellm-complexity-router-cause": "llm_classifier", + }, + ), + ( + {"router_type": "complexity", "tier": "REASONING", "cause": "literal_keyword_match"}, + { + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-cause": "literal_keyword_match", + }, + ), + ({"router_type": "quality", "tier": "premium", "cause": "quality_tier"}, {}), + ({"router_type": "complexity", "score": True}, {}), + ({"router_type": "complexity", "score": float("nan")}, {}), + ({"router_type": "complexity", "score": float("inf")}, {}), + ({"router_type": "complexity", "tier": "研究", "cause": "bad\r\nX-Injected: true"}, {}), + ({"router_type": "complexity", "tier_litellm_params": {"reasoning_effort": 1}}, {}), + ({"router_type": "complexity", "tier_litellm_params": "invalid"}, {}), + ([], {}), + (None, {}), + ], +) +def test_complexity_router_decision_headers_omits_absent_or_invalid_fields( + decision: object, + expected: Mapping[str, str], +) -> None: + assert dict(complexity_router_decision_headers({"metadata": {"routing_decision": decision}})) == expected + + +@pytest.mark.parametrize( + "litellm_decision, metadata_decision, expected", + [ + ( + {"router_type": "complexity", "tier": "SIMPLE", "cause": "heuristic_scorer"}, + {"router_type": "complexity", "tier": "REASONING", "tier_litellm_params": {"reasoning_effort": "xhigh"}}, + {"x-litellm-complexity-router-tier": "SIMPLE", "x-litellm-complexity-router-cause": "heuristic_scorer"}, + ), + ( + {"router_type": "quality", "tier": "premium"}, + {"router_type": "complexity", "tier": "FORGED", "cause": "heuristic_scorer"}, + {}, + ), + ( + {}, + {"router_type": "complexity", "tier": "FORGED", "cause": "heuristic_scorer"}, + {}, + ), + ], +) +def test_complexity_router_decision_headers_never_falls_back_from_internal_metadata( + litellm_decision: Mapping[str, object], + metadata_decision: Mapping[str, object], + expected: Mapping[str, str], +) -> None: + headers = complexity_router_decision_headers( + { + "litellm_metadata": {"routing_decision": litellm_decision}, + "metadata": {"routing_decision": metadata_decision}, + } + ) + assert dict(headers) == expected + + +def test_replace_complexity_router_headers_drops_stale_values() -> None: + assert replace_complexity_router_headers( + { + "x-existing": "keep", + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-reasoning-effort": "xhigh", + }, + {"x-litellm-complexity-router-tier": "SIMPLE"}, + ) == {"x-existing": "keep", "x-litellm-complexity-router-tier": "SIMPLE"} + + def test_add_fallback_headers_to_streaming_wrapper(): response = StreamingWrapper() diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 5b2af3cf02e..8d83f4ca8a6 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -73,7 +73,7 @@ def assert_native_request( headers: HTTPMessage, body: object, ) -> None: - if route not in {"ocr", "azure_ocr", "transcription", "messages", "chat_completions"}: + if route not in {"ocr", "azure_ocr", "azure_di", "transcription", "messages", "chat_completions"}: raise AssertionError(f"unexpected route marker: {route!r}") if outcome not in {"success", "429", "hang"}: raise AssertionError(f"unexpected outcome marker: {outcome!r}") @@ -92,6 +92,13 @@ def assert_native_request( assert body["model"] == "mistral-ocr-2505" assert body["document"]["document_url"] == "data:application/pdf;base64,YWJj" return + if route == "azure_di": + assert path.startswith("/documentintelligence/documentModels/prebuilt-read:analyze?") + assert "api-version=2024-11-30" in path + assert "pages=1%2C3" in path + assert headers.get("ocp-apim-subscription-key") == "di-key" + assert body == {"base64Source": "YWJj"} + return if route == "transcription": assert path == "/model/mistral.voxtral-mini-3b-2507/converse" assert headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") @@ -115,6 +122,8 @@ def native_response(status: int, route: str | None) -> bytes: return b'{"error":"native-rate-limit"}' if route in {"ocr", "azure_ocr"}: return b'{"pages":[{"index":0,"markdown":"native-ocr"}]}' + if route == "azure_di": + return b'{"status":"succeeded","analyzeResult":{"pages":[]}}' if route == "transcription": return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' return ANTHROPIC_RESPONSE @@ -194,10 +203,22 @@ def azure_ocr_kwargs(api_base: str) -> dict[str, object]: "api_base": api_base, "custom_llm_provider": "azure_ai", "extra_headers": { - "Authorization": "Bearer prepared-azure-token", "x-test-outcome": "success", "x-test-route": "azure_ocr", }, + "optional_params": {"azure_ad_token": "prepared-azure-token"}, + } + + +def azure_di_kwargs(api_base: str) -> dict[str, object]: + return { + "model": "doc-intelligence/prebuilt-read", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "api_key": "di-key", + "api_base": api_base, + "custom_llm_provider": "azure_ai", + "extra_headers": {"x-test-outcome": "success", "x-test-route": "azure_di"}, + "optional_params": {"req_format": "native", "pages": [0, 2]}, } @@ -232,6 +253,8 @@ def exercise_sync(native: object, api_base: str) -> None: else: raise AssertionError(f"{route} accepted a 429 response") assert_success("ocr", native.ocr(**azure_ocr_kwargs(api_base))) + di_response: Final = native.ocr(**azure_di_kwargs(api_base)) + assert di_response["provider_native_response"]["status"] == "succeeded" async def exercise_async(native: object, api_base: str) -> None: @@ -245,6 +268,8 @@ async def exercise_async(native: object, api_base: str) -> None: else: raise AssertionError(f"a{route} accepted a 429 response") assert_success("ocr", await native.aocr(**azure_ocr_kwargs(api_base))) + di_response: Final = await native.aocr(**azure_di_kwargs(api_base)) + assert di_response["provider_native_response"]["status"] == "succeeded" async def exercise_async_concurrency(native: object, api_base: str) -> None: diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 2df91204390..71aa79cc4bb 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -8,16 +8,26 @@ cases need the extension and are skipped when it is not built. from __future__ import annotations import json +from types import MappingProxyType from typing import Final import pytest +import tiktoken +from tokenizers import Tokenizer import litellm +from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as bridge +from litellm.utils import claude_json_str MODEL: Final = "claude-sonnet-4-5-20250929" +CL100K_MODEL: Final = "gpt-4" +O200K_MODEL: Final = "gpt-4o" +TOKENIZERS: Final[tuple[bridge.RustTokenizer, ...]] = ("anthropic", "cl100k_base", "o200k_base") +RANK_FILE_LINES: Final = MappingProxyType({"cl100k_base": 100_256, "o200k_base": 199_998}) BODY: Final = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "hello"}]}).encode() @@ -44,69 +54,122 @@ class _RecordingCounter: return {"model": MODEL, "input_tokens": 42} -class _DecliningCounter: - def __init__(self, tokenizer_json: str) -> None: - pass +class _RecordingFactory: + """Stands in for the native `TokenCounter` class: callable for tokenizer JSON, `from_*_ranks` for rank files.""" + + def __init__(self) -> None: + self.counters: list[_RecordingCounter] = [] + self.rank_files: list[str] = [] + + def __call__(self, tokenizer_json: str) -> _RecordingCounter: + counter = _RecordingCounter(tokenizer_json) + self.counters.append(counter) + return counter + + def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: + self.rank_files.append(rank_file) + return self("cl100k_base") + + def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: + self.rank_files.append(rank_file) + return self("o200k_base") + + +class _RaisingCounter: + def __init__(self, error: Exception) -> None: + self.error = error async def acount_request(self, body: bytes) -> object: - raise _FakeDeclined("request has no messages") + raise self.error -class _FailingCounter: - def __init__(self, tokenizer_json: str) -> None: - pass +class _RaisingFactory: + """Every counter it builds, for either tokenizer, raises `error` on count.""" - async def acount_request(self, body: bytes) -> object: - raise RuntimeError("encode failed") + def __init__(self, error: Exception) -> None: + self.error = error + + def __call__(self, tokenizer_json: str) -> _RaisingCounter: + return _RaisingCounter(self.error) + + def from_cl100k_ranks(self, rank_file: str) -> _RaisingCounter: + return _RaisingCounter(self.error) + + def from_o200k_ranks(self, rank_file: str) -> _RaisingCounter: + return _RaisingCounter(self.error) @pytest.fixture(autouse=True) def _reset_bridge(monkeypatch: pytest.MonkeyPatch): bridge.TOKEN_COUNTER.reset() - bridge._anthropic_counter.cache_clear() + bridge._counter.cache_clear() configuration.reset_rust_configuration() monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) yield bridge.TOKEN_COUNTER.reset() - bridge._anthropic_counter.cache_clear() + bridge._counter.cache_clear() configuration.reset_rust_configuration() @pytest.mark.asyncio -async def test_disabled_bridge_never_constructs_a_counter() -> None: - constructed: list[str] = [] - - def factory(tokenizer_json: str) -> _RecordingCounter: - constructed.append(tokenizer_json) - return _RecordingCounter(tokenizer_json) - +@pytest.mark.parametrize("tokenizer", TOKENIZERS) +async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.RustTokenizer) -> None: + factory: Final = _RecordingFactory() litellm.rust(False) bridge.TOKEN_COUNTER.override(factory) - assert await bridge.count_anthropic_input_tokens(BODY) is None - assert constructed == [] + assert await bridge.count_input_tokens(BODY, tokenizer) is None + assert factory.counters == [] @pytest.mark.asyncio async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> None: - counters: list[_RecordingCounter] = [] - - def factory(tokenizer_json: str) -> _RecordingCounter: - counter = _RecordingCounter(tokenizer_json) - counters.append(counter) - return counter - + factory: Final = _RecordingFactory() litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_anthropic_input_tokens(BODY) - second: Final = await bridge.count_anthropic_input_tokens(BODY) + first: Final = await bridge.count_input_tokens(BODY, "anthropic") + second: Final = await bridge.count_input_tokens(BODY, "anthropic") assert first == bridge.InputTokenCount(model=MODEL, input_tokens=42) assert second == first - assert len(counters) == 1 - assert counters[0].bodies == [BODY, BODY] - assert json.loads(counters[0].tokenizer_json)["model"]["type"] == "BPE" + assert len(factory.counters) == 1 + assert factory.counters[0].bodies == [BODY, BODY] + assert json.loads(factory.counters[0].tokenizer_json)["model"]["type"] == "BPE" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) +async def test_tiktoken_counter_is_built_from_the_vendored_rank_file_once(tokenizer: bridge.RustTokenizer) -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + bridge.TOKEN_COUNTER.override(factory) + + first: Final = await bridge.count_input_tokens(BODY, tokenizer) + second: Final = await bridge.count_input_tokens(BODY, tokenizer) + + assert first == second == bridge.InputTokenCount(model=MODEL, input_tokens=42) + assert len(factory.rank_files) == 1 + assert factory.rank_files[0].startswith("IQ== 0\n") + assert factory.rank_files[0].count("\n") == RANK_FILE_LINES[tokenizer] + assert factory.counters[0].tokenizer_json == tokenizer + assert factory.counters[0].bodies == [BODY, BODY] + + +@pytest.mark.asyncio +async def test_each_tokenizer_gets_its_own_cached_counter() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + bridge.TOKEN_COUNTER.override(factory) + + await bridge.count_input_tokens(BODY, "anthropic") + await bridge.count_input_tokens(BODY, "cl100k_base") + await bridge.count_input_tokens(BODY, "o200k_base") + await bridge.count_input_tokens(BODY, "anthropic") + await bridge.count_input_tokens(BODY, "o200k_base") + + assert [counter.tokenizer_json for counter in factory.counters][1:] == ["cl100k_base", "o200k_base"] + assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] @pytest.mark.asyncio @@ -114,38 +177,131 @@ async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) litellm.rust(True) monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) - assert await bridge.count_anthropic_input_tokens(BODY) is None + assert [await bridge.count_input_tokens(BODY, tokenizer) for tokenizer in TOKENIZERS] == [None, None, None] @pytest.mark.asyncio -async def test_declined_request_falls_back() -> None: +@pytest.mark.parametrize("tokenizer", TOKENIZERS) +async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) - bridge.TOKEN_COUNTER.override(_DecliningCounter) + bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) - assert await bridge.count_anthropic_input_tokens(BODY) is None + assert await bridge.count_input_tokens(BODY, tokenizer) is None @pytest.mark.asyncio -async def test_runtime_failure_falls_back() -> None: +@pytest.mark.parametrize("tokenizer", TOKENIZERS) +async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) - bridge.TOKEN_COUNTER.override(_FailingCounter) + bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) - assert await bridge.count_anthropic_input_tokens(BODY) is None + assert await bridge.count_input_tokens(BODY, tokenizer) is None @pytest.mark.parametrize( ("model", "expected"), - ((MODEL, True), ("claude-3-5-sonnet-20241022", False), ("gpt-4o", False), ("my-router-alias", False)), + ( + (MODEL, "anthropic"), + ("claude-3-5-sonnet-20241022", "cl100k_base"), + ("gpt-4", "cl100k_base"), + ("gpt-4-turbo", "cl100k_base"), + ("gpt-3.5-turbo", "cl100k_base"), + ("azure/gpt-35-turbo", "cl100k_base"), + ("gemini/gemini-2.5-pro", "cl100k_base"), + ("mistral/mistral-large-latest", "cl100k_base"), + ("my-router-alias", "cl100k_base"), + ("azure/gpt-4o", "cl100k_base"), + ("command-r-plus", "cl100k_base"), + ("gpt-4o", "o200k_base"), + ("gpt-4o-mini", "o200k_base"), + ("gpt-4o-2024-08-06", "o200k_base"), + ("chatgpt-4o-latest", "o200k_base"), + ("gpt-4.1", "o200k_base"), + ("gpt-5", "o200k_base"), + ("gpt-5-mini", "o200k_base"), + ("o1", "o200k_base"), + ("o3", "o200k_base"), + ("o3-mini", "o200k_base"), + ("o4-mini", "o200k_base"), + ("replicate/meta/llama-2-70b-chat", None), + ("meta-llama/Llama-3-8b", None), + ), ) -def test_uses_anthropic_tokenizer_mirrors_python_tokenizer_selection(model: str, expected: bool) -> None: - assert bridge.uses_anthropic_tokenizer(model) is expected +def test_rust_tokenizer_mirrors_python_tokenizer_selection(model: str, expected: bridge.RustTokenizer | None) -> None: + assert bridge.rust_tokenizer(model) == expected -@pytest.mark.parametrize("flag", ("disable_hf_tokenizer_download", "disable_token_counter")) -def test_uses_anthropic_tokenizer_respects_python_opt_outs(monkeypatch: pytest.MonkeyPatch, flag: str) -> None: - monkeypatch.setattr(litellm, flag, True) +@pytest.mark.parametrize( + ("model", "python_encoding"), + (("text-davinci-003", "p50k_base"), ("gpt-oss-120b", "o200k_harmony")), +) +def test_rust_tokenizer_declines_tiktoken_encodings_rust_does_not_have( + monkeypatch: pytest.MonkeyPatch, model: str, python_encoding: str +) -> None: + monkeypatch.setattr(litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {model}) - assert bridge.uses_anthropic_tokenizer(MODEL) is False + assert openai_tokenizer_encoding(model).name == python_encoding + assert bridge.rust_tokenizer(model) is None + + +def test_rust_tokenizer_declines_the_cohere_tokenizer_download(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "cohere_models", litellm.cohere_models | {"command-r-plus"}) + + assert bridge.rust_tokenizer("command-r-plus") is None + + +@pytest.mark.parametrize("legacy_model", ("gpt-3.5-turbo-0301", "gpt-35-turbo-0301")) +def test_rust_tokenizer_declines_legacy_message_accounting_python_prices_differently( + monkeypatch: pytest.MonkeyPatch, legacy_model: str +) -> None: + monkeypatch.setattr( + litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"gpt-3.5-turbo-0301"} + ) + monkeypatch.setattr(litellm, "azure_llms", {**litellm.azure_llms, "gpt-35-turbo-0301": "azure"}) + messages: Final = [{"role": "user", "name": "bob", "content": "hello there"}] + + assert litellm.token_counter(model=legacy_model, messages=messages) != litellm.token_counter( + model=CL100K_MODEL, messages=messages + ) + assert bridge.rust_tokenizer(legacy_model) is None + assert bridge.rust_tokenizer(CL100K_MODEL) == "cl100k_base" + + +@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", "o3")) +def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: str) -> None: + text: Final = ( + "Hello, world! camelCase ABCdef \u00e9\u00e8 12345 \u3053\u3093\u306b\u3061\u306f <|endoftext|>\r\n" * 9 + ) + python_count: Final = litellm.token_counter(model=model, text=text) + cl100k_count: Final = len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=())) + o200k_count: Final = len(tiktoken.get_encoding("o200k_base").encode(text, disallowed_special=())) + assert cl100k_count != o200k_count + match bridge.rust_tokenizer(model): + case "cl100k_base": + assert python_count == cl100k_count + case "o200k_base": + assert python_count == o200k_count + case "anthropic": + assert python_count == len(Tokenizer.from_str(claude_json_str).encode(text).ids) + assert python_count not in {cl100k_count, o200k_count} + case None: + pytest.fail(f"{model} must have a Rust tokenizer") + + +def test_disabled_hf_download_routes_anthropic_models_to_cl100k_like_python(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + + assert bridge.rust_tokenizer(MODEL) == "cl100k_base" + assert bridge.rust_tokenizer("meta-llama/Llama-3-8b") == "cl100k_base" + assert bridge.rust_tokenizer(O200K_MODEL) == "o200k_base" + + +def test_disabled_token_counter_declines_every_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_token_counter", True) + + assert bridge.rust_tokenizer(MODEL) is None + assert bridge.rust_tokenizer(CL100K_MODEL) is None + assert bridge.rust_tokenizer(O200K_MODEL) is None PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = ( @@ -182,7 +338,16 @@ PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = ( }, { "model": MODEL, - "messages": [{"role": "user", "content": "x " * 20_000}], + "messages": [{"role": "user", "content": "x " * 500}], + }, + { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": "I'VE got 1234567 things; it's \"fine\"...\r\n\r\n caf\u00e9 \u0645\u0631\u062d\u0628\u0627 \U0001f600 <|endoftext|>", + } + ], }, {"model": MODEL, "prompt": "Write a haiku about ships.", "max_tokens": 20}, {"model": MODEL, "prompt": ["first prompt", "second prompt"]}, @@ -190,7 +355,7 @@ PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = ( "model": MODEL, "instructions": "be terse", "input": [ - {"role": "user", "content": [{"type": "input_text", "text": "Summarise caf\u00e9 menus \u2014 \"ok\"?\n"}]}, + {"role": "user", "content": [{"type": "input_text", "text": 'Summarise caf\u00e9 menus \u2014 "ok"?\n'}]}, {"role": "assistant", "content": "Sure."}, ], }, @@ -202,27 +367,63 @@ PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = ( ) +PARITY_MODELS: Final[tuple[tuple[str, bridge.RustTokenizer], ...]] = ( + (MODEL, "anthropic"), + (CL100K_MODEL, "cl100k_base"), + (O200K_MODEL, "o200k_base"), + ("gpt-5", "o200k_base"), +) + + @pytest.mark.asyncio +@pytest.mark.parametrize(("model", "tokenizer"), PARITY_MODELS) @pytest.mark.parametrize("request_body", PARITY_REQUESTS) async def test_native_count_matches_python_budget_counter( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object] + monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str, tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) litellm.rust(True) + body: Final = json.dumps(request_body).replace(MODEL, model) - rust_count: Final = await bridge.count_anthropic_input_tokens(json.dumps(request_body).encode()) - python_count: Final = _count_input_tokens(request_body=request_body, model=MODEL) + rust_count: Final = await bridge.count_input_tokens(body.encode(), tokenizer) + python_count: Final = _count_input_tokens(request_body=json.loads(body), model=model) assert rust_count is not None - assert rust_count.model == request_body.get("model") + assert rust_count.model == json.loads(body).get("model") assert rust_count.input_tokens == python_count +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "tokenizer"), ((CL100K_MODEL, "cl100k_base"), (O200K_MODEL, "o200k_base"))) +async def test_tiktoken_counts_long_text_exactly_where_python_chunks( + monkeypatch: pytest.MonkeyPatch, model: str, tokenizer: bridge.RustTokenizer +) -> None: + """Python encodes tiktoken text in fixed-size chunks (drift of up to one token per chunk boundary); Rust does not.""" + native: Final = pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + litellm.rust(True) + text: Final = "x " * 20_000 + body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} + encoding: Final = tiktoken.get_encoding(tokenizer) + exact: Final = 3 + len(encoding.encode("user")) + len(encoding.encode(text)) + 3 + chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) + + rust_count: Final = await bridge.count_input_tokens(json.dumps(body).encode(), tokenizer) + python_count: Final = _count_input_tokens(request_body=body, model=model) + + assert rust_count is not None + assert rust_count.input_tokens == exact + assert python_count is not None + assert exact < python_count <= exact + chunks + + DECLINED_REQUESTS: Final[tuple[dict[str, object], ...]] = ( { "model": MODEL, - "messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}}]}], + "messages": [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}}]} + ], }, {"model": MODEL, "prompt": 1.5}, {"model": MODEL, "documents": [{"score": 0.5}]}, @@ -231,12 +432,13 @@ DECLINED_REQUESTS: Final[tuple[dict[str, object], ...]] = ( @pytest.mark.asyncio +@pytest.mark.parametrize("tokenizer", TOKENIZERS) @pytest.mark.parametrize("request_body", DECLINED_REQUESTS) async def test_native_declines_shapes_python_prices_differently( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object] + monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) litellm.rust(True) - assert await bridge.count_anthropic_input_tokens(json.dumps(request_body).encode()) is None + assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None diff --git a/tests/test_litellm/test_fireworks_serverless_model_costs.py b/tests/test_litellm/test_fireworks_serverless_model_costs.py index 701938f5677..1303f46e8fa 100644 --- a/tests/test_litellm/test_fireworks_serverless_model_costs.py +++ b/tests/test_litellm/test_fireworks_serverless_model_costs.py @@ -14,6 +14,8 @@ import os import pytest +from litellm import completion_cost +from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info @@ -56,17 +58,38 @@ def test_bare_fireworks_ids_resolve_through_prefixed_entries(): assert info["max_output_tokens"] == expected["max_output_tokens"] +def test_deepseek_v4p1_flash_twin_costs(local_model_cost_map): + for model in ( + "fireworks_ai/deepseek-v4p1-flash", + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + ): + response = ModelResponse( + model=model, + choices=[Choices(index=0, message=Message(role="assistant", content="ok"))], + usage=Usage(prompt_tokens=1000, completion_tokens=1000, total_tokens=2000), + ) + cost = completion_cost(completion_response=response, model=model) + assert cost == pytest.approx(8.8e-04) + + TWIN_PINNED_PRICES = { "deepseek-v4-flash-0731": { "input_cost_per_token": 2.2e-07, "cache_read_input_token_cost": 7e-09, "output_cost_per_token": 6.6e-07, }, + "deepseek-v4p1-flash": { + "input_cost_per_token": 2.2e-07, + "cache_read_input_token_cost": 7e-09, + "output_cost_per_token": 6.6e-07, + "supports_vision": True, + "max_output_tokens": 393216, + }, } -def test_deepseek_v4_flash_0731_twins_pin_published_pricing(model_data): - """Both 0731 entries carry the price published at docs.fireworks.ai/serverless/pricing.""" +def test_deepseek_v4_flash_twins_pin_published_pricing(model_data): + """Both entries of each Flash twin pair carry the price published at docs.fireworks.ai/serverless/pricing.""" for bare_suffix, expected in TWIN_PINNED_PRICES.items(): for key in ( f"fireworks_ai/{bare_suffix}", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 2a97e92396a..f5e9b2091a0 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8,7 +8,7 @@ import threading from datetime import datetime from collections.abc import Awaitable, Callable, Mapping from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -2622,6 +2622,78 @@ def test_adopt_fallback_response_headers_keeps_identity_when_fallback_has_none() assert wrapper.fallback_headers_adopted is True +@pytest.mark.asyncio +@pytest.mark.parametrize("response_kind", ["object", "dict", "async-generator"]) +async def test_set_response_headers_exposes_complexity_decision_on_every_response_shape( + response_kind: Literal["object", "dict", "async-generator"], +) -> None: + class HeaderResponse: + def __init__(self) -> None: + self._hidden_params: dict[str, object] = {} + + response: object + if response_kind == "object": + response = HeaderResponse() + elif response_kind == "dict": + response = {} + else: + response = _AsyncList() + + router = Router(model_list=[]) + result = await router.set_response_headers( + response=response, + request_kwargs={ + "metadata": { + "routing_decision": { + "router_type": "complexity", + "tier": "SIMPLE", + "cause": "heuristic_scorer", + "score": 0.25, + "tier_litellm_params": {"reasoning_effort": "low"}, + } + } + }, + ) + hidden_params = result["_hidden_params"] if isinstance(result, dict) else result._hidden_params + additional_headers = hidden_params["additional_headers"] + + assert additional_headers == { + "x-litellm-model-group": None, + "x-litellm-complexity-router-tier": "SIMPLE", + "x-litellm-complexity-router-cause": "heuristic_scorer", + "x-litellm-complexity-router-score": "0.25", + "x-litellm-complexity-router-reasoning-effort": "low", + } + + +@pytest.mark.asyncio +async def test_set_response_headers_is_the_only_complexity_header_source_for_proxy_headers() -> None: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + router = Router(model_list=[]) + response = await router.set_response_headers(response={}, request_kwargs={}) + additional_headers = response["_hidden_params"]["additional_headers"] + proxy_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + request_data={ + "metadata": { + "routing_decision": { + "router_type": "complexity", + "tier": "REASONING", + "cause": "heuristic_scorer", + "tier_litellm_params": {"reasoning_effort": "xhigh"}, + } + } + }, + **additional_headers, + ) + + assert not { + key for key in proxy_headers if key.startswith("x-litellm-complexity-router-") + } + + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_adopts_fallback_response_headers(): """LIT-6767: after a successful pre-first-chunk fallback, the wrapper must @@ -3503,6 +3575,26 @@ def _make_router_with_fallback(primary="gpt-4", secondary="gpt-3.5-turbo"): ) +class _InjectedFallbackRouter(Router): + def __init__(self, fallback_response: object) -> None: + super().__init__(model_list=[]) + self._fallback_response: Final = fallback_response + + async def async_function_with_fallbacks_common_utils( + self, + e: Exception, + disable_fallbacks: bool | None, + fallbacks: list | None, + context_window_fallbacks: list | None, + content_policy_fallbacks: list | None, + model_group: str | None, + args: tuple[object, ...], + kwargs: dict[str, object], + include_fallback_errors: bool = False, + ) -> object: + return self._fallback_response + + @pytest.mark.asyncio async def test_aresponses_streaming_iterator_fallback(): """Catches MidStreamFallbackError, re-enters the fallback chain via @@ -3562,6 +3654,63 @@ async def test_aresponses_streaming_iterator_fallback(): assert call_kwargs["disable_fallbacks"] is False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallback_headers", + [ + {"x-fallback-only": "yes"}, + { + "x-fallback-only": "yes", + "x-litellm-complexity-router-tier": "SIMPLE", + }, + ], + ids=["plain-fallback", "complexity-tier-fallback"], +) +async def test_aresponses_streaming_iterator_replaces_complexity_headers_before_fallback_output( + fallback_headers: dict[str, str], +) -> None: + primary_headers: Final = { + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-reasoning-effort": "xhigh", + } + source: Final = _make_responses_iterator( + error=MidStreamFallbackError( + message="primary failed before output", + model="gpt-4", + llm_provider="openai", + is_pre_first_chunk=True, + generated_content="", + ), + hidden_params={"additional_headers": primary_headers}, + ) + fallback_output: Final = MagicMock(type="response.output_text.delta") + fallback: Final = _AsyncList([fallback_output]) + fallback._hidden_params = { + "model_id": "fallback-deployment", + "additional_headers": fallback_headers, + } + router: Final = _InjectedFallbackRouter(fallback) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=source, + initial_kwargs={ + "model": "gpt-4", + "stream": True, + "input": "Hello", + "original_generic_function": litellm.aresponses, + }, + ) + assert wrapped._hidden_params["additional_headers"] == primary_headers + first_output: Final = await wrapped.__anext__() + + assert first_output is fallback_output + assert wrapped.fallback_headers_adopted is True + assert wrapped._hidden_params == { + "model_id": "fallback-deployment", + "additional_headers": fallback_headers, + } + + @pytest.mark.asyncio async def test_aresponses_streaming_iterator_writes_litellm_metadata_on_fallback(): """Regression: model_group must land under "litellm_metadata" (the key @@ -12579,6 +12728,54 @@ async def test_anthropic_messages_fallback_merges_fallback_hidden_params(): assert headers["x-fallback-only"] == "yes" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallback_headers", + [ + {"x-fallback-only": "yes"}, + { + "x-fallback-only": "yes", + "x-litellm-complexity-router-tier": "SIMPLE", + }, + ], + ids=["plain-fallback", "complexity-tier-fallback"], +) +async def test_anthropic_messages_fallback_replaces_complexity_headers_before_output( + fallback_headers: dict[str, str], +) -> None: + primary_headers: Final = { + "x-litellm-complexity-router-tier": "REASONING", + "x-litellm-complexity-router-reasoning-effort": "xhigh", + } + source: Final = _AnthropicMessagesFallbackByteStream( + [_anthropic_messages_overloaded_error_chunk()], + hidden_params={"additional_headers": primary_headers}, + ) + fallback_output: Final = _anthropic_messages_content_chunk("fallback answer") + fallback: Final = _AnthropicMessagesFallbackByteStream( + [fallback_output], + hidden_params={ + "model_id": "fallback-deployment", + "additional_headers": fallback_headers, + }, + ) + router: Final = _InjectedFallbackRouter(fallback) + + wrapped: Final = await router._aanthropic_messages_streaming_iterator( + response=source, + initial_kwargs={"model": "primary"}, + ) + assert wrapped._hidden_params["additional_headers"] == primary_headers + first_output: Final = await wrapped.__anext__() + + assert first_output == fallback_output + assert wrapped.fallback_headers_adopted is True + assert wrapped._hidden_params == { + "model_id": "fallback-deployment", + "additional_headers": fallback_headers, + } + + @pytest.mark.asyncio async def test_aanthropic_messages_with_streaming_fallbacks_deep_copies_nested_metadata(): """Bugbot regression: a shallow .copy() of kwargs still shares the diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx index dad8599e967..ebc97891744 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx @@ -1,11 +1,11 @@ import React from "react"; -import { render, screen, waitFor, within } from "@testing-library/react"; +import { screen, waitFor, within } from "@testing-library/react"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import AddAgentForm from "./add_agent_form"; import * as networking from "@/components/networking"; import type { AgentCreateInfo } from "@/components/networking"; -import { chooseSelectOption } from "../../../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders as render } from "../../../../../tests/test-utils"; vi.mock("@/components/networking", () => ({ createAgentCall: vi.fn(), @@ -351,4 +351,33 @@ describe("AddAgentForm submit payload", () => { expect(await screen.findByText("Agent Created!")).toBeInTheDocument(); expect(within(screen.getByText("Agent Created!").parentElement!).getByText("created-agent")).toBeInTheDocument(); }); + it("blocks creation after clearing the existing key and assigns the reselected key", async () => { + vi.mocked(networking.keyListCall).mockResolvedValue({ + keys: [{ token: "key-maple", key_alias: "Maple key" }], + }); + const user = userEvent.setup(); + renderForm(); + await user.type(await screen.findByLabelText("Agent Name"), "key-selection-agent"); + await user.type(screen.getByLabelText("Display Name"), "Key selection"); + await user.type(screen.getByPlaceholderText("Describe what this agent does..."), "d"); + for (let step = 0; step < 3; step++) { + await user.click(screen.getByRole("button", { name: /^Next/ })); + } + await user.click(screen.getByRole("radio", { name: "Assign an existing key" })); + const keySelector = await screen.findByPlaceholderText("Search by key name…"); + await chooseSelectOption(user, keySelector, "Maple key"); + await user.click(screen.getByRole("button", { name: "Clear" })); + await user.click(screen.getByRole("button", { name: /Create Agent/ })); + expect(networking.createAgentCall).not.toHaveBeenCalled(); + expect(networking.keyUpdateCall).not.toHaveBeenCalled(); + await chooseSelectOption(user, keySelector, "Maple key"); + await user.click(screen.getByRole("button", { name: /Create Agent/ })); + await waitFor(() => + expect(networking.keyUpdateCall).toHaveBeenCalledWith("tok", { + key: "key-maple", + agent_id: "agent-1", + }), + ); + expect(networking.createAgentCall).toHaveBeenCalledTimes(1); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index 108bae977e1..e71fed40209 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -338,6 +338,11 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok return; } + if (keyAssignOption === "existing_key" && !selectedExistingKey) { + toast.error("Please select an existing key to assign"); + return; + } + setIsSubmitting(true); try { const isValid = await form.trigger(); @@ -406,12 +411,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok selectedTeamId, ); setCreatedKeyValue(keyResponse.key || null); - } else if (keyAssignOption === "existing_key") { - if (!selectedExistingKey) { - toast.error("Please select an existing key to assign"); - setIsSubmitting(false); - return; - } + } else if (keyAssignOption === "existing_key" && selectedExistingKey) { await keyUpdateCall(accessToken, { key: selectedExistingKey, agent_id: agentId, @@ -963,8 +963,8 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok setSelectedExistingKey(value || null)} + value={selectedExistingKey} + onValueChange={setSelectedExistingKey} options={existingKeys.map((k) => ({ label: k.key_alias || k.token?.slice(0, 12) + "…", value: k.token, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx index d85d26a21a8..06e332d2cfc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -170,8 +170,8 @@ interface StartFormValidityInputs { models: string[]; routerNames: string[]; direction: ShadowEvalDirection; - baselineModel: string; - judgeModel: string; + baselineModel: string | null; + judgeModel: string | null; percentage: string; maxBudget: string; } @@ -181,13 +181,13 @@ const startFormValidity = (inputs: StartFormValidityInputs) => { const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; const parsedMaxBudget = Number.parseFloat(inputs.maxBudget); const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; - const baselinePicked = inputs.direction === "forward" || inputs.baselineModel !== ""; + const baselinePicked = inputs.direction === "forward" || Boolean(inputs.baselineModel); const targetsPicked = inputs.apiKeyIds.length + inputs.teamIds.length + inputs.userIds.length > 0; const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS; const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1; const routersValid = routerCountValid && routersMatchDirection; const scopeValid = routersValid && (inputs.direction === "reverse" || inputs.models.length <= MAX_MODELS); - const modelsPicked = scopeValid && inputs.judgeModel !== "" && baselinePicked; + const modelsPicked = scopeValid && Boolean(inputs.judgeModel) && baselinePicked; const filled = targetsPicked && modelsPicked; const boundsValid = percentageValid && maxBudgetValid; const valid = Boolean(inputs.accessToken) && filled && boundsValid; @@ -201,7 +201,7 @@ interface StartBodyInputs { models: string[]; routerNames: string[]; direction: ShadowEvalDirection; - baselineModel: string; + baselineModel: string | null; shadowPercentage: number; durationDays: number; maxBudget: number; @@ -215,7 +215,7 @@ const buildStartBody = (inputs: StartBodyInputs) => ({ models: inputs.direction === "forward" ? inputs.models : [], router_names: inputs.routerNames, direction: inputs.direction, - ...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}), + ...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel ?? undefined } : {}), shadow_percentage: inputs.shadowPercentage, duration_days: inputs.durationDays, max_budget: inputs.maxBudget, @@ -230,10 +230,10 @@ export const StartForm: React.FC = () => { const [models, setModels] = useState([]); const [routerNames, setRouterNames] = useState([]); const [direction, setDirection] = useState("forward"); - const [baselineModel, setBaselineModel] = useState(""); + const [baselineModel, setBaselineModel] = useState(null); const [percentage, setPercentage] = useState("10"); const [durationDays, setDurationDays] = useState("7"); - const [judgeModel, setJudgeModel] = useState(""); + const [judgeModel, setJudgeModel] = useState(null); const [maxBudget, setMaxBudget] = useState("10"); const { data: autoRouters } = useAutoRouters(); const configuredGroups = usePlainModelGroups(); @@ -286,6 +286,7 @@ export const StartForm: React.FC = () => { }; const { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid } = startFormValidity(validityInputs); const handleStart = () => { + if (!valid || !judgeModel) return; const bodyInputs: StartBodyInputs = { apiKeyIds, teamIds, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/index.tsx index f3bd74260ad..b0612412059 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/index.tsx @@ -15,7 +15,7 @@ const generateId = () => `entry-${Date.now()}-${Math.random().toString(36).subst const createDefaultEntry = (): ModelEntry => ({ id: generateId(), - model: "", + model: null, input_tokens: 1000, output_tokens: 500, num_requests_per_day: undefined, @@ -28,7 +28,7 @@ const PricingCalculator: React.FC = ({ accessToken, mode const { debouncedFetchForEntry, removeEntry, getMultiModelResult } = useMultiCostEstimate(accessToken); const handleEntryChange = useCallback( - (id: string, field: keyof ModelEntry, value: string | number | undefined) => { + (id: string, field: keyof ModelEntry, value: string | number | null | undefined) => { setEntries((prev) => { const updated = prev.map((entry) => (entry.id === id ? { ...entry, [field]: value } : entry)); const changedEntry = updated.find((e) => e.id === id); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/types.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/types.ts index 250857a74f5..859d2f3e8d9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/types.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/types.ts @@ -4,7 +4,7 @@ export interface PricingCalculatorProps { } export interface PricingFormValues { - model: string; + model: string | null; input_tokens: number; output_tokens: number; num_requests_per_day?: number; @@ -13,7 +13,7 @@ export interface PricingFormValues { export interface ModelEntry { id: string; - model: string; + model: string | null; input_tokens: number; output_tokens: number; num_requests_per_day?: number; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index 1de4e697f64..d45cfc3fe7d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -48,7 +48,10 @@ const GUARDRAIL_MODES = [ ] as const; const submitGuardrailSchema = z.object({ - team_id: z.string().min(1, "Select a team"), + team_id: z + .string() + .nullable() + .pipe(z.string({ error: "Select a team" }).min(1, "Select a team")), guardrail_name: z.string().min(1, "Enter a guardrail name"), mode: z.string().min(1, "Select a mode"), api_base: z.string().min(1, "Enter the API base URL").refine(isValidUrl, "Must be a valid URL"), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx index 984b8135466..5785d0b37a1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx @@ -8,6 +8,8 @@ import { MemoryRow } from "@/components/networking"; import { MemoryTable } from "./MemoryTable"; +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + const makeMemory = (overrides: Partial = {}): MemoryRow => ({ memory_id: "mem-1", key: "user:profile", @@ -36,6 +38,23 @@ const baseProps = { }; describe("MemoryTable", () => { + it("links the User ID and Team ID cells to their detail pages", () => { + render(); + + expect(screen.getByRole("link", { name: "user-42" })).toHaveAttribute("href", "/ui/users?user=user-42"); + expect(screen.getByRole("link", { name: "team-7" })).toHaveAttribute("href", "/ui/teams?team=team-7"); + }); + + it("leaves the proxy admin and dashboard sentinels unlinked", () => { + const sentinelRow = makeMemory({ user_id: "default_user_id", team_id: "litellm-dashboard" }); + render(); + + expect(screen.getByText("default_user_id")).toBeInTheDocument(); + expect(screen.getByText("litellm-dashboard")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "default_user_id" })).not.toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "litellm-dashboard" })).not.toBeInTheDocument(); + }); + it("renders every column header", () => { render(); for (const header of ["ID", "Name", "Preview", "User ID", "Team ID", "Updated"]) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx index 6b2a6b08704..62b8ec624fa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx @@ -14,6 +14,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; +import { teamDetailHref, userDetailHref } from "@/utils/entityLinks"; interface MemoryRowActionsProps { row: MemoryRow; @@ -109,7 +110,10 @@ export const getMemoryTableColumns = ({ header: "User ID", size: 160, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => { + const userId = row.original.user_id; + return ; + }, }, { id: "team_id", @@ -118,7 +122,10 @@ export const getMemoryTableColumns = ({ header: "Team ID", size: 160, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => { + const teamId = row.original.team_id; + return ; + }, }, { id: "updated_at", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx index 5c7dbb18428..8482d0832c3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx @@ -278,7 +278,7 @@ export function AllModelsTable({ options={modelGroupOptions} value={(get(MODEL_NAME_COLUMN_ID) as string) ?? ALL_MODEL_GROUPS_VALUE} onValueChange={(value) => - set(MODEL_NAME_COLUMN_ID, value === ALL_MODEL_GROUPS_VALUE ? undefined : value) + set(MODEL_NAME_COLUMN_ID, value === ALL_MODEL_GROUPS_VALUE ? undefined : value ?? undefined) } placeholder="Filter by Public Model Name" emptyText="No models found" @@ -289,7 +289,7 @@ export function AllModelsTable({ options={accessGroupOptions} value={(get(ACCESS_GROUPS_COLUMN_ID) as string) ?? ALL_MODEL_GROUPS_VALUE} onValueChange={(value) => - set(ACCESS_GROUPS_COLUMN_ID, value === ALL_MODEL_GROUPS_VALUE ? undefined : value) + set(ACCESS_GROUPS_COLUMN_ID, value === ALL_MODEL_GROUPS_VALUE ? undefined : value ?? undefined) } placeholder="Filter by Model Access Group" emptyText="No model access groups found" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 1443c065d9b..1cdce04d07c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -26,7 +26,7 @@ export default function AddModelPanel() { const { data: modelCostMapData } = useModelCostMap(); const { data: credentialsResponse } = useCredentials(); const { data: teams } = useTeams(); - const [selectedProvider, setSelectedProvider] = useState(Providers.Anthropic); + const [selectedProvider, setSelectedProvider] = useState(Providers.Anthropic); const [providerModels, setProviderModels] = useState([]); const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); @@ -57,7 +57,9 @@ export default function AddModelPanel() { selectedProvider={selectedProvider} setSelectedProvider={setSelectedProvider} providerModels={providerModels} - setProviderModelsFn={(provider) => setProviderModels(getProviderModels(provider, modelCostMapData))} + setProviderModelsFn={(provider) => + setProviderModels(provider === null ? [] : getProviderModels(provider, modelCostMapData)) + } getPlaceholder={getPlaceholder} showAdvancedSettings={showAdvancedSettings} setShowAdvancedSettings={setShowAdvancedSettings} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx similarity index 95% rename from ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx index bf6b092a2b0..984996351df 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx @@ -153,6 +153,7 @@ describe("ChatUI", () => { }); it("should allow the user to select a model", async () => { + const user = userEvent.setup(); render( { await waitFor(() => { expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0); }); + await user.click(screen.getByRole("option", { name: "Model 1Mode: chat" })); + expect(screen.getByPlaceholderText("Select a Model")).toHaveValue("Model 1"); + + await user.click(screen.getAllByRole("button", { name: "Clear" })[0]); + const input = screen.getByPlaceholderText("Describe the image you want to generate..."); + fireEvent.change(input, { target: { value: "Contract endpoint check" } }); + expect(screen.getByRole("button", { name: "Send message" })).toBeDisabled(); + fireEvent.keyDown(input, { key: "Enter", code: "Enter" }); + expect(input).toHaveValue("Contract endpoint check"); + expect(makeOpenAIChatCompletionRequest).not.toHaveBeenCalled(); + expect(sessionStorage.getItem("endpointType")).toBeNull(); + + await selectComboboxOption("Select an endpoint", "/v1/chat/completions"); + await selectComboboxOption("Select a Model", "Model 1"); + expect(screen.getByRole("button", { name: "Send message" })).toBeEnabled(); }); it("shows only endpoint-compatible models when chat endpoint is selected", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index eca9e2323ae..ed8679cfdc1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -196,17 +196,17 @@ const ChatUI: React.FC = ({ () => sessionStorage.getItem("customProxyBaseUrl") || "", ); const [inputMessage, setInputMessage] = useState(""); - const [selectedModel, setSelectedModel] = useState(simplified ? fixedModel : undefined); + const [selectedModel, setSelectedModel] = useState(simplified ? fixedModel : null); const [showCustomModelInput, setShowCustomModelInput] = useState(false); const [modelInfo, setModelInfo] = useState([]); const [isLoadingModels, setIsLoadingModels] = useState(false); const [modelLoadError, setModelLoadError] = useState(false); const [agentInfo, setAgentInfo] = useState([]); - const [selectedAgent, setSelectedAgent] = useState(undefined); + const [selectedAgent, setSelectedAgent] = useState(null); const debouncedSetSelectedModel = useDebouncedCallback((value: string) => setSelectedModel(value), { wait: CUSTOM_MODEL_DEBOUNCE_WAIT_MS, }); - const [endpointType, setEndpointType] = useState( + const [endpointType, setEndpointType] = useState( () => sessionStorage.getItem("endpointType") || EndpointType.CHAT, ); const [isLoading, setIsLoading] = useState(false); @@ -327,7 +327,7 @@ const ChatUI: React.FC = ({ }; useEffect(() => { - if (isGetCodeModalVisible) { + if (isGetCodeModalVisible && endpointType !== null) { const code = generateCodeSnippet({ apiKeySource, accessToken, @@ -342,7 +342,7 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, endpointType, - selectedModel, + selectedModel: selectedModel ?? undefined, selectedSdk, selectedVoice, proxySettings, @@ -376,7 +376,8 @@ const ChatUI: React.FC = ({ } catch { // Storage full or unavailable — non-critical, skip persisting. } - sessionStorage.setItem("endpointType", endpointType); + if (endpointType === null) sessionStorage.removeItem("endpointType"); + else sessionStorage.setItem("endpointType", endpointType); sessionStorage.setItem("selectedTags", JSON.stringify(selectedTags)); sessionStorage.setItem("selectedVectorStores", JSON.stringify(selectedVectorStores)); sessionStorage.setItem("selectedGuardrails", JSON.stringify(selectedGuardrails)); @@ -493,7 +494,7 @@ const ChatUI: React.FC = ({ setAgentInfo(agents); // Clear selection if current agent not in list if (selectedAgent && !agents.some((a) => a.agent_name === selectedAgent)) { - setSelectedAgent(undefined); + setSelectedAgent(null); } } catch (error) { console.error("Error fetching agents:", error); @@ -616,10 +617,11 @@ const ChatUI: React.FC = ({ setUploadedAudio(file); }; - const handleEndpointChange = (value: string) => { + const handleEndpointChange = (value: string | null) => { setEndpointType(value); - setSelectedModel(undefined); - setSelectedAgent(undefined); + setGeneratedCode(""); + setSelectedModel(null); + setSelectedAgent(null); setShowCustomModelInput(false); setSelectedMCPDirectTool(undefined); if (value === EndpointType.MCP) { @@ -710,6 +712,11 @@ const ChatUI: React.FC = ({ }; const handleSendMessage = async () => { + if (endpointType === null) { + toast.fromError("Please select an endpoint before sending a request"); + return; + } + if (inputMessage.trim() === "" && endpointType !== EndpointType.TRANSCRIPTION && endpointType !== EndpointType.MCP) return; @@ -1152,7 +1159,7 @@ const ChatUI: React.FC = ({ toast.success("Chat history cleared."); }; - const onModelChange = (value: string) => { + const onModelChange = (value: string | null) => { setSelectedModel(value); setShowCustomModelInput(value === "custom"); @@ -1210,6 +1217,7 @@ const ChatUI: React.FC = ({ : "Describe the image you want to generate..."; const sendDisabled = + endpointType === null || isLoading || (endpointType === EndpointType.MCP ? !(selectedMCPServers.length === 1 && selectedMCPServers[0] !== "__all__" && selectedMCPDirectTool) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointSelector.tsx index 644bab5b5a4..6b286c403a8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointSelector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointSelector.tsx @@ -3,8 +3,8 @@ import React from "react"; import { ENDPOINT_OPTIONS } from "./chatConstants"; interface EndpointSelectorProps { - endpointType: string; // Accept string to avoid type conflicts - onEndpointChange: (value: string) => void; + endpointType: string | null; + onEndpointChange: (value: string | null) => void; className?: string; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/SessionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/SessionManagement.tsx index 35c8e839159..4dbf9168897 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/SessionManagement.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/SessionManagement.tsx @@ -7,7 +7,7 @@ import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; interface SessionManagementProps { - endpointType: string; + endpointType: string | null; responsesSessionId: string | null; useApiSessionManagement: boolean; onToggleSessionManagement: (useApi: boolean) => void; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx index 5659875cf82..8ecc0e0c8bb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx @@ -22,7 +22,8 @@ export function ModelSelector({ value, onChange, models, loading, disabled }: Mo const selectValue = isAddingCustom ? "__custom__" : value || undefined; - const handleSelectChange = (selected: string) => { + const handleSelectChange = (selected: string | null) => { + if (selected === null) return; if (selected === "__custom__") { setIsAddingCustom(true); if (value && !options.includes(value)) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.integration.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.integration.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.tsx index 2830b0fda12..d14165dd849 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.tsx @@ -44,10 +44,10 @@ const policyShape = { .min(1, "Please enter a policy name") .regex(/^[a-zA-Z0-9_-]+$/, "Policy name can only contain letters, numbers, hyphens, and underscores"), description: z.string(), - inherit: z.string(), + inherit: z.string().nullable(), guardrails_add: z.array(z.string()), guardrails_remove: z.array(z.string()), - model_condition: z.string(), + model_condition: z.string().nullable(), }; const policySchema = z.object(policyShape); @@ -57,19 +57,19 @@ type PolicyFormValues = z.infer; const EMPTY_VALUES: PolicyFormValues = { policy_name: "", description: "", - inherit: "", + inherit: null, guardrails_add: [], guardrails_remove: [], - model_condition: "", + model_condition: null, }; const toFormValues = (policy: Policy): PolicyFormValues => ({ policy_name: policy.policy_name, description: policy.description ?? "", - inherit: policy.inherit ?? "", + inherit: policy.inherit ?? null, guardrails_add: policy.guardrails_add || [], guardrails_remove: policy.guardrails_remove || [], - model_condition: policy.condition?.model ?? "", + model_condition: policy.condition?.model ?? null, }); const buildPolicyRequest = (values: PolicyFormValues): PolicyCreateRequest | PolicyUpdateRequest => ({ @@ -529,7 +529,7 @@ const AddPolicyForm: React.FC = ({ {...control} id={id} ref={ref} - value={value} + value={value ?? ""} onChange={onChange} placeholder="Leave empty to apply to all models (e.g., gpt-4.* or bedrock/claude-.*)" /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx index 32e777949d0..c4f76b07a4e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx @@ -68,7 +68,7 @@ const AiSuggestionModal: React.FC = ({ const [suggestions, setSuggestions] = useState(null); const [explanation, setExplanation] = useState(null); const [selectedIds, setSelectedIds] = useState>(new Set()); - const [selectedModel, setSelectedModel] = useState(undefined); + const [selectedModel, setSelectedModel] = useState(null); const [availableModels, setAvailableModels] = useState([]); const [isLoadingModels, setIsLoadingModels] = useState(false); // Test panel state @@ -114,7 +114,7 @@ const AiSuggestionModal: React.FC = ({ setSuggestions(null); setExplanation(null); setSelectedIds(new Set()); - setSelectedModel(undefined); + setSelectedModel(null); setShowTestPanel(false); setTestInputText(""); setIsTestLoading(false); @@ -837,7 +837,7 @@ const AiSuggestionModal: React.FC = ({ ({ label: m, value: m }))} value={selectedModel} - onValueChange={(value) => setSelectedModel(value || undefined)} + onValueChange={setSelectedModel} placeholder={isLoadingModels ? "Loading models..." : "Select a model to analyze your requirements"} emptyText="No models found" disabled={isLoadingModels} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx index 8651be39f0d..da77b348028 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx @@ -343,7 +343,7 @@ const StepCard: React.FC = ({ onChange({ guardrail: value })} + onValueChange={(value) => onChange({ guardrail: value ?? undefined })} placeholder="Select a guardrail" emptyText="No guardrails found" /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx index 410d4ea8467..b90379b149d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx @@ -46,7 +46,7 @@ const TemplateParameterModal: React.FC = ({ }) => { const [parameterValues, setParameterValues] = useState>({}); const [competitorMode, setCompetitorMode] = useState<"ai" | "manual">("ai"); - const [selectedModel, setSelectedModel] = useState(undefined); + const [selectedModel, setSelectedModel] = useState(null); const [availableModels, setAvailableModels] = useState([]); const [isLoadingModels, setIsLoadingModels] = useState(false); const [competitorTags, setCompetitorTags] = useState([]); @@ -72,7 +72,7 @@ const TemplateParameterModal: React.FC = ({ }); setParameterValues(initial); setCompetitorMode("ai"); - setSelectedModel(undefined); + setSelectedModel(null); setCompetitorTags([]); setVariationsMap({}); setIsGenerating(false); @@ -297,7 +297,7 @@ const TemplateParameterModal: React.FC = ({ ({ label: m, value: m }))} value={selectedModel} - onValueChange={(value) => setSelectedModel(value || undefined)} + onValueChange={setSelectedModel} placeholder={isLoadingModels ? "Loading models..." : "Select a model to generate names"} emptyText="No models found" disabled={isLoadingModels} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx index 5c1a443d6c5..77f28b05ea5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx @@ -10,7 +10,7 @@ import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useUpdateProject, ProjectUpdateParams } from "@/app/(dashboard)/hooks/projects/useUpdateProject"; import { ProjectBaseForm } from "./ProjectBaseForm"; -import { projectFormSchema, type ProjectFormValues } from "./projectFormSchema"; +import { projectFormSchema, type ProjectFormValues, type ProjectSubmitValues } from "./projectFormSchema"; import { buildProjectUpdateParams } from "./projectFormUtils"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; @@ -58,7 +58,7 @@ export const toFormValues = (project: ProjectResponse): ProjectFormValues => { return { project_alias: project.project_alias ?? "", - team_id: project.team_id ?? "", + team_id: project.team_id ?? null, description: project.description ?? "", models: project.models ?? [], max_budget: project.litellm_budget_table?.max_budget ?? undefined, @@ -81,7 +81,7 @@ function EditProjectForm({ project, onClose, onSuccess }: Omit { - const submitted: ProjectFormValues = advancedEverOpened + const submitted: ProjectSubmitValues = advancedEverOpened ? values : { ...values, guardrails: undefined, modelLimits: undefined, metadata: undefined }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx index fed1a88fe98..1ad8a1e4953 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx @@ -5,7 +5,7 @@ import { useFieldArray, useWatch, type UseFormReturn } from "react-hook-form"; import { ChevronDown, CircleAlert, Minus, Plus } from "lucide-react"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { ALL_TEAM_MODELS, type ProjectFormValues } from "./projectFormSchema"; +import { ALL_TEAM_MODELS, type ProjectFormValues, type ProjectSubmitValues } from "./projectFormSchema"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Team } from "@/components/key_team_helpers/key_list"; import { fetchTeamModels } from "@/components/organisms/create_key_button"; @@ -32,7 +32,7 @@ const toOptionalNumber = (raw: string): number | undefined => { }; interface ProjectBaseFormProps { - form: UseFormReturn; + form: UseFormReturn; advancedOpen: boolean; onAdvancedOpenChange: (open: boolean) => void; } @@ -94,7 +94,7 @@ export function ProjectBaseForm({ form, advancedOpen, onAdvancedOpenChange }: Pr } }, [selectedTeam, accessToken, userId, userRole]); - const handleTeamChange = (teamId: string) => { + const handleTeamChange = (teamId: string | null) => { const team = teams?.find((t) => t.team_id === teamId) ?? null; setSelectedTeam(team); form.setValue("models", []); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormSchema.ts b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormSchema.ts index 6bfaa831bba..d4c85d89616 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormSchema.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormSchema.ts @@ -16,7 +16,10 @@ const modelLimitSchema = z.object({ export const projectFormSchema = z .object({ project_alias: z.string().min(1, "Please enter a project name"), - team_id: z.string().min(1, "Please select a team"), + team_id: z + .string() + .nullable() + .pipe(z.string({ error: "Please select a team" }).min(1, "Please select a team")), description: z.string().optional(), models: z.array(z.string()), max_budget: z.number().optional(), @@ -48,11 +51,12 @@ export const projectFormSchema = z }); }); -export type ProjectFormValues = z.output; +export type ProjectFormValues = z.input; +export type ProjectSubmitValues = z.output; export const emptyProjectFormValues: ProjectFormValues = { project_alias: "", - team_id: "", + team_id: null, description: undefined, models: [], max_budget: undefined, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx index edbb897fb2f..009abf93d1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx @@ -10,6 +10,8 @@ vi.mock("@/components/networking", () => ({ modelHubCall: vi.fn().mockResolvedValue({ data: [] }), })); +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + const mockPrompts: PromptSpec[] = [ { prompt_id: "prompt-newer", @@ -53,6 +55,15 @@ describe("PromptTable", () => { } }); + it("links the Created By cell to the creator's detail page, leaving the placeholder unlinked", () => { + const prompts = [mockPrompts[0], { ...mockPrompts[1], created_by: "default_user_id" }]; + render(); + + expect(screen.getByRole("link", { name: "user-1" })).toHaveAttribute("href", "/ui/users?user=user-1"); + expect(screen.getByText("default_user_id")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "default_user_id" })).not.toBeInTheDocument(); + }); + it("should display the empty state when data is empty", () => { render(); expect(screen.getByText("No prompts yet")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx index f927a6d1486..db0392bbd6e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx @@ -17,6 +17,7 @@ import { } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; +import { userDetailHref } from "@/utils/entityLinks"; import { extractModel, getProviderFromModelHub, ModelGroupInfo } from "./prompt_utils"; @@ -191,9 +192,16 @@ export const getPromptTableColumns = ({ enableSorting: false, cell: ({ row }) => { const createdBy = row.original.created_by; + if (!createdBy) { + return -; + } return ( - - {createdBy || "-"} + + ); }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/ModelConfigCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/ModelConfigCard.tsx index a14e45de99a..e9b526c9703 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/ModelConfigCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/ModelConfigCard.tsx @@ -6,11 +6,11 @@ import { SettingsIcon } from "lucide-react"; import ModelSelector from "@/components/common_components/ModelSelector"; interface ModelConfigCardProps { - model: string; + model: string | null; temperature?: number; maxTokens?: number; accessToken: string | null; - onModelChange: (model: string) => void; + onModelChange: (model: string | null) => void; onTemperatureChange: (temp: number) => void; onMaxTokensChange: (tokens: number) => void; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx index 04cac01365a..0f001979c28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx @@ -21,7 +21,7 @@ interface PromptEditorHeaderProps { editMode?: boolean; onShowHistory?: () => void; version?: string | null; - promptModel?: string; + promptModel?: string | null; promptVariables?: Record; accessToken: string | null; proxySettings?: { @@ -85,7 +85,7 @@ const PromptEditorHeader: React.FC = ({
{ expect(result).toContain("output:"); expect(result).toContain("format: text"); expect(result).toContain("User: Hello world"); + const cleared = convertToDotPrompt({ ...prompt, model: null }); + expect(cleared).toBe(result.replace("model: gpt-4\n", "")); }); it("should include config parameters when set", () => { @@ -203,6 +205,20 @@ describe("convertToDotPrompt", () => { }); describe("parseExistingPrompt", () => { + it("should keep saved prompts with missing or blank models unassigned", () => { + for (const modelLine of ["", "model: \n"]) { + const prompt = parseExistingPrompt({ + prompt_spec: { + prompt_id: "unassigned-prompt", + litellm_params: { dotprompt_content: `---\n${modelLine}temperature: 0\n---\nUser: Keep this message` }, + }, + }); + + expect(prompt.model).toBeNull(); + expect(convertToDotPrompt(prompt)).not.toMatch(/^model:/m); + } + }); + it("should parse basic dotprompt content", () => { const apiResponse = { prompt_spec: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/utils.ts b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/utils.ts index ef327d93495..3d768e930a9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/utils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/utils.ts @@ -23,7 +23,7 @@ export const extractVariables = (prompt: PromptType): string[] => { export const convertToDotPrompt = (prompt: PromptType): string => { const variables = extractVariables(prompt); - let result = `---\nmodel: ${prompt.model}\n`; + let result = prompt.model ? `---\nmodel: ${prompt.model}\n` : "---\n"; // Add temperature if set if (prompt.config.temperature !== undefined) { @@ -237,7 +237,7 @@ export const parseExistingPrompt = (apiResponse: any): PromptType => { return { name: baseName, - model: parsedFrontmatter.model || "gpt-4o", + model: parsedFrontmatter.model || null, config: parsedFrontmatter.config, tools: parsedFrontmatter.tools, developerMessage: parsedBody.developerMessage, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx index 269f3b0af39..0239844851f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx @@ -20,7 +20,12 @@ import { useZodForm } from "@/lib/forms/useZodForm"; import { fetchClient } from "@/lib/http/api"; import { buildBody, settingsToForm, type DefaultInternalUserParams, type InternalUserSettings } from "./mapper"; -import { defaultUserSettingsSchema, EMPTY_TEAM_ROW, type DefaultUserSettingsFormValues } from "./schema"; +import { + defaultUserSettingsSchema, + EMPTY_TEAM_ROW, + type DefaultUserSettingsFormValues, + type DefaultUserSettingsSubmitValues, +} from "./schema"; const NO_RESET = "never"; @@ -63,7 +68,7 @@ interface RoleOption { description: string; } -type SettingsControl = Control; +type SettingsControl = Control; const TeamPickerField = ({ control, index }: { control: SettingsControl; index: number }) => { const [search, setSearch] = React.useState(""); @@ -233,7 +238,7 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on const { isDirty } = form.formState; const mutation = useMutation({ - mutationFn: (values: DefaultUserSettingsFormValues) => updateSettings(buildBody(values)), + mutationFn: (values: DefaultUserSettingsSubmitValues) => updateSettings(buildBody(values)), onSuccess: (_result, values) => { toast.success("Default user settings updated successfully"); queryClient.invalidateQueries({ queryKey: SETTINGS_QUERY_KEY }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.test.ts index e8b350332c8..f4a7f001480 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildBody, settingsToForm } from "./mapper"; -import type { DefaultUserSettingsFormValues } from "./schema"; +import type { DefaultUserSettingsSubmitValues } from "./schema"; const CONFIGURED_SETTINGS = { user_role: "internal_user", @@ -59,13 +59,13 @@ describe("settingsToForm", () => { it("degrades an unrecognisable team entry to a blank row instead of throwing", () => { expect(settingsToForm({ teams: [{ max_budget_in_team: 5 }, 7] }).teams).toStrictEqual([ - { team_id: "", max_budget_in_team: "", user_role: "user" }, - { team_id: "", max_budget_in_team: "", user_role: "user" }, + { team_id: null, max_budget_in_team: "", user_role: "user" }, + { team_id: null, max_budget_in_team: "", user_role: "user" }, ]); }); }); -const formValues = (overrides: Partial = {}): DefaultUserSettingsFormValues => ({ +const formValues = (overrides: Partial = {}): DefaultUserSettingsSubmitValues => ({ user_role: "internal_user", max_budget: "100", budget_duration: "30d", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.ts b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.ts index 50365081afb..ac528542ae2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/mapper.ts @@ -2,7 +2,12 @@ import { z } from "zod/v4"; import type { components } from "@/lib/http/schema"; -import { EMPTY_TEAM_ROW, type DefaultTeamRowValues, type DefaultUserSettingsFormValues } from "./schema"; +import { + EMPTY_TEAM_ROW, + type DefaultTeamRowValues, + type DefaultUserSettingsFormValues, + type DefaultUserSettingsSubmitValues, +} from "./schema"; export type InternalUserSettings = components["schemas"]["InternalUserSettingsResponse"]; export type DefaultInternalUserParams = components["schemas"]["DefaultInternalUserParams"]; @@ -60,13 +65,13 @@ const textOrNull = (raw: string): string | null => (raw.trim() === "" ? null : r const listOrNull = (items: readonly T[]): T[] | null => (items.length === 0 ? null : [...items]); -const toTeamBody = (team: DefaultTeamRowValues): DefaultTeamBody => ({ +const toTeamBody = (team: DefaultUserSettingsSubmitValues["teams"][number]): DefaultTeamBody => ({ team_id: team.team_id, max_budget_in_team: numberOrNull(team.max_budget_in_team), user_role: team.user_role, }); -export const buildBody = (values: DefaultUserSettingsFormValues): DefaultInternalUserParams => ({ +export const buildBody = (values: DefaultUserSettingsSubmitValues): DefaultInternalUserParams => ({ user_role: asDefaultUserRole(values.user_role), max_budget: numberOrNull(values.max_budget), budget_duration: textOrNull(values.budget_duration), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/schema.ts b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/schema.ts index 7309e3745da..cd66c3addc8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/schema.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/schema.ts @@ -10,14 +10,17 @@ const amountOrEmpty = z ); const defaultTeamRowSchema = z.object({ - team_id: z.string().min(1, "Select a team"), + team_id: z + .string() + .nullable() + .pipe(z.string({ error: "Select a team" }).min(1, "Select a team")), max_budget_in_team: amountOrEmpty, user_role: z.enum(["user", "admin"]), }); -export type DefaultTeamRowValues = z.output; +export type DefaultTeamRowValues = z.input; -export const EMPTY_TEAM_ROW: DefaultTeamRowValues = { team_id: "", max_budget_in_team: "", user_role: "user" }; +export const EMPTY_TEAM_ROW: DefaultTeamRowValues = { team_id: null, max_budget_in_team: "", user_role: "user" }; const defaultUserSettingsShape = { user_role: z.string(), @@ -41,4 +44,5 @@ export const defaultUserSettingsSchema = z.object(defaultUserSettingsShape).supe ); }); -export type DefaultUserSettingsFormValues = z.output; +export type DefaultUserSettingsFormValues = z.input; +export type DefaultUserSettingsSubmitValues = z.output; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx index 875df74652c..8fa76a64af6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx @@ -192,7 +192,7 @@ export function UsersTable({ set("user_role", value)} + onValueChange={(value) => set("user_role", value ?? undefined)} placeholder="Select a role…" emptyText="No roles found" /> @@ -201,7 +201,7 @@ export function UsersTable({ set("team", value)} + onValueChange={(value) => set("team", value ?? undefined)} placeholder="Select a team…" emptyText="No teams found" /> diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index 0f7c356b8cc..83a72e50f5c 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -59,7 +59,7 @@ interface UISettings { interface CreateUserFormValues { user_email?: string; user_role: string; - team_id?: string; + team_id?: string | null; organization_ids?: string[]; metadata?: string; send_invite_email: boolean; diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.test.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.test.tsx index ee2f42ad85b..31cd407a5e6 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.test.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.test.tsx @@ -5,6 +5,8 @@ import { renderWithProviders } from "../../../tests/test-utils"; import DeletedKeysPage from "./DeletedKeysPage"; import { useDeletedKeys, DeletedKeyResponse } from "@/app/(dashboard)/hooks/keys/useKeys"; +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useDeletedKeys: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.test.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.test.tsx index 7e30ef2c135..dd8007be264 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.test.tsx @@ -5,6 +5,8 @@ import { renderWithProviders } from "../../../../tests/test-utils"; import { DeletedKeysTable } from "./DeletedKeysTable"; import { DeletedKeyResponse } from "@/app/(dashboard)/hooks/keys/useKeys"; +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + const makeDeletedKey = (overrides: Partial = {}): DeletedKeyResponse => ({ token: "sk-1234567890abcdef", @@ -86,3 +88,19 @@ it("should show the empty state when there are no deleted keys", () => { expect(screen.getByText("No deleted keys found")).toBeInTheDocument(); }); + +it("links the owner, creator and deleter cells to their user detail pages", () => { + renderWithProviders(); + + expect(screen.getByRole("link", { name: "user-1" })).toHaveAttribute("href", "/ui/users?user=user-1"); + expect(screen.getByRole("link", { name: "creator-1" })).toHaveAttribute("href", "/ui/users?user=creator-1"); + expect(screen.getByRole("link", { name: "deleter-1" })).toHaveAttribute("href", "/ui/users?user=deleter-1"); +}); + +it("leaves the default_user_id placeholder unlinked", () => { + const placeholderKey = makeDeletedKey({ user_id: "default_user_id", created_by: "default_user_id" }); + renderWithProviders(); + + expect(screen.getAllByText("default_user_id")).toHaveLength(2); + expect(screen.queryByRole("link", { name: "default_user_id" })).not.toBeInTheDocument(); +}); diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTableColumns.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTableColumns.tsx index aa7d6380cc3..32185f867b4 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTableColumns.tsx @@ -3,8 +3,9 @@ import { ColumnDef } from "@tanstack/react-table"; import { DataTableSortHeader } from "@/components/shared/DataTable"; -import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; +import { DateCell, IdCell, IdentityCell, MoneyCell } from "@/components/shared/table_cells"; import { DeletedKeyResponse } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { userDetailHref } from "@/utils/entityLinks"; function TruncatedTextCell({ value }: { value: string | null | undefined }) { if (!value) { @@ -17,6 +18,17 @@ function TruncatedTextCell({ value }: { value: string | null | undefined }) { ); } +function UserLinkCell({ userId }: { userId: string | null | undefined }) { + if (!userId) { + return -; + } + return ( + + + + ); +} + export const getDeletedKeysTableColumns = (): ColumnDef[] => [ { id: "token", @@ -89,7 +101,7 @@ export const getDeletedKeysTableColumns = (): ColumnDef[] => header: "User ID", size: 120, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => , }, { id: "created_at", @@ -107,7 +119,7 @@ export const getDeletedKeysTableColumns = (): ColumnDef[] => header: "Created By", size: 120, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => , }, { id: "deleted_at", @@ -125,6 +137,6 @@ export const getDeletedKeysTableColumns = (): ColumnDef[] => header: "Deleted By", size: 120, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => , }, ]; diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.test.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.test.tsx index 952d8764463..d2d25f5e47e 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.test.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.test.tsx @@ -5,6 +5,8 @@ import { renderWithProviders } from "../../../tests/test-utils"; import DeletedTeamsPage from "./DeletedTeamsPage"; import { useDeletedTeams, DeletedTeam } from "@/app/(dashboard)/hooks/teams/useTeams"; +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ useDeletedTeams: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.test.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.test.tsx index e166f6b0d1b..837fb583f30 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.test.tsx @@ -4,6 +4,8 @@ import { renderWithProviders } from "../../../../tests/test-utils"; import { DeletedTeamsTable } from "./DeletedTeamsTable"; import { DeletedTeam } from "@/app/(dashboard)/hooks/teams/useTeams"; +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + const makeDeletedTeam = (overrides: Partial = {}): DeletedTeam => ({ team_id: "team-1", team_alias: "Test Team", @@ -81,3 +83,21 @@ it("renders the shared pagination footer with the server row count", () => { expect(screen.getByTestId("pagination-prev")).toBeEnabled(); expect(screen.getByTestId("pagination-next")).toBeDisabled(); }); + +it("links the organization and deleted by cells, leaving the deleted team id unlinked", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("link", { name: "org-1" })).toHaveAttribute("href", "/ui/organizations?org=org-1"); + expect(screen.getByRole("link", { name: "user-1" })).toHaveAttribute("href", "/ui/users?user=user-1"); + expect(screen.queryByRole("link", { name: "team-1" })).not.toBeInTheDocument(); +}); + +it("leaves the default_user_id placeholder unlinked in the deleted by cell", () => { + const team = makeDeletedTeam({ deleted_by: "default_user_id", organization_id: null }); + renderWithProviders(); + + expect(screen.getByText("default_user_id")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "default_user_id" })).not.toBeInTheDocument(); +}); diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTableColumns.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTableColumns.tsx index e36077fd2c3..172f0417027 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTableColumns.tsx @@ -3,8 +3,20 @@ import { ColumnDef } from "@tanstack/react-table"; import { DataTableSortHeader } from "@/components/shared/DataTable"; -import { DateCell, IdCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; +import { DateCell, IdCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; import { DeletedTeam } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { orgDetailHref, userDetailHref } from "@/utils/entityLinks"; + +function EntityCell({ value, href }: { value: string | null | undefined; href: string | undefined }) { + if (!value) { + return -; + } + return ( + + + + ); +} export const getDeletedTeamsTableColumns = (): ColumnDef[] => [ { @@ -78,7 +90,10 @@ export const getDeletedTeamsTableColumns = (): ColumnDef[] => [ header: "Organization", size: 150, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => { + const orgId = row.original.organization_id; + return ; + }, }, { id: "deleted_at", @@ -97,15 +112,8 @@ export const getDeletedTeamsTableColumns = (): ColumnDef[] => [ size: 120, enableSorting: false, cell: ({ row }) => { - const value = row.original.deleted_by; - if (!value) { - return -; - } - return ( - - {value} - - ); + const deletedBy = row.original.deleted_by; + return ; }, }, ]; diff --git a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx index 535694f14da..f4c9cc7c44c 100644 --- a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx +++ b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx @@ -1,4 +1,4 @@ -import { renderWithProviders, screen } from "../../tests/test-utils"; +import { fireEvent, renderWithProviders, screen } from "../../tests/test-utils"; import { vi } from "vitest"; import { EnvCredentialLoginWarningBanner } from "./EnvCredentialLoginWarningBanner"; import type { HealthReadinessDetailsResponse } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; @@ -23,6 +23,22 @@ const mockRole = (userRole: string) => { }; describe("EnvCredentialLoginWarningBanner", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("should hide the banner when dismissed and stay hidden on remount", () => { + mockRole("Admin"); + mockDetails({ status: "healthy", show_env_credential_login_warning: true }); + const first = renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Dismiss banner" })); + expect(first.container).toBeEmptyDOMElement(); + + first.unmount(); + const second = renderWithProviders(); + expect(second.container).toBeEmptyDOMElement(); + }); + it("should warn an admin when env-credential login is enabled", () => { mockRole("Admin"); mockDetails({ status: "healthy", show_env_credential_login_warning: true }); diff --git a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx index 3a9d50011f1..a2cd531759e 100644 --- a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx +++ b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx @@ -1,26 +1,37 @@ "use client"; -import React from "react"; -import { TriangleAlert } from "lucide-react"; +import React, { useState } from "react"; +import { TriangleAlert, X } from "lucide-react"; +import { Button } from "@/components/ui/button"; import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; import { useAuth } from "@/contexts/AuthContext"; import { isAdminRole } from "@/utils/roles"; +const DISMISS_STORAGE_KEY = "litellm:envCredentialLoginWarningDismissed"; + export const EnvCredentialLoginWarningBanner: React.FC<{ accessToken: string | null }> = ({ accessToken }) => { const { userRole } = useAuth(); const { data: healthData } = useHealthReadinessDetails(accessToken); + const [dismissed, setDismissed] = useState( + () => typeof window !== "undefined" && localStorage.getItem(DISMISS_STORAGE_KEY) === "true", + ); - if (!isAdminRole(userRole) || !healthData?.show_env_credential_login_warning) { + if (dismissed || !isAdminRole(userRole) || !healthData?.show_env_credential_login_warning) { return null; } + const handleDismiss = () => { + localStorage.setItem(DISMISS_STORAGE_KEY, "true"); + setDismissed(true); + }; + return (
onRoutingChange(value === "" ? undefined : value)} + value={routing} + onValueChange={(value) => onRoutingChange(value ?? undefined)} placeholder="Inherit from the request's own compression guardrails" emptyText="No compression guardrails found" aria-label="Routing decision compression" @@ -74,8 +74,8 @@ const CompressionControls: React.FC = ({ value, onChan
onModelChange(value === "" ? undefined : value)} + value={model} + onValueChange={(value) => onModelChange(value ?? undefined)} placeholder="None (no compression)" emptyText="No compression guardrails found" aria-label="Model call compression" diff --git a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.tsx b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.integration.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.tsx rename to ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.integration.test.tsx index ecd5abfa451..495fd207266 100644 --- a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.integration.test.tsx @@ -47,7 +47,7 @@ describe("RouterConfigBuilder", () => { expect(onChange).toHaveBeenCalledWith({ routes: [ expect.objectContaining({ - name: "", + name: null, utterances: [], description: "", score_threshold: 0.5, diff --git a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.ts b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.ts new file mode 100644 index 00000000000..d681c911c46 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { serializeRouterConfig } from "./RouterConfigBuilder"; + +describe("serializeRouterConfig", () => { + it("rejects a cleared route model and preserves selected route settings", () => { + expect(() => serializeRouterConfig({ routes: [{ name: null }] })).toThrow("Please select a model for every route"); + const config = { routes: [{ name: "model-silver", utterances: [], description: "", score_threshold: 0 }] }; + expect(JSON.parse(serializeRouterConfig(config))).toEqual(config); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx index 6dfb9f32091..9713d18872b 100644 --- a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx +++ b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx @@ -15,7 +15,7 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp interface Route { id: string; - model: string; + model: string | null; utterances: string[]; description: string; score_threshold: number; @@ -23,21 +23,28 @@ interface Route { interface SavedRoute { id?: string; - name?: string; - model?: string; + name?: string | null; + model?: string | null; utterances?: string[]; description?: string; score_threshold?: number; } -interface RouterConfig { +export interface RouterConfig { routes?: SavedRoute[]; } +export function serializeRouterConfig(config: RouterConfig | null): string { + if (config?.routes?.some((route) => !(route.name ?? route.model))) { + throw new Error("Please select a model for every route"); + } + return JSON.stringify(config); +} + interface RouterConfigBuilderProps { modelInfo: ModelGroup[]; - value?: RouterConfig; - onChange?: (config: any) => void; + value?: RouterConfig | null; + onChange?: (config: RouterConfig) => void; } interface UtteranceInputProps { @@ -136,7 +143,7 @@ const RouterConfigBuilder: React.FC = ({ modelInfo, va routeIds.push(id); return { id, - model: route.name || route.model || "", + model: route.name || route.model || null, utterances: route.utterances || [], description: route.description || "", score_threshold: route.score_threshold ?? 0.5, @@ -165,7 +172,7 @@ const RouterConfigBuilder: React.FC = ({ modelInfo, va const newRouteId = `route-${Date.now()}`; const updatedRoutes = [ ...routes, - { id: newRouteId, model: "", utterances: [], description: "", score_threshold: 0.5 }, + { id: newRouteId, model: null, utterances: [], description: "", score_threshold: 0.5 }, ]; setRoutes(updatedRoutes); updateConfig(updatedRoutes); diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx index 5684d32a319..0a5a9a5bfb6 100644 --- a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx @@ -61,7 +61,9 @@ const SemanticKeywordMatching: React.FC = ({ { + if (model !== null) onEmbeddingModelChange(model); + }} placeholder="Select an embedding model" emptyText="No embedding models found" aria-label="Embedding model" diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index d7c4a671317..632f4427a82 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -150,7 +150,10 @@ export const getSubmitBlockedReason = ( const autoRouterSchema = (requiresTeamScope: boolean) => z.object({ auto_router_name: z.string().min(1, "Auto router name is required"), - team_id: requiresTeamScope ? z.string().min(1, "Please select a team to continue") : z.string(), + team_id: z + .string() + .nullable() + .refine((teamId) => !requiresTeamScope || Boolean(teamId), "Please select a team to continue"), model_access_group: z.array(z.string()).optional(), }); @@ -158,12 +161,12 @@ type AddAutoRouterFormValues = z.infer>; const EMPTY_FORM_VALUES: AddAutoRouterFormValues = { auto_router_name: "", - team_id: "", + team_id: null, model_access_group: undefined, }; -const teamScopePayload = (requiresTeamScope: boolean, teamId: string): { team_id?: string } => - requiresTeamScope ? { team_id: teamId } : {}; +const teamScopePayload = (requiresTeamScope: boolean, teamId: string | null): { team_id?: string } => + requiresTeamScope && teamId ? { team_id: teamId } : {}; const BlockedReasonTooltip: React.FC<{ reason: string | null; children: React.ReactElement }> = ({ reason, @@ -458,7 +461,7 @@ const AddAutoRouterTab: React.FC = ({ const serverVerdict = await validateAutoRouterConfig( accessToken, complexityRouterConfigPayload as unknown as Record, - requiresTeamScope ? form.getValues("team_id") : undefined, + requiresTeamScope ? form.getValues("team_id") ?? undefined : undefined, ); const dryRunError = dryRunRejection(serverVerdict); if (dryRunError) { @@ -630,9 +633,7 @@ const AddAutoRouterTab: React.FC = ({ "Select the team this auto router belongs to. Only keys for this team will be able to call it.", )} > - {({ id, value, onChange }) => ( - onChange(next ?? "")} /> - )} + {({ id, value, onChange }) => } )} @@ -776,7 +777,7 @@ const AddAutoRouterTab: React.FC = ({ config={buildComplexityRouterConfig(complexityRouterConfigParams)} defaultModel={resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model)} routerName={watchedName} - teamId={requiresTeamScope ? watchedTeamId : undefined} + teamId={requiresTeamScope ? watchedTeamId ?? undefined : undefined} /> )} diff --git a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx index bfa2df8f54d..8cf4b077f39 100644 --- a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx @@ -8,9 +8,9 @@ import { MountedFormField, type MountedFormValues } from "../common_components/M import { Providers } from "../provider_info_helpers"; interface LiteLLMModelNameFieldProps { - selectedProvider: Providers; + selectedProvider: string | null; providerModels: string[]; - getPlaceholder: (provider: Providers) => string; + getPlaceholder: (provider: string) => string; } const LiteLLMModelNameField: React.FC = ({ @@ -123,7 +123,7 @@ const LiteLLMModelNameField: React.FC = ({ id={control.id} value={(control.value as string | undefined) ?? ""} onBlur={control.onBlur} - placeholder={getPlaceholder(selectedProvider)} + placeholder={selectedProvider === null ? "Select a provider first" : getPlaceholder(selectedProvider)} onChange={(event) => { control.onChange(event); if (selectedProvider === Providers.Azure) { @@ -147,7 +147,7 @@ const LiteLLMModelNameField: React.FC = ({ value: "custom", }, { - label: `All ${selectedProvider} Models (Wildcard)`, + label: `All ${selectedProvider ?? "provider"} Models (Wildcard)`, value: "all-wildcard", }, ...providerModels.map((model) => ({ @@ -163,7 +163,7 @@ const LiteLLMModelNameField: React.FC = ({ value={(control.value as string | undefined) ?? ""} onChange={control.onChange} onBlur={control.onBlur} - placeholder={getPlaceholder(selectedProvider)} + placeholder={selectedProvider === null ? "Select a provider first" : getPlaceholder(selectedProvider)} /> ) } diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index e1a47e9792d..4add7b918f7 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -18,7 +18,7 @@ import { provider_map, Providers } from "../provider_info_helpers"; import { labelWithHint } from "@/components/shared/form/LabelWithHint"; interface ProviderSpecificFieldsProps { - selectedProvider: Providers; + selectedProvider: string | null; } const readTextFile = (file: File, onLoaded: (contents: string) => void) => { @@ -168,6 +168,7 @@ const ProviderSpecificFields: React.FC = ({ selecte }, [cacheEntries]); const allFields = React.useMemo(() => { + if (selectedProvider === null) return []; // First try to resolve from the in-memory cache. We support both the // enum/display-name form and the raw provider slug (e.g. "petals"). const cachedFields = diff --git a/ui/litellm-dashboard/src/components/common_components/MemberTable.test.tsx b/ui/litellm-dashboard/src/components/common_components/MemberTable.test.tsx new file mode 100644 index 00000000000..a16f11eab60 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/MemberTable.test.tsx @@ -0,0 +1,215 @@ +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +import type { Member } from "@/components/networking"; + +import { renderWithProviders } from "../../../tests/test-utils"; +import MemberTable, { MemberTableColumn, memberRoleOptions } from "./MemberTable"; + +const MEMBERS: Member[] = [ + { user_id: "u-zed", user_email: "zed@example.com", user_alias: "Zed Ortiz", role: "user" }, + { user_id: "u-nameless", user_email: "mystery@example.com", user_alias: null, role: "user" }, + { user_id: "u-amy", user_email: "amy@example.com", user_alias: "amy chen", role: "admin" }, + { user_id: "u-bob", user_email: null, user_alias: "Bob Lee", role: "user" }, +]; + +const BUDGETS: Record = { "u-zed": 50, "u-nameless": null, "u-amy": 1000, "u-bob": 5 }; + +const budgetColumn: MemberTableColumn = { + title: "Budget", + key: "budget", + sortValue: (member) => BUDGETS[member.user_id ?? ""] ?? null, + render: (member) => {BUDGETS[member.user_id ?? ""] ?? "Unlimited"}, +}; + +const renderTable = (overrides: Partial> = {}) => { + const props = { + members: MEMBERS, + canEdit: true, + onEdit: vi.fn(), + onDelete: vi.fn(), + extraColumns: [budgetColumn], + ...overrides, + }; + renderWithProviders(); + return props; +}; + +const rowIds = (): (string | null)[] => + Array.from(document.querySelectorAll("tbody tr[data-row-id]")).map((row) => row.getAttribute("data-row-id")); + +const search = (value: string) => fireEvent.change(screen.getByTestId("datatable-search"), { target: { value } }); + +describe("MemberTable display", () => { + it("shows each member's name, falling back to a dash when there is none", () => { + renderTable(); + + expect(within(screen.getByRole("row", { name: /zed@example\.com/ })).getByText("Zed Ortiz")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /^name/i })).toBeInTheDocument(); + const cells = within(screen.getByRole("row", { name: /mystery@example\.com/ })).getAllByRole("cell"); + expect(cells[0]).toHaveTextContent("-"); + }); + + it("orders members by name with nameless members last by default", () => { + renderTable(); + + expect(rowIds()).toEqual(["u-amy", "u-bob", "u-zed", "u-nameless"]); + }); + + it("reports the full member count", () => { + renderTable(); + + expect(screen.getByText("4 Members")).toBeInTheDocument(); + }); +}); + +describe("MemberTable search", () => { + it("matches on name case-insensitively", async () => { + renderTable(); + + search("ZED"); + + await waitFor(() => expect(rowIds()).toEqual(["u-zed"])); + }); + + it("matches on email", async () => { + renderTable(); + + search("mystery@"); + + await waitFor(() => expect(rowIds()).toEqual(["u-nameless"])); + }); + + it("matches on user id", async () => { + renderTable(); + + search("u-bob"); + + await waitFor(() => expect(rowIds()).toEqual(["u-bob"])); + }); + + it("still searches names when the first member has no name", async () => { + renderTable({ members: [MEMBERS[1], MEMBERS[0]] }); + + search("ortiz"); + + await waitFor(() => expect(rowIds()).toEqual(["u-zed"])); + }); + + it("does not match on role", async () => { + renderTable(); + + search("admin"); + + await waitFor(() => expect(rowIds()).toEqual([])); + expect(screen.getByText("No members match your search or filters")).toBeInTheDocument(); + }); +}); + +describe("MemberTable sorting", () => { + it("sorts by email with missing emails last in both directions", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("sort-header-user_email")); + expect(rowIds()).toEqual(["u-amy", "u-nameless", "u-zed", "u-bob"]); + + await user.click(screen.getByTestId("sort-header-user_email")); + expect(rowIds()).toEqual(["u-zed", "u-nameless", "u-amy", "u-bob"]); + }); + + it("sorts by role", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("sort-header-role")); + expect(rowIds()[0]).toBe("u-amy"); + + await user.click(screen.getByTestId("sort-header-role")); + expect(rowIds()[3]).toBe("u-amy"); + }); + + it("sorts an extra column numerically by its sort value with blanks last", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("sort-header-budget")); + expect(rowIds()).toEqual(["u-bob", "u-zed", "u-amy", "u-nameless"]); + + await user.click(screen.getByTestId("sort-header-budget")); + expect(rowIds()).toEqual(["u-amy", "u-zed", "u-bob", "u-nameless"]); + }); + + it("flips name order on the second click", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("sort-header-user_alias")); + expect(rowIds()).toEqual(["u-zed", "u-bob", "u-amy", "u-nameless"]); + }); + + it("leaves extra columns without a sort value unsortable", () => { + renderTable({ + extraColumns: [{ title: "Rate Limits", key: "rate_limits", render: () => No Limits }], + }); + + expect(screen.getByRole("columnheader", { name: "Rate Limits" })).toBeInTheDocument(); + expect(screen.queryByTestId("sort-header-rate_limits")).not.toBeInTheDocument(); + }); +}); + +describe("MemberTable role filter", () => { + it("shows only members with the chosen role and clears on reset", async () => { + const user = userEvent.setup(); + renderTable({ roleColumnTitle: "Team Role" }); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await user.click(screen.getByTestId("filter-role")); + await user.click(await screen.findByRole("option", { name: "admin" })); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(rowIds()).toEqual(["u-amy"])); + expect(screen.getByTestId("filter-chip-role")).toHaveTextContent("Team Role"); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await user.click(screen.getByTestId("filter-drawer-reset")); + + await waitFor(() => expect(rowIds()).toHaveLength(4)); + }); + + it("offers the roles present in the roster", () => { + expect(memberRoleOptions(MEMBERS)).toEqual(["admin", "user"]); + expect(memberRoleOptions([{ user_id: "x", role: "" }])).toEqual([]); + }); +}); + +describe("MemberTable actions", () => { + it("passes the clicked member to onEdit and onDelete", async () => { + const user = userEvent.setup(); + const { onEdit, onDelete } = renderTable(); + const row = screen.getByRole("row", { name: /amy@example\.com/ }); + + await user.click(within(row).getByTestId("edit-member")); + await user.click(within(row).getByTestId("delete-member")); + + expect(onEdit).toHaveBeenCalledWith(MEMBERS[2]); + expect(onDelete).toHaveBeenCalledWith(MEMBERS[2]); + }); + + it("hides delete for members the caller excludes", () => { + renderTable({ showDeleteForMember: (member) => member.role !== "admin" }); + + expect(screen.getAllByTestId("delete-member")).toHaveLength(3); + expect( + within(screen.getByRole("row", { name: /amy@example\.com/ })).queryByTestId("delete-member"), + ).not.toBeInTheDocument(); + }); + + it("shows the empty text when there are no members at all", () => { + renderTable({ members: [], emptyText: "No members found" }); + + expect(screen.getByText("No members found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx b/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx index 8a0b7671078..e54994c1ab7 100644 --- a/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx +++ b/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx @@ -1,17 +1,29 @@ -import { SimpleTooltip } from "@/components/ui/tooltip"; +import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table"; +import { Crown, Info, User, UserPlus } from "lucide-react"; +import React, { useState } from "react"; + import { Member } from "@/components/networking"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableSortHeader, + DataTableToolbar, +} from "@/components/shared/DataTable"; import { StatusBadge } from "@/components/shared/table_cells"; import { Button } from "@/components/ui/button"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; -import { Crown, Info, User, UserPlus } from "lucide-react"; -import React from "react"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { SimpleTooltip } from "@/components/ui/tooltip"; + import TableIconActionButton from "./IconActionButton/TableIconActionButtons/TableIconActionButton"; +export type MemberTableSortValue = string | number | null | undefined; + export interface MemberTableColumn { title: React.ReactNode; - key: React.Key; - dataIndex?: keyof Member; - render?: (value: Member[keyof Member], member: Member, index: number) => React.ReactNode; + key: string; + render: (member: Member) => React.ReactNode; + sortValue?: (member: Member) => MemberTableSortValue; } export interface MemberTableProps { @@ -27,12 +39,152 @@ export interface MemberTableProps { emptyText?: string; } -const extraColumnCell = (column: MemberTableColumn, member: Member, index: number): React.ReactNode => { - const value = column.dataIndex ? member[column.dataIndex] : undefined; - return column.render ? column.render(value, member, index) : value; +const ALL_ROLES = "all"; + +export const memberRowId = (member: Member): string => member.user_id ?? member.user_email ?? JSON.stringify(member); + +export const memberRoleOptions = (members: readonly Member[]): string[] => + Array.from(new Set(members.map((member) => member.role).filter((role) => role !== ""))).sort(); + +const isAdminRole = (role: string): boolean => { + const normalized = role.toLowerCase(); + return normalized === "admin" || normalized === "org_admin"; }; -const STICKY_ACTIONS_CLASS = "sticky right-0 w-[120px] bg-background"; +function RoleHeaderTitle({ title, tooltip }: { title: string; tooltip?: string }) { + if (tooltip === undefined) return <>{title}; + return ( + + {title} + + + + + ); +} + +const ACTIONS_COLUMN_WIDTH = 120; + +interface MemberColumnDeps { + canEdit: boolean; + onEdit: (member: Member) => void; + onDelete: (member: Member) => void; + roleColumnTitle: string; + roleTooltip?: string; + extraColumns: MemberTableColumn[]; + showDeleteForMember?: (member: Member) => boolean; +} + +const extraColumnDef = (column: MemberTableColumn): ColumnDef => { + const { sortValue } = column; + if (sortValue === undefined) { + return { + id: column.key, + header: () => {column.title}, + enableSorting: false, + enableGlobalFilter: false, + cell: ({ row }) => column.render(row.original), + }; + } + return { + id: column.key, + accessorFn: (member) => sortValue(member) ?? undefined, + header: ({ column: tableColumn }) => , + sortDescFirst: false, + sortUndefined: "last", + enableGlobalFilter: false, + cell: ({ row }) => column.render(row.original), + }; +}; + +const buildColumns = ({ + canEdit, + onEdit, + onDelete, + roleColumnTitle, + roleTooltip, + extraColumns, + showDeleteForMember, +}: MemberColumnDeps): ColumnDef[] => [ + { + id: "user_alias", + accessorFn: (member) => member.user_alias || undefined, + header: ({ column }) => , + sortingFn: "text", + sortUndefined: "last", + enableGlobalFilter: true, + meta: { title: "Name" }, + cell: ({ row }) => row.original.user_alias || -, + }, + { + id: "user_email", + accessorFn: (member) => member.user_email || undefined, + header: ({ column }) => , + sortingFn: "text", + sortUndefined: "last", + enableGlobalFilter: true, + meta: { title: "User Email" }, + cell: ({ row }) => row.original.user_email || "-", + }, + { + id: "user_id", + accessorFn: (member) => member.user_id ?? undefined, + header: "User ID", + enableSorting: false, + enableGlobalFilter: true, + cell: ({ row }) => + row.original.user_id === "default_user_id" ? ( + + ) : ( + row.original.user_id || "-" + ), + }, + { + id: "role", + accessorFn: (member) => member.role, + header: ({ column }) => ( + } /> + ), + sortingFn: "text", + filterFn: "equalsString", + enableGlobalFilter: false, + meta: { title: roleColumnTitle }, + cell: ({ row }) => ( + + {isAdminRole(row.original.role) ? : } + {row.original.role || "-"} + + ), + }, + ...extraColumns.map(extraColumnDef), + { + id: "actions", + header: "Actions", + size: ACTIONS_COLUMN_WIDTH, + enableSorting: false, + enableGlobalFilter: false, + meta: { pinned: "right" }, + cell: ({ row }) => + canEdit ? ( + + onEdit(row.original)} + /> + {(!showDeleteForMember || showDeleteForMember(row.original)) && ( + onDelete(row.original)} + /> + )} + + ) : null, + }, +]; export default function MemberTable({ members, @@ -46,90 +198,89 @@ export default function MemberTable({ showDeleteForMember, emptyText, }: MemberTableProps) { + const [globalFilter, setGlobalFilter] = useState(""); + const [columnFilters, setColumnFilters] = useState([]); + const [filtersOpen, setFiltersOpen] = useState(false); + + const columnDeps: MemberColumnDeps = { + canEdit, + onEdit, + onDelete, + roleColumnTitle, + roleTooltip, + extraColumns, + showDeleteForMember, + }; + const columns = buildColumns(columnDeps); + const roleFilterItems = [ + { value: ALL_ROLES, label: "All Roles" }, + ...memberRoleOptions(members).map((role) => ({ value: role, label: role })), + ]; + + const isNarrowed = globalFilter !== "" || columnFilters.length > 0; + return (
{members.length} Member{members.length !== 1 ? "s" : ""} - - - - User Email - User ID - - {roleTooltip ? ( - - {roleColumnTitle} - - - - - ) : ( - roleColumnTitle + + {isNarrowed ? "No members match your search or filters" : emptyText ?? "No data"} + + } + toolbar={(table) => ( + <> + setFiltersOpen(true)} + showViewOptions={false} + /> + + {({ get, set }) => ( + + + )} - - {extraColumns.map((column) => ( - {column.title} - ))} - Actions - - - - {members.length === 0 ? ( - - - {emptyText ?? "No data"} - - - ) : ( - members.map((member, memberIndex) => ( - - {member.user_email || "-"} - - {member.user_id === "default_user_id" ? ( - - ) : ( - member.user_id || "-" - )} - - - - {member.role?.toLowerCase() === "admin" || member.role?.toLowerCase() === "org_admin" ? ( - - ) : ( - - )} - {member.role || "-"} - - - {extraColumns.map((column) => ( - {extraColumnCell(column, member, memberIndex)} - ))} - - {canEdit ? ( - - onEdit(member)} - /> - {(!showDeleteForMember || showDeleteForMember(member)) && ( - onDelete(member)} - /> - )} - - ) : null} - - - )) - )} - -
+ + + )} + /> {onAddMember && canEdit && ( diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 8a62e86e842..86f18ee9b12 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -13,7 +13,7 @@ import AccessGroupTagsCombobox from "../add_model/AccessGroupTagsCombobox"; import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceCombobox"; import { modelAvailableCall, modelPatchUpdateCall, validateAutoRouterConfig } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; -import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; +import RouterConfigBuilder, { type RouterConfig, serializeRouterConfig } from "../add_model/RouterConfigBuilder"; import { hydrateTierModelParams } from "../add_model/complexity_router_tiers"; import { type ActiveTierSet, @@ -452,7 +452,7 @@ const EditAutoRouterModal: React.FC = ({ const [modelInfo, setModelInfo] = useState([]); const [showValidationErrors, setShowValidationErrors] = useState(false); const [editingTiers, setEditingTiers] = useState(false); - const [routerConfig, setRouterConfig] = useState(null); + const [routerConfig, setRouterConfig] = useState(null); const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); const [keywordTierRules, setKeywordTierRules] = useState([]); const [escalationKeywords, setEscalationKeywords] = useState([]); @@ -706,7 +706,7 @@ const EditAutoRouterModal: React.FC = ({ // Prepare the updated litellm_params const updatedLitellmParams = { ...modelData.litellm_params, - auto_router_config: JSON.stringify(routerConfig), + auto_router_config: serializeRouterConfig(routerConfig), auto_router_default_model: values.auto_router_default_model, auto_router_embedding_model: values.auto_router_embedding_model || undefined, }; @@ -745,7 +745,7 @@ const EditAutoRouterModal: React.FC = ({ })(); } catch (error) { console.error("Error updating auto router:", error); - toast.fromError("Failed to update auto router configuration"); + toast.fromError(error); } finally { setLoading(false); } diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx index c0c12356c6f..dbc270fa62e 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx @@ -96,10 +96,10 @@ export function BudgetFallbacksEditor({ value, onChange, availableModels }: Budg ({ label: m, value: m }))} - value={entry.primaryModel ?? ""} + value={entry.primaryModel} onValueChange={(v) => { const newFallbacks = entry.fallbackModels.filter((m) => m !== v); - updateEntry(entry.id, { primaryModel: v === "" ? null : v, fallbackModels: newFallbacks }); + updateEntry(entry.id, { primaryModel: v, fallbackModels: newFallbacks }); }} placeholder="Select model" emptyText="No models found" diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx index ef5a3e15b52..0fab5555343 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx @@ -153,8 +153,8 @@ export function ModelMaxBudgetEditor({ ({ label: model, value: model }))} - value={entry.model ?? ""} - onValueChange={(model) => updateEntry(entry.id, { model: model === "" ? null : model })} + value={entry.model} + onValueChange={(model) => updateEntry(entry.id, { model })} placeholder="Select model" emptyText="No models found" disabled={!premiumUser} diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx index 8108cc1c20e..fdc02b4fc41 100644 --- a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx +++ b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx @@ -42,7 +42,7 @@ export default function CredentialModal({ existingCredential = null, }: CredentialModalProps) { const isEdit = mode === "edit"; - const [selectedProvider, setSelectedProvider] = useState( + const [selectedProvider, setSelectedProvider] = useState( (existingCredential?.credential_info.custom_llm_provider as Providers) ?? Providers.OpenAI, ); @@ -110,7 +110,7 @@ export default function CredentialModal({ {(control) => ( { control.onChange(value); - resetCredentialFormOnProviderChange(formAdapter, value as Providers, setSelectedProvider); + resetCredentialFormOnProviderChange(formAdapter, value, setSelectedProvider); }} /> )} diff --git a/ui/litellm-dashboard/src/components/model_add/credential_form_helpers.ts b/ui/litellm-dashboard/src/components/model_add/credential_form_helpers.ts index 7190c539c81..e7b1e861811 100644 --- a/ui/litellm-dashboard/src/components/model_add/credential_form_helpers.ts +++ b/ui/litellm-dashboard/src/components/model_add/credential_form_helpers.ts @@ -1,5 +1,3 @@ -import { Providers } from "../provider_info_helpers"; - interface CredentialFormAdapter { getFieldValue: (field: string) => unknown; resetFields: () => void; @@ -25,8 +23,8 @@ interface CredentialFormAdapter { */ export function resetCredentialFormOnProviderChange( form: CredentialFormAdapter, - newProvider: Providers, - setSelectedProvider: (p: Providers) => void, + newProvider: string | null, + setSelectedProvider: (p: string | null) => void, ): void { const preservedName = form.getFieldValue("credential_name"); form.resetFields(); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d7cf5b6110b..cab073dc808 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2946,6 +2946,7 @@ export interface Member { role: string; user_id: string | null; user_email?: string | null; + user_alias?: string | null; max_budget_in_team?: number | null; tpm_limit?: number | null; rpm_limit?: number | null; diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts index 37a973d5def..311e9357825 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts @@ -202,6 +202,8 @@ export const buildKeyCreatePayload = (input: KeyCreateInput): KeyPayloadResult = endpoint: input.keyOwner === "service_account" ? "service_account" : "standard", payload: { ...withoutKeys(values, dropped), + ...(values.organization_id === null && { organization_id: undefined }), + ...(values.project_id === null && { project_id: undefined }), ...(input.keyOwner === "you" && { user_id: input.userID }), ...(input.keyOwner === "agent" && { agent_id: input.selectedAgentId }), ...(input.autoRotationEnabled && { auto_rotate: true, rotation_interval: input.rotationInterval }), diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 2bf39bf4cd7..0d5d9f5ec8d 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -16,7 +16,7 @@ const state = vi.hoisted(() => ({ can: {} as Record, uiSettings: {} as Record, tags: {} as Record, - teams: [] as { team_id: string; team_alias: string; models: string[] }[], + teams: [] as { team_id: string; team_alias: string; models: string[]; organization_id?: string }[], organizations: [] as { organization_id: string; organization_alias: string }[], accessGroups: [] as { access_group_id: string; access_group_name: string }[], projects: [] as { project_id: string; project_alias: string; team_id?: string; models?: string[] }[], @@ -806,6 +806,36 @@ describe("CreateKey", () => { expect((await createdPayload()).organization_id).toBe("org-1"); }); + it("discards the old project and team when the organization changes", async () => { + state.uiSettings = { enable_projects_ui: true }; + state.organizations = [ + { organization_id: "scope-silver", organization_alias: "Silver" }, + { organization_id: "scope-copper", organization_alias: "Copper" }, + ]; + state.teams = [{ team_id: "group-maple", team_alias: "Maple", organization_id: "scope-silver", models: [] }]; + state.projects = [{ project_id: "project-orbit", project_alias: "Orbit", team_id: "group-maple", models: [] }]; + await openModal({ teams: state.teams as Team[] }); + await nameTheKey(); + await userEvent.click(await screen.findByLabelText("Organization")); + await userEvent.click(await screen.findByRole("option", { name: /Silver/ })); + await userEvent.click(await screen.findByLabelText("Project")); + await userEvent.click(await screen.findByRole("option", { name: /Orbit/ })); + await waitFor(() => expect(screen.getByLabelText("Team")).toHaveValue("Maple")); + expect(screen.getByLabelText("Team")).toBeDisabled(); + + await userEvent.click(screen.getByLabelText("Organization")); + await userEvent.click(await screen.findByRole("option", { name: /Copper/ })); + expect(screen.getByLabelText("Project")).toHaveValue(""); + expect(screen.getByLabelText("Team")).toHaveValue(""); + expect(screen.getByLabelText("Team")).toBeEnabled(); + await submit(); + + const payload = JSON.parse(JSON.stringify(await createdPayload())); + expect(payload.organization_id).toBe("scope-copper"); + expect(payload.team_id).toBeNull(); + expect(payload).not.toHaveProperty("project_id"); + }); + it("drops organization_id when the chosen organization is cleared again", async () => { state.organizations = [{ organization_id: "org-1", organization_alias: "Engineering" }]; await openModal(); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 831cb0cf6a6..82580fd4667 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -592,35 +592,35 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp }; const changeOrganization = (write: FieldWrite) => (orgId: string | null) => { - write(orgId ?? undefined); + write(orgId); setSelectedOrganizationId(orgId); // Clear team and project when org changes setSelectedCreateKeyTeam(null); setSelectedProjectId(null); - form.setValue("team_id", undefined); - form.setValue("project_id", undefined); + form.setValue("team_id", null); + form.setValue("project_id", null); }; const selectTeam = (team: Team | null) => { setSelectedCreateKeyTeam(team); setSelectedProjectId(null); - form.setValue("project_id", undefined); + form.setValue("project_id", null); // Auto-populate org from team for non-admin users if (team?.organization_id) { setSelectedOrganizationId(team.organization_id); form.setValue("organization_id", team.organization_id); } else if (!team) { setSelectedOrganizationId(null); - form.setValue("organization_id", undefined); + form.setValue("organization_id", null); } }; - const changeProject = (write: FieldWrite) => (projectId: string) => { + const changeProject = (write: FieldWrite) => (projectId: string | null) => { write(projectId); if (!projectId) { setSelectedProjectId(null); setSelectedCreateKeyTeam(null); - form.setValue("team_id", undefined); + form.setValue("team_id", null); return; } setSelectedProjectId(projectId); @@ -756,8 +756,8 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp inputId="create-key-agent" placeholder="Select an agent" emptyText="No agents found" - value={selectedAgentId ?? undefined} - onValueChange={(value) => setSelectedAgentId(value === "" ? null : value)} + value={selectedAgentId} + onValueChange={setSelectedAgentId} options={agentsList.map((a) => ({ label: a.agent_name || a.agent_id, value: a.agent_id, @@ -783,7 +783,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp {(control) => ( = ({ team, teams, data, addKey, autoOp {(control) => ( = ({ team, teams, data, addKey, autoOp {(control) => ( = ({ return
Organization not found
; } + const orgMemberById = new Map((orgData.members || []).map((m) => [m.user_id, m])); + const orgMemberFor = (record: Member) => (record.user_id != null ? orgMemberById.get(record.user_id) : undefined); + const orgExtraColumns: MemberTableColumn[] = [ { title: "Spend (USD)", key: "spend", - render: (_: unknown, record: Member) => { - const orgMember = - record.user_id != null ? (orgData.members || []).find((m) => m.user_id === record.user_id) : undefined; - return ; - }, + sortValue: (record: Member) => orgMemberFor(record)?.spend ?? null, + render: (record: Member) => , }, { title: "Created At", key: "created_at", - render: (_: unknown, record: Member) => { - const orgMember = - record.user_id != null ? (orgData.members || []).find((m) => m.user_id === record.user_id) : undefined; - return {orgMember?.created_at ? new Date(orgMember.created_at).toLocaleString() : "-"}; + sortValue: (record: Member) => orgMemberFor(record)?.created_at ?? null, + render: (record: Member) => { + const createdAt = orgMemberFor(record)?.created_at; + return {createdAt ? new Date(createdAt).toLocaleString() : "-"}; }, }, ]; @@ -252,10 +252,12 @@ const OrganizationInfoView: React.FC = ({
({ role: m.user_role || "", user_id: m.user_id, user_email: m.user_email, + user_alias: m.user?.user_alias ?? null, }))} canEdit={canEditOrg} onEdit={(member) => { diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 72ec5990557..de83cd00790 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -453,7 +453,7 @@ export const getPlaceholder = (selectedProvider: string): string => { return providerPlaceholderMap[resolvedProvider] ?? "gpt-3.5-turbo"; }; -export const getProviderModels = (provider: Providers, modelMap: any): Array => { +export const getProviderModels = (provider: string, modelMap: any): Array => { let providerKey = provider; let custom_llm_provider = provider_map[providerKey]; diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index 162fc39a632..8ed8e392ae1 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -390,6 +390,28 @@ describe("DataTable filtering", () => { expect(names()).toEqual(["Alice"]); }); + it("client global filter searches an opted-in column even when the first row has no value", () => { + const nicknameColumns: ColumnDef[] = [ + ...nameEmailColumns, + { + id: "nickname", + accessorFn: (row) => (row.id === "b" ? "Bobby" : undefined), + enableGlobalFilter: true, + header: "Nickname", + }, + ]; + render( + , + ); + expect(names()).toEqual(["Bob"]); + }); + it("server mode never filters locally even when columnFilters is set", () => { render( (columns: ColumnDef[]): Colu return { left: collect("left"), right: collect("right") }; } +function columnCanGlobalFilter(firstRow: TData | undefined, column: Column): boolean { + if (column.columnDef.enableGlobalFilter === true) return true; + if (firstRow === undefined || column.accessorFn === undefined) return false; + const firstValue: unknown = column.accessorFn(firstRow, 0); + return typeof firstValue === "string" || typeof firstValue === "number"; +} + function buildRowModels( sortingMode: SortingMode, paginationMode: PaginationMode, @@ -516,6 +523,7 @@ function useDataTableInstance( onRowSelectionChange: rowSelectionState.onChange, onColumnVisibilityChange: setColumnVisibility, onColumnSizingChange: setColumnSizing, + getColumnCanGlobalFilter: (column) => columnCanGlobalFilter(data[0], column), getCoreRowModel: getCoreRowModel(), ...buildRowModels(sortingMode, paginationMode, filterMode, expansionGuard), ...(getRowId !== undefined ? { getRowId } : {}), diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx similarity index 89% rename from ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx rename to ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx index 04c8d3e9012..2b948ca8420 100644 --- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, renderWithProviders as render, screen, waitFor } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import { useState } from "react"; import { describe, expect, it, vi } from "vitest"; @@ -51,7 +51,7 @@ describe("PaginatedSearchSelect", () => { const onSearchChange = vi.fn(); function Controlled() { - const [value, setValue] = useState(""); + const [value, setValue] = useState(null); return ( { expect(onSearchChange).not.toHaveBeenCalled(); }); - it("still reports a cleared input so the unfiltered page comes back", async () => { + it("should keep a cleared selection empty after a late page arrives and reset the query", async () => { const user = userEvent.setup(); const onSearchChange = vi.fn(); - renderSelect({ onSearchChange, value: "alias-alpha" }); - - await user.click(document.querySelector('[data-slot="combobox-clear"]') as HTMLElement); - - await waitFor(() => expect(onSearchChange).toHaveBeenCalledWith("")); + const onValueChange = vi.fn(); + function Controlled({ options }: { options: SearchSelectOption[] }) { + const [value, setValue] = useState("alias-alpha"); + return ( + { + setValue(next); + onValueChange(next); + }} + /> + ); + } + const { rerender } = render(); + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(onValueChange).toHaveBeenLastCalledWith(null); + rerender( ({ ...option }))} />); + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("")); + expect(screen.getByRole("combobox")).toHaveValue(""); + expect(screen.queryByRole("button", { name: "Clear" })).not.toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox"), "alias-beta"); + expect(onValueChange).toHaveBeenLastCalledWith("alias-beta"); }); it("requests the next page once the list is scrolled near the bottom", async () => { @@ -147,7 +166,7 @@ describe("PaginatedSearchSelect", () => { function ServerBacked() { const [search, setSearch] = useState(""); - const [value, setValue] = useState("alias-alpha"); + const [value, setValue] = useState("alias-alpha"); const freshlyBuiltOptions = OPTIONS.filter((option) => option.label.includes(search)).map((option) => ({ ...option, })); @@ -236,7 +255,7 @@ describe("PaginatedSearchSelect", () => { function Refetching() { const [options, setOptions] = useState([{ label: "Beta Team", value: "team-2" }]); - const [value, setValue] = useState(""); + const [value, setValue] = useState(null); return ( <> { function ServerBacked() { const [search, setSearch] = useState(""); - const [value, setValue] = useState(""); + const [value, setValue] = useState(null); return ( option.label.includes(search))} diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx index 4b7ef7401c7..1314afdf2bf 100644 --- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx @@ -17,8 +17,8 @@ import { usePaginatedCombobox } from "./usePaginatedCombobox"; interface PaginatedSearchSelectProps { options: SearchSelectOption[]; - value?: string; - onValueChange: (value: string) => void; + value?: string | null; + onValueChange: (value: string | null) => void; onSearchChange: (query: string) => void; onLoadMore?: () => void; hasNextPage?: boolean; @@ -86,7 +86,7 @@ export function PaginatedSearchSelect({ }; const selected = useMemo(() => { - if (value === undefined || value === "") return null; + if (value == null || value === "") return null; return ( options.find((option) => option.value === value) ?? (pickedOption?.value === value ? pickedOption : { label: value, value }) @@ -118,7 +118,7 @@ export function PaginatedSearchSelect({ inputValue={typedQuery ?? selected?.label ?? ""} onValueChange={(item: SearchSelectOption | null) => { setPickedOption(item); - onValueChange(item?.value ?? ""); + onValueChange(item?.value ?? null); }} onInputValueChange={(next, eventDetails) => handleTypedInput(next, eventDetails.reason)} onOpenChange={(nextOpen, eventDetails) => handleOpenChange(nextOpen, eventDetails.reason)} @@ -139,7 +139,7 @@ export function PaginatedSearchSelect({ onKeyDown={snapshotWholeSelection} onPaste={snapshotWholeSelection} placeholder={placeholder} - showClear={value !== undefined && value !== ""} + showClear={value != null && value !== ""} className={`w-full ${className ?? ""}`} /> diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx similarity index 76% rename from ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx rename to ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx index 62a510268dd..c981010dff9 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx @@ -1,4 +1,5 @@ -import { fireEvent, render, screen } from "@testing-library/react"; +import { fireEvent, renderWithProviders as render, screen } from "../../../tests/test-utils"; +import { useState } from "react"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; @@ -36,11 +37,30 @@ describe("SearchSelect", () => { expect(screen.getByRole("combobox")).toHaveValue("Growth"); }); - it("shows a clear control only when a value is selected", () => { - const { rerender } = render(); - expect(document.querySelector('[data-slot="combobox-clear"]')).toBeNull(); - rerender(); - expect(document.querySelector('[data-slot="combobox-clear"]')).not.toBeNull(); + it("should clear to null and allow selecting again through the real control", async () => { + const onValueChange = vi.fn(); + const user = userEvent.setup(); + function Controlled() { + const [value, setValue] = useState(null); + return ( + { + setValue(next); + onValueChange(next); + }} + /> + ); + } + render(); + expect(screen.queryByRole("button", { name: "Clear" })).not.toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox"), "Growth"); + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(onValueChange).toHaveBeenLastCalledWith(null); + expect(screen.getByRole("combobox")).toHaveValue(""); + await chooseSelectOption(user, screen.getByRole("combobox"), "Data Team"); + expect(onValueChange).toHaveBeenLastCalledWith("team-3"); }); it("filters the options client-side as you type", async () => { diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx index 53a5371d72d..21d38e9458c 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx @@ -21,8 +21,8 @@ export interface SearchSelectOption { interface SearchSelectProps { options: SearchSelectOption[]; - value?: string; - onValueChange: (value: string) => void; + value?: string | null; + onValueChange: (value: string | null) => void; placeholder?: string; emptyText?: string; disabled?: boolean; @@ -51,9 +51,7 @@ export function SearchSelect({ "aria-label": ariaLabel, }: SearchSelectProps) { const selected = - value === undefined || value === "" - ? null - : options.find((option) => option.value === value) ?? { label: value, value }; + value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value }; const items = selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options; @@ -61,7 +59,7 @@ export function SearchSelect({ onValueChange(item?.value ?? "")} + onValueChange={(item: SearchSelectOption | null) => onValueChange(item?.value ?? null)} isItemEqualToValue={(a: SearchSelectOption, b: SearchSelectOption) => a.value === b.value} itemToStringLabel={(item: SearchSelectOption) => item.label} filter={matchesQuery} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx index c715210fdde..c6720ecc7b1 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx @@ -4,6 +4,10 @@ import { describe, expect, it, vi } from "vitest"; import { IdCell } from "./id_cell"; +const { routerPushMock } = vi.hoisted(() => ({ routerPushMock: vi.fn() })); + +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: routerPushMock }) })); + const { copyToClipboardMock } = vi.hoisted(() => ({ copyToClipboardMock: vi.fn() })); vi.mock("@/utils/dataUtils", async (importOriginal) => ({ @@ -83,4 +87,23 @@ describe("IdCell", () => { render(); expect(screen.getByTestId("key-id-cell")).toHaveTextContent("k-1"); }); + + it("renders the id as a link and routes client side when href is set", async () => { + const user = userEvent.setup(); + render(); + + const link = screen.getByRole("link", { name: "user-42" }); + expect(link).toHaveAttribute("href", "/ui/users?user=user-42"); + expect(link).toHaveClass("cursor-pointer"); + + await user.click(link); + expect(routerPushMock).toHaveBeenCalledWith("/ui/users?user=user-42"); + }); + + it("stays plain text when href is undefined", () => { + render(); + + expect(screen.getByText("default_user_id").tagName).toBe("SPAN"); + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx index 33c7f835e64..47d0d750223 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx @@ -3,6 +3,7 @@ import { Copy } from "lucide-react"; import * as React from "react"; +import { useEntityLinkClick } from "@/components/shared/EntityLink"; import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; @@ -13,6 +14,7 @@ export type IdCellVariant = "pill" | "plain"; interface IdCellProps { value: string | null | undefined; variant?: IdCellVariant; + href?: string; onClick?: (value: string) => void; copyable?: boolean; copyLabel?: string; @@ -38,6 +40,7 @@ const VARIANT_CLASS: Record export function IdCell({ value, variant = "pill", + href, onClick, copyable = false, copyLabel = "Copy ID", @@ -52,16 +55,17 @@ export function IdCell({ return {fallback}; } + const linked = !!href && !disabled; const clickable = !!onClick && !disabled; const classes = cn( VARIANT_CLASS[variant].base, - clickable && VARIANT_CLASS[variant].clickable, + (linked || clickable) && VARIANT_CLASS[variant].clickable, truncate && "block max-w-[15ch] truncate", disabled && "opacity-50", className, ); - const idElement = clickable ? ( + const unlinkedElement = clickable ? ( @@ -71,6 +75,14 @@ export function IdCell({ ); + const idElement = linked ? ( + + {value} + + ) : ( + unlinkedElement + ); + const withTooltip = ; if (!copyable) { @@ -94,3 +106,21 @@ export function IdCell({ ); } + +interface IdLinkProps extends React.ComponentPropsWithoutRef<"a"> { + href: string; + dataTestId?: string; +} + +const IdLink = React.forwardRef(function IdLink( + { href, dataTestId, children, ...props }, + ref, +) { + const handleClick = useEntityLinkClick(href); + + return ( + + {children} + + ); +}); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 988e2e082aa..8e3a17c2622 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -333,7 +333,10 @@ const teamUpdateFieldsSchema = z.object({ modelLimits: z .array( z.object({ - model: z.string().min(1, "Missing model"), + model: z + .string() + .nullable() + .refine((model) => Boolean(model), "Missing model"), tpm: z.number().nullish(), rpm: z.number().nullish(), }), @@ -1879,7 +1882,7 @@ const TeamInfoView: React.FC = ({ onChange(next === "" ? null : next)} + onValueChange={onChange} options={userOrganizations.map((org) => ({ value: org.organization_id ?? "", label: org.organization_alias || org.organization_id || "", diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx index 6711514bfe9..2c119bb5848 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -1,4 +1,4 @@ -import { screen, within } from "@testing-library/react"; +import { fireEvent, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; @@ -168,6 +168,27 @@ describe("TeamMembersComponent", () => { expect(table).toHaveTextContent("admin"); }); + it("clears the member search when a different team is shown", () => { + const props = { + canEditTeam: false, + handleMemberDelete: mockHandleMemberDelete, + setSelectedEditMember: mockSetSelectedEditMember, + setIsEditMemberModalVisible: mockSetIsEditMemberModalVisible, + setIsAddMemberModalVisible: mockSetIsAddMemberModalVisible, + }; + const { rerender } = renderWithProviders(); + + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "user2" } }); + expect(screen.queryByText("user1@test.com")).not.toBeInTheDocument(); + + const otherTeam = createMockTeamData({ team_id: "team-456" }); + rerender(); + + expect(screen.getByTestId("datatable-search")).toHaveValue(""); + expect(screen.getAllByText("user1@test.com").length).toBeGreaterThanOrEqual(1); + expect(screen.getAllByText("user2@test.com").length).toBeGreaterThanOrEqual(1); + }); + it("should render Add Member button", () => { renderWithProviders( ), key: "model_scope", - render: (_: unknown, record: Member) => { + render: (record: Member) => { const models = getUserAllowedModels(record.user_id); if (!models) { return (all team models); @@ -141,9 +141,8 @@ export default function TeamMemberTab({ ), key: "spend", - render: (_: unknown, record: Member) => ( - - ), + sortValue: (record: Member) => getUserCurrentCycleSpend(record.user_id), + render: (record: Member) => , }, { title: ( @@ -155,19 +154,22 @@ export default function TeamMemberTab({ ), key: "total_spend", - render: (_: unknown, record: Member) => , + sortValue: (record: Member) => getUserTotalSpend(record.user_id), + render: (record: Member) => , }, { title: "Team Member Budget (USD)", key: "budget", - render: (_: unknown, record: Member) => ( + sortValue: (record: Member) => getUserBudget(record.user_id), + render: (record: Member) => ( ), }, { title: "Budget Reset", key: "budget_reset", - render: (_: unknown, record: Member) => , + sortValue: (record: Member) => getUserBudgetReset(record.user_id), + render: (record: Member) => , }, { title: ( @@ -179,12 +181,13 @@ export default function TeamMemberTab({ ), key: "rate_limits", - render: (_: unknown, record: Member) => {getUserRateLimits(record.user_id)}, + render: (record: Member) => {getUserRateLimits(record.user_id)}, }, ]; return ( { diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx similarity index 98% rename from ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx rename to ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx index b2c2a381b42..7c2226f0369 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx @@ -1483,11 +1483,11 @@ describe("KeyEditView", () => { }); }); - it("submits organization_id as null after the organization is cleared", async () => { + it("clears the organization and its dependent team in the update payload", async () => { const onSubmit = vi.fn().mockResolvedValue(undefined); renderWithProviders( {}} onSubmit={onSubmit} accessToken="" @@ -1504,9 +1504,33 @@ describe("KeyEditView", () => { await userEvent.click(screen.getByRole("button", { name: /save changes/i })); await waitFor(() => { - expect(onSubmit).toHaveBeenCalledWith(expect.objectContaining({ organization_id: null })); + expect(onSubmit).toHaveBeenCalledWith(expect.objectContaining({ organization_id: null, team_id: null })); }); - expect(JSON.parse(JSON.stringify(onSubmit.mock.calls[0][0]))).toHaveProperty("organization_id", null); + expect(JSON.parse(JSON.stringify(onSubmit.mock.calls[0][0]))).toMatchObject({ + organization_id: null, + team_id: null, + }); + }); + + it("keeps project key relationships locked and omits unsupported project updates", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderWithProviders( + {}} + onSubmit={onSubmit} + accessToken="" + userID="" + userRole="Admin" + premiumUser={false} + />, + ); + expect(await screen.findByRole("combobox", { name: "Organization" })).toBeDisabled(); + expect(screen.getByRole("combobox", { name: "Team ID" })).toBeDisabled(); + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)); + expect(onSubmit.mock.calls[0][0]).toMatchObject({ organization_id: "org-1", team_id: "group-maple" }); + expect(onSubmit.mock.calls[0][0]).not.toHaveProperty("project_id"); }); }); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index e464dfe5008..f0d63f9671d 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -305,7 +305,7 @@ export function KeyEditView({ const handleOrganizationChange = (setField: (value: string | null) => void, orgId: string | null) => { setField(orgId); setSelectedOrganizationId(orgId); - form.setValue("team_id", undefined); + form.setValue("team_id", null); }; const handleTeamChange = (setField: (value: string | null) => void, teamId: string | null) => { @@ -316,7 +316,7 @@ export function KeyEditView({ form.setValue("organization_id", selectedTeam.organization_id); } else if (!teamId) { setSelectedOrganizationId(null); - form.setValue("organization_id", undefined); + form.setValue("organization_id", null); } }; @@ -769,14 +769,15 @@ export function KeyEditView({ "Organization", "The organization this key belongs to. Selecting an organization filters the available teams.", )} + description={hasProject ? "Organization is locked because this key belongs to a project" : undefined} > {({ value, onChange, id }) => ( handleOrganizationChange(onChange, orgId)} /> )} @@ -786,15 +787,13 @@ export function KeyEditView({ control={form.control} name="team_id" label="Team ID" - description={ - enableProjectsUI && hasProject ? "Team is locked because this key belongs to a project" : undefined - } + description={hasProject ? "Team is locked because this key belongs to a project" : undefined} > {({ value, onChange, id }) => (