diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index b89c3c2f33f..4cd4ea2fa60 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -10,7 +10,7 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import _get_masked_values from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object from litellm.repositories.credentials_repository import CredentialsRepository from litellm.types.utils import CreateCredentialItem, CredentialItem @@ -34,6 +34,22 @@ class CredentialHelperUtils: credential_info=credential.credential_info or {}, ) + @staticmethod + def decrypt_credential_values(credential: CredentialItem) -> CredentialItem: + decrypted_values = { + key: ( + decrypt_value_helper(value=value, key=key, return_original_value=True) or value + if isinstance(value, str) + else value + ) + for key, value in (credential.credential_values or {}).items() + } + return CredentialItem( + credential_name=credential.credential_name, + credential_values=decrypted_values, + credential_info=credential.credential_info or {}, + ) + @router.post( "/credentials", @@ -146,7 +162,23 @@ async def get_credential_by_name( """ [BETA] endpoint. This might change unexpectedly. """ + from litellm.proxy.proxy_server import prisma_client + try: + if prisma_client is not None: + db_credential = await CredentialsRepository(prisma_client).find_by_name(credential_name) + if db_credential is not None: + plaintext = CredentialHelperUtils.decrypt_credential_values(db_credential) + return CredentialItem( + credential_name=plaintext.credential_name, + credential_values=_get_masked_values( + plaintext.credential_values, + unmasked_length=4, + number_of_asterisks=4, + ), + credential_info=plaintext.credential_info, + ) + for credential in litellm.credential_list: if credential.credential_name == credential_name: masked_credential = CredentialItem( @@ -317,7 +349,6 @@ async def update_credential( }, ) - # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) new_name = merged_credential.credential_name existing_in_memory: CredentialItem | None = None for cred in litellm.credential_list: @@ -327,21 +358,24 @@ async def update_credential( if existing_in_memory is not None: in_memory_values = dict(existing_in_memory.credential_values or {}) - if credential.credential_values: - in_memory_values.update(credential.credential_values) in_memory_info = dict(existing_in_memory.credential_info or {}) - if credential.credential_info: - in_memory_info.update(credential.credential_info) - updated_in_memory = CredentialItem( - credential_name=new_name, - credential_values=in_memory_values, - credential_info=in_memory_info, - ) - # Remove old entry if renamed, then use upsert_credentials to handle duplicates - if new_name != credential_name: - litellm.credential_list = [c for c in litellm.credential_list if c.credential_name != credential_name] - CredentialAccessor.upsert_credentials([updated_in_memory]) + else: + plaintext_db = CredentialHelperUtils.decrypt_credential_values(db_credential) + in_memory_values = dict(plaintext_db.credential_values or {}) + in_memory_info = dict(plaintext_db.credential_info or {}) + if credential.credential_values: + in_memory_values.update(credential.credential_values) + if credential.credential_info: + in_memory_info.update(credential.credential_info) + updated_in_memory = CredentialItem( + credential_name=new_name, + credential_values=in_memory_values, + credential_info=in_memory_info, + ) + if new_name != credential_name: + litellm.credential_list = [c for c in litellm.credential_list if c.credential_name != credential_name] + CredentialAccessor.upsert_credentials([updated_in_memory]) return {"success": True, "message": "Credential updated successfully"} except Exception as e: - return handle_exception_on_proxy(e) + raise handle_exception_on_proxy(e) diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index 9d918736262..d2e7906df2c 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -24,20 +24,40 @@ export const INTERNAL_USER_STORAGE_PATH = storagePath("internalUser.storageState export const INTERNAL_VIEWER_STORAGE_PATH = storagePath("internalViewer.storageState.json"); export const TEAM_ADMIN_STORAGE_PATH = storagePath("teamAdmin.storageState.json"); -// Seeded user identities (match seed.sql) +// Seeded user identities (match fixtures/seed.ts) export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin"; export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; +export const E2E_ADMIN_VIEWER_USER_ID = "e2e-admin-viewer"; +export const E2E_ADMIN_VIEWER_EMAIL = "adminviewer@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; +export const E2E_INTERNAL_VIEWER_USER_ID = "e2e-internal-viewer"; +export const E2E_INTERNAL_VIEWER_EMAIL = "viewer@test.local"; +export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin"; +export const E2E_TEAM_ADMIN_EMAIL = "teamadmin@test.local"; +export const E2E_INVITABLE_USER_ID = "e2e-invitable-user"; +export const E2E_INVITABLE_USER_EMAIL = "invitable@test.local"; +export const E2E_INTERNAL_NOTEAM_USER_ID = "e2e-internal-noteam"; +export const E2E_INTERNAL_NOTEAM_EMAIL = "noteam@test.local"; +export const E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID = "e2e-invitable-by-team-admin"; +export const E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL = "invitable-team@test.local"; +export const E2E_REMOVABLE_MEMBER_USER_ID = "e2e-removable-member"; +export const E2E_REMOVABLE_MEMBER_EMAIL = "removable@test.local"; +export const E2E_USER_PASSWORD = "test"; -// Key aliases for seeded test keys (match seed.sql) +// Organization and its budget (match fixtures/seed.ts) +export const E2E_ORG_ID = "e2e-org-main"; +export const E2E_ORG_ALIAS = "E2E Organization"; +export const E2E_ORG_BUDGET_ID = "e2e-budget-org"; + +// Key aliases for seeded test keys (match fixtures/seed.ts) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey"; export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey"; export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey"; export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey"; -// Team identifiers (match seed.sql) +// Team identifiers (match fixtures/seed.ts) export const E2E_TEAM_CRUD_ID = "e2e-team-crud"; export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD"; export const E2E_TEAM_DELETE_ID = "e2e-team-delete"; diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql deleted file mode 100644 index a1218633cdb..00000000000 --- a/tests/e2e/ui/fixtures/seed.sql +++ /dev/null @@ -1,86 +0,0 @@ --- E2E Test Seed Data --- Idempotent: deletes all e2e-* rows then re-inserts deterministic data. - --- 1. Clean up in dependency order -DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%'; -DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%'; - --- 2. Budget (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by") -VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin'); - --- 3. Organization (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_OrganizationTable" ( - "organization_id", "organization_alias", "budget_id", - "metadata", "models", "spend", "model_spend", - "created_by", "updated_by" -) VALUES ( - 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', - '{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb, - 'e2e-proxy-admin', 'e2e-proxy-admin' -); - --- 4. Users (password hash is scrypt of "test") -INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") -VALUES - ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); - --- 5. Teams (members_with_roles is required JSON) -INSERT INTO "LiteLLM_TeamTable" ( - "team_id", "team_alias", "organization_id", "admins", "members", - "members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked" -) VALUES - ('e2e-team-crud', 'E2E Team CRUD', NULL, - '{"e2e-team-admin"}', - '{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}', - '[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-delete', 'E2E Team Delete', NULL, - '{"e2e-team-admin"}', '{"e2e-team-admin"}', - '[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-org', 'E2E Team In Org', 'e2e-org-main', - '{}', '{"e2e-internal-user"}', - '[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-no-admin', 'E2E Team No Admin', NULL, - '{}', '{"e2e-invitable-user"}', - '[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false); - --- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at) -INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend") -VALUES - ('e2e-team-admin', 'e2e-team-crud', 0.0), - ('e2e-internal-user', 'e2e-team-crud', 0.0), - ('e2e-internal-viewer', 'e2e-team-crud', 0.0), - ('e2e-removable-member', 'e2e-team-crud', 0.0), - ('e2e-team-admin', 'e2e-team-delete', 0.0), - ('e2e-internal-user', 'e2e-team-org', 0.0), - ('e2e-invitable-user', 'e2e-team-no-admin', 0.0); - --- 7. Verification Tokens (API Keys) -INSERT INTO "LiteLLM_VerificationToken" ( - "token", "key_name", "key_alias", "user_id", "team_id", - "models", "spend", "max_budget", "expires", "metadata" -) VALUES - ('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb); diff --git a/tests/e2e/ui/fixtures/seed.ts b/tests/e2e/ui/fixtures/seed.ts new file mode 100644 index 00000000000..707c7b7bafd --- /dev/null +++ b/tests/e2e/ui/fixtures/seed.ts @@ -0,0 +1,272 @@ +import type { APIRequestContext } from "@playwright/test"; +import { + E2E_ADMIN_VIEWER_EMAIL, + E2E_ADMIN_VIEWER_USER_ID, + E2E_DELETE_KEY_ALIAS, + E2E_INTERNAL_NOTEAM_EMAIL, + E2E_INTERNAL_NOTEAM_USER_ID, + E2E_INTERNAL_USER_EMAIL, + E2E_INTERNAL_USER_ID, + E2E_INTERNAL_USER_KEY_ALIAS, + E2E_INTERNAL_VIEWER_EMAIL, + E2E_INTERNAL_VIEWER_USER_ID, + E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL, + E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID, + E2E_INVITABLE_USER_EMAIL, + E2E_INVITABLE_USER_ID, + E2E_ORG_ALIAS, + E2E_ORG_BUDGET_ID, + E2E_ORG_ID, + E2E_PROXY_ADMIN_EMAIL, + E2E_PROXY_ADMIN_USER_ID, + E2E_REGENERATE_KEY_ALIAS, + E2E_REMOVABLE_MEMBER_EMAIL, + E2E_REMOVABLE_MEMBER_USER_ID, + E2E_TEAM_ADMIN_EMAIL, + E2E_TEAM_ADMIN_USER_ID, + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_CRUD_ID, + E2E_TEAM_DELETE_ALIAS, + E2E_TEAM_DELETE_ID, + E2E_TEAM_NO_ADMIN_ALIAS, + E2E_TEAM_NO_ADMIN_ID, + E2E_TEAM_ORG_ALIAS, + E2E_TEAM_ORG_ID, + E2E_UPDATE_LIMITS_KEY_ALIAS, + E2E_USER_PASSWORD, + E2E_VIEWER_KEY_ALIAS, +} from "../constants"; + +type UserRole = "proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer"; +type TeamMemberRole = "admin" | "user"; + +type SeedUser = { readonly userId: string; readonly email: string; readonly role: UserRole }; +type SeedTeamMember = { readonly user_id: string; readonly role: TeamMemberRole }; +type SeedTeam = { + readonly teamId: string; + readonly alias: string; + readonly organizationId: string | null; + readonly models: readonly string[]; + readonly members: readonly SeedTeamMember[]; +}; +type SeedKey = { + readonly alias: string; + readonly userId: string; + readonly teamId: string | null; + readonly models: readonly string[]; +}; + +const OPENAI_MODEL = "fake-openai-gpt-4"; +const ANTHROPIC_MODEL = "fake-anthropic-claude"; +const ORG_MAX_BUDGET = 1000; + +// The master key's own user is auto-added as an admin of every team it creates, +// which would make the proxy admin a member of teams the suite needs it to be a +// stranger to. Removed after each team is created. +const MASTER_KEY_USER_ID = "default_user_id"; + +const USERS: readonly SeedUser[] = [ + { userId: E2E_PROXY_ADMIN_USER_ID, email: E2E_PROXY_ADMIN_EMAIL, role: "proxy_admin" }, + { userId: E2E_ADMIN_VIEWER_USER_ID, email: E2E_ADMIN_VIEWER_EMAIL, role: "proxy_admin_viewer" }, + { userId: E2E_INTERNAL_USER_ID, email: E2E_INTERNAL_USER_EMAIL, role: "internal_user" }, + { userId: E2E_INTERNAL_VIEWER_USER_ID, email: E2E_INTERNAL_VIEWER_EMAIL, role: "internal_user_viewer" }, + { userId: E2E_TEAM_ADMIN_USER_ID, email: E2E_TEAM_ADMIN_EMAIL, role: "internal_user" }, + { userId: E2E_INVITABLE_USER_ID, email: E2E_INVITABLE_USER_EMAIL, role: "internal_user" }, + { userId: E2E_INTERNAL_NOTEAM_USER_ID, email: E2E_INTERNAL_NOTEAM_EMAIL, role: "internal_user" }, + { + userId: E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID, + email: E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL, + role: "internal_user", + }, + { userId: E2E_REMOVABLE_MEMBER_USER_ID, email: E2E_REMOVABLE_MEMBER_EMAIL, role: "internal_user" }, +]; + +const TEAMS: readonly SeedTeam[] = [ + { + teamId: E2E_TEAM_CRUD_ID, + alias: E2E_TEAM_CRUD_ALIAS, + organizationId: null, + models: [OPENAI_MODEL, ANTHROPIC_MODEL], + members: [ + { user_id: E2E_TEAM_ADMIN_USER_ID, role: "admin" }, + { user_id: E2E_INTERNAL_USER_ID, role: "user" }, + { user_id: E2E_INTERNAL_VIEWER_USER_ID, role: "user" }, + { user_id: E2E_REMOVABLE_MEMBER_USER_ID, role: "user" }, + ], + }, + { + teamId: E2E_TEAM_DELETE_ID, + alias: E2E_TEAM_DELETE_ALIAS, + organizationId: null, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_TEAM_ADMIN_USER_ID, role: "admin" }], + }, + { + teamId: E2E_TEAM_ORG_ID, + alias: E2E_TEAM_ORG_ALIAS, + organizationId: E2E_ORG_ID, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_INTERNAL_USER_ID, role: "user" }], + }, + { + teamId: E2E_TEAM_NO_ADMIN_ID, + alias: E2E_TEAM_NO_ADMIN_ALIAS, + organizationId: null, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_INVITABLE_USER_ID, role: "user" }], + }, +]; + +const KEYS: readonly SeedKey[] = [ + { + alias: E2E_UPDATE_LIMITS_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_DELETE_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_REGENERATE_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_INTERNAL_USER_KEY_ALIAS, + userId: E2E_INTERNAL_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_VIEWER_KEY_ALIAS, + userId: E2E_INTERNAL_VIEWER_USER_ID, + teamId: null, + models: [OPENAI_MODEL], + }, +]; + +type JsonBody = Record; + +const jsonHeaders = (masterKey: string): Record => ({ + Authorization: `Bearer ${masterKey}`, + "Content-Type": "application/json", +}); + +async function post( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.post(url, { headers, data }); + if (!res.ok()) { + throw new Error(`Seeding call POST ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +async function postAllowingMissing( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.post(url, { headers, data }); + if (!res.ok() && res.status() !== 404) { + throw new Error(`Seeding call POST ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +async function deleteAllowingMissing( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.delete(url, { headers, data }); + if (!res.ok() && res.status() !== 404) { + throw new Error(`Seeding call DELETE ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +// /team/delete and /user/delete are all-or-nothing: one unknown id in the batch +// aborts the whole call and deletes nothing, so each id goes in its own request. +async function removeFixtures( + api: APIRequestContext, + apiBase: string, + headers: Record, +): Promise { + await postAllowingMissing(api, `${apiBase}/key/delete`, { key_aliases: KEYS.map((key) => key.alias) }, headers); + for (const team of TEAMS) { + await postAllowingMissing(api, `${apiBase}/team/delete`, { team_ids: [team.teamId] }, headers); + } + for (const user of USERS) { + await postAllowingMissing(api, `${apiBase}/user/delete`, { user_ids: [user.userId] }, headers); + } + await deleteAllowingMissing(api, `${apiBase}/organization/delete`, { organization_ids: [E2E_ORG_ID] }, headers); + await postAllowingMissing(api, `${apiBase}/budget/delete`, { id: E2E_ORG_BUDGET_ID }, headers); +} + +async function createFixtures( + api: APIRequestContext, + apiBase: string, + headers: Record, +): Promise { + await post(api, `${apiBase}/budget/new`, { budget_id: E2E_ORG_BUDGET_ID, max_budget: ORG_MAX_BUDGET }, headers); + await post( + api, + `${apiBase}/organization/new`, + { organization_id: E2E_ORG_ID, organization_alias: E2E_ORG_ALIAS, budget_id: E2E_ORG_BUDGET_ID }, + headers, + ); + + for (const user of USERS) { + await post( + api, + `${apiBase}/user/new`, + { user_id: user.userId, user_email: user.email, user_role: user.role, auto_create_key: false }, + headers, + ); + await post(api, `${apiBase}/user/update`, { user_id: user.userId, password: E2E_USER_PASSWORD }, headers); + } + + for (const team of TEAMS) { + await post( + api, + `${apiBase}/team/new`, + { + team_id: team.teamId, + team_alias: team.alias, + organization_id: team.organizationId, + models: team.models, + members_with_roles: team.members, + }, + headers, + ); + await postAllowingMissing( + api, + `${apiBase}/team/member_delete`, + { team_id: team.teamId, user_id: MASTER_KEY_USER_ID }, + headers, + ); + } + + for (const key of KEYS) { + await post( + api, + `${apiBase}/key/generate`, + { key_alias: key.alias, user_id: key.userId, team_id: key.teamId, models: key.models }, + headers, + ); + } +} + +export async function seedFixtures(api: APIRequestContext, apiBase: string, masterKey: string): Promise { + const headers = jsonHeaders(masterKey); + await removeFixtures(api, apiBase, headers); + await createFixtures(api, apiBase, headers); +} diff --git a/tests/e2e/ui/fixtures/users.ts b/tests/e2e/ui/fixtures/users.ts index 731234b5ea9..b92ded48c41 100644 --- a/tests/e2e/ui/fixtures/users.ts +++ b/tests/e2e/ui/fixtures/users.ts @@ -1,6 +1,11 @@ import { ADMIN_STORAGE_PATH, ADMIN_VIEWER_STORAGE_PATH, + E2E_ADMIN_VIEWER_EMAIL, + E2E_INTERNAL_USER_EMAIL, + E2E_INTERNAL_VIEWER_EMAIL, + E2E_TEAM_ADMIN_EMAIL, + E2E_USER_PASSWORD, INTERNAL_USER_STORAGE_PATH, INTERNAL_VIEWER_STORAGE_PATH, TEAM_ADMIN_STORAGE_PATH, @@ -20,20 +25,20 @@ export const users: Record = { password: process.env.LITELLM_MASTER_KEY || "sk-1234", }, [Role.ProxyAdminViewer]: { - email: "adminviewer@test.local", - password: "test", + email: E2E_ADMIN_VIEWER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.InternalUser]: { - email: "internal@test.local", - password: "test", + email: E2E_INTERNAL_USER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.InternalUserViewer]: { - email: "viewer@test.local", - password: "test", + email: E2E_INTERNAL_VIEWER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.TeamAdmin]: { - email: "teamadmin@test.local", - password: "test", + email: E2E_TEAM_ADMIN_EMAIL, + password: E2E_USER_PASSWORD, }, }; diff --git a/tests/e2e/ui/globalSetup.ts b/tests/e2e/ui/globalSetup.ts index 6068a51c88e..021916fd03a 100644 --- a/tests/e2e/ui/globalSetup.ts +++ b/tests/e2e/ui/globalSetup.ts @@ -1,5 +1,6 @@ import { chromium, expect, request } from "@playwright/test"; import { users, Role, STORAGE_PATHS } from "./fixtures/users"; +import { seedFixtures } from "./fixtures/seed"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; import * as fs from "fs"; import * as path from "path"; @@ -29,6 +30,12 @@ async function globalSetup() { if (!settingsRes.ok()) { throw new Error(`Enabling enable_projects_ui failed (${settingsRes.status()}): ${await settingsRes.text()}`); } + + // The users, teams and keys every spec reads are created here over the + // management API rather than by loading SQL into the database, so the suite + // can run against any deployment it can reach (a local proxy, CI, or a remote + // one whose database it has no direct access to). + await seedFixtures(api, `${UI_BASE_URL}${rootPath}`, masterKey); await api.dispose(); for (const role of Object.values(Role)) { diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 858eb401c8e..1bc1cdd2034 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -3,7 +3,8 @@ set -euo pipefail # ================================================================ # UI E2E Test Runner (Consolidated) -# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. +# Starts postgres, starts mock + proxy, runs Playwright (which seeds +# its fixtures over the management API in globalSetup). # All tests target the proxy on port 4000 (which serves both API # and UI from the built Next.js static export). # @@ -56,7 +57,7 @@ done # --- Database setup --- if [ "$IS_CI" = "false" ]; then - for cmd in docker psql; do + for cmd in docker pg_isready; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } done for port in 4000 5432 8090; do @@ -174,17 +175,6 @@ if [ "$PROXY_READY" -ne 1 ]; then fi echo "Proxy is ready." -# --- Seed database --- -echo "=== Seeding database ===" -DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') -DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') -DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') -DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') -DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') - -PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ - -f "$SCRIPT_DIR/fixtures/seed.sql" - # --- Playwright --- echo "=== Installing Playwright dependencies ===" cd "$SCRIPT_DIR" diff --git a/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts b/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts index 1b048198456..d46c6ff8d55 100644 --- a/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts @@ -1,4 +1,5 @@ import { test, expect } from "@playwright/test"; +import { E2E_INTERNAL_NOTEAM_EMAIL, E2E_USER_PASSWORD } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; @@ -12,11 +13,14 @@ test.describe("Internal User with no team memberships", () => { test.use({ storageState: { cookies: [], origins: [] } }); test("Create Key team dropdown is empty when the user belongs to no teams", async ({ page }) => { - // Log in via the form as the no-team seeded user. + // Log in via the form as the no-team seeded user (created by fixtures/seed.ts). await page.goto("/ui/login"); - await page.getByPlaceholder("Enter your username").fill("noteam@test.local"); - await page.getByPlaceholder("Enter your password").fill("test"); + await page.getByPlaceholder("Enter your username").fill(E2E_INTERNAL_NOTEAM_EMAIL); + await page.getByPlaceholder("Enter your password").fill(E2E_USER_PASSWORD); await page.getByRole("button", { name: "Login", exact: true }).click(); + await page.waitForURL((url) => url.pathname.includes("/ui") && !url.pathname.includes("/login"), { + timeout: 30_000, + }); await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 30_000 }); expect(new URL(page.url()).pathname).not.toMatch(/\/connect$/); await navigateToPage(page, Page.ApiKeys); diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index bb8806a9c01..85276ed1e62 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -59,16 +59,11 @@ test.describe("Add Model", () => { const createdModelId = (await createResponse.json()).model_info?.id; expect(createdModelId, "model id from /model/new").toBeTruthy(); - // Navigate to Models + Endpoints - await page.goto("/ui"); - await page.getByText("Models + Endpoints").click(); - - // The Model ID cell is the drill-in control; the row itself is not clickable. - const modelIdCell = page.getByTestId(`model-id-${createdModelId}`); - await expect(modelIdCell).toBeVisible({ timeout: 10_000 }); - await modelIdCell.click(); - - await expect(page.getByText("Back to Models").first()).toBeVisible({ timeout: 10_000 }); + // Deep-link into the detail view. Searching the paginated All Models table is + // flaky on a shared stage DB with many deployments, and team-scoped models are + // rewritten to model_name_{team_id}_{uuid} so a name search is also unreliable. + await page.goto(`/ui?page=models&model=${createdModelId}`); + await expect(page.getByText("Back to Models").first()).toBeVisible({ timeout: 15_000 }); // Edit Settings → change TPM/RPM → Save await page.getByRole("button", { name: "Edit Settings" }).click(); diff --git a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts index e67dcb96f36..f526d3d50b2 100644 --- a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts @@ -63,16 +63,11 @@ test.describe("Clear custom pricing on a deployment", () => { }); test("UI sends null for cleared pricing and backend removes the override", async ({ page }) => { - // Navigate to the model detail view. - await page.goto("/ui"); - await page.getByText("Models + Endpoints").click(); - - // The Model ID cell is the drill-in control; the row itself is not clickable. - const modelIdCell = page.getByTestId(`model-id-${createdModelId}`); - await expect(modelIdCell).toBeVisible({ timeout: 15_000 }); - await modelIdCell.click(); + // Deep-link into the detail view so a crowded paginated models table on a + // shared stage DB cannot hide the row we just created. + await page.goto(`/ui?page=models&model=${createdModelId}`); await expect(page.getByText("Back to Models").first()).toBeVisible({ - timeout: 10_000, + timeout: 15_000, }); // Sanity: the seeded pricing is shown in the detail view (77.7000 / 99.9000 diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index c44957ea737..83f3fee1a2f 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -97,8 +97,13 @@ test.describe("Proxy Admin - Keys", () => { await navigateToPage(page, Page.ApiKeys); await dismissFeedbackPopup(page); + const search = page.getByPlaceholder(/search/i).first(); + if (await search.isVisible({ timeout: 2_000 }).catch(() => false)) { + await search.fill(E2E_DELETE_KEY_ALIAS); + } + const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS }); - await expect(keyRow).toBeVisible({ timeout: 10_000 }); + await expect(keyRow).toBeVisible({ timeout: 15_000 }); await keyRow.locator("button").first().click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); @@ -163,16 +168,14 @@ test.describe("Proxy Admin - Keys", () => { const keyName = `e2e-admin-specific-${Date.now()}`; await page.getByTestId("base-input").fill(keyName); - // Open the model multi-select and pick a single specific model. Use - // getByRole("option", ...) to avoid the strict-mode collision between - // the option container and its inner text node. + // Open the model multi-select and pick a single specific model. Type to + // filter first: on a stage gateway with many model groups the unfiltered + // list never materialises every option as a DOM node. const modelName = "fake-openai-gpt-4"; await page.locator(".ant-select-selection-overflow").click(); + await page.keyboard.type(modelName); const option = page.locator(".ant-select-dropdown:visible").getByRole("option", { name: modelName, exact: true }); - await option.waitFor({ state: "attached" }); - // Dispatch the click via the DOM — antd's dropdown can render the option - // off-viewport during the open animation, which trips Playwright's - // visibility/stability checks. The click handler fires regardless. + await option.waitFor({ state: "attached", timeout: 15_000 }); await option.evaluate((el: HTMLElement) => el.click()); await page.keyboard.press("Escape"); diff --git a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts index 17ff62f37cf..5d7767636f4 100644 --- a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts @@ -94,8 +94,13 @@ test.describe("Proxy Admin - Teams", () => { await navigateToPage(page, Page.Teams); await dismissFeedbackPopup(page); + const search = page.getByPlaceholder(/search/i).first(); + if (await search.isVisible({ timeout: 2_000 }).catch(() => false)) { + await search.fill(E2E_TEAM_DELETE_ALIAS); + } + const teamRow = page.locator("tr", { hasText: E2E_TEAM_DELETE_ALIAS }).first(); - await expect(teamRow).toBeVisible({ timeout: 10_000 }); + await expect(teamRow).toBeVisible({ timeout: 15_000 }); // Actions live in a kebab menu: open it, then click "Delete team". await teamRow.locator('[data-testid^="team-actions-"]').click(); await page.getByTestId("team-action-delete").click(); diff --git a/tests/e2e/ui/tests/settings/routerSettings.spec.ts b/tests/e2e/ui/tests/settings/routerSettings.spec.ts index 631d2814664..cdf9d8db1f5 100644 --- a/tests/e2e/ui/tests/settings/routerSettings.spec.ts +++ b/tests/e2e/ui/tests/settings/routerSettings.spec.ts @@ -78,13 +78,21 @@ test.describe("Router Settings - Fallbacks", () => { const primarySelect = modal.locator(".ant-select").filter({ hasText: "Select primary model" }); await primarySelect.click(); await page.keyboard.type(PRIMARY); - await page.keyboard.press("Enter"); + const primaryOption = page + .locator(".ant-select-dropdown:visible") + .getByRole("option", { name: PRIMARY, exact: true }); + await primaryOption.waitFor({ state: "attached", timeout: 15_000 }); + await primaryOption.evaluate((el: HTMLElement) => el.click()); await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 }); const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" }); await fallbackSelect.click(); await page.keyboard.type(FALLBACK); - await page.keyboard.press("Enter"); + const fallbackOption = page + .locator(".ant-select-dropdown:visible") + .getByRole("option", { name: FALLBACK, exact: true }); + await fallbackOption.waitFor({ state: "attached", timeout: 15_000 }); + await fallbackOption.evaluate((el: HTMLElement) => el.click()); await page.keyboard.press("Escape"); // The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the // selection has been recorded. diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index 18b43ec89b2..a05e2572760 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from "@playwright/test"; import { E2E_INTERNAL_USER_KEY_ALIAS, + E2E_REMOVABLE_MEMBER_USER_ID, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_CRUD_ID, TEAM_ADMIN_STORAGE_PATH, @@ -71,7 +72,7 @@ test.describe("Team Admin", () => { // Seeded members appear in the roster by user_id (members_with_roles has no // email), so match the row on the user_id rather than the email. - const row = page.locator("tr", { hasText: "e2e-removable-member" }).first(); + const row = page.locator("tr", { hasText: E2E_REMOVABLE_MEMBER_USER_ID }).first(); await expect(row).toBeVisible({ timeout: 10_000 }); await row.getByTestId("delete-member").click(); diff --git a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py new file mode 100644 index 00000000000..054fdc5e52c --- /dev/null +++ b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py @@ -0,0 +1,180 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +import litellm +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.credential_endpoints import endpoints as credential_endpoints +from litellm.proxy.credential_endpoints.endpoints import ( + CredentialHelperUtils, + router, + update_credential, + update_db_credential, +) +from litellm.types.utils import CredentialItem + + +def _auth() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="test-user", user_role=LitellmUserRoles.PROXY_ADMIN) + + +def _app() -> TestClient: + test_app = FastAPI() + test_app.include_router(router) + test_app.dependency_overrides[user_api_key_auth] = _auth + return TestClient(test_app) + + +def test_update_db_credential_merges_api_base_without_dropping_other_values() -> None: + db_credential = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_key": "enc-key", "api_base": "https://api.openai.com/v1"}, + credential_info={"custom_llm_provider": "openai"}, + ) + update_patch = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_base": "https://proxy.e2e.example.com/v1"}, + credential_info={}, + ) + + with patch.object( + CredentialHelperUtils, + "encrypt_credential_values", + side_effect=lambda cred, new_encryption_key=None: cred, + ): + merged = update_db_credential(db_credential, update_patch) + + assert merged.credential_values["api_base"] == "https://proxy.e2e.example.com/v1" + assert merged.credential_values["api_key"] == "enc-key" + + +@pytest.mark.asyncio +async def test_update_credential_always_upserts_in_memory_even_when_absent() -> None: + db_credential = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_key": "enc-key", "api_base": "https://api.openai.com/v1"}, + credential_info={}, + ) + update_patch = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_base": "https://proxy.e2e.example.com/v1"}, + credential_info={}, + ) + repo = MagicMock() + repo.find_by_name = AsyncMock(return_value=db_credential) + repo.update_by_name = AsyncMock() + prisma = MagicMock() + previous = list(litellm.credential_list) + litellm.credential_list = [] + + try: + with ( + patch.object(credential_endpoints, "CredentialsRepository", return_value=repo), + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch.object( + CredentialHelperUtils, + "encrypt_credential_values", + side_effect=lambda cred, new_encryption_key=None: cred, + ), + patch.object( + CredentialHelperUtils, + "decrypt_credential_values", + return_value=CredentialItem( + credential_name="e2e-cred", + credential_values={"api_key": "sk-plain", "api_base": "https://api.openai.com/v1"}, + credential_info={}, + ), + ), + patch.object(credential_endpoints, "jsonify_object", side_effect=lambda x: x), + patch.object(credential_endpoints.CredentialAccessor, "upsert_credentials") as upsert, + ): + result = await update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=update_patch, + credential_name="e2e-cred", + user_api_key_dict=_auth(), + ) + + assert result["success"] is True + upsert.assert_called_once() + written = upsert.call_args.args[0][0] + assert written.credential_values["api_base"] == "https://proxy.e2e.example.com/v1" + assert written.credential_values["api_key"] == "sk-plain" + finally: + litellm.credential_list = previous + + +def test_get_credential_by_name_prefers_db_over_stale_memory() -> None: + client = _app() + db_credential = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_key": "enc-key", "api_base": "https://proxy.e2e.example.com/v1"}, + credential_info={}, + ) + stale = CredentialItem( + credential_name="e2e-cred", + credential_values={"api_key": "sk-plain", "api_base": "https://api.openai.com/v1"}, + credential_info={}, + ) + previous = list(litellm.credential_list) + litellm.credential_list = [stale] + repo = MagicMock() + repo.find_by_name = AsyncMock(return_value=db_credential) + + try: + with ( + patch.object(credential_endpoints, "CredentialsRepository", return_value=repo), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch.object( + CredentialHelperUtils, + "decrypt_credential_values", + return_value=CredentialItem( + credential_name="e2e-cred", + credential_values={ + "api_key": "sk-plain", + "api_base": "https://proxy.e2e.example.com/v1", + }, + credential_info={}, + ), + ), + ): + response = client.get("/credentials/by_name/e2e-cred") + finally: + litellm.credential_list = previous + + assert response.status_code == 200 + body = response.json() + assert body["credential_values"]["api_base"] == "https://proxy.e2e.example.com/v1" + + +@pytest.mark.asyncio +async def test_update_credential_raises_proxy_exception_instead_of_returning_200() -> None: + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch.object( + credential_endpoints, + "CredentialsRepository", + side_effect=HTTPException(status_code=403, detail="forbidden"), + ), + patch.object( + credential_endpoints, + "handle_exception_on_proxy", + side_effect=lambda e: e, + ), + ): + with pytest.raises(HTTPException): + await update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="e2e-cred", + credential_values={"api_base": "https://x"}, + credential_info={}, + ), + credential_name="e2e-cred", + user_api_key_dict=_auth(), + )