mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-05 08:10:14 +00:00
feat: add Jina as embedding provider for code indexing
- Add Jina to EmbedderProvider type and model profiles - Implement JinaEmbedder class with multi-vector embeddings support - Configure jina-embeddings-v4 model with code.query downstream task - Add UI components for Jina provider selection and API key input - Include proper error handling and rate limiting - Add localization support for Jina-related messages
This commit is contained in:
parent
cc0f9e3604
commit
b56695ea7d
14 changed files with 412 additions and 5 deletions
|
|
@ -21,7 +21,9 @@ export const CODEBASE_INDEX_DEFAULTS = {
|
|||
export const codebaseIndexConfigSchema = z.object({
|
||||
codebaseIndexEnabled: z.boolean().optional(),
|
||||
codebaseIndexQdrantUrl: z.string().optional(),
|
||||
codebaseIndexEmbedderProvider: z.enum(["openai", "ollama", "openai-compatible", "gemini", "mistral"]).optional(),
|
||||
codebaseIndexEmbedderProvider: z
|
||||
.enum(["openai", "ollama", "openai-compatible", "gemini", "mistral", "jina"])
|
||||
.optional(),
|
||||
codebaseIndexEmbedderBaseUrl: z.string().optional(),
|
||||
codebaseIndexEmbedderModelId: z.string().optional(),
|
||||
codebaseIndexEmbedderModelDimension: z.number().optional(),
|
||||
|
|
@ -48,6 +50,7 @@ export const codebaseIndexModelsSchema = z.object({
|
|||
"openai-compatible": z.record(z.string(), z.object({ dimension: z.number() })).optional(),
|
||||
gemini: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
|
||||
mistral: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
|
||||
jina: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
|
||||
})
|
||||
|
||||
export type CodebaseIndexModels = z.infer<typeof codebaseIndexModelsSchema>
|
||||
|
|
@ -64,6 +67,7 @@ export const codebaseIndexProviderSchema = z.object({
|
|||
codebaseIndexOpenAiCompatibleModelDimension: z.number().optional(),
|
||||
codebaseIndexGeminiApiKey: z.string().optional(),
|
||||
codebaseIndexMistralApiKey: z.string().optional(),
|
||||
codebaseIndexJinaApiKey: z.string().optional(),
|
||||
})
|
||||
|
||||
export type CodebaseIndexProvider = z.infer<typeof codebaseIndexProviderSchema>
|
||||
|
|
|
|||
|
|
@ -186,6 +186,7 @@ export const SECRET_STATE_KEYS = [
|
|||
"codebaseIndexOpenAiCompatibleApiKey",
|
||||
"codebaseIndexGeminiApiKey",
|
||||
"codebaseIndexMistralApiKey",
|
||||
"codebaseIndexJinaApiKey",
|
||||
"huggingFaceApiKey",
|
||||
] as const satisfies readonly (keyof ProviderSettings)[]
|
||||
export type SecretState = Pick<ProviderSettings, (typeof SECRET_STATE_KEYS)[number]>
|
||||
|
|
|
|||
|
|
@ -2036,6 +2036,9 @@ export const webviewMessageHandler = async (
|
|||
settings.codebaseIndexMistralApiKey,
|
||||
)
|
||||
}
|
||||
if (settings.codebaseIndexJinaApiKey !== undefined) {
|
||||
await provider.contextProxy.storeSecret("codebaseIndexJinaApiKey", settings.codebaseIndexJinaApiKey)
|
||||
}
|
||||
|
||||
// Send success response first - settings are saved regardless of validation
|
||||
await provider.postMessageToWebview({
|
||||
|
|
@ -2157,6 +2160,7 @@ export const webviewMessageHandler = async (
|
|||
))
|
||||
const hasGeminiApiKey = !!(await provider.context.secrets.get("codebaseIndexGeminiApiKey"))
|
||||
const hasMistralApiKey = !!(await provider.context.secrets.get("codebaseIndexMistralApiKey"))
|
||||
const hasJinaApiKey = !!(await provider.context.secrets.get("codebaseIndexJinaApiKey"))
|
||||
|
||||
provider.postMessageToWebview({
|
||||
type: "codeIndexSecretStatus",
|
||||
|
|
@ -2166,6 +2170,7 @@ export const webviewMessageHandler = async (
|
|||
hasOpenAiCompatibleApiKey,
|
||||
hasGeminiApiKey,
|
||||
hasMistralApiKey,
|
||||
hasJinaApiKey,
|
||||
},
|
||||
})
|
||||
break
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@
|
|||
"openAiCompatibleConfigMissing": "OpenAI Compatible configuration missing for embedder creation",
|
||||
"geminiConfigMissing": "Gemini configuration missing for embedder creation",
|
||||
"mistralConfigMissing": "Mistral configuration missing for embedder creation",
|
||||
"jinaConfigMissing": "Jina configuration missing for embedder creation",
|
||||
"invalidEmbedderType": "Invalid embedder type configured: {{embedderProvider}}",
|
||||
"vectorDimensionNotDeterminedOpenAiCompatible": "Could not determine vector dimension for model '{{modelId}}' with provider '{{provider}}'. Please ensure the 'Embedding Dimension' is correctly set in the OpenAI-Compatible provider settings.",
|
||||
"vectorDimensionNotDetermined": "Could not determine vector dimension for model '{{modelId}}' with provider '{{provider}}'. Check model profiles or configuration.",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export class CodeIndexConfigManager {
|
|||
private openAiCompatibleOptions?: { baseUrl: string; apiKey: string }
|
||||
private geminiOptions?: { apiKey: string }
|
||||
private mistralOptions?: { apiKey: string }
|
||||
private jinaOptions?: { apiKey: string }
|
||||
private qdrantUrl?: string = "http://localhost:6333"
|
||||
private qdrantApiKey?: string
|
||||
private searchMinScore?: number
|
||||
|
|
@ -69,6 +70,7 @@ export class CodeIndexConfigManager {
|
|||
const openAiCompatibleApiKey = this.contextProxy?.getSecret("codebaseIndexOpenAiCompatibleApiKey") ?? ""
|
||||
const geminiApiKey = this.contextProxy?.getSecret("codebaseIndexGeminiApiKey") ?? ""
|
||||
const mistralApiKey = this.contextProxy?.getSecret("codebaseIndexMistralApiKey") ?? ""
|
||||
const jinaApiKey = this.contextProxy?.getSecret("codebaseIndexJinaApiKey") ?? ""
|
||||
|
||||
// Update instance variables with configuration
|
||||
this.codebaseIndexEnabled = codebaseIndexEnabled ?? true
|
||||
|
|
@ -104,6 +106,8 @@ export class CodeIndexConfigManager {
|
|||
this.embedderProvider = "gemini"
|
||||
} else if (codebaseIndexEmbedderProvider === "mistral") {
|
||||
this.embedderProvider = "mistral"
|
||||
} else if (codebaseIndexEmbedderProvider === "jina") {
|
||||
this.embedderProvider = "jina"
|
||||
} else {
|
||||
this.embedderProvider = "openai"
|
||||
}
|
||||
|
|
@ -124,6 +128,7 @@ export class CodeIndexConfigManager {
|
|||
|
||||
this.geminiOptions = geminiApiKey ? { apiKey: geminiApiKey } : undefined
|
||||
this.mistralOptions = mistralApiKey ? { apiKey: mistralApiKey } : undefined
|
||||
this.jinaOptions = jinaApiKey ? { apiKey: jinaApiKey } : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -141,6 +146,7 @@ export class CodeIndexConfigManager {
|
|||
openAiCompatibleOptions?: { baseUrl: string; apiKey: string }
|
||||
geminiOptions?: { apiKey: string }
|
||||
mistralOptions?: { apiKey: string }
|
||||
jinaOptions?: { apiKey: string }
|
||||
qdrantUrl?: string
|
||||
qdrantApiKey?: string
|
||||
searchMinScore?: number
|
||||
|
|
@ -160,6 +166,7 @@ export class CodeIndexConfigManager {
|
|||
openAiCompatibleApiKey: this.openAiCompatibleOptions?.apiKey ?? "",
|
||||
geminiApiKey: this.geminiOptions?.apiKey ?? "",
|
||||
mistralApiKey: this.mistralOptions?.apiKey ?? "",
|
||||
jinaApiKey: this.jinaOptions?.apiKey ?? "",
|
||||
qdrantUrl: this.qdrantUrl ?? "",
|
||||
qdrantApiKey: this.qdrantApiKey ?? "",
|
||||
}
|
||||
|
|
@ -184,6 +191,7 @@ export class CodeIndexConfigManager {
|
|||
openAiCompatibleOptions: this.openAiCompatibleOptions,
|
||||
geminiOptions: this.geminiOptions,
|
||||
mistralOptions: this.mistralOptions,
|
||||
jinaOptions: this.jinaOptions,
|
||||
qdrantUrl: this.qdrantUrl,
|
||||
qdrantApiKey: this.qdrantApiKey,
|
||||
searchMinScore: this.currentSearchMinScore,
|
||||
|
|
@ -221,6 +229,11 @@ export class CodeIndexConfigManager {
|
|||
const qdrantUrl = this.qdrantUrl
|
||||
const isConfigured = !!(apiKey && qdrantUrl)
|
||||
return isConfigured
|
||||
} else if (this.embedderProvider === "jina") {
|
||||
const apiKey = this.jinaOptions?.apiKey
|
||||
const qdrantUrl = this.qdrantUrl
|
||||
const isConfigured = !!(apiKey && qdrantUrl)
|
||||
return isConfigured
|
||||
}
|
||||
return false // Should not happen if embedderProvider is always set correctly
|
||||
}
|
||||
|
|
@ -292,6 +305,7 @@ export class CodeIndexConfigManager {
|
|||
const currentModelDimension = this.modelDimension
|
||||
const currentGeminiApiKey = this.geminiOptions?.apiKey ?? ""
|
||||
const currentMistralApiKey = this.mistralOptions?.apiKey ?? ""
|
||||
const currentJinaApiKey = this.jinaOptions?.apiKey ?? ""
|
||||
const currentQdrantUrl = this.qdrantUrl ?? ""
|
||||
const currentQdrantApiKey = this.qdrantApiKey ?? ""
|
||||
|
||||
|
|
@ -318,6 +332,11 @@ export class CodeIndexConfigManager {
|
|||
return true
|
||||
}
|
||||
|
||||
const prevJinaApiKey = prev?.jinaApiKey ?? ""
|
||||
if (prevJinaApiKey !== currentJinaApiKey) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for model dimension changes (generic for all providers)
|
||||
if (prevModelDimension !== currentModelDimension) {
|
||||
return true
|
||||
|
|
@ -375,6 +394,7 @@ export class CodeIndexConfigManager {
|
|||
openAiCompatibleOptions: this.openAiCompatibleOptions,
|
||||
geminiOptions: this.geminiOptions,
|
||||
mistralOptions: this.mistralOptions,
|
||||
jinaOptions: this.jinaOptions,
|
||||
qdrantUrl: this.qdrantUrl,
|
||||
qdrantApiKey: this.qdrantApiKey,
|
||||
searchMinScore: this.currentSearchMinScore,
|
||||
|
|
|
|||
278
src/services/code-index/embedders/jina.ts
Normal file
278
src/services/code-index/embedders/jina.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces"
|
||||
import { getModelQueryPrefix } from "../../../shared/embeddingModels"
|
||||
import { t } from "../../../i18n"
|
||||
import {
|
||||
withValidationErrorHandling,
|
||||
formatEmbeddingError,
|
||||
getErrorMessageForStatus,
|
||||
} from "../shared/validation-helpers"
|
||||
import type { HttpError } from "../shared/validation-helpers"
|
||||
import { TelemetryEventName } from "@roo-code/types"
|
||||
import { TelemetryService } from "@roo-code/telemetry"
|
||||
import {
|
||||
MAX_BATCH_TOKENS,
|
||||
MAX_ITEM_TOKENS,
|
||||
MAX_BATCH_RETRIES as MAX_RETRIES,
|
||||
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
|
||||
} from "../constants"
|
||||
|
||||
interface JinaEmbeddingRequest {
|
||||
model: string
|
||||
input: string[]
|
||||
encoding_type?: "float" | "base64"
|
||||
task?: string
|
||||
dimensions?: number
|
||||
late_chunking?: boolean
|
||||
embedding_type?: "float" | "base64" | "binary" | "ubinary"
|
||||
}
|
||||
|
||||
interface JinaEmbeddingResponse {
|
||||
model: string
|
||||
object: "list"
|
||||
usage: {
|
||||
total_tokens: number
|
||||
prompt_tokens: number
|
||||
}
|
||||
data: Array<{
|
||||
object: "embedding"
|
||||
index: number
|
||||
embedding: number[] | string
|
||||
}>
|
||||
}
|
||||
|
||||
/**
|
||||
* Jina implementation of the embedder interface with batching and rate limiting
|
||||
* Uses jina-embeddings-v4 with multi-vector embeddings for code search
|
||||
*/
|
||||
export class JinaEmbedder implements IEmbedder {
|
||||
private readonly apiKey: string
|
||||
private readonly baseUrl = "https://api.jina.ai/v1"
|
||||
private readonly defaultModelId: string
|
||||
|
||||
/**
|
||||
* Creates a new Jina embedder
|
||||
* @param apiKey Jina API key
|
||||
* @param modelId Optional model identifier (defaults to jina-embeddings-v4)
|
||||
*/
|
||||
constructor(apiKey: string, modelId?: string) {
|
||||
this.apiKey = apiKey
|
||||
this.defaultModelId = modelId || "jina-embeddings-v4"
|
||||
}
|
||||
|
||||
get embedderInfo(): EmbedderInfo {
|
||||
return { name: "jina" }
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates embeddings for the given texts with batching and rate limiting
|
||||
* @param texts Array of text strings to embed
|
||||
* @param model Optional model identifier
|
||||
* @returns Promise resolving to embedding response
|
||||
*/
|
||||
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
|
||||
const modelToUse = model || this.defaultModelId
|
||||
|
||||
// Apply model-specific query prefix if required
|
||||
const queryPrefix = getModelQueryPrefix("jina", modelToUse)
|
||||
const processedTexts = queryPrefix
|
||||
? texts.map((text) => {
|
||||
// Prevent double-prefixing
|
||||
if (text.startsWith(queryPrefix)) {
|
||||
return text
|
||||
}
|
||||
return queryPrefix + text
|
||||
})
|
||||
: texts
|
||||
|
||||
let attempt = 0
|
||||
let lastError: Error | null = null
|
||||
|
||||
while (attempt < MAX_RETRIES) {
|
||||
attempt++
|
||||
|
||||
try {
|
||||
const batches = this.createBatches(processedTexts)
|
||||
const allEmbeddings: number[][] = []
|
||||
let totalPromptTokens = 0
|
||||
let totalTokens = 0
|
||||
|
||||
for (const batch of batches) {
|
||||
const response = await this.fetchEmbeddings(batch, modelToUse)
|
||||
|
||||
// Extract embeddings from response
|
||||
const embeddings = response.data
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.map((item) => {
|
||||
if (typeof item.embedding === "string") {
|
||||
throw new Error("Base64/binary embeddings are not supported")
|
||||
}
|
||||
return item.embedding
|
||||
})
|
||||
|
||||
allEmbeddings.push(...embeddings)
|
||||
totalPromptTokens += response.usage.prompt_tokens
|
||||
totalTokens += response.usage.total_tokens
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: allEmbeddings,
|
||||
usage: {
|
||||
promptTokens: totalPromptTokens,
|
||||
totalTokens: totalTokens,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error as Error
|
||||
|
||||
if (error && typeof error === "object" && "status" in error) {
|
||||
const errorStatus = (error as any).status
|
||||
if (errorStatus === 401) {
|
||||
throw new Error(t("embeddings:authenticationFailed"))
|
||||
} else if (errorStatus === 429) {
|
||||
// Rate limit - retry with exponential backoff
|
||||
const delay = INITIAL_DELAY_MS * Math.pow(2, attempt - 1)
|
||||
console.warn(
|
||||
t("embeddings:rateLimitRetry", {
|
||||
delayMs: delay,
|
||||
attempt,
|
||||
maxRetries: MAX_RETRIES,
|
||||
}),
|
||||
)
|
||||
await new Promise((resolve) => setTimeout(resolve, delay))
|
||||
continue
|
||||
} else {
|
||||
throw new Error(
|
||||
t("embeddings:failedWithStatus", {
|
||||
attempts: attempt,
|
||||
statusCode: error.status,
|
||||
errorMessage: error.message,
|
||||
}),
|
||||
)
|
||||
}
|
||||
} else if (error instanceof Error) {
|
||||
throw new Error(
|
||||
t("embeddings:failedWithError", {
|
||||
attempts: attempt,
|
||||
errorMessage: error.message,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we've exhausted all retries
|
||||
if (lastError) {
|
||||
throw new Error(
|
||||
t("embeddings:failedMaxAttempts", {
|
||||
attempts: MAX_RETRIES,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(t("embeddings:unknownError"))
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the embedder configuration by testing connectivity and credentials
|
||||
* @returns Promise resolving to validation result
|
||||
*/
|
||||
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
|
||||
return withValidationErrorHandling(async () => {
|
||||
const testText = "function hello() { return 'world'; }"
|
||||
const response = await this.fetchEmbeddings([testText], this.defaultModelId)
|
||||
|
||||
// Validate response structure
|
||||
if (!response.data || !Array.isArray(response.data) || response.data.length === 0) {
|
||||
throw new Error(t("embeddings:validation.invalidResponse"))
|
||||
}
|
||||
|
||||
const embedding = response.data[0].embedding
|
||||
if (!Array.isArray(embedding) || embedding.length === 0) {
|
||||
throw new Error(t("embeddings:validation.invalidResponse"))
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
}, "jina")
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates batches of texts based on token limits
|
||||
*/
|
||||
private createBatches(texts: string[]): string[][] {
|
||||
const batches: string[][] = []
|
||||
let currentBatch: string[] = []
|
||||
let currentBatchTokens = 0
|
||||
|
||||
for (const text of texts) {
|
||||
// Rough token estimation (1 token ≈ 4 characters)
|
||||
const estimatedTokens = Math.ceil(text.length / 4)
|
||||
|
||||
// Check if this item exceeds the max item tokens
|
||||
if (estimatedTokens > MAX_ITEM_TOKENS) {
|
||||
console.warn(
|
||||
t("embeddings:textExceedsTokenLimit", {
|
||||
index: texts.indexOf(text),
|
||||
itemTokens: estimatedTokens,
|
||||
maxTokens: MAX_ITEM_TOKENS,
|
||||
}),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding this text would exceed batch limits, start a new batch
|
||||
if (currentBatch.length > 0 && currentBatchTokens + estimatedTokens > MAX_BATCH_TOKENS) {
|
||||
batches.push(currentBatch)
|
||||
currentBatch = []
|
||||
currentBatchTokens = 0
|
||||
}
|
||||
|
||||
currentBatch.push(text)
|
||||
currentBatchTokens += estimatedTokens
|
||||
}
|
||||
|
||||
// Don't forget the last batch
|
||||
if (currentBatch.length > 0) {
|
||||
batches.push(currentBatch)
|
||||
}
|
||||
|
||||
return batches
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches embeddings from Jina API
|
||||
*/
|
||||
private async fetchEmbeddings(texts: string[], model: string): Promise<JinaEmbeddingResponse> {
|
||||
const request: JinaEmbeddingRequest = {
|
||||
model,
|
||||
input: texts,
|
||||
encoding_type: "float",
|
||||
// Use code.query task for code search embeddings
|
||||
task: "code.query",
|
||||
// Request full 2048 dimensions for jina-embeddings-v4
|
||||
dimensions: model === "jina-embeddings-v4" ? 2048 : undefined,
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/embeddings`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text().catch(() => "Unknown error")
|
||||
const error = { status: response.status, message: errorData } as any
|
||||
throw formatEmbeddingError(error, MAX_RETRIES)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JinaEmbeddingResponse
|
||||
|
||||
// Capture telemetry
|
||||
// Log telemetry for successful embedding creation
|
||||
// Note: Currently only CODE_INDEX_ERROR event is available for code indexing
|
||||
|
||||
return data
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ export interface CodeIndexConfig {
|
|||
openAiCompatibleOptions?: { baseUrl: string; apiKey: string }
|
||||
geminiOptions?: { apiKey: string }
|
||||
mistralOptions?: { apiKey: string }
|
||||
jinaOptions?: { apiKey: string }
|
||||
qdrantUrl?: string
|
||||
qdrantApiKey?: string
|
||||
searchMinScore?: number
|
||||
|
|
@ -35,6 +36,7 @@ export type PreviousConfigSnapshot = {
|
|||
openAiCompatibleApiKey?: string
|
||||
geminiApiKey?: string
|
||||
mistralApiKey?: string
|
||||
jinaApiKey?: string
|
||||
qdrantUrl?: string
|
||||
qdrantApiKey?: string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ export interface EmbeddingResponse {
|
|||
}
|
||||
}
|
||||
|
||||
export type AvailableEmbedders = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral"
|
||||
export type AvailableEmbedders = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" | "jina"
|
||||
|
||||
export interface EmbedderInfo {
|
||||
name: AvailableEmbedders
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ export interface ICodeIndexManager {
|
|||
}
|
||||
|
||||
export type IndexingState = "Standby" | "Indexing" | "Indexed" | "Error"
|
||||
export type EmbedderProvider = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral"
|
||||
export type EmbedderProvider = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" | "jina"
|
||||
|
||||
export interface IndexProgressUpdate {
|
||||
systemStatus: IndexingState
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { CodeIndexOllamaEmbedder } from "./embedders/ollama"
|
|||
import { OpenAICompatibleEmbedder } from "./embedders/openai-compatible"
|
||||
import { GeminiEmbedder } from "./embedders/gemini"
|
||||
import { MistralEmbedder } from "./embedders/mistral"
|
||||
import { JinaEmbedder } from "./embedders/jina"
|
||||
import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../../shared/embeddingModels"
|
||||
import { QdrantVectorStore } from "./vector-store/qdrant-client"
|
||||
import { codeParser, DirectoryScanner, FileWatcher } from "./processors"
|
||||
|
|
@ -70,6 +71,11 @@ export class CodeIndexServiceFactory {
|
|||
throw new Error(t("embeddings:serviceFactory.mistralConfigMissing"))
|
||||
}
|
||||
return new MistralEmbedder(config.mistralOptions.apiKey, config.modelId)
|
||||
} else if (provider === "jina") {
|
||||
if (!config.jinaOptions?.apiKey) {
|
||||
throw new Error(t("embeddings:serviceFactory.jinaConfigMissing"))
|
||||
}
|
||||
return new JinaEmbedder(config.jinaOptions.apiKey, config.modelId)
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
|
|
|
|||
|
|
@ -255,7 +255,7 @@ export interface WebviewMessage {
|
|||
// Global state settings
|
||||
codebaseIndexEnabled: boolean
|
||||
codebaseIndexQdrantUrl: string
|
||||
codebaseIndexEmbedderProvider: "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral"
|
||||
codebaseIndexEmbedderProvider: "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" | "jina"
|
||||
codebaseIndexEmbedderBaseUrl?: string
|
||||
codebaseIndexEmbedderModelId: string
|
||||
codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers
|
||||
|
|
@ -269,6 +269,7 @@ export interface WebviewMessage {
|
|||
codebaseIndexOpenAiCompatibleApiKey?: string
|
||||
codebaseIndexGeminiApiKey?: string
|
||||
codebaseIndexMistralApiKey?: string
|
||||
codebaseIndexJinaApiKey?: string
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Defines profiles for different embedding models, including their dimensions.
|
||||
*/
|
||||
|
||||
export type EmbedderProvider = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" // Add other providers as needed
|
||||
export type EmbedderProvider = "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" | "jina" // Add other providers as needed
|
||||
|
||||
export interface EmbeddingModelProfile {
|
||||
dimension: number
|
||||
|
|
@ -53,6 +53,11 @@ export const EMBEDDING_MODEL_PROFILES: EmbeddingModelProfiles = {
|
|||
mistral: {
|
||||
"codestral-embed-2505": { dimension: 1536, scoreThreshold: 0.4 },
|
||||
},
|
||||
jina: {
|
||||
"jina-embeddings-v4": { dimension: 2048, scoreThreshold: 0.4 },
|
||||
"jina-embeddings-v3": { dimension: 1024, scoreThreshold: 0.4 },
|
||||
"jina-clip-v2": { dimension: 1024, scoreThreshold: 0.4 },
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -143,6 +148,9 @@ export function getDefaultModelId(provider: EmbedderProvider): string {
|
|||
case "mistral":
|
||||
return "codestral-embed-2505"
|
||||
|
||||
case "jina":
|
||||
return "jina-embeddings-v4"
|
||||
|
||||
default:
|
||||
// Fallback for unknown providers
|
||||
console.warn(`Unknown provider for default model ID: ${provider}. Falling back to OpenAI default.`)
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ interface LocalCodeIndexSettings {
|
|||
codebaseIndexOpenAiCompatibleApiKey?: string
|
||||
codebaseIndexGeminiApiKey?: string
|
||||
codebaseIndexMistralApiKey?: string
|
||||
codebaseIndexJinaApiKey?: string
|
||||
}
|
||||
|
||||
// Validation schema for codebase index settings
|
||||
|
|
@ -136,6 +137,14 @@ const createValidationSchema = (provider: EmbedderProvider, t: any) => {
|
|||
.min(1, t("settings:codeIndex.validation.modelSelectionRequired")),
|
||||
})
|
||||
|
||||
case "jina":
|
||||
return baseSchema.extend({
|
||||
codebaseIndexJinaApiKey: z.string().min(1, t("settings:codeIndex.validation.jinaApiKeyRequired")),
|
||||
codebaseIndexEmbedderModelId: z
|
||||
.string()
|
||||
.min(1, t("settings:codeIndex.validation.modelSelectionRequired")),
|
||||
})
|
||||
|
||||
default:
|
||||
return baseSchema
|
||||
}
|
||||
|
|
@ -628,6 +637,9 @@ export const CodeIndexPopover: React.FC<CodeIndexPopoverProps> = ({
|
|||
<SelectItem value="mistral">
|
||||
{t("settings:codeIndex.mistralProvider")}
|
||||
</SelectItem>
|
||||
<SelectItem value="jina">
|
||||
{t("settings:codeIndex.jinaProvider")}
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
|
@ -1020,6 +1032,71 @@ export const CodeIndexPopover: React.FC<CodeIndexPopoverProps> = ({
|
|||
</>
|
||||
)}
|
||||
|
||||
{currentSettings.codebaseIndexEmbedderProvider === "jina" && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">
|
||||
{t("settings:codeIndex.jinaApiKeyLabel")}
|
||||
</label>
|
||||
<VSCodeTextField
|
||||
type="password"
|
||||
value={currentSettings.codebaseIndexJinaApiKey || ""}
|
||||
onInput={(e: any) =>
|
||||
updateSetting("codebaseIndexJinaApiKey", e.target.value)
|
||||
}
|
||||
placeholder={t("settings:codeIndex.jinaApiKeyPlaceholder")}
|
||||
className={cn("w-full", {
|
||||
"border-red-500": formErrors.codebaseIndexJinaApiKey,
|
||||
})}
|
||||
/>
|
||||
{formErrors.codebaseIndexJinaApiKey && (
|
||||
<p className="text-xs text-vscode-errorForeground mt-1 mb-0">
|
||||
{formErrors.codebaseIndexJinaApiKey}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">
|
||||
{t("settings:codeIndex.modelLabel")}
|
||||
</label>
|
||||
<VSCodeDropdown
|
||||
value={currentSettings.codebaseIndexEmbedderModelId}
|
||||
onChange={(e: any) =>
|
||||
updateSetting("codebaseIndexEmbedderModelId", e.target.value)
|
||||
}
|
||||
className={cn("w-full", {
|
||||
"border-red-500": formErrors.codebaseIndexEmbedderModelId,
|
||||
})}>
|
||||
<VSCodeOption value="" className="p-2">
|
||||
{t("settings:codeIndex.selectModel")}
|
||||
</VSCodeOption>
|
||||
{getAvailableModels().map((modelId) => {
|
||||
const model =
|
||||
codebaseIndexModels?.[
|
||||
currentSettings.codebaseIndexEmbedderProvider
|
||||
]?.[modelId]
|
||||
return (
|
||||
<VSCodeOption key={modelId} value={modelId} className="p-2">
|
||||
{modelId}{" "}
|
||||
{model
|
||||
? t("settings:codeIndex.modelDimensions", {
|
||||
dimension: model.dimension,
|
||||
})
|
||||
: ""}
|
||||
</VSCodeOption>
|
||||
)
|
||||
})}
|
||||
</VSCodeDropdown>
|
||||
{formErrors.codebaseIndexEmbedderModelId && (
|
||||
<p className="text-xs text-vscode-errorForeground mt-1 mb-0">
|
||||
{formErrors.codebaseIndexEmbedderModelId}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Qdrant Settings */}
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">
|
||||
|
|
|
|||
|
|
@ -55,6 +55,9 @@
|
|||
"mistralProvider": "Mistral",
|
||||
"mistralApiKeyLabel": "API Key:",
|
||||
"mistralApiKeyPlaceholder": "Enter your Mistral API key",
|
||||
"jinaProvider": "Jina",
|
||||
"jinaApiKeyLabel": "API Key:",
|
||||
"jinaApiKeyPlaceholder": "Enter your Jina API key",
|
||||
"openaiCompatibleProvider": "OpenAI Compatible",
|
||||
"openAiKeyLabel": "OpenAI API Key",
|
||||
"openAiKeyPlaceholder": "Enter your OpenAI API key",
|
||||
|
|
@ -120,6 +123,7 @@
|
|||
"modelDimensionRequired": "Model dimension is required",
|
||||
"geminiApiKeyRequired": "Gemini API key is required",
|
||||
"mistralApiKeyRequired": "Mistral API key is required",
|
||||
"jinaApiKeyRequired": "Jina API key is required",
|
||||
"ollamaBaseUrlRequired": "Ollama base URL is required",
|
||||
"baseUrlRequired": "Base URL is required",
|
||||
"modelDimensionMinValue": "Model dimension must be greater than 0"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue