scriptoriumadmin/skillhub
1
0
Fork 0
mirror of https://github.com/iflytek/skillhub.git synced 2026-10-10 03:27:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat(search): add semantic reranking foundation

Browse source
This commit is contained in:
yun-zhi-ztl 2026-03-15 17:07:48 +08:00
parent 04eb468a19
commit 3a5e8d711e
13 changed files with 319 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
docs/04-search-architecture.md
View file

@ -107,10 +107,17 @@ PostgreSQL 全文搜索索引：表增加 `search_vector tsvector` 生成列，
| 阶段 | 实现 | 索引粒度 | 切换方式 |
|------|------|---------|---------|
| 一期 | PostgreSQL Full-Text (tsvector + GIN) | 每 skill 一条（latest_version_id） | 默认 |
| 一点五期 | PostgreSQL Full-Text + 语义向量重排 | 每 skill 一条（latest_version_id） | 配置 `skillhub.search.semantic.enabled=true` |
| 二期 | ES / OpenSearch | 每 skill_version 一条 + skill 聚合文档 | 配置 `search.provider=elasticsearch` |
| 三期 | 向量检索 | 每 skill_version 多条（chunk 级） | 配置 `search.provider=vector` |
| 四期 | 混合排序 | 关键词 + 向量混合 | 配置 `search.provider=hybrid` |
当前代码实现已落在“一点五期”：
- 仍然使用 PostgreSQL 全文搜索作为主召回
- 搜索文档表新增 `semantic_vector` 缓存字段
- relevance 排序下，对全文候选集追加语义向量重排
- 语义向量不可用时自动降级为现有全文相关度排序
### 5.3 SPI 演进策略
一期 SPI 接口（`SearchIndexService` / `SearchQueryService`）的入参是 `SkillSearchDocument`（skill 粒度）。二期切换到 ES 时：

5
server/skillhub-app/src/main/resources/application.yml
View file

@ -87,6 +87,11 @@ skillhub:
search:
engine: postgres
rebuild-on-startup: false
semantic:
enabled: true
weight: 0.35
candidate-multiplier: 8
max-candidates: 120
publish:
max-file-count: 100
max-single-file-size: 1048576 # 1MB

2
server/skillhub-app/src/main/resources/db/migration/V11__skill_search_semantic_vector.sql Normal file
View file

@ -0,0 +1,2 @@
ALTER TABLE skill_search_document
ADD COLUMN semantic_vector TEXT;

13
server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillSearchDocumentEntity.java
View file

@ -35,6 +35,9 @@ public class SkillSearchDocumentEntity {
@Column(name = "search_text", columnDefinition = "TEXT")
private String searchText;
@Column(name = "semantic_vector", columnDefinition = "TEXT")
private String semanticVector;
@Column(nullable = false, length = 20)
private String visibility;
@ -56,6 +59,7 @@ public class SkillSearchDocumentEntity {
String summary,
String keywords,
String searchText,
String semanticVector,
String visibility,
String status) {
this.skillId = skillId;
@ -66,6 +70,7 @@ public class SkillSearchDocumentEntity {
this.summary = summary;
this.keywords = keywords;
this.searchText = searchText;
this.semanticVector = semanticVector;
this.visibility = visibility;
this.status = status;
}
@ -117,6 +122,10 @@ public class SkillSearchDocumentEntity {
return visibility;
}
public String getSemanticVector() {
return semanticVector;
}
public String getStatus() {
return status;
}
@ -154,6 +163,10 @@ public class SkillSearchDocumentEntity {
this.searchText = searchText;
}
public void setSemanticVector(String semanticVector) {
this.semanticVector = semanticVector;
}
public void setVisibility(String visibility) {
this.visibility = visibility;
}

3
server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillSearchDocumentJpaRepository.java
View file

@ -3,10 +3,13 @@ package com.iflytek.skillhub.infra.jpa;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Repository
public interface SkillSearchDocumentJpaRepository extends JpaRepository<SkillSearchDocumentEntity, Long> {
Optional<SkillSearchDocumentEntity> findBySkillId(Long skillId);
List<SkillSearchDocumentEntity> findBySkillIdIn(Collection<Long> skillIds);
void deleteBySkillId(Long skillId);
}

83
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/HashingSearchEmbeddingService.java Normal file
View file

@ -0,0 +1,83 @@
package com.iflytek.skillhub.search;
import java.util.Arrays;
import java.util.Locale;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
@Service
public class HashingSearchEmbeddingService implements SearchEmbeddingService {
private static final Pattern TOKEN_SPLITTER = Pattern.compile("[^\\p{L}\\p{N}_]+");
private static final int DIMENSIONS = 64;
@Override
public String embed(String text) {
double[] vector = buildVector(text);
return Arrays.stream(vector)
.mapToObj(value -> String.format(Locale.ROOT, "%.6f", value))
.collect(Collectors.joining(","));
}
@Override
public double similarity(String text, String serializedVector) {
if (serializedVector == null || serializedVector.isBlank()) {
return 0D;
}
double[] left = buildVector(text);
double[] right = parseVector(serializedVector);
if (left.length != right.length || left.length == 0) {
return 0D;
}
double dot = 0D;
for (int i = 0; i < left.length; i++) {
dot += left[i] * right[i];
}
return dot;
}
private double[] buildVector(String text) {
double[] vector = new double[DIMENSIONS];
if (text == null || text.isBlank()) {
return vector;
}
TOKEN_SPLITTER.splitAsStream(text.toLowerCase(Locale.ROOT))
.map(String::trim)
.filter(token -> !token.isBlank())
.forEach(token -> {
int hash = token.hashCode();
int index = Math.floorMod(hash, DIMENSIONS);
double weight = 1D + Math.min(token.length(), 12) / 12D;
vector[index] += weight;
});
normalize(vector);
return vector;
}
private double[] parseVector(String serializedVector) {
String[] parts = serializedVector.split(",");
double[] vector = new double[parts.length];
for (int i = 0; i < parts.length; i++) {
vector[i] = Double.parseDouble(parts[i]);
}
normalize(vector);
return vector;
}
private void normalize(double[] vector) {
double magnitude = 0D;
for (double value : vector) {
magnitude += value * value;
}
if (magnitude == 0D) {
return;
}
double norm = Math.sqrt(magnitude);
for (int i = 0; i < vector.length; i++) {
vector[i] = vector[i] / norm;
}
}
}

7
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/SearchEmbeddingService.java Normal file
View file

@ -0,0 +1,7 @@
package com.iflytek.skillhub.search;
public interface SearchEmbeddingService {
String embed(String text);
double similarity(String text, String serializedVector);
}

1
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/SkillSearchDocument.java
View file

@ -9,6 +9,7 @@ public record SkillSearchDocument(
String summary,
String keywords,
String searchText,
String semanticVector,
String visibility,
String status
) {}

20
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/postgres/PostgresFullTextIndexService.java
View file

@ -2,6 +2,7 @@ package com.iflytek.skillhub.search.postgres;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository;
import com.iflytek.skillhub.search.SearchEmbeddingService;
import com.iflytek.skillhub.search.SearchIndexService;
import com.iflytek.skillhub.search.SkillSearchDocument;
import org.springframework.stereotype.Service;
@ -14,9 +15,12 @@ import java.util.Optional;
public class PostgresFullTextIndexService implements SearchIndexService {
private final SkillSearchDocumentJpaRepository repository;
private final SearchEmbeddingService searchEmbeddingService;
public PostgresFullTextIndexService(SkillSearchDocumentJpaRepository repository) {
public PostgresFullTextIndexService(SkillSearchDocumentJpaRepository repository,
SearchEmbeddingService searchEmbeddingService) {
this.repository = repository;
this.searchEmbeddingService = searchEmbeddingService;
}
@Override
@ -33,6 +37,7 @@ public class PostgresFullTextIndexService implements SearchIndexService {
entity.setSummary(document.summary());
entity.setKeywords(document.keywords());
entity.setSearchText(document.searchText());
entity.setSemanticVector(buildSemanticVector(document));
entity.setVisibility(document.visibility());
entity.setStatus(document.status());
repository.save(entity);
@ -46,6 +51,7 @@ public class PostgresFullTextIndexService implements SearchIndexService {
document.summary(),
document.keywords(),
document.searchText(),
buildSemanticVector(document),
document.visibility(),
document.status()
);
@ -66,4 +72,16 @@ public class PostgresFullTextIndexService implements SearchIndexService {
public void remove(Long skillId) {
repository.deleteBySkillId(skillId);
}
private String buildSemanticVector(SkillSearchDocument document) {
return searchEmbeddingService.embed(String.join("\n",
safe(document.title()),
safe(document.summary()),
safe(document.keywords()),
safe(document.searchText())));
}
private String safe(String value) {
return value == null ? "" : value;
}
}

101
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/postgres/PostgresFullTextQueryService.java
View file

@ -1,13 +1,21 @@
package com.iflytek.skillhub.search.postgres;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository;
import com.iflytek.skillhub.search.SearchEmbeddingService;
import com.iflytek.skillhub.search.SearchQuery;
import com.iflytek.skillhub.search.SearchQueryService;
import com.iflytek.skillhub.search.SearchResult;
import jakarta.persistence.EntityManager;
import jakarta.persistence.Query;
import java.util.Comparator;
import java.util.HashMap;
import org.springframework.stereotype.Service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@ -20,9 +28,32 @@ public class PostgresFullTextQueryService implements SearchQueryService {
private static final String TITLE_SQL = "LOWER(title)";
private final EntityManager entityManager;
private final SkillSearchDocumentJpaRepository searchDocumentRepository;
private final SearchEmbeddingService searchEmbeddingService;
private final boolean semanticEnabled;
private final double semanticWeight;
private final int candidateMultiplier;
private final int maxCandidates;
public PostgresFullTextQueryService(EntityManager entityManager) {
this(entityManager, null, null, false, 0.35D, 8, 120);
}
@Autowired
public PostgresFullTextQueryService(EntityManager entityManager,
SkillSearchDocumentJpaRepository searchDocumentRepository,
SearchEmbeddingService searchEmbeddingService,
@Value("${skillhub.search.semantic.enabled:true}") boolean semanticEnabled,
@Value("${skillhub.search.semantic.weight:0.35}") double semanticWeight,
@Value("${skillhub.search.semantic.candidate-multiplier:8}") int candidateMultiplier,
@Value("${skillhub.search.semantic.max-candidates:120}") int maxCandidates) {
this.entityManager = entityManager;
this.searchDocumentRepository = searchDocumentRepository;
this.searchEmbeddingService = searchEmbeddingService;
this.semanticEnabled = semanticEnabled;
this.semanticWeight = semanticWeight;
this.candidateMultiplier = candidateMultiplier;
this.maxCandidates = maxCandidates;
}
@Override
@ -31,6 +62,18 @@ public class PostgresFullTextQueryService implements SearchQueryService {
String tsQuery = buildPrefixTsQuery(normalizedKeyword);
boolean hasKeyword = tsQuery != null;
boolean useShortPrefixTitleSearch = hasKeyword && normalizedKeyword.length() <= SHORT_PREFIX_LENGTH;
boolean useSemanticRerank = semanticEnabled
&& hasKeyword
&& "relevance".equals(query.sortBy())
&& searchDocumentRepository != null
&& searchEmbeddingService != null;
int requestedOffset = query.page() * query.size();
int sqlLimit = query.size();
int sqlOffset = requestedOffset;
if (useSemanticRerank) {
sqlLimit = Math.min(Math.max((query.page() + 1) * query.size() * candidateMultiplier, query.size() * candidateMultiplier), maxCandidates);
sqlOffset = 0;
}
Set<Long> memberNamespaceIds = query.visibilityScope().memberNamespaceIds().isEmpty()
? Set.of(-1L)
: query.visibilityScope().memberNamespaceIds();
@ -114,8 +157,8 @@ public class PostgresFullTextQueryService implements SearchQueryService {
nativeQuery.setParameter("titleLike", "%" + normalizedKeyword.toLowerCase() + "%");
}
nativeQuery.setParameter("limit", query.size());
nativeQuery.setParameter("offset", query.page() * query.size());
nativeQuery.setParameter("limit", sqlLimit);
nativeQuery.setParameter("offset", sqlOffset);
@SuppressWarnings("unchecked")
List<Long> skillIds = (List<Long>) nativeQuery.getResultList().stream()
@ -152,9 +195,60 @@ public class PostgresFullTextQueryService implements SearchQueryService {
long total = ((Number) countQuery.getSingleResult()).longValue();
if (useSemanticRerank && !skillIds.isEmpty()) {
skillIds = rerankBySemanticSimilarity(skillIds, normalizedKeyword, requestedOffset, query.size());
}
return new SearchResult(skillIds, total, query.page(), query.size());
}
private List<Long> rerankBySemanticSimilarity(List<Long> candidateSkillIds,
String normalizedKeyword,
int requestedOffset,
int pageSize) {
Map<Long, SkillSearchDocumentEntity> documentsBySkillId = new HashMap<>();
for (SkillSearchDocumentEntity entity : searchDocumentRepository.findBySkillIdIn(candidateSkillIds)) {
documentsBySkillId.put(entity.getSkillId(), entity);
}
int totalCandidates = Math.max(candidateSkillIds.size(), 1);
List<RankedSkill> rankedSkills = new java.util.ArrayList<>(candidateSkillIds.size());
for (int index = 0; index < candidateSkillIds.size(); index++) {
Long skillId = candidateSkillIds.get(index);
SkillSearchDocumentEntity entity = documentsBySkillId.get(skillId);
double baseScore = 1D - (index / (double) totalCandidates);
double semanticScore = computeSemanticScore(normalizedKeyword, entity);
double combinedScore = (baseScore * (1D - semanticWeight)) + (semanticScore * semanticWeight);
rankedSkills.add(new RankedSkill(skillId, combinedScore));
}
return rankedSkills.stream()
.sorted(Comparator.comparingDouble(RankedSkill::score).reversed())
.skip(requestedOffset)
.limit(pageSize)
.map(RankedSkill::skillId)
.toList();
}
private double computeSemanticScore(String normalizedKeyword, SkillSearchDocumentEntity entity) {
if (entity == null) {
return 0D;
}
String serializedVector = entity.getSemanticVector();
if (serializedVector == null || serializedVector.isBlank()) {
serializedVector = searchEmbeddingService.embed(String.join("\n",
safe(entity.getTitle()),
safe(entity.getSummary()),
safe(entity.getKeywords()),
safe(entity.getSearchText())));
}
return searchEmbeddingService.similarity(normalizedKeyword, serializedVector);
}
private String safe(String value) {
return value == null ? "" : value;
}
private String normalizeKeyword(String keyword) {
if (keyword == null || keyword.isBlank()) {
return null;
@ -183,4 +277,7 @@ public class PostgresFullTextQueryService implements SearchQueryService {
.reduce((left, right) -> left + " & " + right)
.orElse(null);
}
private record RankedSkill(Long skillId, double score) {
}
}

1
server/skillhub-search/src/main/java/com/iflytek/skillhub/search/postgres/PostgresSearchRebuildService.java
View file

@ -88,6 +88,7 @@ public class PostgresSearchRebuildService implements SearchRebuildService {
skill.getSummary(),
"",
searchText,
null,
skill.getVisibility().name(),
skill.getStatus().name()
));

29
server/skillhub-search/src/test/java/com/iflytek/skillhub/search/HashingSearchEmbeddingServiceTest.java Normal file
View file

@ -0,0 +1,29 @@
package com.iflytek.skillhub.search;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
class HashingSearchEmbeddingServiceTest {
private final HashingSearchEmbeddingService service = new HashingSearchEmbeddingService();
@Test
void embedShouldBeDeterministic() {
String first = service.embed("self improving skill");
String second = service.embed("self improving skill");
assertThat(first).isEqualTo(second);
}
@Test
void similarityShouldFavorCloserText() {
String relevantVector = service.embed("self improvement productivity habit tracker");
String noisyVector = service.embed("web search keywords company research");
double relevant = service.similarity("self improvement", relevantVector);
double noisy = service.similarity("self improvement", noisyVector);
assertThat(relevant).isGreaterThan(noisy);
}
}

50
server/skillhub-search/src/test/java/com/iflytek/skillhub/search/postgres/PostgresFullTextQueryServiceTest.java
View file

@ -1,5 +1,8 @@
package com.iflytek.skillhub.search.postgres;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository;
import com.iflytek.skillhub.search.HashingSearchEmbeddingService;
import com.iflytek.skillhub.search.SearchQuery;
import com.iflytek.skillhub.search.SearchVisibilityScope;
import jakarta.persistence.EntityManager;
@ -165,4 +168,51 @@ class PostgresFullTextQueryServiceTest {
verify(nativeQuery).setParameter("tsQuery", "self:* & improving:*");
verify(countQuery).setParameter("tsQuery", "self:* & improving:*");
}
@Test
void semanticRerankShouldPromoteSemanticallyRelevantCandidate() {
EntityManager entityManager = mock(EntityManager.class);
Query nativeQuery = mock(Query.class);
Query countQuery = mock(Query.class);
SkillSearchDocumentJpaRepository repository = mock(SkillSearchDocumentJpaRepository.class);
HashingSearchEmbeddingService embeddingService = new HashingSearchEmbeddingService();
when(entityManager.createNativeQuery(anyString()))
.thenReturn(nativeQuery)
.thenReturn(countQuery);
when(nativeQuery.setParameter(anyString(), org.mockito.ArgumentMatchers.any())).thenReturn(nativeQuery);
when(countQuery.setParameter(anyString(), org.mockito.ArgumentMatchers.any())).thenReturn(countQuery);
when(nativeQuery.getResultList()).thenReturn(List.of(2L, 1L));
when(countQuery.getSingleResult()).thenReturn(2L);
when(repository.findBySkillIdIn(List.of(2L, 1L))).thenReturn(List.of(
new SkillSearchDocumentEntity(1L, 1L, "global", "user-1", "Self Improvement Coach",
"Build better habits", "habits,self improvement", "habit tracker and self improvement guide",
embeddingService.embed("habit tracker and self improvement guide"), "PUBLIC", "ACTIVE"),
new SkillSearchDocumentEntity(2L, 1L, "global", "user-2", "Web Search Exa",
"Research assistant", "keywords,search", "web search keywords company research",
embeddingService.embed("web search keywords company research"), "PUBLIC", "ACTIVE")
));
PostgresFullTextQueryService service = new PostgresFullTextQueryService(
entityManager,
repository,
embeddingService,
true,
0.6D,
8,
120
);
var result = service.search(new SearchQuery(
"self improvement",
null,
new SearchVisibilityScope(null, Set.of(), Set.of()),
"relevance",
0,
2
));
verify(nativeQuery).setParameter("limit", 16);
verify(nativeQuery).setParameter("offset", 0);
assertThat(result.skillIds()).containsExactly(1L, 2L);
}
}