scriptoriumadmin/skillhub
1
0
Fork 0
mirror of https://github.com/iflytek/skillhub.git synced 2026-08-27 11:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(search): observe label search rebuild failures

Browse source 
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS 2026-08-04 10:43:24 +08:00
parent 55e5c1e10e
commit b6cd414588
4 changed files with 169 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java
View file

@ -60,4 +60,11 @@ public class SkillHubMetrics {
"operation", operation
).increment();
}
public void incrementSearchRebuildFailure(String trigger) {
meterRegistry.counter(
"skillhub.search.rebuild.failure",
"trigger", trigger
).increment();
}
}

34
server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java
View file

@ -1,5 +1,6 @@
package com.iflytek.skillhub.service;
import com.iflytek.skillhub.metrics.SkillHubMetrics;
import com.iflytek.skillhub.search.SearchRebuildService;
import java.util.List;
import org.slf4j.Logger;
@ -11,22 +12,21 @@ import org.springframework.stereotype.Service;
public class LabelSearchSyncService {
static final int REBUILD_BATCH_SIZE = 50;
static final int REBUILD_MAX_ATTEMPTS = 2;
private static final Logger log = LoggerFactory.getLogger(LabelSearchSyncService.class);
private final SearchRebuildService searchRebuildService;
private final SkillHubMetrics metrics;
public LabelSearchSyncService(SearchRebuildService searchRebuildService) {
public LabelSearchSyncService(SearchRebuildService searchRebuildService, SkillHubMetrics metrics) {
this.searchRebuildService = searchRebuildService;
this.metrics = metrics;
}
@Async("skillhubEventExecutor")
public void rebuildSkill(Long skillId) {
try {
searchRebuildService.rebuildBySkill(skillId);
} catch (RuntimeException ex) {
log.error("Failed to rebuild search document for skill {}", skillId, ex);
}
rebuildBySkill(skillId, "single", "Failed to rebuild search document for skill {}");
}
@Async("skillhubEventExecutor")
@ -38,11 +38,7 @@ public class LabelSearchSyncService {
for (int i = 0; i < normalizedSkillIds.size(); i += REBUILD_BATCH_SIZE) {
List<Long> batch = normalizedSkillIds.subList(i, Math.min(i + REBUILD_BATCH_SIZE, normalizedSkillIds.size()));
for (Long skillId : batch) {
try {
searchRebuildService.rebuildBySkill(skillId);
} catch (RuntimeException ex) {
log.error("Failed to rebuild search document for skill {} after label change", skillId, ex);
}
rebuildBySkill(skillId, "batch", "Failed to rebuild search document for skill {} after label change");
}
}
}
@ -51,4 +47,20 @@ public class LabelSearchSyncService {
public void rebuildSkillsAsync(List<Long> skillIds) {
rebuildSkills(skillIds);
}
private void rebuildBySkill(Long skillId, String trigger, String finalFailureMessage) {
for (int attempt = 1; attempt <= REBUILD_MAX_ATTEMPTS; attempt++) {
try {
searchRebuildService.rebuildBySkill(skillId);
return;
} catch (RuntimeException ex) {
if (attempt < REBUILD_MAX_ATTEMPTS) {
log.warn("Retrying search document rebuild for skill {} after attempt {}", skillId, attempt, ex);
continue;
}
metrics.incrementSearchRebuildFailure(trigger);
log.error(finalFailureMessage, skillId, ex);
}
}
}
}

64
server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java
View file

@ -2,6 +2,7 @@ package com.iflytek.skillhub.service;
import com.iflytek.skillhub.SkillhubApplication;
import com.iflytek.skillhub.TestRedisConfig;
import com.iflytek.skillhub.auth.rbac.RbacService;
import com.iflytek.skillhub.domain.label.LabelDefinition;
import com.iflytek.skillhub.domain.label.LabelDefinitionRepository;
import com.iflytek.skillhub.domain.label.LabelTranslation;
@ -14,6 +15,8 @@ import com.iflytek.skillhub.domain.namespace.NamespaceType;
import com.iflytek.skillhub.domain.skill.Skill;
import com.iflytek.skillhub.domain.skill.SkillRepository;
import com.iflytek.skillhub.domain.skill.SkillVisibility;
import com.iflytek.skillhub.dto.AdminLabelUpdateRequest;
import com.iflytek.skillhub.dto.LabelTranslationItemRequest;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity;
import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository;
import com.iflytek.skillhub.search.SearchEmbeddingService;
@ -23,6 +26,7 @@ import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -58,6 +62,9 @@ class LabelSearchSyncIntegrationTest {
@Autowired
private SkillLabelAppService skillLabelAppService;
@Autowired
private LabelAdminAppService labelAdminAppService;
@Autowired
private NamespaceRepository namespaceRepository;
@ -82,10 +89,14 @@ class LabelSearchSyncIntegrationTest {
@MockBean
private SearchEmbeddingService searchEmbeddingService;
@MockBean
private RbacService rbacService;
@BeforeEach
void setUp() {
when(searchEmbeddingService.embed(anyString())).thenReturn("");
when(searchEmbeddingService.similarity(anyString(), anyString())).thenReturn(0.0d);
when(rbacService.getUserRoleCodes(anyString())).thenReturn(Set.of("SUPER_ADMIN"));
}
@Test
@ -193,6 +204,41 @@ class LabelSearchSyncIntegrationTest {
.contains(f.labelDisplayName);
}
@Test
void updatingLabelTranslations_rebuildsAffectedSkillSearchKeywords() throws Exception {
Fixture f = createFixture();
String updatedEnglish = "DeepLearning" + UUID.randomUUID().toString().substring(0, 8);
String updatedChinese = "深度学习" + UUID.randomUUID().toString().substring(0, 8);
skillLabelAppService.attachLabel(
f.namespaceSlug, f.skillSlug, f.labelSlug, f.ownerId, f.ownerRoles, auditContext());
SkillSearchDocumentEntity afterAttach = awaitIndexedDocument(f.skillId);
assertThat(afterAttach.getKeywords()).contains(f.labelDisplayName);
labelAdminAppService.update(
f.labelSlug,
new AdminLabelUpdateRequest(
LabelType.RECOMMENDED,
true,
0,
List.of(
new LabelTranslationItemRequest("en", updatedEnglish),
new LabelTranslationItemRequest("zh-CN", updatedChinese)
)
),
f.ownerId,
auditContext()
);
awaitKeywordPresent(f.skillId, updatedEnglish);
SkillSearchDocumentEntity afterUpdate = skillSearchDocumentJpaRepository.findBySkillId(f.skillId)
.orElseThrow();
assertThat(afterUpdate.getKeywords())
.contains(updatedEnglish)
.contains(updatedChinese)
.doesNotContain(f.labelDisplayName);
}
private SkillSearchDocumentEntity awaitIndexedDocument(Long skillId) throws InterruptedException {
Instant deadline = Instant.now().plus(Duration.ofSeconds(15));
Optional<SkillSearchDocumentEntity> indexed = skillSearchDocumentJpaRepository.findBySkillId(skillId);
@ -222,6 +268,24 @@ class LabelSearchSyncIntegrationTest {
+ skillId + " but keywords were: " + keywords);
}
private void awaitKeywordPresent(Long skillId, String keyword) throws InterruptedException {
Instant deadline = Instant.now().plus(Duration.ofSeconds(15));
while (Instant.now().isBefore(deadline)) {
Optional<SkillSearchDocumentEntity> indexed =
skillSearchDocumentJpaRepository.findBySkillId(skillId);
if (indexed.isPresent() && indexed.get().getKeywords().contains(keyword)) {
return;
}
Thread.sleep(100L);
}
String keywords = skillSearchDocumentJpaRepository.findBySkillId(skillId)
.map(SkillSearchDocumentEntity::getKeywords)
.orElse("<no document>");
throw new AssertionError(
"Expected keyword '" + keyword + "' to be present in index for skill "
+ skillId + " but keywords were: " + keywords);
}
private AuditRequestContext auditContext() {
return new AuditRequestContext("127.0.0.1", "junit");
}

76
server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java
View file

@ -1,12 +1,18 @@
package com.iflytek.skillhub.service;
import com.iflytek.skillhub.metrics.SkillHubMetrics;
import com.iflytek.skillhub.search.SearchRebuildService;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
@ -15,7 +21,11 @@ class LabelSearchSyncServiceTest {
@Test
void rebuildSkillsShouldSkipNullsAndDuplicatesWhileProcessingLargeLists() {
SearchRebuildService rebuildService = mock(SearchRebuildService.class);
LabelSearchSyncService service = new LabelSearchSyncService(rebuildService);
SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
LabelSearchSyncService service = new LabelSearchSyncService(
rebuildService,
new SkillHubMetrics(meterRegistry)
);
List<Long> skillIds = new ArrayList<>();
skillIds.add(null);
for (long i = 1; i <= 120; i++) {
@ -30,5 +40,69 @@ class LabelSearchSyncServiceTest {
verify(rebuildService).rebuildBySkill(i);
}
verifyNoMoreInteractions(rebuildService);
assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull();
}
@Test
void rebuildSkillShouldRetryTransientFailureWithoutRecordingFailureMetric() {
SearchRebuildService rebuildService = mock(SearchRebuildService.class);
SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
LabelSearchSyncService service = new LabelSearchSyncService(
rebuildService,
new SkillHubMetrics(meterRegistry)
);
doThrow(new IllegalStateException("temporary index failure"))
.doNothing()
.when(rebuildService)
.rebuildBySkill(42L);
service.rebuildSkill(42L);
verify(rebuildService, times(2)).rebuildBySkill(42L);
assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull();
}
@Test
void rebuildSkillShouldRecordFailureMetricAfterRetriesAreExhausted() {
SearchRebuildService rebuildService = mock(SearchRebuildService.class);
SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
LabelSearchSyncService service = new LabelSearchSyncService(
rebuildService,
new SkillHubMetrics(meterRegistry)
);
doThrow(new IllegalStateException("index unavailable"))
.when(rebuildService)
.rebuildBySkill(42L);
service.rebuildSkill(42L);
verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(42L);
assertThat(meterRegistry.get("skillhub.search.rebuild.failure")
.tag("trigger", "single")
.counter()
.count()).isEqualTo(1.0d);
}
@Test
void rebuildSkillsShouldRecordBatchFailureMetricAndContinue() {
SearchRebuildService rebuildService = mock(SearchRebuildService.class);
SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
LabelSearchSyncService service = new LabelSearchSyncService(
rebuildService,
new SkillHubMetrics(meterRegistry)
);
doThrow(new IllegalStateException("temporary index failure"))
.when(rebuildService)
.rebuildBySkill(2L);
service.rebuildSkills(List.of(1L, 2L, 3L));
verify(rebuildService).rebuildBySkill(1L);
verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(2L);
verify(rebuildService).rebuildBySkill(3L);
assertThat(meterRegistry.get("skillhub.search.rebuild.failure")
.tag("trigger", "batch")
.counter()
.count()).isEqualTo(1.0d);
}
}