mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-15 09:11:18 +00:00
205 lines
7.6 KiB
Go
205 lines
7.6 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
|
|
"github.com/iflytek/skillhub/github-sync-go/internal/github"
|
|
)
|
|
|
|
type shrinkingSearchFixture struct{ curatedRepositoryFixture }
|
|
|
|
func (shrinkingSearchFixture) SearchRepositories(_ context.Context, _, _ string,
|
|
_ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) {
|
|
if page != 1 {
|
|
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page")
|
|
}
|
|
repositories := []github.Repository{{ID: 11, Name: "first"}, {ID: 12, Name: "second"}}
|
|
for i := range repositories {
|
|
repositories[i].Owner.Login = "public-owner"
|
|
}
|
|
return github.SearchPage{Repositories: repositories}, github.Quota{}, nil
|
|
}
|
|
|
|
func TestProcessDiscoveryRevisitsShrinkingPage(t *testing.T) {
|
|
previous := `{"v":1,"entry":"SEARCH_RULE","page":1,"offset":3}`
|
|
var reported controlplane.DiscoveryBatch
|
|
var completed bool
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
switch {
|
|
case strings.HasSuffix(request.URL.Path, "/discovered-repositories"):
|
|
if err := json.NewDecoder(request.Body).Decode(&reported); err != nil ||
|
|
reported.PreviousCheckpoint == nil || *reported.PreviousCheckpoint != previous {
|
|
writer.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
writer.WriteHeader(http.StatusAccepted)
|
|
case strings.HasSuffix(request.URL.Path, "/complete"):
|
|
var body struct {
|
|
DiscoveryCheckpoint string `json:"discoveryCheckpoint"`
|
|
}
|
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil ||
|
|
body.DiscoveryCheckpoint != reported.Checkpoint {
|
|
writer.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
completed = true
|
|
writer.Header().Set("Content-Type", "application/json")
|
|
_, _ = writer.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`))
|
|
default:
|
|
writer.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := &controlplane.ClaimedTask{
|
|
TaskID: "shrinking-page", Kind: "DISCOVERY",
|
|
Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)},
|
|
Discovery: &controlplane.DiscoverySnapshot{
|
|
EntryType: "SEARCH_RULE", Checkpoint: previous, CandidateBudget: 3,
|
|
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
|
|
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
|
|
}},
|
|
SearchWindow: &controlplane.SearchWindow{
|
|
Mode: "FULL_RECHECK", Until: time.Now(),
|
|
},
|
|
},
|
|
}
|
|
if err := processDiscoveryWithGitHub(context.Background(), control, task,
|
|
DiscoveryLimits{MaxPagesPerLease: 2}, shrinkingSearchFixture{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var next discoveryCursor
|
|
if err := json.Unmarshal([]byte(reported.Checkpoint), &next); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !completed || len(reported.Repositories) != 2 ||
|
|
reported.Repositories[0].RepositoryID != 11 || reported.Repositories[1].RepositoryID != 12 ||
|
|
next.Revisit != 1 || next.Selector != 1 {
|
|
t.Fatalf("shrinking page was not revisited: completed=%v batch=%+v cursor=%+v",
|
|
completed, reported.Repositories, next)
|
|
}
|
|
}
|
|
|
|
func TestPreviewSearchResultsUsesOnePagePerSelectorAndPreservesWarnings(t *testing.T) {
|
|
until := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)
|
|
snapshot := &controlplane.DiscoverySnapshot{
|
|
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
|
|
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
|
|
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
|
|
}},
|
|
SearchWindow: &controlplane.SearchWindow{Mode: "FULL_RECHECK", Until: until},
|
|
}
|
|
var calls []string
|
|
search := func(_ context.Context, kind, value string, since *time.Time,
|
|
end time.Time, page int) (github.SearchPage, github.Quota, error) {
|
|
if since != nil || !end.Equal(until) || page != 1 {
|
|
t.Fatalf("unexpected preview search window or page: since=%v end=%s page=%d",
|
|
since, end, page)
|
|
}
|
|
calls = append(calls, kind+":"+value)
|
|
repositories := make([]github.Repository, 6)
|
|
for i := range repositories {
|
|
repositories[i].ID = int64(i + 1)
|
|
repositories[i].Name = "skill"
|
|
repositories[i].Owner.Login = "public-owner"
|
|
}
|
|
return github.SearchPage{
|
|
Repositories: repositories, TotalCount: 1001,
|
|
IncompleteResults: true, LimitReached: true,
|
|
}, github.Quota{}, nil
|
|
}
|
|
|
|
results, err := previewSearchResults(context.Background(), snapshot, search)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" ||
|
|
calls[1] != "TOPIC:agent-skills" || len(results) != 2 {
|
|
t.Fatalf("unexpected selectors: calls=%v results=%+v", calls, results)
|
|
}
|
|
for _, result := range results {
|
|
if result.TotalCount != 1001 || !result.IncompleteResults ||
|
|
!result.LimitReached || len(result.Samples) != 5 {
|
|
t.Fatalf("preview lost estimates or warnings: %+v", result)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPreviewSearchResultsDoesNotReturnPartialResultsAfterFailure(t *testing.T) {
|
|
snapshot := &controlplane.DiscoverySnapshot{
|
|
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
|
|
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
|
|
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
|
|
}},
|
|
SearchWindow: &controlplane.SearchWindow{Until: time.Now()},
|
|
}
|
|
calls := 0
|
|
results, err := previewSearchResults(context.Background(), snapshot,
|
|
func(context.Context, string, string, *time.Time, time.Time, int) (
|
|
github.SearchPage, github.Quota, error) {
|
|
calls++
|
|
if calls == 2 {
|
|
return github.SearchPage{}, github.Quota{}, errors.New("rate limited")
|
|
}
|
|
return github.SearchPage{TotalCount: 1}, github.Quota{}, nil
|
|
})
|
|
if err == nil || len(results) != 0 || calls != 2 {
|
|
t.Fatalf("preview should fail atomically: results=%+v err=%v calls=%d",
|
|
results, err, calls)
|
|
}
|
|
}
|
|
|
|
func TestSelectDiscoveryBatchCountsUniqueRepositoriesAndKeepsRawOffset(t *testing.T) {
|
|
candidates := []controlplane.RepositoryCandidate{
|
|
{RepositoryID: 10}, {RepositoryID: 10}, {RepositoryID: 20},
|
|
{RepositoryID: 20}, {RepositoryID: 30},
|
|
}
|
|
batch, offset := selectDiscoveryBatch(candidates, 0, 2, map[int64]struct{}{})
|
|
if offset != 3 || len(batch) != 2 || batch[0].RepositoryID != 10 || batch[1].RepositoryID != 20 {
|
|
t.Fatalf("first batch = %+v, offset = %d", batch, offset)
|
|
}
|
|
batch, offset = selectDiscoveryBatch(candidates, offset, 1,
|
|
map[int64]struct{}{10: {}, 20: {}})
|
|
if offset != len(candidates) || len(batch) != 1 || batch[0].RepositoryID != 30 {
|
|
t.Fatalf("second batch = %+v, offset = %d", batch, offset)
|
|
}
|
|
}
|
|
|
|
func TestSearchBoundsPartitionsFixedWindowWithoutOverlap(t *testing.T) {
|
|
since := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
|
|
window := &controlplane.SearchWindow{
|
|
Mode: "INCREMENTAL", Since: since, Until: since.Add(7 * time.Second),
|
|
}
|
|
leftFrom, leftUntil, err := searchBounds(window, "L")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rightFrom, rightUntil, err := searchBounds(window, "R")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !leftFrom.Equal(since) || !leftUntil.Add(time.Second).Equal(rightFrom) ||
|
|
!rightUntil.Equal(window.Until) || nextSearchCursor(discoveryCursor{Path: "L"}).Path != "R" {
|
|
t.Fatalf("split windows are not contiguous: [%s, %s], [%s, %s]",
|
|
leftFrom, leftUntil, rightFrom, rightUntil)
|
|
}
|
|
}
|
|
|
|
func TestSelectDiscoveryBatchCanAdvancePastOnlyDuplicates(t *testing.T) {
|
|
candidates := []controlplane.RepositoryCandidate{{RepositoryID: 10}, {RepositoryID: 10}}
|
|
batch, offset := selectDiscoveryBatch(candidates, 0, 1, map[int64]struct{}{10: {}})
|
|
if len(batch) != 0 || offset != len(candidates) {
|
|
t.Fatalf("batch = %+v, offset = %d", batch, offset)
|
|
}
|
|
}
|