skillhub/github-sync-go/internal/worker/discovery_test.go
XiaoSeS 2df4c913cb feat(github-sync): add continuous public repository discovery and skill sync
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
2026-10-20 14:40:46 +08:00

205 lines
7.6 KiB
Go

package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type shrinkingSearchFixture struct{ curatedRepositoryFixture }
func (shrinkingSearchFixture) SearchRepositories(_ context.Context, _, _ string,
_ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) {
if page != 1 {
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page")
}
repositories := []github.Repository{{ID: 11, Name: "first"}, {ID: 12, Name: "second"}}
for i := range repositories {
repositories[i].Owner.Login = "public-owner"
}
return github.SearchPage{Repositories: repositories}, github.Quota{}, nil
}
func TestProcessDiscoveryRevisitsShrinkingPage(t *testing.T) {
previous := `{"v":1,"entry":"SEARCH_RULE","page":1,"offset":3}`
var reported controlplane.DiscoveryBatch
var completed bool
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
switch {
case strings.HasSuffix(request.URL.Path, "/discovered-repositories"):
if err := json.NewDecoder(request.Body).Decode(&reported); err != nil ||
reported.PreviousCheckpoint == nil || *reported.PreviousCheckpoint != previous {
writer.WriteHeader(http.StatusBadRequest)
return
}
writer.WriteHeader(http.StatusAccepted)
case strings.HasSuffix(request.URL.Path, "/complete"):
var body struct {
DiscoveryCheckpoint string `json:"discoveryCheckpoint"`
}
if err := json.NewDecoder(request.Body).Decode(&body); err != nil ||
body.DiscoveryCheckpoint != reported.Checkpoint {
writer.WriteHeader(http.StatusBadRequest)
return
}
completed = true
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`))
default:
writer.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
task := &controlplane.ClaimedTask{
TaskID: "shrinking-page", Kind: "DISCOVERY",
Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)},
Discovery: &controlplane.DiscoverySnapshot{
EntryType: "SEARCH_RULE", Checkpoint: previous, CandidateBudget: 3,
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
}},
SearchWindow: &controlplane.SearchWindow{
Mode: "FULL_RECHECK", Until: time.Now(),
},
},
}
if err := processDiscoveryWithGitHub(context.Background(), control, task,
DiscoveryLimits{MaxPagesPerLease: 2}, shrinkingSearchFixture{}); err != nil {
t.Fatal(err)
}
var next discoveryCursor
if err := json.Unmarshal([]byte(reported.Checkpoint), &next); err != nil {
t.Fatal(err)
}
if !completed || len(reported.Repositories) != 2 ||
reported.Repositories[0].RepositoryID != 11 || reported.Repositories[1].RepositoryID != 12 ||
next.Revisit != 1 || next.Selector != 1 {
t.Fatalf("shrinking page was not revisited: completed=%v batch=%+v cursor=%+v",
completed, reported.Repositories, next)
}
}
func TestPreviewSearchResultsUsesOnePagePerSelectorAndPreservesWarnings(t *testing.T) {
until := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)
snapshot := &controlplane.DiscoverySnapshot{
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
}},
SearchWindow: &controlplane.SearchWindow{Mode: "FULL_RECHECK", Until: until},
}
var calls []string
search := func(_ context.Context, kind, value string, since *time.Time,
end time.Time, page int) (github.SearchPage, github.Quota, error) {
if since != nil || !end.Equal(until) || page != 1 {
t.Fatalf("unexpected preview search window or page: since=%v end=%s page=%d",
since, end, page)
}
calls = append(calls, kind+":"+value)
repositories := make([]github.Repository, 6)
for i := range repositories {
repositories[i].ID = int64(i + 1)
repositories[i].Name = "skill"
repositories[i].Owner.Login = "public-owner"
}
return github.SearchPage{
Repositories: repositories, TotalCount: 1001,
IncompleteResults: true, LimitReached: true,
}, github.Quota{}, nil
}
results, err := previewSearchResults(context.Background(), snapshot, search)
if err != nil {
t.Fatal(err)
}
if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" ||
calls[1] != "TOPIC:agent-skills" || len(results) != 2 {
t.Fatalf("unexpected selectors: calls=%v results=%+v", calls, results)
}
for _, result := range results {
if result.TotalCount != 1001 || !result.IncompleteResults ||
!result.LimitReached || len(result.Samples) != 5 {
t.Fatalf("preview lost estimates or warnings: %+v", result)
}
}
}
func TestPreviewSearchResultsDoesNotReturnPartialResultsAfterFailure(t *testing.T) {
snapshot := &controlplane.DiscoverySnapshot{
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
}},
SearchWindow: &controlplane.SearchWindow{Until: time.Now()},
}
calls := 0
results, err := previewSearchResults(context.Background(), snapshot,
func(context.Context, string, string, *time.Time, time.Time, int) (
github.SearchPage, github.Quota, error) {
calls++
if calls == 2 {
return github.SearchPage{}, github.Quota{}, errors.New("rate limited")
}
return github.SearchPage{TotalCount: 1}, github.Quota{}, nil
})
if err == nil || len(results) != 0 || calls != 2 {
t.Fatalf("preview should fail atomically: results=%+v err=%v calls=%d",
results, err, calls)
}
}
func TestSelectDiscoveryBatchCountsUniqueRepositoriesAndKeepsRawOffset(t *testing.T) {
candidates := []controlplane.RepositoryCandidate{
{RepositoryID: 10}, {RepositoryID: 10}, {RepositoryID: 20},
{RepositoryID: 20}, {RepositoryID: 30},
}
batch, offset := selectDiscoveryBatch(candidates, 0, 2, map[int64]struct{}{})
if offset != 3 || len(batch) != 2 || batch[0].RepositoryID != 10 || batch[1].RepositoryID != 20 {
t.Fatalf("first batch = %+v, offset = %d", batch, offset)
}
batch, offset = selectDiscoveryBatch(candidates, offset, 1,
map[int64]struct{}{10: {}, 20: {}})
if offset != len(candidates) || len(batch) != 1 || batch[0].RepositoryID != 30 {
t.Fatalf("second batch = %+v, offset = %d", batch, offset)
}
}
func TestSearchBoundsPartitionsFixedWindowWithoutOverlap(t *testing.T) {
since := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
window := &controlplane.SearchWindow{
Mode: "INCREMENTAL", Since: since, Until: since.Add(7 * time.Second),
}
leftFrom, leftUntil, err := searchBounds(window, "L")
if err != nil {
t.Fatal(err)
}
rightFrom, rightUntil, err := searchBounds(window, "R")
if err != nil {
t.Fatal(err)
}
if !leftFrom.Equal(since) || !leftUntil.Add(time.Second).Equal(rightFrom) ||
!rightUntil.Equal(window.Until) || nextSearchCursor(discoveryCursor{Path: "L"}).Path != "R" {
t.Fatalf("split windows are not contiguous: [%s, %s], [%s, %s]",
leftFrom, leftUntil, rightFrom, rightUntil)
}
}
func TestSelectDiscoveryBatchCanAdvancePastOnlyDuplicates(t *testing.T) {
candidates := []controlplane.RepositoryCandidate{{RepositoryID: 10}, {RepositoryID: 10}}
batch, offset := selectDiscoveryBatch(candidates, 0, 1, map[int64]struct{}{10: {}})
if len(batch) != 0 || offset != len(candidates) {
t.Fatalf("batch = %+v, offset = %d", batch, offset)
}
}