scriptoriumadmin/skillhub
1
0
Fork 0
mirror of https://github.com/iflytek/skillhub.git synced 2026-11-14 09:01:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

feat(github-sync): add continuous public repository discovery and skill sync

Browse source 
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS 2026-10-20 14:40:46 +08:00
parent d424c91f1f
commit 2df4c913cb
534 changed files with 55079 additions and 950 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.env.release.draft
View file

@ -78,6 +78,7 @@ SKILLHUB_STORAGE_S3_ENDPOINT=https://oss-cn-shanghai.aliyuncs.com
SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=
SKILLHUB_STORAGE_S3_BUCKET=skillhub-prod
# Set both static keys, or leave both empty to use the deployment's IAM credentials.
SKILLHUB_STORAGE_S3_ACCESS_KEY=TODO_fill_real_access_key
SKILLHUB_STORAGE_S3_SECRET_KEY=TODO_fill_real_secret_key
SKILLHUB_STORAGE_S3_REGION=cn-shanghai
@ -90,6 +91,25 @@ SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=false
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false
SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY=PT10M
# Set true for storage providers such as Aliyun OSS that reject aws-chunked uploads.
SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=false
# Optional public GitHub Skill sync. Keep the github-sync Compose profile disabled
# until Java and Go have the same random service token (at least 32 characters).
# For S3, set a positive staging prefix budget and allow the internal presigned PUT host.
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=0
SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN=
SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS=
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=
# Optional system GitHub credential pool: v1=<Base64-encoded 32-byte key>.
# Leave both blank for anonymous public API access.
SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS=
SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION=
# Outbound GitHub proxy only; Java control-plane and staging PUT remain direct.
SKILLHUB_GITHUB_SYNC_HTTP_PROXY=
SKILLHUB_GITHUB_SYNC_HTTPS_PROXY=
SKILLHUB_GITHUB_SYNC_NO_PROXY=
# Bootstrap admin is only for first login / first bootstrap.
BOOTSTRAP_ADMIN_ENABLED=true
BOOTSTRAP_ADMIN_USER_ID=docker-admin

14
.env.release.example
View file

@ -94,6 +94,20 @@ SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=false
SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false
SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY=PT10M
# When enabling GitHub sync with S3, set the staging prefix byte budget from real storage capacity.
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=0
# Enable the optional github-sync profile only after filling the shared Java/Go token.
SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN=
SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS=
# S3/MinIO: comma-separated internal hosts (host:port) used by presigned PUT URLs.
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=
# Optional system credential pool; both fields stay blank for anonymous public API access.
SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS=
SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION=
# Optional GitHub outbound proxy; Java internal and staging PUT requests stay direct.
SKILLHUB_GITHUB_SYNC_HTTP_PROXY=
SKILLHUB_GITHUB_SYNC_HTTPS_PROXY=
SKILLHUB_GITHUB_SYNC_NO_PROXY=
# Bootstrap local admin account for first login. Rotate or disable after initial setup.
BOOTSTRAP_ADMIN_ENABLED=true

57
.github/workflows/pr-github-sync.yml vendored Normal file
View file

@ -0,0 +1,57 @@
name: PR GitHub Sync
on:
pull_request:
paths:
- 'github-sync-go/**'
- '.github/workflows/pr-github-sync.yml'
permissions:
contents: read
jobs:
go:
name: Go checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-go@v5
with:
go-version-file: github-sync-go/go.mod
cache-dependency-path: github-sync-go/go.mod
- name: Test and vet
working-directory: github-sync-go
run: |
go test -count=1 ./...
go vet ./...
image:
name: Image (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: ubuntu-latest
- arch: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: docker/setup-buildx-action@v3
- name: Build image
uses: docker/build-push-action@v6
with:
context: github-sync-go
platforms: linux/${{ matrix.arch }}
tags: skillhub-github-sync-pr:${{ matrix.arch }}-${{ github.sha }}
load: true
- name: Check runtime tools and user
run: |
docker run --rm --platform linux/${{ matrix.arch }} \
--entrypoint sh skillhub-github-sync-pr:${{ matrix.arch }}-${{ github.sha }} \
-ec 'test "$(id -u)" = 10001; git --version >/dev/null; tini --version >/dev/null; test -x /usr/local/bin/github-sync'

12
.github/workflows/publish-images.yml vendored
View file

@ -45,17 +45,23 @@ jobs:
platforms: linux/amd64,linux/arm64
image: ghcr.io/${{ github.repository_owner }}/skillhub-scanner
mirror_image: skillhub-scanner
- name: github-sync
context: ./github-sync-go
dockerfile: ./github-sync-go/Dockerfile
platforms: linux/amd64,linux/arm64
image: ghcr.io/${{ github.repository_owner }}/skillhub-github-sync
mirror_image: skillhub-github-sync
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:

13
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build build-backend build-backend-app build-builtin-skills build-cli build-frontend build-web check clean cli-install db-reset dev dev-all dev-all-down dev-all-reset dev-down dev-logs dev-server dev-server-restart dev-status dev-web docs-build docs-dev docs-preview generate-api help lint-cli lint-web namespace-smoke suite-smoke suite-bundle-smoke parallel-down parallel-init parallel-sync parallel-up pr publish-cli publish-cli-major publish-cli-minor staging staging-down staging-logs test test-backend test-backend-app test-builtin-skills test-cli test-e2e-frontend test-e2e-smoke-frontend test-frontend test-redis-cluster test-web typecheck-cli typecheck-web validate-release-config web-deps web-install web-install-ci
.PHONY: build build-backend build-backend-app build-builtin-skills build-cli build-frontend build-web check clean cli-install db-reset dev dev-all dev-all-down dev-all-reset dev-down dev-github-sync dev-logs dev-server dev-server-restart dev-status dev-web docs-build docs-dev docs-preview generate-api help lint-cli lint-web namespace-smoke suite-smoke suite-bundle-smoke parallel-down parallel-init parallel-sync parallel-up pr publish-cli publish-cli-major publish-cli-minor staging staging-down staging-logs test test-backend test-backend-app test-builtin-skills test-cli test-e2e-frontend test-e2e-smoke-frontend test-frontend test-redis-cluster test-web typecheck-cli typecheck-web validate-release-config web-deps web-install web-install-ci
DEV_DIR := .dev
DEV_SERVER_PID := $(DEV_DIR)/server.pid
@ -129,6 +129,17 @@ dev-all: ## 一键启动本地开发环境（依赖 + scanner + 后端 + 前端
dev-server: ## 启动后端开发服务器
cd server && bash -lc '$(DEV_SERVER_PREPARE) && exec env $(DEV_SERVER_SCANNER_ENV) $(DEV_SERVER_CMD)'
dev-github-sync: ## 前台运行可选的本地 GitHub 采集服务（需先给 Java 和 Go 配置同一服务密钥）
@test -n "$${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-}" || { echo "SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN is required" >&2; exit 1; }
@mkdir -p $(DEV_DIR)/github-sync-tmp
cd github-sync-go && go build -o ../$(DEV_DIR)/github-sync ./cmd/github-sync
SKILLHUB_GITHUB_SYNC_JAVA_URL=$(DEV_API_URL)/internal/github-sync/v1 \
SKILLHUB_GITHUB_SYNC_TEMP_ROOT=$(abspath $(DEV_DIR))/github-sync-tmp \
SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES=$${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912} \
SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES=$${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912} \
SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES=$${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728} \
$(DEV_DIR)/github-sync
dev-server-restart: ## 重启后端开发服务器
@mkdir -p $(DEV_DIR)
@$(DEV_PROCESS) stop --pid-file $(DEV_SERVER_PID)

28
charts/skillhub/README.md
View file

@ -57,6 +57,34 @@ helm -n skillhub upgrade -i skillhub ./charts/skillhub \
`secrets.initialRoleGrantsJson`，或在 `existingSecret` 中提供
`auth-initial-role-grants-json`。这些值仅用于数据库首次初始化；之后在后台
“系统配置”中管理。规则格式及管理入口见 [登录开关与外部账号首次授权](../../docs/923-auth-system-settings.md)。
### 可选的公开 GitHub 采集服务
Go 采集 Pod 默认不部署。需要启用时，在受保护的 values 文件中配置：
```yaml
githubSync:
enabled: true
# S3/MinIO 模式按暂存前缀容量填写正数字节预算
stagingCapacityLimitBytes: 1073741824
# S3/MinIO 模式填写预签名 PUT 的内部主机及端口，例如 minio:9000
allowedS3Hosts: ""
secrets:
githubSyncServiceToken: "<至少32字符的随机服务密钥>"
```
Chart 把同一服务密钥注入 Java 与 Go。使用 `existingSecret` 时，该 Secret 必须包含
`github-sync-service-token` 键，不再在 values 中填写明文。Go 固定单副本，使用
768 MiB 内存型临时卷；默认单仓库预算 512 MiB，预留 128 MiB，容器内存上限
1.5 GiB。需要 GitHub 出站代理时设置 `githubSync.httpProxy`、
`githubSync.httpsProxy` 和 `githubSync.noProxy`；Go 到 Java 的内部请求始终直连。
运行字段和当前实现边界见 [Go 服务说明](../../github-sync-go/README.md)。
系统级公开 GitHub Token 由 Java 加密保存。需同时向 Java 提供
`github-sync-credential-keys`（`v1=<32 字节随机密钥的 Base64>`，可逗号分隔保留旧版本）
和 `github-sync-active-credential-key-version` 两个 Secret 键。使用 `existingSecret`
时直接写入该 Secret；Chart 自建 Secret 时对应 `secrets.githubSyncCredentialKeys` 与
`secrets.githubSyncActiveCredentialKeyVersion`，不要将真实值提交到版本控制。不配置时
Go 使用匿名公开 API 额度，后台创建凭证会明确拒绝。
> **Ingress values 迁移：** 当前版本只支持结构化的 `ingress.hosts[]` 和
> `ingress.tls[]`。旧的 `ingress.host`、`ingress.tls.enabled` 与

9
charts/skillhub/templates/_helpers.tpl
View file

@ -70,6 +70,15 @@ app.kubernetes.io/component: scanner
app.kubernetes.io/component: scanner
{{- end }}
{{- define "skillhub.githubSync.labels" -}}
{{ include "skillhub.labels" . }}
app.kubernetes.io/component: github-sync
{{- end }}
{{- define "skillhub.githubSync.selectorLabels" -}}
{{ include "skillhub.selectorLabels" . }}
app.kubernetes.io/component: github-sync
{{- end }}
{{- /* Bitnami PostgreSQL subchart 完整名称 */}}
{{- define "skillhub.postgresql.fullname" -}}
{{- if .Values.postgresql.fullnameOverride -}}

121
charts/skillhub/templates/github-sync-deployment.yaml Normal file
View file

@ -0,0 +1,121 @@
{{- if .Values.githubSync.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "skillhub.fullname" . }}-github-sync
labels:
{{- include "skillhub.githubSync.labels" . | nindent 4 }}
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "skillhub.githubSync.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "skillhub.githubSync.selectorLabels" . | nindent 8 }}
annotations:
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
{{- range $key, $val := .Values.githubSync.podAnnotations }}
{{ $key }}: {{ $val | quote }}
{{- end }}
spec:
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
{{- with .Values.githubSync.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: github-sync
image: {{ .Values.githubSync.image.registry | default .Values.images.registry }}/skillhub-github-sync:{{ .Values.githubSync.image.tag | default .Values.images.tag | default (printf "v%s" .Chart.AppVersion) }}
imagePullPolicy: {{ .Values.images.pullPolicy }}
ports:
- name: health
containerPort: 8090
env:
- name: SKILLHUB_GITHUB_SYNC_JAVA_URL
value: {{ printf "http://%s-server:%v/internal/github-sync/v1" (include "skillhub.fullname" .) .Values.server.service.port | quote }}
- name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "skillhub.secretName" . }}
key: github-sync-service-token
- name: SKILLHUB_GITHUB_SYNC_TEMP_ROOT
value: /var/lib/skillhub-github-sync
- name: SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS
value: {{ .Values.githubSync.discoveryWorkers | quote }}
- name: SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS
value: {{ .Values.githubSync.repositoryWorkers | quote }}
- name: SKILLHUB_GITHUB_SYNC_POLL_INTERVAL
value: {{ .Values.githubSync.pollInterval | quote }}
- name: SKILLHUB_GITHUB_SYNC_MAX_PAGES_PER_LEASE
value: {{ .Values.githubSync.maxPagesPerLease | quote }}
- name: SKILLHUB_GITHUB_SYNC_MAX_TREE_REQUESTS
value: {{ .Values.githubSync.maxTreeRequests | quote }}
- name: SKILLHUB_GITHUB_SYNC_MAX_TREE_ENTRIES
value: {{ .Values.githubSync.maxTreeEntries | quote }}
- name: SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES
value: {{ .Values.githubSync.maxWorkspaceBytes | quote }}
- name: SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES
value: {{ .Values.githubSync.totalWorkspaceBytes | quote }}
- name: SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES
value: {{ .Values.githubSync.minFreeBytes | quote }}
- name: SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS
value: {{ .Values.githubSync.allowedS3Hosts | quote }}
{{- if .Values.githubSync.httpProxy }}
- name: HTTP_PROXY
value: {{ .Values.githubSync.httpProxy | quote }}
{{- end }}
{{- if .Values.githubSync.httpsProxy }}
- name: HTTPS_PROXY
value: {{ .Values.githubSync.httpsProxy | quote }}
{{- end }}
{{- if .Values.githubSync.noProxy }}
- name: NO_PROXY
value: {{ .Values.githubSync.noProxy | quote }}
{{- end }}
volumeMounts:
- name: workspace
mountPath: /var/lib/skillhub-github-sync
resources:
{{- toYaml .Values.githubSync.resources | nindent 12 }}
startupProbe:
httpGet:
path: /health/live
port: health
periodSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
path: /health/ready
port: health
periodSeconds: 10
livenessProbe:
httpGet:
path: /health/live
port: health
periodSeconds: 15
volumes:
- name: workspace
emptyDir:
medium: Memory
sizeLimit: {{ .Values.githubSync.tmpfsSizeLimit | quote }}
{{- with .Values.githubSync.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.githubSync.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.githubSync.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

8
charts/skillhub/templates/secret.yaml
View file

@ -55,6 +55,14 @@ stringData:
{{- end }}
skillhub-download-anon-cookie-secret: {{ $downloadSecret | quote }}
{{- if .Values.githubSync.enabled }}
github-sync-service-token: {{ .Values.secrets.githubSyncServiceToken | quote }}
{{- end }}
{{- if .Values.secrets.githubSyncCredentialKeys }}
github-sync-credential-keys: {{ .Values.secrets.githubSyncCredentialKeys | quote }}
github-sync-active-credential-key-version: {{ .Values.secrets.githubSyncActiveCredentialKeyVersion | quote }}
{{- end }}
# OAuth2 GitHub (optional)
{{- if .Values.secrets.oauth2GithubClientId }}
oauth2-github-client-id: {{ .Values.secrets.oauth2GithubClientId | quote }}

21
charts/skillhub/templates/server-deployment.yaml
View file

@ -79,6 +79,27 @@ spec:
value: {{ $profiles | quote }}
- name: SKILLHUB_SUITE_REVIEW_WRITES_ENABLED
value: {{ .Values.server.suiteReviewWritesEnabled | quote }}
- name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES
value: {{ .Values.githubSync.stagingCapacityLimitBytes | quote }}
{{- if .Values.githubSync.enabled }}
- name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS
valueFrom:
secretKeyRef:
name: {{ include "skillhub.secretName" . }}
key: github-sync-service-token
{{- end }}
- name: SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS
valueFrom:
secretKeyRef:
name: {{ include "skillhub.secretName" . }}
key: github-sync-credential-keys
optional: true
- name: SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION
valueFrom:
secretKeyRef:
name: {{ include "skillhub.secretName" . }}
key: github-sync-active-credential-key-version
optional: true
# Database
- name: SPRING_DATASOURCE_URL

18
charts/skillhub/templates/validate.yaml
View file

@ -121,6 +121,24 @@
{{- fail "s3.publicEndpoint must be an absolute HTTP(S) URL" -}}
{{- end -}}
{{- if .Values.githubSync.enabled -}}
{{- if not .Values.server.enabled -}}
{{- fail "githubSync.enabled requires server.enabled" -}}
{{- end -}}
{{- if and (not .Values.existingSecret) (lt (len .Values.secrets.githubSyncServiceToken) 32) -}}
{{- fail "secrets.githubSyncServiceToken must contain at least 32 characters, or use existingSecret with github-sync-service-token" -}}
{{- end -}}
{{- if and .Values.s3.enabled (not .Values.githubSync.allowedS3Hosts) -}}
{{- fail "githubSync.allowedS3Hosts is required when s3.enabled=true" -}}
{{- end -}}
{{- if or (lt (.Values.githubSync.discoveryWorkers | int) 1) (lt (.Values.githubSync.repositoryWorkers | int) 1) -}}
{{- fail "githubSync worker counts must be positive" -}}
{{- end -}}
{{- if lt (.Values.githubSync.totalWorkspaceBytes | int64) (mul (.Values.githubSync.repositoryWorkers | int64) (.Values.githubSync.maxWorkspaceBytes | int64) | int64) -}}
{{- fail "githubSync.totalWorkspaceBytes must cover all repository workers" -}}
{{- end -}}
{{- end -}}
{{- if not .Values.secrets.allowAutoGenerated -}}
{{- if not .Values.existingSecret -}}
{{- if not (or .Values.secrets.bootstrapAdminPassword .Values.bootstrapAdmin.password) -}}

17
charts/skillhub/tests/configuration-contracts.sh
View file

@ -75,6 +75,23 @@ assert_rejected web-dns-empty-label --set-string web.clusterDomain=cluster..loca
assert_rejected web-dns-invalid-label --set-string web.clusterDomain=-cluster.local
render verify "$CHART_DIR" >"$TMP_DIR/default.yaml"
if grep -Fq 'name: verify-skillhub-github-sync' "$TMP_DIR/default.yaml"; then
fail "GitHub sync deployment must stay disabled by default"
fi
grep -A1 -F 'name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES' "$TMP_DIR/default.yaml" \
| grep -Fq 'value: "0"' \
|| fail "server must receive the default GitHub sync staging capacity budget"
render github-sync "$CHART_DIR" \
--set githubSync.enabled=true \
--set githubSync.stagingCapacityLimitBytes=1073741824 \
--set-string secrets.githubSyncServiceToken=github-sync-service-token-at-least-32-characters \
>"$TMP_DIR/github-sync.yaml"
grep -Fq 'name: github-sync-skillhub-github-sync' "$TMP_DIR/github-sync.yaml" \
|| fail "enabled GitHub sync must render its deployment"
grep -A1 -F 'name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES' "$TMP_DIR/github-sync.yaml" \
| grep -Fq 'value: "1073741824"' \
|| fail "server must receive the configured GitHub sync staging capacity budget"
assert_rejected github-sync-negative-capacity --set githubSync.stagingCapacityLimitBytes=-1
grep -Fq 'name: POSTGRESQL_MAX_CONNECTIONS' "$TMP_DIR/default.yaml"
grep -Fq 'value: "verify-postgresql"' "$TMP_DIR/default.yaml"
grep -Fq 'value: "verify-redis-master"' "$TMP_DIR/default.yaml"

37
charts/skillhub/values.schema.json
View file

@ -202,7 +202,10 @@
"oauth2DingtalkClientSecret": { "type": "string" },
"scannerLlmApiKey": { "type": "string" },
"scannerLlmBaseUrl": { "type": "string" },
"scannerLlmModel": { "type": "string" }
"scannerLlmModel": { "type": "string" },
"githubSyncServiceToken": { "type": "string" },
"githubSyncCredentialKeys": { "type": "string" },
"githubSyncActiveCredentialKeyVersion": { "type": "string" }
}
},
"postgresql": {
@ -302,10 +305,40 @@
}
},
"server": { "$ref": "#/definitions/serverComponent" },
"githubSync": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "image", "discoveryWorkers", "repositoryWorkers", "pollInterval", "maxPagesPerLease", "maxTreeRequests", "maxTreeEntries", "maxWorkspaceBytes", "totalWorkspaceBytes", "minFreeBytes", "tmpfsSizeLimit", "allowedS3Hosts", "httpProxy", "httpsProxy", "noProxy", "resources", "imagePullSecrets", "podAnnotations", "nodeSelector", "tolerations", "affinity"],
"properties": {
"enabled": { "type": "boolean" },
"image": { "$ref": "#/definitions/image" },
"stagingCapacityLimitBytes": { "type": "integer", "minimum": 0 },
"discoveryWorkers": { "type": "integer", "minimum": 1, "maximum": 32 },
"repositoryWorkers": { "type": "integer", "minimum": 1, "maximum": 32 },
"pollInterval": { "type": "string", "pattern": "^[1-9][0-9]*(s|m|h)$" },
"maxPagesPerLease": { "type": "integer", "minimum": 1 },
"maxTreeRequests": { "type": "integer", "minimum": 1 },
"maxTreeEntries": { "type": "integer", "minimum": 1 },
"maxWorkspaceBytes": { "type": "integer", "minimum": 1 },
"totalWorkspaceBytes": { "type": "integer", "minimum": 1 },
"minFreeBytes": { "type": "integer", "minimum": 0 },
"tmpfsSizeLimit": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" },
"allowedS3Hosts": { "type": "string" },
"httpProxy": { "type": "string" },
"httpsProxy": { "type": "string" },
"noProxy": { "type": "string" },
"resources": { "$ref": "#/definitions/resources" },
"imagePullSecrets": { "type": "array", "items": { "type": "object" } },
"podAnnotations": { "$ref": "#/definitions/stringMap" },
"nodeSelector": { "$ref": "#/definitions/stringMap" },
"tolerations": { "type": "array", "items": { "type": "object" } },
"affinity": { "type": "object" }
}
},
"web": { "$ref": "#/definitions/webComponent" },
"scanner": { "$ref": "#/definitions/scannerComponent" }
},
"required": ["images", "auth", "builtinSkills", "ingress", "s3", "session", "bootstrapAdmin", "springProfilesActive", "secrets", "postgresql", "externalDatabase", "redis", "externalRedis", "server", "web", "scanner"],
"required": ["images", "auth", "builtinSkills", "ingress", "s3", "session", "bootstrapAdmin", "springProfilesActive", "secrets", "postgresql", "externalDatabase", "redis", "externalRedis", "server", "githubSync", "web", "scanner"],
"definitions": {
"dnsLabel": {
"type": "string",

42
charts/skillhub/values.yaml
View file

@ -118,6 +118,10 @@ secrets:
scannerLlmApiKey: ""
scannerLlmBaseUrl: ""
scannerLlmModel: ""
githubSyncServiceToken: ""
# 格式：v1=<32 字节密钥的 Base64>。轮换时保留旧版本并追加新版本。
githubSyncCredentialKeys: ""
githubSyncActiveCredentialKeyVersion: ""
# ============================================================================
# PostgreSQL 配置（Bitnami）
@ -386,6 +390,44 @@ server:
enabled: false
minAvailable: 1
# 公开 GitHub Skill 采集服务；默认不创建任务执行 Pod。
githubSync:
enabled: false
# S3/MinIO 暂存前缀容量预算（字节）；启用同步前按部署存储能力设置正值。
stagingCapacityLimitBytes: 0
image:
registry: ""
tag: ""
# 与 Java 的 SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS 使用同一服务密钥。
# 在 secrets.githubSyncServiceToken 或 existingSecret 中配置。
discoveryWorkers: 1
repositoryWorkers: 1
pollInterval: 5s
maxPagesPerLease: 20
maxTreeRequests: 100
maxTreeEntries: 100000
maxWorkspaceBytes: 536870912
totalWorkspaceBytes: 536870912
minFreeBytes: 134217728
tmpfsSizeLimit: 768Mi
# S3 模式需填写上传票据指向的内部存储主机及端口，例如 minio:9000。
allowedS3Hosts: ""
httpProxy: ""
httpsProxy: ""
noProxy: ""
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 1000m
memory: 1536Mi
imagePullSecrets: []
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ============================================================================
# Web 配置
# ============================================================================

49
compose.release.yml
View file

@ -46,6 +46,19 @@ services:
timeout: 5s
retries: 10
# A named volume masks the app-owned directory created in server/Dockerfile.
storage-permissions:
image: ${SKILLHUB_SERVER_IMAGE:-ghcr.io/iflytek/skillhub-server}:${SKILLHUB_VERSION:-latest}
user: "0:0"
entrypoint: ["chown", "100:101", "/var/lib/skillhub/storage"]
restart: "no"
network_mode: none
read_only: true
cap_drop: [ALL]
cap_add: [CHOWN]
volumes:
- skillhub_storage:/var/lib/skillhub/storage
server:
image: ${SKILLHUB_SERVER_IMAGE:-ghcr.io/iflytek/skillhub-server}:${SKILLHUB_VERSION:-latest}
restart: unless-stopped
@ -91,6 +104,10 @@ services:
SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING: ${SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING:-false}
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: ${SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET:-false}
SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY: ${SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY:-PT10M}
SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS:-}
SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS: ${SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS:-}
SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION: ${SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION:-}
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES:-0}
SKILLHUB_SECURITY_SCANNER_ENABLED: ${SKILLHUB_SECURITY_SCANNER_ENABLED:-true}
SKILLHUB_SECURITY_SCANNER_URL: http://skill-scanner:8000
SKILLHUB_SECURITY_SCANNER_MODE: upload
@ -149,6 +166,8 @@ services:
volumes:
- skillhub_storage:/var/lib/skillhub/storage
depends_on:
storage-permissions:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
@ -186,6 +205,36 @@ services:
retries: 12
start_period: 10s
github-sync:
profiles: [github-sync]
image: ${SKILLHUB_GITHUB_SYNC_IMAGE:-ghcr.io/iflytek/skillhub-github-sync}:${SKILLHUB_VERSION:-latest}
restart: unless-stopped
environment:
SKILLHUB_GITHUB_SYNC_JAVA_URL: http://server:8080/internal/github-sync/v1
SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-}
SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS: ${SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS:-1}
SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS: ${SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS:-1}
SKILLHUB_GITHUB_SYNC_POLL_INTERVAL: ${SKILLHUB_GITHUB_SYNC_POLL_INTERVAL:-5s}
SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912}
SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912}
SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES: ${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728}
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS: ${SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS:-}
HTTP_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTP_PROXY:-}
HTTPS_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTPS_PROXY:-}
NO_PROXY: ${SKILLHUB_GITHUB_SYNC_NO_PROXY:-}
tmpfs:
- /var/lib/skillhub-github-sync:size=${SKILLHUB_GITHUB_SYNC_TMPFS_BYTES:-805306368},mode=1777
mem_limit: ${SKILLHUB_GITHUB_SYNC_MEMORY_LIMIT_BYTES:-1610612736}
depends_on:
server:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-Y", "off", "-qO-", "http://127.0.0.1:8090/health/ready"]
interval: 15s
timeout: 5s
retries: 5
start_period: 30s
volumes:
postgres_data:
redis_data:

14
deploy/k8s/README.md
View file

@ -62,6 +62,9 @@ cp secret.yaml.example secret.yaml
| redis-password | Redis 数据节点密码 | 否 |
| redis-sentinel-password | Redis Sentinel 独立密码 | 否 |
| bootstrap-admin-password | 管理员密码 | 是 |
| github-sync-service-token | 可选 Go 采集服务与 Java 之间的共享服务密钥，启用时至少 32 字符 | 仅启用采集时必填 |
| github-sync-credential-keys | 系统级 GitHub Token 加密密钥环，格式 `v1=<32 字节随机密钥的 Base64>` | 后台配置 Token 时必填 |
| github-sync-active-credential-key-version | 写入新密文时使用的密钥版本，例如 `v1` | 配置加密密钥环时必填 |
| oauth2-github-client-id | GitHub OAuth ID | 否 |
| oauth2-github-client-secret | GitHub OAuth 密钥 | 否 |
| skill-scanner-llm-api-key | LLM API 密钥 | 否 |
@ -166,6 +169,17 @@ kubectl apply -k overlays/with-infra/ # 或 overlays/external/
## 部署架构
### 可选的公开 GitHub 采集服务
`base/github-sync-deployment.yaml` 不在默认 Kustomize 清单中。先在 `skillhub-secret` 配置 `github-sync-service-token`，并更新 Java Pod 使其读取该 Secret；再按实际镜像版本与存储网络配置此 Deployment，然后单独运行：
```bash
kubectl apply -n skillhub -f deploy/k8s/base/github-sync-deployment.yaml
kubectl rollout status -n skillhub deployment/skillhub-github-sync
```
Go Pod 通过 `skillhub-server:8080` 领取任务。S3/MinIO 模式还要把 `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` 设为预签名 PUT 的内部主机及端口；需要 GitHub 出站代理时，在可选 Deployment 中填写 `HTTP_PROXY`、`HTTPS_PROXY`、`NO_PROXY`，默认空值不启用。代理地址如包含凭证，应改由 Kubernetes Secret 注入，勿写入清单或 ConfigMap。Go 对内部 Java 和暂存对象上传使用直连客户端。工作目录为 768 MiB 内存型 `emptyDir`，并按 512 MiB 单仓库预算和 128 MiB 预留空间配置。完整变量说明见 [Go 服务 README](../../github-sync-go/README.md)。
```
┌─────────────────────────────────────────────────────────────┐
│ skillhub namespace │

26
deploy/k8s/base/backend-deployment.yaml
View file

@ -14,6 +14,9 @@ spec:
labels:
app.kubernetes.io/name: skillhub-server
spec:
securityContext:
fsGroup: 101
fsGroupChangePolicy: OnRootMismatch
containers:
- name: server
image: ghcr.io/iflytek/skillhub-server:edge
@ -24,6 +27,29 @@ spec:
env:
- name: SPRING_PROFILES_ACTIVE
value: docker
- name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS
valueFrom:
secretKeyRef:
name: skillhub-secret
key: github-sync-service-token
optional: true
- name: SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS
valueFrom:
secretKeyRef:
name: skillhub-secret
key: github-sync-credential-keys
optional: true
- name: SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION
valueFrom:
secretKeyRef:
name: skillhub-secret
key: github-sync-active-credential-key-version
optional: true
- name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES
valueFrom:
configMapKeyRef:
name: skillhub-config
key: github-sync-staging-capacity-limit-bytes
# Database
- name: SPRING_DATASOURCE_URL

2
deploy/k8s/base/configmap.yaml
View file

@ -30,6 +30,8 @@ data:
# 存储配置
# local: 本地存储（默认）, s3: S3/OSS 对象存储
skillhub-storage-provider: local
# S3/MinIO 模式启用 GitHub 同步前，按隔离暂存区容量预算填写正数字节数。
github-sync-staging-capacity-limit-bytes: "0"
# 技能扫描器配置
skill-scanner-enabled: "true"

94
deploy/k8s/base/github-sync-deployment.yaml Normal file
View file

@ -0,0 +1,94 @@
# 可选部署：先在 skillhub-secret 配置 github-sync-service-token，再单独 apply 此文件。
apiVersion: apps/v1
kind: Deployment
metadata:
name: skillhub-github-sync
labels:
app.kubernetes.io/name: skillhub-github-sync
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: skillhub-github-sync
template:
metadata:
labels:
app.kubernetes.io/name: skillhub-github-sync
spec:
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
containers:
- name: github-sync
image: ghcr.io/iflytek/skillhub-github-sync:edge
imagePullPolicy: IfNotPresent
ports:
- name: health
containerPort: 8090
env:
- name: SKILLHUB_GITHUB_SYNC_JAVA_URL
value: http://skillhub-server:8080/internal/github-sync/v1
- name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: skillhub-secret
key: github-sync-service-token
- name: SKILLHUB_GITHUB_SYNC_TEMP_ROOT
value: /var/lib/skillhub-github-sync
- name: SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS
value: "1"
- name: SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS
value: "1"
- name: SKILLHUB_GITHUB_SYNC_POLL_INTERVAL
value: 5s
- name: SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES
value: "536870912"
- name: SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES
value: "536870912"
- name: SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES
value: "134217728"
# S3/MinIO 模式必须填入内部上传主机及端口，例如 minio:9000。
- name: SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS
value: ""
# 按部署网络填写 GitHub 出站代理；Java 和对象存储由 Go 客户端直连。
- name: HTTP_PROXY
value: ""
- name: HTTPS_PROXY
value: ""
- name: NO_PROXY
value: ""
volumeMounts:
- name: workspace
mountPath: /var/lib/skillhub-github-sync
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1536Mi
startupProbe:
httpGet:
path: /health/live
port: health
periodSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
path: /health/ready
port: health
periodSeconds: 10
livenessProbe:
httpGet:
path: /health/live
port: health
periodSeconds: 15
volumes:
- name: workspace
emptyDir:
medium: Memory
sizeLimit: 768Mi

6
deploy/k8s/base/secret.yaml.example
View file

@ -23,6 +23,12 @@ stringData:
# Bootstrap 管理员密码（敏感）
bootstrap-admin-password: ChangeMe!2026
# 公开 GitHub 采集服务身份；启用独立 Go Deployment 时填写至少 32 字符的随机值
github-sync-service-token: ""
# AES-256-GCM 密钥环：v1=<32 字节随机密钥的 Base64>，可逗号分隔保留旧版本。
github-sync-credential-keys: ""
github-sync-active-credential-key-version: ""
# GitHub OAuth（可选，用于 GitHub 登录）
oauth2-github-client-id: ""
oauth2-github-client-secret: ""

33
docker-compose.staging.yml
View file

@ -34,6 +34,10 @@ services:
SKILLHUB_STORAGE_S3_REGION: us-east-1
SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true"
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "true"
SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS:-}
SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS: ${SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS:-}
SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION: ${SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION:-}
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES:-0}
BOOTSTRAP_ADMIN_ENABLED: "true"
BOOTSTRAP_ADMIN_USER_ID: staging-admin
BOOTSTRAP_ADMIN_USERNAME: admin
@ -84,3 +88,32 @@ services:
timeout: 5s
retries: 6
start_period: 10s
github-sync:
profiles: [github-sync]
build: ./github-sync-go
image: skillhub-github-sync:staging
environment:
SKILLHUB_GITHUB_SYNC_JAVA_URL: http://server:8080/internal/github-sync/v1
SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-}
SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912}
SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912}
SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES: ${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728}
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS: minio:9000
HTTP_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTP_PROXY:-}
HTTPS_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTPS_PROXY:-}
NO_PROXY: ${SKILLHUB_GITHUB_SYNC_NO_PROXY:-}
tmpfs:
- /var/lib/skillhub-github-sync:size=${SKILLHUB_GITHUB_SYNC_TMPFS_BYTES:-805306368},mode=1777
mem_limit: ${SKILLHUB_GITHUB_SYNC_MEMORY_LIMIT_BYTES:-1610612736}
depends_on:
server:
condition: service_healthy
minio:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-Y", "off", "-qO-", "http://127.0.0.1:8090/health/ready"]
interval: 15s
timeout: 5s
retries: 5
start_period: 30s

5
docs/01-system-architecture.md
View file

@ -24,7 +24,8 @@ server/
├── skillhub-auth # OAuth2 认证 + RBAC + 授权判定
├── skillhub-search # 搜索 SPI + PostgreSQL 全文实现
├── skillhub-storage # 对象存储抽象 + LocalFile/S3 双实现
└── skillhub-infra # JPA、通用工具、配置基础
├── skillhub-infra # JPA、通用工具、配置基础
└── skillhub-github-sync # GitHub 来源与同步控制面模型；仍在 Java 应用进程内
```
## 4. 模块依赖方向（依赖倒置，禁止领域层依赖基础设施）
@ -35,6 +36,7 @@ infra → domain # infra 实现 domain 定义的 Repository 接口
auth → domain # auth 引用 UserAccount 等领域实体
search → domain # search 引用 SkillSearchDocument 等领域模型
storage → (独立抽象) # 纯 SPI，不依赖 domain
app, infra → github-sync → domain, storage
```
核心原则：
@ -115,6 +117,7 @@ web/
skillhub/
├── server/ # Maven 多模块 Java 后端
│ └── Dockerfile # 后端多阶段构建
├── github-sync-go/ # 独立 Go 采集服务（实施中）
├── web/ # React 前端
│ ├── Dockerfile # 前端多阶段构建
│ ├── nginx.conf.template # Nginx 运行时模板

6
docs/02-domain-model.md
View file

@ -16,7 +16,7 @@
| id | bigint | 主键 |
| slug | varchar(64) | URL 友好标识 |
| display_name | varchar(128) | 展示名 |
| type | enum | `GLOBAL` / `TEAM` |
| type | enum | `GLOBAL` / `CATALOG` / `TEAM` |
| description | text | 描述 |
| avatar_url | varchar(512) | 头像 |
| status | enum | `ACTIVE` / `FROZEN` / `ARCHIVED` |
@ -25,11 +25,13 @@
| updated_at | datetime | |
- `GLOBAL` 类型全局唯一（只有一个 `@global`），由平台管理员管理
- `CATALOG` 为系统管理的来源目录；首个 `@github` 通过迁移创建，普通用户不能修改或删除
- `@github` 迁移同时创建不可登录的系统 actor `github-sync-publisher`；若 namespace slug 或 actor ID 已被占用，迁移明确失败而不接管已有记录
- `TEAM` 类型对应部门/团队，可创建多个
- 技能完整寻址：`@{namespace_slug}/{skill_slug}`
- slug 唯一约束：`slug`
- slug 格式校验：`[a-z0-9]([a-z0-9-]*[a-z0-9])?`，长度 2-64，且不得包含连续两个以上的连字符 `--`（为兼容层坐标映射保留）
- slug 保留词列表（用户创建 namespace 时不可使用）：`admin`, `api`, `dashboard`, `search`, `auth`, `me`, `global`, `system`, `static`, `assets`, `health`
- slug 保留词列表（用户创建 namespace 时不可使用）：`admin`, `api`, `dashboard`, `search`, `auth`, `me`, `global`, `github`, `system`, `static`, `assets`, `health`；`github` 仅在 namespace 创建入口保留，不禁止同名 Skill
- 系统内置 namespace（`@global`）在数据库初始化时由 Flyway 脚本预置，绕过 slug 校验规则。保留词校验仅作用于用户创建 namespace 的接口
- 状态语义：
- `ACTIVE`：正常使用

36
docs/09-deployment.md
View file

@ -93,6 +93,15 @@ make dev-all-down
make dev-all-reset
```
可选的 GitHub 公开 Skill 采集服务在宿主机前台运行。先为 Java 和 Go 设置同一枚本地服务密钥，再启动 `make dev-all` 与 `make dev-github-sync`；后者按 `Ctrl-C` 停止，不影响 Java 和其他开发依赖。默认使用本地存储，Go 临时目录为 `.dev/github-sync-tmp`。如需使用 S3/MinIO，应按[同步运行手册](github-sync-operations.md)配置两侧的存储地址和允许上传主机。
```bash
export SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS="<本机生成的至少 32 字符密钥>"
export SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN="$SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS"
make dev-all
make dev-github-sync
```
## 5 单机交付环境
### 5.1 启动
@ -103,6 +112,22 @@ make validate-release-config
docker compose --env-file .env.release -f compose.release.yml up -d
```
公开 GitHub Skill 采集服务使用独立的 `github-sync` profile，默认不启动。设置同一条至少
32 字符的服务密钥到 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN` 与 Java 认可的
`SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 后，才能显式启动；S3/MinIO 模式还需设置
`SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` 为内部上传主机及端口。配置和当前实现边界见
[Go 服务说明](../github-sync-go/README.md)，启停、后台操作与排障见
[管理员运行手册](github-sync-operations.md)。
使用 S3/MinIO 时，给 Java 服务设置正值的 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES`，
表示 GitHub 同步隔离暂存前缀可使用的字节预算。未设置且暂存区已有对象时容量告警为 `1`；
扫描前缀不完整或读取失败时同样告警。预算应结合对象存储容量与试运行数据确定，
具体指标和扫描边界见 [内部协议说明](api/github-sync-internal.md#暂存容量监控)。
如需在后台配置公开读取 GitHub Token，Java 服务还需设置
`SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS`（例如 `v1=<32 字节随机密钥的 Base64>`）和
`SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION`（例如 `v1`）。两项缺省时保留匿名读取能力，后台不能保存 Token。轮换时同时保留旧、新版本密钥，待旧密文被替换后再移除旧版本；不要把密钥或 Token 写入 Compose 文件、仓库或命令历史。管理接口和状态见 [GitHub 同步管理接口](api/github-sync-admin.md)。
默认访问地址：
- Web UI: `SKILLHUB_PUBLIC_BASE_URL`
@ -214,13 +239,15 @@ docker compose --env-file .env.release -f compose.release.yml up -d --force-recr
1. 检出代码
2. 登录 GHCR
3. 分别构建 `server/Dockerfile` 与 `web/Dockerfile`
3. 分别构建 `server`、`web`、`scanner` 与 `github-sync-go` 的 Dockerfile
4. 推送镜像：
- `ghcr.io/iflytek/skillhub-server`
- `ghcr.io/iflytek/skillhub-web`
- `ghcr.io/iflytek/skillhub-scanner`
- `ghcr.io/iflytek/skillhub-github-sync`
5. 写入 `edge` / `vX.Y.Z` / `latest` / `sha-*` 标签
6. 同时发布多架构 manifest：`server` / `web` 覆盖 `linux/amd64`、`linux/arm64` 与
`linux/riscv64`，`scanner` 暂保持 `linux/amd64` 与 `linux/arm64`
`linux/riscv64`，`scanner` 与 `github-sync` 覆盖 `linux/amd64`、`linux/arm64`
## 7 配置管理
@ -743,3 +770,8 @@ Flyway 仍是唯一 schema 变更入口：
- 路径：`server/skillhub-app/src/main/resources/db/migration/`
- 命名：`V{version}__{description}.sql`
- 启动策略：应用容器启动时自动执行迁移
GitHub 同步的 V76 迁移会创建系统管理的 `@github` 目录和 `github-sync-publisher` 系统账户。若升级前已有同名 namespace 或账户，迁移会明确失败并回滚，不会接管或改写原数据。部署前应检查这两个标识是否被占用；遇到冲突时先处理现有数据的归属，再重新启动迁移。
本次 GitHub 同步从 main 的 V75 schema 继续追加 V76–V119；main 已发行 V1–V75 不改名、不改 checksum。升级前备份数据库和对象存储，先在副本验证迁移与业务。早期未发布开发分支使用过 V66–V109 的不同编号，它们与 main 历史冲突，不能直接套用本次迁移，也不应通过 Flyway repair 掩盖 checksum/版本冲突。一次性开发库应使用新的隔离库重新建立；需要保留早期分支数据时须另行设计并验收数据迁移。回退镜像不等于数据库降级。

193
docs/api/github-deep-discovery.md Normal file
View file

@ -0,0 +1,193 @@
# 官方 GitHub 深度探索接口增量（D2）
> V119统一管理补充：平台全部任务现在还要求 `platform-pause/v1`。当前Go声明四项官方能力加平台暂停能力，缺少后者的旧实例不能领取 `@github` 任务，也不显示为兼容实例。TEAM规则不变。管理员入口合并至“技能同步 → 采集配置”，详见[统一管理接口](github-sync-admin.md#统一技能同步管理)。
管理端机器可读合同见 [管理 OpenAPI](github-deep-discovery.openapi.yaml)，Java–Go见 [内部 OpenAPI](github-sync-internal.openapi.yaml)。本文固定当前协议；验证入口和限制见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。
正式行为以 [需求](../../openspec/changes/add-github-skill-sync/deep-discovery.md)、[设计](../../openspec/changes/add-github-skill-sync/deep-discovery-design.md) 为准。本文固定字段与交互合同，物理表、索引与迁移见同一 OpenSpec 下的[持久化设计](../../openspec/changes/add-github-skill-sync/deep-discovery-storage.md)。
## 1. 已核对的实现基线
| 代码 | 当前事实 | D2 增量 |
| --- | --- | --- |
| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncController.java` | `/api/v1/admin/github-sync`，`SUPER_ADMIN`，已有来源/运行/水位/周期接口 | 独立AdminGithubDeepDiscoveryController已实现配置GET/PUT与六个读模型接口，并完成受控HTTP验证 |
| `server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ApiResponse.java`、`PageResponse.java` | `code/msg/data/timestamp/requestId`，成功 code=0；分页 `items/total/page/size` | 保持包装，分页增加 `asOf` |
| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalController.java` | claim已扩展可选capabilities并按持久能力过滤；官方响应与legacy互斥；官方批次与仓库控制已接线 | 增加能力过滤、独立官方快照和出处；202不变 |
| `github-sync-go/internal/controlplane/client.go` | Go生产工作池声明四项官方能力，执行官方客户端与循环 | 启用前全部Java/Go升级；实例能力心跳已持久化并验证 |
| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncWorkerHeartbeatController.java`、`repository/GithubSyncWorkerPresenceRepository.java` | 保留legacy全局联系时间，同时保存实例ID、协议与能力 | 官方在线判定使用实例能力和最近联系；心跳不证明业务正在推进 |
Web SDK 已从当前 Java 运行时 `/v3/api-docs` 生成。管理页面使用生成类型；Go wire 类型与内部合同保持同步。受控集成验证不替代真实 GitHub 搜索效果对照，完整边界见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。
## 2. 管理端路径与授权
共同前缀：`/api/v1/admin/github-sync/deep-discovery`。所有读写仅 `SUPER_ADMIN`，范围固定为公共 `@github`，不接受客户端指定 namespace。写入沿用会话、CSRF 与审计；记录用户字符串 ID、配置版本及规范化差异。响应 `Cache-Control: no-store`，不包含凭证、租约、上传票据、原始查询响应或异常正文。
| 方法 | 相对路径 | data及实施状态 |
| --- | --- | --- |
| GET | `/configuration` | 当前启停、预算、排除项及version；已实现，隔离HTTP验证通过 |
| PUT | `/configuration` | 整体保存后的配置；请求含expectedVersion；已实现，隔离HTTP验证通过 |
| GET | `/summary?from=...&until=...` | 近期成果、当前积压、联系状态、等待原因、最多5个近期runId |
| GET | `/progress` | 当前策略各渠道的持久分片计数、最近业务进度和额度观测 |
| GET | `/repositories` | 仓库摘要分页 |
| GET | `/repositories/{repositoryId}` | 按需详情及关联sourceId/runId |
| GET | `/repositories/{repositoryId}/provenance` | 出处分页 |
| GET | `/repositories/{repositoryId}/paths` | 最近逐路径结果及Skill/version/review ID分页 |
没有“执行一次”接口。既有 `/runs/{runId}` 及其 `/tasks`、`/paths` 继续承担运行详情；本次不另造运行对象。历史清理后的关联runId可失效，详情返回404时页面提示历史已清理。导航使用现有权限校验，不能拼造未经实现的前端路由。
错误沿用 `ApiResponse` 数值 code，HTTP 400 为参数不合法，401未登录，403权限或CSRF不满足，404目标不在可见官方范围，409配置版本冲突。当前配置接口沿用GlobalExceptionHandler，以HTTP状态码作为数值code；配置冲突使用DomainConflictException及本地化消息。前端不得匹配翻译后的msg。
## 3. 配置并发、排除与预算
V108迁移持久初始化 `version=1, enabled=false, budgetProfile=BALANCED, exclusions=[], strategyVersion=null, updatedBy=null`；strategyVersion在官方目录调度首次事务对账后设为已交付目录的真实版本；这表示目录已装载，不表示结果接收/执行链路完成。三个预算档位是 `ECONOMY/BALANCED/ACCELERATED`。这些值仅命名档位，当前预约节奏以预算设计为准，部署方仍须结合实际额度校准，不能把加速理解为绕过水位或删除探索渠道。
PUT 请求例（示意数据）：
```json
{"expectedVersion":1,"enabled":true,"budgetProfile":"BALANCED","exclusions":[{"kind":"OWNER","value":"example"},{"kind":"REPOSITORY","value":"sample/skills"}]}
```
在同一事务中比较 expectedVersion 与当前 version；相同且有效则保存并递增 version，过期则409且不写任何字段。即便重复相同内容也先比较版本，不静默覆盖其他管理员操作。客户端重新读取并让管理员确认合并，不自动重试旧表单。官方策略升级只更新 strategyVersion，不能覆盖启停、预算和排除项；管理员配置的实际保存才递增version，策略版本单独演进；运行任务同时记录两者。
排除项最多1000条，OWNER 为登录名，REPOSITORY 为 `owner/name`；按合法 GitHub 坐标格式校验、统一大小写并去重，拒绝URL、通配符与查询语法。规范化后整体替换；返回规范化列表。排除按本次已核实的当前坐标匹配，不承诺旧名字跟随改名或转移。
调度语义：关闭/排除停止官方新发现及官方独占未登记候选的后续采集，保留候选、出处、进度。已登记来源按自身设置继续；其他独立启用入口需要的候选继续。取消排除或重开自动续排，不能复活已失效的租约。
## 4. 有界列表与按需详情
- 沿用 `page=0,size=20`；size为1–100，page为0–1000，且 `page*size<=100000`。超出返回400，提示缩小筛选；上限是合同防护，不是深分页性能已达标的证明。默认全部状态，服务端支持 q（owner/name，最长200）、status、channel、paused，多个条件取交集。
- 仓库按 `(firstDiscoveredAt DESC, candidateId DESC)` 稳定排序，firstDiscoveredAt为候选全局首次发现时间；candidateId为本地候选ID，详情仍按稳定repositoryId定位。首次缺省asOf使用服务器时间，后续沿用响应asOf，仅纳入 `firstDiscoveredAt<=asOf` 的候选。出处按 `(lastSeenAt DESC,id DESC)`、路径按 `(skillPath ASC)`，均继续分页。
- asOf是插入截止时间，**不是历史一致性快照**：已有记录的状态、排序时间与过滤条件仍可能变化，跨页可发生移动。客户端固定当前显示页，提示有新数据后手动刷新，禁止轮询自动重排；一致性快照或游标迁移要依据D3执行计划/规模测量另行收敛，不能暗示当前设计消除了页间漂移。
- total仅统计同过滤范围，不在每页重新计算业务成果聚合；实施时采用匹配索引和受限计数/缓存策略，缓存窗口须公开在观测时间中。所有列表最多100项，详情只含摘要；出处、路径、正文、日志不一起打包。
- progress最多3个默认渠道、100项额度观测；若主体超过上限，`quotaObservationsTruncated=true`，不得把截断后列表当完整额度池。等待项每组最多32项；同原因/工作类别合并。
- 关闭抽屉保留筛选与位置；页面离开或隐藏时停止轮询，请求不叠加。每次返回observedAt，asOf和observedAt作用不同。
## 5. 成果、积压与等待口径
近期区间 `[from,until)`，UTC时间，最长90天，from<until且until不得晚于服务端当前时间；这是查询边界，不决定数据保留期限。
| recent字段 | 持久事件和去重键 |
| --- | --- |
| newCandidateRepositories | namespace内候选首次创建时间；当时已有官方出处。按repository ID去重。已有候选后来首次被官方命中不计“新增候选” |
| firstValidRepositories | V115的FIRST_VALID事件及其occurred_at，official_at_event记录事件发生时的官方归因；同仓库一次。无法证明的历史不补造时间，不能用当前BOUND状态、最后命中或续租时间倒推 |
| newSkills | 首次成功完成采集校验并创建新Skill及版本的持久交接事件，按skillId去重；仅下载、202接收、手工绑定已存在Skill不计 |
| updatedVersions | 已有Skill创建新内容版本的交接事件，按versionId去重；与newSkills的首个版本互斥，重放或NO_CHANGE不计 |
这些是采集成果，不是发布数量。事件发生时固定归因，其他入口同时参与则保留共同出处，不宣称官方独占贡献；后补官方出处不能改写过去成果。历史缺少first-valid或归因事件时 `complete=false`、`availableSince`标明完整可用起点；受缺失影响的指标为null，禁止默认为0。完整期间零成果才是0。
current独立统计此刻曾有官方出处的@github仓库：pendingRepositoryChecks为待检查的去重仓库数（含暂停等待），pendingReviewVersions为仍在待审的去重版本数。它不随recent区间变化，关联入口共享项只计一次；当前筛选与来源归属依据数据库权威状态。`current.observedAt` 与 recent 聚合的 summary.observedAt 区分缓存/观测时间。
等待原因区分关闭、排除、正常预算、主限额、次级冷却、凭证不可用、候选容量、审核积压、执行器无联系、能力不足与覆盖缺口。每项包含受影响工作、仍能继续的工作、恢复条件与相关操作类别；retryAt无可靠依据时为null。到时只允许重新领取，不代表GitHub请求已执行。
内部credential-status新增可选officialQuotaObservation，保留旧githubQuota的必填项和状态枚举；新字段明确观测时间、主/次级限流与UNKNOWN，主体由Java按租约凭证关联，不能由Go任意指定。额度按资源类别与可确认的共享主体展示最近观测；不同token不能简单相加。未知主体不承诺独立额度；过期/缺失为UNKNOWN且remaining=null。reset/retry时间只作预计重试，不承诺恢复。凭证认证失败与限流分开；次级限流收紧共享请求并遵守退避，不能不断换token冲击同一限制。
## 6. 内部能力协商与固定快照
保留 `/internal/github-sync/v1`、`protocolMajor=1`、原任务kind、原discovery.entryType、状态和错误枚举。以下兼容扩展已接线，验证范围见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)：
1. claim新增可选 `capabilities: string[]`（最多32项），缺省空表示只支持旧任务。官方发现任务要求 `official-discovery/v1`、`official-checkpoint/v1`、`official-pause/v1` 全部匹配；官方独占候选的REPOSITORY_SYNC任务仅要求新增控制能力 `official-pause/v1`（已有仓库采集协议能力仍适用）；不兼容时不领取该任务，仍可领取适配的旧任务或返回204。未知能力忽略，不能当通配支持。
2. 官方发现仍是kind=DISCOVERY，但仅返回 `officialDiscovery`，不返回legacy discovery；response.requiredCapabilities明示完整能力集合。官方独占候选仓库任务同样携带requiredCapabilities并受领取过滤，以确保暂停合同生效。来源已登记或其他入口独立需要时保留对应既有执行资格。
3. 快照固定formatVersion、策略/查询版本、shardId、渠道、执行模式、结构化query、固定范围和fixedCutoffAt、预算以及已确认checkpoint。README与元数据有不同scope；KEYWORD的value保持1–256字符的有界整段短语，README也按完整短语编译，不拆词，只接受字母、数字、普通空格、减号、下划线、点和斜杠，不接受原始查询语法。不扩大自定义规则KEYWORD的含义；代码搜索不在v1默认渠道和scope中。
4. 作者扩展只接受已有有效种子，附seedRepositoryId、稳定ownerId、当前ownerLogin和expansionDepth=1；同作者复查进度去重，新作者不递归扩散。
5. 快照configurationVersion只供追溯。启停、排除、有效归属、租约与当前准入在领取和关键接收时再次检查；策略快照不会授权绕过最新管理决定。策略切换不在旧任务中热替换query/window。
6. 心跳可选body增加workerId/protocolMajor/capabilities，带body时三个字段必须齐全；老式无body或{}仍有效。Java按进程实例保存服务端接收时间，重启换workerId。可执行状态要求实例近期心跳且其claim声明支持任务能力，不将全局旧心跳冒充官方实例在线。lastProgressAt来自批次/包/检查点业务提交，不能取因renew变化的task.updated_at。
7. 部署顺序必须先Java持久字段和能力过滤，再Go执行能力，最后管理员启用。旧Java会忽略未知claim字段，因此新Go遇到不带官方快照的响应继续按旧合同处理。回滚到不会能力过滤的Java之前必须隔离/排空官方排队及重试任务；仅关闭页面不足以阻止旧worker误领。
## 7. 出处、分片与检查点提交
官方仓库搜索的OfficialQueryRange按 **pushed时间维度** 使用UTC整秒半开区间 `[since,until)`，必须满足 `since<until<=fixedCutoffAt`。Java把计划截止时间向下取整到UTC整秒并固定为fixedCutoffAt；起点从已确认覆盖边界减去保守重叠窗口后向下取整。下一窗口使用实际确认的整秒until衔接并继续重叠，不从未取整的原截止时间开始，因此被本次取整留到后面的不足一秒范围不会丢失；若取整后没有非空区间则等待下一次调度，不伪造窗口。具体重叠宽度在调度校准阶段确定。Go拒绝带小数秒的since/until，不静默截断；其有界编译将不含until转换为GitHub含终点的 `until-1秒`。这些要求只用于官方查询，不改变legacy RepositorySearchWindow或自定义规则窗口语义。
partition首期仅允许省略或空字符串，当前Go对非空值明确拒绝。Java在能力协商和次级分片执行器明确支持之前不得下发非空partition；达到整秒粒度仍不完整时保存GAP，不能假设次级分片已经实现。
`RepositoryCandidate.officialProvenance` 是可选新增字段，但官方任务回报必须提供与任务相符的出处。Java核对channel/strategy/query/version/shard/seed并补run/task/namespace身份及接收时间，不信任Go自报归属。候选按(namespace,repository ID)去重，出处按候选+持久queryId（UUID）合并first/lastSeen；同仓库多渠道可有多条出处，不重复创建Skill版本或待审目标。长期出处不按每个shard/seed增行；详情仅保留最近分片/种子的有界诊断摘要，可随运行保留期清空，queryId及first/lastSeen独立持久保存。
现有批次上限100仓库、previousCheckpoint与checkpoint及202空body保留。官方批次附officialBatch，checkpoint是OfficialCheckpoint对象的JSON字符串，仍限制4096字符并额外限制4096 UTF-8字节。首批previousCheckpoint=null；后续使用Java已确认的原始字符串，禁止客户端重编码后当相同值。
一个事务持久候选/出处、批次幂等摘要、分片结果/子分片与新检查点后才返回202。响应丢失可重放：同前后checkpoint和规范化业务payload返回原结果；payload不同RESULT_CONFLICT，游标已前进且非同批重放CHECKPOINT_CONFLICT。不能只写checkpoint再异步补候选，也不能丢弃超预算部分后推进整页。额度/候选预算中途耗尽必须保存可恢复的页内cursor或重放该页，去重后继续。
coverage仅描述当前分片：IN_PROGRESS继续，COMPLETE该分片已处理，SPLIT已经原子登记覆盖父范围的子分片，GAP保存未覆盖范围与原因。按时间拆分的childRanges必须非重叠且并集等于父范围；同一时间无法继续拆分时，首期直接保存GAP；未来仅在能力协商通过且受控执行器已实现时才能采用非空次级partition，不无限拆分。childRanges最多16，累计拆分深度和子任务数由Java复核；预算耗尽不写虚假的COMPLETE。
官方发现完成仍调用已有complete：最后discoveryCheckpoint必须等于持久确认值；COMPLETE只允许分片已完整完成或已持久交接子分片，GAP/IN_PROGRESS必须INCOMPLETE。父分片SPLIT完成不等于渠道完成，子分片独立计数；保存缺口与自动重试条件。旧checkpoint编码不变。兼容查询复用长期进度，新增或语义改变的queryVersion新建历史补搜；无法迁移的进度记录原因，从可证明安全的边界重放。长期查询进度不能只寄存在会被历史清理删除的task里。出处/成果事件独立保留；原task_hit/batch随终态任务清理级联删除，不能作为长期归因的唯一依据。
发现游标无法解析、版本不支持或身份不匹配时，不调用旧fail反复重试同一游标；调用 `POST /tasks/{taskId}/official-checkpoint-recovery`，回报结构化原因及被拒绝的原游标。Java验证租约与持久游标相同后原子停止旧进度、记录原因、安排原固定范围内的安全重放，返回REPLAY_SCHEDULED及replacementTaskId。若无可证明的更近边界，重放原任务整个固定范围，不猜测跳页；候选/成果依赖幂等去重。旧租约失效、游标不一致分别沿用LEASE_STALE/CHECKPOINT_CONFLICT。
## 8. 暂停、接收与离线安全边界
新增的 `POST /tasks/{taskId}/official-control` 使用LeaseProof，返回CONTINUE或PAUSE与最新配置版本；Go在开始下一页、候选树检查或包准入前查询。该响应不构成永久授权，Java在实际写入时仍核对当前配置、独立入口归属和租约。
- 发现请求已经在途：允许有效租约提交当前受限批次及检查点；关闭/排除命中的候选保留但不启动官方后续采集。页面停用不回滚已持久接收事实。
- 官方发现抵达安全边界后调用新的 `POST /tasks/{taskId}/official-pause`，携带lease、kind=DISCOVERY和最后确认discoveryCheckpoint（从未提交为null）；Java事务中复核当前仍应暂停、游标和代际，释放租约并保存暂停原因，返回新端点专用PAUSED。配置已重开/独立资格变化时返回新端点专用409 CONTROL_CHANGED，重新查control后继续；旧租约返回409 LEASE_STALE。PAUSED不是对旧complete返回枚举的扩展。
- 官方独占候选仓库也调用同一 `official-pause`，携带lease、kind=REPOSITORY_SYNC与repositoryProgress；要求任务requiredCapabilities含official-pause/v1，不要求officialDiscovery快照。**不再用旧complete承担配置暂停**；旧complete的合法commit和未来retryAt要求保持原样。
- repositoryProgress包含observedCommitSha、pathCheckpoint、incompleteSubtrees、pathResults。尚未取得commit时SHA及路径checkpoint为null，两个数组为空；Java确认任务尚无固定commit、已接收包或绑定commit的准入后才接受。已有固定commit时必须提交相同SHA（不符COMMIT_MISMATCH）；若首次在此报告合法SHA，可原子固定该SHA并保存进度。pathResults最多10000、incompleteSubtrees最多1000，遵循原路径规范；不得推进来源完整处理边界。
- 已持久接收包依据Java摄入记录保留，ACCEPTED路径必须引用同任务/commit/路径的ingestionId；漏报既有接收项不能将其撤销。Go先结束在途上传/回报，再提交暂停；暂停事务保存检查点、路径结果/重试路径并释放租约及未用准入占位，不要求retryAt。恢复领取沿用固定commit及acceptedPaths，不重复上传。未接收暂存对象仍受原票据/租约及清理合同管理，暂停不会把它们冒充已接收包。
- Java在暂停事务中重新解析候选当前归属，而非只看领取时source_id=NULL：途中已经BOUND成为正式来源、其他独立启用入口仍需要采集，或配置已恢复允许执行时，返回409 CONTROL_CHANGED，不写暂停进度、不释放当前租约；Go重新查询control并按当前资格继续。只有仍应暂停且租约、commit和路径合法时，保存进度与释放租约才原子成功。
- 候选enqueue及source_id=NULL的REPOSITORY_SYNC claim都检查最新配置/有效入口，不能只门控DISCOVERY。关闭后新包准入受阻，沿用BACKPRESSURE_ACTIVE并通过official-control区分原因；已持准入/已上传包允许以有效租约提交摄入，已接收或上传事实不删除、不回滚。后续新包在恢复前不启动。上传不是Skill创建或发布成功。
- 租约过期、被接管或代际不符时，所有进度/包/暂停写入拒绝；不以幂等为名接受旧租约。最后未确认批次可由新租约重做，业务接收保持幂等。
Go无联系时Java保留任务、检查点和retryAt，优先显示等待采集服务恢复；不继续声称正在执行GitHub请求。多实例仅在没有任何适配实例时显示官方能力不可用；有心跳不等于处理正常。进程启动恢复归部署工具负责，Java不拉起Go。
## 9. 验收与后续扩展
当前预算档位已使用持久预约节奏，具体规则见[预算设计](../../openspec/changes/add-github-skill-sync/deep-discovery-design.md#3-预算凭证与暂停)。未知额度主体不强行合并，统计缺失不补造历史；非空次级分片和代码搜索不属于默认策略。验收须观察能力门、配置并发、多渠道去重、检查点原子、关闭/排除竞态、独立入口继续运行、接管、统计归因以及列表规模，见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。
## 14. 有界顺序时间分片（V112）
- Java新root的工程初值为最大深度32、累计最多126个child（含root最多127节点），单任务仍最多10次搜索请求/10页；并非预展开完整二叉树，也不表示已校准或保证无缺口。`maxSplitDepth`为当前节点剩余深度，`maxChildShards`为当前root剩余累计child额度，允许0，少于2不能二分。
- 每query仍只有一个active任务。SPLIT批次必须恰有两个UTC整秒、无重叠且完整覆盖父范围的子区间；Java在root行锁下复核深度和累计额度，候选、批次、父checkpoint和PENDING子节点原子写入。相同摘要重放不重复登记。Go确认202后以原SPLIT checkpoint调用complete(COMPLETE)；此成功仅终结父任务并交接，不是query覆盖成功。
- scheduler优先取已有root的PENDING子节点，继承固定策略/query/range/cutoff；退休query的已有子树继续，但不创建新root。只有所有叶子均COMPLETE，才按root原始until推进covered_until；GAP或任何未完成叶子均不能推进。现有无ledger任务按其原固定快照与旧预算补建根并结算。
- 已登记SPLIT拒绝checkpoint recovery，不能把父当作未交接范围再次重放。普通恢复只替换当前节点task关联，不改变root或范围。终态FAILED的ACTIVE节点可安全回到PENDING重放；SPLIT父FAILED则保留已登记children，清旧task关联后继续子节点，不重复创建子树。
- ledger包含固定快照，不依赖父task历史。task清理只将ledger.task_id置空，不能级联删除未完成树；完整root在下一root开始前删除，退休query的完整root也按既有history cutoff有界清理。持久query覆盖和candidate出处不删除。
- 策略执行代次升为2/official-search-v2，query语义版本保持不变。代次只阻止旧scheduler回写，**不能保证旧Java接收器理解child结算**；首次启用前全部Java和Go实例须升级并完成迁移与集成验收；旧Go虽然声明相同三个v1能力，却拒绝超过16的child预算，因此本片不支持旧显式opt-in worker混跑。生产 Go 声明完整能力，但能力名相同不能证明旧二进制理解新预算。
- GAP以原checkpoint、INCOMPLETE和未来retryAt请求结算，返回PARTIAL明确未完整；释放query活动指针，先执行PENDING兄弟，再重扫到期GAP。无PENDING时next_due_at取最早GAP retryAt。任一GAP仍阻止根覆盖推进。
- ledger保留GAP原checkpoint（含cursor.revisit累计退避）、原因、retryAt和最后generation。复用PARTIAL task/run时只收紧预算，query/range/cutoff/策略固定；清终结时间及旧租约字段，generation不重置，下次claim继续递增。仅删除该已终结任务的旧batch/task_hit诊断；候选首次发现/官方出处不删除，重扫不能当业务新增。
- 历史清理后task_id置空，ledger仍保留退避和检查点；重建任务仅改检查点shardId，保留cursor/revisit与generation。普通IN_PROGRESS仍原任务RETRY_WAIT，旧自定义任务完成规则不变。
- 默认Go能力保持关闭，尚未执行SQL/HTTP/并发或端到端验收。
## 15. 有效作者扩展（V113）
- 仓库metadata新增可选`ownerId`（稳定GitHub账号ID），保留旧请求构造与缺省语义。Java仅在有效租约、仓库ID相符时保存身份；省略不靠login推断，并清除该仓库旧稳定ID关联，避免转移后误将新login归给旧ID。实际包`validated`成功事务才持久seed证据，证据不随task/ingestion历史清理消失，BOUND或收到上传均不等价。
- 只有同时有metadata/README等非OWNER官方出处的有效仓库才能生成作者查询。按namespace+ownerId去重，固定seedRepositoryId、depth=1；OWNER发现本身不递归生成新作者。被OWNER先发现的库后来独立搜索命中且已有真实验证证据，可成为普通seed。每轮最多物化100个新owner，后续调度继续处理其余。
- 动态query_source=OWNER独立于内置目录退休；其语义身份是ownerId，规范login只是坐标，改名不提升queryVersion；策略代次为3/official-search-v3，query语义版本未变。OWNER任务除三项官方能力外必须声明`official-owner/v1`，search-only opt-in实例不能领取。
- 快照mode=OWNER_RECHECK，range/cutoff只标识固定执行身份与复查截止，不按pushed过滤；不进入时间分片ledger，不更新covered_until。完整枚举只更新last_full_recheck_at及下次复查时间；当前周期复用search_rule_hours，未校准。
- Go每租约通过`GET /user/{account_id}`解析稳定ID的规范login/type，逐仓库复核owner ID，并在末页再次校验身份。lookup后的首空批通过可选officialBatch.ownerLogin确认规范坐标，再继续枚举；该字段进入幂等摘要，Java保存坐标后下一control能响应新login排除，固定快照ownerId不变。改名或旧login复用不得换账号；错误/身份不一致保留未完成进度退避。
- 单租约仍最多10次CORE请求/10页，身份读取与末页复核均计请求，快照maxRequests至少3。每次列页前预留页请求及可能末页复核预算。OWNER检查点页码为正int32，按page/offset持续续页，无10页总上限；整型边界仍有结果时记RESULT_LIMIT GAP。OWNER GAP保持RETRY_WAIT，搜索GAP才轮转为PARTIAL；页列表是可变集合，后续复查弥补改名/新增造成的位移，不宣称一致性全量快照。
- owner排除按动态query最近确认的规范login在入队、claim、control/pause复核；所有返回仓库再按当前owner/name排除，已存在来源和独立入口资格维持第13节规则。暂停确认已接收页，不回滚候选。关闭/owner排除后同配置/租约最多接收一个新的在途批次，重放幂等不消耗该边界。
- 默认能力仍关闭；本片仅编译与静态复审，不代表SQL迁移、GitHub真实枚举、HTTP或跨进程恢复验收通过。
## 16. 管理员读模型与成果证据（V115）
六个只读端点由独立 `AdminGithubDeepDiscoveryReadController` 提供，固定 `@github`、`SUPER_ADMIN`、`no-store`。列表、出处和路径各自限制 page 0–1000、size 1–100（默认20）；列表先分页再聚合出处。`q` 使用字面子串匹配，不解释正则、SQL通配符或GitHub查询。`asOf` 限制插入截止，后续状态仍是最新投影，不声称历史事务快照。
近期指标基于 V115 长期事件；事件与实际候选创建、包验证及版本交接同一事务。候选 `INSERT DO NOTHING RETURNING` 判断真正新建，后补官方出处不计新增；首次有效必须通过真实包验证，不能从 BOUND 推断。发布结果携带实际创建分支的 `newSkill`，新 Skill 与已有 Skill 更新互斥，未变化、路径移动、仅绑定不发成果事件。事件保留当时官方归因及查询身份，不依赖运行日志存活。
迁移记录 `availableSince`。早于该时间的查询指标返回 null、`complete=false`。旧 validated 摄入、绑定及种子保存无时间先前验证标记；缺旧证据的候选进入未知集合，其后验证不能冒充确定的首次有效：只有查询区间内出现这种官方事件时，`firstValidRepositories=null`、`complete=false`，其他可靠指标仍返回数值。旧数据不按现在的出处伪造历史归因。
候选 REJECTED 区分 `NO_VALID_SKILL` 与 `VALIDATION_FAILED`；后者不是“没有技能”。`lastDiscoveredAt`取官方出处最近命中，`lastCheckedAt`取保留任务最近结算（包含失败尝试）。历史完整边界不可证实时 `historyAvailableSince=null`。路径保留 `UNCHANGED` 原始结果；原始错误正文不会充当 reasonCode。待审数量以 `SKILL_VERSION` 类型的真实 PENDING 审核记录按版本去重。
渠道进度包括仍在续跑的退休查询。搜索使用当前树叶子，尚未建立树的到期查询算一个待处理工作单元；作者使用当前一层查询，只有真实 `last_full_recheck_at` 才能证明最近枚举完成。根被下一次复查替换后计数可下降，不是历史累计或全站覆盖率。RUNNING 同时要求有效租约与90秒内业务进展，不使用续租或配置开启代替进度。
## 16. 额度观察与共享冷却（V116）
- 新`POST /tasks/{taskId}/official-quota`接受`{lease, observation}`，只允许当前具official-pause/v1的租约。Java从租约关联凭证（可以匿名），不信任客户端主体声明。旧credential-status仍必填credentialId/status，附加officialQuotaObservation已接收并保持原字段一致性检查。
- 观察按凭证键/resource保存，主体当前未知保持null。保存observedAt/receivedAt/validUntil，时间允许偏差300秒，观察有效期不超过5分钟，过期不显示确定余额；不累计token余额。未知主体不累加额度，也不证明不同凭证额度独立。
- 认证PRIMARY_EXHAUSTED及真实OBSERVED remaining=0写入租约绑定credential/resource的额度记录；匿名才按resource保存primary_until。单个SEARCH耗尽不阻断其他当前可用凭证，也不写成整凭证停用。SECONDARY_COOLDOWN使用部署级cooldown_until；缺省60秒、最多24小时，重复同observedAt不延长。UNKNOWN带retryAfterSeconds也触发共享失败退避，无余额不补造0。
- 新claim先检查共享门控：命中则延期刚领取租约并不返回任务；有任何已配置凭证但当前全部不可用也延期，不回落匿名。仓库CORE的新包准入受同一冷却约束，已有准入重放与上传/接收可以结算。官方control返回QUOTA_COOLDOWN/retryAt，pause按当前服务端时间转RETRY_WAIT；来源自身PAUSED仍优先。旧无官方能力的在途worker没有逐请求control保证，但新claim/admission仍受门控。
- 替换为不同token时同事务清理旧凭证的资源额度与官方观察；同token重存不清余额。官方与legacy租约报告按task→credential锁顺序再次核对fingerprint，替换后的旧报告不能写回额度或失效状态。
- Go在官方REST/Git请求边界同步观察（匿名也报告）；持久回报失败停止本租约后续请求并使用进程共享退避，不通过轮换token继续。PRIMARY进程门控按token摘要和resource，认证失败按token摘要隔离，SECONDARY与回报失败共享全进程；Git缺配额响应只报UNKNOWN，不伪造OBSERVED。
- GitHub剩余额度没有预约账本或“实时余额”承诺；V117官方执行预算是独立的持久节奏控制，不能抵扣或代表GitHub额度。生产Go声明能力，Java配置默认关闭。
### 等待反馈的证据与优先级
渠道等待聚合查询、当前任务、缺口叶子与作者实时排除，不再选取单个最小字符串。详情按当前仓库资格、来源自身暂停、活动任务、实例能力、水位和额度共享保护组合；同原因合并受影响工作，最多32项。历史任务的 DISABLED/EXCLUDED 不覆盖现时已恢复资格，过期冷却标签不表示保护仍生效。
水位判断复用真实采集容量 gate（含未解决坐标计数、暂停后的低水位恢复条件），不是只看可能滞后的 paused 标记。来源自己的暂停显示 DISABLED、影响 SOURCE_UPDATE/PACKAGE_ACQUISITION，并提示恢复该来源；不会提示改探索开关解决来源暂停。
V116共享保护作用于所有任务的新领取；CORE保护也阻止新的FULL包准入，已准入包可结算。旧协议已领取任务不保证逐请求停。`QUOTA_COOLDOWN`表示共享退避保护，不能据此断言GitHub已确认次级限流。SEARCH保护只影响搜索探索，CORE还影响作者探索、候选验证、来源更新与包采集。
联系状态使用全局最近心跳判定是否有采集服务联系；渠道分别核对所需能力，官方独占未绑定候选核对暂停协议能力，已有来源不会因为缺作者扩展能力被宣称停机。`continuingWork`只包含有效租约且90秒内有实际业务进度、当前资格允许、且未被任何已知等待原因阻断的工作类别；没有此证据返回空，不承诺任务正在执行。无实例联系时不会声称其他工作继续。
配置过凭证、但当前资源没有可用凭证时，新领取任务保存 `CREDENTIAL_UNAVAILABLE` 并短暂延后；不默默降级匿名，也不把该状态分类为确定限流。管理员反馈实时按凭证启用/有效状态及该资源最近额度判断，恢复可用后立即不再显示旧凭证等待；后续成功领取同时清除持久任务/查询中的旧凭证或冷却原因。系统从未配置凭证或删除全部凭证时，沿用匿名能力；无可靠凭证恢复时间返回 `retryAt=null`，内部轮询延迟不是恢复承诺。

1175
docs/api/github-deep-discovery.openapi.yaml Normal file
View file

File diff suppressed because it is too large Load diff

223
docs/api/github-sync-admin.md Normal file
View file

@ -0,0 +1,223 @@
# GitHub 同步管理接口（实施中）
本页记录已落地的管理接口；完整业务规则以 [OpenSpec](../../openspec/changes/add-github-skill-sync/README.md) 为准。普通用户会话、GitHub 凭证和 Go 服务密钥均不能代替平台 `SUPER_ADMIN` 权限。写入接口沿用 Web 会话的 CSRF 校验，并记录操作审计。
## 系统级公开 GitHub 凭证
平台 `SUPER_ADMIN` 可通过以下接口维护零枚或多枚系统凭证。Token 在管理员接口中只通过创建或替换请求体传入，不出现在管理列表与响应、持久任务快照、审计详情或 URL。管理响应使用 `Cache-Control: no-store`；写入沿用用户会话 CSRF 与审计。Go 只在受机器身份保护的任务领取响应中临时得到凭证，并以当前租约回报额度、认证失效或限流；限流不会把凭证标成失效。Java 按领取任务所需的 `SEARCH` 或 `CORE` 资源选择凭证：该类别已观测余额为零且重置时间未到时跳过，另一类别仍可使用；显式限流的全局等待时间仍优先生效。无可用凭证时任务仍可匿名访问公开 GitHub API。隔离的 Java HTTP/PostgreSQL 与独立 Go 进程已验证以上三种领取路径；真实 GitHub Token 不参与该测试。
| 方法 | 路径 | 说明 |
| --- | --- | --- |
| `GET` | `/api/v1/admin/github-sync/credentials` | 列出名称、Token SHA-256 前 16 位指纹、状态、密钥版本、密钥是否可用和限流时间，不返回 Token。 |
| `POST` | `/api/v1/admin/github-sync/credentials` | 以 `{"label":"公开读取 1","token":"..."}` 加密新增。 |
| `PUT` | `/api/v1/admin/github-sync/credentials/{id}` | 以同样请求体替换 Token、可修改名称，并恢复为可用状态。 |
| `PUT` | `/api/v1/admin/github-sync/credentials/{id}/enabled` | 以 `{"enabled":false}` 停用，或以 `true` 启用；认证失效项必须先替换 Token。 |
部署需提供 `SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS`（`v1=<32 字节随机密钥的 Base64>`，多个版本用逗号分隔）和 `SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION`（如 `v1`）。不配置时只能匿名运行，创建凭证会被拒绝。轮换密钥时先保留旧版本并追加新版本为 active，旧密文仍可解；替换 Token 后采用新版本加密，确认所有旧密文都更新后才能移除旧密钥。不要把这些值提交到仓库或输出到日志。部署方应把它们放入 Compose 环境文件或 Kubernetes Secret，并确保 Go 到 Java 的内部连接采用可信网络与 TLS 边界。管理员页面已提供凭证表单及已观测额度；Go 会在有效 GitHub API 响应后按当前任务租约回报额度。额度回报失败不会阻断采集，页面数据可能延迟。
## 积压准入水位
| 方法 | 路径 | 说明 |
| --- | --- | --- |
| `GET` | `/api/v1/admin/github-sync/admission-policy` | 读取全平台和单 namespace 的高低水位、候选预算、全平台暂停状态、当前未解决项与 `@github` 候选数量及准入状态。 |
| `GET` | `/api/v1/admin/github-sync/worker-status` | 读取 Go 采集服务的最后在线时间和在线判断；从未连接时 `lastSeenAt` 为空。 |
| `PUT` | `/api/v1/admin/github-sync/admission-policy` | 同时设置两组水位和候选预算；写入操作者和新旧值的审计记录。 |
| `GET` | `/api/v1/admin/github-sync/schedule-policy` | 读取仓库更新、精选用户或组织、广泛搜索三类目标周期。 |
| `PUT` | `/api/v1/admin/github-sync/schedule-policy` | 以 `repositoryHours`、`curatedOwnerHours`、`searchRuleHours`（均为 1–720）更新三类周期与已有入口。 |
| `GET` | `/api/v1/admin/github-sync/sources?page=0&size=20` | 分页查看平台已解析来源、最近任务和逐包摄入状态计数，包含失败包数 `failedPackages`。 |
| `GET` | `/api/v1/admin/github-sync/sources/{sourceId}/skills?page=0&size=20` | 查看来源各 Skill 路径最近一次采集、摄入与审核状态。 |
| `POST` | `/api/v1/admin/github-sync/sources/{sourceId}/rescan` | 将平台来源提前纳入正常调度；已有开放任务时合并请求。 |
| `POST` | `/api/v1/admin/github-sync/sources/{sourceId}/retry-rejected-skill` | 对一个最新审核结果为拒绝的 Skill 路径请求重新采集并送审。请求体：`{"skillPath":"skills/example"}`。 |
| `POST` | `/api/v1/admin/github-sync/conflicts/{ingestionId}/bind-skill` | 平台同名冲突确认绑定到已有 Skill ID，并重试该同步包。 |
| `GET` | `/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns` | 查看平台来源追加的候选目录忽略规则。 |
| `PUT` | `/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns` | 用 `{"ignorePatterns":["examples/legacy"]}` 整体替换规则。 |
| `GET` | `/api/v1/admin/github-sync/runs?page=0&size=20` | 分页查看平台发现及仓库采集运行与后续包处理计数。 |
| `GET` | `/api/v1/admin/github-sync/runs/{runId}` | 查看一轮平台运行的类型、状态、时间、任务数、路径数、包处理阶段计数、最近任务错误码和下次重试时间。 |
| `GET` | `/api/v1/admin/github-sync/runs/{runId}/tasks?page=0&size=20` | 分页查看该轮发现或仓库任务的仓库、模式、状态和错误码。 |
| `GET` | `/api/v1/admin/github-sync/runs/{runId}/paths?page=0&size=20` | 分页查看该轮实际产生的 Skill 路径结果、采集结果、摄入错误码及关联审核状态。 |
运行详情按 `runId` 与平台 namespace 定位，发现运行可能没有 Skill 路径；预览运行以及已按保留策略清理的运行详情返回 404。路径结果来自这轮任务，不把来源的最新路径状态当成历史运行结果。任务和路径分别分页，`page` 从 0 开始，`size` 为 1–100；凭证、任务快照及原始 GitHub 响应不在这些响应中。
逐路径响应增加 `latestBindingIngestionStatus`、`retryRequested` 与 `retryInProgress`。最新绑定项为 `REJECTED` 时，管理员可请求逐 Skill 重试；当前忽略的路径及暂停来源不可重试。请求会合并或提前安排来源任务，必要时把排队中的仅 HEAD 任务升级为完整扫描。每次请求只自动安排一次完整采集；任务终止且未产生新待审项时，管理员可以再次请求，不会因目录缺失形成无限重扫。只有请求后的新包才可用本次重试意图重新送审；审核历史和原拒绝记录保留，成功建立新待审目标后清除重试标记。高水位准入仍生效，重试不保证立即执行。
来源列表附带最近任务的 `latestTaskErrorCode`、`latestTaskErrorDetail` 和 `latestTaskRetryAt`。诊断详情限于 Worker 或 GitHub HTTP 的已定义代码，不含原始 GitHub 响应、Token 或异常文本。隔离环境已验证有效租约的失败回报会同时持久化诊断码和系统凭证的 SEARCH 额度，含 URL 的详情请求返回 400，旧租约重放返回 409。
调度配置初始为仓库 24 小时、精选用户或组织 24 小时、广泛搜索 72 小时。升级部署若仍设置 `SKILLHUB_GITHUB_SYNC_REPOSITORY_CHECK_INTERVAL`，在管理员首次保存后台周期前，仓库完成后的检查时间继续按原环境值计算；`GET` 的 `repositoryHours` 此时为 `null`，`effectiveRepositoryInterval` 返回实际生效的 ISO 8601 时长。保存后后台配置接管仓库周期，并更新已存在的精选仓库、精选用户或组织、搜索规则的目标周期，包括先前在单条入口上设置的周期；已有更早的到期时间不会被延后，已领取/等待重试任务不会因此跳过额度与积压保护。以后新增入口使用类别默认值，单条搜索规则和精选用户仍可另填周期。
`PUT` 请求体：
```json
{
"globalHigh": 1000,
"globalLow": 500,
"namespaceHigh": 200,
"namespaceLow": 100,
"candidatePendingLimit": 1000,
"candidatePerLeaseLimit": 100
}
```
水位数值仅展示字段格式，不是建议默认值。新部署时两组水位为 `null`，Java 拒绝新的逐包采集准入；管理员应结合试运行的待审数量、包大小、对象存储和审核能力决定实际水位。水位要求 `0 ≤ low < high`；候选预算要求 `1 ≤ candidatePerLeaseLimit ≤ candidatePendingLimit`，并限制为 10000/100000 以内。数据库初始候选预算分别为 100 和 1000，试运行后可调整。`candidatePendingLimit` 限制单 namespace 中排队或正在验证的候选仓库总数，`candidatePerLeaseLimit` 是单次发现租约的新增上限。读取与修改均使用项目统一的 `ApiResponse` 包装；`globalPaused` 是执行状态，不由此接口直接写入。
当前响应还包含 `globalUnresolvedCount`、`platformUnresolvedCount`、`platformPendingCandidateCount` 和 `platformPaused`。前两项沿用新包准入使用的去重口径，按最新未解决 `(namespace, repository ID, Skill 路径)` 计数，并包含仍持有效租约的包准入占位；候选数量只统计 `@github` 下的 `PENDING`、`VALIDATING`，与候选水位口径一致。`platformPaused` 是 `@github` namespace 的准入暂停状态，与 `globalPaused` 分开显示。平台页面每 30 秒重读这些运行值；它们是观察时刻的快照，不保证多个计数在并发写入下完全同时。
## 自动发现配置
以下接口要求平台 `SUPER_ADMIN`，所有写入均记审计；返回体为项目统一 `ApiResponse`。
| 方法 | 路径 | 说明 |
| --- | --- | --- |
| `GET` | `/api/v1/admin/github-sync/discovery-entries` | 列出平台搜索规则、精选用户或组织、精选仓库入口。 |
| `POST` | `/api/v1/admin/github-sync/discovery-entries/search-rules` | 创建搜索规则，初始停用。 |
| `POST` | `/api/v1/admin/github-sync/discovery-entries/search-rules/preview` | 按同样的结构化规则请求排队预览，返回预览任务 ID；不保存规则或登记来源。 |
| `GET` | `/api/v1/admin/github-sync/discovery-entries/search-rules/previews/{id}` | 轮询状态、各条件的仓库命中估计、最多五个样例及结果不完整信号。 |
| `PUT` | `/api/v1/admin/github-sync/discovery-entries/search-rules/{id}` | 编辑规则并递增版本；进行中的任务继续使用旧快照。 |
| `POST` | `/api/v1/admin/github-sync/discovery-entries/curated-owners` | 登记精选公开用户或组织并启用枚举。 |
| `POST` | `/api/v1/admin/github-sync/discovery-entries/curated-repositories` | 登记精选公开仓库并启用探测；接受 `ownerLogin` 和 `repositoryName`。 |
| `PUT` | `/api/v1/admin/github-sync/discovery-entries/{id}/enabled` | 启用或停用发现入口；重复设置不改变版本。 |
搜索规则请求示例：
```json
{
"title": "Agent Skill 仓库",
"targetHours": 72,
"selectors": [
{"kind": "KEYWORD", "value": "agent skills", "priority": "CORE"},
{"kind": "TOPIC", "value": "agent-skills", "priority": "EXTENSION"}
]
}
```
精选用户或组织请求体为 `{"ownerLogin":"example","targetHours":24}`；精选仓库请求体为 `{"ownerLogin":"example","repositoryName":"skills"}`；启停请求体为 `{"enabled":true}`。关键词和 topic 只能通过结构化条件提交，不能传入原始 GitHub 查询语句。首次部署预置一条停用规则，不会自行执行广泛搜索；精选用户入口目前会登记仓库线索。精选仓库由 Go 解析公开仓库及稳定 repository ID，Java 首次登记为来源并进入周期采集；重复解析只能指向首次固定的 repository ID。入口固定 ID 后，后续发现任务优先使用同 namespace 来源绑定的当前仓库坐标，避免改名或转移后持续访问最初填写的地址；来源详情中的坐标反映最近一次已核对的上游元数据。达到积压水位时仍可登记来源，但新包采集受准入水位阻止。来源创建可能晚于配置提交，读取入口的 `repositoryId` 可区分待首次解析与已解析。发现任务每分钟检查到期入口，同一入口若已有排队、运行或等待重试的任务，不会重复排队。
发现任务按规则版本和固定搜索窗口执行，核心条件先于扩展条件；同一仓库跨条件命中时按 repository ID 去重。搜索结果超过 1,000 条或 GitHub 返回不完整标志时，Go 拆分原时间窗继续查找，不能将截断页算作完整发现。精选用户和组织的公开仓库先作为带入口出处的候选线索保存，只有后续仓库树与有效 Skill 校验通过才成为正式来源。上述规则、截断处理和两类精选账号的 Java→Go→Java 合同已用受控仓库元数据验证；公开 GitHub 的多仓库规模验收见 6.3。
目前已实现入口配置、发现调度、候选仓库任务排队、有效包校验后的来源绑定、精选仓库直接来源登记、已登记来源的周期任务，以及已校验包的 Skill 版本与同步审核任务创建。高水位或尚未配置水位时，新候选不会进入仓库树检查；已登记来源仍可检查 HEAD，新 commit 在容量恢复后进入完整采集。没有有效 Skill 的候选仓库 7 天后重查；读取失败按 1/3/7 天退避，连续三次标记为不可达并继续低频重试。平台管理页首版提供水位配置、精选仓库与账号登记、搜索规则新建、编辑和入口启停，以及来源、运行、逐路径状态查询和来源忽略路径。系统凭证配置和 TEAM 公开仓库来源页面已有首版。无副作用规则预览已有 Java/Go/Web 首版；审核详情页已接入来源、首次文件树、逐文件变更摘要及受限文本差异预览。公开 API 变更须从实际 Java Springdoc 生成 Web 类型。
预览沿用 Java 任务租约，由 Go 对每个条件只搜索第一页并回写命中估计和公开仓库样例；搜索显式限定 `is:public`，各条件命中可能重叠，不能相加作为去重仓库数。GitHub 元数据搜索可能漏召回。预览不创建候选、来源或待审 Skill，预览运行从正式采集运行列表排除。隔离 PostgreSQL/Redis 环境中，Java→Go→Java 的真实公网双条件预览已完成，候选、来源和发现批次均为 0；受控响应还覆盖核心优先、样例上限、后续条件限流时不提交部分结果和持久重试。原始 GitHub 查询语法被拒绝，预览租约不能写发现批次，完成后的旧租约不能重复回写。精选仓库的跨进程来源登记及周期任务排队由受控 GitHub 元数据验证；可选的公网精选仓库演练遇到限流并进入 `RETRY_WAIT`，待在 6.3 的小范围验收中重试。
来源忽略规则以仓库相对目录前缀表示，最多 64 条，每条最长 1024 字符，不接受 `.`、`..`、空段、反斜杠或控制字符。只有已解析为来源绑定的仓库才能配置；平台默认忽略的 10 类目录始终有效。修改规则会重新安排当前 commit 的完整扫描；旧任务的快照不能把新规则下的来源标记为完整处理。Go 按快照跳过候选，Java 在准入、摄入和发布前按当前规则复核。被忽略的已有 Skill 不会自动删除或撤回。
## 同步审核队列
`GET /api/v1/reviews/sync?status=PENDING` 单独查询 GitHub 同步审核；每项包含 `review`、`ingestionId`、`runId`、`repositoryId`、仓库 owner/name、`skillPath` 和 `commitSha`。可选 `namespaceId`、`page`、`size` 与 `sortDirection`，权限沿用当前审核规则；来源摘要只对当前页通过权限过滤的审核项批量查询。普通 `GET /api/v1/reviews` 和 `/pending` 只查询普通提交。旧审核记录在迁移后标记为 `USER`；同步审核记录使用 `GITHUB_SYNC` 并关联包摄入 ID。后台交接可以创建新同步审核项；同一 Skill 的新内容会把旧项标记为 `SUPERSEDED`，不再允许批准。来源与许可证证据接口、审核详情页只读展示及变更文件的文本差异预览已提供；平台审核区和 TEAM namespace 审核区已增加独立同步队列，并在当前分页内按仓库与运行归组；每个 Skill 从详情页单独批准或拒绝。队列可查看待审、已通过、已拒绝和已被新版本替代的记录。验收应同时覆盖 Java 队列映射、既有审核服务、Web 组件以及含真实同步审核数据的浏览器流程。
同一来源路径的后台发布按数据库事务串行处理，并在事务内重新读取当前绑定。若较新的采集任务已形成待审目标，迟到的旧包会保留为 `SUPERSEDED` 摄入历史，不再创建版本或改写当前绑定。新内容待审期间，上一已发布版本继续可用；旧待审项保留历史但不能再批准。
是否升版依据规范化包文件和许可证证据的内容指纹，不依据 GitHub commit 字符串。不同 commit 的内容相同则记为 `NO_CHANGE`；包内许可证文件变化会产生新指纹并重新进入同步审核。内容指纹只用于同一来源绑定的内容判断，不能证明许可证法律有效性。
每个 Skill 另有持久化的 `syncRevision` 内容修订号。首次同步为 1；与最近同步目标内容不同则加 1；已拒绝内容显式重试时沿用原号；内容回退仍加 1。`GET /api/v1/reviews/sync/{id}/source`、市场技能详情的 `githubSource`、技能版本列表及市场列表的 `publishedVersion` 均返回该字段；市场列表优先显示“第 N 次修订”。原 `version` 字符串继续作为安装与精确定位用的技术版本，不改接口含义。
`GET /api/v1/reviews/sync/{id}/source` 返回单条同步审核的仓库 ID、任务当时的 owner/name、固定 commit、Skill 路径、内容指纹、摄入状态、许可证/NOTICE 证据，以及 `baseVersionId` 和 `fileChanges`。基线为审核提交前最近的已发布版本；首次导入的基线为空，文件列表显示全部新增文件；更新仅列出新增、修改、删除的路径及旧/新 SHA-256、大小，不传文件内容。只有按现有审核规则可查看该审核项的人员能读取；普通审核 ID 返回未找到。证据状态为 `MISSING`、`PRESENT_UNCLASSIFIED` 或 `PRESENT_UNPREVIEWABLE`：`MISSING` 指没有找到许可证类文件，即使存在 NOTICE 也仍为缺失；其余状态只表示取得证据的可预览程度，不推断法律有效性。单文件证据预览受 64 KiB 限制；上游仓库根文件的 Git blob 摘要与 Skill 包文件的 SHA-256 分开标注。同步审核详情页以只读方式展示来源、许可证证据及文件变化；当前版本文件仍使用已有审核预览接口；基线文件由 `GET /api/v1/reviews/sync/{id}/base-file?path=...` 读取。该接口沿用审核查看权限，只允许读取该审核差异中的旧版文件，且单文件不得超过 1 MiB。前端仅对已有文本预览支持的文件类型开放差异按钮。
审核详情首屏使用 `GET /api/v1/reviews/{id}/skill-detail?includeDocumentation=false` 读取技能、版本与文件元数据，进入“技能内容”页签后再读取说明正文。省略参数时默认包含正文，保持已有调用行为。逐文件在线预览调用 `GET /api/v1/reviews/{id}/file?path=...&preview=true` 和 `GET /api/v1/reviews/sync/{id}/base-file?path=...&preview=true`；预览最多允许 256 KiB，服务端同时检查记录大小与实际读取字节数，超限返回错误而非截断正文。完整文件下载不带 `preview` 参数，仍沿用原权限与大小规则。首次导入没有旧文件，不请求 `base-file`。历史记录只在打开“审核记录”页签时加载；来源响应不嵌入差异正文。
`@github` 目录的许可证目前没有自动识别结果；审核者批准这类同步项时，必须在审核意见中写明处理依据。缺少意见的批准请求返回业务错误，不能因为具有超级管理员权限而跳过。TEAM 来源仍展示同样的证据和警告，并沿用人工审核权限。
同步审核来源详情也返回采集 `runId`，详情页显示该 ID，以便从单条审核追溯到采集运行；审核记录自身展示提交者和审核者。首次导入的实际文件树及预览位于详情侧栏，来源卡片的文件列表列出新增路径。安全扫描已有结果时展示扫描卡片；尚在扫描、暂无结果或结果读取失败时，详情页分别显示对应状态。
## TEAM 公开仓库来源
TEAM namespace 的 OWNER、ADMIN 可使用以下接口；成员和其他 namespace 的管理员不能跨范围读取或修改。冻结或归档的 namespace 可读取配置，但不能新增或修改。请求沿用会话及 CSRF 保护，写入留审计。
| 方法 | 路径 | 说明 |
| --- | --- | --- |
| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources` | 列出本团队已配置的公开仓库入口。 |
| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources` | 登记 `ownerLogin`、`repositoryName` 与可选 `visibility`；默认 `NAMESPACE_ONLY`。 |
| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/enabled` | 以 `{"enabled":false}` 暂停、以 `true` 恢复入口和已解析来源。 |
| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/visibility` | 以 `{"visibility":"PUBLIC"}` 或 `NAMESPACE_ONLY` 修改后续导入的可见范围。 |
| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved?page=0&size=20` | 分页查看本团队已解析来源、最近任务及逐包摄入状态计数，包含 `failedPackages`。 |
| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills?page=0&size=20` | 查看本团队来源的逐 Skill 路径状态。 |
| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/rescan` | TEAM 管理员将已解析来源提前纳入正常调度；已有开放任务时合并请求。 |
| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/retry-rejected-skill` | TEAM OWNER/ADMIN 对本团队已拒绝的 Skill 路径请求重新采集并送审；请求体同平台接口。 |
| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/conflicts/{ingestionId}/bind-skill` | TEAM 管理员确认同名冲突或路径变化，并绑定到本 namespace 的已有 Skill ID。 |
| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns` | 查看本团队已解析来源的追加忽略规则。 |
| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns` | 在本团队来源上整体替换追加忽略规则。 |
| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/runs?page=0&size=20` | 分页查看本团队的发现与仓库采集运行。 |
重扫接口不接收包或 commit，也不直接创建待审 Skill。返回 `sourceId`、`status`（`SCHEDULED` 或 `MERGED`）、可选 `taskId`、`taskStatus` 与 `nextCheckAt`。已有 `QUEUED` 任务会提前排序；`LEASED` 任务照常运行；`RETRY_WAIT` 保留原重试时间。没有开放任务时，仅把来源的下次检查时间提前到当前，最多等待正常调度周期。`PAUSED` 来源返回业务错误。正常任务仍按积压准入、GitHub 额度和 HEAD/FULL 判定执行；相同 commit 的完整内容不重新导入。
Go 只解析公开 GitHub 仓库；Java 将首次解析的 repository ID 固定到该 TEAM 入口，在 TEAM namespace 中建立独立来源绑定。页面可用返回体的 `repositoryId` 区分待首次解析与已解析。暂停不撤回既有 Skill；已批准入的在途包可结算，之后不再发放新包准入。可见范围变更不追改已发布 Skill。TEAM 管理页提供公开仓库登记、启停、可见范围、逐 Skill 状态及本 namespace 的运行记录。
来源、运行和逐 Skill 路径列表的 `page` 从 0 开始，`size` 为 1–100，缺省 20。来源返回最近仓库任务的 `latestTaskStatus`、`latestTaskMode`、错误码与 `latestTaskRetryAt`，另按每个 Skill 路径的最新摄入记录状态分别计数 `receivedPackages`、`processingPackages`、`pendingReviewPackages`、`blockedPackages`、`failedPackages`；旧失败在同一路径重新摄入后不再计入来源当前状态。运行列表的阶段计数仍按该次运行的摄入记录统计，因此保留历史运行结果。`trackedSkills` 是已绑定 Skill 数，包含待审核 Skill，**不是已发布数**。运行的 `ACQUISITION_COMPLETE` 只表示采集结束；`receivedPackages` 等字段用于识别 Java 摄入和审核仍在进行。逐路径接口按 Skill 路径返回最近一次采集 outcome、受控原因码、摄入状态和审核状态；来源 ID 必须属于当前平台或 TEAM namespace，否则返回未找到。本期列表不返回包正文或凭证。
平台和 TEAM 管理页面在前台查看时每 30 秒重取来源、运行与已展开的逐 Skill 列表；平台凭证页的已观测额度也同频刷新。手动重扫成功后会立即刷新相关查询；离开页面后不持续轮询。
运行列表的 `nextRetryAt` 是该运行中所有 `RETRY_WAIT` 任务最早的重试时间；没有等待重试的任务时为空。它表示采集任务的下一次尝试，不表示 Java 摄入或审核会在此时间完成。
运行列表的 `lastErrorCode` 是当前处于等待重试、部分完成或失败的采集任务，以及等待重试、阻塞或失败的包处理中，最近更新的一条错误码；没有这类错误时为空。发现运行没有仓库来源详情，仍可通过此字段查看失败原因。错误码只用于定位当前异常，不包含仓库文件内容或凭证。
Go 服务独立于工作池，每 30 秒向内部 `/heartbeat` 发送一次受机器身份保护的心跳。Java 只保存全平台最近一次成功心跳，不保存 Go 主机名或凭证；最近 90 秒内有心跳时 `online=true`。后台每 30 秒刷新，停止或失联后约 90–120 秒显示离线。该状态与高水位造成的“新包准入暂停”是两件事，后台分别显示；`online` 只表示 Go 与 Java 最近可通信，不保证 GitHub 出站访问成功。
## 同名冲突与路径变化人工确认
发布时若来源 Skill 的规范化 slug 已被同 namespace 的 Skill 占用，该摄入项进入 `BLOCKED`，`ingestionErrorCode` 为 `SLUG_CONFLICT`，逐路径接口同时返回 `targetSkillSlug` 和 `ingestionId`。平台 `SUPER_ADMIN` 可处理 `@github` 中的冲突；TEAM 的 OWNER/ADMIN 只能处理本团队 namespace 的冲突。其他状态不能调用绑定接口，跨 namespace 的摄入 ID 返回未找到。
两个绑定接口使用相同请求体：
```json
{"skillId":123,"reason":"已核对为同一技能，保留现有历史版本"}
```
`skillId` 必须是本 namespace 中 slug 相同、未归档且没有普通用户待审版本的 Skill；已经由其他来源管理的 Skill 或已经占用的 GitHub 路径不能再次绑定。`reason` 去除首尾空白后至少 10 字符，最多 1000 字符。成功后返回 `QUEUED_FOR_RETRY`，记录操作者、仓库 ID、来源 ID、路径和处理依据；后台任务会对该固定 commit 的包重新执行发布与审核流程。预留绑定不把当前包指纹当作已发布指纹，因而不会错误地判为无变化。创建同步待审版本时，Skill 的技术所有者转为保留的系统 actor，由 namespace 管理；若创建失败，原所有权保持不变。此操作不会直接批准或公开新内容。
完整仓库扫描结束后，若旧路径消失，且新路径的完整包内容指纹只匹配本仓库一个已绑定 Skill，Java 自动更新路径绑定、沿用 Skill ID，并记录系统审计；相同内容不创建新版本。旧路径仍出现在本次扫描结果时不会被认定为移动。多个候选或内容变化导致无法唯一匹配时，摄入项标记 `PATH_MOVE_AMBIGUOUS`，不得自动新建 Skill；进行中的扫描等待完成；终止的部分扫描不据此推断目录消失或自动移动。若部分状态仅由其他包的确定性失败造成，新路径已接收且所有既有绑定路径都在本次结果中，可作为独立 Skill 进入同步审核；旧路径未得到确认时仍标记 `PATH_MOVE_SCAN_INCOMPLETE`。
平台 `SUPER_ADMIN` 或 TEAM OWNER/ADMIN 可用上述绑定接口处理 `PATH_MOVE_AMBIGUOUS`。逐路径列表显示已绑定的 Skill ID；管理员应核对旧路径及目标 Skill 后提交其 ID 和处理依据。服务端再次验证本次任务为完整扫描、新路径已采集、旧路径在该快照中确实不存在、目标 Skill 未归档且已由同 namespace、同仓库来源管理。成功后保留 Skill ID 与历史版本，记录操作者及旧新路径，并重新处理固定 commit 的包；内容变化才创建待审版本，相同内容只更新来源路径。接口本身不批准内容。`PATH_MOVE_SCAN_INCOMPLETE` 不能人工确认为移动，须等后续完整重扫。被当前忽略规则覆盖的旧路径或新路径也不能通过人工确认。
来源绑定记录上游 `UNKNOWN/PRESENT/MISSING`、最近核对 commit 和首次缺失时间，并通过逐路径响应的 `upstreamPresence`、`presenceCommitSha`、`missingSince` 展示。`UNKNOWN` 表示尚无有效完整核对；`PRESENT` 和 `MISSING` 都只说明最近一次有效完整扫描的结果，不是实时 GitHub 状态。仅完整扫描且忽略规则未变化时对账；部分扫描、暂停来源和当前忽略的路径不触发缺失推断，后续完整重扫看到旧路径会恢复 `PRESENT`。不可达来源成功完成扫描后恢复为 `ACTIVE`，旧快照不能覆盖较新的状态；过期扫描或规则变化的路径移动摄入项标记 `PATH_MOVE_STALE_SNAPSHOT`。目录消失本身不会删除、撤回或隐藏已发布 Skill。
## 统一技能同步管理
产品方案见[统一管理入口](../../openspec/changes/add-github-skill-sync/unified-admin.md)，状态与一致性约束见[实施合同](../../openspec/changes/add-github-skill-sync/unified-admin-contract.md)。下列接口固定管理平台 `@github`，需要 `SUPER_ADMIN` 会话；写请求沿用 CSRF，返回既有 `ApiResponse` 包装并设置 `Cache-Control: no-store`。TEAM 继续使用 namespace 来源接口。
### 仓库列表与详情
根路径为 `/api/v1/admin/github-sync`：
| 方法与路径 | 查询参数与行为 |
| --- | --- |
| `GET /repositories` | `page=0`、`size=20`（1..100）、可选 `asOf`、`q`（最长200字符）、`originType`、`state`、`sourceStatus`。按 namespace 与 GitHub 稳定仓库 ID 合并候选和来源，筛选后分页。 |
| `GET /repositories/{repositoryId}` | 返回仓库摘要及最近检查/运行；仅其他 namespace 存在的仓库同样返回404。 |
| `GET /repositories/{repositoryId}/origins` | `page`、`size`、`asOf`；分页展示各发现方式的持久出处，按类型与来源键去重。 |
| `GET /repositories/{repositoryId}/paths` | `page`、`size`、`asOf`；复用逐 Skill 路径及审核安全投影，按需读取。 |
分页返回 `items / total / page / size / asOf`。非法枚举或分页参数返回400。`asOf` 是新增记录的时间截止点，防止新发现不断挤动当前分页，不代表跨请求冻结全部状态；同次列表请求的总数与列表使用一致事务快照。
仓库摘要字段：`repositoryId`、可空 `sourceId / candidateId`、`owner / name`、`firstSeenAt`、`state`、可空 `sourceStatus / candidateStatus`、`originTypes`。`state` 取 `REGISTERED / PENDING / VALIDATING / NO_VALID_SKILL / VALIDATION_FAILED / UNREACHABLE / BOUND_WITHOUT_SOURCE`，只表达登记阶段，不表示审核通过或市场可见。已登记来源状态另用 `ACTIVE / PAUSED / UNREACHABLE`。出处类型为 `CURATED_REPOSITORY / CURATED_OWNER / SEARCH_RULE / OFFICIAL_DISCOVERY / TEAM_CONFIG`，不会因此扩大到 TEAM 数据。
详情返回 `repository` 及可空 `latestRunId / lastCheckedAt / nextCheckAt / lastErrorCode`。出处返回 `originType / originKey / firstSeenAt / lastSeenAt / entryId / channel / queryId / runId / seedRepositoryId`；没有实际观测的字段为空，已清理详情的运行不提供失效链接。旧 `/sources` 和官方探索专用接口保留兼容。
### 指定仓库与运行
指定仓库重复添加返回同一 namespace、同类型、同目标的已有入口；不恢复其停用状态，不修改既有到期时间或重复排任务。尚未解析稳定仓库 ID 的入口在采集配置中展示，不计入统一仓库数量。
入口响应增加可空 `latestExecution`：`taskId / runId / status / errorCode / retryAt`，按持久任务的规则 ID 关联，不凭名称猜测。未知字段为空，不返回原始任务快照或外部异常正文。
既有 `GET /runs` 增加可选 `kind / status`，保留原分页调用。`kind` 为 `DISCOVERY / REPOSITORY_SYNC`，状态为 `QUEUED / LEASED / RETRY_WAIT / PARTIAL / ACQUISITION_COMPLETE / FAILED`。列表排除预览和已清理详情；`createdAt` 是创建时间，不是实际开始时间。
每项增加 `targetCount` 及可空 `target: {type, value}`：单目标显示摘要，多目标显示数量；同仓库优先按稳定 ID 去重，缺失时才按规范化坐标区分。HEAD_ONLY/FULL 仍在仓库任务详情中区分，不拆成虚构运行类型。
### 平台暂停与恢复
`GET /platform-control` 返回：
| 字段 | 说明 |
| --- | --- |
| `namespaceId`、`version` | 平台 namespace 与单调控制版本。 |
| `desiredPaused` | 管理员保存的暂停意图。 |
| `state` | `RUNNING / PAUSING / PAUSED`；RUNNING只代表没有暂停，不证明Go在线。 |
| `pendingStopCount` | 当前暂停版本仍未确认停止的在途任务数。 |
| `expiredUnconfirmedCount` | 上述任务中租约已过期但仍无停止确认的数量。 |
| `updatedAt` | 持久状态更新时间。 |
`PUT /platform-control` 请求示例：
```json
{"expectedVersion": 3, "paused": true}
```
请求使用最近读取的版本，响应结构同 GET。并发版本冲突返回409，重新读取后再决定操作；相同版本、相同意图不反复推进版本。变更记录管理员审计。
暂停独立于自动探索和容量策略，阻止平台的新发现、检查和下载，TEAM不受影响。Go执行线程到安全点后确认；过期或失联不能被当作已停止。Java已经接收的包及人工审核继续，恢复不覆盖各来源和配置独立启停。部署须配套支持 `platform-pause/v1` 的Go版本，详细边界见[内部协议](github-sync-internal.md#9-平台github采集安全暂停)。

322
docs/api/github-sync-internal.md Normal file
View file

@ -0,0 +1,322 @@
# GitHub 同步内部接口说明
> **用途：当前 Java–Go 内部协议说明。** Java 已实现按类型 `claim`、`renew`、系统凭证临时下发与当前租约的额度/认证失效/限流回报、发现候选批次接收与结算、候选及已登记来源的仓库任务调度、仓库 `HEAD_ONLY` / `FULL` 结算、分类 `fail` 与受控诊断码、逐包准入、上传授权、摄入回报、后台归档校验，以及已校验包到同步 Skill 版本和独立审核任务的持久交接；Go 已实现常驻双工作池、发现执行器、选择性包获取和仓库任务处理器，并接入可选的 Compose、Helm 与 K8s 部署入口。规则预览不创建来源或待审项；具体集成验收入口和边界见 [验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。`fail` 可在有效租约内同时回报系统凭证额度与受控诊断码；不持久原始异常文本。接口目标合同以同目录的 [OpenAPI YAML](github-sync-internal.openapi.yaml) 为准；产品行为、来源归属和审核规则以 [OpenSpec change](../../openspec/changes/add-github-skill-sync/README.md) 为准。本文解释流程，不另建一套协议。
## 1. 调用边界
- **调用方向：** Go 主动请求 Java。Java 保存配置、持久任务、租约、路径结果、包摄入与审核状态；Go 不连接 Java 数据库或 Redis，也不调用普通用户发布接口。
- **基础路径：** `/internal/github-sync/v1`，只供内部网络访问，不暴露为面向浏览器的公开 API，也不加入 Web SDK。
- **认证：** 所有接口使用独立服务身份 `Authorization: Bearer <service-secret>`。浏览器会话、普通用户/API Token、GitHub Token 均不能代替服务身份。本地流式上传还须提供 `X-Upload-Ticket`。
- **当前 Java 配置：** `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 接受逗号分隔的新旧服务密钥，每枚至少 32 个字符；不配置时内部接口拒绝所有请求。`SKILLHUB_GITHUB_SYNC_LEASE_DURATION` 初值为 `PT5M`，需在小范围联调中校准。
- **上传授权：** `SKILLHUB_GITHUB_SYNC_UPLOAD_TICKET_TTL` 默认 `PT5M`，最长一小时；S3/MinIO 的暂存 PUT 优先以 `SKILLHUB_STORAGE_S3_ENDPOINT` 签名，公开 GET 继续使用 `SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT`，本地存储使用 Java 同源相对路径与独立上传票据。Go 的 S3 目标主机须由部署方显式允许。回报前 Java 检查对象存在及实际大小；SHA-256 和包内容由后续摄入工作流再验证。
- **暂存清理：** `SKILLHUB_GITHUB_SYNC_SIGNED_UPLOAD_CLEANUP_GRACE` 默认 `PT1H`，`SKILLHUB_GITHUB_SYNC_UPLOAD_CLEANUP_INTERVAL_MS` 默认 `300000`。未被摄入接受且已失效的上传，以及已完成版本复制、无变化或校验终止失败的对象可以清理；S3 票据须先经过宽限期，删除失败保留领取状态供后续重试。仍在校验、等待版本绑定或因冲突阻断的对象不会按固定 TTL 清理。
- **包上限变化：** Java 在逐包回报时先核对已持久接收的幂等结果，再按当前包上限复核尚未接受的票据与实际对象大小。票据签发后上限调低或 S3/MinIO 实际对象超限时，回报 `TOO_LARGE`；仅在票据从未接收状态成功转为失败后尝试删除对象，避免并发回报删除已接收包。删除失败由暂存清理任务补偿，本地已上传对象也可立即进入补偿清理。
- **后台校验：** `SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_LEASE` 默认 `PT30M`，`SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_INTERVAL_MS` 默认 `5000`。Java 从 `RECEIVED` 领取单包，复核归档大小、SHA-256、ZIP 符号链接及条目边界，并复用现有 Skill 包校验器；通过后标记 `VALIDATED`，仍未创建版本或审核目标。失败留错误码，未接受或确定失败的对象由清理任务按状态处理。
- **后台发布交接：** `SKILLHUB_GITHUB_SYNC_PUBLICATION_LEASE` 默认 `PT4H`，`SKILLHUB_GITHUB_SYNC_PUBLICATION_INTERVAL_MS` 默认 `5000`。Java 独立领取 `VALIDATED` 包，锁定摄入记录并重新核对归档与内容指纹；同内容转 `NO_CHANGE`，新内容在同一数据库事务中创建 Skill 版本、来源路径绑定与 `GITHUB_SYNC` 审核任务，再转 `READY_FOR_REVIEW`。审核批准、拒绝或被更新目标取代后，摄入状态结算；待审前不会公开发布。归档按受限临时目录处理，真实容量与对象存储行为仍须联调。
- **许可证证据：** Go 从固定 commit 的 Tree 元数据挑选仓库根目录的 LICENSE、LICENCE、COPYING、NOTICE、COPYRIGHT 及常见后缀文件，最多 16 个；单文件最多 64 KiB、预览内容合计最多 256 KiB。超过预览限制的文件仍记录 Git blob 摘要与大小，不下载全文。Go 在 ZIP 中使用 `__skillhub_sync_evidence__.json` 清单及 `__skillhub_sync_evidence__/` 文件区传递证据；Java 重新校验大小和已携带文件的 Git blob 摘要，把仓库根证据纳入正式内容指纹并单独保存证据快照。这些内部条目不进入 Skill 安装包。缺失或无法预览的许可证信息保留为审核警告；当前不自动判定 SPDX 许可证，也不自动批准。
- **仓库观测周期：** `SKILLHUB_GITHUB_SYNC_REPOSITORY_CHECK_INTERVAL` 初值为 `PT24H`，只在后台尚未保存仓库目标周期时作为兼容回退。平台管理员通过 `/api/v1/admin/github-sync/schedule-policy` 配置仓库更新、精选用户或组织及广泛搜索的类别目标周期；`HEAD_ONLY` 和完整仓库任务完成后，Java 按当前后台值设置来源下次检查时间。
- **候选忽略：** 已登记来源的仓库任务快照携带 `ignorePatterns`，Go 在固定 commit 的树上只过滤候选 Skill 根，不删除父包内的普通文件。Java 对当前来源规则复核：新包准入返回 `PATH_IGNORED`，已接收包在摄入或发布前转为忽略终态。规则变化后旧任务不能推进来源的完整处理检查点；已发布内容不自动撤回。
- **调度周期：** `SKILLHUB_GITHUB_SYNC_DISCOVERY_SCHEDULER_INTERVAL_MS`、`SKILLHUB_GITHUB_SYNC_CANDIDATE_SCHEDULER_INTERVAL_MS`、`SKILLHUB_GITHUB_SYNC_SOURCE_SCHEDULER_INTERVAL_MS` 初值均为 `60000`。已登记来源的排队批次按 namespace 轮流选取到期来源，仓库任务领取优先已登记来源，并参考各 namespace 最近成功领取时间，避免单一 namespace 持续占用领取名额；候选来源仍按精选用户优先于搜索候选。Java 只在配置了水位且低于恢复边界时启动新候选树检查；已登记来源在反压期间按到期计划改做 `HEAD_ONLY`。仓库完整处理检查点要等待完整任务及其相关包校验结束，不随单次包回报提前推进。
- **仓库改名与转移：** Go 在仓库元数据请求中仅跟随 `https://api.github.com` 内的有限次数跳转，取得响应后先核对稳定的 repository ID，再通过当前任务租约调用 `repository-metadata`。Java 再核对 ID 并更新来源或候选坐标与任务快照；后续 HEAD、树和包获取使用 GitHub 返回的新坐标，市场链接优先使用来源的当前坐标。跨主机跳转不会携带系统凭证继续请求。旧名称被别的仓库占用时，ID 不匹配会阻断处理；旧地址的跳转失效后，当前方案不能仅凭旧坐标自动找回新地址，需由后续发现命中同一 ID 或管理员处理。跳转处理依据 [GitHub REST API 最佳实践](https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api#follow-redirects)；GitHub 也说明[转移后的旧地址可能被复用而失去跳转](https://docs.github.com/en/repositories/creating-and-managing-repositories/transferring-a-repository)。
- **历史清理：** `SKILLHUB_GITHUB_SYNC_COMPLETED_DETAIL_RETENTION` 默认 `P90D`，`SKILLHUB_GITHUB_SYNC_HISTORY_CLEANUP_INTERVAL_MS` 默认 `3600000`。调度器每轮限量清理已完成且无恢复引用的发现任务及空运行；候选入口归属独立保存，不因任务清理而丢失精选优先级或来源归属。已完成的仓库任务在包摄入均已结算且不再承担候选恢复后缩减快照和检查点；被更新结果取代的逐路径明细删除，来源仍需展示的最新路径状态保留。仓库任务/运行 ID 继续支撑已发布版本和审核追溯，过期运行不再出现在后台运行列表。未完成任务、待审或冲突项及其恢复数据不按期限清理；容器日志由部署环境管理。
- **逐包续跑：** 仓库任务领取响应的 `acceptedPaths` 来自同一任务、固定 commit 下已被 Java 持久接收的包；即使检查点尚未提交，Go 也跳过这些路径。`retryPaths` 标识需要重试的失败路径，即使位于检查点之前仍要处理。两者均由 Java 的逐路径持久状态计算，不由 Go 本地文件推断。两个独立 Go HTTP 客户端进程验证已接收包不重复上传；实际仓库 worker 再用受控 GitHub Tree 跨进程验证失败路径重访、Java 持久 `retryAt`、到期重领和终态结算。两条链路均在本地和 MinIO 模式通过；GitHub Tree 网络请求仍由受控 fixture 代替。
- **请求关联：** 每次调用均带独立 `X-Request-Id`；它只用于日志关联，不能充当幂等键。Go 到 Java 的 HTTP 客户端应与 GitHub 出站代理客户端分开。
- **协议版本：** 路径主版本为 `v1`，`claim` 请求还传 `protocolMajor: 1`。不兼容主版本返回 `426 PROTOCOL_UNSUPPORTED`；兼容的可选字段增加不得改变已有状态含义。
- **敏感数据：** `claim` 响应中的可选公开读取 GitHub Token、租约令牌、上传 URL/票据只在执行时使用，不能进入持久任务、日志、异常、指标标签或仓库 URL。服务身份与 GitHub 凭证是两种独立密钥。
以下示例中的 ID、SHA、路径和占位符只用于说明格式，不代表现有环境数据或可直接调用的服务。
## 2. 一次任务的调用顺序
```mermaid
sequenceDiagram
participant Go as Go 采集服务
participant Java as SkillHub Java
participant Stage as 临时对象存储
Go->>Java: POST /tasks/claim
Java-->>Go: 200 任务快照 + 当前租约 / 204 无任务
loop 任务仍在执行
Go->>Java: POST /tasks/{id}/renew
Java-->>Go: 新的 leaseExpiresAt
end
alt 发现任务
Go->>Java: POST /tasks/{id}/discovered-repositories
Java-->>Go: 202 候选和游标已持久化
else 仓库 FULL 任务
Go->>Java: POST /tasks/{id}/package-admissions（每个新包下载前）
Java-->>Go: 200 准入 ID / 409 积压暂停
alt 已获准
Go->>Java: POST /tasks/{id}/upload-authorizations（逐包）
Java-->>Go: 临时对象上传授权
Go->>Stage: PUT 单个 Skill 包
Go->>Java: POST /tasks/{id}/packages/report（逐包）
Java-->>Go: 202 摄入记录已持久化
else 积压暂停
Note over Go,Java: 保存路径进度，不开始下载下一个包
end
else 仓库 HEAD_ONLY 任务
Note over Go,Java: 只观测当前 commit，不上传包
end
Go->>Java: POST /tasks/{id}/complete 或 /fail
Java-->>Go: 任务采集状态已持久化
Note over Java: 后台另行复核、扫描、建待审目标；人工审核后才发布
```
`claim` 和 `renew` 建立当前租约代际。Go 的发现池与仓库池分别用 `kind` 领取；只在租约有效时进行上传授权、批次回报、逐包回报和任务结算，过期后须停止，等待 Java 重派。发现任务和仓库任务共用接口，但 `discovery`、`repository` 与 `mode` 由任务类型决定。仓库第一次领取时 `fixedCommitSha` 可为空，由 Go 固定本轮 commit；续跑时 Java 下发原固定 commit、`pathCheckpoint`、`incompleteSubtrees`、`retryPaths` 和 `acceptedPaths`。
发现批次须携带 `previousCheckpoint` 与新的 `checkpoint`。首批的前置值为 `null`；Java 在有效租约内原子保存候选、任务命中和新游标，相同批次重放不重复计数。任务快照中的 `candidateBudget` 限制当前租约新增线索数；Java 另以管理策略中的 `candidatePendingLimit` 限制该 namespace 中排队或正在验证的候选总数。前置游标已过期、同游标内容冲突或超出任一预算时拒绝整批，不能把搜索命中直接视为正式来源。重领任务时，`discovery.checkpoint` 由 Java 已持久游标覆盖原始快照；待验证候选未被消费前不会无限扩张。
发现任务调用 `complete` 时必须引用最后一批已确认的 `discoveryCheckpoint`。覆盖完整返回 `ACQUISITION_COMPLETE`；尚有未查页面或需延期的窗口时使用 `INCOMPLETE` 和未来的 `retryAt`，任务进入 `RETRY_WAIT`。这只结算搜索采集，不代表候选仓库已验证或 Skill 已发布。
`SEARCH_PREVIEW` 是发现池的独立任务快照。Go 对各结构化条件最多查询 GitHub 搜索第一页，经 `POST /tasks/{taskId}/preview-results` 回传每条件总量估计、不完整/千条上限信号与最多五个公开仓库样例；Java 在有效租约内原子写入任务快照并结算。预览不调用发现候选批次接口，不创建候选、来源、包或待审 Skill。Java→Go→Java 的两条件完成与第二条件限流场景已用受控搜索页联调：限流时只保存未来重试时间，不留下部分预览结果。各条件可能命中同一仓库，命中数不能直接相加；元数据搜索也可能漏召回。
`CURATED_REPOSITORY` 任务携带固定的 `owner` 与 `repositoryName`。Go 通过公开 GitHub API 解析 repository ID，最多回报一个仓库；Java 将首次解析的 ID 固定到精选入口，并直接登记平台来源。后续任务若解析为另一个 ID，Java 拒绝该批次。此类明确配置的来源不等待 Skill 候选验证；积压水位仍控制后续包准入。
`SEARCH_RULE` 的 `discovery.criteria.selectors` 是固定规则版本：每项为 `KEYWORD` 或 `TOPIC`，并标记 `CORE` 或 `EXTENSION`；`searchWindow` 固定本轮增量或全量重查时间范围。`KEYWORD` 仅搜索仓库名称、描述和 topics，`TOPIC` 限定 topic；每项单独查询并按 repository ID 合并。Go 不接受原始 GitHub 查询语法，不搜索 README 或代码；分页、GitHub 每查询最多 1,000 条的上限和 `incomplete_results` 必须纳入检查点与不完整结果处理。[GitHub 仓库搜索语法](https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories)、[REST 搜索限制](https://docs.github.com/en/rest/search/search)。
高水位期间，发现任务仍可在 GitHub 额度与 `candidateBudget` 内回报去重的仓库线索和游标；Java 暂不把这些线索转换为已验证来源或安排新候选仓库树检查。预算满时保存游标并延后。已登记仓库只执行 `HEAD_ONLY`；Go 不领取新的 `FULL` 任务，已领取的 `FULL` 任务在下一个包前须重新申请准入。低水位恢复后，Java 重新核对候选仓库与来源的当前上游状态，再安排包采集。
## 3. 接口一览
以下路径均相对于 `/internal/github-sync/v1`。所有请求都要服务身份和 `X-Request-Id`；`{taskId}`、`{uploadId}` 在 YAML 中定义为 UUID。
| 方法与路径 | Go 提交 | Java 的成功确认 | 主要边界 |
| --- | --- | --- | --- |
| `POST /tasks/claim` | `workerId`、`protocolMajor`、`kind` | `200 ClaimedTask`；无任务 `204` | Java 按发现/仓库类型原子领取，返回任务快照、租约和可选临时 GitHub 凭证。 |
| `POST /tasks/{taskId}/renew` | `lease`（代际、令牌） | `200 leaseExpiresAt` | 仅当前代际可续租；旧租约 `409`。 |
| `POST /tasks/{taskId}/preview-results` | 当前 `lease`、各条件命中估计与样例 | `200 ACQUISITION_COMPLETE` | 只允许预览任务，不写发现候选。 |
| `POST /tasks/{taskId}/credential-status` | 当前租约、凭证 ID、`OBSERVED`、`AUTH_INVALID` 或 `RATE_LIMITED`；`OBSERVED` 必须带额度，`RATE_LIMITED` 必须带 `retryAt` | `204` | 只有该租约领取时绑定的凭证可更新；认证失效和限流分别处理，旧 Token 不能误伤已替换的新 Token。 |
| `POST /tasks/{taskId}/discovered-repositories` | 当前租约、公开仓库候选批次、续跑 `checkpoint`、延后统计 | `202`，批次与游标已持久化 | 仅发现任务使用；搜索命中不是正式来源，更不是已发布 Skill。 |
| `POST /tasks/{taskId}/package-admissions` | 当前租约、固定 commit、Skill 路径 | `200 admissionId`；高水位 `409 BACKPRESSURE_ACTIVE` | 仅仓库 `FULL` 任务使用；Go 在获取文件内容前申请，Java 原子预留容量。 |
| `POST /tasks/{taskId}/upload-authorizations` | 当前租约、`admissionId`、固定 commit、Skill 路径、包大小与 SHA-256 | `200 UploadAuthorization` | 仅仓库 `FULL` 任务使用；授权绑定已准入包和临时对象。 |
| `PUT /uploads/{uploadId}/content` | 二进制包、上传票据 | `204`，本地临时对象已写入 | 仅 `LOCAL_STREAM` 模式使用；接收时执行大小硬上限。 |
| `POST /tasks/{taskId}/packages/report` | 当前租约、`admissionId`、固定 commit、Skill 路径、上传 ID、包大小与 SHA-256 | `202 ingestionId`、`RECEIVED` | Java 已持久接收摄入记录；准入占位转为未解决摄入项，不表示内容校验、审核或发布成功。 |
| `POST /tasks/{taskId}/complete` | 租约、覆盖情况、逐路径结果、观测 commit、可选额度/重试时间 | `200`，Java 权威任务状态 | `COMPLETE`、`INCOMPLETE`、`HEAD_ONLY` 的检查点推进规则不同。 |
| `POST /tasks/{taskId}/fail` | 租约、分类原因、可选 `retryAt`、额度与受控诊断码 | `200`，失败/重试状态已持久化 | 仅接受 `WORKER_TIMEOUT`、`WORKER_ERROR` 或 OpenAPI 所列的 GitHub HTTP 诊断码，不保存原始异常文本。 |
### 领取任务示例
Go 发送：
```json
{
"workerId": "github-sync-worker-1",
"protocolMajor": 1,
"kind": "REPOSITORY_SYNC"
}
```
Java 对仓库 `FULL` 任务的响应示意（省略可选的 `credential`）：
```json
{
"taskId": "11111111-1111-4111-8111-111111111111",
"runId": "22222222-2222-4222-8222-222222222222",
"kind": "REPOSITORY_SYNC",
"mode": "FULL",
"namespaceId": 33,
"sourceId": 44,
"lease": { "generation": 2, "token": "<lease-token>" },
"leaseExpiresAt": "2026-09-28T12:15:00Z",
"repository": { "repositoryId": 123456, "owner": "example", "name": "skills" },
"maxPackageBytes": 104857600,
"maxSingleFileBytes": 10485760,
"maxFilesPerPackage": 100,
"maxUploadBytes": 104857600
}
```
发现任务的 `discovery` 快照示意：
```json
{
"entryType": "SEARCH_RULE",
"ruleVersion": 3,
"criteria": {
"selectors": [
{ "kind": "KEYWORD", "value": "agent skills", "priority": "CORE" },
{ "kind": "TOPIC", "value": "agent-skills", "priority": "EXTENSION" }
]
},
"searchWindow": {
"mode": "INCREMENTAL",
"since": "2026-09-27T00:00:00Z",
"until": "2026-09-28T00:00:00Z"
},
"candidateBudget": 100
}
```
若响应为 `204`，Go 按运行配置退避后再轮询；不能把空队列记为同步成功。示例展示当前发布策略的默认值，部署的有效配置可改变下发上限。
### 逐包上传与回报
包下载前的准入请求示意：
```json
{
"lease": { "generation": 2, "token": "<lease-token>" },
"commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"skillPath": "skills/example"
}
```
Java 准入后返回 `{"admissionId":"66666666-6666-4666-8666-666666666666"}`；该 ID 仅随当前租约、commit 和路径有效。若返回 `409 BACKPRESSURE_ACTIVE`，Go 不获取该包的文件内容。
1. Go 从固定 commit 的树元数据枚举到一个 `SKILL.md` 根目录，完成路径及大小预检后，先调用 `package-admissions`。Java 原子检查全局与 namespace 积压及未完成准入占位，返回绑定当前任务、commit 和路径的 `admissionId`；高水位返回 `409 BACKPRESSURE_ACTIVE`。Go 在准入被拒时不下载文件，保存路径进度并结算部分任务。已获准且正在处理的包可以完成；租约失效后不得继续下载。
2. 准入成功后，Go 只把所需文件下载到受限本地临时目录。按当前有效发布政策预检，当前 `application.yml` 配置为最多 100 个文件、单文件 10 MiB、包总量 100 MiB；逐包打包并计算**上传对象**的 SHA-256 与字节数。此摘要用于传输完整性；正式 Skill 内容指纹由 Java 解包后按规范化文件计算。
3. Go 携带 `admissionId` 调用 `upload-authorizations`，取得 `uploadId`、`mode`、`uploadUrl`、到期时间和必要标头。S3/MinIO 模式使用短时预签名 PUT；本地模式调用受服务身份与上传票据保护的 Java 流式 PUT。Go 仅向部署配置允许的存储主机上传，不跟随重定向。
4. PUT 成功后，Go 携带 `admissionId` 调用 `packages/report`，只交 `uploadId` 与完整性信息，不提供任意下载 URL。Java 在短事务中记录任务、commit、规范化路径、对象引用和摘要，返回同一 `ingestionId`，并将准入占位转换为未解决摄入项。Java 用自己的存储适配器由 `uploadId` 定位临时对象，再核对实际对象、解包规则、许可证、正式内容哈希、安全扫描和审核目标。未使用的准入在任务结算或租约失效后释放。
5. Go 可以继续处理同仓库其他包，但每个包都要重新申请准入。单包失败记录在路径结果中，不能让其他包失去已完成进度。
取得文件之前，Go 先通过固定 commit 的 GitHub Tree 元数据找到候选 `SKILL.md`，按子树处理截断响应，并依据 blob 大小和类型做预检。`blob:none` 仍可能在本地保存整个仓库的树元数据，稀疏检出会按需获取所选文件；因此工作目录预算要包含 `.git`、检出文件和归档，不能只算最终 Skill 包。逐包回报 `202` 后清理可释放的本地文件；接近预算时用 `pathCheckpoint`、`deferralReason: LOCAL_DISK_BUDGET` 和 `retryAt` 回报阶段结果，清理本批目录，从同一固定 commit 续跑。不完整范围不触发上游删除判断。[GitHub Tree API](https://docs.github.com/en/rest/git/trees)和[Git partial clone 文档](https://git-scm.com/docs/partial-clone)分别说明了树响应截断与延迟获取对象的边界。
`LOCAL_DISK_BUDGET` 延后只适用于清理工作目录后仍能处理下一批的情形。若当前容量连固定 commit 元数据或一个合规包都无法处理，Go 以 `coverage: INCOMPLETE`、`deferralReason: CAPACITY` 回报容量阻塞，Java 等待容量或规则调整后的管理员重试；不能给同一条件下的任务设置极短 `retryAt` 形成领取循环。预检须包含文件落地、归档及安全余量，且工作区应有部署可验证的独立容量边界；Kubernetes 临时存储限制可能通过驱逐 Pod 生效，不能视作对每次写入的精确拦截。[Kubernetes 临时存储资源说明](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)。
逐包回报请求示意：
```json
{
"lease": { "generation": 2, "token": "<lease-token>" },
"admissionId": "66666666-6666-4666-8666-666666666666",
"commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"skillPath": "skills/example",
"uploadId": "55555555-5555-4555-8555-555555555555",
"sizeBytes": 4096,
"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
}
```
S3/MinIO 预签名 PUT 的 `maxBytes` 是 Go 应遵守、Java 随后复核的边界，不能仅凭 PUT 签名宣称对象存储接收时已强制最大字节数。临时对象须存放在隔离区域并有容量告警、超额清理；本地流式入口须在接收时执行硬限制。已被 Java 接收但尚未处理的对象不能只按固定 TTL 删除；失败对象需有清理与重采集记录。
### 暂存容量监控
Java 每隔 `skillhub.github-sync.upload-cleanup-interval-ms` 采样一次暂存账面预留量与本地磁盘容量。`skillhub.github.sync.staging.reserved.bytes` 汇总尚未清理的上传票据所声明的字节数；它不能反映绕过客户端限制的超额 PUT。使用本地存储时，`skillhub.github.sync.staging.local.available.bytes` 是存储目录所在文件系统的可用字节数；低于一个允许的最大 Skill 包大小或读取失败时，`skillhub.github.sync.staging.local.capacity.warning` 为 `1`，并输出警告日志。
S3/MinIO 模式下，Java 默认每小时以最多 100 次列举请求、10,000 个对象的上限扫描 `github-sync/staging/` 前缀。完整扫描时，`skillhub.github.sync.staging.s3.used.bytes` 与 `.s3.objects` 记录实际字节数和对象数，`.s3.scan.complete` 为 `1`；扫描不完整或读取失败时，实际字节数为 `-1`，`.s3.capacity.warning` 为 `1`，防止把截断结果误判为有余量。部署方须在启用 S3 同步前配置 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES` 为该暂存前缀可使用的正数字节预算；达到预算或未配置预算且暂存区已有对象时也告警。没有同步对象时不会仅因预算未配置而告警。`SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_SAMPLE_INTERVAL_MS` 可调整采样周期，默认 `3600000`。这项预算只约束告警判断，不代表对象存储服务的真实剩余空间或上传硬配额；仍需按存储服务的总容量/配额监控其余对象。指标是否通过 Actuator 暴露取决于部署的端点配置。
### 完成与失败
`complete` 的 `coverage` 明确三种情况：
| 值 | 允许的效果 |
| --- | --- |
| `COMPLETE` | 固定 commit 的树枚举完整，逐路径结果已持久化；仅在所有需处理路径均有持久采集结论且无待重试范围时，Java 可推进 `lastFullyProcessedCommit`。这仍不表示审核通过。 |
| `INCOMPLETE` | 树枚举或包采集未覆盖全部目标时，保存已完成路径、失败子树或 `pathCheckpoint`、延后原因和重试时间；不得据此判断上游删除，也不能推进完整处理检查点。 |
| `HEAD_ONLY` | 只保存 `lastObservedCommit`，用于审核积压时的低成本观测；不得上传包或推进 `lastFullyProcessedCommit`。 |
`FULL` 仓库任务的 `pathCheckpoint` 是按字典序处理过的最后一个规范化 Skill 根路径；Java 拒绝倒退。`INCOMPLETE` 保存失败子树和待重试路径，并在重新领取时返回固定 commit、检查点与重试目标。`COMPLETE` 若仍有活动包准入或待重试路径，返回 `409 COVERAGE_CONFLICT`；已持久接收的包可用 `ingestionId` 核对，但不能仅靠结算请求伪造 `ACCEPTED`。存在终态失败路径时任务状态为 `PARTIAL`，该路径保留错误记录。
发现任务的 `complete` 还应保存发现游标和额度情况。任何 `retryAt` 均由 Java 持久调度；Go 不长时间占租约等待 GitHub 额度恢复。若来源在扫描期间变为不可读，使用 `fail` 分类报告，不以 `complete` 掩盖失败。
## 4. 幂等与恢复
| 场景 | Java/Go 约定 |
| --- | --- |
| Go 停机或租约到期 | Java 按持久检查点重派，生成新租约代际；旧执行者收到 `409 LEASE_EXPIRED` 或 `LEASE_STALE` 后停止。 |
| 发现任务提交批次后退出 | Java 先持久保存候选和发现检查点；新 Go 进程在下一代租约中读取检查点，重放同一批次仍只保留一条批次及候选命中，然后完成任务。应使用 Java HTTP、PostgreSQL 与两个独立 Go 进程验证。 |
| 同一个包回报重复送达 | 同任务、commit、规范化路径和相同摘要返回原 `ingestionId`；不同摘要返回 `409 RESULT_CONFLICT`。 |
| Java 已返回 `202` 后后台处理失败 | Go 不因审核耗时重复上传；Java 从摄入记录恢复。对象损坏或丢失时，Java 明确安排该路径重采集。 |
| Java 校验或发布进程在领取后退出 | `VALIDATING` 与 `VALIDATED` 分别保留持久租约及尝试次数；租约过期后其他 Java 实例可重新领取同一摄入项。过期校验令牌不能回写结果，发布仍以同一 `ingestionId` 幂等交接。应分别验证独立 JVM 退出与完整应用容器重启恢复。 |
| 同内容在新轮次出现 | Java 计算正式内容/合规指纹后决定无变化，不重复创建版本或待审项。Go 不自行判定 `UNCHANGED`。 |
| 未完整枚举目录 | 已完整部分继续处理，失败子树单独续跑；绝不据此删除既有 Skill。 |
`ACQUISITION_COMPLETE` 是仓库采集任务状态；`RECEIVED` 是包摄入状态；审核与发布属于现有 Skill 生命周期。详细状态转换见 [状态与检查点设计](../../openspec/changes/add-github-skill-sync/state-model.md)。
## 5. 错误处理
错误体包含 `code`、可脱敏展示的 `message`，可重试时可包含 `retryAt`。Go 必须按 `code` 和当前任务状态决定停止、重领、延期或报告，不能把所有非 2xx 统一重试。
| HTTP / code | 处理原则 |
| --- | --- |
| `401 AUTH_INVALID` | 服务身份失效；停止领取并告警，不尝试普通用户或 GitHub Token。 |
| `426 PROTOCOL_UNSUPPORTED` | 协议主版本不兼容；停止该实例领取，按同版本镜像和合同排查。 |
| `409 LEASE_EXPIRED`、`LEASE_STALE` | 当前执行者失去任务所有权；停止回报，等待重新领取。 |
| `409 COMMIT_MISMATCH`、`RESULT_CONFLICT` | 不覆盖已有结果；保留诊断信息，重新读取 Java 任务快照或交人工处理。 |
| `409 BACKPRESSURE_ACTIVE` | 停止启动后续包下载，持久回报部分路径进度；不当作包失败。Java 到低水位后重新安排当前上游版本。 |
| `413 PACKAGE_TOO_LARGE` | 记录该包终态失败，其他包继续；不能上传截断包。 |
| `422 PATH_INVALID`、`OBJECT_INVALID`、`DIGEST_MISMATCH` | 拒绝该对象或路径；核对打包/传输原因，不创建正式版本。 |
| `GITHUB_RATE_LIMITED`、`RETRYABLE_FAILURE` | 保存 GitHub 指示的 `retryAt` 或可恢复失败信息，由 Java 延后任务；不占租约空等。 |
| `SOURCE_UNREADABLE` | 与限流区分；按来源不可达复查策略运行，已发布 Skill 保持原样。 |
GitHub 自身的 HTTP 状态并不直接等于内部接口 HTTP 状态；Go 应通过 `/fail` 回报分类、额度和可重试时间。`403/429` 可能涉及权限或限流，需结合 GitHub 响应信息分类；不能通过不断轮换凭证规避限流。
## 6. 联调边界与待定参数
首轮联调应覆盖：无任务 `204`、领取后停机重派、旧租约回报、同包重复回报与摘要冲突、完整/不完整树、`HEAD_ONLY`、本地与 S3/MinIO 上传、包超限、Go 回报 `202` 后 Java 重启、限流和来源不可读。可观察结果应分别落在任务、包摄入、审核记录中；不能只看任务接口返回 `200` 就断言 Skill 已发布。
已确认首期自动发现只用仓库搜索、精选入口和后续仓库树验证，不做全站代码搜索；`criteria` 已锁定为独立的关键词/topic 条件与核心/扩展优先级，搜索窗口随任务固定。包限制沿用当前有效发布政策、广泛搜索默认每 7 天全量重查一次、已完成运行/任务详情默认保留 90 天；未完成任务与恢复数据不按此期限清理，容器日志由部署环境管理。租约和轮询值以当前绑定配置为准；全平台/单 namespace 未解决同步项的高低水位由管理员配置，候选预算初值见管理接口。首期 S3/MinIO 采用上传前 Go 预检和 Java 摄入前复核；若将来要求对象存储接收时硬拒绝超限对象，再验证条件上传方案。字段变更须先更新 OpenAPI YAML，再同步修改本文示例和对应实现。
## 7. 采集服务在线报告
Go 服务启动后每 30 秒调用 `POST /internal/github-sync/v1/heartbeat`，包含现有机器身份 Bearer 凭证，不传 GitHub Token、主机信息或任务内容。Java 返回 `204`，仅记录最近一次成功心跳。心跳独立于发现和仓库工作池，长任务运行期间仍会发送；调用失败不会中断在途任务。平台管理员通过 `GET /api/v1/admin/github-sync/worker-status` 查看最近在线时间。该时间表示 Go 与 Java 最近连通，不代表 GitHub 出站访问成功。
## 8. 官方深度探索合同与当前接线状态
[深度探索增量说明](github-deep-discovery.md) 定义新管理接口、能力协商、固定策略快照、多渠道出处、检查点和暂停边界。同目录 OpenAPI 标有 `x-implementation-status: proposed-not-implemented` 的字段/路径是待实现设计，不能视为当前 Java/Go 已支持。
保留现有任务 kind、legacy discovery.entryType、状态/错误枚举和批次202空响应。新 claim 可选 capabilities 缺省为空；只有完整声明官方所需能力的实例才能领取含 officialDiscovery 的任务。V119起平台任务还统一要求 `platform-pause/v1`，因此空能力请求不再领取平台任务，TEAM规则不变。旧空心跳继续有效，但不能证明官方能力在线；实例/能力心跳是新增持久化工作。配置在领取和关键接收时按最新状态复核；暂停不影响已登记来源与其他独立启用入口，已接收进度不回滚。
发布时先实现 Java 的能力过滤再启用官方任务；回滚到旧 Java 前须隔离或排空官方任务，不能只关管理页面。旧协议的联调结果不覆盖这些新增行为；实现后再分别验证 Java、Go、管理页面与跨进程恢复。
官方独占候选的仓库暂停也使用新增 `official-pause`，仅要求 `official-pause/v1` 控制能力，不要求官方发现快照。新端点支持尚未取得commit时的空SHA/空路径进度，以及已固定commit的路径与已接收包进度；原 `complete` 对commit和未来retryAt的要求保持不变。任务途中成为正式来源或另有独立启用入口时返回新端点专用CONTROL_CHANGED，重新查询控制状态后继续。详见增量说明第8节。
### 官方执行与分片
Java已接通配置、版本化目录、持久query、能力领取、官方批次/检查点、发现与仓库任务control/pause/recovery，以及顺序时间分片。仓库临时隔离已由最新enabled/排除/独立入口资格替换。来源自身PAUSED返回SOURCE_PAUSED，已绑定来源不受官方开关影响。
SPLIT原子登记两子范围，预算为剩余深度0..32、root剩余累计child数0..126；全叶COMPLETE才推进根覆盖。官方GAP以INCOMPLETE请求结算为PARTIAL并轮转兄弟，普通IN_PROGRESS延期仍为RETRY_WAIT。GAP ledger保留原checkpoint、retryAt和租约代数；到期复用终结任务，或在历史已清理时从ledger恢复。旧自定义完成语义不变。详见增量说明第14节。
生产Go工作池现声明四项官方能力，Java配置仍默认关闭。首次启用前必须全部Java与Go升级并完成迁移和集成验收；三个v1能力名不能区分旧Go的16-child上限和新126预算，不能混跑旧显式opt-in worker。策略generation也不能保护旧Java接收器。作者扩展见增量说明第15节；统计口径见该说明第5节，协议文档本身不代表运行验收通过。
### 作者扩展（V113）
metadata可选ownerId；OWNER快照另要求official-owner/v1，只由实际验证且具有独立非OWNER官方出处的seed触发一层扩展。OWNER_RECHECK不使用时间分片和pushed覆盖。其GAP完成保持RETRY_WAIT，页码为正int32、按租约预算续页；10页是单租约上限。CORE请求预算包含身份解析与末页身份复核，完整结果只记录作者复查时间。详见[增量说明第15节](github-deep-discovery.md#15-有效作者扩展v113)。
### 官方实例心跳
带 `workerId`、`protocolMajor=1`、`capabilities` 的心跳按实例持久记录服务端接收时间；空 body 或 `{}` 保留旧全局联系行为，不证明具备官方能力。后台根据完整官方能力集合（含作者扩展）与最近 90 秒联系统计兼容实例；配置启用、实例可联系、业务最近进度是三个不同事实。没有兼容实例但近期有旧式联系时为 `INCOMPATIBLE`；全部联系超时为 `UNREACHABLE`；从未联系为 `NEVER_SEEN`。
实例记录按既有 `completed-detail-retention` 和 `history-cleanup-interval-ms` 清理，每次最多 10 批、每批 500 行；不删除任务、检查点或旧全局联系记录。实例心跳本身不证明多实例并发业务正确。
### 额度观察与共享冷却（V116）
`POST /tasks/{taskId}/official-quota`以有效官方租约接收`{lease, observation}`，无credentialId亦可用于匿名请求；旧credential-status必填保持不变，附加官方观察已接线。PRIMARY按资源冷却，SECONDARY部署级共享退避；未知主体不聚合余额。control新增QUOTA_COOLDOWN及retryAt，pause根据当前门控延期，既有准入可结算。Go持久回报失败停止后续GitHub请求并进程共享退避。具体时间/状态边界见[增量说明第16节](github-deep-discovery.md#16-额度观察与共享冷却v116)。跨进程并发限流需使用真实依赖独立验证。
## 9. 平台@github采集安全暂停
合同见[统一管理实施合同](../../openspec/changes/add-github-skill-sync/unified-admin-contract.md)。新增能力 `platform-pause/v1` 覆盖所有平台采集任务，包括普通发现、搜索预览、官方探索和仓库检查采集；TEAM 不受该平台开关控制。新 Java 不向缺少此能力的旧 Go 领取者发放平台任务；受控任务的 claim 返回 `platformControlVersion`，TEAM 此字段为空。
### 读取任务控制
`POST /internal/github-sync/v1/tasks/{taskId}/platform-control`：
```json
{"lease":{"generation":2,"token":"<领取时取得的租约令牌>"}}
```
返回 `{"action":"CONTINUE","platformVersion":3}` 或 `{"action":"PAUSE","platformVersion":4}`。Go 在网络请求和包处理安全点检查，并以5秒为目标间隔观察进行中的任务。仅有效租约或当前暂停版本捕获的原租约证明可以读取；过期的当前暂停证明只用于安全停止，不允许继续采集。
### 安全停止确认
`POST /internal/github-sync/v1/tasks/{taskId}/platform-pause`：
| 字段 | 说明 |
| --- | --- |
| `lease` | 原领取的代际和令牌。 |
| `observedVersion` | 最近读取到的暂停版本。 |
| `kind` | `DISCOVERY` 或 `REPOSITORY_SYNC`，必须与任务匹配。 |
| `discoveryCheckpoint` | 发现任务最后已被 Java 确认的游标；从未保存批次时为空。 |
| `repositoryProgress` | 仓库任务的 `observedCommitSha / pathCheckpoint / incompleteSubtrees / pathResults`；复用仓库进度验证，不能推进完整扫描边界。 |
发现任务不得携带仓库进度。仓库任务必须携带进度对象；尚未取得 commit 时 SHA 和路径检查点为空，数组为空，并且服务端也必须尚未绑定 commit 或准入。已取得 commit 的 FULL 任务复用现有结果与摄入校验；HEAD_ONLY 不接受包路径结果。成功返回 `{"status":"PAUSED"}`，相同确认可以幂等重试。
只有执行线程停止外部工作、完成必要结算与本地清理后才能确认，不能让控制轮询协程携带旧检查点代为确认。有效租约可原子保存已确认进度并释放任务；过期或已结算任务只能以当前暂停捕获的证明确认停止，不能写新进度，也不能影响新一代租约。恢复后旧版本返回 `409 CONTROL_CHANGED`，需重新读取；证明失效返回 `409 LEASE_STALE`，进度或类型冲突沿用 `RESULT_CONFLICT / CHECKPOINT_CONFLICT / COMMIT_MISMATCH`。范围不匹配返回 `409 PLATFORM_SCOPE_MISMATCH`；缺少 lease 返回 `422 LEASE_REQUIRED`，非法JSON或参数返回 `400 INVALID_REQUEST`。
### 状态与接收边界
Java持久保存平台暂停意图，claim和新包准入与暂停共享事务门控。租约过期或心跳丢失不等于已停止，当前暂停版本尚有未确认任务时管理端保持 `PAUSING`；保留期清理也不能删除这些记录来伪造 `PAUSED`。恢复递增控制版本，不强制等待失联 worker，但不允许旧代际回写新工作。
Java已接受摄入的边界是 `acceptReport` 事务成功，不是对象存储 PUT 完成。暂停前已完整打包且有有效准入/租约的包允许结算；不完整本地包清理并释放准入。已接受包后续校验、安全扫描、待审创建与人工审核继续运行。平台暂停不修改各发现入口的独立启停，恢复时按既有状态及持久进度继续。
运行证据与未完成范围见 [OpenSpec 验收记录](../../openspec/changes/add-github-skill-sync/verification.md)，接口定义本身不等同端到端验收通过。

1508
docs/api/github-sync-internal.openapi.yaml Normal file
View file

File diff suppressed because it is too large Load diff

111
docs/github-sync-operations.md Normal file
View file

@ -0,0 +1,111 @@
# GitHub 公开 Skill 同步：管理员运行手册
本功能只采集 GitHub 公开仓库。SkillHub Java 保存配置、来源、任务、审核和发布状态；独立 Go 服务轮询 Java 的内部接口执行发现与采集。Go 停止后，Java 中已保存的任务和检查点仍在；重新启动 Go 后由租约机制继续处理。功能仍在实施验收阶段，生产启用前须完成 [OpenSpec 验收任务](../openspec/changes/add-github-skill-sync/tasks.md) 中未勾选的部署检查。
## 启用前配置
1. 检查 `@github` namespace 和 `github-sync-publisher` 账户没有被现有用户数据占用；V76 迁移遇到冲突会失败并回滚，见[部署文档](09-deployment.md)。
2. 在 Java 与 Go 两侧配置同一条至少 32 字符的服务密钥。Compose 分别使用 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS`（Java，可配置轮换窗口内的多枚）和 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN`（Go，单枚）。Helm 的 `secrets.githubSyncServiceToken` 或 `existingSecret` 中的 `github-sync-service-token` 会注入两侧。密钥放在受保护的环境文件或 Kubernetes Secret，不要放入命令参数、仓库或日志。
3. 在管理后台先设置全平台、单 namespace 的未解决项高低水位，再启用采集。新部署的水位为空，Java 此时拒绝新的逐包准入。候选预算初值为每次发现租约 100、每 namespace 待验证 1000；水位和预算都应结合试运行调整，字段与范围见[管理接口说明](api/github-sync-admin.md#积压准入水位)。
4. 如需保存系统级 GitHub Token，Java 还须配置 `SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS` 与 `SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION`。不配置时仍可匿名读取公开资源，但后台不能保存 Token。只为公开读取创建低权限凭证，不授予私有仓库或写入权限；具体访问能力以 [GitHub 官方端点权限说明](https://docs.github.com/en/rest/authentication/permissions-required-for-fine-grained-personal-access-tokens) 和实际受控请求为准。Token 在 Java 侧加密保存，Go 只在领取任务时获得当次凭证；更换加密密钥时，旧版本须保留到旧密文完成替换。
5. Go 的临时目录必须是独立受限目录。默认单仓库工作区 512 MiB、总工作区 512 MiB、保留空闲 128 MiB，部署提供 768 MiB 内存型临时卷。增加仓库并发数时，同时检查 `REPOSITORY_WORKERS × MAX_WORKSPACE_BYTES ≤ TOTAL_WORKSPACE_BYTES`，且临时卷至少容纳总预算与保留空闲。同一目录一次只供一个 Go 进程使用；多实例各自使用独立临时目录。S3/MinIO 模式还须设置 Go 的 `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` 和 Java 的 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES`。
6. Java 校验摄入包和生成待审版本时也会使用自己的系统临时目录存放 ZIP 与解压文件。每次处理前会检查至少有“当前 ZIP 大小 + 当前发布包大小上限”的可用空间；这只是单次检查，不预留并发处理容量。部署时要让 Java 临时目录可写，并为同时运行的校验和发布留足空间；它不使用 Go 的专用临时卷。
在 staging Compose 中，Java 的 `SKILLHUB_STORAGE_S3_ENDPOINT=http://minio:9000` 用于生成 Go 可访问的暂存 PUT 签名，`SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://localhost:9000` 用于宿主机访问的下载链接。Go 的允许主机填写 `minio:9000`；不要把只供宿主机访问的 `localhost:9000` 当作 Go 容器内的对象存储地址。
Go 镜像由 `tini` 作为 PID 1 转发停止信号并回收 Git 子进程。停机后以 Java 持久租约和检查点恢复；需分别验证本地存储与 MinIO 模式。
发布版 Compose 在启动 Java 前运行一次 `storage-permissions`，把命名卷的挂载根目录设为镜像中 `app` 用户的 UID/GID `100:101`；Docker 新建卷会覆盖镜像内目录的属主。基础 K8s Deployment 和 Helm 使用 `fsGroup: 101` 处理本地存储 PVC；实际存储驱动仍须支持该权限策略。卷权限不正确会导致暂存包上传失败，部署验收必须检查非 root 用户的实际写入能力。
Go 到 Java 的内部 HTTP 连接默认直连。跨主机部署时要在可信网络和 TLS 边界内传输服务密钥与当次 GitHub Token；本功能不要求额外部署 mTLS、队列或 KMS。Compose 从 `.env.release` 的 `SKILLHUB_GITHUB_SYNC_HTTP_PROXY`、`SKILLHUB_GITHUB_SYNC_HTTPS_PROXY`、`SKILLHUB_GITHUB_SYNC_NO_PROXY` 注入标准代理变量；Helm 对应 `githubSync.httpProxy`、`githubSync.httpsProxy`、`githubSync.noProxy`。代理只用于 GitHub 出站访问，不用于内部 Java 请求或对象存储暂存 PUT。
## 官方深度探索的启用与观察
管理员统一入口为 `/dashboard/github-sync`，打开“采集配置”设置自动探索；旧 `/dashboard/github-deep-discovery` 链接会转到这里。自动探索默认关闭。先完成 Java 的数据库迁移及全部 Java/Go 实例升级，再由管理员启用；不要让早期显式声明官方能力但不理解新分片预算的 worker 混跑。Go 生产入口自动声明官方探索及 `platform-pause/v1` 能力，不需要另加探索环境开关。管理员保存的是启用意图，不是立即完成一次搜索的命令，页面没有“执行一次”按钮。
启用后的正常链路：Java 周期性生成到期查询任务，Go 轮询领取，分页搜索并回报候选；Java 保存去重仓库、多个发现出处与检查点，再通过既有仓库采集链路检查 Skill、生成版本并进入独立同步审核队列。发现仓库数量不等于 Skill 数量，也不等于已发布数量。
- **概览与采集配置**：查看准备状态、等待原因和最近业务进度；探索配置集中在抽屉中。服务“已联系”只说明心跳有效，不能据此判断 GitHub 请求正在执行。
- **仓库**：统一查看自动探索、指定仓库及其他配置发现的仓库，按登记阶段、来源状态、出处和仓库坐标筛选；同一个 GitHub 仓库合并展示多个出处。服务端分页，默认每页20条。新结果不会自动打乱当前页，使用刷新查看。超过允许浏览范围时缩小筛选；总数不是当前已下载的条数。
- **详情抽屉**：按需查看来源出处、Skill 路径结果，以及已有来源、运行和审核入口。关联运行被保留期清理后可能无法再打开，不表示仓库或版本被删除。
- **资源预算**：节省、均衡、加速都保留同样渠道。实际预约规则和试运行初值见[设计第3节](../openspec/changes/add-github-skill-sync/deep-discovery-design.md#3-预算凭证与暂停)。预算不代表 GitHub 允许额度，也不统计后续全部下载成本；主额度、共享冷却与容量保护仍可使采集暂停。
- **多个 Token**：用于同一系统级凭证池。不能把显示余额简单相加，也不能保证每枚 Token 有独立额度。池内凭证全部不可用时等待修复，不自动退回匿名继续冲击请求。
关闭探索后保留进度、候选和出处，暂停官方新发现及仅由它触发的未登记候选采集；已登记来源和其他独立启用入口按自身开关继续。重新开启由 Go 续领持久进度，无需手工创建运行。到达预计重试时间只表示任务可能可领；Go 停机时仍不会执行 GitHub 请求，恢复联系后才能继续。
只需停用官方新发现时关闭自动探索；需要暂停整个平台采集时使用“暂停平台采集”，观察在途任务到安全边界确认。已接收包可完成结算，不删除版本或历史作为回退。回退到不理解官方任务的旧 Java 前，需要隔离或排空官方队列并停止对应 worker；单纯关闭页面不能约束旧版本服务。完整反向版本回退仍在验收范围内，不推荐直接降级数据库。
验证范围见[验证说明](../openspec/changes/add-github-skill-sync/verification.md)。受控 GitHub fixture、合成仓库性能数据与公网采集结果必须分别记录；不得把测试候选当作真实探索成果。
## 从源码构建 Java 镜像
从仓库根目录使用 `server/Dockerfile` 构建时，容器不会自动读取宿主机的 Maven settings。需要构建期 Maven 镜像源、代理或仓库认证时，可通过可选的 BuildKit secret 提供受保护的 settings 文件：
```bash
docker buildx build --load -f server/Dockerfile -t skillhub-server:local \
--secret id=maven_settings,src="${MAVEN_SETTINGS_FILE:?请先设置受保护的settings.xml文件路径}" \
server
```
`MAVEN_SETTINGS_FILE` 指向部署方自行维护的文件。无需自定义 settings 时去掉 `--secret` 参数，沿用 Maven 默认配置；提供时仅在两条 Maven 构建指令期间挂载到 `/root/.m2/settings.xml`，不通过 ARG、COPY 或最终镜像保存。此配置不替代 Docker 拉取基础镜像或 Maven Wrapper 下载自身发行包的网络配置。Secret 用法见 [Docker 构建密钥说明](https://docs.docker.com/build/building/secrets/)。
源码镜像执行 `clean package`，防止共享构建上下文中的宿主 `target/` 产物被误当成本次源码编译结果；`Dockerfile.dev` 仍用于明确选择预构建 JAR 的流程。
## 启动、停用和恢复
本地开发使用宿主机 Java 与 Go：在同一 shell 导出匹配的 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 和 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN` 后，先运行 `make dev-all`，再在另一个终端运行 `make dev-github-sync`。第二个终端需继承同一枚 Go 服务密钥；该目标构建并前台启动 Go，不修改开发 Compose 的依赖服务。默认临时目录为 `.dev/github-sync-tmp`，工作区、总预算和保留空闲值可用对应环境变量调整。
Go 启动时独占临时目录，只清理上次异常退出遗留的 `skillhub-github-package-*` 包工作区；其他文件不会被清理。若目录已被另一 Go 进程占用，启动会失败并输出 `TEMP_WORKSPACE_UNAVAILABLE`，应检查重复实例或目录配置。清理本地遗留包不改变 Java 持久化的任务、固定 commit 和路径检查点。
发布版 Compose 在受保护的 `.env.release` 已完成配置后，先只检查合并配置，再启动独立 Go 服务：
```bash
docker compose --env-file .env.release -f compose.release.yml --profile github-sync config --quiet
docker compose --env-file .env.release -f compose.release.yml --profile github-sync up -d github-sync
docker compose --env-file .env.release -f compose.release.yml --profile github-sync ps github-sync
```
临时停止领取新任务和恢复轮询：
```bash
docker compose --env-file .env.release -f compose.release.yml --profile github-sync stop github-sync
docker compose --env-file .env.release -f compose.release.yml --profile github-sync up -d github-sync
```
停用某条发现规则或指定入口，使用后台“技能同步 → 采集配置”的启停操作；这不会删除已登记来源或已发布 Skill。TEAM 来源在对应 namespace 管理页启停。停用后，已排队或等待重试的发现任务不会再领取；已经取得租约的任务不会被强制中断。恢复入口后，原等待任务可继续领取。Helm 默认 `githubSync.enabled: false`，启用和 Secret 字段见 [Chart 说明](../charts/skillhub/README.md#可选的公开-github-采集服务)；基础 K8s Deployment 默认不在 Kustomize 清单中，见 [K8s 说明](../deploy/k8s/README.md)。
## 后台操作顺序
1. 打开“技能同步”，日常页面只有“概览、仓库、运行记录”三个页签。先看概览的准备清单：采集服务、准入水位和采集入口分别显示实际状态；凭证可选，读取失败或未知不会显示为就绪。页面不会替管理员自动启用探索。
2. 点击“采集配置”打开抽屉。自动探索设置启停、预算及排除项；指定仓库只需填写 GitHub URL 或 `owner/repository`，固定进入 `@github`。重复添加返回已有入口，不重新启用已停用配置。尚未解析仓库 ID 的配置留在抽屉中展示执行状态，不虚增仓库数量。“其他采集方式”折叠展示指定账号和自定义规则；搜索预览沿用既有 Go 执行机制，预览不创建正式来源。首期不采集私有仓库。
3. 在“仓库”查看合并后的登记与采集状态，进入抽屉按需加载出处、Skill 路径、关联运行与审核。注册为来源不代表已发布。首次发现后的 HEAD 检查只记录仓库提交，后续 FULL 任务才逐包采集；相同提交的周期任务只检查 HEAD。提交、忽略规则变化或既有手动重扫/路径重试仍受水位、额度及重试约束。部分目录失败不会丢弃其他已接收包。
4. 在“运行记录”按类型和状态筛选。真实运行类型是“发现”与“仓库检查与采集”；HEAD_ONLY/FULL 阶段在详情中区分。列表时间为创建时间，单目标显示安全摘要，多目标显示数量；发现运行可能没有 Skill 路径结果。运行历史与仓库当前状态不是同一时间口径。
5. 点击“系统设置”打开容量保护、采集周期和系统凭证配置。Token 仅在添加表单中输入，关闭后清空，不回显明文。同步审核保持独立队列，未通过审核的新版本不会成为市场最新版本。
6. TEAM 管理员继续在所属 namespace 管理公开来源；本平台页面不新增 TEAM 管理入口。TEAM Skill 默认 `NAMESPACE_ONLY`，`@github` 已发布 Skill 全局可见。目录消失、部分扫描、来源暂停和不可达都不会自动删除或撤回已发布版本。
### 平台暂停与恢复
“暂停平台采集”只约束 `@github`，不影响 TEAM，也不修改各采集入口自己的开关。暂停后阻止新任务领取及新包准入；Go 将在途任务停在安全边界，保存确认进度后回报。尚有任务未确认时显示“暂停中”；租约过期或服务离线不会被当作已经停止。全部确认或原本没有在途任务时才显示“已暂停”。界面同时展示尚未确认和过期未确认数量。
已由 Java 正式接收的包可继续校验、安全扫描并进入审核；仅上传到暂存存储不等于已接收。暂停期间仍可保存配置，恢复后按原进度和独立启停状态继续。恢复采集只改变控制意图，不代表 Go 已在线或任务已开始。若服务离线，先恢复服务连接；不要靠删除任务消除“暂停中”。
V119 引入持久平台控制与任务确认记录。Java/Go 必须配套升级：新 Java 只向声明 `platform-pause/v1` 的 worker 发放新的平台任务，旧 Go 会等待升级，TEAM 沿用原规则。在途旧 worker 无法证明安全停止时保持待确认；升级前应安排其有界停止。停止观察目标间隔为5秒，实际确认时间仍受在途请求超时和网络状态影响，不是即时停止承诺。
公开市场的“开源技能”入口为 `/github`，页面注明首期仅收录 GitHub 公开仓库；调用现有市场搜索并固定 `@github` namespace，仅显示当前已发布、公开可见的 Skill；可以搜索、按最近更新或下载量排序及分页。页面数量来自市场查询，不包含发现候选或待审包。点击 Skill 后，在详情页查看来源仓库、路径、固定提交和许可证。当前页面不展示尚无可靠聚合接口的“精选仓库数量”或 GitHub 星标数。页面中性表面、文字和边框继承全站明暗主题，局部语义颜色 token 仅用于状态、数字和细线，定义见 `web/src/features/github-sync/github-sync-colors.css`。
页面权限、状态字段、配置请求和手动操作的精确合同见[管理接口说明](api/github-sync-admin.md)。
## 故障排查
| 现象 | 先检查 | 处理 |
| --- | --- | --- |
| Go 容器未就绪 | `ps github-sync`、Go `/health/ready`；Java 内部地址与服务密钥是否匹配 | 修复配置和网络后重启 Go；就绪要求两个工作池都曾成功访问 Java 领取接口。 |
| 没有新的待审包 | 平台是否暂停、水位是否设置或已到高水位、来源/规则是否启用、Go 是否运行且支持平台暂停能力 | 先恢复容量或修正配置；高水位时已登记来源只检查 HEAD，新包不会下载。 |
| 任务显示 `CAPACITY` | 临时卷、单仓库/总工作区预算、最低空闲值 | 合规单包装不下时保留固定 commit 与检查点，约 24 小时后再试；修正容量配置后再观察。 |
| 任务显示 `LOCAL_DISK_BUDGET` | 临时卷实际剩余空间和容器内存 | 运行中接近空间保护线时约 15 分钟后重试；检查并发数与临时卷容量。 |
| 摄入校验或待审版本生成显示 `LOCAL_CAPACITY` | Java 容器的系统临时目录可用空间，区别于 Go 临时卷和对象存储容量 | 扩充 Java 临时空间或降低同时处理量后，等待持久重试；不要删除已接收的暂存对象。 |
| GitHub 限流或凭证失效 | 后台“凭证”中 `SEARCH`/`CORE` 额度、重置时间、认证状态 | 等待限流重置；只有认证失效才替换 Token，不用反复轮换来规避限流。 |
| S3/MinIO 上传或摄入失败 | 预签名 PUT 的内部主机是否在 Go 允许列表、Java 暂存前缀预算与对象实际大小 | 按任务和路径错误码排查；不要把票据声明的大小误认为对象存储接收时的硬限制。 |
| 上游路径显示缺失 | 最近一次扫描是否完整、来源是否暂停、忽略规则是否变化 | 先查看固定 commit 和扫描覆盖状态；部分扫描不据此认定 Skill 已删除。 |
服务指标与暂存容量告警见[内部协议说明](api/github-sync-internal.md#暂存容量监控)。Go 镜像在 8090 端口提供 `/health/live`、`/health/ready`、`/metrics`。暂存对象和容器日志的保留、对象存储整体剩余容量由部署侧监控；`/metrics` 不暴露 Token 或 Skill 正文。

2
docs/prds/security-audit-ui-v1.0-prd.md
View file

@ -1,5 +1,7 @@
# PRD: 前端安全审核信息展示
> 本文保留 2026-03-22 的原始设计记录。2026-10-10 审核详情页改版以 [审核指南](../skillhub/guide/review.md) 和 `openspec/changes/add-github-skill-sync/specs/github-sync-review/spec.md` 为准：顶部常显安全状态摘要，完整扫描卡片与发现项放在“来源与安全”页签。下文关于卡片插入位置和空结果不渲染的描述仅为历史方案；无结果不得解释为安全通过，扫描中禁止通过的规则不变。
**版本**: v1.0
**日期**: 2026-03-22
**状态**: Draft

2
docs/prds/skill-file-browser-sidebar-v1.0-prd.md
View file

@ -1,5 +1,7 @@
# 技能详情与审核页文件浏览侧边栏 - 产品需求文档 (PRD)
> 2026-10-09 更新：本文件记录早期侧栏方案。**审核详情页**现采用顶部摘要和页级 Tab，文件树移入“内容与文件”Tab；下文有关审核页常驻右侧栏及其三个内层 Tab 的条款不再适用。技能市场详情页的侧栏需求仍按本文件执行。审核页最新行为见 [审核指南](../skillhub/guide/review.md)。
## 需求说明
### 背景

16
docs/skillhub/guide/review.md
View file

@ -69,15 +69,16 @@ SkillHub 提供了完整的审核工作流，确保发布到注册中心的技
1. 访问 `/dashboard/reviews`
2. 查看待审核列表
3. 点击技能包查看详情：
- 查看元数据（名称、描述、版本）
- 浏览文件列表
- 在线查看文件内容
- 下载完整包进行本地测试
- 查看合规声明快照和相对上一发布版本的差异
- 顶部核对技能名称、描述、状态、提交者和版本；GitHub 同步项显示独立的内容修订号、来源仓库与短 commit，完整安装用技术版本可在技术信息中查看。顶部的安全与许可证状态用于定位问题，点击后在“来源与安全”查看依据；没有结果、读取失败或许可证未识别都不能理解为通过
- 在“技能内容”查看待审版本说明文档和文件树。说明文档在进入此页签后读取；选中单个文件后才读取其存储内容。如果存储对象已丢失，预览会提示失败
- GitHub 更新项默认打开“变更对比”，逐文件核对新增、修改和删除；首次导入和普通技能默认打开“技能内容”。超过 64 KiB 或 1000 行时跳过逐行比较，超过 256 KiB 时不在线加载内容。当前版本文件可按原权限下载；已删除的旧文件还受基线接口 1 MiB 上限约束，超限时页面会说明无法从审核页下载
- 在“来源与安全”核对固定 commit、许可证证据、完整扫描卡片和发现项；普通技能也可查看安全信息。扫描尚未完成时不能批准；GitHub 来源许可证依据缺失时按页面提示处理
- 在“审核记录”查看提交历史和已完成的审核意见；该页签打开时才请求历史
- 套件审核在内容区核对待审快照。快照加载失败时可重试，加载成功前“通过”按钮不可用
![流程图](/diagrams/review-flow.png)
4. 做出决策：
4. 在页面底部固定的审核栏填写意见并做出决策；切换页签保留意见，切换到另一个审核对象时清空意见、驳回表单和未确认弹窗：
- **批准**：技能包正式发布，开发者收到通知
- **拒绝**：填写拒绝原因，开发者可修改后重新提交
@ -186,8 +187,11 @@ GET /api/v1/reviews/{id}
**获取审核中的技能包详情**：
```bash
GET /api/v1/reviews/{id}/skill-detail
GET /api/v1/reviews/{id}/skill-detail?includeDocumentation=false
```
`includeDocumentation` 缺省为 `true`，保持原响应行为。设为 `false` 时仍返回技能、版本、文件元数据与说明文件路径，`documentationContent` 为空且不读取对象存储正文；审核详情首屏使用此模式，进入“技能内容”后再取正文。
**下载审核包**：
```bash
GET /api/v1/reviews/{id}/download

6
github-sync-go/.dockerignore Normal file
View file

@ -0,0 +1,6 @@
*
!go.mod
!cmd/
!cmd/**
!internal/
!internal/**

27
github-sync-go/Dockerfile Normal file
View file

@ -0,0 +1,27 @@
FROM --platform=$BUILDPLATFORM golang:1.27.1-alpine3.24 AS build
WORKDIR /src
COPY go.mod ./
COPY cmd/ ./cmd/
COPY internal/ ./internal/
ARG TARGETOS
ARG TARGETARCH
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \
go build -trimpath -ldflags="-s -w" -o /github-sync ./cmd/github-sync
FROM alpine:3.24.2
RUN apk add --no-cache ca-certificates git tini && \
addgroup -S -g 10001 skillhub && \
adduser -S -D -u 10001 -G skillhub skillhub && \
mkdir -p /var/lib/skillhub-github-sync && \
chown skillhub:skillhub /var/lib/skillhub-github-sync
COPY --from=build /github-sync /usr/local/bin/github-sync
USER 10001:10001
ENV SKILLHUB_GITHUB_SYNC_TEMP_ROOT=/var/lib/skillhub-github-sync
EXPOSE 8090
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
CMD wget -Y off -qO- http://127.0.0.1:8090/health/live >/dev/null || exit 1
ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/github-sync"]

100
github-sync-go/README.md Normal file
View file

@ -0,0 +1,100 @@
# GitHub 公开 Skill 采集服务
此目录是独立 Go 服务。它从 SkillHub Java 的内部接口领取发现或仓库任务，向 GitHub 读取公开仓库，逐个 Skill 包上传到 Java 授权的临时对象，再回报采集结果。来源配置、任务状态、审核和发布归 Java 管理。首期不读取私有仓库。
## 当前接入状态
Go 的发现、仓库树枚举、逐包采集、续租和回报代码已接入单进程双工作池，Java 的配置、来源转换、Skill 版本与审核流程也已接入。本地存储与 MinIO 应分别验收；ARM64 整镜像与长期容量须独立验证。启动容器本身不会让未经审核的 Skill 出现在公开市场。
官方深度探索已有独立搜索执行路径：metadata、README 和 topic 查询按固定 UTC 整秒半开窗口执行，逐页检查控制状态，保存分页/页内游标和出处，预算耗尽后保留 IN_PROGRESS。截断或不完整页面中已返回的仓库仍按候选预算保存；结果上限内可访问的页面继续采集，缺口原因跨页保留，最终按预算提交时间子分片或记录 GAP。GAP 按持久游标递增退避（暂为 5 分钟至 6 小时），普通工作预算续排暂为 1 秒；这些值仍需调度校准。批次响应丢失可重放相同请求，坏检查点请求服务端安全重放。非空二级分片暂不支持；SPLIT 由 Java 快照中的剩余深度与根范围子任务额度控制，额度为 0 或 1 时不能二分。父分片交接后不代表全部覆盖；恢复到已确认的 SPLIT 检查点只重试父任务结算，不重查父范围。
带 `official-pause/v1` 标记的候选仓库任务已接入控制检查：metadata/HEAD、每个树请求、下一个包准入及结算前查询；已获准包完成上传和回报后再暂停。暂停保存固定 commit、路径进度和未完成子树，尚未取得 commit 时使用空值；已接收包由 Java 摄入记录保留。候选途中绑定来源或获得独立入口时按最新控制结果继续，来源自身暂停以 `SOURCE_PAUSED` 区分。
作者扩展使用独立 `official-owner/v1` 能力：按稳定账号 ID 获取当前 login，复用 public owner 仓库分页并逐仓库核对 owner ID，终页再次核验账号身份。身份查询与分页均计 CORE 请求预算，终页核验预留请求额度；检查点页码支持正 int32，达到上限仍有下一页时记录 GAP。作者列表不按 pushed 时间过滤、不做时间二分，GAP 保持 RETRY_WAIT；已确认检查点支持暂停与跨租约续跑。GitHub 元数据上报可选 ownerId，供 Java 的有效 seed 事务使用。
官方配额观察通过租约绑定的 `official-quota` 接口持久化，包含匿名请求；REST 的 metadata、HEAD、commit、tree、search、owner 请求以及可能联网的 Git fetch / sparse checkout / checkout 都有请求守门。主配额耗尽按资源退避；二级限流与观察持久化失败触发进程共享退避，不能通过更换官方任务 token 绕过。回报失败锁住当前租约后续请求，进程失败退避暂为 1、2、4、8、15 分钟；Git 无 REST 响应头只报 UNKNOWN，失败附短退避，成功不虚构余额。已生成 artifact 的上传与结果回报不受 Git 请求守门影响。进程退避不是跨实例持久状态；跨实例冷却由 Java 的官方配额状态和控制 / 领取门控负责。
生产工作池通过 `ClaimWithCapabilities` 和 `HeartbeatWithCapabilities` 声明 `official-discovery/v1`、`official-checkpoint/v1`、`official-pause/v1`、`official-owner/v1` 和 `platform-pause/v1`；旧客户端方法仍可调用，但V119之后缺少平台暂停能力的实例不能领取平台任务，TEAM规则不变。Java 官方探索配置默认关闭，管理员启用后才允许领取官方任务。首次启用前必须整体升级 Java、数据库和 Go；不混跑早期显式 opt-in 的旧执行器。真实 GitHub 召回、预算档位校准和并发生产验收需要独立验证。策略目录及版本由 Java 维护。
平台 `@github` 的全部采集还受独立 `platform-pause/v1` 控制：Java 保存暂停意图，Go 在请求安全点及目标5秒观察间隔检查，停止外部工作并完成清理后由执行线程确认。暂停中不是已停止，离线与租约过期不能代替确认；TEAM不受平台开关控制。Java已接收的包可继续校验和审核，恢复不重置检查点或各入口独立启停。部署需配套升级Java/Go，操作见[管理员手册](../docs/github-sync-operations.md#平台暂停与恢复)，协议见[内部接口第9节](../docs/api/github-sync-internal.md#9-平台github采集安全暂停)。
## 构建与启动
```bash
cd github-sync-go
go build ./...
```
先在仓库根目录运行 `make staging` 准备预发布服务，再用同一个 Compose 项目启动 Go 容器：
```bash
SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN='<至少 32 字符的随机服务密钥>' \
SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS='<同一服务密钥>' \
docker compose -p skillhub-staging -f docker-compose.yml -f docker-compose.staging.yml \
--profile github-sync up -d github-sync
```
上述命令是参数示例；正式环境应通过受保护的环境文件或容器 Secret 提供密钥，不把真实值写进命令历史、仓库或日志。发布 Compose 使用 `compose.release.yml` 和相同的 `github-sync` profile。默认不启动 Go 服务；Helm 对应 `githubSync.enabled: false`。
| 环境变量 | 用途 | 默认值或要求 |
| --- | --- | --- |
| `SKILLHUB_GITHUB_SYNC_JAVA_URL` | Java 内部合同根地址 | 必填；Compose 已设置。 |
| `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN` | Go 调用 Java 的服务身份 | 必填，至少 32 字符；Java 的 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 须包含它。 |
| `SKILLHUB_GITHUB_SYNC_TEMP_ROOT` | 受限临时目录 | 必填绝对路径；镜像默认 `/var/lib/skillhub-github-sync`。同一目录一次只能供一个 Go 进程使用。 |
| `SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS` / `SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS` | 两类工作池并发数 | 各 1。 |
| `SKILLHUB_GITHUB_SYNC_POLL_INTERVAL` | 无任务或领取失败后的轮询间隔 | `5s`。 |
| `SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES` | 单仓库工作目录预算 | Compose/Helm 默认 512 MiB。 |
| `SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES` | 所有仓库 worker 的总预算 | Compose/Helm 默认 512 MiB。 |
| `SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES` | 临时目录预留空间 | Compose/Helm 默认 128 MiB。 |
| `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` | 允许上传的对象存储主机和端口，逗号分隔 | S3/MinIO 模式必填；应与 Java 返回的内部预签名 PUT 主机一致。 |
| `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` | GitHub 出站代理 | 按网络环境设置；Go→Java 内部请求与暂存对象上传直连，不走该代理。 |
Compose 从 `.env.release` 中的 `SKILLHUB_GITHUB_SYNC_HTTP_PROXY`、`SKILLHUB_GITHUB_SYNC_HTTPS_PROXY`、`SKILLHUB_GITHUB_SYNC_NO_PROXY` 显式注入对应标准代理变量；staging 覆盖文件也使用这三个宿主变量。留空时不会自动继承宿主机的代理配置。Helm 使用 `githubSync.httpProxy`、`githubSync.httpsProxy`、`githubSync.noProxy`。
镜像提供 `GET /health/live`、`GET /health/ready` 和 `GET /metrics`，端口为 8090。就绪状态要求发现池和仓库池都曾成功访问 Java 领取接口。`/metrics` 只暴露固定种类的低基数计数，不包含服务密钥或 Skill 正文。
运行镜像使用 `tini` 作为 PID 1，以回收 Git 子进程并转发停止信号；应避免直接由 Go 充当 PID 1 时无法完整回收 Git 子进程。停止 Go 后，Java 保存的租约和逐包检查点供下一实例续跑。
临时目录使用独立的 768 MiB tmpfs，内存限制默认 1.5 GiB；tmpfs 占用计入容器内存。采集会预检并逐包释放目录，仍应监控容器内存与容量阻塞状态。S3/MinIO 预签名 PUT 的大小由 Go 上传前和 Java 摄入前复核；不能把 URL 票据字段当作对象存储的接收时硬限制。
Go 启动时会独占临时目录，并只清理该目录下本服务命名的遗留包工作区（`skillhub-github-package-*`）。进程被强制终止后，重启可先释放这些本地文件，再按 Java 保存的固定 commit 和检查点续跑；同一临时目录已有运行中的 Go 进程时，新进程启动失败。多实例部署须为每个实例分配独立临时目录。
S3/MinIO 开启桶自动创建时，Java 会在签发首张预签名 PUT 票据前建桶。对象存储仍可能接收超过票据声明大小的直接 PUT；Java 在接收采集回报前按真实对象大小复核，超限对象标失败并清理。
Go 在下载包文件前，为 Git 对象、检出文件、ZIP 和至少 16 MiB 基础 Git 元数据预留工作区；预留不足的仓库任务回报容量延后，不启动该包下载。仓库树元数据的实际大小无法事先精确获知，因此 Git 每一步会复查真实目录占用和最低剩余空间，写 ZIP 前也会复查剩余空间。下载前估算不是磁盘硬配额。
若单包在当前配置下根本无法容纳，任务以 `CAPACITY` 延后约 24 小时，保留固定 commit 和已完成路径检查点；Java 释放本次包准入，不生成摄入包。若是运行时暂存空间逼近阈值，使用 `LOCAL_DISK_BUDGET` 延后约 15 分钟。两者都不会在同一租约内继续下载后续包；先检查临时目录容量、`MAX_WORKSPACE_BYTES`、`TOTAL_WORKSPACE_BYTES` 和 `MIN_FREE_BYTES` 的组合，再决定是否调整配置。
归档写入时如果先触及工作区剩余预算，或文件系统返回空间/配额不足，也按 `LOCAL_DISK_BUDGET` 延后；只有归档达到任务下发的上传大小上限才记为 `ARCHIVE_TOO_LARGE`。
需要复核公开 GitHub 的实际受限 Git 获取时，可在 `github-sync-go/` 运行 `SKILLHUB_GITHUB_SYNC_LIVE=1 go test ./internal/repository -run TestLiveAcquireFixedPublicSkill -count=1 -v`。它使用固定公开 commit，检查 ZIP 文件、摘要和临时目录清理；普通 `go test ./...` 不访问 GitHub。
测试专用的 `SKILLHUB_GITHUB_SYNC_LIVE_SMALL_VOLUME=1` 将上述用例的单包工作区预算设为 32 MiB、保留空闲设为 8 MiB，便于在小 tmpfs 中复核。容器复核时应使用非 root 身份、只读根文件系统和有硬容量限制的 tmpfs，按网络配置显式提供 GitHub 出站代理。该用例只处理一个小 Skill，不代表大仓库峰值或完整 worker 停机续跑。
同一固定 commit 的双 Skill 顺序采集可用 `SKILLHUB_GITHUB_SYNC_LIVE=1 go test ./internal/repository -run TestLiveAcquireTwoFixedPublicSkillsSequentially -count=1 -v` 复核。测试在首包关闭后确认临时目录为空，再采集第二个 Skill；输出分别给出归档和采集结束时的工作目录字节数。这是单仓库小范围演练，不是并发或大规模容量基线。
较多文件的公开仓库可用 `SKILLHUB_GITHUB_SYNC_LIVE_LARGE_REPO=1 go test ./internal/repository -run '^TestLiveAcquireLargerPublicRepositorySkill$' -count=1 -v` 复核一个固定 commit 的单 Skill 稀疏获取。此用例要求临时文件系统至少容纳 64 MiB 工作区及 16 MiB 保留空间；建议在有硬容量限制的临时卷中运行。它不覆盖完整 GitHub Tree API 枚举、多包并发或进程瞬时峰值。
要手工复核 Go/Java 包格式互通，给上述 Go 测试额外设置 `SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE` 为一个尚不存在的临时 ZIP 路径；随后在 `server/` 给 `GithubSyncArchiveValidatorInteropTest` 设置同一路径的 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE` 并运行 Maven 定向测试，最后删除临时 ZIP。两个变量仅供测试使用，不参与服务启动。
要继续复核实际 Go→Java HTTP 合同，将同一个 ZIP 路径传给 `GithubSyncHttpIngestionInteropTest`：在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest' -Dsurefire.failIfNoSpecifiedTests=false test`。此测试需要本机 Go 与 Docker；它在隔离的 PostgreSQL、Redis 容器和随机 HTTP 端口上让真正的 Go 控制面客户端完成领取、准入、本地暂存上传、重复与冲突回报、仓库完成，再验证 Java 摄入校验、后台发布、Scanner outbox、独立同步待审任务及审核前市场版本指针为空。Scanner 消费者默认调用隔离的 HTTP 测试桩取得安全结果后，超级管理员批准同步审核，才将版本发布为市场最新版本。第二次同内容完整摄入必须落为 `NO_CHANGE`，不能新增版本或审核项；TEAM 来源另验证 `NAMESPACE_ONLY` Skill 被拒后不发布，同内容重采也不重复审核。显式请求逐 Skill 重试后，即使 commit 和包内容相同，下一次完整采集也必须建立新的同步待审项，并清除重试标记。测试还让 Go 对第一完整 Java 应用进程回报包，停掉该进程后由第二进程从同一数据库与存储自动续跑，检查唯一版本和同步待审项。加上 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 后，同一测试会启动隔离的 MinIO，验证改写预签名 URL 的对象 key 被拒且未生成对象、原签名正常上传、首次建桶、对象摄入以及超额对象被 Java 拒绝并清理。测试结束后删除临时 ZIP。新增用例还会在 Go 持有仓库任务租约时关闭 Java，确认续租连接失败，再在原端口启动 Java，由第二个 Go 进程接管并完成已接收路径；测试提前设置租约到期时间，避免等待生产环境的完整租约周期。
如需验证实际仓库 worker 的双 Skill 逐包路径，先用上面的 `SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE` 从固定公开 commit 生成第一个 Skill 的参考 ZIP，再在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_LIVE_WORKER=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerAcquiresTwoPublicSkillsThroughJava' -Dsurefire.failIfNoSpecifiedTests=false test`。Go 使用受控 Tree 元数据绕开 GitHub REST 额度，从公开固定 commit 实际稀疏获取两个 Skill；Java 完成领取、准入、上传、校验并生成两条独立同步待审项，第一个摄入包的 SHA-256 与参考 ZIP 一致。隔离 Scanner HTTP 测试桩将两包判为安全后，测试逐条批准：首条批准时另一条仍未发布，两条批准后均为已发布版本；TEAM 的两条 Skill 对匿名访问均返回 403。额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可复核隔离 MinIO 模式。此用例不验证真实 GitHub REST Tree 枚举或真实 Scanner 判定；运行后删除参考 ZIP。
同一固定 commit 的在途恢复用例可在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_LIVE_INTERRUPT_INTEROP=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerResumesAfterSigtermBetweenReceiptAndAcknowledgement' -Dsurefire.failIfNoSpecifiedTests=false test`；加 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可复核 MinIO。Go 测试进程在首包已被 Java 持久接收、确认尚未送达时收到 SIGTERM；新进程领取到期租约并只采集剩余包。测试检查首包摄入 ID、路径结果和准入次数不变，及中断后工作区为空。Tree 元数据受控，此用例仍不能代替完整服务容器在小临时卷中的停启演练。
实际服务镜像的在途续跑可在仓库根目录先执行 `docker build -t skillhub-github-sync:interop github-sync-go`，再在 `server/` 执行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_INTEROP_IMAGE=skillhub-github-sync:interop ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#serviceContainerResumesPublicRepositoryOnSmallTmpfs' -Dsurefire.failIfNoSpecifiedTests=false test`；额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可验证隔离 MinIO。参考 ZIP 依上文单包采集命令生成，测试从 `SKILLHUB_GITHUB_SYNC_LIVE_TOKEN` 或本机 `gh auth token` 取得公开读取凭证；容器需要可用的 GitHub 出站代理。此显式公网用例在 96 MiB tmpfs 中首包持久接收后停止容器，由新容器按租约续跑真实 Tree 的 43 个合规包，并检查首包不重复摄入、容器进程表无僵尸。完成后删除参考 ZIP。
要把同一双 Skill 用例改为真实 GitHub REST Tree 元数据，在上述 Maven 命令额外设置 `SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE=1`。测试进程从 `SKILLHUB_GITHUB_SYNC_LIVE_TOKEN` 或本机 `gh auth token` 读取凭证，仅用于公开 API 的 GET 请求；测试代码不写入文件或日志，也不影响正式服务的 Java 凭证池。Go 核对真实 repository ID、固定 commit 和完整递归 Tree 响应后，仅选择上述两个 Skill 的元数据，以控制此次联调的包数。可以先在 `github-sync-go/` 用 `SKILLHUB_GITHUB_SYNC_LIVE_TREE=1 go test ./internal/repository -run '^TestLiveEnumerateFixedPublicGitHubTree$' -count=1 -v` 验证完整 Tree 枚举及所有候选 Skill 的元数据预检；预检采用当前 `application.yml` 的 100 文件、单文件 10 MiB、包总量 100 MiB 限制，不下载其余包。真实 Tree 双包联调已在本地存储与 MinIO 模式通过，但不能据此认定全部 44 个 Skill、其他仓库或高并发容量已验收。
如需在同一固定公开仓库中顺序获取所有通过预检的包，可在 `github-sync-go/` 运行 `SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES=1 go test ./internal/repository -run '^TestLiveAcquireAllFixedPublicSkillsSequentially$' -count=1 -v -timeout=16m`。此显式公网用例从真实 Tree 生成包计划，在单包 64 MiB 工作区预算和 16 MiB 保留空间下逐包执行 Git 获取，核对 ZIP 条目、摘要，并在下一包开始前确认工作区为空。当前固定 commit 的 43 个可采包全部通过，另 1 个因 312 个文件超过 100 文件政策而被拒。用例不经过 Java 摄入或 Scanner；完成时的目录占用不等于 Git 执行期间的严格瞬时峰值。
如需检查整仓的 Go→Java 接收与后台处理，可先按上文生成单包参考 ZIP，再在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/creator.zip SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE=1 SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES_INTEROP=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerReportsEveryEligiblePublicSkillThroughJava' -Dsurefire.failIfNoSpecifiedTests=false test`；加 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 验证隔离 MinIO。此显式用例把包上限恢复为当前 100 文件、100 MiB 政策，约需数分钟，验证 43 包接收后有 42 个同步待审项、1 个同 namespace slug 冲突，另一个超文件数目录不下载；待审前无市场最新版本。用例不自动批准待审项，不代表跨仓库并发或长期容量验收。
需要验证真实 Scanner 时，先从当前 `scanner/Dockerfile` 构建并启动独立 Scanner，确认其 `/health` 可访问；然后为上述定向测试额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_SCANNER_URL=http://127.0.0.1:<scanner-port>`。单包测试可用 `GithubSyncHttpIngestionInteropTest#goArchiveCanTravelThroughHttpReceiptAndPersistedIngestion`，双包 worker 测试可用 `GithubSyncHttpIngestionInteropTest#repositoryWorkerAcquiresTwoPublicSkillsThroughJava`。该模式使用正式 Compose 相同的 `upload` 扫描方式；不设置此变量时仍使用 HTTP 测试桩。真实 Scanner 的结论取决于测试包内容和当前规则，不应固定假设所有包都安全。
内部接口字段、错误码和恢复语义见 [合同说明](../docs/api/github-sync-internal.md) 与 [OpenAPI](../docs/api/github-sync-internal.openapi.yaml)。
上述互操作测试还从同一公开采集包生成四个受控内容 revision，用不同固定 commit 模拟上游连续更新。它验证旧审核被新目标替代、旧目标不能批准、已发布版本在新审核期间保持可用，以及较旧包迟到时不能覆盖较新的来源绑定。这是协议和发布事务测试，不代表已对四个真实 GitHub commit 做公网采集。
同一测试继续以另一个固定 commit 回报完全相同的包，验证 `NO_CHANGE`；再只调整包内 `LICENSE.txt`，验证内容指纹和同步待审目标发生变化。这两步也使用受控 revision，不代表对应 commit 曾在 GitHub 公网真实存在。
摄入后还会启动独立 Java JVM 领取校验租约并退出，再由测试中的 Java 应用实例在租约过期后完成校验；发布阶段重复相同的进程丢失与恢复检查。这验证持久租约和后台续跑，不替代完整应用容器的停止、重新启动及部署配置验收。

225
github-sync-go/cmd/github-sync/main.go Normal file
View file

@ -0,0 +1,225 @@
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/worker"
)
type config struct {
JavaURL string
Token string
HealthAddr string
Pool worker.PoolConfig
}
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
settings, err := loadConfig()
if err != nil {
slog.Error("GitHub sync configuration invalid", "code", "INVALID_CONFIGURATION")
os.Exit(1)
}
workspaceGuard, err := prepareTempWorkspace(settings.Pool.Repository.TempRoot)
if err != nil {
slog.Error("GitHub sync temporary workspace unavailable", "code", "TEMP_WORKSPACE_UNAVAILABLE")
os.Exit(1)
}
defer workspaceGuard.Close()
control, err := controlplane.NewClient(settings.JavaURL, settings.Token)
if err != nil {
slog.Error("GitHub sync control plane unavailable", "code", "INVALID_CONTROL_PLANE")
os.Exit(1)
}
listener, err := net.Listen("tcp", settings.HealthAddr)
if err != nil {
slog.Error("GitHub sync health listener unavailable", "code", "HEALTH_BIND_FAILED")
os.Exit(1)
}
counters := &worker.Counters{}
server := &http.Server{Handler: healthHandler(counters), ReadHeaderTimeout: 5 * time.Second}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
serveDone := make(chan error, 1)
go func() { serveDone <- server.Serve(listener) }()
poolDone := make(chan error, 1)
go func() { poolDone <- worker.Run(ctx, control, settings.Pool, counters) }()
poolStopped := false
failed := false
select {
case <-ctx.Done():
case err := <-serveDone:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
slog.Error("GitHub sync health server stopped", "code", "HEALTH_SERVER_FAILED")
failed = true
}
stop()
case err := <-poolDone:
poolStopped = true
if err != nil && !errors.Is(err, context.Canceled) {
slog.Error("GitHub sync worker stopped", "code", "WORKER_STOPPED")
failed = true
}
stop()
}
counters.DiscoveryReady.Store(false)
counters.RepositoryReady.Store(false)
shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = server.Shutdown(shutdown)
if !poolStopped {
select {
case <-poolDone:
case <-time.After(20 * time.Second):
slog.Warn("GitHub sync shutdown timed out", "code", "SHUTDOWN_TIMEOUT")
failed = true
}
}
if failed {
os.Exit(1)
}
}
func healthHandler(counters *worker.Counters) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", func(response http.ResponseWriter, _ *http.Request) {
response.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("GET /health/ready", func(response http.ResponseWriter, _ *http.Request) {
if !counters.DiscoveryReady.Load() || !counters.RepositoryReady.Load() {
response.WriteHeader(http.StatusServiceUnavailable)
return
}
response.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("GET /metrics", func(response http.ResponseWriter, _ *http.Request) {
response.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
fmt.Fprintf(response, "skillhub_github_sync_claims_total{kind=\"discovery\"} %d\n", counters.DiscoveryClaims.Load())
fmt.Fprintf(response, "skillhub_github_sync_claims_total{kind=\"repository\"} %d\n", counters.RepositoryClaims.Load())
fmt.Fprintf(response, "skillhub_github_sync_errors_total{kind=\"discovery\"} %d\n", counters.DiscoveryErrors.Load())
fmt.Fprintf(response, "skillhub_github_sync_errors_total{kind=\"repository\"} %d\n", counters.RepositoryErrors.Load())
fmt.Fprintf(response, "skillhub_github_sync_active_tasks{kind=\"discovery\"} %d\n", counters.DiscoveryActive.Load())
fmt.Fprintf(response, "skillhub_github_sync_active_tasks{kind=\"repository\"} %d\n", counters.RepositoryActive.Load())
})
return mux
}
func loadConfig() (config, error) {
var settings config
settings.JavaURL = os.Getenv("SKILLHUB_GITHUB_SYNC_JAVA_URL")
settings.Token = os.Getenv("SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN")
settings.HealthAddr = valueOrDefault("SKILLHUB_GITHUB_SYNC_HEALTH_ADDR", ":8090")
var err error
if settings.Pool.DiscoveryWorkers, err = positiveInt("SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS", 1); err != nil {
return config{}, err
}
if settings.Pool.RepositoryWorkers, err = positiveInt("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", 1); err != nil {
return config{}, err
}
if settings.Pool.Discovery.MaxPagesPerLease, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_PAGES_PER_LEASE", 20); err != nil {
return config{}, err
}
if settings.Pool.Repository.MaxTreeRequests, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_TREE_REQUESTS", 100); err != nil {
return config{}, err
}
if settings.Pool.Repository.MaxTreeEntries, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_TREE_ENTRIES", 100000); err != nil {
return config{}, err
}
settings.Pool.PollInterval, err = time.ParseDuration(valueOrDefault("SKILLHUB_GITHUB_SYNC_POLL_INTERVAL", "5s"))
if err != nil || settings.Pool.PollInterval < time.Second {
return config{}, errors.New("invalid poll interval")
}
settings.Pool.Repository.TempRoot = os.Getenv("SKILLHUB_GITHUB_SYNC_TEMP_ROOT")
settings.Pool.Repository.AllowedS3Hosts = splitHosts(os.Getenv("SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS"))
settings.Pool.Repository.MaxWorkspaceBytes, err = positiveBytes("SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES")
if err != nil {
return config{}, err
}
settings.Pool.Repository.MinFreeBytes, err = nonnegativeBytes("SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES")
if err != nil {
return config{}, err
}
totalWorkspaceBytes, err := positiveBytes("SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES")
if err != nil || int64(settings.Pool.RepositoryWorkers) >
totalWorkspaceBytes/settings.Pool.Repository.MaxWorkspaceBytes {
return config{}, errors.New("repository workers exceed total workspace budget")
}
if !filepath.IsAbs(settings.Pool.Repository.TempRoot) || settings.JavaURL == "" || settings.Token == "" {
return config{}, errors.New("missing GitHub sync service configuration")
}
info, err := os.Stat(settings.Pool.Repository.TempRoot)
if err != nil || !info.IsDir() {
return config{}, errors.New("GitHub sync temporary directory is unavailable")
}
var filesystem syscall.Statfs_t
if err := syscall.Statfs(settings.Pool.Repository.TempRoot, &filesystem); err != nil ||
uint64(totalWorkspaceBytes)+uint64(settings.Pool.Repository.MinFreeBytes) >
uint64(filesystem.Blocks)*uint64(filesystem.Bsize) {
return config{}, errors.New("temporary filesystem is smaller than configured workspace budget")
}
probe, err := os.MkdirTemp(settings.Pool.Repository.TempRoot, "write-check-")
if err != nil {
return config{}, errors.New("GitHub sync temporary directory is not writable")
}
if err := os.Remove(probe); err != nil {
return config{}, errors.New("GitHub sync temporary directory cannot be cleaned")
}
return settings, nil
}
func valueOrDefault(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
func positiveInt(name string, fallback int) (int, error) {
value, err := strconv.Atoi(valueOrDefault(name, strconv.Itoa(fallback)))
if err != nil || value < 1 {
return 0, errors.New("invalid positive worker configuration")
}
return value, nil
}
func positiveBytes(name string) (int64, error) {
value, err := strconv.ParseInt(os.Getenv(name), 10, 64)
if err != nil || value < 1 {
return 0, errors.New("invalid workspace byte budget")
}
return value, nil
}
func nonnegativeBytes(name string) (int64, error) {
value, err := strconv.ParseInt(os.Getenv(name), 10, 64)
if err != nil || value < 0 {
return 0, errors.New("invalid minimum free byte budget")
}
return value, nil
}
func splitHosts(value string) []string {
if strings.TrimSpace(value) == "" {
return nil
}
var hosts []string
for _, item := range strings.Split(value, ",") {
if host := strings.TrimSpace(item); host != "" {
hosts = append(hosts, host)
}
}
return hosts
}

27
github-sync-go/cmd/github-sync/main_test.go Normal file
View file

@ -0,0 +1,27 @@
package main
import (
"strings"
"testing"
)
func TestLoadConfigBoundsRepositoryWorkersByTotalWorkspace(t *testing.T) {
t.Setenv("SKILLHUB_GITHUB_SYNC_JAVA_URL", "http://127.0.0.1:8080/internal/github-sync/v1")
t.Setenv("SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN", strings.Repeat("t", 32))
t.Setenv("SKILLHUB_GITHUB_SYNC_TEMP_ROOT", t.TempDir())
t.Setenv("SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES", "33554432")
t.Setenv("SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES", "67108864")
t.Setenv("SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES", "8388608")
t.Setenv("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", "2")
settings, err := loadConfig()
if err != nil || settings.Pool.RepositoryWorkers != 2 {
t.Fatalf("two workers within the total budget should start: workers=%d, err=%v",
settings.Pool.RepositoryWorkers, err)
}
t.Setenv("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", "3")
if _, err := loadConfig(); err == nil {
t.Fatal("workers exceeding the total workspace budget must be rejected")
}
}

40
github-sync-go/cmd/github-sync/workspace.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"errors"
"os"
"path/filepath"
"strings"
"syscall"
)
const packageWorkspacePrefix = "skillhub-github-package-"
// Keep the directory locked while the worker runs so another process cannot
// remove active packages from a shared local workspace during startup.
func prepareTempWorkspace(root string) (*os.File, error) {
guard, err := os.Open(root)
if err != nil {
return nil, err
}
if err := syscall.Flock(int(guard.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = guard.Close()
return nil, errors.New("GitHub sync temporary workspace is already in use")
}
entries, err := os.ReadDir(root)
if err != nil {
_ = guard.Close()
return nil, err
}
for _, entry := range entries {
if !strings.HasPrefix(entry.Name(), packageWorkspacePrefix) ||
len(entry.Name()) == len(packageWorkspacePrefix) {
continue
}
if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil {
_ = guard.Close()
return nil, err
}
}
return guard, nil
}

58
github-sync-go/cmd/github-sync/workspace_test.go Normal file
View file

@ -0,0 +1,58 @@
package main
import (
"os"
"path/filepath"
"testing"
)
func TestPrepareTempWorkspaceCleansOnlyAbandonedPackages(t *testing.T) {
root := t.TempDir()
abandoned := filepath.Join(root, packageWorkspacePrefix+"abandoned")
if err := os.Mkdir(abandoned, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(abandoned, "object"), []byte("old"), 0600); err != nil {
t.Fatal(err)
}
unrelated := filepath.Join(root, "operator-note")
if err := os.WriteFile(unrelated, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
guard, err := prepareTempWorkspace(root)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = guard.Close() })
if _, err := os.Stat(abandoned); !os.IsNotExist(err) {
t.Fatalf("abandoned package remains: %v", err)
}
if content, err := os.ReadFile(unrelated); err != nil || string(content) != "keep" {
t.Fatalf("unrelated file changed: %q, %v", content, err)
}
if _, err := prepareTempWorkspace(root); err == nil {
t.Fatal("a second worker must not clean an active workspace")
}
}
func TestPrepareTempWorkspaceCanRecoverAfterPreviousOwnerExits(t *testing.T) {
root := t.TempDir()
guard, err := prepareTempWorkspace(root)
if err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(root, packageWorkspacePrefix+"interrupted"), 0700); err != nil {
t.Fatal(err)
}
if err := guard.Close(); err != nil {
t.Fatal(err)
}
guard, err = prepareTempWorkspace(root)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = guard.Close() })
if entries, err := os.ReadDir(root); err != nil || len(entries) != 0 {
t.Fatalf("restarted worker did not clear its old package: %v, %v", entries, err)
}
}

3
github-sync-go/go.mod Normal file
View file

@ -0,0 +1,3 @@
module github.com/iflytek/skillhub/github-sync-go
go 1.27.1

592
github-sync-go/internal/controlplane/client.go Normal file
View file

@ -0,0 +1,592 @@
package controlplane
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
const maxResponseBytes = 16 << 20
// Client speaks the versioned internal API. It never uses the GitHub proxy
// transport and never sends a service token to a redirect target.
type Client struct {
baseURL string
token string
http *http.Client
}
func NewClient(baseURL, serviceToken string) (*Client, error) {
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" ||
parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" ||
len(strings.TrimSpace(serviceToken)) < 32 {
return nil, errors.New("invalid internal API URL or missing service token")
}
return &Client{
baseURL: strings.TrimRight(baseURL, "/"),
token: strings.TrimSpace(serviceToken),
http: &http.Client{
Timeout: 20 * time.Second,
Transport: &http.Transport{
Proxy: nil,
MaxIdleConnsPerHost: 4,
ResponseHeaderTimeout: 15 * time.Second,
},
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
},
}, nil
}
func (c *Client) Heartbeat(ctx context.Context) error {
_, err := c.post(ctx, "/heartbeat", struct{}{}, nil)
return err
}
type LeaseProof struct {
Generation int64 `json:"generation"`
Token string `json:"token"`
}
type PackageAdmission struct {
AdmissionID string `json:"admissionId"`
}
type ClaimedTask struct {
PlatformControlVersion *int64 `json:"platformControlVersion,omitempty"`
TaskID string `json:"taskId"`
RunID string `json:"runId"`
Kind string `json:"kind"`
Mode string `json:"mode,omitempty"`
NamespaceID int64 `json:"namespaceId"`
SourceID int64 `json:"sourceId,omitempty"`
Lease LeaseProof `json:"lease"`
LeaseExpiresAt time.Time `json:"leaseExpiresAt"`
Discovery *DiscoverySnapshot `json:"discovery,omitempty"`
OfficialDiscovery *OfficialDiscoverySnapshot `json:"officialDiscovery,omitempty"`
RequiredCapabilities []string `json:"requiredCapabilities,omitempty"`
Repository *RepositorySnapshot `json:"repository,omitempty"`
Credential *GithubCredential `json:"credential,omitempty"`
MaxPackageBytes int64 `json:"maxPackageBytes,omitempty"`
MaxSingleFileBytes int64 `json:"maxSingleFileBytes,omitempty"`
MaxFilesPerPackage int `json:"maxFilesPerPackage,omitempty"`
MaxUploadBytes int64 `json:"maxUploadBytes,omitempty"`
}
type GithubCredential struct {
CredentialID string `json:"credentialId"`
Token string `json:"token"`
}
type GithubQuota struct {
Resource string `json:"resource"`
Remaining int `json:"remaining"`
ResetAt time.Time `json:"resetAt"`
}
type DiscoverySnapshot struct {
EntryType string `json:"entryType"`
RuleID string `json:"ruleId,omitempty"`
RuleVersion int64 `json:"ruleVersion"`
Criteria *SearchCriteria `json:"criteria,omitempty"`
SearchWindow *SearchWindow `json:"searchWindow,omitempty"`
Owner string `json:"owner,omitempty"`
RepositoryName string `json:"repositoryName,omitempty"`
Checkpoint string `json:"checkpoint,omitempty"`
CandidateBudget int `json:"candidateBudget,omitempty"`
}
type SearchCriteria struct {
Selectors []SearchSelector `json:"selectors"`
}
type SearchSelector struct {
Kind string `json:"kind"`
Value string `json:"value"`
Priority string `json:"priority"`
}
type SearchWindow struct {
Mode string `json:"mode"`
Since time.Time `json:"since,omitempty"`
Until time.Time `json:"until"`
}
type RepositorySnapshot struct {
RepositoryID int64 `json:"repositoryId"`
Owner string `json:"owner"`
Name string `json:"name"`
FixedCommitSHA string `json:"fixedCommitSha,omitempty"`
PathCheckpoint string `json:"pathCheckpoint,omitempty"`
IncompleteSubtrees []string `json:"incompleteSubtrees,omitempty"`
RetryPaths []string `json:"retryPaths,omitempty"`
AcceptedPaths []string `json:"acceptedPaths,omitempty"`
IgnorePatterns []string `json:"ignorePatterns,omitempty"`
}
type RepositoryPathResult struct {
SkillPath string `json:"skillPath"`
Outcome string `json:"outcome"`
IngestionID string `json:"ingestionId,omitempty"`
Reason string `json:"reason,omitempty"`
}
type RepositoryCompletion struct {
CommitSHA string
PathCheckpoint string
IncompleteSubtrees []string
PathResults []RepositoryPathResult
RetryAt time.Time
DeferralReason string
}
type RepositoryCandidate struct {
RepositoryID int64 `json:"repositoryId"`
Owner string `json:"owner"`
Name string `json:"name"`
DefaultBranch string `json:"defaultBranch,omitempty"`
OfficialProvenance []OfficialProvenance `json:"officialProvenance,omitempty"`
}
// RepositoryMetadata is the canonical public identity returned by GitHub for
// the stable repository ID in a leased task.
type RepositoryMetadata struct {
OwnerID int64 `json:"ownerId,omitempty"`
RepositoryID int64 `json:"repositoryId"`
Owner string `json:"owner"`
Name string `json:"name"`
DefaultBranch string `json:"defaultBranch"`
}
func (c *Client) UpdateRepositoryMetadata(ctx context.Context, taskID string,
lease LeaseProof, metadata RepositoryMetadata) error {
if metadata.RepositoryID <= 0 || metadata.Owner == "" || metadata.Name == "" ||
metadata.DefaultBranch == "" || metadata.OwnerID < 0 {
return errors.New("invalid GitHub repository metadata")
}
request := map[string]any{
"lease": lease, "repositoryId": metadata.RepositoryID, "owner": metadata.Owner,
"name": metadata.Name, "defaultBranch": metadata.DefaultBranch,
}
if metadata.OwnerID > 0 {
request["ownerId"] = metadata.OwnerID
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/repository-metadata", request, nil)
return err
}
type DiscoveryBatch struct {
PreviousCheckpoint *string `json:"previousCheckpoint"`
Checkpoint string `json:"checkpoint"`
Repositories []RepositoryCandidate `json:"repositories"`
DeferredCount int `json:"deferredCount,omitempty"`
DeferralReason string `json:"deferralReason,omitempty"`
OfficialBatch *OfficialDiscoveryBatch `json:"officialBatch,omitempty"`
}
type PreviewSelectorResult struct {
Kind string `json:"kind"`
Value string `json:"value"`
Priority string `json:"priority"`
TotalCount int `json:"totalCount"`
IncompleteResults bool `json:"incompleteResults"`
LimitReached bool `json:"limitReached"`
Samples []RepositoryCandidate `json:"samples"`
}
func (c *Client) CompletePreview(ctx context.Context, taskID string, lease LeaseProof,
results []PreviewSelectorResult) error {
if len(results) < 1 || len(results) > 20 {
return errors.New("invalid preview result count")
}
var response struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/preview-results", map[string]any{
"lease": lease, "selectors": results,
}, &response)
if err != nil {
return err
}
if response.Status != "ACQUISITION_COMPLETE" {
return errors.New("internal API did not complete preview")
}
return nil
}
// APIError intentionally omits the server's free-form message, which can
// contain sensitive upstream details. Callers can act on Code and RetryAt.
type APIError struct {
Status int
Code string
RetryAt time.Time
}
func (e *APIError) Error() string {
return fmt.Sprintf("internal API returned HTTP %d (%s)", e.Status, e.SafeCode())
}
// SafeCode is suitable for logs. The response code is untrusted HTTP content.
func (e *APIError) SafeCode() string {
switch e.Code {
case "ADMISSION_STALE", "BACKPRESSURE_ACTIVE", "CANDIDATE_BUDGET_EXCEEDED",
"CHECKPOINT_CONFLICT", "COMMIT_MISMATCH", "CONTROL_CHANGED", "PLATFORM_SCOPE_MISMATCH", "COVERAGE_CONFLICT",
"COVERAGE_UNSUPPORTED", "INVALID_REQUEST", "LEASE_STALE",
"PACKAGE_TOO_LARGE", "PATH_IGNORED", "PROTOCOL_UNSUPPORTED",
"REPOSITORY_MISMATCH", "RESULT_CONFLICT", "UPLOAD_MISMATCH",
"UPLOAD_NOT_READY", "UPLOAD_TICKET_INVALID", "SERVICE_UNAVAILABLE":
return e.Code
default:
return "CONTROL_PLANE_ERROR"
}
}
func (c *Client) Claim(ctx context.Context, workerID, kind string) (*ClaimedTask, error) {
return c.ClaimWithCapabilities(ctx, workerID, kind, nil)
}
// ClaimWithCapabilities is opt-in. Capability declarations must only be supplied
// by an executor that implements their full lifecycle, including safe pause.
func (c *Client) ClaimWithCapabilities(ctx context.Context, workerID, kind string, capabilities []string) (*ClaimedTask, error) {
if workerID == "" || len(workerID) > 128 ||
(kind != "DISCOVERY" && kind != "REPOSITORY_SYNC") {
return nil, errors.New("invalid worker ID or task kind")
}
if err := validateCapabilities(capabilities); err != nil {
return nil, err
}
var task ClaimedTask
request := map[string]any{
"workerId": workerID, "protocolMajor": 1, "kind": kind,
}
if len(capabilities) > 0 {
request["capabilities"] = capabilities
}
status, err := c.post(ctx, "/tasks/claim", request, &task)
if err != nil {
return nil, err
}
if status == http.StatusNoContent {
return nil, nil
}
if task.TaskID == "" || task.RunID == "" || task.NamespaceID <= 0 ||
task.Lease.Generation < 1 || task.Lease.Token == "" || task.LeaseExpiresAt.IsZero() ||
task.Kind != kind {
return nil, errors.New("internal API returned an incomplete task")
}
if err := validateOfficialTask(task, capabilities); err != nil {
return nil, err
}
if task.OfficialDiscovery != nil {
return &task, nil
}
if task.Kind == "DISCOVERY" && (task.Discovery == nil || task.Discovery.RuleVersion < 1 ||
task.Discovery.CandidateBudget < 1 ||
(task.Discovery.EntryType == "SEARCH_RULE" &&
(task.Discovery.Criteria == nil || task.Discovery.SearchWindow == nil)) ||
(task.Discovery.EntryType == "CURATED_OWNER" && task.Discovery.Owner == "")) {
return nil, errors.New("internal API returned an incomplete discovery task")
}
if task.Kind == "DISCOVERY" && task.Discovery.EntryType == "CURATED_REPOSITORY" &&
(task.Discovery.Owner == "" || task.Discovery.RepositoryName == "") {
return nil, errors.New("internal API returned an incomplete discovery task")
}
if task.Kind == "REPOSITORY_SYNC" && (task.Repository == nil || task.Repository.RepositoryID <= 0 ||
task.Repository.Owner == "" || task.Repository.Name == "" ||
(task.Mode != "FULL" && task.Mode != "HEAD_ONLY")) {
return nil, errors.New("internal API returned an incomplete repository task")
}
if task.Kind == "REPOSITORY_SYNC" && task.Mode == "FULL" &&
(task.MaxPackageBytes < 1 || task.MaxSingleFileBytes < 1 ||
task.MaxFilesPerPackage < 1 || task.MaxUploadBytes < 1) {
return nil, errors.New("internal API returned no package limits")
}
return &task, nil
}
func (c *Client) Renew(ctx context.Context, taskID string, lease LeaseProof) (time.Time, error) {
var response struct {
LeaseExpiresAt time.Time `json:"leaseExpiresAt"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/renew", map[string]any{
"lease": lease,
}, &response)
if err != nil {
return time.Time{}, err
}
if response.LeaseExpiresAt.IsZero() {
return time.Time{}, errors.New("internal API returned no lease expiration")
}
return response.LeaseExpiresAt, nil
}
// AdmitPackage reserves capacity before the worker requests any file content for this Skill.
func (c *Client) AdmitPackage(ctx context.Context, taskID string, lease LeaseProof,
commitSHA, skillPath string) (PackageAdmission, error) {
var admission PackageAdmission
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/package-admissions", map[string]any{
"lease": lease, "commitSha": commitSHA, "skillPath": skillPath,
}, &admission)
if err != nil {
return PackageAdmission{}, err
}
if admission.AdmissionID == "" {
return PackageAdmission{}, errors.New("internal API returned no package admission ID")
}
return admission, nil
}
// ReportDiscoveryBatch persists leads and advances an opaque checkpoint atomically.
// A replay with the same batch succeeds; a stale previous checkpoint is rejected.
func (c *Client) ReportDiscoveryBatch(ctx context.Context, taskID string,
lease LeaseProof, batch DiscoveryBatch) error {
if err := validateOfficialBatch(batch); err != nil {
return err
}
if batch.Checkpoint == "" || len(batch.Checkpoint) > 4096 || len(batch.Repositories) > 100 ||
batch.DeferredCount < 0 || (batch.DeferredCount > 0 && batch.DeferralReason == "") {
return errors.New("invalid discovery batch")
}
for _, repository := range batch.Repositories {
if repository.RepositoryID <= 0 || repository.Owner == "" || repository.Name == "" {
return errors.New("invalid discovery repository candidate")
}
}
if batch.Repositories == nil {
batch.Repositories = []RepositoryCandidate{}
}
status, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/discovered-repositories",
struct {
Lease LeaseProof `json:"lease"`
DiscoveryBatch
}{Lease: lease, DiscoveryBatch: batch}, nil)
if err != nil {
return err
}
if status != http.StatusAccepted {
return errors.New("internal API did not accept discovery batch")
}
if observer, ok := ctx.Value(checkpointObserverKey{}).(func(string)); ok {
observer(batch.Checkpoint)
}
return nil
}
// CompleteDiscovery closes a fully covered search or releases its lease for a
// delayed continuation. The checkpoint must already have been acknowledged.
func (c *Client) CompleteDiscovery(ctx context.Context, taskID string, lease LeaseProof,
checkpoint string, retryAt time.Time, deferralReason string) (string, error) {
return c.completeDiscovery(ctx, taskID, lease, checkpoint, retryAt, deferralReason, "RETRY_WAIT")
}
func (c *Client) completeDiscovery(ctx context.Context, taskID string, lease LeaseProof,
checkpoint string, retryAt time.Time, deferralReason, deferredStatus string) (string, error) {
if checkpoint == "" || len(checkpoint) > 4096 ||
(retryAt.IsZero() && deferralReason != "") ||
(!retryAt.IsZero() && !retryAt.After(time.Now())) {
return "", errors.New("invalid discovery completion")
}
coverage := "COMPLETE"
if !retryAt.IsZero() {
coverage = "INCOMPLETE"
}
request := map[string]any{
"lease": lease, "coverage": coverage, "discoveryCheckpoint": checkpoint,
}
if !retryAt.IsZero() {
request["retryAt"] = retryAt
if deferralReason != "" {
request["deferralReason"] = deferralReason
}
}
var response struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", request, &response)
if err != nil {
return "", err
}
if (coverage == "COMPLETE" && response.Status != "ACQUISITION_COMPLETE") ||
(coverage == "INCOMPLETE" && response.Status != deferredStatus) {
return "", errors.New("internal API returned inconsistent discovery status")
}
return response.Status, nil
}
// CompleteHeadOnly records an observed commit without claiming package acquisition finished.
func (c *Client) CompleteHeadOnly(ctx context.Context, taskID string, lease LeaseProof, observedCommitSHA string) error {
if len(observedCommitSHA) != 40 {
return errors.New("observed commit SHA must have 40 characters")
}
var response struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", map[string]any{
"lease": lease, "coverage": "HEAD_ONLY", "observedCommitSha": observedCommitSHA,
}, &response)
if err != nil {
return err
}
if response.Status != "ACQUISITION_COMPLETE" {
return errors.New("internal API did not complete HEAD_ONLY task")
}
return nil
}
// CompleteRepository closes a fixed-revision acquisition or records a delayed
// continuation. Accepted paths must already have been acknowledged by Java.
func (c *Client) CompleteRepository(ctx context.Context, taskID string, lease LeaseProof,
completion RepositoryCompletion) (string, error) {
if len(completion.CommitSHA) != 40 || len(completion.PathCheckpoint) > 1024 ||
len(completion.IncompleteSubtrees) > 1000 || len(completion.PathResults) > 10000 {
return "", errors.New("invalid repository completion")
}
for _, digit := range completion.CommitSHA {
if digit < '0' || digit > '9' && (digit < 'a' || digit > 'f') {
return "", errors.New("invalid repository commit SHA")
}
}
coverage := "COMPLETE"
if !completion.RetryAt.IsZero() {
if !completion.RetryAt.After(time.Now()) {
return "", errors.New("repository retry time must be in the future")
}
coverage = "INCOMPLETE"
} else if completion.DeferralReason != "" || len(completion.IncompleteSubtrees) > 0 {
return "", errors.New("complete repository has deferred coverage")
}
for _, result := range completion.PathResults {
if result.SkillPath == "" || (result.Outcome == "ACCEPTED") != (result.IngestionID != "") {
return "", errors.New("invalid repository path result")
}
switch result.Outcome {
case "ACCEPTED", "IGNORED", "TERMINAL_FAILURE":
case "RETRYABLE_FAILURE":
if coverage == "COMPLETE" {
return "", errors.New("complete repository has retryable path")
}
default:
return "", errors.New("invalid repository path outcome")
}
}
request := map[string]any{
"lease": lease, "coverage": coverage, "observedCommitSha": completion.CommitSHA,
"pathResults": completion.PathResults, "incompleteSubtrees": completion.IncompleteSubtrees,
}
if completion.PathCheckpoint != "" {
request["pathCheckpoint"] = completion.PathCheckpoint
}
if coverage == "INCOMPLETE" {
request["retryAt"] = completion.RetryAt
if completion.DeferralReason != "" {
request["deferralReason"] = completion.DeferralReason
}
}
var response struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", request, &response)
if err != nil {
return "", err
}
if coverage == "COMPLETE" && response.Status != "ACQUISITION_COMPLETE" && response.Status != "PARTIAL" ||
coverage == "INCOMPLETE" && response.Status != "RETRY_WAIT" {
return "", errors.New("internal API returned inconsistent repository status")
}
return response.Status, nil
}
// Fail releases the lease and returns the Java-owned retry time.
func (c *Client) Fail(ctx context.Context, taskID string, lease LeaseProof,
reason, detail string, retryAt time.Time, quota *GithubQuota) (time.Time, error) {
request := map[string]any{"lease": lease, "reason": reason}
if detail != "" {
request["detail"] = detail
}
if quota != nil {
request["githubQuota"] = quota
}
if !retryAt.IsZero() {
request["retryAt"] = retryAt
}
var response struct {
Status string `json:"status"`
RetryAt time.Time `json:"retryAt"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/fail", request, &response)
if err != nil {
return time.Time{}, err
}
if response.Status != "RETRY_WAIT" || response.RetryAt.IsZero() {
return time.Time{}, errors.New("internal API returned no retry schedule")
}
return response.RetryAt, nil
}
// ReportCredentialStatus updates only the credential bound to this current task lease.
func (c *Client) ReportCredentialStatus(ctx context.Context, taskID string, lease LeaseProof,
credentialID, status string, retryAt time.Time, quota *GithubQuota) error {
request := map[string]any{"lease": lease, "credentialId": credentialID, "status": status}
if !retryAt.IsZero() {
request["retryAt"] = retryAt
}
if quota != nil {
request["githubQuota"] = quota
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/credential-status", request, nil)
return err
}
func (c *Client) post(ctx context.Context, path string, body any, target any) (int, error) {
encoded, err := json.Marshal(body)
if err != nil {
return 0, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(encoded))
if err != nil {
return 0, err
}
requestID := make([]byte, 16)
if _, err := rand.Read(requestID); err != nil {
return 0, err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("X-Request-Id", hex.EncodeToString(requestID))
request.Header.Set("Content-Type", "application/json")
response, err := c.http.Do(request)
if err != nil {
return 0, err
}
defer response.Body.Close()
limited := io.LimitReader(response.Body, maxResponseBytes+1)
if response.StatusCode < 200 || response.StatusCode >= 300 {
var apiError struct {
Code string `json:"code"`
RetryAt time.Time `json:"retryAt"`
}
_ = json.NewDecoder(limited).Decode(&apiError)
return response.StatusCode, &APIError{Status: response.StatusCode, Code: apiError.Code, RetryAt: apiError.RetryAt}
}
if response.StatusCode == http.StatusNoContent || target == nil {
return response.StatusCode, nil
}
contents, err := io.ReadAll(limited)
if err != nil || len(contents) > maxResponseBytes {
return 0, errors.New("internal API response exceeds size limit")
}
if err := json.Unmarshal(contents, target); err != nil {
return 0, fmt.Errorf("decode internal API response: %w", err)
}
return response.StatusCode, nil
}

259
github-sync-go/internal/controlplane/official.go Normal file
View file

@ -0,0 +1,259 @@
package controlplane
import (
"errors"
"slices"
"time"
"unicode/utf8"
)
const (
OfficialDiscoveryCapability = "official-discovery/v1"
OfficialCheckpointCapability = "official-checkpoint/v1"
OfficialPauseCapability = "official-pause/v1"
OfficialOwnerCapability = "official-owner/v1"
)
// OfficialDiscoverySnapshot is an immutable Java-owned policy and shard snapshot.
// Decoding it does not imply that this worker implements the official executor.
type OfficialDiscoverySnapshot struct {
FormatVersion int `json:"formatVersion"`
ConfigurationVersion int64 `json:"configurationVersion"`
StrategyVersion string `json:"strategyVersion"`
QueryID string `json:"queryId"`
QueryVersion int64 `json:"queryVersion"`
ShardID string `json:"shardId"`
Channel string `json:"channel"`
Mode string `json:"mode"`
Query OfficialQuery `json:"query"`
Range OfficialQueryRange `json:"range"`
FixedCutoffAt time.Time `json:"fixedCutoffAt"`
SeedRepositoryID int64 `json:"seedRepositoryId,omitempty"`
OwnerID int64 `json:"ownerId,omitempty"`
OwnerLogin string `json:"ownerLogin,omitempty"`
ExpansionDepth int `json:"expansionDepth,omitempty"`
Checkpoint string `json:"checkpoint,omitempty"`
Budget OfficialBudget `json:"budget"`
}
type OfficialQuery struct {
Kind string `json:"kind"`
Value string `json:"value"`
Scope string `json:"scope"`
}
type OfficialQueryRange struct {
Since time.Time `json:"since"`
Until time.Time `json:"until"`
Partition string `json:"partition,omitempty"`
}
type OfficialBudget struct {
Profile string `json:"profile"`
MaxRequests int64 `json:"maxRequests"`
MaxPages int64 `json:"maxPages"`
CandidateBudget int64 `json:"candidateBudget"`
MaxSplitDepth int64 `json:"maxSplitDepth"`
MaxChildShards int64 `json:"maxChildShards"`
}
type OfficialCheckpoint struct {
FormatVersion int `json:"formatVersion"`
StrategyVersion string `json:"strategyVersion"`
QueryID string `json:"queryId"`
QueryVersion int64 `json:"queryVersion"`
ShardID string `json:"shardId"`
NextPage int64 `json:"nextPage"`
Cursor string `json:"cursor,omitempty"`
Coverage string `json:"coverage"`
}
type OfficialProvenance struct {
Channel string `json:"channel"`
QueryID string `json:"queryId"`
QueryVersion int64 `json:"queryVersion"`
StrategyVersion string `json:"strategyVersion"`
ShardID string `json:"shardId"`
SeedRepositoryID int64 `json:"seedRepositoryId,omitempty"`
}
type OfficialDiscoveryBatch struct {
OwnerLogin string `json:"ownerLogin,omitempty"`
ShardID string `json:"shardId"`
Coverage string `json:"coverage"`
RequestsUsed int64 `json:"requestsUsed"`
ChildRanges []OfficialQueryRange `json:"childRanges,omitempty"`
GapReason string `json:"gapReason,omitempty"`
}
type OfficialTaskControl struct {
Action string `json:"action"`
ConfigurationVersion int64 `json:"configurationVersion"`
Reason string `json:"reason,omitempty"`
RetryAt *time.Time `json:"retryAt,omitempty"`
}
// Null commit/checkpoint values must stay explicit on the wire. No commit is
// required when pausing before the first repository read has completed.
type OfficialRepositoryProgress struct {
ObservedCommitSHA *string `json:"observedCommitSha"`
PathCheckpoint *string `json:"pathCheckpoint"`
IncompleteSubtrees []string `json:"incompleteSubtrees"`
PathResults []RepositoryPathResult `json:"pathResults"`
}
type OfficialCheckpointRecovery struct {
Lease LeaseProof `json:"lease"`
Reason string `json:"reason"`
RejectedCheckpoint string `json:"rejectedCheckpoint"`
}
type OfficialQuotaObservation struct {
Resource string `json:"resource"`
ObservedAt time.Time `json:"observedAt"`
State string `json:"state"`
Remaining *int `json:"remaining,omitempty"`
ResetAt *time.Time `json:"resetAt,omitempty"`
RetryAfterSeconds *int `json:"retryAfterSeconds,omitempty"`
}
func validateCapabilities(capabilities []string) error {
if len(capabilities) > 32 {
return errors.New("too many worker capabilities")
}
for _, capability := range capabilities {
if !boundedOfficialString(capability, 128) {
return errors.New("invalid worker capability")
}
}
return nil
}
func boundedOfficialString(value string, maximum int) bool {
return value != "" && utf8.ValidString(value) && utf8.RuneCountInString(value) <= maximum
}
func validOfficialTime(value time.Time) bool {
_, offset := value.Zone()
return !value.IsZero() && value.Nanosecond() == 0 && offset == 0
}
func validOfficialRange(window OfficialQueryRange) bool {
return validOfficialTime(window.Since) && validOfficialTime(window.Until) &&
window.Since.Before(window.Until) && window.Partition == ""
}
func validateOfficialTask(task ClaimedTask, capabilities []string) error {
if err := validateCapabilities(task.RequiredCapabilities); err != nil {
return err
}
for _, required := range task.RequiredCapabilities {
if !slices.Contains(capabilities, required) {
return errors.New("internal API returned a task requiring an undeclared capability")
}
}
if task.PlatformControlVersion != nil && (*task.PlatformControlVersion < 1 || !slices.Contains(task.RequiredCapabilities, PlatformPauseCapability)) {
return errors.New("internal API returned invalid platform control capability")
}
if slices.Contains(task.RequiredCapabilities, PlatformPauseCapability) && task.PlatformControlVersion == nil {
return errors.New("internal API omitted platform control version")
}
snapshot := task.OfficialDiscovery
if snapshot == nil {
if task.Kind == "DISCOVERY" {
for _, required := range task.RequiredCapabilities {
if required != PlatformPauseCapability {
return errors.New("internal API returned capabilities without an official discovery snapshot")
}
}
}
return nil
}
if task.Kind != "DISCOVERY" || task.Discovery != nil || task.Repository != nil {
return errors.New("internal API returned conflicting discovery snapshots")
}
for _, required := range []string{OfficialDiscoveryCapability, OfficialCheckpointCapability, OfficialPauseCapability} {
if !slices.Contains(task.RequiredCapabilities, required) {
return errors.New("internal API omitted a required official discovery capability")
}
}
if snapshot.FormatVersion != 1 || snapshot.ConfigurationVersion < 1 || snapshot.QueryVersion < 1 ||
!boundedOfficialString(snapshot.StrategyVersion, 128) || !boundedOfficialString(snapshot.QueryID, 128) ||
snapshot.ShardID == "" || !boundedOfficialString(snapshot.Query.Value, 256) ||
!validOfficialRange(snapshot.Range) || !validOfficialTime(snapshot.FixedCutoffAt) ||
snapshot.Range.Until.After(snapshot.FixedCutoffAt) || len(snapshot.Checkpoint) > 4096 ||
!utf8.ValidString(snapshot.Checkpoint) {
return errors.New("internal API returned an invalid official discovery snapshot")
}
if !slices.Contains([]string{"HISTORICAL", "INCREMENTAL", "GAP_RECHECK", "OWNER_RECHECK"}, snapshot.Mode) {
return errors.New("internal API returned an unsupported official discovery mode")
}
budget := snapshot.Budget
if !slices.Contains([]string{"ECONOMY", "BALANCED", "ACCELERATED"}, budget.Profile) ||
budget.MaxRequests < 1 || budget.MaxPages < 1 || budget.CandidateBudget < 1 ||
budget.MaxSplitDepth < 0 || budget.MaxSplitDepth > 32 || budget.MaxChildShards < 0 || budget.MaxChildShards > 126 {
return errors.New("internal API returned an invalid official discovery budget")
}
switch snapshot.Channel {
case "REPOSITORY_METADATA":
if snapshot.Query.Scope != "NAME_DESCRIPTION_TOPICS" ||
(snapshot.Query.Kind != "KEYWORD" && snapshot.Query.Kind != "TOPIC") {
return errors.New("invalid official metadata query")
}
case "README":
if snapshot.Query.Scope != "README" || snapshot.Query.Kind != "KEYWORD" {
return errors.New("invalid official README query")
}
case "OWNER_EXPANSION":
if !slices.Contains(task.RequiredCapabilities, OfficialOwnerCapability) || snapshot.Mode != "OWNER_RECHECK" || budget.MaxRequests < 3 || budget.MaxSplitDepth != 0 || budget.MaxChildShards != 0 || snapshot.Range.Partition != "" || snapshot.Query.Scope != "OWNER_REPOSITORIES" || snapshot.Query.Kind != "OWNER" ||
snapshot.SeedRepositoryID <= 0 || snapshot.OwnerID <= 0 ||
!boundedOfficialString(snapshot.OwnerLogin, 39) || snapshot.ExpansionDepth != 1 ||
snapshot.Query.Value != snapshot.OwnerLogin {
return errors.New("invalid official owner expansion")
}
default:
return errors.New("unsupported official discovery channel")
}
if snapshot.Channel != "OWNER_EXPANSION" &&
(snapshot.SeedRepositoryID != 0 || snapshot.OwnerID != 0 || snapshot.OwnerLogin != "" || snapshot.ExpansionDepth != 0) {
return errors.New("official search contains owner expansion metadata")
}
return nil
}
func validateOfficialBatch(batch DiscoveryBatch) error {
official := batch.OfficialBatch
for _, repository := range batch.Repositories {
if len(repository.OfficialProvenance) > 16 ||
(official == nil) != (len(repository.OfficialProvenance) == 0) {
return errors.New("invalid official provenance count or batch")
}
for _, provenance := range repository.OfficialProvenance {
if (official.OwnerLogin != "" && provenance.Channel != "OWNER_EXPANSION") || !boundedOfficialString(provenance.QueryID, 128) || provenance.QueryVersion < 1 ||
!boundedOfficialString(provenance.StrategyVersion, 128) || provenance.ShardID != official.ShardID ||
!slices.Contains([]string{"REPOSITORY_METADATA", "README", "OWNER_EXPANSION"}, provenance.Channel) ||
(provenance.Channel == "OWNER_EXPANSION" && provenance.SeedRepositoryID < 1) ||
(provenance.Channel != "OWNER_EXPANSION" && provenance.SeedRepositoryID != 0) {
return errors.New("invalid official repository provenance")
}
}
}
if official == nil {
return nil
}
if (official.OwnerLogin != "" && !boundedOfficialString(official.OwnerLogin, 39)) || official.ShardID == "" || official.RequestsUsed < 0 || len(official.ChildRanges) > 16 ||
!slices.Contains([]string{"IN_PROGRESS", "COMPLETE", "SPLIT", "GAP"}, official.Coverage) ||
(official.Coverage == "SPLIT") != (len(official.ChildRanges) > 0) ||
(official.Coverage == "GAP") != (official.GapReason != "") {
return errors.New("invalid official discovery batch")
}
if official.GapReason != "" && !slices.Contains([]string{"RESULT_LIMIT", "INCOMPLETE_RESULTS", "UNSPLITTABLE", "WORK_BUDGET"}, official.GapReason) {
return errors.New("invalid official discovery gap reason")
}
for _, child := range official.ChildRanges {
if !validOfficialRange(child) {
return errors.New("invalid official child range")
}
}
return nil
}

246
github-sync-go/internal/controlplane/official_client.go Normal file
View file

@ -0,0 +1,246 @@
package controlplane
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/url"
"slices"
"time"
"unicode/utf8"
)
// CompleteOfficialDiscovery releases a search GAP as PARTIAL for its siblings;
// owner GAPs retain RETRY_WAIT. The acknowledged checkpoint is sent unchanged.
func (c *Client) CompleteOfficialDiscovery(ctx context.Context, taskID string, lease LeaseProof,
checkpoint string, retryAt time.Time, deferralReason, channel string) (string, error) {
var progress OfficialCheckpoint
if err := json.Unmarshal([]byte(checkpoint), &progress); err != nil || progress.FormatVersion != 1 ||
!slices.Contains([]string{"REPOSITORY_METADATA", "README", "OWNER_EXPANSION"}, channel) ||
(channel == "OWNER_EXPANSION" && progress.Coverage == "SPLIT") {
return "", errors.New("invalid official discovery completion checkpoint")
}
deferredStatus := "RETRY_WAIT"
switch progress.Coverage {
case "GAP":
if retryAt.IsZero() {
return "", errors.New("official GAP completion requires a retry time")
}
if channel != "OWNER_EXPANSION" {
deferredStatus = "PARTIAL"
}
case "IN_PROGRESS":
if retryAt.IsZero() {
return "", errors.New("unfinished official discovery requires a retry time")
}
case "COMPLETE", "SPLIT":
if !retryAt.IsZero() {
return "", errors.New("finished official shard cannot be deferred")
}
default:
return "", errors.New("unsupported official discovery completion coverage")
}
return c.completeDiscovery(ctx, taskID, lease, checkpoint, retryAt, deferralReason, deferredStatus)
}
// HeartbeatWithCapabilities is separate from the legacy empty heartbeat so
// protocol scaffolding cannot accidentally advertise an executable capability.
func (c *Client) HeartbeatWithCapabilities(ctx context.Context, workerID string, capabilities []string) error {
if !boundedOfficialString(workerID, 128) {
return errors.New("invalid worker ID")
}
if err := validateCapabilities(capabilities); err != nil {
return err
}
if capabilities == nil {
capabilities = []string{}
}
status, err := c.post(ctx, "/heartbeat", map[string]any{
"workerId": workerID, "protocolMajor": 1, "capabilities": capabilities,
}, nil)
if err != nil {
return err
}
if status != http.StatusNoContent {
return errors.New("internal API did not accept worker heartbeat")
}
return nil
}
func (c *Client) GetOfficialTaskControl(ctx context.Context, taskID string, lease LeaseProof) (OfficialTaskControl, error) {
var control OfficialTaskControl
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-control", map[string]any{"lease": lease}, &control)
if err != nil {
return OfficialTaskControl{}, err
}
if control.ConfigurationVersion < 1 || (control.Action != "CONTINUE" && control.Action != "PAUSE") ||
(control.Action == "PAUSE" && !slices.Contains([]string{"DISABLED", "SOURCE_PAUSED", "EXCLUDED", "CANDIDATE_CAPACITY", "REVIEW_BACKPRESSURE", "QUOTA_COOLDOWN"}, control.Reason)) {
return OfficialTaskControl{}, errors.New("internal API returned invalid official task control")
}
if control.Reason == "QUOTA_COOLDOWN" && (control.RetryAt == nil || control.RetryAt.IsZero()) {
return OfficialTaskControl{}, errors.New("quota cooldown requires retry time")
}
return control, nil
}
// PauseOfficialDiscovery saves only the last acknowledged checkpoint. A nil
// checkpoint is sent as null when no batch has been accepted yet.
func (c *Client) PauseOfficialDiscovery(ctx context.Context, taskID string, lease LeaseProof, checkpoint *string) error {
if checkpoint != nil && (len(*checkpoint) > 4096 || !utf8.ValidString(*checkpoint)) {
return errors.New("invalid official discovery pause checkpoint")
}
return c.pauseOfficial(ctx, taskID, map[string]any{
"lease": lease, "kind": "DISCOVERY", "discoveryCheckpoint": checkpoint,
})
}
// PauseOfficialRepository also supports pausing before a commit is known. The
// server atomically checks the latest ownership, accepted paths and fixed commit.
func (c *Client) PauseOfficialRepository(ctx context.Context, taskID string, lease LeaseProof, progress OfficialRepositoryProgress) error {
if progress.ObservedCommitSHA == nil {
if progress.PathCheckpoint != nil || len(progress.IncompleteSubtrees) != 0 || len(progress.PathResults) != 0 {
return errors.New("official repository pause has progress without a commit")
}
} else if !validOfficialCommit(*progress.ObservedCommitSHA) {
return errors.New("invalid official repository pause commit")
}
if (progress.PathCheckpoint != nil && (!utf8.ValidString(*progress.PathCheckpoint) || utf8.RuneCountInString(*progress.PathCheckpoint) > 1024)) ||
len(progress.IncompleteSubtrees) > 1000 || len(progress.PathResults) > 10000 {
return errors.New("official repository pause exceeds progress limits")
}
for _, path := range progress.IncompleteSubtrees {
if !boundedOfficialString(path, 1024) {
return errors.New("invalid official incomplete subtree")
}
}
seen := make(map[string]bool, len(progress.PathResults))
for _, result := range progress.PathResults {
if !boundedOfficialString(result.SkillPath, 1024) || seen[result.SkillPath] ||
!utf8.ValidString(result.Reason) || utf8.RuneCountInString(result.Reason) > 500 ||
!slices.Contains([]string{"ACCEPTED", "IGNORED", "RETRYABLE_FAILURE", "TERMINAL_FAILURE"}, result.Outcome) ||
(result.Outcome == "ACCEPTED") != (result.IngestionID != "") {
return errors.New("invalid official repository pause path result")
}
seen[result.SkillPath] = true
}
if progress.IncompleteSubtrees == nil {
progress.IncompleteSubtrees = []string{}
}
if progress.PathResults == nil {
progress.PathResults = []RepositoryPathResult{}
}
return c.pauseOfficial(ctx, taskID, map[string]any{
"lease": lease, "kind": "REPOSITORY_SYNC", "repositoryProgress": progress,
})
}
func validOfficialCommit(commit string) bool {
if len(commit) != 40 {
return false
}
for _, digit := range commit {
if (digit < '0' || digit > '9') && (digit < 'a' || digit > 'f') {
return false
}
}
return true
}
// On APIError.Code CONTROL_CHANGED, callers must re-read control and retain
// their lease. This endpoint-specific code does not extend legacy enums.
func (c *Client) pauseOfficial(ctx context.Context, taskID string, request any) error {
var response struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-pause", request, &response)
if err != nil {
return err
}
if response.Status != "PAUSED" {
return errors.New("internal API did not pause official task")
}
return nil
}
// RecoverOfficialCheckpoint returns the replacement task ID, not a new lease.
// The server has retired the old task; the caller must stop writing to it.
func (c *Client) RecoverOfficialCheckpoint(ctx context.Context, taskID string, recovery OfficialCheckpointRecovery) (string, error) {
if !slices.Contains([]string{"MALFORMED", "FORMAT_UNSUPPORTED", "IDENTITY_MISMATCH"}, recovery.Reason) ||
len(recovery.RejectedCheckpoint) > 4096 || !utf8.ValidString(recovery.RejectedCheckpoint) {
return "", errors.New("invalid official checkpoint recovery")
}
var response struct {
Status string `json:"status"`
ReplacementTaskID string `json:"replacementTaskId"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-checkpoint-recovery", recovery, &response)
if err != nil {
return "", err
}
if response.Status != "REPLAY_SCHEDULED" || response.ReplacementTaskID == "" {
return "", errors.New("internal API did not schedule checkpoint replay")
}
return response.ReplacementTaskID, nil
}
// ReportOfficialCredentialStatus carries the optional observation while leaving
// the legacy credential-status enum and payload unchanged for existing callers.
func (c *Client) ReportOfficialCredentialStatus(ctx context.Context, taskID string, lease LeaseProof,
credentialID, status string, retryAt time.Time, quota *GithubQuota, observation OfficialQuotaObservation) error {
if !slices.Contains([]string{"OBSERVED", "AUTH_INVALID", "RATE_LIMITED"}, status) ||
(status == "RATE_LIMITED") != !retryAt.IsZero() || credentialID == "" ||
(status == "OBSERVED" && quota == nil) ||
(observation.State == "UNKNOWN" && quota != nil) ||
!slices.Contains([]string{"CORE", "SEARCH", "GRAPHQL", "OTHER"}, observation.Resource) || observation.ObservedAt.IsZero() ||
!slices.Contains([]string{"OBSERVED", "UNKNOWN", "PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN", "AUTH_INVALID"}, observation.State) ||
(observation.Remaining != nil && *observation.Remaining < 0) ||
(observation.ResetAt != nil && observation.ResetAt.IsZero()) ||
(observation.RetryAfterSeconds != nil && *observation.RetryAfterSeconds < 0) ||
(observation.State == "UNKNOWN" && observation.Remaining != nil) ||
((observation.State == "OBSERVED" || observation.State == "PRIMARY_EXHAUSTED") && (observation.Remaining == nil || observation.ResetAt == nil)) {
return errors.New("invalid official credential observation")
}
request := map[string]any{
"lease": lease, "credentialId": credentialID, "status": status, "officialQuotaObservation": observation,
}
if quota != nil {
request["githubQuota"] = quota
}
if !retryAt.IsZero() {
request["retryAt"] = retryAt
}
responseStatus, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/credential-status", request, nil)
if err != nil {
return err
}
if responseStatus != http.StatusNoContent {
return errors.New("internal API did not accept official credential observation")
}
return nil
}
// ReportOfficialQuota lets Java derive the quota subject from the lease, including
// anonymous requests. It never accepts a caller-selected credential or subject.
func (c *Client) ReportOfficialQuota(ctx context.Context, taskID string, lease LeaseProof, observation OfficialQuotaObservation) error {
now := time.Now()
if !slices.Contains([]string{"CORE", "SEARCH", "GRAPHQL", "OTHER"}, observation.Resource) || observation.ObservedAt.IsZero() ||
observation.ObservedAt.Before(now.Add(-5*time.Minute)) || observation.ObservedAt.After(now.Add(5*time.Minute)) ||
!slices.Contains([]string{"OBSERVED", "UNKNOWN", "PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN", "AUTH_INVALID"}, observation.State) ||
(observation.Remaining != nil && *observation.Remaining < 0) ||
(observation.ResetAt != nil && (observation.ResetAt.IsZero() || observation.ResetAt.After(now.Add(24*time.Hour)))) ||
(observation.RetryAfterSeconds != nil && (*observation.RetryAfterSeconds < 0 || *observation.RetryAfterSeconds > 86400)) ||
(observation.State == "UNKNOWN" && (observation.Remaining != nil || observation.ResetAt != nil)) ||
(observation.State == "PRIMARY_EXHAUSTED" && (observation.Remaining == nil || *observation.Remaining != 0 || observation.ResetAt == nil || !observation.ResetAt.After(now))) ||
((observation.State == "OBSERVED" || observation.State == "PRIMARY_EXHAUSTED") && (observation.Remaining == nil || observation.ResetAt == nil)) {
return errors.New("invalid official quota observation")
}
status, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-quota", map[string]any{"lease": lease, "observation": observation}, nil)
if err != nil {
return err
}
if status != http.StatusNoContent {
return errors.New("internal API did not accept official quota observation")
}
return nil
}

48
github-sync-go/internal/controlplane/platform.go Normal file
View file

@ -0,0 +1,48 @@
package controlplane
import (
"context"
"errors"
"net/url"
)
const PlatformPauseCapability = "platform-pause/v1"
type PlatformControl struct {
Action string `json:"action"`
PlatformVersion int64 `json:"platformVersion"`
}
type PlatformPause struct {
Lease LeaseProof `json:"lease"`
ObservedVersion int64 `json:"observedVersion"`
Kind string `json:"kind"`
DiscoveryCheckpoint *string `json:"discoveryCheckpoint,omitempty"`
RepositoryProgress *OfficialRepositoryProgress `json:"repositoryProgress,omitempty"`
}
func (c *Client) GetPlatformControl(ctx context.Context, task string, lease LeaseProof) (PlatformControl, error) {
var result PlatformControl
_, err := c.post(ctx, "/tasks/"+url.PathEscape(task)+"/platform-control", struct {
Lease LeaseProof `json:"lease"`
}{lease}, &result)
if err == nil && (result.PlatformVersion < 1 || result.Action != "CONTINUE" && result.Action != "PAUSE") {
err = errors.New("invalid platform control")
}
return result, err
}
func (c *Client) PausePlatform(ctx context.Context, task string, request PlatformPause) error {
var result struct {
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(task)+"/platform-pause", request, &result)
if err == nil && result.Status != "PAUSED" {
err = errors.New("platform pause not acknowledged")
}
return err
}
type checkpointObserverKey struct{}
func WithCheckpointObserver(ctx context.Context, observer func(string)) context.Context {
return context.WithValue(ctx, checkpointObserverKey{}, observer)
}

21
github-sync-go/internal/controlplane/platform_test.go Normal file
View file

@ -0,0 +1,21 @@
package controlplane
import "testing"
func TestPlatformCapabilityIsValidForLegacyDiscoveryAndRequiresControlVersion(t *testing.T) {
version := int64(1)
for _, entry := range []string{"CURATED_REPOSITORY", "SEARCH_RULE", "SEARCH_PREVIEW", "CURATED_OWNER"} {
task := ClaimedTask{Kind: "DISCOVERY", Discovery: &DiscoverySnapshot{EntryType: entry}, RequiredCapabilities: []string{PlatformPauseCapability}, PlatformControlVersion: &version}
if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err != nil {
t.Fatalf("%s: %v", entry, err)
}
task.PlatformControlVersion = nil
if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err == nil {
t.Fatal("accepted missing platform version")
}
}
task := ClaimedTask{Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version}
if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err == nil {
t.Fatal("accepted undeclared platform capability")
}
}

73
github-sync-go/internal/controlplane/testdata/discoveryinterop/main.go vendored Normal file
View file

@ -0,0 +1,73 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
const checkpoint = "interop-discovery-checkpoint"
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
phase := os.Getenv("SKILLHUB_INTEROP_DISCOVERY_PHASE")
if phase != "report" && phase != "resume" {
return errors.New("invalid discovery interoperability phase")
}
client, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
task, err := client.ClaimWithCapabilities(ctx, "interop-discovery-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
return err
}
if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Discovery == nil || task.Discovery.EntryType != "CURATED_OWNER" ||
task.Discovery.Owner != "interop-owner" {
return errors.New("Java did not lease the expected discovery task")
}
if expected := os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID"); expected != "" &&
(task.Credential == nil || task.Credential.CredentialID != expected ||
task.Credential.Token != os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN")) {
return errors.New("Java did not lease the expected fixture credential")
}
if phase == "report" && (task.Lease.Generation != 1 || task.Discovery.Checkpoint != "") ||
phase == "resume" && (task.Lease.Generation != 2 || task.Discovery.Checkpoint != checkpoint) {
return errors.New("Java did not preserve the expected discovery lease and checkpoint")
}
batch := controlplane.DiscoveryBatch{
Checkpoint: checkpoint,
Repositories: []controlplane.RepositoryCandidate{{
RepositoryID: 424242, Owner: "interop-owner", Name: "public-skills",
DefaultBranch: "main",
}},
}
if err := client.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch); err != nil {
return err
}
if phase == "report" {
fmt.Println("REPORTED")
return nil
}
status, err := client.CompleteDiscovery(ctx, task.TaskID, task.Lease,
checkpoint, time.Time{}, "")
if err != nil || status != "ACQUISITION_COMPLETE" {
return errors.New("reclaimed discovery lease did not complete the saved checkpoint")
}
fmt.Println("COMPLETED")
return nil
}

301
github-sync-go/internal/controlplane/testdata/httpinterop/main.go vendored Normal file
View file

@ -0,0 +1,301 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
javaURL := os.Getenv("SKILLHUB_INTEROP_JAVA_URL")
serviceToken := os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")
taskID := os.Getenv("SKILLHUB_INTEROP_TASK_ID")
commitSHA := os.Getenv("SKILLHUB_INTEROP_COMMIT_SHA")
skillPath := os.Getenv("SKILLHUB_INTEROP_SKILL_PATH")
archivePath := os.Getenv("SKILLHUB_INTEROP_ARCHIVE_PATH")
if taskID == "" || commitSHA == "" || skillPath == "" || archivePath == "" {
return errors.New("missing HTTP interoperability fixture")
}
client, err := controlplane.NewClient(javaURL, serviceToken)
if err != nil {
return err
}
archive, err := os.Open(archivePath)
if err != nil {
return err
}
defer archive.Close()
info, err := archive.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 {
return errors.New("invalid interoperability archive")
}
digest := sha256.New()
if _, err := io.Copy(digest, archive); err != nil {
return err
}
digestHex := hex.EncodeToString(digest.Sum(nil))
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
task, err := client.ClaimWithCapabilities(ctx, "interop-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
return err
}
if task == nil || task.TaskID != taskID || task.Mode != "FULL" ||
task.Repository == nil || task.Repository.FixedCommitSHA != commitSHA {
if task == nil {
return errors.New("Java did not lease a repository task")
}
return fmt.Errorf("Java leased task %s in mode %s, expected task %s in FULL mode",
task.TaskID, task.Mode, taskID)
}
mode := os.Getenv("SKILLHUB_INTEROP_MODE")
if mode == "resume-only" {
expectedIngestionID := os.Getenv("SKILLHUB_INTEROP_EXPECT_INGESTION_ID")
if expectedIngestionID == "" || len(task.Repository.AcceptedPaths) != 1 ||
task.Repository.AcceptedPaths[0] != skillPath || task.Lease.Generation < 2 {
return errors.New("reclaimed lease did not return the persisted package path")
}
status, err := client.CompleteRepository(ctx, taskID, task.Lease,
controlplane.RepositoryCompletion{CommitSHA: commitSHA,
PathResults: []controlplane.RepositoryPathResult{{
SkillPath: skillPath, Outcome: "ACCEPTED", IngestionID: expectedIngestionID,
}}})
if err != nil || status != "ACQUISITION_COMPLETE" {
return errors.New("reclaimed lease could not complete the persisted package")
}
fmt.Println(expectedIngestionID)
return nil
}
if mode != "" && mode != "report-only" && mode != "wait-for-java-outage" {
return errors.New("invalid HTTP interoperability mode")
}
admission, err := client.AdmitPackage(ctx, taskID, task.Lease, commitSHA, skillPath)
if err != nil {
return err
}
authorization, err := client.AuthorizeUpload(ctx, taskID, task.Lease,
admission.AdmissionID, commitSHA, skillPath, info.Size(), digestHex)
if err != nil {
return err
}
expectedMode := os.Getenv("SKILLHUB_INTEROP_UPLOAD_MODE")
if expectedMode == "" {
expectedMode = "LOCAL_STREAM"
}
if authorization.Mode != expectedMode {
return errors.New("Java returned an unexpected staging upload mode")
}
var allowedS3Hosts []string
if expectedMode == "S3_PRESIGNED_PUT" {
allowedHost := os.Getenv("SKILLHUB_INTEROP_ALLOWED_S3_HOST")
if allowedHost == "" {
return errors.New("missing allowed S3 fixture host")
}
allowedS3Hosts = []string{allowedHost}
if err := verifySignedUploadCannotWriteAnotherKey(ctx, authorization); err != nil {
return err
}
}
if err := client.UploadPackage(ctx, authorization, archivePath, allowedS3Hosts); err != nil {
return err
}
ingestionID, err := client.ReportPackage(ctx, taskID, task.Lease,
admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), digestHex)
if err != nil {
return err
}
replayedID, err := client.ReportPackage(ctx, taskID, task.Lease,
admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), digestHex)
if err != nil || replayedID != ingestionID {
return errors.New("Java did not replay the same package receipt")
}
forgedDigest := digestHex[:63] + "0"
if digestHex[63] == '0' {
forgedDigest = digestHex[:63] + "1"
}
_, err = client.ReportPackage(ctx, taskID, task.Lease,
admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), forgedDigest)
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Code != "RESULT_CONFLICT" {
return errors.New("Java accepted a conflicting package report")
}
if mode == "report-only" {
fmt.Println(ingestionID)
return nil
}
if mode == "wait-for-java-outage" {
if err := awaitJavaOutage(ctx, client, task, ingestionID); err != nil {
return err
}
fmt.Println(ingestionID)
return nil
}
status, err := client.CompleteRepository(ctx, taskID, task.Lease,
controlplane.RepositoryCompletion{
CommitSHA: commitSHA,
PathResults: []controlplane.RepositoryPathResult{{
SkillPath: skillPath, Outcome: "ACCEPTED", IngestionID: ingestionID,
}},
})
if err != nil {
return err
}
if status != "ACQUISITION_COMPLETE" {
return errors.New("Java did not complete the repository acquisition")
}
if oversizedTaskID := os.Getenv("SKILLHUB_INTEROP_OVERSIZED_TASK_ID"); oversizedTaskID != "" {
if err := verifyOversizedS3Object(ctx, client, oversizedTaskID, commitSHA,
skillPath, info.Size(), digestHex, allowedS3Hosts); err != nil {
return err
}
}
fmt.Println(ingestionID)
return nil
}
func awaitJavaOutage(ctx context.Context, client *controlplane.Client,
task *controlplane.ClaimedTask, ingestionID string) error {
readyPath := os.Getenv("SKILLHUB_INTEROP_READY_MARKER")
outagePath := os.Getenv("SKILLHUB_INTEROP_OUTAGE_MARKER")
if readyPath == "" || outagePath == "" {
return errors.New("missing Java outage fixture markers")
}
if err := os.WriteFile(readyPath, []byte(ingestionID), 0600); err != nil {
return err
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
if _, err := os.Stat(outagePath); err == nil {
break
} else if !os.IsNotExist(err) {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
renewContext, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := client.Renew(renewContext, task.TaskID, task.Lease)
if err == nil {
return errors.New("Java accepted renewal while the service was stopped")
}
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, context.Canceled) {
return errors.New("Java outage check expired before observing a connection failure")
}
var transportError *url.Error
if !errors.As(err, &transportError) {
return errors.New("Java outage check did not observe a transport failure")
}
return nil
}
func verifySignedUploadCannotWriteAnotherKey(ctx context.Context,
authorization controlplane.UploadAuthorization) error {
target, err := url.Parse(authorization.UploadURL)
if err != nil || target.Scheme == "" || target.Host == "" {
return errors.New("invalid signed upload target")
}
target.Path += ".forbidden"
target.RawPath = ""
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
target.String(), bytes.NewReader([]byte("forbidden")))
if err != nil {
return errors.New("could not create changed-key upload probe")
}
for name, value := range authorization.Headers {
request.Header.Set(name, value)
}
probeClient := &http.Client{Transport: &http.Transport{Proxy: nil},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
response, err := probeClient.Do(request)
if err != nil {
return errors.New("changed-key upload probe did not reach object storage")
}
response.Body.Close()
if response.StatusCode != http.StatusForbidden {
return fmt.Errorf("changed-key upload unexpectedly returned HTTP %d", response.StatusCode)
}
return nil
}
// This deliberately bypasses UploadPackage's client-side size guard to prove
// Java rejects and removes an oversized object accepted by a signed PUT URL.
func verifyOversizedS3Object(ctx context.Context, client *controlplane.Client,
taskID, commitSHA, skillPath string, size int64, digest string,
allowedHosts []string) error {
if len(allowedHosts) != 1 {
return errors.New("oversized object fixture requires S3 staging")
}
task, err := client.ClaimWithCapabilities(ctx, "interop-oversize-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
return err
}
if task == nil || task.TaskID != taskID {
if task == nil {
return errors.New("Java had no repository task for the oversized object fixture")
}
return fmt.Errorf("Java leased repository task %s instead of oversized fixture task %s",
task.TaskID, taskID)
}
admission, err := client.AdmitPackage(ctx, taskID, task.Lease, commitSHA, skillPath)
if err != nil {
return err
}
authorization, err := client.AuthorizeUpload(ctx, taskID, task.Lease,
admission.AdmissionID, commitSHA, skillPath, size, digest)
if err != nil {
return err
}
if authorization.Mode != "S3_PRESIGNED_PUT" || authorization.MaxBytes < size ||
authorization.MaxBytes > 1<<20 {
return errors.New("invalid oversized object fixture authorization")
}
maliciousBody := bytes.NewReader(make([]byte, authorization.MaxBytes+1))
request, err := http.NewRequestWithContext(ctx, http.MethodPut, authorization.UploadURL, maliciousBody)
if err != nil {
return err
}
for name, value := range authorization.Headers {
request.Header.Set(name, value)
}
uploadClient := &http.Client{Transport: &http.Transport{Proxy: nil},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
response, err := uploadClient.Do(request)
if err != nil {
return err
}
response.Body.Close()
if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent {
return errors.New("S3 did not accept the intentionally oversized object")
}
_, err = client.ReportPackage(ctx, taskID, task.Lease, admission.AdmissionID,
commitSHA, skillPath, authorization, size, digest)
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Status != http.StatusRequestEntityTooLarge ||
apiError.Code != "PACKAGE_TOO_LARGE" {
return errors.New("Java did not reject the oversized S3 object")
}
return nil
}

168
github-sync-go/internal/controlplane/upload.go Normal file
View file

@ -0,0 +1,168 @@
package controlplane
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
type UploadAuthorization struct {
UploadID string `json:"uploadId"`
Method string `json:"method"`
Mode string `json:"mode"`
UploadURL string `json:"uploadUrl"`
UploadTicket string `json:"uploadTicket,omitempty"`
ExpiresAt time.Time `json:"expiresAt"`
MaxBytes int64 `json:"maxBytes"`
Headers map[string]string `json:"headers,omitempty"`
}
func (c *Client) AuthorizeUpload(ctx context.Context, taskID string, lease LeaseProof,
admissionID, commitSHA, skillPath string, sizeBytes int64, digest string) (UploadAuthorization, error) {
if taskID == "" || admissionID == "" || sizeBytes < 1 || len(digest) != 64 {
return UploadAuthorization{}, errors.New("invalid package upload request")
}
var authorization UploadAuthorization
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/upload-authorizations", map[string]any{
"lease": lease, "admissionId": admissionID, "commitSha": commitSHA,
"skillPath": skillPath, "sizeBytes": sizeBytes, "sha256": digest,
}, &authorization)
if err != nil {
return UploadAuthorization{}, err
}
if authorization.UploadID == "" || authorization.Method != http.MethodPut ||
(authorization.Mode != "LOCAL_STREAM" && authorization.Mode != "S3_PRESIGNED_PUT") ||
authorization.UploadURL == "" || authorization.ExpiresAt.IsZero() ||
authorization.MaxBytes < sizeBytes || authorization.ExpiresAt.Before(time.Now()) {
return UploadAuthorization{}, errors.New("internal API returned an invalid upload authorization")
}
return authorization, nil
}
func (c *Client) ReportPackage(ctx context.Context, taskID string, lease LeaseProof,
admissionID, commitSHA, skillPath string, authorization UploadAuthorization,
sizeBytes int64, digest string) (string, error) {
if taskID == "" || admissionID == "" || authorization.UploadID == "" ||
sizeBytes < 1 || len(digest) != 64 {
return "", errors.New("invalid package report")
}
var response struct {
IngestionID string `json:"ingestionId"`
Status string `json:"status"`
}
_, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/packages/report", map[string]any{
"lease": lease, "admissionId": admissionID, "commitSha": commitSHA,
"skillPath": skillPath, "uploadId": authorization.UploadID,
"sizeBytes": sizeBytes, "sha256": digest,
}, &response)
if err != nil {
return "", err
}
if response.IngestionID == "" || response.Status != "RECEIVED" {
return "", errors.New("internal API returned an invalid package receipt")
}
return response.IngestionID, nil
}
// UploadPackage never sends the Java service token to an object-storage URL.
// S3 hosts must be explicitly configured by the deployment, including their port.
func (c *Client) UploadPackage(ctx context.Context, authorization UploadAuthorization,
archivePath string, allowedS3Hosts []string) error {
archive, err := os.Open(archivePath)
if err != nil {
return err
}
defer archive.Close()
info, err := archive.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() || info.Size() < 1 || info.Size() > authorization.MaxBytes {
return errors.New("archive does not match upload size policy")
}
parsed, err := url.Parse(authorization.UploadURL)
if err != nil || parsed.User != nil || parsed.Fragment != "" {
return errors.New("invalid upload URL")
}
var target string
var client *http.Client
local := authorization.Mode == "LOCAL_STREAM"
if local {
expected := "/internal/github-sync/v1/uploads/" + authorization.UploadID + "/content"
if parsed.IsAbs() || parsed.Host != "" || parsed.Path != expected || parsed.RawQuery != "" ||
authorization.UploadTicket == "" {
return errors.New("invalid local upload target")
}
base, err := url.Parse(c.baseURL)
if err != nil {
return err
}
target = base.Scheme + "://" + base.Host + expected
client = directUploadClient()
} else {
if authorization.Mode != "S3_PRESIGNED_PUT" || (parsed.Scheme != "https" && parsed.Scheme != "http") ||
parsed.Host == "" || len(allowedS3Hosts) == 0 {
return errors.New("invalid object-storage upload target")
}
allowed := false
for _, host := range allowedS3Hosts {
allowed = allowed || strings.EqualFold(parsed.Host, host)
}
if !allowed {
return errors.New("object-storage host is not allowed")
}
target = authorization.UploadURL
client = directUploadClient()
}
request, err := http.NewRequestWithContext(ctx, http.MethodPut, target, archive)
if err != nil {
return errors.New("cannot prepare package upload request")
}
request.ContentLength = info.Size()
for name, value := range authorization.Headers {
if strings.EqualFold(name, "authorization") || strings.EqualFold(name, "host") ||
strings.EqualFold(name, "x-upload-ticket") {
return errors.New("upload authorization contains a forbidden header")
}
request.Header.Set(name, value)
}
if local {
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("X-Upload-Ticket", authorization.UploadTicket)
request.Header.Set("Content-Type", "application/octet-stream")
requestID := make([]byte, 16)
if _, err := rand.Read(requestID); err != nil {
return err
}
request.Header.Set("X-Request-Id", hex.EncodeToString(requestID))
}
response, err := client.Do(request)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return errors.New("package upload transport failed")
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode != http.StatusNoContent && response.StatusCode != http.StatusOK {
return fmt.Errorf("package upload returned HTTP %d", response.StatusCode)
}
return nil
}
func directUploadClient() *http.Client {
return &http.Client{
Timeout: 10 * time.Minute,
Transport: &http.Transport{Proxy: nil, MaxIdleConnsPerHost: 2},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
}

60
github-sync-go/internal/controlplane/upload_test.go Normal file
View file

@ -0,0 +1,60 @@
package controlplane
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"testing"
)
func TestStagingUploadUsesDirectTransport(t *testing.T) {
transport, ok := directUploadClient().Transport.(*http.Transport)
if !ok || transport.Proxy != nil {
t.Fatal("staging upload must not use the GitHub proxy")
}
}
func TestS3UploadUsesOnlySignedHeaders(t *testing.T) {
const contents = "Skill package fixture"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodPut || request.URL.Path != "/staging/package.zip" {
t.Errorf("unexpected upload target: %s %s", request.Method, request.URL.Path)
}
if request.Header.Get("Authorization") != "" || request.Header.Get("X-Upload-Ticket") != "" {
t.Error("service credentials reached object storage")
}
if request.Header.Get("Content-Type") != "application/zip" {
t.Errorf("signed content type was not forwarded: %q", request.Header.Get("Content-Type"))
}
body, err := io.ReadAll(request.Body)
if err != nil || string(body) != contents {
t.Errorf("unexpected uploaded package: %q, %v", body, err)
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
archive := filepath.Join(t.TempDir(), "package.zip")
if err := os.WriteFile(archive, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
client, err := NewClient(server.URL+"/internal/github-sync/v1", "a-32-character-internal-service-token")
if err != nil {
t.Fatal(err)
}
parsed, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
authorization := UploadAuthorization{
Mode: "S3_PRESIGNED_PUT", UploadURL: server.URL + "/staging/package.zip",
MaxBytes: int64(len(contents)), Headers: map[string]string{"Content-Type": "application/zip"},
}
if err := client.UploadPackage(context.Background(), authorization, archive, []string{parsed.Host}); err != nil {
t.Fatal(err)
}
}

319
github-sync-go/internal/github/client.go Normal file
View file

@ -0,0 +1,319 @@
package github
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
)
const (
apiBaseURL = "https://api.github.com"
apiVersion = "2026-03-10"
maxResponseBytes = 9 << 20
)
var ErrResponseTooLarge = errors.New("GitHub API response exceeds size limit")
var (
ownerPattern = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$`)
repoPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,100}$`)
shaPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
)
type Client struct {
http *http.Client
token string
quotaObserver func(context.Context, Quota)
beforeRequest func(context.Context) error
afterRequest func(context.Context, Quota, error) error
}
func (c *Client) SetQuotaObserver(observer func(context.Context, Quota)) {
c.quotaObserver = observer
}
// SetRequestPolicy installs fail-closed hooks for official acquisition only.
func (c *Client) SetRequestPolicy(before func(context.Context) error, after func(context.Context, Quota, error) error) {
c.beforeRequest, c.afterRequest = before, after
}
func NewClient(token string) *Client {
return &Client{
http: &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxIdleConnsPerHost: 4,
ResponseHeaderTimeout: 20 * time.Second,
},
CheckRedirect: func(request *http.Request, previous []*http.Request) error {
if len(previous) >= 4 || request.URL.Scheme != "https" ||
request.URL.Host != "api.github.com" || request.URL.User != nil ||
request.URL.Fragment != "" {
return errors.New("unsafe GitHub API redirect")
}
return nil
},
},
token: token,
}
}
type Repository struct {
ID int64 `json:"id"`
Name string `json:"name"`
Private bool `json:"private"`
DefaultBranch string `json:"default_branch"`
Owner struct {
Login string `json:"login"`
ID int64 `json:"id"`
} `json:"owner"`
}
type Reference struct {
Object struct {
Type string `json:"type"`
SHA string `json:"sha"`
} `json:"object"`
}
type Commit struct {
SHA string `json:"sha"`
Tree struct {
SHA string `json:"sha"`
} `json:"tree"`
}
type TreeEntry struct {
Path string `json:"path"`
Mode string `json:"mode"`
Type string `json:"type"`
Size int64 `json:"size"`
SHA string `json:"sha"`
}
type Tree struct {
SHA string `json:"sha"`
Truncated bool `json:"truncated"`
Entries []TreeEntry `json:"tree"`
}
type Quota struct {
Resource string
Remaining int
ResetAt time.Time
}
type APIError struct {
Status int
Code string
RetryAt time.Time
Quota Quota
SecondaryRateLimit bool
}
func (e *APIError) Error() string {
return fmt.Sprintf("GitHub API returned HTTP %d (%s)", e.Status, e.Code)
}
func (c *Client) GetRepository(ctx context.Context, owner, name string) (Repository, Quota, error) {
if err := validateRepository(owner, name); err != nil {
return Repository{}, Quota{}, err
}
var repository Repository
quota, err := c.get(ctx, "/repos/"+owner+"/"+name, &repository)
if err != nil {
return Repository{}, quota, err
}
if repository.Private || repository.ID <= 0 || repository.Owner.Login == "" || repository.Name == "" {
return Repository{}, quota, errors.New("repository is private or has incomplete metadata")
}
return repository, quota, nil
}
func (c *Client) GetHead(ctx context.Context, repository Repository) (string, Quota, error) {
if err := validateRepository(repository.Owner.Login, repository.Name); err != nil {
return "", Quota{}, err
}
if repository.DefaultBranch == "" {
return "", Quota{}, errors.New("repository has no default branch")
}
// Branch names may contain '/', so escape each segment while retaining the separator.
segments := strings.Split(repository.DefaultBranch, "/")
for i, segment := range segments {
if segment == "" || segment == "." || segment == ".." {
return "", Quota{}, errors.New("invalid default branch")
}
segments[i] = url.PathEscape(segment)
}
var reference Reference
quota, err := c.get(ctx, "/repos/"+repository.Owner.Login+"/"+repository.Name+
"/git/ref/heads/"+strings.Join(segments, "/"), &reference)
if err != nil {
return "", quota, err
}
if reference.Object.Type != "commit" || !shaPattern.MatchString(reference.Object.SHA) {
return "", quota, errors.New("default branch did not resolve to a commit")
}
return reference.Object.SHA, quota, nil
}
func (c *Client) GetCommitTreeSHA(ctx context.Context, owner, name, commitSHA string) (string, Quota, error) {
if err := validateRepository(owner, name); err != nil {
return "", Quota{}, err
}
if !shaPattern.MatchString(commitSHA) {
return "", Quota{}, errors.New("invalid commit SHA")
}
var commit Commit
quota, err := c.get(ctx, "/repos/"+owner+"/"+name+"/git/commits/"+commitSHA, &commit)
if err != nil {
return "", quota, err
}
if commit.SHA != commitSHA || !shaPattern.MatchString(commit.Tree.SHA) {
return "", quota, errors.New("GitHub commit does not match the fixed revision")
}
return commit.Tree.SHA, quota, nil
}
func (c *Client) GetTree(ctx context.Context, owner, name, treeSHA string, recursive bool) (Tree, Quota, error) {
if err := validateRepository(owner, name); err != nil {
return Tree{}, Quota{}, err
}
if !shaPattern.MatchString(treeSHA) {
return Tree{}, Quota{}, errors.New("invalid tree SHA")
}
path := "/repos/" + owner + "/" + name + "/git/trees/" + treeSHA
if recursive {
path += "?recursive=1"
}
var tree Tree
quota, err := c.get(ctx, path, &tree)
if err != nil {
return Tree{}, quota, err
}
if tree.SHA != treeSHA {
return Tree{}, quota, errors.New("GitHub tree does not match the fixed revision")
}
return tree, quota, nil
}
func validateRepository(owner, name string) error {
if !ownerPattern.MatchString(owner) || !repoPattern.MatchString(name) || name == "." || name == ".." {
return errors.New("invalid GitHub repository coordinate")
}
return nil
}
// PublicCloneURL never embeds a credential in the remote URL.
func PublicCloneURL(owner, name string) (string, error) {
if err := validateRepository(owner, name); err != nil {
return "", err
}
return "https://github.com/" + owner + "/" + name + ".git", nil
}
func (c *Client) get(ctx context.Context, path string, target any) (quota Quota, resultErr error) {
quota.Remaining = -1
if err := CheckAcquisitionControl(ctx); err != nil {
return quota, err
}
if c.beforeRequest != nil {
if err := c.beforeRequest(ctx); err != nil {
return quota, err
}
}
if c.afterRequest != nil {
defer func() {
if err := c.afterRequest(ctx, quota, resultErr); err != nil {
resultErr = err
}
}()
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, apiBaseURL+path, nil)
if err != nil {
return Quota{}, err
}
request.Header.Set("Accept", "application/vnd.github+json")
request.Header.Set("X-GitHub-Api-Version", apiVersion)
request.Header.Set("User-Agent", "SkillHub-GitHub-Sync")
if c.token != "" {
request.Header.Set("Authorization", "Bearer "+c.token)
}
response, err := c.http.Do(request)
if err != nil {
return quota, err
}
defer response.Body.Close()
quota = parseQuota(response.Header)
if c.quotaObserver != nil && quota.Remaining >= 0 && !quota.ResetAt.IsZero() {
c.quotaObserver(ctx, quota)
}
if response.StatusCode != http.StatusOK {
return quota, classifyError(response, quota)
}
contents, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
if err != nil {
return quota, err
}
if len(contents) > maxResponseBytes {
return quota, ErrResponseTooLarge
}
if err := json.Unmarshal(contents, target); err != nil {
return quota, fmt.Errorf("decode GitHub API response: %w", err)
}
return quota, nil
}
func parseQuota(header http.Header) Quota {
quota := Quota{Resource: header.Get("X-RateLimit-Resource"), Remaining: -1}
if remaining, err := strconv.Atoi(header.Get("X-RateLimit-Remaining")); err == nil {
quota.Remaining = remaining
}
if reset, err := strconv.ParseInt(header.Get("X-RateLimit-Reset"), 10, 64); err == nil {
quota.ResetAt = time.Unix(reset, 0).UTC()
}
return quota
}
func classifyError(response *http.Response, quota Quota) error {
status := response.StatusCode
classified := &APIError{Status: status, Quota: quota, Code: "RETRYABLE_FAILURE"}
var payload struct {
Message string `json:"message"`
}
_ = json.NewDecoder(io.LimitReader(response.Body, 8192)).Decode(&payload)
if status == http.StatusUnauthorized {
classified.Code = "AUTH_INVALID"
} else if status == http.StatusNotFound || status == http.StatusForbidden {
classified.Code = "SOURCE_UNREADABLE"
}
if status == http.StatusTooManyRequests ||
(status == http.StatusForbidden && (response.Header.Get("X-RateLimit-Remaining") == "0" ||
response.Header.Get("Retry-After") != "" || strings.Contains(strings.ToLower(payload.Message), "rate limit"))) {
classified.Code = "GITHUB_RATE_LIMITED"
classified.SecondaryRateLimit = response.Header.Get("Retry-After") != "" ||
strings.Contains(strings.ToLower(payload.Message), "secondary rate limit") || quota.Remaining > 0
classified.RetryAt = rateLimitRetryAt(response.Header, quota)
}
return classified
}
func rateLimitRetryAt(header http.Header, quota Quota) time.Time {
if seconds, err := strconv.Atoi(header.Get("Retry-After")); err == nil && seconds > 0 {
return time.Now().Add(time.Duration(seconds) * time.Second)
}
if !quota.ResetAt.IsZero() && quota.Remaining == 0 {
return quota.ResetAt
}
return time.Now().Add(time.Minute)
}

62
github-sync-go/internal/github/client_redirect_test.go Normal file
View file

@ -0,0 +1,62 @@
package github
import (
"context"
"io"
"net/http"
"strings"
"testing"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return fn(request)
}
func TestGetRepositoryFollowsSameHostRenameRedirect(t *testing.T) {
client := NewClient("fixture-token")
requests := 0
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
requests++
if request.Header.Get("Authorization") != "Bearer fixture-token" {
t.Fatal("GitHub credential missing from same-host request")
}
switch request.URL.Path {
case "/repos/old-owner/old-name":
return &http.Response{StatusCode: http.StatusMovedPermanently,
Header: http.Header{"Location": {"https://api.github.com/repositories/42"}},
Body: io.NopCloser(strings.NewReader("")), Request: request}, nil
case "/repositories/42":
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"id":42,"name":"new-name","owner":{"login":"new-owner"},"default_branch":"main","private":false}`)),
Request: request}, nil
default:
t.Fatalf("unexpected GitHub path %s", request.URL.Path)
return nil, nil
}
})
repository, _, err := client.GetRepository(context.Background(), "old-owner", "old-name")
if err != nil || repository.ID != 42 || repository.Owner.Login != "new-owner" ||
repository.Name != "new-name" || requests != 2 {
t.Fatalf("unexpected renamed repository result: %+v, requests=%d, error=%v", repository, requests, err)
}
}
func TestGetRepositoryRejectsCrossHostRedirect(t *testing.T) {
client := NewClient("fixture-token")
requests := 0
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
requests++
if request.URL.Host != "api.github.com" {
t.Fatal("GitHub credential request escaped API host")
}
return &http.Response{StatusCode: http.StatusMovedPermanently,
Header: http.Header{"Location": {"https://example.invalid/repos/new-owner/new-name"}},
Body: io.NopCloser(strings.NewReader("")), Request: request}, nil
})
_, _, err := client.GetRepository(context.Background(), "old-owner", "old-name")
if err == nil || requests != 1 {
t.Fatalf("cross-host redirect must fail before a second request: requests=%d, error=%v", requests, err)
}
}

93
github-sync-go/internal/github/client_status_test.go Normal file
View file

@ -0,0 +1,93 @@
package github
import (
"context"
"errors"
"io"
"net/http"
"strconv"
"strings"
"testing"
"time"
)
func TestRepositoryStatusClassificationAndQuota(t *testing.T) {
reset := time.Now().Add(5 * time.Minute).Truncate(time.Second)
tests := []struct {
name string
status int
message string
headers http.Header
code string
wantRetryTime time.Time
}{
{"invalid credential", http.StatusUnauthorized, `{"message":"Bad credentials"}`,
nil, "AUTH_INVALID", time.Time{}},
{"private or missing", http.StatusNotFound, `{"message":"Not Found"}`,
nil, "SOURCE_UNREADABLE", time.Time{}},
{"forbidden without rate limit", http.StatusForbidden, `{"message":"Resource not accessible"}`,
nil, "SOURCE_UNREADABLE", time.Time{}},
{"primary rate limit", http.StatusForbidden, `{"message":"API rate limit exceeded"}`,
http.Header{"X-Ratelimit-Remaining": {"0"},
"X-Ratelimit-Reset": {strconv.FormatInt(reset.Unix(), 10)},
"X-Ratelimit-Resource": {"core"}}, "GITHUB_RATE_LIMITED", reset},
{"server failure", http.StatusInternalServerError, `{"message":"Server error"}`,
nil, "RETRYABLE_FAILURE", time.Time{}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client := NewClient("fixture-token")
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
if request.Header.Get("Authorization") != "Bearer fixture-token" {
t.Fatal("GitHub credential was not sent")
}
return &http.Response{StatusCode: test.status, Header: test.headers,
Body: io.NopCloser(strings.NewReader(test.message)), Request: request}, nil
})
_, quota, err := client.GetRepository(context.Background(), "public-owner", "skills")
var apiError *APIError
if !errors.As(err, &apiError) || apiError.Code != test.code {
t.Fatalf("status %d: expected %s, got %v", test.status, test.code, err)
}
if !test.wantRetryTime.IsZero() &&
(!apiError.RetryAt.Equal(test.wantRetryTime) || quota.Remaining != 0 ||
quota.Resource != "core") {
t.Fatalf("rate limit headers were not preserved: error=%+v quota=%+v", apiError, quota)
}
})
}
}
func TestSecondaryRateLimitRetryAfter(t *testing.T) {
client := NewClient("fixture-token")
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusTooManyRequests,
Header: http.Header{"Retry-After": {"120"}},
Body: io.NopCloser(strings.NewReader(`{"message":"You have exceeded a secondary rate limit"}`)),
Request: request}, nil
})
before := time.Now()
_, _, err := client.GetRepository(context.Background(), "public-owner", "skills")
var apiError *APIError
if !errors.As(err, &apiError) || apiError.Code != "GITHUB_RATE_LIMITED" ||
apiError.RetryAt.Before(before.Add(120*time.Second)) ||
apiError.RetryAt.After(time.Now().Add(121*time.Second)) {
t.Fatalf("secondary rate limit did not honor Retry-After: %v", err)
}
}
func TestAnonymousRepositoryRequestHasNoAuthorizationHeader(t *testing.T) {
client := NewClient("")
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
if request.Header.Get("Authorization") != "" {
t.Fatal("anonymous GitHub request contained an authorization header")
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"id":42,"name":"skills","owner":{"login":"public-owner"},"default_branch":"main","private":false}`)),
Request: request}, nil
})
repository, _, err := client.GetRepository(context.Background(), "public-owner", "skills")
if err != nil || repository.ID != 42 {
t.Fatalf("anonymous public repository lookup failed: repository=%+v error=%v", repository, err)
}
}

193
github-sync-go/internal/github/discovery.go Normal file
View file

@ -0,0 +1,193 @@
package github
import (
"context"
"errors"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
)
const repositoryPageSize = 100
var topicPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`)
type SearchPage struct {
Repositories []Repository
TotalCount int
IncompleteResults bool
LimitReached bool
HasMore bool
}
type OwnerType string
const (
UserOwner OwnerType = "User"
OrganizationOwner OwnerType = "Organization"
)
type OwnerPage struct {
Repositories []Repository
HasMore bool
}
// SearchRepositories accepts only structured selectors. Callers must retain the
// incomplete/limit signals and split the fixed window instead of marking it done.
func (c *Client) SearchRepositories(ctx context.Context, kind, value string,
since *time.Time, until time.Time, page int) (SearchPage, Quota, error) {
if page < 1 || page > 10 || until.IsZero() || (since != nil && since.After(until)) {
return SearchPage{}, Quota{}, errors.New("invalid search page or window")
}
selector, err := searchSelector(kind, value)
if err != nil {
return SearchPage{}, Quota{}, err
}
windowEnd := until.UTC().Format("2006-01-02T15:04:05+00:00")
window := "pushed:*.." + windowEnd
if since != nil {
window = "pushed:" + since.UTC().Format("2006-01-02T15:04:05+00:00") + ".." + windowEnd
}
return c.searchRepositoryPage(ctx, selector+" is:public "+window, page)
}
// searchRepositoryPage preserves GitHub's coverage signals for both custom and
// official discovery. Its callers compile and validate the query before use.
func (c *Client) searchRepositoryPage(ctx context.Context, compiled string, page int) (SearchPage, Quota, error) {
query := url.Values{}
query.Set("q", compiled)
query.Set("sort", "updated")
query.Set("order", "desc")
query.Set("per_page", strconv.Itoa(repositoryPageSize))
query.Set("page", strconv.Itoa(page))
var response struct {
TotalCount *int `json:"total_count"`
IncompleteResults *bool `json:"incomplete_results"`
Items []Repository `json:"items"`
}
quota, err := c.get(ctx, "/search/repositories?"+query.Encode(), &response)
if err != nil {
return SearchPage{}, quota, err
}
if response.TotalCount == nil || response.IncompleteResults == nil || *response.TotalCount < 0 {
return SearchPage{}, quota, errors.New("GitHub search returned incomplete metadata")
}
accessible := min(*response.TotalCount, 1000)
return SearchPage{
Repositories: publicRepositories(response.Items),
TotalCount: *response.TotalCount,
IncompleteResults: *response.IncompleteResults,
LimitReached: *response.TotalCount > 1000,
HasMore: page*repositoryPageSize < accessible,
}, quota, nil
}
func (c *Client) GetOwnerType(ctx context.Context, owner string) (OwnerType, Quota, error) {
if !ownerPattern.MatchString(owner) {
return "", Quota{}, errors.New("invalid GitHub owner")
}
var response struct {
Type string `json:"type"`
}
quota, err := c.get(ctx, "/users/"+owner, &response)
if err != nil {
return "", quota, err
}
ownerType := OwnerType(response.Type)
if ownerType != UserOwner && ownerType != OrganizationOwner {
return "", quota, errors.New("GitHub owner is not a user or organization")
}
return ownerType, quota, nil
}
// OwnerIdentity is resolved through GitHub's durable account ID endpoint.
type OwnerIdentity struct {
ID int64 `json:"id"`
Login string `json:"login"`
Type OwnerType `json:"type"`
}
func (c *Client) GetOwnerByID(ctx context.Context, id int64) (OwnerIdentity, Quota, error) {
if id < 1 {
return OwnerIdentity{}, Quota{}, errors.New("invalid GitHub owner ID")
}
var owner OwnerIdentity
quota, err := c.get(ctx, "/user/"+strconv.FormatInt(id, 10), &owner)
if err != nil {
return OwnerIdentity{}, quota, err
}
if owner.ID != id || !ownerPattern.MatchString(owner.Login) || (owner.Type != UserOwner && owner.Type != OrganizationOwner) {
return OwnerIdentity{}, quota, errors.New("GitHub returned inconsistent owner identity")
}
return owner, quota, nil
}
// ListPublicOwnerRepositories uses the public-only endpoint for the account type
// resolved by GetOwnerType. Candidate budgeting and page checkpoints belong to the worker.
func (c *Client) ListPublicOwnerRepositories(ctx context.Context, owner string,
ownerType OwnerType, page int) (OwnerPage, Quota, error) {
if !ownerPattern.MatchString(owner) || page < 1 ||
(ownerType != UserOwner && ownerType != OrganizationOwner) {
return OwnerPage{}, Quota{}, errors.New("invalid GitHub owner or page")
}
path := "/users/" + owner + "/repos"
query := url.Values{}
if ownerType == OrganizationOwner {
path = "/orgs/" + owner + "/repos"
query.Set("type", "public")
} else {
query.Set("type", "owner")
}
query.Set("sort", "full_name")
query.Set("direction", "asc")
query.Set("per_page", strconv.Itoa(repositoryPageSize))
query.Set("page", strconv.Itoa(page))
var repositories []Repository
quota, err := c.get(ctx, path+"?"+query.Encode(), &repositories)
if err != nil {
return OwnerPage{}, quota, err
}
return OwnerPage{
Repositories: publicRepositories(repositories),
HasMore: len(repositories) == repositoryPageSize,
}, quota, nil
}
func searchSelector(kind, value string) (string, error) {
value = strings.TrimSpace(value)
if value == "" || utf8.RuneCountInString(value) > 100 {
return "", errors.New("invalid search selector")
}
switch kind {
case "KEYWORD":
for _, char := range value {
if !unicode.IsLetter(char) && !unicode.IsDigit(char) && char != ' ' &&
char != '-' && char != '_' {
return "", errors.New("keyword contains unsupported search syntax")
}
}
return `"` + value + `" in:name,description,topics`, nil
case "TOPIC":
if !topicPattern.MatchString(value) {
return "", errors.New("invalid GitHub topic selector")
}
return "topic:" + strings.ToLower(value), nil
default:
return "", errors.New("unsupported search selector kind")
}
}
func publicRepositories(repositories []Repository) []Repository {
public := make([]Repository, 0, len(repositories))
for _, repository := range repositories {
if !repository.Private && repository.ID > 0 &&
validateRepository(repository.Owner.Login, repository.Name) == nil {
public = append(public, repository)
}
}
return public
}

33
github-sync-go/internal/github/discovery_live_test.go Normal file
View file

@ -0,0 +1,33 @@
package github
import (
"context"
"os"
"testing"
"time"
)
// Opt-in check of the structured search contract against the public GitHub API.
func TestLiveStructuredRepositorySearch(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_SEARCH") != "1" {
t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_SEARCH=1 to use the public GitHub API")
}
client := NewClient("")
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
until := time.Now().UTC()
for _, selector := range []struct{ kind, value string }{
{"KEYWORD", "agent skills"},
{"TOPIC", "agent-skills"},
} {
page, _, err := client.SearchRepositories(ctx, selector.kind, selector.value,
nil, until, 1)
if err != nil {
t.Fatalf("%s search failed: %v", selector.kind, err)
}
if page.TotalCount < 0 || len(page.Repositories) > repositoryPageSize {
t.Fatalf("%s search returned invalid metadata: total=%d page=%d",
selector.kind, page.TotalCount, len(page.Repositories))
}
}
}

78
github-sync-go/internal/github/discovery_test.go Normal file
View file

@ -0,0 +1,78 @@
package github
import (
"context"
"io"
"net/http"
"strings"
"testing"
"time"
)
func TestSearchRepositoriesReportsLimitAndIncompleteCoverage(t *testing.T) {
client := NewClient("")
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
query := request.URL.Query()
if request.URL.Path != "/search/repositories" || query.Get("page") != "1" ||
query.Get("q") != `"agent skills" in:name,description,topics is:public pushed:2026-09-01T00:00:00+00:00..2026-09-02T00:00:00+00:00` {
t.Fatalf("unexpected search request: %s", request.URL.String())
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"total_count":1001,"incomplete_results":true,"items":[{"id":1,"name":"skills","owner":{"login":"owner"},"private":false}]}`)),
Request: request}, nil
})
from := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
until := from.Add(24 * time.Hour)
page, _, err := client.SearchRepositories(context.Background(), "KEYWORD", "agent skills", &from, until, 1)
if err != nil || !page.LimitReached || !page.IncompleteResults || !page.HasMore ||
len(page.Repositories) != 1 || page.Repositories[0].ID != 1 {
t.Fatalf("search coverage = %+v, error = %v", page, err)
}
}
func TestCuratedUserAndOrganizationEnumerateOnlyPublicRepositories(t *testing.T) {
for _, testcase := range []struct {
name string
ownerType OwnerType
listPath string
filterType string
}{
{name: "user", ownerType: UserOwner, listPath: "/users/public-owner/repos", filterType: "owner"},
{name: "organization", ownerType: OrganizationOwner,
listPath: "/orgs/public-owner/repos", filterType: "public"},
} {
t.Run(testcase.name, func(t *testing.T) {
client := NewClient("")
calls := 0
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
calls++
body := `{"type":"` + string(testcase.ownerType) + `"}`
if calls == 1 {
if request.URL.Path != "/users/public-owner" {
t.Fatalf("unexpected owner lookup: %s", request.URL.Path)
}
} else {
query := request.URL.Query()
if request.URL.Path != testcase.listPath || query.Get("type") != testcase.filterType ||
query.Get("page") != "1" || query.Get("per_page") != "100" {
t.Fatalf("unexpected repository listing: %s", request.URL.String())
}
body = `[{"id":7,"name":"skills","private":false,"owner":{"login":"public-owner"}},` +
`{"id":8,"name":"hidden","private":true,"owner":{"login":"public-owner"}}]`
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(body)), Request: request}, nil
})
ownerType, _, err := client.GetOwnerType(context.Background(), "public-owner")
if err != nil || ownerType != testcase.ownerType {
t.Fatalf("owner type = %s, error = %v", ownerType, err)
}
page, _, err := client.ListPublicOwnerRepositories(context.Background(),
"public-owner", ownerType, 1)
if err != nil || calls != 2 || page.HasMore || len(page.Repositories) != 1 ||
page.Repositories[0].ID != 7 {
t.Fatalf("public repository page = %+v, error = %v", page, err)
}
})
}
}

102
github-sync-go/internal/github/official_discovery.go Normal file
View file

@ -0,0 +1,102 @@
package github
import (
"context"
"errors"
"strings"
"time"
"unicode"
"unicode/utf8"
)
type OfficialQueryKind string
type OfficialQueryScope string
const (
OfficialKeyword OfficialQueryKind = "KEYWORD"
OfficialTopic OfficialQueryKind = "TOPIC"
OfficialMetadataScope OfficialQueryScope = "NAME_DESCRIPTION_TOPICS"
OfficialReadmeScope OfficialQueryScope = "README"
)
// OfficialRepositoryQuery is the executable search portion of a Java-owned
// policy snapshot. Policy versions and query catalogs remain in the control plane.
type OfficialRepositoryQuery struct {
Kind OfficialQueryKind
Value string
Scope OfficialQueryScope
}
// OfficialRepositoryRange is a half-open pushed-time interval. Partition values
// require an agreed compiler and are currently rejected rather than ignored.
type OfficialRepositoryRange struct {
Since time.Time
Until time.Time
Partition string
}
// CompileOfficialRepositoryQuery only accepts structured repository searches.
// It does not compile code search or owner expansion. Whole-second boundaries
// avoid silently changing coverage when translating to GitHub's inclusive range.
func CompileOfficialRepositoryQuery(query OfficialRepositoryQuery, window OfficialRepositoryRange,
fixedCutoffAt time.Time) (string, error) {
_, sinceOffset := window.Since.Zone()
_, untilOffset := window.Until.Zone()
_, cutoffOffset := fixedCutoffAt.Zone()
if window.Since.IsZero() || window.Until.IsZero() || fixedCutoffAt.IsZero() ||
!window.Since.Before(window.Until) || window.Until.After(fixedCutoffAt) ||
window.Since.Nanosecond() != 0 || window.Until.Nanosecond() != 0 || fixedCutoffAt.Nanosecond() != 0 ||
sinceOffset != 0 || untilOffset != 0 || cutoffOffset != 0 {
return "", errors.New("invalid official search window or cutoff")
}
if window.Partition != "" {
return "", errors.New("unsupported official search partition")
}
value := strings.TrimSpace(query.Value)
if !utf8.ValidString(value) || value == "" || utf8.RuneCountInString(value) > 256 {
return "", errors.New("invalid official search value")
}
var selector string
switch query.Kind {
case OfficialKeyword:
for _, char := range value {
if !unicode.IsLetter(char) && !unicode.IsDigit(char) && char != ' ' &&
char != '-' && char != '_' && char != '.' && char != '/' {
return "", errors.New("official keyword contains unsupported search syntax")
}
}
switch query.Scope {
case OfficialMetadataScope:
selector = `"` + value + `" in:name,description,topics`
case OfficialReadmeScope:
selector = `"` + value + `" in:readme`
default:
return "", errors.New("unsupported official keyword scope")
}
case OfficialTopic:
if query.Scope != OfficialMetadataScope || !topicPattern.MatchString(value) {
return "", errors.New("invalid official topic or scope")
}
selector = "topic:" + strings.ToLower(value)
default:
return "", errors.New("unsupported official repository query kind")
}
const timestampFormat = "2006-01-02T15:04:05+00:00"
return selector + " is:public pushed:" + window.Since.UTC().Format(timestampFormat) +
".." + window.Until.Add(-time.Second).UTC().Format(timestampFormat), nil
}
// SearchOfficialRepositories executes one page; the caller retains incomplete
// coverage and limit signals and owns durable checkpoints and shard splitting.
func (c *Client) SearchOfficialRepositories(ctx context.Context, query OfficialRepositoryQuery,
window OfficialRepositoryRange, fixedCutoffAt time.Time, page int) (SearchPage, Quota, error) {
if page < 1 || page > 10 {
return SearchPage{}, Quota{}, errors.New("invalid official search page")
}
compiled, err := CompileOfficialRepositoryQuery(query, window, fixedCutoffAt)
if err != nil {
return SearchPage{}, Quota{}, err
}
return c.searchRepositoryPage(ctx, compiled, page)
}

94
github-sync-go/internal/github/official_discovery_test.go Normal file
View file

@ -0,0 +1,94 @@
package github
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"time"
)
func TestOfficialQueryHalfOpenUTCAndScopes(t *testing.T) {
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
for _, scope := range []OfficialQueryScope{OfficialQueryScope("NAME_DESCRIPTION_TOPICS"), OfficialQueryScope("README")} {
t.Run(string(scope), func(t *testing.T) {
q, err := CompileOfficialRepositoryQuery(OfficialRepositoryQuery{Kind: "KEYWORD", Value: "agent skills", Scope: scope}, OfficialRepositoryRange{Since: start, Until: start.Add(2 * time.Second)}, start.Add(time.Hour))
if err != nil || !strings.Contains(q, "00:00:01+00:00") || strings.Contains(q, "00:00:02+00:00") {
t.Fatalf("half-open query %q, %v", q, err)
}
if scope == "README" && !strings.Contains(q, "in:readme") {
t.Fatalf("missing README qualifier %q", q)
}
})
}
for _, kind := range []string{"fractional", "partition", "cutoff", "injection"} {
t.Run(kind, func(t *testing.T) {
window := OfficialRepositoryRange{Since: start, Until: start.Add(time.Second)}
cutoff := start.Add(time.Hour)
value := "skills"
switch kind {
case "fractional":
window.Since = start.Add(time.Nanosecond)
case "partition":
window.Partition = "stars:>1"
case "cutoff":
cutoff = start
case "injection":
value = "skills user:someone"
}
if _, err := CompileOfficialRepositoryQuery(OfficialRepositoryQuery{Kind: "KEYWORD", Value: value, Scope: "NAME_DESCRIPTION_TOPICS"}, window, cutoff); err == nil {
t.Fatal("invalid official query accepted")
}
})
}
}
func TestOfficialOwnerStableIDResponseValidation(t *testing.T) {
for _, body := range []string{`{"id":42,"login":"renamed","type":"User"}`, `{"id":42,"login":"renamed","type":"Organization"}`, `{"id":43,"login":"renamed","type":"User"}`, `{"id":42,"login":"bad/name","type":"User"}`, `{"id":42,"login":"renamed","type":"Bot"}`} {
t.Run(body, func(t *testing.T) {
c := NewClient("")
c.http.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.URL.Path != "/user/42" {
t.Errorf("lookup not by stable ID: %s", r.URL.Path)
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
owner, _, err := c.GetOwnerByID(context.Background(), 42)
valid := strings.Contains(body, `"id":42`) && !strings.Contains(body, "bad/name") && !strings.Contains(body, "Bot")
if (err == nil) != valid {
t.Fatalf("owner %+v err %v", owner, err)
}
})
}
}
func TestOfficialRESTPolicyStopsRequestsAfterObservationFailure(t *testing.T) {
c := NewClient("")
calls, observations := 0, 0
stopped := false
c.http.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) {
calls++
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"id":42,"login":"owner","type":"User"}`)), Request: r}, nil
})
c.SetRequestPolicy(func(context.Context) error {
if stopped {
return errors.New("cooldown")
}
return nil
}, func(_ context.Context, q Quota, _ error) error {
observations++
if q.Remaining != -1 {
t.Error("missing headers produced known remaining")
}
stopped = true
return errors.New("persistence failed")
})
for i := 0; i < 2; i++ {
if _, _, err := c.GetOwnerByID(context.Background(), 42); err == nil {
t.Fatal("policy error was ignored")
}
}
if calls != 1 || observations != 1 {
t.Fatalf("calls=%d observations=%d", calls, observations)
}
}

16
github-sync-go/internal/github/request_control.go Normal file
View file

@ -0,0 +1,16 @@
package github
import "context"
type acquisitionControlKey struct{}
// WithAcquisitionControl composes platform stopping with the existing quota policy.
func WithAcquisitionControl(ctx context.Context, before func(context.Context) error) context.Context {
return context.WithValue(ctx, acquisitionControlKey{}, before)
}
func CheckAcquisitionControl(ctx context.Context) error {
if before, ok := ctx.Value(acquisitionControlKey{}).(func(context.Context) error); ok {
return before(ctx)
}
return ctx.Err()
}

22
github-sync-go/internal/github/request_control_test.go Normal file
View file

@ -0,0 +1,22 @@
package github
import (
"context"
"errors"
"testing"
)
func TestPlatformControlFencesEveryGithubReadBeforeQuotaAndNetwork(t *testing.T) {
stop := errors.New("platform paused")
ctx := WithAcquisitionControl(context.Background(), func(context.Context) error { return stop })
client := NewClient("")
client.SetRequestPolicy(func(context.Context) error { t.Fatal("quota hook ran after platform stop"); return nil }, nil)
// All acquisition channels (search, owner listing, metadata, HEAD and Tree)
// pass through get; no external request may occur after the control denial.
for _, path := range []string{"/search/repositories", "/search/code", "/users/fixture/repos", "/repos/fixture/skills", "/repos/fixture/skills/git/ref/heads/main", "/repos/fixture/skills/git/trees/fixture"} {
var payload any
if _, err := client.get(ctx, path, &payload); !errors.Is(err, stop) {
t.Fatalf("%s: %v", path, err)
}
}
}

34
github-sync-go/internal/github/tree_test.go Normal file
View file

@ -0,0 +1,34 @@
package github
import (
"context"
"io"
"net/http"
"strings"
"testing"
)
func TestFixedCommitTreeRejectsMismatchedResponses(t *testing.T) {
const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
const treeSHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
client := NewClient("")
client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
var body string
switch request.URL.Path {
case "/repos/owner/repo/git/commits/" + commitSHA:
body = `{"sha":"cccccccccccccccccccccccccccccccccccccccc","tree":{"sha":"` + treeSHA + `"}}`
case "/repos/owner/repo/git/trees/" + treeSHA:
body = `{"sha":"dddddddddddddddddddddddddddddddddddddddd","truncated":false,"tree":[]}`
default:
t.Fatalf("unexpected GitHub tree request: %s", request.URL.String())
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(body)), Request: request}, nil
})
if _, _, err := client.GetCommitTreeSHA(context.Background(), "owner", "repo", commitSHA); err == nil {
t.Fatal("commit response with a different SHA must be rejected")
}
if _, _, err := client.GetTree(context.Background(), "owner", "repo", treeSHA, false); err == nil {
t.Fatal("tree response with a different SHA must be rejected")
}
}

576
github-sync-go/internal/repository/acquire.go Normal file
View file

@ -0,0 +1,576 @@
package repository
import (
"archive/zip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
var (
ErrLocalDiskBudget = errors.New("local Git workspace disk budget exceeded")
ErrCapacity = errors.New("local disk cannot hold a compliant package")
ErrArchiveTooLarge = errors.New("Skill archive exceeds upload limit")
)
const (
evidenceManifestPath = "__skillhub_sync_evidence__.json"
evidenceEntryPrefix = "__skillhub_sync_evidence__/"
gitMetadataReserve = 16 << 20
archiveOverheadReserve = 1 << 20
)
type rootEvidenceManifest struct {
FormatVersion int `json:"formatVersion"`
RootFiles []RootEvidenceFile `json:"rootFiles"`
}
type AcquisitionLimits struct {
MaxWorkspaceBytes int64
MaxArchiveBytes int64
MinFreeBytes int64
}
// Artifact owns its isolated temporary directory until Close is called.
type Artifact struct {
Path string
Size int64
SHA256 string
dir string
once sync.Once
closeErr error
}
func (artifact *Artifact) Close() error {
artifact.once.Do(func() { artifact.closeErr = os.RemoveAll(artifact.dir) })
return artifact.closeErr
}
// AcquirePackage fetches one fixed public commit without repository blobs,
// checks out only the preflighted files, then builds one bounded ZIP. The
// caller must obtain Java package admission before invoking this function.
func AcquirePackage(ctx context.Context, owner, name, commitSHA, tempRoot string,
plan PackagePlan, limits AcquisitionLimits) (*Artifact, error) {
if !validGitSHA(commitSHA) || (plan.Root != "." && !validPath(plan.Root)) ||
len(plan.Files) == 0 ||
plan.TotalBytes < 0 || limits.MaxWorkspaceBytes < 1 || limits.MaxArchiveBytes < 1 ||
limits.MinFreeBytes < 0 || limits.MaxWorkspaceBytes > math.MaxInt64-limits.MinFreeBytes {
return nil, errors.New("invalid fixed revision, package plan, or acquisition limits")
}
remote, err := github.PublicCloneURL(owner, name)
if err != nil {
return nil, err
}
var evidenceBytes int64
for _, file := range plan.Evidence {
if !validPath(file.RepositoryPath) || strings.Contains(file.RepositoryPath, "/") ||
!rootEvidenceName.MatchString(file.RepositoryPath) || !validGitSHA(file.SHA) ||
file.Size < 0 {
return nil, errors.New("invalid root license or notice evidence")
}
if file.Bundled {
if file.Size > maxEvidenceFileBytes ||
file.Size > maxEvidenceTotalBytes-evidenceBytes {
return nil, errors.New("root evidence exceeds acquisition limit")
}
evidenceBytes += file.Size
}
}
if len(plan.Evidence) > maxRootEvidenceFiles ||
!canReserveWorkspace(plan.TotalBytes, evidenceBytes, limits) {
return nil, ErrCapacity
}
patterns, allowed, err := prepareSparseCheckout(plan)
if err != nil {
return nil, err
}
if err := ensureFreeSpace(tempRoot, limits.MaxWorkspaceBytes+limits.MinFreeBytes); err != nil {
return nil, err
}
workDir, err := os.MkdirTemp(tempRoot, "skillhub-github-package-")
if err != nil {
return nil, localDiskWriteError(err)
}
keep := false
defer func() {
if !keep {
_ = os.RemoveAll(workDir)
}
}()
repoDir := filepath.Join(workDir, "repo")
if err := os.Mkdir(repoDir, 0700); err != nil {
return nil, localDiskWriteError(err)
}
for _, step := range []struct {
name string
args []string
}{
{"init", []string{"init", "--quiet"}},
{"remote", []string{"remote", "add", "origin", remote}},
{"fetch", []string{"fetch", "--quiet", "--no-tags", "--filter=blob:none", "--depth=1", "origin", commitSHA}},
} {
if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes,
limits.MinFreeBytes, step.args...); err != nil {
return nil, fmt.Errorf("Git %s: %w", step.name, err)
}
}
if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes,
limits.MinFreeBytes, patterns...); err != nil {
return nil, fmt.Errorf("Git sparse checkout: %w", err)
}
if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes,
limits.MinFreeBytes,
"checkout", "--quiet", "--detach", "--force", "FETCH_HEAD"); err != nil {
return nil, fmt.Errorf("Git fixed checkout: %w", err)
}
head, err := os.ReadFile(filepath.Join(repoDir, ".git", "HEAD"))
if err != nil || strings.TrimSpace(string(head)) != commitSHA {
return nil, errors.New("checked-out commit differs from fixed revision")
}
if err := verifyCheckout(repoDir, allowed); err != nil {
return nil, err
}
archivePath := filepath.Join(workDir, "package.zip")
usedBytes, err := directoryBytes(workDir)
if err != nil {
return nil, err
}
remainingBytes := limits.MaxWorkspaceBytes - usedBytes
if remainingBytes < 1 {
return nil, ErrLocalDiskBudget
}
archiveLimit := min(limits.MaxArchiveBytes, remainingBytes)
if err := requireMinimumFree(workDir, archiveLimit+limits.MinFreeBytes); err != nil {
return nil, err
}
size, digest, err := writePackageArchive(ctx, repoDir, archivePath, plan, archiveLimit)
if err != nil {
return nil, archiveWriteError(err, archiveLimit, limits.MaxArchiveBytes)
}
if size+usedBytes > limits.MaxWorkspaceBytes {
return nil, ErrLocalDiskBudget
}
keep = true
return &Artifact{Path: archivePath, Size: size, SHA256: digest, dir: workDir}, nil
}
// Validate the complete file plan before creating a workspace or fetching Git objects.
func prepareSparseCheckout(plan PackagePlan) ([]string, map[string]PackageFile, error) {
patterns := make([]string, 0, len(plan.Files)+len(plan.Evidence)+4)
patterns = append(patterns, "sparse-checkout", "set", "--no-cone", "--")
allowed := make(map[string]PackageFile, len(plan.Files))
hasSkillMD := false
var expectedTotal int64
for _, file := range plan.Files {
if !validPath(file.RepositoryPath) || !validPath(file.RelativePath) ||
!validGitSHA(file.SHA) || file.Size < 0 {
return nil, nil, errors.New("invalid preflighted file")
}
if plan.Root != "." && !strings.HasPrefix(file.RepositoryPath, plan.Root+"/") {
return nil, nil, errors.New("preflighted file is outside Skill root")
}
relativePath := file.RepositoryPath
if plan.Root != "." {
relativePath = strings.TrimPrefix(file.RepositoryPath, plan.Root+"/")
}
if file.RelativePath != relativePath || file.Size > plan.TotalBytes-expectedTotal {
return nil, nil, errors.New("preflighted file does not match package plan")
}
expectedTotal += file.Size
if file.RelativePath == "SKILL.md" {
hasSkillMD = true
}
if _, duplicate := allowed[file.RepositoryPath]; duplicate {
return nil, nil, errors.New("duplicate preflighted file")
}
allowed[file.RepositoryPath] = file
patterns = append(patterns, sparseFilePattern(file.RepositoryPath))
}
for _, evidence := range plan.Evidence {
if !evidence.Bundled {
continue
}
if selected, exists := allowed[evidence.RepositoryPath]; exists {
if selected.SHA != evidence.SHA || selected.Size != evidence.Size {
return nil, nil, errors.New("root evidence conflicts with package file")
}
continue
}
allowed[evidence.RepositoryPath] = PackageFile{
RepositoryPath: evidence.RepositoryPath,
SHA: evidence.SHA, Size: evidence.Size,
}
patterns = append(patterns, sparseFilePattern(evidence.RepositoryPath))
}
if !hasSkillMD || expectedTotal != plan.TotalBytes {
return nil, nil, errors.New("package plan has no SKILL.md or mismatched total size")
}
return patterns, allowed, nil
}
// Reserve room for Git objects, checked-out files, a ZIP, and baseline Git
// metadata before any network fetch. Actual usage is still checked at each Git
// step because repository tree metadata has no predictable size.
func canReserveWorkspace(packageBytes, evidenceBytes int64, limits AcquisitionLimits) bool {
if packageBytes < 0 || evidenceBytes < 0 || packageBytes > math.MaxInt64-evidenceBytes ||
limits.MaxWorkspaceBytes <= gitMetadataReserve || limits.MaxArchiveBytes < 1 {
return false
}
rawBytes := packageBytes + evidenceBytes
available := limits.MaxWorkspaceBytes - gitMetadataReserve
if rawBytes > available/2 {
return false
}
available -= 2 * rawBytes
archiveBytes := limits.MaxArchiveBytes
if rawBytes <= limits.MaxArchiveBytes-archiveOverheadReserve {
archiveBytes = rawBytes + archiveOverheadReserve
}
return archiveBytes <= available
}
func archiveWriteError(err error, workspaceLimit, uploadLimit int64) error {
if errors.Is(err, ErrArchiveTooLarge) && workspaceLimit < uploadLimit {
return ErrLocalDiskBudget
}
return localDiskWriteError(err)
}
func localDiskWriteError(err error) error {
if errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) {
return ErrLocalDiskBudget
}
return err
}
func sparseFilePattern(path string) string {
var pattern strings.Builder
pattern.WriteByte('/')
for _, char := range path {
if char == '*' || char == '?' || char == '[' || char == ']' || char == '\\' {
pattern.WriteByte('\\')
}
pattern.WriteRune(char)
}
return pattern.String()
}
func gitEnvironment() []string {
environment := make([]string, 0, len(os.Environ())+5)
for _, entry := range os.Environ() {
if strings.HasPrefix(entry, "GIT_") || strings.HasPrefix(entry, "HOME=") {
continue
}
environment = append(environment, entry)
}
return append(environment,
"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null",
"GIT_TERMINAL_PROMPT=0", "GIT_ALLOW_PROTOCOL=https", "GIT_LFS_SKIP_SMUDGE=1")
}
type gitRequestPolicyKey struct{}
type gitRequestPolicy struct {
before func(context.Context) error
after func(context.Context, error) error
}
// WithGitRequestPolicy guards commands that may fetch lazy partial-clone blobs.
// Archive creation and artifact delivery do not perform GitHub requests.
func WithGitRequestPolicy(ctx context.Context, before func(context.Context) error, after func(context.Context, error) error) context.Context {
return context.WithValue(ctx, gitRequestPolicyKey{}, gitRequestPolicy{before, after})
}
func runBoundedGit(ctx context.Context, repoDir, workDir string,
maxBytes, minFreeBytes int64, args ...string) (resultErr error) {
if err := github.CheckAcquisitionControl(ctx); err != nil {
return err
}
var afterRequest func(context.Context, error) error
if policy, ok := ctx.Value(gitRequestPolicyKey{}).(gitRequestPolicy); ok {
if err := policy.before(ctx); err != nil {
return err
}
if len(args) > 0 && (args[0] == "fetch" || args[0] == "checkout" || args[0] == "sparse-checkout") {
afterRequest = policy.after
}
}
if err := checkWorkspace(workDir, maxBytes, minFreeBytes); err != nil {
return err
}
command := exec.CommandContext(ctx, "git", append([]string{
"-c", "credential.helper=", "-c", "core.hooksPath=/dev/null",
"-c", "http.followRedirects=false",
}, args...)...)
command.Dir = repoDir
command.Env = gitEnvironment()
command.Stdout = io.Discard
command.Stderr = io.Discard
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := command.Start(); err != nil {
return localDiskWriteError(err)
}
if afterRequest != nil {
defer func() {
if err := afterRequest(ctx, resultErr); err != nil {
resultErr = err
}
}()
}
done := make(chan error, 1)
go func() { done <- command.Wait() }()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case err := <-done:
if checkErr := checkWorkspace(workDir, maxBytes, minFreeBytes); checkErr != nil {
return checkErr
}
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if errors.Is(requireMinimumFree(workDir, 1), ErrLocalDiskBudget) {
return ErrLocalDiskBudget
}
return errors.New("Git command failed")
}
return nil
case <-ticker.C:
err := checkWorkspace(workDir, maxBytes, minFreeBytes)
if err != nil {
_ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL)
<-done
return err
}
case <-ctx.Done():
_ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL)
<-done
return ctx.Err()
}
}
}
func checkWorkspace(workDir string, maxBytes, minFreeBytes int64) error {
size, err := directoryBytes(workDir)
if err != nil {
return err
}
if size > maxBytes {
return ErrLocalDiskBudget
}
return requireMinimumFree(workDir, minFreeBytes)
}
func requireMinimumFree(path string, bytes int64) error {
err := ensureFreeSpace(path, bytes)
if errors.Is(err, ErrCapacity) {
return ErrLocalDiskBudget
}
return err
}
func directoryBytes(root string) (int64, error) {
var total int64
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
if path != root && errors.Is(walkErr, os.ErrNotExist) {
return nil
}
return walkErr
}
if entry.IsDir() {
return nil
}
info, err := entry.Info()
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
total += info.Size()
return nil
})
return total, err
}
func ensureFreeSpace(path string, required int64) error {
var stats syscall.Statfs_t
if err := syscall.Statfs(path, &stats); err != nil {
return err
}
blockSize := uint64(stats.Bsize)
if blockSize == 0 {
return ErrCapacity
}
requiredBlocks := uint64(required) / blockSize
if uint64(required)%blockSize != 0 {
requiredBlocks++
}
if stats.Bavail < requiredBlocks {
return ErrCapacity
}
return nil
}
func verifyCheckout(repoDir string, allowed map[string]PackageFile) error {
seen := make(map[string]struct{}, len(allowed))
err := filepath.WalkDir(repoDir, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
relative, err := filepath.Rel(repoDir, path)
if err != nil {
return err
}
if relative == ".git" {
return filepath.SkipDir
}
if entry.IsDir() {
return nil
}
file, exists := allowed[filepath.ToSlash(relative)]
if !exists || entry.Type()&os.ModeSymlink != 0 {
return errors.New("checkout contains an unexpected file")
}
info, err := entry.Info()
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
return errors.New("checked-out file differs from tree metadata")
}
seen[file.RepositoryPath] = struct{}{}
return nil
})
if err != nil {
return err
}
if len(seen) != len(allowed) {
return errors.New("checkout omitted preflighted files")
}
return nil
}
type limitedArchiveWriter struct {
writer io.Writer
limit int64
written int64
}
func (writer *limitedArchiveWriter) Write(data []byte) (int, error) {
if int64(len(data)) > writer.limit-writer.written {
return 0, ErrArchiveTooLarge
}
n, err := writer.writer.Write(data)
writer.written += int64(n)
return n, err
}
func writePackageArchive(ctx context.Context, repoDir, archivePath string,
plan PackagePlan, maxBytes int64) (int64, string, error) {
if err := ctx.Err(); err != nil {
return 0, "", err
}
archive, err := os.OpenFile(archivePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return 0, "", err
}
defer archive.Close()
digest := sha256.New()
limited := &limitedArchiveWriter{writer: io.MultiWriter(archive, digest), limit: maxBytes}
zipWriter := zip.NewWriter(limited)
for _, file := range plan.Files {
if err := appendArchiveFile(ctx, zipWriter, repoDir, file.RepositoryPath,
file.RelativePath, file.Size); err != nil {
return 0, "", err
}
}
for _, evidence := range plan.Evidence {
if evidence.Bundled {
if err := appendArchiveFile(ctx, zipWriter, repoDir, evidence.RepositoryPath,
evidenceEntryPrefix+evidence.RepositoryPath, evidence.Size); err != nil {
return 0, "", err
}
}
}
manifest, err := json.Marshal(rootEvidenceManifest{
FormatVersion: 1, RootFiles: append([]RootEvidenceFile{}, plan.Evidence...),
})
if err != nil {
return 0, "", err
}
entry, err := zipWriter.Create(evidenceManifestPath)
if err != nil {
return 0, "", err
}
if _, err := entry.Write(manifest); err != nil {
return 0, "", err
}
if err := ctx.Err(); err != nil {
return 0, "", err
}
if err := zipWriter.Close(); err != nil {
return 0, "", err
}
if err := archive.Sync(); err != nil {
return 0, "", err
}
if err := ctx.Err(); err != nil {
return 0, "", err
}
return limited.written, hex.EncodeToString(digest.Sum(nil)), nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (reader contextReader) Read(data []byte) (int, error) {
if err := reader.ctx.Err(); err != nil {
return 0, err
}
return reader.reader.Read(data)
}
func appendArchiveFile(ctx context.Context, zipWriter *zip.Writer, repoDir, repositoryPath,
archivePath string, expectedSize int64) error {
if err := ctx.Err(); err != nil {
return err
}
input, err := os.Open(filepath.Join(repoDir, filepath.FromSlash(repositoryPath)))
if err != nil {
return err
}
entry, err := zipWriter.Create(archivePath)
if err == nil {
var copied int64
if expectedSize == math.MaxInt64 {
err = errors.New("preflighted file is too large")
} else {
copied, err = io.Copy(entry, io.LimitReader(contextReader{ctx, input}, expectedSize+1))
}
if err == nil && copied != expectedSize {
err = errors.New("file changed while creating package")
}
}
closeErr := input.Close()
if err != nil {
return err
}
return closeErr
}

804
github-sync-go/internal/repository/acquire_test.go Normal file
View file

@ -0,0 +1,804 @@
package repository
import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"math"
"os"
"os/exec"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"testing"
"time"
)
type cancelAfterFirstRead struct {
cancel context.CancelFunc
reads int
}
func (reader *cancelAfterFirstRead) Read(data []byte) (int, error) {
reader.reads++
if reader.reads > 1 {
return 0, io.EOF
}
data[0] = 'x'
reader.cancel()
return 1, nil
}
func TestArchiveReaderStopsAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
source := &cancelAfterFirstRead{cancel: cancel}
_, err := io.Copy(io.Discard, contextReader{ctx: ctx, reader: source})
if !errors.Is(err, context.Canceled) || source.reads != 1 {
t.Fatalf("archive reader continued after cancellation: reads=%d error=%v", source.reads, err)
}
}
func TestWritePackageArchiveDoesNotStartAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
path := filepath.Join(t.TempDir(), "package.zip")
_, _, err := writePackageArchive(ctx, t.TempDir(), path, PackagePlan{}, 1024)
if !errors.Is(err, context.Canceled) {
t.Fatalf("canceled archive returned %v", err)
}
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("canceled archive created a file: %v", err)
}
}
func TestCanReserveWorkspaceIncludesGitCheckoutAndArchive(t *testing.T) {
const mib = int64(1 << 20)
limits := AcquisitionLimits{MaxWorkspaceBytes: 64 * mib, MaxArchiveBytes: 24 * mib}
if canReserveWorkspace(20*mib, 0, limits) {
t.Fatal("a workspace smaller than the combined Git, checkout, archive and metadata reserve was accepted")
}
limits.MaxWorkspaceBytes = 80 * mib
if !canReserveWorkspace(20*mib, 0, limits) {
t.Fatal("a workspace with enough reserved capacity was rejected")
}
if canReserveWorkspace(math.MaxInt64, 1, limits) {
t.Fatal("overflowing package and evidence sizes were accepted")
}
}
func TestArchiveWriteErrorDistinguishesWorkspaceAndUploadLimits(t *testing.T) {
if err := archiveWriteError(ErrArchiveTooLarge, 1024, 2048); !errors.Is(err, ErrLocalDiskBudget) {
t.Fatalf("workspace-limited archive was not deferred as disk budget: %v", err)
}
if err := archiveWriteError(ErrArchiveTooLarge, 2048, 2048); !errors.Is(err, ErrArchiveTooLarge) {
t.Fatalf("upload-limited archive was not rejected as too large: %v", err)
}
for _, diskError := range []error{syscall.ENOSPC, syscall.EDQUOT} {
writeError := &os.PathError{Op: "write", Path: "package.zip", Err: diskError}
if err := archiveWriteError(writeError, 2048, 2048); !errors.Is(err, ErrLocalDiskBudget) {
t.Fatalf("filesystem write limit %v was not deferred: %v", diskError, err)
}
}
}
func TestAcquirePackageDefersWhenArchiveExceedsRemainingWorkspace(t *testing.T) {
const workspaceMax = int64(20 << 20)
const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
fakeBin := t.TempDir()
git := filepath.Join(fakeBin, "git")
script := `#!/bin/sh
case " $* " in
*" checkout "*)
mkdir -p .git
printf '%s\n' "$SKILLHUB_TEST_COMMIT" > .git/HEAD
head -c 10000 /dev/urandom > SKILL.md
truncate -s "$((SKILLHUB_TEST_WORKSPACE_MAX - 20000))" .git/filler
;;
esac
`
if err := os.WriteFile(git, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("SKILLHUB_TEST_COMMIT", commitSHA)
t.Setenv("SKILLHUB_TEST_WORKSPACE_MAX", strconv.FormatInt(workspaceMax, 10))
root := t.TempDir()
artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA, root,
PackagePlan{Root: ".", TotalBytes: 10000, Files: []PackageFile{{
RepositoryPath: "SKILL.md", RelativePath: "SKILL.md",
SHA: strings.Repeat("b", 40), Size: 10000,
}}}, AcquisitionLimits{
MaxWorkspaceBytes: workspaceMax, MaxArchiveBytes: 1 << 20,
})
if artifact != nil || !errors.Is(err, ErrLocalDiskBudget) || errors.Is(err, ErrArchiveTooLarge) {
t.Fatalf("workspace-limited archive returned artifact=%v, error=%v", artifact, err)
}
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatalf("capacity deferral left temporary files: %v, %v", entries, err)
}
}
func TestAcquirePackageCancellationRemovesGitWorkspace(t *testing.T) {
fakeBin := t.TempDir()
started := filepath.Join(fakeBin, "git-started")
git := filepath.Join(fakeBin, "git")
if err := os.WriteFile(git, []byte("#!/bin/sh\n: > \"$SKILLHUB_TEST_GIT_STARTED\"\nexec sleep 60\n"), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("SKILLHUB_TEST_GIT_STARTED", started)
root := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
finished := make(chan error, 1)
go func() {
artifact, err := AcquirePackage(ctx, "openai", "skills", strings.Repeat("a", 40), root,
PackagePlan{Root: "skills/demo", TotalBytes: 1, Files: []PackageFile{{
RepositoryPath: "skills/demo/SKILL.md", RelativePath: "SKILL.md",
SHA: strings.Repeat("b", 40), Size: 1,
}}}, AcquisitionLimits{MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 1 << 20})
if artifact != nil {
_ = artifact.Close()
}
finished <- err
}()
deadline := time.Now().Add(5 * time.Second)
for {
if _, err := os.Stat(started); err == nil {
break
} else if !os.IsNotExist(err) {
t.Fatal(err)
}
if time.Now().After(deadline) {
cancel()
t.Fatal("Git command did not start")
}
time.Sleep(20 * time.Millisecond)
}
cancel()
select {
case err := <-finished:
if !errors.Is(err, context.Canceled) {
t.Fatalf("canceled acquisition returned %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("canceled Git command did not stop")
}
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatalf("canceled acquisition left temporary files: %v, %v", entries, err)
}
}
func TestAcquirePackageStopsOnSIGTERM(t *testing.T) {
if os.Getenv("SKILLHUB_TEST_SIGNAL_CHILD") == "1" {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM)
defer stop()
_, err := AcquirePackage(ctx, "openai", "skills", strings.Repeat("a", 40),
os.Getenv("SKILLHUB_TEST_TEMP_ROOT"), PackagePlan{
Root: "skills/demo", TotalBytes: 1, Files: []PackageFile{{
RepositoryPath: "skills/demo/SKILL.md", RelativePath: "SKILL.md",
SHA: strings.Repeat("b", 40), Size: 1,
}},
}, AcquisitionLimits{MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 1 << 20})
if !errors.Is(err, context.Canceled) {
t.Fatalf("SIGTERM did not cancel Git acquisition: %v", err)
}
return
}
fakeBin := t.TempDir()
started := filepath.Join(fakeBin, "git-started")
git := filepath.Join(fakeBin, "git")
if err := os.WriteFile(git, []byte("#!/bin/sh\nprintf '%s' \"$$\" > \"$SKILLHUB_TEST_GIT_STARTED\"\nexec sleep 60\n"), 0700); err != nil {
t.Fatal(err)
}
root := t.TempDir()
binary, err := os.Executable()
if err != nil {
t.Fatal(err)
}
command := exec.Command(binary, "-test.run=^TestAcquirePackageStopsOnSIGTERM$", "-test.v")
command.Env = append(os.Environ(),
"PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"),
"SKILLHUB_TEST_GIT_STARTED="+started,
"SKILLHUB_TEST_SIGNAL_CHILD=1", "SKILLHUB_TEST_TEMP_ROOT="+root)
var output bytes.Buffer
command.Stdout, command.Stderr = &output, &output
if err := command.Start(); err != nil {
t.Fatal(err)
}
defer func() {
if command.ProcessState == nil {
_ = command.Process.Kill()
if data, err := os.ReadFile(started); err == nil {
if pid, err := strconv.Atoi(string(data)); err == nil {
_ = syscall.Kill(-pid, syscall.SIGKILL)
}
}
}
}()
deadline := time.Now().Add(5 * time.Second)
for {
if _, err := os.Stat(started); err == nil {
break
} else if !os.IsNotExist(err) {
t.Fatal(err)
}
if time.Now().After(deadline) {
t.Fatalf("child Git command did not start: %s", output.String())
}
time.Sleep(20 * time.Millisecond)
}
if err := command.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
finished := make(chan error, 1)
go func() { finished <- command.Wait() }()
select {
case err := <-finished:
if err != nil {
t.Fatalf("SIGTERM child did not exit cleanly: %v, %s", err, output.String())
}
case <-time.After(10 * time.Second):
t.Fatalf("SIGTERM child did not stop: %s", output.String())
}
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatalf("SIGTERM left temporary files: %v, %v", entries, err)
}
}
// This immutable public commit exercises the real HTTPS partial-clone path.
// Run manually with SKILLHUB_GITHUB_SYNC_LIVE=1; regular tests remain offline.
func TestLiveAcquireFixedPublicSkill(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE") != "1" {
t.Skip("requires an explicit public GitHub smoke test")
}
const commitSHA = "49f948faa9258a0c61caceaf225e179651397431"
plan, want := liveSkillCreatorPlan()
tempRoot := t.TempDir()
limits := AcquisitionLimits{
MaxWorkspaceBytes: 512 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 128 << 20,
}
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_SMALL_VOLUME") == "1" {
limits.MaxWorkspaceBytes = 32 << 20
limits.MinFreeBytes = 8 << 20
}
stopSampling := make(chan struct{})
peakSample := make(chan int64, 1)
go func() {
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
var peak int64
for {
select {
case <-ticker.C:
if size, err := directoryBytes(tempRoot); err == nil && size > peak {
peak = size
}
case <-stopSampling:
peakSample <- peak
return
}
}
}()
artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA,
tempRoot, plan, limits)
finalSample, finalSampleErr := directoryBytes(tempRoot)
close(stopSampling)
sampledPeakBytes := <-peakSample
if finalSampleErr == nil && finalSample > sampledPeakBytes {
sampledPeakBytes = finalSample
}
if err != nil {
t.Fatal(err)
}
archiveBytes, err := os.ReadFile(artifact.Path)
if err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(archiveBytes)
if artifact.Size != int64(len(archiveBytes)) || artifact.SHA256 != hex.EncodeToString(digest[:]) {
t.Fatal("archive size or digest does not match the generated artifact")
}
if capture := os.Getenv("SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE"); capture != "" {
output, err := os.OpenFile(capture, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
t.Fatal(err)
}
_, writeErr := output.Write(archiveBytes)
closeErr := output.Close()
if writeErr != nil || closeErr != nil {
t.Fatalf("write captured archive: %v, close: %v", writeErr, closeErr)
}
}
workspaceBytes, err := directoryBytes(artifact.dir)
if err != nil {
t.Fatal(err)
}
t.Logf("fixed commit %s: archive=%d bytes, workspace after acquisition=%d bytes, sampled peak=%d bytes",
commitSHA, artifact.Size, workspaceBytes, sampledPeakBytes)
archive, err := zip.OpenReader(artifact.Path)
if err != nil {
t.Fatal(err)
}
got := make([]string, 0, len(archive.File))
for _, entry := range archive.File {
got = append(got, entry.Name)
}
if err := archive.Close(); err != nil {
t.Fatal(err)
}
sort.Strings(got)
if len(got) != len(want) {
t.Fatalf("archive has %d entries, want %d: %v", len(got), len(want), got)
}
for index := range want {
if got[index] != want[index] {
t.Fatalf("archive entries = %v, want %v", got, want)
}
}
dir := artifact.dir
if err := artifact.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("temporary package directory remains after Close: %v", err)
}
}
// This repository has thousands of tracked paths at the fixed commit. The
// opt-in smoke test checks that a single Skill stays within a small workspace
// even when the remote repository tree is substantially larger.
func TestLiveAcquireLargerPublicRepositorySkill(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_LARGE_REPO") != "1" {
t.Skip("requires an explicit public GitHub smoke test")
}
const commitSHA = "e325b0ee302fdf66abb1990ad7957e1a1c7aba5a"
plan := liveLargerPublicRepositoryPlan()
tempRoot := t.TempDir()
artifact, err := AcquirePackage(context.Background(), "SangHynh", "agent-skills",
commitSHA, tempRoot, plan, AcquisitionLimits{
MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 8 << 20, MinFreeBytes: 16 << 20,
})
if err != nil {
t.Fatal(err)
}
archive, err := zip.OpenReader(artifact.Path)
if err != nil {
_ = artifact.Close()
t.Fatal(err)
}
if len(archive.File) != 2 || archive.File[0].Name != "SKILL.md" ||
archive.File[1].Name != evidenceManifestPath {
_ = archive.Close()
_ = artifact.Close()
t.Fatalf("unexpected Skill archive entries: %+v", archive.File)
}
if err := archive.Close(); err != nil {
_ = artifact.Close()
t.Fatal(err)
}
workspaceBytes, err := directoryBytes(tempRoot)
if err != nil || workspaceBytes >= 64<<20 {
_ = artifact.Close()
t.Fatalf("workspace exceeded configured limit: %d bytes, error %v", workspaceBytes, err)
}
t.Logf("fixed commit %s: archive=%d bytes, workspace after acquisition=%d bytes",
commitSHA, artifact.Size, workspaceBytes)
if err := artifact.Close(); err != nil {
t.Fatal(err)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("temporary files remain after package cleanup: %v, %v", remaining, err)
}
}
func liveLargerPublicRepositoryPlan() PackagePlan {
const root = "skills/00-andruia-consultant"
return PackagePlan{Root: root, TotalBytes: 2894, Files: []PackageFile{{
RepositoryPath: root + "/SKILL.md", RelativePath: "SKILL.md",
SHA: "e173357610b8204ce3d676ca9172b3463b02ce97", Size: 2894,
}}}
}
func liveSkillCreatorPlan() (PackagePlan, []string) {
const skillRoot = "skills/.system/skill-creator"
files := []struct {
path string
sha string
size int64
}{
{"LICENSE.txt", "d645695673349e3947e8e5ae42332d0ac3164cd7", 11358},
{"SKILL.md", "72bc0b97e7a6476254a9d5c424c9971748402ec3", 18664},
{"agents/openai.yaml", "f5ce09ed118b72908b7308fcfc1619f59e89f6ea", 192},
{"references/openai_yaml.md", "da5629f8de500ec6f81291d239a2d9c68d1d9044", 2130},
{"scripts/generate_openai_yaml.py", "1a9d784f8704999daf55a2b9036094949d9af3f8", 6614},
{"scripts/init_skill.py", "f90703eca8159a21a9684ed1df974c702d92f814", 14483},
{"scripts/quick_validate.py", "0547b4041a5f58fa19892079a114a1df98286406", 3293},
}
plan := PackagePlan{Root: skillRoot}
want := make([]string, 0, len(files)+1)
for _, file := range files {
plan.Files = append(plan.Files, PackageFile{
RepositoryPath: skillRoot + "/" + file.path, RelativePath: file.path,
SHA: file.sha, Size: file.size,
})
plan.TotalBytes += file.size
want = append(want, file.path)
}
want = append(want, evidenceManifestPath)
sort.Strings(want)
return plan, want
}
// Two Skill roots in one fixed repository revision must use isolated package
// workspaces: closing the first package releases its Git objects before the
// second package starts.
func TestLiveAcquireTwoFixedPublicSkillsSequentially(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE") != "1" {
t.Skip("requires an explicit public GitHub smoke test")
}
const commitSHA = "49f948faa9258a0c61caceaf225e179651397431"
creator, _ := liveSkillCreatorPlan()
goal := liveDefineGoalPlan()
tempRoot := t.TempDir()
for _, plan := range []PackagePlan{creator, goal} {
artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA,
tempRoot, plan, AcquisitionLimits{
MaxWorkspaceBytes: 512 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 128 << 20,
})
if err != nil {
t.Fatalf("acquire %s: %v", plan.Root, err)
}
archive, err := zip.OpenReader(artifact.Path)
if err != nil {
t.Fatal(err)
}
if len(archive.File) != len(plan.Files)+1 {
t.Fatalf("%s: archive entries = %d, want %d",
plan.Root, len(archive.File), len(plan.Files)+1)
}
if err := archive.Close(); err != nil {
t.Fatal(err)
}
workspaceBytes, err := directoryBytes(tempRoot)
if err != nil || workspaceBytes >= 32<<20 {
t.Fatalf("%s: unexpected workspace bytes %d: %v", plan.Root, workspaceBytes, err)
}
t.Logf("%s: archive=%d bytes, workspace after acquisition=%d bytes",
plan.Root, artifact.Size, workspaceBytes)
if err := artifact.Close(); err != nil {
t.Fatal(err)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("%s: temporary files remain after package cleanup: %v, %v",
plan.Root, remaining, err)
}
}
}
func liveDefineGoalPlan() PackagePlan {
return PackagePlan{
Root: "skills/.curated/define-goal",
Files: []PackageFile{
{RepositoryPath: "skills/.curated/define-goal/LICENSE.txt", RelativePath: "LICENSE.txt",
SHA: "13e25df86ce06eb6488e6a6bc5c5847f5dedc352", Size: 10776},
{RepositoryPath: "skills/.curated/define-goal/SKILL.md", RelativePath: "SKILL.md",
SHA: "87f111bd700e0d993465f7ac741847b5daee57d6", Size: 5647},
{RepositoryPath: "skills/.curated/define-goal/agents/openai.yaml", RelativePath: "agents/openai.yaml",
SHA: "28c7e7a644d7ae3f5616c7ddd408c573139999f0", Size: 208},
},
TotalBytes: 16631,
}
}
// Run in the production image with a 72 MiB /tmp tmpfs. Both packages use
// independent Git workspaces while the volume provides a hard physical limit.
func TestLiveAcquireTwoFixedPublicSkillsConcurrentlyOnSmallVolume(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CONCURRENT") != "1" {
t.Skip("requires an explicit concurrent public GitHub smoke test")
}
const commitSHA = "49f948faa9258a0c61caceaf225e179651397431"
tempRoot := t.TempDir()
var filesystem syscall.Statfs_t
if err := syscall.Statfs(tempRoot, &filesystem); err != nil {
t.Fatal(err)
}
volumeBytes := int64(filesystem.Blocks) * int64(filesystem.Bsize)
if volumeBytes > 80<<20 {
t.Fatalf("concurrent smoke test requires a small tmpfs: volume=%d bytes", volumeBytes)
}
creator, creatorFiles := liveSkillCreatorPlan()
goal := liveDefineGoalPlan()
type request struct {
owner, name, commit string
plan PackagePlan
}
requests := []request{
{owner: "openai", name: "skills", commit: commitSHA, plan: creator},
{owner: "openai", name: "skills", commit: commitSHA, plan: goal},
}
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CROSS_REPOSITORY") == "1" {
goal = liveLargerPublicRepositoryPlan()
requests[1] = request{owner: "SangHynh", name: "agent-skills",
commit: "e325b0ee302fdf66abb1990ad7957e1a1c7aba5a", plan: goal}
}
expectedEntries := map[string]int{
creator.Root: len(creatorFiles),
goal.Root: len(goal.Files) + 1,
}
limits := AcquisitionLimits{
MaxWorkspaceBytes: 32 << 20, MaxArchiveBytes: 8 << 20, MinFreeBytes: 8 << 20,
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
type result struct {
root string
artifact *Artifact
err error
}
results := make(chan result, len(requests))
start := make(chan struct{})
for _, requested := range requests {
go func(requested request) {
<-start
artifact, err := AcquirePackage(ctx, requested.owner, requested.name,
requested.commit, tempRoot, requested.plan, limits)
results <- result{root: requested.plan.Root, artifact: artifact, err: err}
}(requested)
}
close(start)
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
var sampledPeak int64
var overlappingWorkspaces bool
artifacts := make([]*Artifact, 0, len(requests))
for completed := 0; completed < len(requests); {
select {
case outcome := <-results:
completed++
if outcome.err != nil {
t.Errorf("acquire %s: %v", outcome.root, outcome.err)
continue
}
artifacts = append(artifacts, outcome.artifact)
archive, err := zip.OpenReader(outcome.artifact.Path)
if err != nil {
t.Errorf("open %s archive: %v", outcome.root, err)
continue
}
if len(archive.File) != expectedEntries[outcome.root] {
t.Errorf("%s archive entries=%d, want=%d", outcome.root,
len(archive.File), expectedEntries[outcome.root])
}
if err := archive.Close(); err != nil {
t.Errorf("close %s archive: %v", outcome.root, err)
}
case <-ticker.C:
if completed == 0 && len(results) == 0 {
if entries, err := os.ReadDir(tempRoot); err == nil && len(entries) == len(requests) {
overlappingWorkspaces = true
}
}
if used, err := directoryBytes(tempRoot); err == nil {
sampledPeak = max(sampledPeak, used)
} else {
t.Errorf("measure concurrent workspaces: %v", err)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
if len(artifacts) != len(requests) {
t.Errorf("completed packages=%d, want=%d", len(artifacts), len(requests))
}
if !overlappingWorkspaces {
t.Error("the two package workspaces did not overlap during acquisition")
}
if used, err := directoryBytes(tempRoot); err != nil || used > 64<<20 {
t.Errorf("concurrent workspaces exceeded total budget: bytes=%d error=%v", used, err)
} else {
sampledPeak = max(sampledPeak, used)
}
for _, artifact := range artifacts {
if err := artifact.Close(); err != nil {
t.Errorf("remove %s: %v", artifact.Path, err)
}
}
if remaining, err := os.ReadDir(tempRoot); err != nil || len(remaining) != 0 {
t.Errorf("concurrent workspaces were not cleared: entries=%d error=%v", len(remaining), err)
}
t.Logf("concurrent fixed commit packages=%d cross_repository=%t volume_bytes=%d sampled_peak_bytes=%d overlapping_workspaces=%t",
len(artifacts), os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CROSS_REPOSITORY") == "1",
volumeBytes, sampledPeak, overlappingWorkspaces)
}
// Fill both Git metadata directories near the configured aggregate budget.
// The fake Git executable avoids network variance while retaining the real
// workspace accounting and ZIP creation path.
func TestConcurrentAcquisitionNearSmallVolumeBudget(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_CONCURRENT_CAPACITY_TEST") != "1" {
t.Skip("requires an explicit small-volume capacity test")
}
const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
const mib = int64(1 << 20)
root := t.TempDir()
var filesystem syscall.Statfs_t
if err := syscall.Statfs(root, &filesystem); err != nil {
t.Fatal(err)
}
if volume := int64(filesystem.Blocks) * int64(filesystem.Bsize); volume > 80*mib {
t.Fatalf("capacity test requires a small tmpfs: volume=%d bytes", volume)
}
fakeBin := t.TempDir()
script := `#!/bin/sh
case " $* " in
*" checkout "*)
mkdir -p .git
printf '%s\n' "$SKILLHUB_TEST_COMMIT" > .git/HEAD
head -c 10000 /dev/zero > SKILL.md
dd if=/dev/zero of=.git/filler bs=1048576 count=28 2>/dev/null
sleep 0.2
;;
esac
`
if err := os.WriteFile(filepath.Join(fakeBin, "git"), []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("SKILLHUB_TEST_COMMIT", commitSHA)
if resolved, err := exec.LookPath("git"); err != nil || resolved != filepath.Join(fakeBin, "git") {
t.Fatalf("fake Git executable was not selected: %q, %v", resolved, err)
}
plan := PackagePlan{Root: ".", TotalBytes: 10000, Files: []PackageFile{{
RepositoryPath: "SKILL.md", RelativePath: "SKILL.md", SHA: strings.Repeat("b", 40), Size: 10000,
}}}
limits := AcquisitionLimits{MaxWorkspaceBytes: 32 * mib,
MaxArchiveBytes: mib, MinFreeBytes: 8 * mib}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
results := make(chan struct {
artifact *Artifact
err error
}, 2)
start := make(chan struct{})
for range 2 {
go func() {
<-start
artifact, err := AcquirePackage(ctx, "openai", "skills", commitSHA, root, plan, limits)
results <- struct {
artifact *Artifact
err error
}{artifact, err}
}()
}
close(start)
artifacts := make([]*Artifact, 0, 2)
for range 2 {
select {
case outcome := <-results:
if outcome.err != nil {
t.Errorf("near-budget acquisition failed: %v", outcome.err)
} else {
artifacts = append(artifacts, outcome.artifact)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
used, err := directoryBytes(root)
if err != nil || used < 56*mib || used > 64*mib {
t.Errorf("unexpected aggregate workspace occupancy: bytes=%d error=%v", used, err)
}
if err := syscall.Statfs(root, &filesystem); err != nil ||
int64(filesystem.Bavail)*int64(filesystem.Bsize) < 8*mib {
t.Errorf("temporary filesystem lost its 8 MiB reserve: %v", err)
}
for _, artifact := range artifacts {
if err := artifact.Close(); err != nil {
t.Errorf("clean near-budget package: %v", err)
}
}
if remaining, err := os.ReadDir(root); err != nil || len(remaining) != 0 {
t.Errorf("near-budget workspaces remain: entries=%d error=%v", len(remaining), err)
}
t.Logf("near-budget concurrent packages=%d aggregate_workspace_bytes=%d",
len(artifacts), used)
}
func TestWorkspaceChecksDirectorySizeAndFreeSpace(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "object"), make([]byte, 32), 0600); err != nil {
t.Fatal(err)
}
if err := checkWorkspace(root, 31, 0); !errors.Is(err, ErrLocalDiskBudget) {
t.Fatalf("workspace over its byte limit: %v", err)
}
if err := checkWorkspace(root, 32, 0); err != nil {
t.Fatalf("workspace at its byte limit: %v", err)
}
var stats syscall.Statfs_t
if err := syscall.Statfs(root, &stats); err != nil {
t.Fatal(err)
}
available := uint64(stats.Bavail) * uint64(stats.Bsize)
if available > math.MaxInt64-1 {
t.Skip("filesystem is too large for an int64 free-space boundary")
}
if err := requireMinimumFree(root, int64(available)+1); !errors.Is(err, ErrLocalDiskBudget) {
t.Fatalf("minimum free space was not protected: %v", err)
}
}
func TestDirectoryBytesDoesNotIgnoreMissingWorkspace(t *testing.T) {
_, err := directoryBytes(filepath.Join(t.TempDir(), "missing-workspace"))
if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("missing workspace must fail accounting: %v", err)
}
}
func TestAcquirePackageRejectsCapacityBeforeNetworkFetch(t *testing.T) {
const mib = int64(1 << 20)
plan := PackagePlan{
Root: ".", TotalBytes: 20 * mib,
Files: []PackageFile{{RepositoryPath: "SKILL.md", RelativePath: "SKILL.md",
SHA: skillBlobSHA, Size: 20 * mib}},
}
artifact, err := AcquirePackage(context.Background(), "owner", "repo", rootTreeSHA,
t.TempDir(), plan, AcquisitionLimits{MaxWorkspaceBytes: 64 * mib,
MaxArchiveBytes: 24 * mib, MinFreeBytes: 0})
if artifact != nil || !errors.Is(err, ErrCapacity) {
t.Fatalf("expected pre-fetch capacity block, got artifact=%v, error=%v", artifact, err)
}
}
func TestSparseCheckoutKeepsOnlySelectedSkillFile(t *testing.T) {
root := t.TempDir()
repoDir := filepath.Join(root, "repo")
for _, directory := range []string{filepath.Join(repoDir, "selected"), filepath.Join(repoDir, "sibling")} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
for _, path := range []string{"selected/SKILL.md", "sibling/SKILL.md"} {
if err := os.WriteFile(filepath.Join(repoDir, path), []byte("name: fixture\n"), 0600); err != nil {
t.Fatal(err)
}
}
for _, arguments := range [][]string{
{"init", "--quiet"}, {"add", "."},
{"-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "-m", "fixture"},
} {
command := exec.Command("git", arguments...)
command.Dir = repoDir
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("git fixture setup failed: %v: %s", err, output)
}
}
if err := runBoundedGit(context.Background(), repoDir, root, 64<<20, 0,
"sparse-checkout", "set", "--no-cone", "--", sparseFilePattern("selected/SKILL.md")); err != nil {
t.Fatal(err)
}
if err := runBoundedGit(context.Background(), repoDir, root, 64<<20, 0,
"checkout", "--quiet", "--detach", "--force", "HEAD"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(repoDir, "selected/SKILL.md")); err != nil {
t.Fatalf("planned Skill file missing: %v", err)
}
if _, err := os.Stat(filepath.Join(repoDir, "sibling/SKILL.md")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("unplanned sibling file was checked out: %v", err)
}
}

69
github-sync-go/internal/repository/capacity_test.go Normal file
View file

@ -0,0 +1,69 @@
package repository
import (
"context"
"fmt"
"runtime"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type capacityTreeReader struct {
tree github.Tree
calls int
}
func (reader *capacityTreeReader) GetTree(_ context.Context, _, _, sha string,
_ bool) (github.Tree, github.Quota, error) {
reader.calls++
if sha != reader.tree.SHA {
return github.Tree{}, github.Quota{}, fmt.Errorf("unexpected tree SHA")
}
return reader.tree, github.Quota{}, nil
}
func TestSyntheticRepositoryMetadataCapacity(t *testing.T) {
const repositories = 1000
const skillsPerRepository = 10
const expectedSkills = repositories * skillsPerRepository
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
started := time.Now()
requests := 0
plans := 0
for repositoryIndex := range repositories {
sha := fmt.Sprintf("%040x", repositoryIndex+1)
entries := make([]github.TreeEntry, 0, skillsPerRepository)
for skillIndex := range skillsPerRepository {
entries = append(entries, github.TreeEntry{
Path: fmt.Sprintf("skills/skill-%02d/SKILL.md", skillIndex),
Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 64,
})
}
reader := &capacityTreeReader{tree: github.Tree{SHA: sha, Entries: entries}}
snapshot, err := EnumerateTree(context.Background(), reader, "fixture", "repo", sha,
TreeLimits{MaxTreeRequests: 8, MaxEntries: 100_000})
if err != nil || len(snapshot.IncompleteSubtrees) != 0 {
t.Fatalf("repository %d metadata failed: snapshot=%+v, error=%v", repositoryIndex, snapshot, err)
}
roots := RootsFromSnapshot(snapshot, nil)
preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{
MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
})
if err != nil || len(roots.Paths) != skillsPerRepository ||
len(preflight.Plans) != skillsPerRepository || len(preflight.Failures) != 0 {
t.Fatalf("repository %d preflight failed: roots=%d, plans=%d, failures=%d, error=%v",
repositoryIndex, len(roots.Paths), len(preflight.Plans), len(preflight.Failures), err)
}
requests += reader.calls
plans += len(preflight.Plans)
}
runtime.ReadMemStats(&after)
if requests != repositories || plans != expectedSkills {
t.Fatalf("unexpected synthetic coverage: requests=%d, plans=%d", requests, plans)
}
t.Logf("synthetic metadata baseline: repositories=%d, skill paths=%d, tree requests=%d, elapsed=%s, allocated=%d bytes",
repositories, plans, requests, time.Since(started), after.TotalAlloc-before.TotalAlloc)
}

58
github-sync-go/internal/repository/official_quota_test.go Normal file
View file

@ -0,0 +1,58 @@
package repository
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
)
func TestOfficialGitQuotaPolicy(t *testing.T) {
for _, kind := range []string{"blocked-before-start", "fetch-report-failure", "checkout-report-failure", "local-init"} {
t.Run(kind, func(t *testing.T) {
dir := t.TempDir()
bin := t.TempDir()
if err := os.WriteFile(filepath.Join(bin, "git"), []byte("#!/bin/sh\nexit 0\n"), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin)
before, after := 0, 0
denied := errors.New("quota policy denied")
ctx := WithGitRequestPolicy(context.Background(), func(context.Context) error {
before++
if kind == "blocked-before-start" {
return denied
}
return nil
}, func(context.Context, error) error { after++; return denied })
command := "fetch"
if kind == "checkout-report-failure" {
command = "checkout"
}
if kind == "local-init" {
command = "init"
}
err := runBoundedGit(ctx, dir, dir, 1<<20, 0, command)
if before != 1 {
t.Fatalf("before=%d", before)
}
if kind == "local-init" {
if err != nil || after != 0 {
t.Fatal("local init fabricated quota observation")
}
return
}
if !errors.Is(err, denied) {
t.Fatalf("policy failure swallowed: %v", err)
}
wantAfter := 1
if kind == "blocked-before-start" {
wantAfter = 0
}
if after != wantAfter {
t.Fatalf("after=%d want=%d", after, wantAfter)
}
})
}
}

74
github-sync-go/internal/repository/package_boundaries_test.go Normal file
View file

@ -0,0 +1,74 @@
package repository
import (
"fmt"
"reflect"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestSourceIgnoreRemovalRestoresOnlySelectedSkill(t *testing.T) {
snapshot := TreeSnapshot{Entries: []github.TreeEntry{}}
for _, path := range []string{"skills/custom/SKILL.md", "skills/good/SKILL.md", "docs/demo/SKILL.md", "examples/demo/SKILL.md", "tests/demo/SKILL.md"} {
snapshot.Entries = append(snapshot.Entries, github.TreeEntry{Path: path, Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1})
}
for _, testcase := range []struct {
name string
ignores []string
want []string
}{
{"ignored", []string{"skills/custom"}, []string{"docs/demo", "examples/demo", "skills/good"}},
{"restored", nil, []string{"docs/demo", "examples/demo", "skills/custom", "skills/good"}},
} {
t.Run(testcase.name, func(t *testing.T) {
roots := RootsFromSnapshot(snapshot, testcase.ignores)
if !reflect.DeepEqual(roots.Paths, testcase.want) {
t.Fatalf("roots=%v want=%v", roots.Paths, testcase.want)
}
})
}
}
func TestPackagePolicyExactBoundariesAndSubmoduleIsolation(t *testing.T) {
limits := PackageLimits{MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20}
for _, tc := range []struct {
name string
files int
size int64
submodule bool
wantFailure string
}{
{"100-files", 100, 1, false, ""}, {"101-files", 101, 1, false, "PACKAGE_TOO_LARGE"},
{"10MiB-file", 1, 10 << 20, false, ""}, {"10MiB-plus-one", 1, (10 << 20) + 1, false, "PACKAGE_TOO_LARGE"},
{"100MiB-package", 10, 10 << 20, false, ""}, {"110MiB-package", 11, 10 << 20, false, "PACKAGE_TOO_LARGE"},
{"submodule", 1, 1, true, "SUBMODULE"},
} {
t.Run(tc.name, func(t *testing.T) {
entries := []github.TreeEntry{{Path: "good/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}}
for i := 0; i < tc.files; i++ {
name := fmt.Sprintf("file-%03d.txt", i)
if i == 0 {
name = "SKILL.md"
}
entries = append(entries, github.TreeEntry{Path: "selected/" + name, Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: tc.size})
}
if tc.submodule {
entries = append(entries, github.TreeEntry{Path: "selected/vendor", Type: "commit", Mode: "160000", SHA: skillBlobSHA})
}
result, err := PreflightPackages(TreeSnapshot{Entries: entries}, []string{"good", "selected"}, limits)
if err != nil {
t.Fatal(err)
}
if tc.wantFailure == "" {
if len(result.Plans) != 2 || len(result.Failures) != 0 {
t.Fatalf("boundary rejected: %+v", result)
}
} else {
if len(result.Plans) != 1 || result.Plans[0].Root != "good" || len(result.Failures) != 1 || result.Failures[0].Code != tc.wantFailure || result.Failures[0].Retryable {
t.Fatalf("failure not isolated: %+v", result)
}
}
})
}
}

226
github-sync-go/internal/repository/package_plan.go Normal file
View file

@ -0,0 +1,226 @@
package repository
import (
"errors"
"regexp"
"sort"
"strings"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type PackageLimits struct {
MaxFiles int
MaxSingleFileBytes int64
MaxPackageBytes int64
}
type PackageFile struct {
RepositoryPath string
RelativePath string
SHA string
Size int64
}
type PackagePlan struct {
Root string
Files []PackageFile
Evidence []RootEvidenceFile
TotalBytes int64
}
// RootEvidenceFile is repository-level license or notice metadata at the fixed commit.
// Large files keep their Git blob digest in the manifest without being downloaded.
type RootEvidenceFile struct {
RepositoryPath string `json:"path"`
SHA string `json:"gitSha"`
Size int64 `json:"size"`
Bundled bool `json:"bundled"`
}
const (
maxRootEvidenceFiles = 16
maxEvidenceFileBytes = 64 << 10
maxEvidenceTotalBytes = 256 << 10
)
var rootEvidenceName = regexp.MustCompile(`(?i)^(license|licence|copying|notice|copyright)(?:[-_.][a-z0-9][a-z0-9._-]*)?$`)
type PackageFailure struct {
Root string
Code string
Retryable bool
}
type PackagePreflight struct {
Plans []PackagePlan
Failures []PackageFailure
}
type packageAccumulator struct {
plan PackagePlan
seen map[string]struct{}
hasSkillMD bool
failure *PackageFailure
}
// PreflightPackages assigns every blob to its nearest Skill root. Ignored
// candidate directories are still counted when they belong to a parent Skill.
// An incomplete range never produces a package plan for an affected root.
func PreflightPackages(snapshot TreeSnapshot, roots []string, limits PackageLimits) (PackagePreflight, error) {
if limits.MaxFiles < 1 || limits.MaxSingleFileBytes < 1 || limits.MaxPackageBytes < 1 {
return PackagePreflight{}, errors.New("package limits must be positive")
}
packages := make(map[string]*packageAccumulator, len(roots))
for _, root := range roots {
if root != "." && !validPath(root) {
return PackagePreflight{}, errors.New("invalid Skill root")
}
if _, exists := packages[root]; exists {
return PackagePreflight{}, errors.New("duplicate Skill root")
}
packages[root] = &packageAccumulator{
plan: PackagePlan{Root: root}, seen: make(map[string]struct{}),
}
}
for root, accumulator := range packages {
for _, subtree := range snapshot.IncompleteSubtrees {
if subtree != "." && !validPath(subtree) {
return PackagePreflight{}, errors.New("invalid incomplete subtree")
}
if subtree == "." || root == "." || subtree == root ||
strings.HasPrefix(subtree, root+"/") || strings.HasPrefix(root, subtree+"/") {
accumulator.failure = &PackageFailure{Root: root, Code: "TREE_INCOMPLETE", Retryable: true}
break
}
}
}
entries := append([]github.TreeEntry(nil), snapshot.Entries...)
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
evidence, evidenceFailure := planRootEvidence(entries)
for _, entry := range entries {
if !validPath(entry.Path) {
return PackagePreflight{}, errors.New("invalid repository tree path")
}
root := nearestSkillRoot(entry.Path, packages)
if root == "" {
continue
}
accumulator := packages[root]
if accumulator.failure != nil || entry.Type == "tree" {
continue
}
relativePath := entry.Path
if root != "." {
relativePath = strings.TrimPrefix(entry.Path, root+"/")
}
if _, duplicate := accumulator.seen[relativePath]; duplicate {
accumulator.failure = &PackageFailure{Root: root, Code: "DUPLICATE_PATH"}
continue
}
accumulator.seen[relativePath] = struct{}{}
if relativePath == evidenceManifestPath || strings.HasPrefix(relativePath, evidenceEntryPrefix) {
accumulator.failure = &PackageFailure{Root: root, Code: "RESERVED_EVIDENCE_PATH"}
continue
}
switch {
case entry.Type == "commit" || entry.Mode == "160000":
accumulator.failure = &PackageFailure{Root: root, Code: "SUBMODULE"}
case entry.Type == "blob" && entry.Mode == "120000":
accumulator.failure = &PackageFailure{Root: root, Code: "SYMLINK"}
case entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755"):
accumulator.failure = &PackageFailure{Root: root, Code: "UNSUPPORTED_ENTRY"}
case entry.Size < 0 || !validGitSHA(entry.SHA):
accumulator.failure = &PackageFailure{Root: root, Code: "INVALID_BLOB_METADATA", Retryable: true}
case entry.Size > limits.MaxSingleFileBytes ||
entry.Size > limits.MaxPackageBytes-accumulator.plan.TotalBytes ||
len(accumulator.plan.Files) >= limits.MaxFiles:
accumulator.failure = &PackageFailure{Root: root, Code: "PACKAGE_TOO_LARGE"}
default:
accumulator.plan.Files = append(accumulator.plan.Files, PackageFile{
RepositoryPath: entry.Path, RelativePath: relativePath,
SHA: entry.SHA, Size: entry.Size,
})
accumulator.plan.TotalBytes += entry.Size
if relativePath == "SKILL.md" {
accumulator.hasSkillMD = true
}
}
}
result := PackagePreflight{}
for _, root := range sortedAccumulatorRoots(packages) {
accumulator := packages[root]
if accumulator.failure != nil {
result.Failures = append(result.Failures, *accumulator.failure)
} else if evidenceFailure != "" {
result.Failures = append(result.Failures,
PackageFailure{Root: root, Code: evidenceFailure})
} else if !accumulator.hasSkillMD {
result.Failures = append(result.Failures,
PackageFailure{Root: root, Code: "MISSING_SKILL_MD"})
} else {
accumulator.plan.Evidence = append([]RootEvidenceFile(nil), evidence...)
result.Plans = append(result.Plans, accumulator.plan)
}
}
return result, nil
}
func planRootEvidence(entries []github.TreeEntry) ([]RootEvidenceFile, string) {
result := make([]RootEvidenceFile, 0)
var bundledBytes int64
for _, entry := range entries {
if strings.Contains(entry.Path, "/") || !rootEvidenceName.MatchString(entry.Path) {
continue
}
if len(result) >= maxRootEvidenceFiles {
return nil, "TOO_MANY_ROOT_EVIDENCE_FILES"
}
if entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755") ||
entry.Size < 0 || !validGitSHA(entry.SHA) {
return nil, "UNSAFE_ROOT_EVIDENCE"
}
bundled := entry.Size <= maxEvidenceFileBytes &&
entry.Size <= maxEvidenceTotalBytes-bundledBytes
if bundled {
bundledBytes += entry.Size
}
result = append(result, RootEvidenceFile{
RepositoryPath: entry.Path, SHA: entry.SHA,
Size: entry.Size, Bundled: bundled,
})
}
return result, ""
}
func nearestSkillRoot(path string, packages map[string]*packageAccumulator) string {
for directory := parent(path); ; directory = parent(directory) {
if _, exists := packages[directory]; exists {
return directory
}
if directory == "." {
return ""
}
}
}
func sortedAccumulatorRoots(packages map[string]*packageAccumulator) []string {
roots := make([]string, 0, len(packages))
for root := range packages {
roots = append(roots, root)
}
sort.Strings(roots)
return roots
}
func validGitSHA(value string) bool {
if len(value) != 40 {
return false
}
for _, char := range value {
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
return false
}
}
return true
}

85
github-sync-go/internal/repository/package_plan_test.go Normal file
View file

@ -0,0 +1,85 @@
package repository
import (
"reflect"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestIgnoredCandidateStaysInParentPackageAndNestedSkillIsSeparate(t *testing.T) {
snapshot := TreeSnapshot{Entries: []github.TreeEntry{
{Path: "skills/parent/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/parent/test/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/parent/test/example.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4},
{Path: "skills/parent/child/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/parent/child/example.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4},
}}
roots := RootsFromSnapshot(snapshot, nil)
if !reflect.DeepEqual(roots.Paths, []string{"skills/parent", "skills/parent/child"}) {
t.Fatalf("unexpected Skill roots: %v", roots.Paths)
}
preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{
MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
})
if err != nil || len(preflight.Failures) != 0 || len(preflight.Plans) != 2 {
t.Fatalf("unexpected package preflight: %+v, error=%v", preflight, err)
}
if got := packagePaths(preflight.Plans[0]); !reflect.DeepEqual(got,
[]string{"SKILL.md", "test/SKILL.md", "test/example.txt"}) {
t.Fatalf("parent package paths = %v", got)
}
if got := packagePaths(preflight.Plans[1]); !reflect.DeepEqual(got,
[]string{"SKILL.md", "example.txt"}) {
t.Fatalf("child package paths = %v", got)
}
}
func TestDefaultAndSourceIgnoredDirectoriesDoNotBecomeSkillRoots(t *testing.T) {
ignored := []string{
".git", "node_modules", ".venv", "venv", "dist", "build",
"target", "__pycache__", "test", "tests",
}
if len(defaultIgnoredSegments) != len(ignored) {
t.Fatalf("default ignored directory count = %d", len(defaultIgnoredSegments))
}
snapshot := TreeSnapshot{Entries: []github.TreeEntry{
{Path: "skills/good/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/custom/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
}}
for _, segment := range ignored {
snapshot.Entries = append(snapshot.Entries, github.TreeEntry{
Path: "skills/" + segment + "/SKILL.md", Type: "blob",
Mode: "100644", SHA: skillBlobSHA, Size: 12,
})
}
roots := RootsFromSnapshot(snapshot, []string{"skills/custom"})
if !reflect.DeepEqual(roots.Paths, []string{"skills/good"}) {
t.Fatalf("ignored directories became Skill roots: %v", roots.Paths)
}
}
func TestUnsafeChildPackageDoesNotBlockParent(t *testing.T) {
snapshot := TreeSnapshot{Entries: []github.TreeEntry{
{Path: "skills/parent/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/parent/child/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
{Path: "skills/parent/child/link", Type: "blob", Mode: "120000", SHA: skillBlobSHA, Size: 5},
}}
roots := RootsFromSnapshot(snapshot, nil)
preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{
MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
})
if err != nil || len(preflight.Plans) != 1 || preflight.Plans[0].Root != "skills/parent" ||
len(preflight.Failures) != 1 || preflight.Failures[0].Root != "skills/parent/child" ||
preflight.Failures[0].Code != "SYMLINK" {
t.Fatalf("unsafe child isolation failed: %+v, error=%v", preflight, err)
}
}
func packagePaths(plan PackagePlan) []string {
paths := make([]string, 0, len(plan.Files))
for _, file := range plan.Files {
paths = append(paths, file.RelativePath)
}
return paths
}

20
github-sync-go/internal/repository/platform_control_test.go Normal file
View file

@ -0,0 +1,20 @@
package repository
import (
"context"
"errors"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestPlatformControlFencesGitStepsWithoutStartingSubprocess(t *testing.T) {
stopped := errors.New("platform paused")
ctx := github.WithAcquisitionControl(context.Background(), func(context.Context) error { return stopped })
ctx = WithGitRequestPolicy(ctx, func(context.Context) error { t.Fatal("quota policy ran after platform stopped"); return nil }, nil)
for _, step := range []string{"fetch", "sparse-checkout", "checkout"} {
if err := runBoundedGit(ctx, "does-not-exist", "does-not-exist", 1, 0, step); !errors.Is(err, stopped) {
t.Fatalf("%s: %v", step, err)
}
}
}

80
github-sync-go/internal/repository/root_evidence_boundaries_test.go Normal file
View file

@ -0,0 +1,80 @@
package repository
import (
"fmt"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestRootEvidenceBoundariesPreserveDigestWithoutUnboundedBundling(t *testing.T) {
for _, tc := range []struct {
name string
sizes []int64
wantBundled int
wantFailure string
}{
{"single-at-64KiB", []int64{64 << 10}, 1, ""},
{"single-over-64KiB", []int64{(64 << 10) + 1}, 0, ""},
{"total-at-256KiB", []int64{64 << 10, 64 << 10, 64 << 10, 64 << 10}, 4, ""},
{"total-over-256KiB", []int64{64 << 10, 64 << 10, 64 << 10, 64 << 10, 1}, 4, ""},
{"16-evidence-files", make([]int64, 16), 16, ""},
{"17-evidence-files", make([]int64, 17), 0, "TOO_MANY_ROOT_EVIDENCE_FILES"},
} {
t.Run(tc.name, func(t *testing.T) {
entries := []github.TreeEntry{{Path: "skills/demo/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}}
for i, size := range tc.sizes {
entries = append(entries, github.TreeEntry{Path: fmt.Sprintf("LICENSE-%02d", i), Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: size})
}
// Unrelated root files are not license evidence and do not consume its budget.
entries = append(entries, github.TreeEntry{Path: "README.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1})
result, err := PreflightPackages(TreeSnapshot{Entries: entries}, []string{"skills/demo"}, PackageLimits{MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20})
if err != nil {
t.Fatal(err)
}
if tc.wantFailure != "" {
if len(result.Plans) != 0 || len(result.Failures) != 1 || result.Failures[0].Code != tc.wantFailure {
t.Fatalf("evidence limit must block plan: %+v", result)
}
return
}
if len(result.Plans) != 1 || len(result.Failures) != 0 {
t.Fatalf("unexpected plan: %+v", result)
}
plan := result.Plans[0]
if len(plan.Evidence) != len(tc.sizes) || len(plan.Files) != 1 || plan.TotalBytes != 1 {
t.Fatalf("evidence metadata lost or conflated with Skill files: %+v", plan)
}
bundled := 0
var bundledBytes int64
for i, evidence := range plan.Evidence {
if evidence.RepositoryPath != fmt.Sprintf("LICENSE-%02d", i) || evidence.SHA != skillBlobSHA || evidence.Size != tc.sizes[i] {
t.Fatalf("unbundled evidence must retain path, digest and size: %+v", evidence)
}
if evidence.Bundled {
bundled++
bundledBytes += evidence.Size
}
}
if bundled != tc.wantBundled || bundledBytes > 256<<10 {
t.Fatalf("bundled=%d bytes=%d want=%d", bundled, bundledBytes, tc.wantBundled)
}
})
}
}
func TestRootLicenseAndNoticeAreBothRetained(t *testing.T) {
entries := []github.TreeEntry{
{Path: "LICENSE", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4},
{Path: "NOTICE.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 5},
}
evidence, failure := planRootEvidence(entries)
if failure != "" || len(evidence) != 2 {
t.Fatalf("license and notice not retained: %v %s", evidence, failure)
}
for i, file := range evidence {
if file.RepositoryPath != entries[i].Path || file.SHA != skillBlobSHA || !file.Bundled || file.Size != entries[i].Size {
t.Fatalf("invalid evidence: %+v", file)
}
}
}

208
github-sync-go/internal/repository/tree.go Normal file
View file

@ -0,0 +1,208 @@
package repository
import (
"context"
"errors"
"sort"
"strings"
"unicode"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
var defaultIgnoredSegments = map[string]struct{}{
".git": {}, "node_modules": {}, ".venv": {}, "venv": {}, "dist": {},
"build": {}, "target": {}, "__pycache__": {}, "test": {}, "tests": {},
}
type TreeReader interface {
GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error)
}
type TreeLimits struct {
MaxTreeRequests int
MaxEntries int
}
type SkillRoots struct {
Paths []string
IncompleteSubtrees []string
EntriesExamined int
}
type TreeSnapshot struct {
Entries []github.TreeEntry
IncompleteSubtrees []string
EntriesExamined int
}
type pendingTree struct {
SHA string
Prefix string
}
// EnumerateTree retains file metadata for package preflight. Truncated recursive
// trees are rewalked one subtree at a time; failed siblings are reported explicitly.
func EnumerateTree(ctx context.Context, reader TreeReader, owner, name, rootTreeSHA string,
limits TreeLimits) (TreeSnapshot, error) {
if limits.MaxTreeRequests < 1 || limits.MaxEntries < 1 {
return TreeSnapshot{}, errors.New("tree request and entry limits must be positive")
}
root, _, err := reader.GetTree(ctx, owner, name, rootTreeSHA, true)
if err != nil && !errors.Is(err, github.ErrResponseTooLarge) {
return TreeSnapshot{}, err
}
if err == nil && !root.Truncated && len(root.Entries) <= limits.MaxEntries {
result := TreeSnapshot{EntriesExamined: len(root.Entries)}
for _, entry := range root.Entries {
if !validPath(entry.Path) {
result.IncompleteSubtrees = []string{"."}
continue
}
result.Entries = append(result.Entries, entry)
}
return result, nil
}
result := TreeSnapshot{}
incomplete := make(map[string]struct{})
queue := []pendingTree{{SHA: rootTreeSHA, Prefix: "."}}
requests := 1
for len(queue) > 0 {
if err := ctx.Err(); err != nil {
for _, left := range queue {
incomplete[left.Prefix] = struct{}{}
}
result.IncompleteSubtrees = sortedKeys(incomplete)
return result, err
}
current := queue[0]
queue = queue[1:]
if requests >= limits.MaxTreeRequests || result.EntriesExamined >= limits.MaxEntries {
incomplete[current.Prefix] = struct{}{}
for _, left := range queue {
incomplete[left.Prefix] = struct{}{}
}
break
}
tree, _, treeErr := reader.GetTree(ctx, owner, name, current.SHA, false)
requests++
if apiError, ok := treeErr.(*github.APIError); ok &&
(apiError.Code == "GITHUB_RATE_LIMITED" || apiError.Code == "AUTH_INVALID") {
incomplete[current.Prefix] = struct{}{}
for _, left := range queue {
incomplete[left.Prefix] = struct{}{}
}
result.IncompleteSubtrees = sortedKeys(incomplete)
return result, treeErr
}
if treeErr != nil || tree.Truncated {
incomplete[current.Prefix] = struct{}{}
continue
}
if len(tree.Entries) > limits.MaxEntries-result.EntriesExamined {
incomplete[current.Prefix] = struct{}{}
continue
}
for _, entry := range tree.Entries {
path := entry.Path
if current.Prefix != "." {
path = current.Prefix + "/" + path
}
result.EntriesExamined++
if !validPath(path) {
incomplete[current.Prefix] = struct{}{}
continue
}
entry.Path = path
result.Entries = append(result.Entries, entry)
if entry.Type == "tree" {
queue = append(queue, pendingTree{SHA: entry.SHA, Prefix: path})
}
}
}
result.IncompleteSubtrees = sortedKeys(incomplete)
sort.Slice(result.Entries, func(i, j int) bool {
return result.Entries[i].Path < result.Entries[j].Path
})
return result, nil
}
// DiscoverSkillRoots treats incomplete ranges as unknown. A candidate inside an
// ignored directory is skipped, but that directory's files remain available to
// a parent Skill package.
func DiscoverSkillRoots(ctx context.Context, reader TreeReader, owner, name, rootTreeSHA string,
limits TreeLimits, addedIgnoredPrefixes []string) (SkillRoots, error) {
for _, prefix := range addedIgnoredPrefixes {
if !validPath(prefix) || prefix == "." {
return SkillRoots{}, errors.New("invalid additional ignored path prefix")
}
}
snapshot, err := EnumerateTree(ctx, reader, owner, name, rootTreeSHA, limits)
return RootsFromSnapshot(snapshot, addedIgnoredPrefixes), err
}
func RootsFromSnapshot(snapshot TreeSnapshot, addedIgnoredPrefixes []string) SkillRoots {
result := SkillRoots{
IncompleteSubtrees: snapshot.IncompleteSubtrees,
EntriesExamined: snapshot.EntriesExamined,
}
roots := make(map[string]struct{})
for _, entry := range snapshot.Entries {
if candidateRoot(entry.Path, entry, addedIgnoredPrefixes) {
roots[parent(entry.Path)] = struct{}{}
}
}
result.Paths = sortedKeys(roots)
return result
}
func candidateRoot(path string, entry github.TreeEntry, addedIgnoredPrefixes []string) bool {
return entry.Type == "blob" && (path == "SKILL.md" || strings.HasSuffix(path, "/SKILL.md")) &&
!ignoredDirectory(parent(path), addedIgnoredPrefixes)
}
func ignoredDirectory(path string, addedIgnoredPrefixes []string) bool {
if path == "." {
return false
}
for _, segment := range strings.Split(path, "/") {
if _, ignored := defaultIgnoredSegments[segment]; ignored {
return true
}
}
for _, prefix := range addedIgnoredPrefixes {
if path == prefix || strings.HasPrefix(path, prefix+"/") {
return true
}
}
return false
}
func validPath(path string) bool {
if path == "" || path == "." || strings.HasPrefix(path, "/") || strings.HasSuffix(path, "/") ||
strings.Contains(path, "\\") || strings.IndexFunc(path, unicode.IsControl) >= 0 {
return false
}
for _, segment := range strings.Split(path, "/") {
if segment == "" || segment == "." || segment == ".." {
return false
}
}
return true
}
func parent(path string) string {
if cut := strings.LastIndexByte(path, '/'); cut >= 0 {
return path[:cut]
}
return "."
}
func sortedKeys(values map[string]struct{}) []string {
result := make([]string, 0, len(values))
for value := range values {
result = append(result, value)
}
sort.Strings(result)
return result
}

261
github-sync-go/internal/repository/tree_live_test.go Normal file
View file

@ -0,0 +1,261 @@
package repository
import (
"archive/zip"
"context"
"crypto/sha256"
"encoding/hex"
"io"
"os"
"os/exec"
"slices"
"sort"
"strings"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type countedLiveTreeReader struct {
client *github.Client
calls int
}
const fixedPublicCommitSHA = "49f948faa9258a0c61caceaf225e179651397431"
func loadFixedPublicTree(t *testing.T, ctx context.Context) (TreeSnapshot, int) {
t.Helper()
anonymous := os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_ANONYMOUS") == "1"
token := ""
if !anonymous {
token = os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TOKEN")
if token == "" {
output, err := exec.CommandContext(ctx, "gh", "auth", "token").Output()
if err != nil {
t.Fatal("no configured GitHub test token or usable local gh authentication")
}
token = strings.TrimSpace(string(output))
}
if token == "" {
t.Fatal("GitHub test token is empty")
}
}
client := github.NewClient(token)
metadata, _, err := client.GetRepository(ctx, "openai", "skills")
if err != nil {
t.Fatal(err)
}
if metadata.ID != 1103643573 || metadata.Private || metadata.Owner.Login != "openai" {
t.Fatal("public repository metadata differs from the fixed fixture")
}
treeSHA, _, err := client.GetCommitTreeSHA(ctx, metadata.Owner.Login,
metadata.Name, fixedPublicCommitSHA)
if err != nil {
t.Fatal(err)
}
if treeSHA != "224b363928698e143876aef008232ed73d8e6f81" {
t.Fatal("fixed commit resolved to an unexpected tree")
}
reader := &countedLiveTreeReader{client: client}
snapshot, err := EnumerateTree(ctx, reader, metadata.Owner.Login, metadata.Name,
treeSHA, TreeLimits{MaxTreeRequests: 10, MaxEntries: 5000})
if err != nil {
t.Fatal(err)
}
return snapshot, 2 + reader.calls
}
func (reader *countedLiveTreeReader) GetTree(ctx context.Context, owner, name, sha string,
recursive bool) (github.Tree, github.Quota, error) {
reader.calls++
return reader.client.GetTree(ctx, owner, name, sha, recursive)
}
// This opt-in test checks the real GitHub REST metadata path for a fixed public
// commit. SKILLHUB_GITHUB_SYNC_LIVE_ANONYMOUS=1 avoids local gh credentials;
// otherwise any credential stays in process memory and is never included in output.
func TestLiveEnumerateFixedPublicGitHubTree(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TREE") != "1" {
t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_TREE=1 to query the public GitHub Tree API")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
snapshot, apiOperations := loadFixedPublicTree(t, ctx)
roots := RootsFromSnapshot(snapshot, nil)
for _, expected := range []string{
"skills/.system/skill-creator", "skills/.curated/define-goal",
} {
if !slices.Contains(roots.Paths, expected) {
t.Fatalf("real GitHub tree omitted expected Skill root %q", expected)
}
}
if len(snapshot.IncompleteSubtrees) != 0 || len(snapshot.Entries) < 1000 ||
apiOperations != 3 {
t.Fatalf("real tree was incomplete: entries=%d incomplete=%d treeCalls=%d",
len(snapshot.Entries), len(snapshot.IncompleteSubtrees), apiOperations-2)
}
if len(roots.Paths) != 44 {
t.Fatalf("fixed public tree changed Skill root count: got=%d want=44", len(roots.Paths))
}
// Match the effective publish policy in Java's application.yml. This reads only
// Tree metadata; it never downloads the 44 candidate packages.
policy := PackageLimits{
MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
}
preflight, err := PreflightPackages(snapshot, roots.Paths, policy)
if err != nil {
t.Fatal(err)
}
if len(preflight.Plans)+len(preflight.Failures) != len(roots.Paths) {
t.Fatalf("preflight lost Skill roots: roots=%d plans=%d failures=%d",
len(roots.Paths), len(preflight.Plans), len(preflight.Failures))
}
const rejectedRoot = "skills/.curated/cloudflare-deploy"
if len(preflight.Plans) != 43 || len(preflight.Failures) != 1 ||
preflight.Failures[0].Root != rejectedRoot ||
preflight.Failures[0].Code != "PACKAGE_TOO_LARGE" {
t.Fatalf("unexpected fixed-tree package policy result: plans=%d failures=%+v",
len(preflight.Plans), preflight.Failures)
}
rejectedFiles := 0
for _, entry := range snapshot.Entries {
if strings.HasPrefix(entry.Path, rejectedRoot+"/") && entry.Type == "blob" {
rejectedFiles++
}
}
if rejectedFiles <= policy.MaxFiles {
t.Fatalf("rejected root does not exceed file-count policy: files=%d limit=%d",
rejectedFiles, policy.MaxFiles)
}
failureCounts := make(map[string]int)
for _, failure := range preflight.Failures {
failureCounts[failure.Code]++
t.Logf("package preflight rejected root: path=%s code=%s files=%d limit=%d",
failure.Root, failure.Code, rejectedFiles, policy.MaxFiles)
}
failureCodes := make([]string, 0, len(failureCounts))
for code := range failureCounts {
failureCodes = append(failureCodes, code)
}
sort.Strings(failureCodes)
var totalBytes, largestPackageBytes int64
maxPackageFiles := 0
for _, plan := range preflight.Plans {
totalBytes += plan.TotalBytes
largestPackageBytes = max(largestPackageBytes, plan.TotalBytes)
maxPackageFiles = max(maxPackageFiles, len(plan.Files))
}
t.Logf("fixed public tree: api_operations=%d entries=%d skill_roots=%d incomplete=%d",
apiOperations, len(snapshot.Entries), len(roots.Paths), len(snapshot.IncompleteSubtrees))
t.Logf("package preflight: eligible=%d rejected=%d declared_bytes=%d largest_package_bytes=%d max_package_files=%d",
len(preflight.Plans), len(preflight.Failures), totalBytes, largestPackageBytes, maxPackageFiles)
for _, code := range failureCodes {
t.Logf("package preflight rejection: code=%s count=%d", code, failureCounts[code])
}
}
// Opt in separately from the Tree-only test: this makes one bounded Git
// acquisition per eligible Skill and closes each workspace before the next.
func TestLiveAcquireAllFixedPublicSkillsSequentially(t *testing.T) {
if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES") != "1" {
t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES=1 for full public repository acquisition")
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
started := time.Now()
snapshot, apiOperations := loadFixedPublicTree(t, ctx)
if apiOperations != 3 || len(snapshot.IncompleteSubtrees) != 0 {
t.Fatalf("fixed public Tree is incomplete: operations=%d incomplete=%d",
apiOperations, len(snapshot.IncompleteSubtrees))
}
roots := RootsFromSnapshot(snapshot, nil)
preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{
MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
})
if err != nil {
t.Fatal(err)
}
if len(preflight.Plans) != 43 || len(preflight.Failures) != 1 ||
preflight.Failures[0].Code != "PACKAGE_TOO_LARGE" {
t.Fatalf("unexpected package preflight: plans=%d failures=%+v",
len(preflight.Plans), preflight.Failures)
}
tempRoot := t.TempDir()
limits := AcquisitionLimits{
MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 16 << 20,
}
var totalArchiveBytes, largestWorkspaceAfterAcquisition int64
for index, plan := range preflight.Plans {
func() {
artifact, err := AcquirePackage(ctx, "openai", "skills", fixedPublicCommitSHA,
tempRoot, plan, limits)
if err != nil {
t.Fatalf("acquire package %d/%d %s: %v",
index+1, len(preflight.Plans), plan.Root, err)
}
defer func() {
if err := artifact.Close(); err != nil {
t.Fatalf("clean package %s: %v", plan.Root, err)
}
}()
archive, err := zip.OpenReader(artifact.Path)
if err != nil {
t.Fatalf("open package %s: %v", plan.Root, err)
}
defer archive.Close()
want := map[string]struct{}{evidenceManifestPath: {}}
for _, file := range plan.Files {
want[file.RelativePath] = struct{}{}
}
for _, evidence := range plan.Evidence {
if evidence.Bundled {
want[evidenceEntryPrefix+evidence.RepositoryPath] = struct{}{}
}
}
if len(archive.File) != len(want) {
t.Fatalf("package %s has %d ZIP entries, expected %d",
plan.Root, len(archive.File), len(want))
}
for _, file := range archive.File {
if _, expected := want[file.Name]; !expected {
t.Fatalf("package %s contains unexpected ZIP path %s", plan.Root, file.Name)
}
delete(want, file.Name)
}
if len(want) != 0 {
t.Fatalf("package %s omitted %d planned ZIP entries", plan.Root, len(want))
}
stored, err := os.Open(artifact.Path)
if err != nil {
t.Fatalf("read package %s: %v", plan.Root, err)
}
digest := sha256.New()
storedBytes, copyErr := io.Copy(digest, stored)
closeErr := stored.Close()
if copyErr != nil || closeErr != nil || storedBytes != artifact.Size ||
hex.EncodeToString(digest.Sum(nil)) != artifact.SHA256 {
t.Fatalf("package %s archive size or digest mismatch: copy=%v close=%v",
plan.Root, copyErr, closeErr)
}
workspaceBytes, err := directoryBytes(tempRoot)
if err != nil || workspaceBytes > limits.MaxWorkspaceBytes {
t.Fatalf("package %s exceeded workspace budget: bytes=%d error=%v",
plan.Root, workspaceBytes, err)
}
largestWorkspaceAfterAcquisition = max(largestWorkspaceAfterAcquisition, workspaceBytes)
totalArchiveBytes += artifact.Size
}()
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("package %d/%d left a workspace: entries=%d error=%v",
index+1, len(preflight.Plans), len(remaining), err)
}
if (index+1)%10 == 0 {
t.Logf("sequential packages completed=%d/%d", index+1, len(preflight.Plans))
}
}
t.Logf("fixed public repository acquisition: api_operations=%d eligible=%d rejected=%d git_acquisitions=%d total_archive_bytes=%d largest_workspace_after_acquisition=%d elapsed=%s",
apiOperations, len(preflight.Plans), len(preflight.Failures), len(preflight.Plans),
totalArchiveBytes, largestWorkspaceAfterAcquisition, time.Since(started))
}

95
github-sync-go/internal/repository/tree_test.go Normal file
View file

@ -0,0 +1,95 @@
package repository
import (
"context"
"errors"
"reflect"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
const (
rootTreeSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
skillsTreeSHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
goodTreeSHA = "cccccccccccccccccccccccccccccccccccccccc"
badTreeSHA = "dddddddddddddddddddddddddddddddddddddddd"
skillBlobSHA = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
)
type fixtureTreeReader struct {
trees map[string]github.Tree
errors map[string]error
calls []string
}
func (reader *fixtureTreeReader) GetTree(_ context.Context, _, _, sha string,
recursive bool) (github.Tree, github.Quota, error) {
key := sha
if recursive {
key += ":recursive"
}
reader.calls = append(reader.calls, key)
if err := reader.errors[key]; err != nil {
return github.Tree{}, github.Quota{}, err
}
return reader.trees[key], github.Quota{}, nil
}
func TestEnumerateTreeContinuesCompleteSiblingAfterTruncation(t *testing.T) {
reader := &fixtureTreeReader{
trees: map[string]github.Tree{
rootTreeSHA + ":recursive": {SHA: rootTreeSHA, Truncated: true},
rootTreeSHA: {SHA: rootTreeSHA, Entries: []github.TreeEntry{
{Path: "skills", Type: "tree", SHA: skillsTreeSHA},
}},
skillsTreeSHA: {SHA: skillsTreeSHA, Entries: []github.TreeEntry{
{Path: "good", Type: "tree", SHA: goodTreeSHA},
{Path: "bad", Type: "tree", SHA: badTreeSHA},
}},
goodTreeSHA: {SHA: goodTreeSHA, Entries: []github.TreeEntry{
{Path: "SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
}},
},
errors: map[string]error{badTreeSHA: errors.New("temporary tree failure")},
}
snapshot, err := EnumerateTree(context.Background(), reader, "owner", "repo", rootTreeSHA,
TreeLimits{MaxTreeRequests: 8, MaxEntries: 20})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(snapshot.IncompleteSubtrees, []string{"skills/bad"}) ||
!reflect.DeepEqual(reader.calls, []string{
rootTreeSHA + ":recursive", rootTreeSHA, skillsTreeSHA, goodTreeSHA, badTreeSHA,
}) {
t.Fatalf("incorrect partial tree coverage: %+v, calls=%v", snapshot, reader.calls)
}
roots := RootsFromSnapshot(snapshot, nil)
if !reflect.DeepEqual(roots.Paths, []string{"skills/good"}) {
t.Fatalf("complete sibling root was lost: %+v", roots)
}
preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{
MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20,
})
if err != nil || len(preflight.Plans) != 1 || preflight.Plans[0].Root != "skills/good" ||
len(preflight.Failures) != 0 {
t.Fatalf("complete sibling was blocked: %+v, error=%v", preflight, err)
}
}
func TestEnumerateTreeFallsBackWhenRecursiveResponseIsTooLarge(t *testing.T) {
reader := &fixtureTreeReader{
trees: map[string]github.Tree{rootTreeSHA: {
SHA: rootTreeSHA, Entries: []github.TreeEntry{
{Path: "SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12},
},
}},
errors: map[string]error{rootTreeSHA + ":recursive": github.ErrResponseTooLarge},
}
snapshot, err := EnumerateTree(context.Background(), reader, "owner", "repo", rootTreeSHA,
TreeLimits{MaxTreeRequests: 2, MaxEntries: 2})
if err != nil || len(snapshot.IncompleteSubtrees) != 0 ||
!reflect.DeepEqual(RootsFromSnapshot(snapshot, nil).Paths, []string{"."}) {
t.Fatalf("oversized recursive tree did not fall back: %+v, error=%v", snapshot, err)
}
}

401
github-sync-go/internal/worker/discovery.go Normal file
View file

@ -0,0 +1,401 @@
package worker
import (
"context"
"encoding/json"
"errors"
"sort"
"strings"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type DiscoveryLimits struct {
MaxPagesPerLease int
}
type discoveryGitHub interface {
SetQuotaObserver(func(context.Context, github.Quota))
GetRepository(context.Context, string, string) (github.Repository, github.Quota, error)
GetOwnerType(context.Context, string) (github.OwnerType, github.Quota, error)
ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error)
SearchRepositories(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error)
}
type discoveryCursor struct {
Version int `json:"v"`
Entry string `json:"entry"`
Selector int `json:"selector,omitempty"`
Page int `json:"page"`
Offset int `json:"offset,omitempty"`
Path string `json:"path,omitempty"`
Revisit int `json:"revisit,omitempty"`
}
// ProcessDiscovery persists each cursor only after its candidate batch is
// accepted. The cursor contains no token or process-local business state.
func ProcessDiscovery(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits DiscoveryLimits) error {
if task == nil {
return errors.New("invalid discovery task or worker limits")
}
if task.OfficialDiscovery != nil {
return processOfficialDiscovery(ctx, control, task, limits, github.NewClient(credentialToken(task)))
}
return processDiscoveryWithGitHub(ctx, control, task, limits, github.NewClient(credentialToken(task)))
}
func processDiscoveryWithGitHub(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits DiscoveryLimits, upstream discoveryGitHub) error {
if task == nil || task.Kind != "DISCOVERY" || task.Discovery == nil || task.OfficialDiscovery != nil ||
limits.MaxPagesPerLease < 1 || task.Discovery.CandidateBudget < 1 || upstream == nil {
return errors.New("invalid discovery task or worker limits")
}
snapshot := task.Discovery
if snapshot.EntryType == "SEARCH_PREVIEW" {
return processSearchPreview(ctx, control, task)
}
upstream.SetQuotaObserver(observeCredentialQuota(control, task))
checkpoint := snapshot.Checkpoint
var cursor discoveryCursor
if checkpoint == "" {
cursor = discoveryCursor{Version: 1, Entry: snapshot.EntryType, Page: 1}
encoded, err := encodeCursor(cursor)
if err != nil {
return err
}
if err := control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease,
controlplane.DiscoveryBatch{Checkpoint: encoded, Repositories: []controlplane.RepositoryCandidate{}}); err != nil {
return err
}
checkpoint = encoded
} else if err := json.Unmarshal([]byte(checkpoint), &cursor); err != nil {
return errors.New("invalid persisted discovery cursor")
}
if cursor.Version != 1 || cursor.Entry != snapshot.EntryType || cursor.Page < 1 ||
cursor.Offset < 0 || cursor.Selector < 0 || cursor.Revisit < 0 || len(cursor.Path) > 64 ||
strings.Trim(cursor.Path, "LR") != "" {
return errors.New("persisted discovery cursor does not match task")
}
if snapshot.EntryType == "CURATED_REPOSITORY" {
if snapshot.Owner == "" || snapshot.RepositoryName == "" {
return errors.New("curated repository task has no coordinate")
}
if cursor.Page == 1 {
repository, _, err := upstream.GetRepository(ctx, snapshot.Owner, snapshot.RepositoryName)
if err != nil {
return completeDiscoveryError(ctx, control, task, checkpoint, err)
}
next := cursor
next.Page = 2
checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next,
toCandidates([]github.Repository{repository}), 0)
if err != nil {
return err
}
} else if cursor.Page != 2 {
return errors.New("invalid curated repository cursor")
}
_, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "")
return err
}
spent := 0
pages := 0
seen := make(map[int64]struct{})
var ownerType github.OwnerType
if snapshot.EntryType == "CURATED_OWNER" {
var err error
ownerType, _, err = upstream.GetOwnerType(ctx, snapshot.Owner)
if err != nil {
return completeDiscoveryError(ctx, control, task, checkpoint, err)
}
} else if snapshot.EntryType != "SEARCH_RULE" || snapshot.Criteria == nil ||
snapshot.SearchWindow == nil || len(snapshot.Criteria.Selectors) == 0 {
return errors.New("discovery task has no valid entry configuration")
}
var selectors []controlplane.SearchSelector
if snapshot.EntryType == "SEARCH_RULE" {
selectors = selectorsFor(snapshot)
}
for {
if ctx.Err() != nil {
return ctx.Err()
}
if snapshot.EntryType == "SEARCH_RULE" && cursor.Selector >= len(selectors) {
_, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "")
return err
}
if spent >= snapshot.CandidateBudget || pages >= limits.MaxPagesPerLease {
return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(30*time.Minute))
}
var candidates []controlplane.RepositoryCandidate
var hasMore bool
var next discoveryCursor
if snapshot.EntryType == "CURATED_OWNER" {
page, _, err := upstream.ListPublicOwnerRepositories(ctx, snapshot.Owner, ownerType, cursor.Page)
if err != nil {
return completeDiscoveryError(ctx, control, task, checkpoint, err)
}
pages++
candidates = toCandidates(page.Repositories)
hasMore = page.HasMore
next = cursor
} else {
from, until, err := searchBounds(snapshot.SearchWindow, cursor.Path)
if err != nil {
return err
}
selector := selectors[cursor.Selector]
page, _, err := upstream.SearchRepositories(ctx, selector.Kind, selector.Value,
&from, until, cursor.Page)
if err != nil {
return completeDiscoveryError(ctx, control, task, checkpoint, err)
}
pages++
if page.LimitReached || page.IncompleteResults {
if !from.Before(until) {
return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(time.Hour))
}
next = cursor
next.Path += "L"
next.Page, next.Offset = 1, 0
var err error
checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next, nil, 0)
if err != nil {
return err
}
cursor = next
continue
}
candidates = toCandidates(page.Repositories)
hasMore = page.HasMore
next = cursor
}
if cursor.Offset > len(candidates) {
// GitHub search pages can shrink between leases. Revisit the current
// page and let Java deduplicate previously reported repository IDs.
// Bump the cursor generation so replay cannot reuse an older checkpoint.
if cursor.Revisit == int(^uint(0)>>1) {
return errors.New("discovery page revisit limit reached")
}
cursor.Revisit++
cursor.Offset = 0
next = cursor
}
remaining := snapshot.CandidateBudget - spent
batch, end := selectDiscoveryBatch(candidates, cursor.Offset, remaining, seen)
deferred := len(candidates) - end
if end < len(candidates) {
next.Offset = end
} else if hasMore {
next.Page++
next.Offset = 0
} else if snapshot.EntryType == "CURATED_OWNER" {
next.Page++
next.Offset = 0
} else {
next = nextSearchCursor(cursor)
}
var err error
checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next, batch, deferred)
if err != nil {
var apiError *controlplane.APIError
if errors.As(err, &apiError) && apiError.Code == "CANDIDATE_BUDGET_EXCEEDED" {
return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(30*time.Minute))
}
return err
}
for _, candidate := range batch {
seen[candidate.RepositoryID] = struct{}{}
}
spent += len(batch)
cursor = next
if snapshot.EntryType == "CURATED_OWNER" && !hasMore && deferred == 0 {
_, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "")
return err
}
}
}
// The offset tracks raw page positions so a resumed lease does not depend on
// the in-memory seen set. Java also deduplicates repository IDs across leases.
func selectDiscoveryBatch(candidates []controlplane.RepositoryCandidate, offset, budget int,
seen map[int64]struct{}) ([]controlplane.RepositoryCandidate, int) {
batch := make([]controlplane.RepositoryCandidate, 0, min(budget, len(candidates)-offset))
selected := make(map[int64]struct{})
end := offset
for end < len(candidates) && len(batch) < budget {
candidate := candidates[end]
end++
if _, exists := seen[candidate.RepositoryID]; exists {
continue
}
if _, exists := selected[candidate.RepositoryID]; exists {
continue
}
selected[candidate.RepositoryID] = struct{}{}
batch = append(batch, candidate)
}
return batch, end
}
// A preview only reads one search page per selector. Counts are estimates from
// repository metadata and can overlap; no discovery candidates are reported.
func processSearchPreview(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask) error {
upstream := github.NewClient(credentialToken(task))
upstream.SetQuotaObserver(observeCredentialQuota(control, task))
return processSearchPreviewWithSearch(ctx, control, task, upstream.SearchRepositories)
}
func processSearchPreviewWithSearch(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask,
search func(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error)) error {
results, err := previewSearchResults(ctx, task.Discovery, search)
if err != nil {
return err
}
return control.CompletePreview(ctx, task.TaskID, task.Lease, results)
}
func previewSearchResults(ctx context.Context, snapshot *controlplane.DiscoverySnapshot,
search func(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error)) ([]controlplane.PreviewSelectorResult, error) {
if snapshot == nil || snapshot.Criteria == nil || snapshot.SearchWindow == nil ||
len(snapshot.Criteria.Selectors) < 1 || len(snapshot.Criteria.Selectors) > 20 ||
snapshot.SearchWindow.Until.IsZero() || snapshot.Checkpoint != "" {
return nil, errors.New("invalid search preview snapshot")
}
results := make([]controlplane.PreviewSelectorResult, 0, len(snapshot.Criteria.Selectors))
for _, selector := range selectorsFor(snapshot) {
page, _, err := search(ctx, selector.Kind, selector.Value,
nil, snapshot.SearchWindow.Until, 1)
if err != nil {
return nil, err
}
samples := toCandidates(page.Repositories)
if len(samples) > 5 {
samples = samples[:5]
}
results = append(results, controlplane.PreviewSelectorResult{
Kind: selector.Kind, Value: selector.Value, Priority: selector.Priority,
TotalCount: page.TotalCount, IncompleteResults: page.IncompleteResults,
LimitReached: page.LimitReached, Samples: samples,
})
}
return results, nil
}
func selectorsFor(snapshot *controlplane.DiscoverySnapshot) []controlplane.SearchSelector {
selectors := append([]controlplane.SearchSelector(nil), snapshot.Criteria.Selectors...)
sort.SliceStable(selectors, func(i, j int) bool {
return selectors[i].Priority == "CORE" && selectors[j].Priority != "CORE"
})
return selectors
}
func toCandidates(repositories []github.Repository) []controlplane.RepositoryCandidate {
result := make([]controlplane.RepositoryCandidate, 0, len(repositories))
for _, candidate := range repositories {
result = append(result, controlplane.RepositoryCandidate{
RepositoryID: candidate.ID, Owner: candidate.Owner.Login,
Name: candidate.Name, DefaultBranch: candidate.DefaultBranch,
})
}
return result
}
func reportDiscoveryProgress(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, previous string, next discoveryCursor,
repositories []controlplane.RepositoryCandidate, deferred int) (string, error) {
encoded, err := encodeCursor(next)
if err != nil {
return "", err
}
if encoded == previous {
return "", errors.New("discovery cursor did not advance")
}
reason := ""
if deferred > 0 {
reason = "BUDGET"
}
err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, controlplane.DiscoveryBatch{
PreviousCheckpoint: &previous, Checkpoint: encoded, Repositories: repositories,
DeferredCount: deferred, DeferralReason: reason,
})
if err != nil {
return "", err
}
return encoded, nil
}
func encodeCursor(cursor discoveryCursor) (string, error) {
encoded, err := json.Marshal(cursor)
if err != nil || len(encoded) > 4096 {
return "", errors.New("discovery cursor exceeds protocol limit")
}
return string(encoded), nil
}
func searchBounds(window *controlplane.SearchWindow, path string) (time.Time, time.Time, error) {
if window.Until.IsZero() || window.Mode != "FULL_RECHECK" && window.Mode != "INCREMENTAL" ||
window.Mode == "INCREMENTAL" && window.Since.IsZero() {
return time.Time{}, time.Time{}, errors.New("invalid fixed search window")
}
lower := int64(0)
if window.Mode == "INCREMENTAL" {
lower = window.Since.Unix()
}
upper := window.Until.Unix()
if lower > upper {
return time.Time{}, time.Time{}, errors.New("invalid fixed search bounds")
}
for _, step := range path {
if lower >= upper {
return time.Time{}, time.Time{}, errors.New("discovery split cursor exceeds search window")
}
middle := lower + (upper-lower)/2
if step == 'L' {
upper = middle
} else if step == 'R' {
lower = middle + 1
} else {
return time.Time{}, time.Time{}, errors.New("invalid discovery split cursor")
}
}
return time.Unix(lower, 0).UTC(), time.Unix(upper, 0).UTC(), nil
}
func nextSearchCursor(current discoveryCursor) discoveryCursor {
next := current
next.Page, next.Offset = 1, 0
for len(next.Path) > 0 {
last := next.Path[len(next.Path)-1]
next.Path = next.Path[:len(next.Path)-1]
if last == 'L' {
next.Path += "R"
return next
}
}
next.Selector++
return next
}
func deferDiscovery(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, checkpoint, reason string, retryAt time.Time) error {
_, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, futureRetry(retryAt), reason)
return err
}
func completeDiscoveryError(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, checkpoint string, err error) error {
var upstream *github.APIError
if errors.As(err, &upstream) && upstream.Code == "GITHUB_RATE_LIMITED" {
if reportErr := reportCredentialProblem(ctx, control, task, err); reportErr != nil {
return reportErr
}
return deferDiscovery(ctx, control, task, checkpoint, "RATE_LIMIT", upstream.RetryAt)
}
return err
}

73
github-sync-go/internal/worker/discovery_curated_interop_test.go Normal file
View file

@ -0,0 +1,73 @@
package worker
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type curatedRepositoryFixture struct{}
func (curatedRepositoryFixture) SetQuotaObserver(func(context.Context, github.Quota)) {}
func (curatedRepositoryFixture) GetRepository(_ context.Context, owner, name string) (github.Repository, github.Quota, error) {
if owner != "openai" || name != "skills" {
return github.Repository{}, github.Quota{}, errors.New("unexpected curated repository coordinate")
}
repository := github.Repository{ID: 987654321, Name: name, DefaultBranch: "main"}
repository.Owner.Login = owner
return repository, github.Quota{}, nil
}
func (curatedRepositoryFixture) GetOwnerType(context.Context, string) (github.OwnerType, github.Quota, error) {
return "", github.Quota{}, errors.New("unexpected owner lookup")
}
func (curatedRepositoryFixture) ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error) {
return github.OwnerPage{}, github.Quota{}, errors.New("unexpected owner repository listing")
}
func (curatedRepositoryFixture) SearchRepositories(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error) {
return github.SearchPage{}, github.Quota{}, errors.New("unexpected repository search")
}
// Opt-in cross-process check of Java's curated source binding contract.
// The optional live mode also probes GitHub but depends on its public quota.
func TestInteropCuratedPublicRepository(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_CURATED_REPOSITORY") != "1" {
t.Skip("requires a Java HTTP curated repository fixture")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-curated-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") {
t.Fatal("Java did not lease the queued curated repository task")
}
if task.Discovery == nil || task.Discovery.EntryType != "CURATED_REPOSITORY" ||
task.Discovery.Owner != "openai" || task.Discovery.RepositoryName != "skills" {
t.Fatalf("Java returned unexpected curated repository coordinates: %+v", task.Discovery)
}
if task.Credential != nil {
t.Fatal("Java should have leased an anonymous public repository task")
}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
limits := DiscoveryLimits{MaxPagesPerLease: 2}
if os.Getenv("SKILLHUB_INTEROP_CURATED_MODE") == "live" {
return ProcessDiscovery(ctx, control, task, limits)
}
return processDiscoveryWithGitHub(ctx, control, task, limits, curatedRepositoryFixture{})
}); err != nil {
t.Fatal(err)
}
}

68
github-sync-go/internal/worker/discovery_owner_interop_test.go Normal file
View file

@ -0,0 +1,68 @@
package worker
import (
"context"
"errors"
"os"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type curatedOwnerFixture struct {
curatedRepositoryFixture
owner string
ownerType github.OwnerType
}
func (fixture curatedOwnerFixture) GetOwnerType(_ context.Context, owner string) (github.OwnerType, github.Quota, error) {
if owner != fixture.owner {
return "", github.Quota{}, errors.New("unexpected curated owner")
}
return fixture.ownerType, github.Quota{}, nil
}
func (fixture curatedOwnerFixture) ListPublicOwnerRepositories(_ context.Context, owner string,
ownerType github.OwnerType, page int) (github.OwnerPage, github.Quota, error) {
if owner != fixture.owner || ownerType != fixture.ownerType || page != 1 {
return github.OwnerPage{}, github.Quota{}, errors.New("unexpected public owner repository page")
}
repository := github.Repository{ID: 987654324, Name: "public-skills", DefaultBranch: "main"}
if fixture.ownerType == github.OrganizationOwner {
repository.ID++
}
repository.Owner.Login = owner
return github.OwnerPage{Repositories: []github.Repository{repository}}, github.Quota{}, nil
}
func TestInteropCuratedOwnerDiscovery(t *testing.T) {
ownerType := github.OwnerType(os.Getenv("SKILLHUB_INTEROP_OWNER_TYPE"))
if ownerType == "" {
t.Skip("requires a Java HTTP curated owner fixture")
}
if ownerType != github.UserOwner && ownerType != github.OrganizationOwner {
t.Fatal("invalid curated owner fixture type")
}
owner := os.Getenv("SKILLHUB_INTEROP_OWNER_LOGIN")
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-owner-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Discovery == nil || task.Discovery.EntryType != "CURATED_OWNER" ||
task.Discovery.Owner != owner || task.Credential != nil {
t.Fatal("Java did not lease the expected anonymous curated owner task")
}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processDiscoveryWithGitHub(ctx, control, task,
DiscoveryLimits{MaxPagesPerLease: 2}, curatedOwnerFixture{owner: owner, ownerType: ownerType})
}); err != nil {
t.Fatal(err)
}
}

73
github-sync-go/internal/worker/discovery_pagination_interop_test.go Normal file
View file

@ -0,0 +1,73 @@
package worker
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type changingSearchPageFixture struct {
curatedRepositoryFixture
phase string
}
func (fixture changingSearchPageFixture) SearchRepositories(_ context.Context, kind, _ string,
_ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) {
if page != 1 {
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page")
}
if kind == "TOPIC" && fixture.phase == "shrunk" {
return github.SearchPage{}, github.Quota{}, nil
}
if kind != "KEYWORD" {
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search selector")
}
ids := []int64{9400001, 9400002, 9400003, 9400004, 9400005}
if fixture.phase == "shrunk" {
ids = []int64{9400002, 9400006}
}
repositories := make([]github.Repository, 0, len(ids))
for _, id := range ids {
repository := github.Repository{ID: id, Name: "public-skills", DefaultBranch: "main"}
repository.Owner.Login = "fixture-owner"
repositories = append(repositories, repository)
}
return github.SearchPage{Repositories: repositories}, github.Quota{}, nil
}
// Java starts the same discovery task in two Go processes. The second lease
// reads a shorter GitHub page than the first and must not lose the new ID.
func TestInteropChangingSearchPage(t *testing.T) {
phase := os.Getenv("SKILLHUB_INTEROP_SEARCH_PAGE_PHASE")
if phase == "" {
t.Skip("requires a Java HTTP changing search page fixture")
}
if phase != "initial" && phase != "shrunk" {
t.Fatal("invalid search page fixture phase")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-changing-page-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Discovery == nil || task.Discovery.EntryType != "SEARCH_RULE" ||
task.Discovery.CandidateBudget != 3 {
t.Fatal("Java did not lease the expected bounded search task")
}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processDiscoveryWithGitHub(ctx, control, task,
DiscoveryLimits{MaxPagesPerLease: 5}, changingSearchPageFixture{phase: phase})
}); err != nil {
t.Fatal(err)
}
}

98
github-sync-go/internal/worker/discovery_preview_interop_test.go Normal file
View file

@ -0,0 +1,98 @@
package worker
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
// Opt-in wire check: Java creates the preview task and persists the result;
// this process exercises the real worker completion/failure path with fixed search pages.
func TestInteropSearchPreview(t *testing.T) {
phase := os.Getenv("SKILLHUB_INTEROP_PREVIEW_PHASE")
if phase == "" {
t.Skip("requires a Java HTTP preview fixture")
}
if phase != "success" && phase != "limit" && phase != "live" {
t.Fatal("invalid preview fixture phase")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-preview-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Discovery == nil || task.Discovery.EntryType != "SEARCH_PREVIEW" ||
task.Discovery.Criteria == nil || len(task.Discovery.Criteria.Selectors) != 2 {
t.Fatal("Java did not lease the expected multi-selector preview")
}
if expected := os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID"); expected != "" {
if task.Credential == nil || task.Credential.CredentialID != expected ||
task.Credential.Token != os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN") {
t.Fatal("Java did not lease the expected fixture credential")
}
} else if os.Getenv("SKILLHUB_INTEROP_EXPECT_ANONYMOUS") == "1" && task.Credential != nil {
t.Fatal("Java should have leased an anonymous public request")
}
if phase == "live" {
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processSearchPreview(ctx, control, task)
}); err != nil {
t.Fatal(err)
}
return
}
var calls []string
search := func(_ context.Context, kind, value string, since *time.Time,
until time.Time, page int) (github.SearchPage, github.Quota, error) {
if since != nil || until.IsZero() || page != 1 {
t.Fatal("preview did not use the first page of its fixed window")
}
calls = append(calls, kind+":"+value)
if kind == "TOPIC" && phase == "limit" {
return github.SearchPage{}, github.Quota{}, &github.APIError{
Status: 429, Code: "GITHUB_RATE_LIMITED", RetryAt: time.Now().Add(10 * time.Minute),
}
}
if kind == "KEYWORD" {
repositories := make([]github.Repository, 6)
for index := range repositories {
repositories[index].ID = int64(900001 + index)
repositories[index].Name = "public-skill"
repositories[index].Owner.Login = "fixture-owner"
}
return github.SearchPage{Repositories: repositories, TotalCount: 1001,
IncompleteResults: true, LimitReached: true}, github.Quota{}, nil
}
repository := github.Repository{ID: 900101, Name: "topic-skill"}
repository.Owner.Login = "fixture-owner"
return github.SearchPage{Repositories: []github.Repository{repository},
TotalCount: 4}, github.Quota{}, nil
}
// The worker's lease wrapper reports upstream errors to Java, including retryAt.
err = executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processSearchPreviewWithSearch(ctx, control, task, search)
})
if phase == "success" && err != nil || phase == "limit" && err == nil {
t.Fatalf("unexpected worker outcome: phase=%s err=%v", phase, err)
}
if phase == "limit" {
var upstream *github.APIError
if !errors.As(err, &upstream) || upstream.Code != "GITHUB_RATE_LIMITED" {
t.Fatalf("rate limit was not preserved: %v", err)
}
}
if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" ||
calls[1] != "TOPIC:agent-skills" {
t.Fatalf("unexpected preview search order: %v", calls)
}
}

99
github-sync-go/internal/worker/discovery_rule_interop_test.go Normal file
View file

@ -0,0 +1,99 @@
package worker
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type searchRuleFixture struct {
curatedRepositoryFixture
calls []string
windows []searchRuleWindow
saturate bool
reportedLimit bool
}
type searchRuleWindow struct {
from time.Time
until time.Time
}
func (fixture *searchRuleFixture) SearchRepositories(_ context.Context, kind, value string,
since *time.Time, until time.Time, page int) (github.SearchPage, github.Quota, error) {
if since == nil || until.IsZero() || page != 1 {
return github.SearchPage{}, github.Quota{}, errors.New("search did not use its fixed first page")
}
fixture.calls = append(fixture.calls, kind+":"+value)
fixture.windows = append(fixture.windows, searchRuleWindow{from: *since, until: until})
if fixture.saturate && kind == "KEYWORD" && !fixture.reportedLimit {
fixture.reportedLimit = true
return github.SearchPage{TotalCount: 1001, LimitReached: true,
IncompleteResults: true}, github.Quota{}, nil
}
first := github.Repository{ID: 987654322, Name: "skill-one", DefaultBranch: "main"}
first.Owner.Login = "fixture-owner"
if kind == "KEYWORD" && value == "agent skills" {
return github.SearchPage{Repositories: []github.Repository{first}, TotalCount: 1}, github.Quota{}, nil
}
if kind == "TOPIC" && value == "agent-skills" {
second := github.Repository{ID: 987654323, Name: "skill-two", DefaultBranch: "main"}
second.Owner.Login = "fixture-owner"
return github.SearchPage{Repositories: []github.Repository{first, second}, TotalCount: 2}, github.Quota{}, nil
}
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search selector")
}
// Java supplies a versioned rule and fixed window; Go reports deduplicated leads.
func TestInteropSearchRuleDiscovery(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_SEARCH_RULE") != "1" {
t.Skip("requires a Java HTTP search rule fixture")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-search-rule-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Discovery == nil || task.Discovery.EntryType != "SEARCH_RULE" ||
task.Discovery.RuleVersion != 2 || task.Discovery.SearchWindow == nil ||
task.Discovery.SearchWindow.Mode != "FULL_RECHECK" || task.Credential != nil {
t.Fatal("Java did not lease the expected anonymous versioned search rule")
}
fixture := &searchRuleFixture{saturate: os.Getenv("SKILLHUB_INTEROP_SEARCH_SATURATED") == "1"}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processDiscoveryWithGitHub(ctx, control, task,
DiscoveryLimits{MaxPagesPerLease: 5}, fixture)
}); err != nil {
t.Fatal(err)
}
if len(fixture.windows) == 0 || !fixture.windows[0].from.Equal(time.Unix(0, 0)) ||
fixture.windows[0].until.Unix() != task.Discovery.SearchWindow.Until.Unix() {
t.Fatal("discovery changed the Java-provided full search window")
}
if fixture.saturate {
if len(fixture.calls) != 4 || fixture.calls[0] != "KEYWORD:agent skills" ||
fixture.calls[1] != "KEYWORD:agent skills" ||
fixture.calls[2] != "KEYWORD:agent skills" ||
fixture.calls[3] != "TOPIC:agent-skills" ||
!fixture.windows[1].from.Equal(fixture.windows[0].from) ||
!fixture.windows[2].from.Equal(fixture.windows[1].until.Add(time.Second)) ||
!fixture.windows[2].until.Equal(fixture.windows[0].until) ||
!fixture.windows[3].from.Equal(fixture.windows[0].from) ||
!fixture.windows[3].until.Equal(fixture.windows[0].until) {
t.Fatalf("incomplete search did not split and cover the fixed window: %v", fixture.calls)
}
} else if len(fixture.calls) != 2 || fixture.calls[0] != "KEYWORD:agent skills" ||
fixture.calls[1] != "TOPIC:agent-skills" {
t.Fatalf("discovery did not execute core before extension: %v", fixture.calls)
}
}

205
github-sync-go/internal/worker/discovery_test.go Normal file
View file

@ -0,0 +1,205 @@
package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type shrinkingSearchFixture struct{ curatedRepositoryFixture }
func (shrinkingSearchFixture) SearchRepositories(_ context.Context, _, _ string,
_ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) {
if page != 1 {
return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page")
}
repositories := []github.Repository{{ID: 11, Name: "first"}, {ID: 12, Name: "second"}}
for i := range repositories {
repositories[i].Owner.Login = "public-owner"
}
return github.SearchPage{Repositories: repositories}, github.Quota{}, nil
}
func TestProcessDiscoveryRevisitsShrinkingPage(t *testing.T) {
previous := `{"v":1,"entry":"SEARCH_RULE","page":1,"offset":3}`
var reported controlplane.DiscoveryBatch
var completed bool
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
switch {
case strings.HasSuffix(request.URL.Path, "/discovered-repositories"):
if err := json.NewDecoder(request.Body).Decode(&reported); err != nil ||
reported.PreviousCheckpoint == nil || *reported.PreviousCheckpoint != previous {
writer.WriteHeader(http.StatusBadRequest)
return
}
writer.WriteHeader(http.StatusAccepted)
case strings.HasSuffix(request.URL.Path, "/complete"):
var body struct {
DiscoveryCheckpoint string `json:"discoveryCheckpoint"`
}
if err := json.NewDecoder(request.Body).Decode(&body); err != nil ||
body.DiscoveryCheckpoint != reported.Checkpoint {
writer.WriteHeader(http.StatusBadRequest)
return
}
completed = true
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`))
default:
writer.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
task := &controlplane.ClaimedTask{
TaskID: "shrinking-page", Kind: "DISCOVERY",
Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)},
Discovery: &controlplane.DiscoverySnapshot{
EntryType: "SEARCH_RULE", Checkpoint: previous, CandidateBudget: 3,
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
}},
SearchWindow: &controlplane.SearchWindow{
Mode: "FULL_RECHECK", Until: time.Now(),
},
},
}
if err := processDiscoveryWithGitHub(context.Background(), control, task,
DiscoveryLimits{MaxPagesPerLease: 2}, shrinkingSearchFixture{}); err != nil {
t.Fatal(err)
}
var next discoveryCursor
if err := json.Unmarshal([]byte(reported.Checkpoint), &next); err != nil {
t.Fatal(err)
}
if !completed || len(reported.Repositories) != 2 ||
reported.Repositories[0].RepositoryID != 11 || reported.Repositories[1].RepositoryID != 12 ||
next.Revisit != 1 || next.Selector != 1 {
t.Fatalf("shrinking page was not revisited: completed=%v batch=%+v cursor=%+v",
completed, reported.Repositories, next)
}
}
func TestPreviewSearchResultsUsesOnePagePerSelectorAndPreservesWarnings(t *testing.T) {
until := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC)
snapshot := &controlplane.DiscoverySnapshot{
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
}},
SearchWindow: &controlplane.SearchWindow{Mode: "FULL_RECHECK", Until: until},
}
var calls []string
search := func(_ context.Context, kind, value string, since *time.Time,
end time.Time, page int) (github.SearchPage, github.Quota, error) {
if since != nil || !end.Equal(until) || page != 1 {
t.Fatalf("unexpected preview search window or page: since=%v end=%s page=%d",
since, end, page)
}
calls = append(calls, kind+":"+value)
repositories := make([]github.Repository, 6)
for i := range repositories {
repositories[i].ID = int64(i + 1)
repositories[i].Name = "skill"
repositories[i].Owner.Login = "public-owner"
}
return github.SearchPage{
Repositories: repositories, TotalCount: 1001,
IncompleteResults: true, LimitReached: true,
}, github.Quota{}, nil
}
results, err := previewSearchResults(context.Background(), snapshot, search)
if err != nil {
t.Fatal(err)
}
if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" ||
calls[1] != "TOPIC:agent-skills" || len(results) != 2 {
t.Fatalf("unexpected selectors: calls=%v results=%+v", calls, results)
}
for _, result := range results {
if result.TotalCount != 1001 || !result.IncompleteResults ||
!result.LimitReached || len(result.Samples) != 5 {
t.Fatalf("preview lost estimates or warnings: %+v", result)
}
}
}
func TestPreviewSearchResultsDoesNotReturnPartialResultsAfterFailure(t *testing.T) {
snapshot := &controlplane.DiscoverySnapshot{
Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{
{Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"},
{Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"},
}},
SearchWindow: &controlplane.SearchWindow{Until: time.Now()},
}
calls := 0
results, err := previewSearchResults(context.Background(), snapshot,
func(context.Context, string, string, *time.Time, time.Time, int) (
github.SearchPage, github.Quota, error) {
calls++
if calls == 2 {
return github.SearchPage{}, github.Quota{}, errors.New("rate limited")
}
return github.SearchPage{TotalCount: 1}, github.Quota{}, nil
})
if err == nil || len(results) != 0 || calls != 2 {
t.Fatalf("preview should fail atomically: results=%+v err=%v calls=%d",
results, err, calls)
}
}
func TestSelectDiscoveryBatchCountsUniqueRepositoriesAndKeepsRawOffset(t *testing.T) {
candidates := []controlplane.RepositoryCandidate{
{RepositoryID: 10}, {RepositoryID: 10}, {RepositoryID: 20},
{RepositoryID: 20}, {RepositoryID: 30},
}
batch, offset := selectDiscoveryBatch(candidates, 0, 2, map[int64]struct{}{})
if offset != 3 || len(batch) != 2 || batch[0].RepositoryID != 10 || batch[1].RepositoryID != 20 {
t.Fatalf("first batch = %+v, offset = %d", batch, offset)
}
batch, offset = selectDiscoveryBatch(candidates, offset, 1,
map[int64]struct{}{10: {}, 20: {}})
if offset != len(candidates) || len(batch) != 1 || batch[0].RepositoryID != 30 {
t.Fatalf("second batch = %+v, offset = %d", batch, offset)
}
}
func TestSearchBoundsPartitionsFixedWindowWithoutOverlap(t *testing.T) {
since := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
window := &controlplane.SearchWindow{
Mode: "INCREMENTAL", Since: since, Until: since.Add(7 * time.Second),
}
leftFrom, leftUntil, err := searchBounds(window, "L")
if err != nil {
t.Fatal(err)
}
rightFrom, rightUntil, err := searchBounds(window, "R")
if err != nil {
t.Fatal(err)
}
if !leftFrom.Equal(since) || !leftUntil.Add(time.Second).Equal(rightFrom) ||
!rightUntil.Equal(window.Until) || nextSearchCursor(discoveryCursor{Path: "L"}).Path != "R" {
t.Fatalf("split windows are not contiguous: [%s, %s], [%s, %s]",
leftFrom, leftUntil, rightFrom, rightUntil)
}
}
func TestSelectDiscoveryBatchCanAdvancePastOnlyDuplicates(t *testing.T) {
candidates := []controlplane.RepositoryCandidate{{RepositoryID: 10}, {RepositoryID: 10}}
batch, offset := selectDiscoveryBatch(candidates, 0, 1, map[int64]struct{}{10: {}})
if len(batch) != 0 || offset != len(candidates) {
t.Fatalf("batch = %+v, offset = %d", batch, offset)
}
}

23
github-sync-go/internal/worker/heartbeat_interop_test.go Normal file
View file

@ -0,0 +1,23 @@
package worker
import (
"context"
"os"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
func TestInteropWorkerHeartbeat(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_HEARTBEAT") != "1" {
t.Skip("requires a Java HTTP worker heartbeat fixture")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
if err := control.Heartbeat(context.Background()); err != nil {
t.Fatal(err)
}
}

202
github-sync-go/internal/worker/lease.go Normal file
View file

@ -0,0 +1,202 @@
package worker
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
// ExecuteRepository renews the Java lease while processing and stops all
// downstream work as soon as renewal fails or shutdown begins.
func ExecuteRepository(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits RepositoryLimits) error {
if task == nil || task.Kind != "REPOSITORY_SYNC" || task.LeaseExpiresAt.IsZero() {
return errors.New("invalid leased repository task")
}
return executeLeasedTask(ctx, control, task, func(workContext context.Context) error {
return ProcessRepository(workContext, control, task, limits)
})
}
func ExecuteDiscovery(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits DiscoveryLimits) error {
if task == nil || task.Kind != "DISCOVERY" || task.LeaseExpiresAt.IsZero() {
return errors.New("invalid leased discovery task")
}
return executeLeasedTask(ctx, control, task, func(workContext context.Context) error {
return ProcessDiscovery(workContext, control, task, limits)
})
}
func executeLeasedTask(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, process func(context.Context) error) error {
workContext, cancel := context.WithCancel(ctx)
defer cancel()
renewDone := make(chan error, 1)
go func() {
renewDone <- keepLease(workContext, cancel, control, task)
}()
workErr := runPlatformTask(workContext, control, task, process)
cancel()
renewErr := <-renewDone
if ctx.Err() != nil {
return ctx.Err()
}
if renewErr != nil {
return renewErr
}
if workErr == nil {
return nil
}
if errors.Is(workErr, context.Canceled) {
return workErr
}
failureReason, retryAt := classifyFailure(workErr, task.SourceID != 0)
reportContext, reportCancel := context.WithTimeout(ctx, 20*time.Second)
defer reportCancel()
credentialErr := reportCredentialProblem(reportContext, control, task, workErr)
var upstream *github.APIError
var quota *controlplane.GithubQuota
if errors.As(workErr, &upstream) {
quota = credentialQuota(upstream.Quota)
}
_, reportErr := control.Fail(reportContext, task.TaskID, task.Lease,
failureReason, safeFailureDetail(workErr), retryAt, quota)
if reportErr != nil {
return reportErr
}
if task.PlatformControlVersion != nil {
// Fail has settled all acquisition; acknowledge a pause racing that report.
_, ackErr := (&platformExecution{control: control, task: task}).acknowledge(reportContext)
if ackErr != nil {
return ackErr
}
}
if credentialErr != nil {
return credentialErr
}
return workErr
}
func safeFailureDetail(err error) string {
var upstream *github.APIError
if errors.As(err, &upstream) {
return fmt.Sprintf("GITHUB_HTTP_%d:%s", upstream.Status, upstream.Code)
}
if errors.Is(err, context.DeadlineExceeded) {
return "WORKER_TIMEOUT"
}
return "WORKER_ERROR"
}
func reportCredentialProblem(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, err error) error {
// Official requests already persisted their lease-derived observation. The
// legacy RATE_LIMITED status cools the entire credential, including CORE
// after a SEARCH-only primary limit, so it must not be reported again.
if task.OfficialDiscovery != nil || repositoryNeedsOfficialControl(task) {
return nil
}
if task.Credential == nil || task.Credential.CredentialID == "" {
return nil
}
var upstream *github.APIError
if !errors.As(err, &upstream) {
return nil
}
quota := credentialQuota(upstream.Quota)
switch upstream.Code {
case "AUTH_INVALID":
return control.ReportCredentialStatus(ctx, task.TaskID, task.Lease,
task.Credential.CredentialID, "AUTH_INVALID", time.Time{}, quota)
case "GITHUB_RATE_LIMITED":
return control.ReportCredentialStatus(ctx, task.TaskID, task.Lease,
task.Credential.CredentialID, "RATE_LIMITED", futureRetry(upstream.RetryAt), quota)
default:
return nil
}
}
func credentialQuota(observed github.Quota) *controlplane.GithubQuota {
if observed.Remaining < 0 || observed.ResetAt.IsZero() {
return nil
}
resource := strings.ToUpper(observed.Resource)
switch resource {
case "CORE", "SEARCH", "GRAPHQL":
default:
resource = "OTHER"
}
return &controlplane.GithubQuota{
Resource: resource, Remaining: observed.Remaining, ResetAt: observed.ResetAt,
}
}
func observeCredentialQuota(control *controlplane.Client,
task *controlplane.ClaimedTask) func(context.Context, github.Quota) {
if task.Credential == nil || task.Credential.CredentialID == "" {
return nil
}
return func(ctx context.Context, observed github.Quota) {
quota := credentialQuota(observed)
if quota == nil {
return
}
reportCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
_ = control.ReportCredentialStatus(reportCtx, task.TaskID, task.Lease,
task.Credential.CredentialID, "OBSERVED", time.Time{}, quota)
}
}
func keepLease(ctx context.Context, cancel context.CancelFunc, control *controlplane.Client,
task *controlplane.ClaimedTask) error {
expiresAt := task.LeaseExpiresAt
for {
remaining := time.Until(expiresAt)
if remaining <= 0 {
cancel()
return errors.New("GitHub sync lease expired")
}
wait := remaining / 3
if wait > time.Minute {
wait = time.Minute
}
if wait < time.Second {
wait = time.Second
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return nil
case <-timer.C:
}
var err error
expiresAt, err = control.Renew(ctx, task.TaskID, task.Lease)
if err != nil {
cancel()
return err
}
}
}
func classifyFailure(err error, registeredSource bool) (string, time.Time) {
var upstream *github.APIError
if errors.As(err, &upstream) {
switch upstream.Code {
case "GITHUB_RATE_LIMITED":
return "GITHUB_RATE_LIMITED", futureRetry(upstream.RetryAt)
case "SOURCE_UNREADABLE":
if registeredSource {
return "SOURCE_UNREADABLE", time.Time{}
}
}
}
return "RETRYABLE_ACQUISITION", time.Time{}
}

54
github-sync-go/internal/worker/lease_test.go Normal file
View file

@ -0,0 +1,54 @@
package worker
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
func TestLeasedTaskStopsWorkWhenJavaRenewalFails(t *testing.T) {
var renewCalls atomic.Int32
var otherCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/internal/github-sync/v1/tasks/fixture/renew" {
renewCalls.Add(1)
response.Header().Set("Content-Type", "application/json")
response.WriteHeader(http.StatusServiceUnavailable)
_, _ = response.Write([]byte(`{"code":"SERVICE_UNAVAILABLE"}`))
return
}
otherCalls.Add(1)
response.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
client, err := controlplane.NewClient(server.URL+"/internal/github-sync/v1",
strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
task := &controlplane.ClaimedTask{
TaskID: "fixture", Lease: controlplane.LeaseProof{Generation: 1, Token: "lease"},
LeaseExpiresAt: time.Now().Add(3 * time.Second),
}
workReleased := false
err = executeLeasedTask(context.Background(), client, task, func(ctx context.Context) error {
defer func() { workReleased = true }()
<-ctx.Done()
return ctx.Err()
})
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Code != "SERVICE_UNAVAILABLE" {
t.Fatalf("renewal failure was not returned: %v", err)
}
if !workReleased || renewCalls.Load() != 1 || otherCalls.Load() != 0 {
t.Fatalf("worker did not release work after failed renewal: released=%t renew=%d other=%d",
workReleased, renewCalls.Load(), otherCalls.Load())
}
}

341
github-sync-go/internal/worker/official_discovery.go Normal file
View file

@ -0,0 +1,341 @@
package worker
import (
"context"
"encoding/json"
"errors"
"math"
"slices"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
// These are continuation delays, not calibrated budget profile allocations.
const officialBudgetRetryDelay = time.Second
type officialDiscoveryGitHub interface {
SearchOfficialRepositories(context.Context, github.OfficialRepositoryQuery, github.OfficialRepositoryRange,
time.Time, int) (github.SearchPage, github.Quota, error)
}
// The outer checkpoint identifies immutable Java-owned scope; cursor holds only
// bounded page-local progress. Revisit makes replay of a changed page unambiguous.
type officialPageCursor struct {
Offset int `json:"offset"`
Revisit int64 `json:"revisit"`
GapReason string `json:"gapReason,omitempty"`
}
func processOfficialDiscovery(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
limits DiscoveryLimits, upstream officialDiscoveryGitHub) error {
if task == nil || task.Kind != "DISCOVERY" || task.OfficialDiscovery == nil || task.Discovery != nil ||
control == nil || upstream == nil || limits.MaxPagesPerLease < 1 {
return errors.New("invalid official discovery task")
}
if _, err := installOfficialQuotaGuard(control, task, upstream); err != nil {
return err
}
snapshot := task.OfficialDiscovery
for _, capability := range []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability} {
if !slices.Contains(task.RequiredCapabilities, capability) {
return errors.New("official discovery task omits required lifecycle capabilities")
}
}
if snapshot.Channel == "OWNER_EXPANSION" {
ownerAPI, ok := upstream.(officialOwnerGitHub)
if !ok {
return errors.New("GitHub client lacks owner enumeration")
}
return processOfficialOwner(ctx, control, task, limits, ownerAPI)
}
if snapshot.FormatVersion != 1 || snapshot.QueryID == "" || snapshot.QueryVersion < 1 || snapshot.ShardID == "" ||
snapshot.StrategyVersion == "" || snapshot.Budget.CandidateBudget < 1 || snapshot.Budget.MaxPages < 1 || snapshot.Budget.MaxRequests < 1 ||
(snapshot.Channel != "REPOSITORY_METADATA" && snapshot.Channel != "README") ||
(snapshot.Channel == "README" && snapshot.Query.Scope != "README") ||
(snapshot.Channel == "REPOSITORY_METADATA" && snapshot.Query.Scope != "NAME_DESCRIPTION_TOPICS") {
return errors.New("unsupported official discovery snapshot or channel")
}
query := github.OfficialRepositoryQuery{Kind: github.OfficialQueryKind(snapshot.Query.Kind), Value: snapshot.Query.Value,
Scope: github.OfficialQueryScope(snapshot.Query.Scope)}
window := github.OfficialRepositoryRange{Since: snapshot.Range.Since, Until: snapshot.Range.Until, Partition: snapshot.Range.Partition}
if _, err := github.CompileOfficialRepositoryQuery(query, window, snapshot.FixedCutoffAt); err != nil {
return err
}
checkpoint := snapshot.Checkpoint
paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if err != nil || paused {
return err
}
progress, cursor, recoveryReason := decodeOfficialProgress(snapshot)
if recoveryReason != "" {
_, err := control.RecoverOfficialCheckpoint(ctx, task.TaskID, controlplane.OfficialCheckpointRecovery{
Lease: task.Lease, Reason: recoveryReason, RejectedCheckpoint: checkpoint,
})
return err
}
if progress.Coverage == "COMPLETE" || progress.Coverage == "SPLIT" {
_, err := control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", task.OfficialDiscovery.Channel)
return err
}
if checkpoint == "" || progress.Coverage == "GAP" {
if progress.Coverage == "GAP" {
cursor.Revisit++
}
progress.Coverage, progress.NextPage, cursor.Offset = "IN_PROGRESS", 1, 0
cursor.GapReason = ""
checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 0, nil, "", "")
if err != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, err)
}
}
var requests, spent int64
seen := make(map[int64]struct{})
for {
if ctx.Err() != nil {
return ctx.Err()
}
paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if err != nil || paused {
return err
}
if requests >= min(snapshot.Budget.MaxPages, snapshot.Budget.MaxRequests, int64(limits.MaxPagesPerLease)) || spent >= snapshot.Budget.CandidateBudget {
return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialBudgetRetryDelay))
}
page, _, searchErr := upstream.SearchOfficialRepositories(ctx, query, window, snapshot.FixedCutoffAt, int(progress.NextPage))
requests++
if searchErr != nil {
// Account for the attempted request without advancing the page/offset.
cursor.Revisit++
checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 1, nil, "", "")
if err != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, err)
}
var upstreamError *github.APIError
if errors.As(searchErr, &upstreamError) && upstreamError.Code == "GITHUB_RATE_LIMITED" {
retryAt := upstreamError.RetryAt
if !retryAt.After(time.Now()) {
retryAt = time.Now().Add(time.Minute)
}
return completeOfficialDeferred(ctx, control, task, checkpoint, "RATE_LIMIT", retryAt)
}
return searchErr
}
if page.LimitReached || (page.HasMore && progress.NextPage >= 10) {
if cursor.GapReason == "" {
cursor.GapReason = "RESULT_LIMIT"
}
}
if page.IncompleteResults {
cursor.GapReason = "INCOMPLETE_RESULTS"
}
candidates := toCandidates(page.Repositories)
if cursor.Offset > len(candidates) {
cursor.Offset = 0
cursor.Revisit++
}
batch, end := selectDiscoveryBatch(candidates, cursor.Offset, int(min(snapshot.Budget.CandidateBudget-spent, 100)), seen)
provenance := controlplane.OfficialProvenance{Channel: snapshot.Channel, QueryID: snapshot.QueryID,
QueryVersion: snapshot.QueryVersion, StrategyVersion: snapshot.StrategyVersion, ShardID: snapshot.ShardID}
for index := range batch {
batch[index].OfficialProvenance = []controlplane.OfficialProvenance{provenance}
}
next, nextCursor := progress, cursor
var children []controlplane.OfficialQueryRange
gapReason := ""
if end < len(candidates) {
// Even an incomplete page may contain useful public repositories.
// Drain it across leases before handing off or recording a gap.
nextCursor.Offset = end
} else {
nextCursor.Offset = end
if cursor.GapReason != "" {
children = officialSplitRanges(snapshot)
}
if len(children) > 0 {
next.Coverage = "SPLIT"
} else if page.HasMore && progress.NextPage < 10 && cursor.GapReason != "INCOMPLETE_RESULTS" {
// A result cap still allows the accessible pages to be harvested.
// Retain the gap signal so page ten can never claim full coverage.
next.NextPage++
nextCursor.Offset = 0
} else if cursor.GapReason != "" {
next.Coverage, gapReason = "GAP", cursor.GapReason
if snapshot.Range.Until.Unix()-snapshot.Range.Since.Unix() < 2 {
gapReason = "UNSPLITTABLE"
}
} else {
next.Coverage = "COMPLETE"
}
}
acceptedCheckpoint, reportErr := reportOfficialProgress(ctx, control, task, checkpoint, next, nextCursor, batch, 1, children, gapReason, "")
if reportErr != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, reportErr)
}
checkpoint, progress, cursor = acceptedCheckpoint, next, nextCursor
for _, candidate := range batch {
seen[candidate.RepositoryID] = struct{}{}
}
spent += int64(len(batch))
if progress.Coverage == "GAP" {
return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialGapBackoff(cursor.Revisit)))
}
if progress.Coverage == "COMPLETE" || progress.Coverage == "SPLIT" {
_, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", task.OfficialDiscovery.Channel)
return err
}
}
}
func decodeOfficialProgress(snapshot *controlplane.OfficialDiscoverySnapshot) (controlplane.OfficialCheckpoint, officialPageCursor, string) {
progress := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: snapshot.StrategyVersion,
QueryID: snapshot.QueryID, QueryVersion: snapshot.QueryVersion, ShardID: snapshot.ShardID, NextPage: 1, Coverage: "IN_PROGRESS"}
cursor := officialPageCursor{}
if snapshot.Checkpoint == "" {
return progress, cursor, ""
}
progress = controlplane.OfficialCheckpoint{}
if len(snapshot.Checkpoint) > 4096 || json.Unmarshal([]byte(snapshot.Checkpoint), &progress) != nil {
return progress, cursor, "MALFORMED"
}
if progress.FormatVersion != 1 {
return progress, cursor, "FORMAT_UNSUPPORTED"
}
if progress.StrategyVersion != snapshot.StrategyVersion || progress.QueryID != snapshot.QueryID ||
progress.QueryVersion != snapshot.QueryVersion || progress.ShardID != snapshot.ShardID {
return progress, cursor, "IDENTITY_MISMATCH"
}
ownerChannel := snapshot.Channel == "OWNER_EXPANSION"
if progress.NextPage < 1 || progress.NextPage > math.MaxInt32 ||
(!ownerChannel && (progress.NextPage > 11 || (progress.NextPage == 11 && progress.Coverage != "COMPLETE" && progress.Coverage != "SPLIT"))) ||
(ownerChannel && progress.Coverage == "SPLIT") ||
!slices.Contains([]string{"IN_PROGRESS", "COMPLETE", "SPLIT", "GAP"}, progress.Coverage) ||
len(progress.Cursor) > 1024 || (progress.Cursor != "" && json.Unmarshal([]byte(progress.Cursor), &cursor) != nil) ||
cursor.Offset < 0 || cursor.Offset > 100 || cursor.Revisit < 0 || cursor.Revisit >= math.MaxInt64-16 {
return progress, cursor, "MALFORMED"
}
if progress.Cursor != "" {
var fields map[string]json.RawMessage
if json.Unmarshal([]byte(progress.Cursor), &fields) != nil || (len(fields) != 2 && len(fields) != 3) ||
len(fields["offset"]) == 0 || len(fields["revisit"]) == 0 ||
string(fields["offset"]) == "null" || string(fields["revisit"]) == "null" ||
(len(fields) == 3 && len(fields["gapReason"]) == 0) ||
(cursor.GapReason != "" && !slices.Contains([]string{"RESULT_LIMIT", "INCOMPLETE_RESULTS"}, cursor.GapReason)) {
return progress, cursor, "MALFORMED"
}
}
return progress, cursor, ""
}
func reportOfficialProgress(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
previous string, progress controlplane.OfficialCheckpoint, cursor officialPageCursor,
repositories []controlplane.RepositoryCandidate, requests int64, children []controlplane.OfficialQueryRange, gapReason, ownerLogin string) (string, error) {
if ownerLogin != "" && task.OfficialDiscovery.Channel != "OWNER_EXPANSION" {
return previous, errors.New("only owner expansion can report canonical owner login")
}
encodedCursor, err := json.Marshal(cursor)
if err != nil {
return previous, err
}
progress.Cursor = string(encodedCursor)
encoded, err := json.Marshal(progress)
if err != nil || len(encoded) > 4096 || string(encoded) == previous {
return previous, errors.New("official discovery checkpoint did not advance or exceeds limit")
}
var previousCheckpoint *string
if previous != "" {
previousCheckpoint = &previous
}
batch := controlplane.DiscoveryBatch{
PreviousCheckpoint: previousCheckpoint, Checkpoint: string(encoded), Repositories: repositories,
OfficialBatch: &controlplane.OfficialDiscoveryBatch{ShardID: progress.ShardID, Coverage: progress.Coverage,
RequestsUsed: requests, ChildRanges: children, GapReason: gapReason, OwnerLogin: ownerLogin},
}
err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch)
if err != nil && ctx.Err() == nil {
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Status >= 500 {
// The first response may have been lost after commit. Replay the
// identical payload; Java's batch digest prevents double accounting.
err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch)
}
}
if err != nil {
return previous, err
}
return string(encoded), nil
}
// Child tasks are registered atomically by Java. The snapshot carries remaining
// depth and root-wide child capacity; zero/one remaining child cannot bisect.
// Go never scans unacknowledged children; Java rechecks the root budget on write.
func officialSplitRanges(snapshot *controlplane.OfficialDiscoverySnapshot) []controlplane.OfficialQueryRange {
start, end := snapshot.Range.Since.Unix(), snapshot.Range.Until.Unix()
if snapshot.Budget.MaxSplitDepth < 1 || snapshot.Budget.MaxChildShards < 2 || end-start < 2 {
return nil
}
middle := time.Unix(start+(end-start)/2, 0).UTC()
return []controlplane.OfficialQueryRange{
{Since: snapshot.Range.Since, Until: middle},
{Since: middle, Until: snapshot.Range.Until},
}
}
func pauseOfficialDiscoveryIfNeeded(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, checkpoint string) (bool, error) {
for attempts := 0; attempts < 3; attempts++ {
state, err := control.GetOfficialTaskControl(ctx, task.TaskID, task.Lease)
if err != nil || state.Action == "CONTINUE" {
return false, err
}
var acknowledged *string
if checkpoint != "" {
acknowledged = &checkpoint
}
err = control.PauseOfficialDiscovery(ctx, task.TaskID, task.Lease, acknowledged)
if err == nil {
return true, nil
}
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Code != "CONTROL_CHANGED" {
return false, err
}
}
return false, errors.New("official discovery control changed repeatedly")
}
// A refused batch never advances the acknowledged checkpoint. Control is
// re-read before deciding whether to pause or release the lease for capacity.
func handleOfficialBatchFailure(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
checkpoint string, reportErr error) error {
var apiError *controlplane.APIError
if !errors.As(reportErr, &apiError) ||
(apiError.Code != "BACKPRESSURE_ACTIVE" && apiError.Code != "CANDIDATE_BUDGET_EXCEEDED") {
return reportErr
}
paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if err != nil || paused {
return err
}
retryAt := time.Now().Add(time.Minute)
if checkpoint == "" {
// Complete requires a persisted checkpoint. If the very first batch
// was refused, release through fail without inventing any progress.
_, err := control.Fail(ctx, task.TaskID, task.Lease, "RETRYABLE_ACQUISITION",
"", retryAt, nil)
return err
}
return completeOfficialDeferred(ctx, control, task, checkpoint, "CAPACITY", retryAt)
}
func completeOfficialDeferred(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
checkpoint, reason string, retryAt time.Time) error {
_, err := control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, retryAt, reason, task.OfficialDiscovery.Channel)
return err
}
// A repeatedly unresolved fixed range retains its retry generation across
// leases. Normal work-budget exhaustion does not use this gap backoff.
func officialGapBackoff(revisit int64) time.Duration {
return min(5*time.Minute*time.Duration(int64(1)<<min(revisit, 7)), 6*time.Hour)
}

510
github-sync-go/internal/worker/official_discovery_test.go Normal file
View file

@ -0,0 +1,510 @@
package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type officialFixture struct {
before func(context.Context) error
after func(context.Context, github.Quota, error) error
search func(int) (github.SearchPage, error)
owner func(int) github.OwnerIdentity
list func(string, int) github.OwnerPage
searches, owners, lists int
}
func (f *officialFixture) SetRequestPolicy(before func(context.Context) error, after func(context.Context, github.Quota, error) error) {
f.before, f.after = before, after
}
func (f *officialFixture) observe(ctx context.Context, resource string, err error) error {
if e := f.after(ctx, github.Quota{Resource: resource, Remaining: 100, ResetAt: time.Now().Add(time.Hour)}, err); e != nil {
return e
}
return err
}
func (f *officialFixture) SearchOfficialRepositories(ctx context.Context, _ github.OfficialRepositoryQuery, _ github.OfficialRepositoryRange, _ time.Time, page int) (github.SearchPage, github.Quota, error) {
if err := f.before(ctx); err != nil {
return github.SearchPage{}, github.Quota{}, err
}
f.searches++
p, err := f.search(page)
return p, github.Quota{}, f.observe(ctx, "SEARCH", err)
}
func (f *officialFixture) GetOwnerByID(ctx context.Context, id int64) (github.OwnerIdentity, github.Quota, error) {
if err := f.before(ctx); err != nil {
return github.OwnerIdentity{}, github.Quota{}, err
}
f.owners++
o := f.owner(f.owners)
return o, github.Quota{}, f.observe(ctx, "CORE", nil)
}
func (f *officialFixture) ListPublicOwnerRepositories(ctx context.Context, login string, _ github.OwnerType, page int) (github.OwnerPage, github.Quota, error) {
if err := f.before(ctx); err != nil {
return github.OwnerPage{}, github.Quota{}, err
}
f.lists++
p := f.list(login, page)
return p, github.Quota{}, f.observe(ctx, "CORE", nil)
}
type officialHarness struct {
t *testing.T
control *controlplane.Client
accepted string
attempts []controlplane.DiscoveryBatch
completions []struct {
Coverage string
DiscoveryCheckpoint string
RetryAt time.Time
}
paused *string
pause bool
rejectCandidate bool
loseCandidateResponse bool
quotaFailure bool
observations []controlplane.OfficialQuotaObservation
channel string
}
func resetOfficialQuota(t *testing.T) {
reset := func() {
officialProcessQuota.Lock()
defer officialProcessQuota.Unlock()
officialProcessQuota.globalUntil = time.Time{}
officialProcessQuota.resourceUntil = make(map[string]time.Time)
officialProcessQuota.failures = 0
officialProcessQuota.lastFailure = time.Time{}
}
reset()
t.Cleanup(reset)
}
func newOfficialHarness(t *testing.T, channel string) *officialHarness {
t.Helper()
resetOfficialQuota(t)
h := &officialHarness{t: t, channel: channel}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case strings.HasSuffix(r.URL.Path, "/official-control"):
if h.pause {
_, _ = w.Write([]byte(`{"action":"PAUSE","configurationVersion":1,"reason":"DISABLED"}`))
} else {
_, _ = w.Write([]byte(`{"action":"CONTINUE","configurationVersion":1}`))
}
case strings.HasSuffix(r.URL.Path, "/official-quota"):
var body struct {
Observation controlplane.OfficialQuotaObservation
CredentialID string
SubjectKey string
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
if body.CredentialID != "" || body.SubjectKey != "" {
t.Error("client selected quota subject")
}
h.observations = append(h.observations, body.Observation)
if h.quotaFailure {
w.WriteHeader(503)
} else {
w.WriteHeader(204)
}
case strings.HasSuffix(r.URL.Path, "/discovered-repositories"):
var b controlplane.DiscoveryBatch
if err := json.NewDecoder(r.Body).Decode(&b); err != nil {
t.Error(err)
}
h.attempts = append(h.attempts, b)
if h.rejectCandidate && len(b.Repositories) > 0 {
h.pause = true
w.WriteHeader(409)
_, _ = w.Write([]byte(`{"code":"BACKPRESSURE_ACTIVE"}`))
return
}
if b.Checkpoint != h.accepted {
previous := ""
if b.PreviousCheckpoint != nil {
previous = *b.PreviousCheckpoint
}
if previous != h.accepted {
t.Errorf("CAS previous %q != accepted %q", previous, h.accepted)
}
h.accepted = b.Checkpoint
}
if h.loseCandidateResponse && len(b.Repositories) > 0 {
h.loseCandidateResponse = false
w.WriteHeader(503)
return
}
w.WriteHeader(202)
case strings.HasSuffix(r.URL.Path, "/official-pause"):
var body struct{ DiscoveryCheckpoint *string }
_ = json.NewDecoder(r.Body).Decode(&body)
h.paused = body.DiscoveryCheckpoint
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
case strings.HasSuffix(r.URL.Path, "/complete"):
var body struct {
Coverage string
DiscoveryCheckpoint string
RetryAt time.Time
}
_ = json.NewDecoder(r.Body).Decode(&body)
h.completions = append(h.completions, body)
if body.DiscoveryCheckpoint != h.accepted {
t.Error("completion did not retain accepted checkpoint")
}
status := "ACQUISITION_COMPLETE"
if body.Coverage == "INCOMPLETE" {
status = "RETRY_WAIT"
var cp controlplane.OfficialCheckpoint
_ = json.Unmarshal([]byte(h.accepted), &cp)
if cp.Coverage == "GAP" && h.channel != "OWNER_EXPANSION" {
status = "PARTIAL"
}
}
_ = json.NewEncoder(w).Encode(map[string]string{"status": status})
default:
t.Errorf("unexpected control path %s", r.URL.Path)
w.WriteHeader(404)
}
}))
t.Cleanup(server.Close)
var err error
h.control, err = controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
return h
}
func officialTask(channel string) *controlplane.ClaimedTask {
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
task := &controlplane.ClaimedTask{TaskID: "official-task", Kind: "DISCOVERY", Lease: controlplane.LeaseProof{Generation: 1, Token: strings.Repeat("a", 64)}, RequiredCapabilities: []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability}, OfficialDiscovery: &controlplane.OfficialDiscoverySnapshot{FormatVersion: 1, ConfigurationVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", Channel: channel, Mode: "HISTORICAL", Query: controlplane.OfficialQuery{Kind: "KEYWORD", Value: "skills", Scope: "NAME_DESCRIPTION_TOPICS"}, Range: controlplane.OfficialQueryRange{Since: start, Until: start.Add(8 * time.Second)}, FixedCutoffAt: start.Add(time.Minute), Budget: controlplane.OfficialBudget{Profile: "BALANCED", MaxRequests: 10, MaxPages: 10, CandidateBudget: 100}}}
if channel == "OWNER_EXPANSION" {
s := task.OfficialDiscovery
s.Mode = "OWNER_RECHECK"
s.OwnerID = 42
s.OwnerLogin = "old-name"
s.SeedRepositoryID = 11
s.ExpansionDepth = 1
s.Query = controlplane.OfficialQuery{Kind: "OWNER", Value: "old-name", Scope: "OWNER_REPOSITORIES"}
task.RequiredCapabilities = append(task.RequiredCapabilities, controlplane.OfficialOwnerCapability)
}
return task
}
func officialRepo(id, ownerID int64) github.Repository {
r := github.Repository{ID: id, Name: "skills"}
r.Owner.Login = "new-name"
r.Owner.ID = ownerID
return r
}
func officialRun(t *testing.T, h *officialHarness, task *controlplane.ClaimedTask, f *officialFixture) {
t.Helper()
if err := processOfficialDiscovery(context.Background(), h.control, task, DiscoveryLimits{MaxPagesPerLease: 10}, f); err != nil {
t.Fatal(err)
}
}
func decodedCP(t *testing.T, s string) controlplane.OfficialCheckpoint {
t.Helper()
var cp controlplane.OfficialCheckpoint
if err := json.Unmarshal([]byte(s), &cp); err != nil {
t.Fatal(err)
}
return cp
}
func TestOfficialSearchPaginationAndReplay(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
h.loseCandidateResponse = true
f := &officialFixture{search: func(page int) (github.SearchPage, error) {
return github.SearchPage{Repositories: []github.Repository{officialRepo(int64(page), 42)}, HasMore: page < 2}, nil
}}
officialRun(t, h, officialTask(h.channel), f)
if f.searches != 2 || len(h.attempts) != 4 || len(h.completions) != 1 {
t.Fatalf("searches=%d batches=%d completions=%d", f.searches, len(h.attempts), len(h.completions))
}
first, _ := json.Marshal(h.attempts[1])
replay, _ := json.Marshal(h.attempts[2])
if string(first) != string(replay) {
t.Fatal("response-loss replay changed payload")
}
if cp := decodedCP(t, h.accepted); cp.Coverage != "COMPLETE" || cp.NextPage != 2 {
t.Fatalf("checkpoint %+v", cp)
}
}
func TestOfficialRejectedBatchPausesLastAcknowledgedCheckpoint(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
h.rejectCandidate = true
f := &officialFixture{search: func(int) (github.SearchPage, error) {
return github.SearchPage{Repositories: []github.Repository{officialRepo(1, 42)}}, nil
}}
officialRun(t, h, officialTask(h.channel), f)
if h.paused == nil || *h.paused != h.attempts[0].Checkpoint || *h.paused == h.attempts[1].Checkpoint || len(h.completions) != 0 {
t.Fatal("rejected candidate advanced checkpoint or completed")
}
}
func TestOfficialSearchCoverageBoundaries(t *testing.T) {
for _, kind := range []string{"split", "gap", "unsplittable"} {
t.Run(kind, func(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
task := officialTask(h.channel)
want := "GAP"
if kind == "split" {
task.OfficialDiscovery.Budget.MaxSplitDepth = 1
task.OfficialDiscovery.Budget.MaxChildShards = 2
want = "SPLIT"
}
if kind == "unsplittable" {
task.OfficialDiscovery.Range.Until = task.OfficialDiscovery.Range.Since.Add(time.Second)
}
f := &officialFixture{search: func(int) (github.SearchPage, error) {
return github.SearchPage{Repositories: []github.Repository{officialRepo(1, 42)}, IncompleteResults: true}, nil
}}
officialRun(t, h, task, f)
b := h.attempts[len(h.attempts)-1]
if b.OfficialBatch.Coverage != want || len(b.Repositories) != 1 {
t.Fatalf("lost coverage/candidates: %+v", b)
}
if want == "SPLIT" {
ranges := b.OfficialBatch.ChildRanges
if len(ranges) != 2 || !ranges[0].Since.Equal(task.OfficialDiscovery.Range.Since) || !ranges[0].Until.Equal(ranges[1].Since) || !ranges[1].Until.Equal(task.OfficialDiscovery.Range.Until) {
t.Fatal("split does not partition parent")
}
} else if !h.completions[0].RetryAt.After(time.Now().Add(4 * time.Minute)) {
t.Fatal("GAP lacks backoff")
}
if kind == "unsplittable" && b.OfficialBatch.GapReason != "UNSPLITTABLE" {
t.Fatal("missing unsplittable reason")
}
})
}
}
func TestOfficialSplitResumeDoesNotSearch(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
task := officialTask(h.channel)
cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 1, Coverage: "SPLIT", Cursor: `{"offset":0,"revisit":0}`}
b, _ := json.Marshal(cp)
task.OfficialDiscovery.Checkpoint = string(b)
h.accepted = string(b)
f := &officialFixture{search: func(int) (github.SearchPage, error) {
t.Fatal("resumed parent was searched")
return github.SearchPage{}, nil
}}
officialRun(t, h, task, f)
if f.searches != 0 || len(h.completions) != 1 {
t.Fatal("split resume did not only finish parent")
}
}
func TestOfficialOwnerIdentityAndDepth(t *testing.T) {
for _, kind := range []string{"renamed", "wrong-repo-owner", "rename-at-empty-tail", "recursive"} {
t.Run(kind, func(t *testing.T) {
h := newOfficialHarness(t, "OWNER_EXPANSION")
task := officialTask(h.channel)
f := &officialFixture{owner: func(call int) github.OwnerIdentity {
login := "new-name"
if kind == "rename-at-empty-tail" && call == 2 {
login = "newer-name"
}
return github.OwnerIdentity{ID: 42, Login: login, Type: github.UserOwner}
}, list: func(login string, page int) github.OwnerPage {
if login != "new-name" {
t.Error("used stale snapshot login")
}
if kind == "rename-at-empty-tail" {
return github.OwnerPage{}
}
id := int64(42)
if kind == "wrong-repo-owner" {
id = 99
}
return github.OwnerPage{Repositories: []github.Repository{officialRepo(9, id)}}
}}
if kind == "recursive" {
task.OfficialDiscovery.ExpansionDepth = 2
if err := processOfficialDiscovery(context.Background(), h.control, task, DiscoveryLimits{MaxPagesPerLease: 10}, f); err == nil || f.owners != 0 {
t.Fatal("recursive expansion made requests")
}
return
}
officialRun(t, h, task, f)
if h.attempts[0].OfficialBatch.OwnerLogin != "new-name" {
t.Fatal("canonical login not persisted before enumeration")
}
cp := decodedCP(t, h.accepted)
if kind == "renamed" {
if cp.Coverage != "COMPLETE" || f.owners != 2 || h.attempts[len(h.attempts)-1].Repositories[0].OfficialProvenance[0].SeedRepositoryID != 11 {
t.Fatal("valid owner expansion incomplete")
}
} else if cp.Coverage != "IN_PROGRESS" {
t.Fatal("identity race falsely completed")
}
})
}
}
func TestOfficialOwnerResumeBeyondSearchLimit(t *testing.T) {
h := newOfficialHarness(t, "OWNER_EXPANSION")
task := officialTask(h.channel)
cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 12, Coverage: "IN_PROGRESS", Cursor: `{"offset":0,"revisit":0}`}
b, _ := json.Marshal(cp)
task.OfficialDiscovery.Checkpoint = string(b)
h.accepted = string(b)
f := &officialFixture{owner: func(int) github.OwnerIdentity {
return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner}
}, list: func(_ string, page int) github.OwnerPage {
if page != 12 {
t.Error("lost resumed page")
}
return github.OwnerPage{}
}}
officialRun(t, h, task, f)
if decodedCP(t, h.accepted).Coverage != "COMPLETE" {
t.Fatal("owner page beyond ten rejected")
}
}
func TestOfficialQuotaPersistenceFailureStopsLeaseAndOtherTokens(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
h.quotaFailure = true
f := &officialFixture{search: func(int) (github.SearchPage, error) { return github.SearchPage{HasMore: true}, nil }}
err := processOfficialDiscovery(context.Background(), h.control, officialTask(h.channel), DiscoveryLimits{MaxPagesPerLease: 10}, f)
if err == nil || f.searches != 1 || len(h.observations) != 1 {
t.Fatalf("did not fail closed: %v searches=%d", err, f.searches)
}
if err := (&officialQuotaGuard{resource: "CORE"}).before(context.Background()); err == nil {
t.Fatal("different resource/token bypassed persistence cooldown")
}
}
func TestOfficialQuotaResourceAndAnonymousSemantics(t *testing.T) {
for _, kind := range []string{"primary", "secondary", "anonymous-unknown", "git-failure", "auth"} {
t.Run(kind, func(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
g := &officialQuotaGuard{control: h.control, task: officialTask(h.channel), resource: "SEARCH"}
quota := github.Quota{Resource: "SEARCH", Remaining: 0, ResetAt: time.Now().Add(time.Hour)}
var err error
switch kind {
case "primary":
err = g.after(context.Background(), quota, nil)
case "secondary":
quota.Remaining = 5
err = g.after(context.Background(), quota, &github.APIError{Code: "GITHUB_RATE_LIMITED", SecondaryRateLimit: true, RetryAt: time.Now().Add(time.Minute)})
case "anonymous-unknown":
err = g.after(context.Background(), github.Quota{Remaining: -1}, nil)
case "git-failure":
err = g.afterGit(context.Background(), errors.New("git failed"))
case "auth":
err = g.after(context.Background(), github.Quota{Remaining: -1}, &github.APIError{Code: "AUTH_INVALID"})
}
obs := h.observations[0]
if kind == "anonymous-unknown" {
if err != nil || obs.State != "UNKNOWN" || obs.Remaining != nil || obs.ResetAt != nil {
t.Fatal("unknown manufactured quota")
}
return
}
if err == nil {
t.Fatal("limit/error did not stop current lease")
}
coreErr := (&officialQuotaGuard{resource: "CORE"}).before(context.Background())
if (kind == "primary") != (coreErr == nil) {
t.Fatal("primary resource isolation or secondary global sharing broken")
}
if kind == "git-failure" && (obs.Resource != "OTHER" || obs.State != "UNKNOWN" || obs.RetryAfterSeconds == nil) {
t.Fatal("git invented REST quota")
}
})
}
}
func TestOfficialSearchLimitHarvestsAccessiblePagesBeforeGap(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
f := &officialFixture{search: func(page int) (github.SearchPage, error) {
return github.SearchPage{Repositories: []github.Repository{officialRepo(int64(page), 42)}, HasMore: true, LimitReached: true}, nil
}}
officialRun(t, h, officialTask(h.channel), f)
last := h.attempts[len(h.attempts)-1]
if f.searches != 10 || last.OfficialBatch.Coverage != "GAP" || last.OfficialBatch.GapReason != "RESULT_LIMIT" {
t.Fatal("accessible search pages discarded or falsely complete")
}
total := 0
for _, b := range h.attempts {
total += len(b.Repositories)
}
if total != 10 {
t.Fatalf("stored %d candidates", total)
}
}
func TestOfficialOwnerBudgetResumeAndPause(t *testing.T) {
h := newOfficialHarness(t, "OWNER_EXPANSION")
task := officialTask(h.channel)
task.OfficialDiscovery.Budget.MaxRequests = 3
task.OfficialDiscovery.Budget.CandidateBudget = 1
f := &officialFixture{owner: func(int) github.OwnerIdentity {
return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner}
}, list: func(_ string, page int) github.OwnerPage {
return github.OwnerPage{Repositories: []github.Repository{officialRepo(1, 42), officialRepo(2, 42)}}
}}
officialRun(t, h, task, f)
cp := decodedCP(t, h.accepted)
var cursor officialPageCursor
_ = json.Unmarshal([]byte(cp.Cursor), &cursor)
if cp.Coverage != "IN_PROGRESS" || cp.NextPage != 1 || cursor.Offset != 1 || f.owners != 1 {
t.Fatal("candidate budget lost page offset")
}
task.OfficialDiscovery.Checkpoint = h.accepted
task.Lease.Generation++
h.pause = true
officialRun(t, h, task, f)
if h.paused == nil || *h.paused != task.OfficialDiscovery.Checkpoint || f.lists != 1 {
t.Fatal("paused resume performed upstream request")
}
h.pause = false
officialRun(t, h, task, f)
last := h.attempts[len(h.attempts)-1]
if len(last.Repositories) != 1 || last.Repositories[0].RepositoryID != 2 || last.OfficialBatch.Coverage != "COMPLETE" || last.OfficialBatch.RequestsUsed != 2 {
t.Fatal("resume duplicated candidate or skipped tail identity cost")
}
}
func TestOfficialSearchFailurePersistsAttemptWithoutAdvancingPage(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
f := &officialFixture{search: func(int) (github.SearchPage, error) {
return github.SearchPage{}, errors.New("controlled upstream error")
}}
if err := processOfficialDiscovery(context.Background(), h.control, officialTask(h.channel), DiscoveryLimits{MaxPagesPerLease: 10}, f); err == nil {
t.Fatal("upstream error swallowed")
}
last := h.attempts[len(h.attempts)-1]
cp := decodedCP(t, h.accepted)
if len(h.attempts) != 2 || last.OfficialBatch.RequestsUsed != 1 || cp.NextPage != 1 || cp.Coverage != "IN_PROGRESS" || len(h.completions) != 0 {
t.Fatal("failed search advanced or lost request accounting")
}
}
func TestOfficialOwnerPageOverflowIsGap(t *testing.T) {
h := newOfficialHarness(t, "OWNER_EXPANSION")
task := officialTask(h.channel)
cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 2147483647, Coverage: "IN_PROGRESS", Cursor: `{"offset":0,"revisit":0}`}
encoded, _ := json.Marshal(cp)
task.OfficialDiscovery.Checkpoint = string(encoded)
h.accepted = string(encoded)
f := &officialFixture{owner: func(int) github.OwnerIdentity {
return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner}
}, list: func(_ string, page int) github.OwnerPage {
if page != 2147483647 {
t.Fatal("page overflow before request")
}
return github.OwnerPage{HasMore: true}
}}
officialRun(t, h, task, f)
last := h.attempts[len(h.attempts)-1]
if decodedCP(t, h.accepted).NextPage != 2147483647 || last.OfficialBatch.Coverage != "GAP" || last.OfficialBatch.GapReason != "RESULT_LIMIT" || h.completions[0].Coverage != "INCOMPLETE" {
t.Fatal("owner overflow claimed completion")
}
}

180
github-sync-go/internal/worker/official_owner.go Normal file
View file

@ -0,0 +1,180 @@
package worker
import (
"context"
"errors"
"math"
"slices"
"strings"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type officialOwnerGitHub interface {
GetOwnerByID(context.Context, int64) (github.OwnerIdentity, github.Quota, error)
ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error)
}
// Owner rechecks enumerate a changing public collection, not a pushed-time window.
// Stable IDs fence account renames and login reuse between API requests.
func processOfficialOwner(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
limits DiscoveryLimits, upstream officialOwnerGitHub) error {
snapshot := task.OfficialDiscovery
if !slices.Contains(task.RequiredCapabilities, controlplane.OfficialOwnerCapability) ||
snapshot.FormatVersion != 1 || snapshot.QueryID == "" || snapshot.QueryVersion < 1 || snapshot.ShardID == "" || snapshot.StrategyVersion == "" ||
snapshot.OwnerID < 1 || snapshot.SeedRepositoryID < 1 || snapshot.ExpansionDepth != 1 ||
snapshot.OwnerLogin == "" || snapshot.Query.Value != snapshot.OwnerLogin || snapshot.Range.Partition != "" ||
snapshot.Mode != "OWNER_RECHECK" || snapshot.Query.Kind != "OWNER" || snapshot.Query.Scope != "OWNER_REPOSITORIES" ||
snapshot.Budget.MaxRequests < 3 || snapshot.Budget.MaxPages < 1 || snapshot.Budget.CandidateBudget < 1 ||
snapshot.Budget.MaxSplitDepth != 0 || snapshot.Budget.MaxChildShards != 0 {
return errors.New("invalid official owner expansion snapshot")
}
checkpoint := snapshot.Checkpoint
paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if paused || err != nil {
return err
}
progress, cursor, recoveryReason := decodeOfficialProgress(snapshot)
if recoveryReason != "" {
_, err = control.RecoverOfficialCheckpoint(ctx, task.TaskID, controlplane.OfficialCheckpointRecovery{
Lease: task.Lease, Reason: recoveryReason, RejectedCheckpoint: checkpoint,
})
return err
}
if progress.Coverage == "COMPLETE" {
_, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", snapshot.Channel)
return err
}
if progress.Coverage == "GAP" {
cursor.Revisit++
progress.Coverage, progress.NextPage, cursor.Offset, cursor.GapReason = "IN_PROGRESS", 1, 0, ""
}
owner, _, lookupErr := upstream.GetOwnerByID(ctx, snapshot.OwnerID)
canonicalLogin := ""
if lookupErr == nil {
if owner.ID != snapshot.OwnerID || owner.Login == "" {
lookupErr = errors.New("owner identity changed")
} else {
canonicalLogin = owner.Login
}
}
cursor.Revisit++
checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 1, nil, "", canonicalLogin)
if err != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, err)
}
if lookupErr != nil {
return deferOfficialOwnerError(ctx, control, task, checkpoint, lookupErr)
}
requests, pages, spent := int64(1), int64(0), int64(0)
seen := make(map[int64]struct{})
for {
paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if paused || err != nil {
return err
}
// Reserve a second request to verify even an empty last page. With a minimum
// budget of three, a fresh lease can always resolve identity and finish a page.
if requests+2 > snapshot.Budget.MaxRequests || pages >= min(snapshot.Budget.MaxPages, int64(limits.MaxPagesPerLease)) || spent >= snapshot.Budget.CandidateBudget {
return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialBudgetRetryDelay))
}
page, _, pageErr := upstream.ListPublicOwnerRepositories(ctx, owner.Login, owner.Type, int(progress.NextPage))
requests++
pages++
used := int64(1)
if pageErr == nil {
for _, repository := range page.Repositories {
if repository.Owner.ID != snapshot.OwnerID {
pageErr = errors.New("repository owner identity changed during enumeration")
break
}
}
}
if pageErr != nil {
cursor.Revisit++
checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, used, nil, "", canonicalLogin)
if err != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, err)
}
return deferOfficialOwnerError(ctx, control, task, checkpoint, pageErr)
}
candidates := toCandidates(page.Repositories)
if cursor.Offset > len(candidates) {
cursor.Offset = 0
cursor.Revisit++
}
batch, end := selectDiscoveryBatch(candidates, cursor.Offset, int(min(snapshot.Budget.CandidateBudget-spent, 100)), seen)
next, nextCursor := progress, cursor
nextCursor.Offset = end
gapReason := ""
if end == len(candidates) {
if page.HasMore {
if progress.NextPage == math.MaxInt32 {
next.Coverage, gapReason = "GAP", "RESULT_LIMIT"
} else {
next.NextPage++
nextCursor.Offset = 0
}
} else {
paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint)
if paused || err != nil {
return err
}
verified, _, verifyErr := upstream.GetOwnerByID(ctx, snapshot.OwnerID)
requests++
used++
if verifyErr == nil && (verified.ID != owner.ID || verified.Type != owner.Type || !strings.EqualFold(verified.Login, owner.Login)) {
verifyErr = errors.New("owner identity changed before final page confirmation")
}
if verifyErr != nil {
// Retain the page; no candidate or completion from an unconfirmed tail is
// committed. Retry resolves the canonical login again using the stable ID.
cursor.Revisit++
checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, used, nil, "", canonicalLogin)
if err != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, err)
}
return deferOfficialOwnerError(ctx, control, task, checkpoint, verifyErr)
}
next.Coverage = "COMPLETE"
}
}
provenance := controlplane.OfficialProvenance{Channel: snapshot.Channel, QueryID: snapshot.QueryID,
QueryVersion: snapshot.QueryVersion, StrategyVersion: snapshot.StrategyVersion, ShardID: snapshot.ShardID,
SeedRepositoryID: snapshot.SeedRepositoryID}
for i := range batch {
batch[i].OfficialProvenance = []controlplane.OfficialProvenance{provenance}
}
accepted, reportErr := reportOfficialProgress(ctx, control, task, checkpoint, next, nextCursor, batch, used, nil, gapReason, canonicalLogin)
if reportErr != nil {
return handleOfficialBatchFailure(ctx, control, task, checkpoint, reportErr)
}
checkpoint, progress, cursor = accepted, next, nextCursor
for _, candidate := range batch {
seen[candidate.RepositoryID] = struct{}{}
}
spent += int64(len(batch))
if progress.Coverage == "COMPLETE" {
_, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", snapshot.Channel)
return err
}
if progress.Coverage == "GAP" {
return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialGapBackoff(cursor.Revisit)))
}
}
}
func deferOfficialOwnerError(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
checkpoint string, acquisitionErr error) error {
reason, retryAt := "BUDGET", time.Now().Add(time.Minute)
var apiError *github.APIError
if errors.As(acquisitionErr, &apiError) && apiError.Code == "GITHUB_RATE_LIMITED" {
reason = "RATE_LIMIT"
if apiError.RetryAt.After(retryAt) {
retryAt = apiError.RetryAt
}
}
return completeOfficialDeferred(ctx, control, task, checkpoint, reason, retryAt)
}

68
github-sync-go/internal/worker/official_process_interop_test.go Normal file
View file

@ -0,0 +1,68 @@
package worker
import (
"context"
"encoding/json"
"os"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
// This test is compiled and launched as a separate OS process by the Java
// PostgreSQL interop test. Only GitHub is a fixture; control-plane HTTP is real.
func TestInteropOfficialDiscoveryProcess(t *testing.T) {
phase := os.Getenv("SKILLHUB_INTEROP_OFFICIAL_PROCESS")
if phase == "" {
t.Skip("requires Java official discovery process fixture")
}
if phase != "before-kill" && phase != "resume" {
t.Fatal("invalid interop phase")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
if err := control.HeartbeatWithCapabilities(context.Background(), "interop-official-"+phase, supportedCapabilities()); err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-official-"+phase, "DISCOVERY", supportedCapabilities())
if err != nil {
t.Fatal(err)
}
if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || task.OfficialDiscovery == nil || task.Credential != nil {
t.Fatal("Java did not return expected anonymous official task")
}
if phase == "resume" {
var cp controlplane.OfficialCheckpoint
if json.Unmarshal([]byte(task.OfficialDiscovery.Checkpoint), &cp) != nil || cp.NextPage != 2 || cp.Coverage != "IN_PROGRESS" || task.Lease.Generation < 2 {
t.Fatal("claim lost persisted checkpoint or generation")
}
} else if task.OfficialDiscovery.Checkpoint != "" {
t.Fatal("first process received unexpected prior progress")
}
fixture := &officialFixture{search: func(page int) (github.SearchPage, error) {
if phase == "before-kill" && page == 2 {
if err := os.WriteFile(os.Getenv("SKILLHUB_INTEROP_BARRIER"), []byte("PAGE_TWO_AFTER_ACK"), 0600); err != nil {
t.Fatal(err)
}
select {} // Java kills this exact executable, including its renewal goroutine.
}
if (phase == "before-kill" && page != 1) || (phase == "resume" && page != 2) {
t.Fatalf("unexpected %s search page %d", phase, page)
}
repository := officialRepo(987654900+int64(page), 42)
repository.DefaultBranch = "main"
return github.SearchPage{Repositories: []github.Repository{repository}, HasMore: page == 1}, nil
}}
err = executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processOfficialDiscovery(ctx, control, task, DiscoveryLimits{MaxPagesPerLease: 10}, fixture)
})
if err != nil {
t.Fatal(err)
}
if phase != "resume" || fixture.searches != 1 {
t.Fatal("process did not resume exclusively from second page")
}
}

192
github-sync-go/internal/worker/official_quota.go Normal file
View file

@ -0,0 +1,192 @@
package worker
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"sync"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
// A single bounded process gate prevents token rotation from bypassing secondary
// cooldown or a failed persistence attempt. Primary limits are scoped to the credential revision and resource.
var officialProcessQuota = struct {
sync.Mutex
globalUntil time.Time
resourceUntil map[string]time.Time
failures int
lastFailure time.Time
}{resourceUntil: make(map[string]time.Time)}
type officialRequestPolicy interface {
SetRequestPolicy(func(context.Context) error, func(context.Context, github.Quota, error) error)
}
type officialQuotaGuard struct {
control *controlplane.Client
task *controlplane.ClaimedTask
resource string
stopped error
}
func installOfficialQuotaGuard(control *controlplane.Client, task *controlplane.ClaimedTask, upstream any) (*officialQuotaGuard, error) {
client, ok := upstream.(officialRequestPolicy)
if !ok {
return nil, errors.New("official GitHub client lacks quota request policy")
}
resource := "CORE"
if task.OfficialDiscovery != nil && task.OfficialDiscovery.Channel != "OWNER_EXPANSION" {
resource = "SEARCH"
}
guard := &officialQuotaGuard{control: control, task: task, resource: resource}
client.SetRequestPolicy(guard.before, guard.after)
return guard, nil
}
func (g *officialQuotaGuard) before(ctx context.Context) error {
if ctx.Err() != nil {
return ctx.Err()
}
if g.stopped != nil {
return g.stopped
}
officialProcessQuota.Lock()
blocked := time.Now().Before(officialProcessQuota.globalUntil) || time.Now().Before(officialProcessQuota.resourceUntil[g.quotaKey(g.resource)]) || time.Now().Before(officialProcessQuota.resourceUntil[g.quotaKey("*")])
officialProcessQuota.Unlock()
if blocked {
g.stopped = errors.New("official GitHub requests are in process quota cooldown")
}
return g.stopped
}
func (g *officialQuotaGuard) after(ctx context.Context, quota github.Quota, requestErr error) error {
return g.observe(ctx, quota, requestErr, g.resource)
}
func (g *officialQuotaGuard) afterGit(ctx context.Context, requestErr error) error {
// Git HTTPS exposes no REST quota headers. UNKNOWN is not a fabricated CORE
// observation; a failed transport requests a conservative shared cooldown.
return g.observe(ctx, github.Quota{Remaining: -1}, requestErr, "OTHER")
}
func (g *officialQuotaGuard) beforeGit(ctx context.Context) error {
if err := g.before(ctx); err != nil {
return err
}
// Another worker may have persisted a deployment-wide cooldown while this
// package was admitted. Finish local delivery, but do not fetch more blobs.
control, err := g.control.GetOfficialTaskControl(ctx, g.task.TaskID, g.task.Lease)
if err != nil {
g.stopped = err
} else if control.Action == "PAUSE" && control.Reason == "QUOTA_COOLDOWN" {
g.stopped = errors.New("official Git acquisition is in quota cooldown")
}
return g.stopped
}
func (g *officialQuotaGuard) observe(ctx context.Context, observed github.Quota, requestErr error, resource string) error {
now := time.Now().UTC()
observation := controlplane.OfficialQuotaObservation{Resource: resource, ObservedAt: now, State: "UNKNOWN"}
quota := credentialQuota(observed)
if quota != nil && quota.ResetAt.Before(now.Add(24*time.Hour)) {
observation.Resource, observation.State = quota.Resource, "OBSERVED"
observation.Remaining, observation.ResetAt = &quota.Remaining, &quota.ResetAt
}
var upstreamError *github.APIError
var until time.Time
global := false
if errors.As(requestErr, &upstreamError) && upstreamError.Code == "AUTH_INVALID" {
observation.State, until = "AUTH_INVALID", now.Add(time.Minute)
global = g.task == nil || g.task.Credential == nil || g.task.Credential.Token == ""
} else if errors.As(requestErr, &upstreamError) && upstreamError.Code == "GITHUB_RATE_LIMITED" {
if quota != nil && quota.Remaining == 0 && quota.ResetAt.After(now) && quota.ResetAt.Before(now.Add(24*time.Hour)) && !upstreamError.SecondaryRateLimit {
observation.State, until = "PRIMARY_EXHAUSTED", quota.ResetAt
} else {
observation.State, until, global = "SECONDARY_COOLDOWN", upstreamError.RetryAt, true
if !until.After(now) {
until = now.Add(time.Minute)
}
if until.After(now.Add(24 * time.Hour)) {
until = now.Add(24 * time.Hour)
}
}
} else if quota != nil && observation.State == "OBSERVED" && quota.Remaining == 0 && quota.ResetAt.After(now) {
observation.State, until = "PRIMARY_EXHAUSTED", quota.ResetAt
} else if requestErr != nil && observation.State == "UNKNOWN" {
until, global = now.Add(time.Minute), true
}
if !until.IsZero() {
seconds := int(until.Sub(now).Seconds())
if seconds < 1 {
seconds = 1
}
observation.RetryAfterSeconds = &seconds
key := g.quotaKey(observation.Resource)
if observation.State == "AUTH_INVALID" {
key = g.quotaKey("*")
}
rememberOfficialCooldown(key, until, global)
}
// Missing headers always retain nil balances and reset time, including Git.
reportCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := g.control.ReportOfficialQuota(reportCtx, g.task.TaskID, g.task.Lease, observation); err != nil {
rememberOfficialPersistenceFailure(now)
g.stopped = errors.New("official quota persistence failed; GitHub requests stopped")
return g.stopped
}
if !until.IsZero() {
g.stopped = errors.New("official GitHub quota requires cooldown")
if requestErr != nil {
return requestErr
}
return g.stopped
}
return nil
}
// Token digests are process-local keys, never persisted or logged. A rotated token
// must not inherit the previous token's primary exhaustion. Anonymous requests
// share one key; different tokens are not assumed to have independent balances.
func (g *officialQuotaGuard) quotaKey(resource string) string {
if g.task == nil || g.task.Credential == nil || g.task.Credential.Token == "" {
return "anonymous:" + resource
}
digest := sha256.Sum256([]byte(g.task.Credential.Token))
return hex.EncodeToString(digest[:]) + ":" + resource
}
func rememberOfficialCooldown(resource string, until time.Time, global bool) {
officialProcessQuota.Lock()
defer officialProcessQuota.Unlock()
for key, expiry := range officialProcessQuota.resourceUntil {
if !expiry.After(time.Now()) {
delete(officialProcessQuota.resourceUntil, key)
}
}
if global {
if until.After(officialProcessQuota.globalUntil) {
officialProcessQuota.globalUntil = until
}
} else if until.After(officialProcessQuota.resourceUntil[resource]) {
officialProcessQuota.resourceUntil[resource] = until
}
}
func rememberOfficialPersistenceFailure(now time.Time) {
officialProcessQuota.Lock()
defer officialProcessQuota.Unlock()
if now.Sub(officialProcessQuota.lastFailure) > 15*time.Minute {
officialProcessQuota.failures = 0
}
officialProcessQuota.failures = min(officialProcessQuota.failures+1, 5)
officialProcessQuota.lastFailure = now
until := now.Add(min(time.Minute*time.Duration(1<<uint(officialProcessQuota.failures-1)), 15*time.Minute))
if until.After(officialProcessQuota.globalUntil) {
officialProcessQuota.globalUntil = until
}
}

84
github-sync-go/internal/worker/official_quota_test.go Normal file
View file

@ -0,0 +1,84 @@
package worker
import (
"context"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestOfficialPrimaryCooldownIsCredentialAndResourceScoped(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
guard := func(token, resource string) *officialQuotaGuard {
task := officialTask(h.channel)
if token != "" {
task.Credential = &controlplane.GithubCredential{CredentialID: "same-slot", Token: token}
}
return &officialQuotaGuard{control: h.control, task: task, resource: resource}
}
a := guard("fixture-token-a", "SEARCH")
if err := a.after(context.Background(), github.Quota{Resource: "SEARCH", Remaining: 0, ResetAt: time.Now().Add(time.Hour)}, nil); err == nil {
t.Fatal("exhausted task should stop")
}
for _, tc := range []struct {
name, token, resource string
blocked bool
}{
{"same credential", "fixture-token-a", "SEARCH", true},
{"other resource", "fixture-token-a", "CORE", false},
{"replacement credential", "fixture-token-b", "SEARCH", false},
{"anonymous", "", "SEARCH", false},
} {
t.Run(tc.name, func(t *testing.T) {
if blocked := guard(tc.token, tc.resource).before(context.Background()) != nil; blocked != tc.blocked {
t.Fatalf("blocked=%t want %t", blocked, tc.blocked)
}
})
}
anonymous := guard("", "SEARCH")
_ = anonymous.after(context.Background(), github.Quota{Resource: "SEARCH", Remaining: 0, ResetAt: time.Now().Add(time.Hour)}, nil)
if guard("", "SEARCH").before(context.Background()) == nil {
t.Fatal("anonymous exhaustion not shared")
}
if guard("fixture-token-b", "SEARCH").before(context.Background()) != nil {
t.Fatal("anonymous exhaustion blocked authenticated credential")
}
secondary := guard("fixture-token-b", "SEARCH")
_ = secondary.after(context.Background(), github.Quota{Remaining: -1}, &github.APIError{Code: "GITHUB_RATE_LIMITED", SecondaryRateLimit: true, RetryAt: time.Now().Add(time.Minute)})
if guard("fixture-token-c", "CORE").before(context.Background()) == nil {
t.Fatal("rotation bypassed shared secondary cooldown")
}
}
func TestOfficialCooldownPrunesExpiredCredentialKeys(t *testing.T) {
resetOfficialQuota(t)
officialProcessQuota.resourceUntil["expired"] = time.Now().Add(-time.Minute)
rememberOfficialCooldown("active", time.Now().Add(time.Minute), false)
if _, present := officialProcessQuota.resourceUntil["expired"]; present {
t.Fatal("expired credential retained")
}
if len(officialProcessQuota.resourceUntil) != 1 {
t.Fatal("active cooldown lost")
}
}
func TestOfficialInvalidCredentialDoesNotBlockHealthyToken(t *testing.T) {
h := newOfficialHarness(t, "REPOSITORY_METADATA")
guard := func(token, resource string) *officialQuotaGuard {
task := officialTask(h.channel)
task.Credential = &controlplane.GithubCredential{CredentialID: "same-slot", Token: token}
return &officialQuotaGuard{control: h.control, task: task, resource: resource}
}
invalid := guard("invalid-fixture", "SEARCH")
if err := invalid.after(context.Background(), github.Quota{Remaining: -1}, &github.APIError{Code: "AUTH_INVALID"}); err == nil {
t.Fatal("invalid credential not stopped")
}
if guard("invalid-fixture", "CORE").before(context.Background()) == nil {
t.Fatal("invalid token reused for another resource")
}
if guard("healthy-fixture", "SEARCH").before(context.Background()) != nil {
t.Fatal("invalid token blocked a healthy replacement")
}
}

79
github-sync-go/internal/worker/official_repository.go Normal file
View file

@ -0,0 +1,79 @@
package worker
import (
"context"
"errors"
"slices"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
"github.com/iflytek/skillhub/github-sync-go/internal/repository"
)
func repositoryNeedsOfficialControl(task *controlplane.ClaimedTask) bool {
return slices.Contains(task.RequiredCapabilities, controlplane.OfficialPauseCapability)
}
// The claim's source ID is not authoritative: a candidate can become bound or
// gain an independent origin during acquisition. Java resolves current ownership.
func pauseOfficialRepositoryIfNeeded(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, completion *controlplane.RepositoryCompletion) (bool, error) {
if err := github.CheckAcquisitionControl(ctx); err != nil {
return false, err
}
if !repositoryNeedsOfficialControl(task) {
return false, nil
}
for attempts := 0; attempts < 3; attempts++ {
state, err := control.GetOfficialTaskControl(ctx, task.TaskID, task.Lease)
if err != nil || state.Action == "CONTINUE" {
return false, err
}
progress := controlplane.OfficialRepositoryProgress{
IncompleteSubtrees: completion.IncompleteSubtrees,
PathResults: completion.PathResults,
}
if completion.CommitSHA != "" {
progress.ObservedCommitSHA = &completion.CommitSHA
}
if completion.PathCheckpoint != "" {
progress.PathCheckpoint = &completion.PathCheckpoint
}
// Accepted packages remain in Java's ingestion records. We only send
// actual local path outcomes, never manufacture ACCEPTED receipt IDs.
err = control.PauseOfficialRepository(ctx, task.TaskID, task.Lease, progress)
if err == nil {
return true, nil
}
var apiError *controlplane.APIError
if !errors.As(err, &apiError) || apiError.Code != "CONTROL_CHANGED" {
return false, err
}
}
return false, errors.New("official repository control changed repeatedly")
}
// A tree walk can issue multiple requests after a truncated response. Guard
// each request, and cancel the walk after control ends the task so enumeration
// cannot swallow a control error and proceed to acquire packages.
type officialRepositoryTreeReader struct {
reader repository.TreeReader
before func(context.Context) (bool, error)
stop context.CancelFunc
stopped bool
err error
}
func (r *officialRepositoryTreeReader) GetTree(ctx context.Context, owner, name, sha string, recursive bool) (github.Tree, github.Quota, error) {
if !r.stopped {
paused, err := r.before(ctx)
if paused || err != nil {
r.stopped, r.err = true, err
r.stop()
}
}
if r.stopped {
return github.Tree{}, github.Quota{}, context.Canceled
}
return r.reader.GetTree(ctx, owner, name, sha, recursive)
}

205
github-sync-go/internal/worker/platform.go Normal file
View file

@ -0,0 +1,205 @@
package worker
import (
"context"
"errors"
"sync"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
var errPlatformPaused = errors.New("platform collection paused")
type platformContextKey struct{}
// The monitor only cancels acquisition. The execution goroutine acknowledges
// stopping after all work and cleanup have returned.
type platformExecution struct {
control *controlplane.Client
task *controlplane.ClaimedTask
cancel context.CancelFunc
mu sync.Mutex
paused bool
delivering bool
observeErr error
checkpoint string
completion *controlplane.RepositoryCompletion
}
func (p *platformExecution) before(ctx context.Context) error {
p.mu.Lock()
defer p.mu.Unlock()
if p.paused {
return errPlatformPaused
}
if p.observeErr != nil {
return p.observeErr
}
state, err := p.control.GetPlatformControl(ctx, p.task.TaskID, p.task.Lease)
if err != nil {
p.observeErr = err
p.cancel()
return err
}
if state.Action == "PAUSE" {
p.paused = true
if !p.delivering {
p.cancel()
}
return errPlatformPaused
}
return nil
}
// A fully built, already-admitted artifact can settle under its live lease.
// Only platform pause cancellation is deferred; shutdown and lease fencing still
// cancel the parent context and therefore the upload.
func beginPlatformDelivery(ctx context.Context) error {
state, ok := ctx.Value(platformContextKey{}).(*platformExecution)
if !ok {
return ctx.Err()
}
state.mu.Lock()
defer state.mu.Unlock()
if state.paused {
return errPlatformPaused
}
if state.observeErr != nil {
return state.observeErr
}
if err := ctx.Err(); err != nil {
return err
}
state.delivering = true
return nil
}
func endPlatformDelivery(ctx context.Context) {
state, ok := ctx.Value(platformContextKey{}).(*platformExecution)
if !ok {
return
}
state.mu.Lock()
defer state.mu.Unlock()
state.delivering = false
if state.paused {
state.cancel()
}
}
func runPlatformTask(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask,
process func(context.Context) error) error {
if task.PlatformControlVersion == nil {
return process(ctx)
}
work, cancel := context.WithCancel(ctx)
defer cancel()
state := &platformExecution{control: control, task: task, cancel: cancel}
if task.Repository != nil {
state.completion = &controlplane.RepositoryCompletion{
CommitSHA: task.Repository.FixedCommitSHA,
PathCheckpoint: task.Repository.PathCheckpoint,
IncompleteSubtrees: append([]string(nil), task.Repository.IncompleteSubtrees...),
}
}
if task.Discovery != nil {
state.checkpoint = task.Discovery.Checkpoint
}
if task.OfficialDiscovery != nil {
state.checkpoint = task.OfficialDiscovery.Checkpoint
}
work = context.WithValue(work, platformContextKey{}, state)
work = github.WithAcquisitionControl(work, state.before)
work = controlplane.WithCheckpointObserver(work, func(value string) { state.checkpoint = value })
monitor, stop := context.WithCancel(ctx)
done := make(chan struct{})
go func() {
defer close(done)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-monitor.Done():
return
case <-ticker.C:
if state.before(monitor) != nil {
return
}
}
}
}()
err := state.before(work)
if err == nil {
err = process(work)
}
stop()
<-done
// Use the lease-renewal context, not the cancelled acquisition context.
// A pause may have captured the proof while normal completion was settling.
acknowledged, ackErr := state.acknowledge(ctx)
if ackErr != nil {
return ackErr
}
if acknowledged {
return nil
}
if errors.Is(err, errPlatformPaused) || errors.Is(err, context.Canceled) && ctx.Err() == nil {
// A rapid resume invalidated the pause epoch. Release via the normal retry
// report, retaining Java's confirmed progress; never restart stale local work.
return errors.New("platform control changed during acquisition")
}
return err
}
func (p *platformExecution) acknowledge(ctx context.Context) (bool, error) {
for attempt := 0; attempt < 3; attempt++ {
state, err := p.control.GetPlatformControl(ctx, p.task.TaskID, p.task.Lease)
var api *controlplane.APIError
if errors.As(err, &api) && api.Code == "LEASE_STALE" {
return false, nil
}
if err != nil {
return false, err
}
if state.Action != "PAUSE" {
return false, nil
}
request := controlplane.PlatformPause{Lease: p.task.Lease, ObservedVersion: state.PlatformVersion, Kind: p.task.Kind}
if p.task.Kind == "DISCOVERY" {
if p.checkpoint != "" {
request.DiscoveryCheckpoint = &p.checkpoint
}
} else {
progress := &controlplane.OfficialRepositoryProgress{IncompleteSubtrees: []string{}, PathResults: []controlplane.RepositoryPathResult{}}
if p.completion != nil {
if p.completion.CommitSHA != "" {
progress.ObservedCommitSHA = &p.completion.CommitSHA
}
if p.completion.PathCheckpoint != "" {
progress.PathCheckpoint = &p.completion.PathCheckpoint
}
progress.IncompleteSubtrees = append(progress.IncompleteSubtrees, p.completion.IncompleteSubtrees...)
progress.PathResults = append(progress.PathResults, p.completion.PathResults...)
}
request.RepositoryProgress = progress
}
err = p.control.PausePlatform(ctx, p.task.TaskID, request)
if err == nil {
return true, nil
}
if !errors.As(err, &api) {
// The ACK may have committed before its response was lost. Read the
// current epoch again and repeat the idempotent stop confirmation.
if ctx.Err() != nil {
return false, ctx.Err()
}
continue
}
if api.Code != "CONTROL_CHANGED" {
return false, err
}
}
return false, errors.New("platform control changed repeatedly")
}

267
github-sync-go/internal/worker/platform_test.go Normal file
View file

@ -0,0 +1,267 @@
package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
func TestPlatformPauseAcknowledgesOnlyAfterWorkCleanup(t *testing.T) {
for _, kind := range []string{"DISCOVERY", "REPOSITORY_SYNC"} {
t.Run(kind, func(t *testing.T) {
var paused, cleaned, acked atomic.Bool
var received controlplane.PlatformPause
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-control") {
action := "CONTINUE"
if paused.Load() {
action = "PAUSE"
}
_ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 2})
return
}
if strings.HasSuffix(r.URL.Path, "/platform-pause") {
if !cleaned.Load() {
t.Error("ack before acquisition cleanup")
}
_ = json.NewDecoder(r.Body).Decode(&received)
acked.Store(true)
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
return
}
t.Errorf("unexpected request %s", r.URL.Path)
w.WriteHeader(500)
}))
defer server.Close()
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
task := &controlplane.ClaimedTask{TaskID: "fixture", Kind: kind, PlatformControlVersion: &version, Discovery: &controlplane.DiscoverySnapshot{Checkpoint: "confirmed"}}
err := runPlatformTask(context.Background(), client, task, func(ctx context.Context) error {
defer cleaned.Store(true)
if kind == "REPOSITORY_SYNC" {
ctx.Value(platformContextKey{}).(*platformExecution).completion = &controlplane.RepositoryCompletion{CommitSHA: strings.Repeat("a", 40), PathCheckpoint: "received"}
}
paused.Store(true)
return github.CheckAcquisitionControl(ctx)
})
if err != nil || !acked.Load() {
t.Fatalf("pause=%v ack=%t", err, acked.Load())
}
if received.ObservedVersion != 2 || received.Kind != kind {
t.Fatalf("wrong request %+v", received)
}
if kind == "DISCOVERY" && (received.DiscoveryCheckpoint == nil || *received.DiscoveryCheckpoint != "confirmed") {
t.Fatal("lost checkpoint")
}
if kind == "REPOSITORY_SYNC" && (received.RepositoryProgress == nil || *received.RepositoryProgress.PathCheckpoint != "received") {
t.Fatal("lost repository progress")
}
})
}
}
func TestPlatformControlDoesNotPollTeam(t *testing.T) {
called := false
err := runPlatformTask(context.Background(), nil, &controlplane.ClaimedTask{}, func(context.Context) error { called = true; return nil })
if err != nil || !called {
t.Fatal(err)
}
}
func TestPlatformFinalControlAcknowledgesNormalAndOfficialSettlement(t *testing.T) {
for _, result := range []error{nil, errors.New("acquisition failed")} {
var calls atomic.Int32
var acknowledgements atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-pause") {
acknowledgements.Add(1)
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
return
}
action := "CONTINUE"
if calls.Add(1) > 1 {
action = "PAUSE"
}
_ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 2})
}))
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { return result })
server.Close()
if err != nil || acknowledgements.Load() != 1 {
t.Fatalf("terminal ack: %v", err)
}
}
}
func TestPlatformChangedEpochNeverAcknowledgesNewRunningState(t *testing.T) {
var controls, acks atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-pause") {
acks.Add(1)
w.WriteHeader(409)
_, _ = w.Write([]byte(`{"code":"CONTROL_CHANGED"}`))
return
}
action := "PAUSE"
if controls.Add(1) > 2 {
action = "CONTINUE"
}
_ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 3})
}))
defer server.Close()
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
err := runPlatformTask(ctx, client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { t.Fatal("paused work started"); return nil })
if err == nil || acks.Load() != 1 {
t.Fatalf("epoch handling err=%v acks=%d", err, acks.Load())
}
}
// Runs against the real Java HTTP server launched by the platform-control PG test.
// No GitHub network or credentials are used; the acquisition waits until the
// production control monitor cancels it, then the production executor sends ACK.
func TestPlatformJavaInterop(t *testing.T) {
base := os.Getenv("SKILLHUB_PLATFORM_INTEROP_URL")
if base == "" {
t.Skip("Java interoperability fixture not supplied")
}
client, err := controlplane.NewClient(base, os.Getenv("SKILLHUB_PLATFORM_INTEROP_TOKEN"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
task, err := client.ClaimWithCapabilities(ctx, "platform-interop", "DISCOVERY", []string{controlplane.PlatformPauseCapability})
if err != nil || task == nil {
t.Fatalf("real Java claim: %v", err)
}
if task.Discovery.Checkpoint != "confirmed-page" {
t.Fatal("Java claim lost persisted checkpoint")
}
if task.PlatformControlVersion == nil {
t.Fatal("platform claim missing control version")
}
err = executeLeasedTask(ctx, client, task, func(work context.Context) error {
if err := os.WriteFile(os.Getenv("SKILLHUB_PLATFORM_INTEROP_READY"), []byte("ready"), 0600); err != nil {
return err
}
<-work.Done()
return work.Err()
})
if err != nil {
t.Fatal(err)
}
}
func TestPlatformPauseBeforeRepositoryStartsUsesPersistedProgress(t *testing.T) {
var received controlplane.PlatformPause
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-control") {
_, _ = w.Write([]byte(`{"action":"PAUSE","platformVersion":2}`))
return
}
_ = json.NewDecoder(r.Body).Decode(&received)
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
}))
defer server.Close()
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
task := &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version,
Repository: &controlplane.RepositorySnapshot{FixedCommitSHA: strings.Repeat("a", 40), PathCheckpoint: "confirmed", IncompleteSubtrees: []string{"unknown"}}}
err := runPlatformTask(context.Background(), client, task, func(context.Context) error { t.Fatal("paused worker started"); return nil })
if err != nil {
t.Fatal(err)
}
if received.RepositoryProgress == nil || received.RepositoryProgress.ObservedCommitSHA == nil || *received.RepositoryProgress.ObservedCommitSHA != strings.Repeat("a", 40) || *received.RepositoryProgress.PathCheckpoint != "confirmed" || len(received.RepositoryProgress.IncompleteSubtrees) != 1 {
t.Fatalf("lost persisted progress: %+v", received.RepositoryProgress)
}
}
func TestPlatformPauseAllowsCompletedArtifactSettlementBeforeAck(t *testing.T) {
var paused, settled atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-control") {
action := "CONTINUE"
if paused.Load() {
action = "PAUSE"
}
_ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 2})
return
}
if !settled.Load() {
t.Error("ACK before completed artifact settlement")
}
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
}))
defer server.Close()
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version}, func(ctx context.Context) error {
if err := beginPlatformDelivery(ctx); err != nil {
return err
}
paused.Store(true)
state := ctx.Value(platformContextKey{}).(*platformExecution)
if err := state.before(ctx); !errors.Is(err, errPlatformPaused) {
t.Fatalf("expected pause: %v", err)
}
if ctx.Err() != nil {
t.Fatal("platform pause cancelled completed artifact settlement")
}
settled.Store(true)
endPlatformDelivery(ctx)
if ctx.Err() == nil {
t.Fatal("acquisition was not cancelled after settlement")
}
return ctx.Err()
})
if err != nil {
t.Fatal(err)
}
}
func TestPlatformAckResponseLossRetriesIdempotently(t *testing.T) {
var acknowledgements atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/platform-control") {
_, _ = w.Write([]byte(`{"action":"PAUSE","platformVersion":2}`))
return
}
if acknowledgements.Add(1) == 1 {
connection, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
_ = connection.Close()
return
}
_, _ = w.Write([]byte(`{"status":"PAUSED"}`))
}))
defer server.Close()
client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
version := int64(1)
err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { t.Fatal("paused work started"); return nil })
if err != nil || acknowledgements.Load() != 2 {
t.Fatalf("ack recovery=%v calls=%d", err, acknowledgements.Load())
}
}

164
github-sync-go/internal/worker/pool.go Normal file
View file

@ -0,0 +1,164 @@
package worker
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"sync"
"sync/atomic"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
type PoolConfig struct {
DiscoveryWorkers int
RepositoryWorkers int
PollInterval time.Duration
Discovery DiscoveryLimits
Repository RepositoryLimits
}
type Counters struct {
DiscoveryClaims atomic.Uint64
RepositoryClaims atomic.Uint64
DiscoveryErrors atomic.Uint64
RepositoryErrors atomic.Uint64
DiscoveryActive atomic.Int64
RepositoryActive atomic.Int64
DiscoveryReady atomic.Bool
RepositoryReady atomic.Bool
}
// supportedCapabilities describes the lifecycle implemented by the executable.
// Java configuration still determines whether official work is scheduled.
func supportedCapabilities() []string {
return []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability,
controlplane.OfficialPauseCapability, controlplane.OfficialOwnerCapability, controlplane.PlatformPauseCapability}
}
// Run starts separate bounded claim loops. Each loop claims only its task kind,
// so no lease waits in a queue behind work from the other pool.
func Run(ctx context.Context, control *controlplane.Client,
config PoolConfig, counters *Counters) error {
if control == nil || counters == nil || config.DiscoveryWorkers < 1 ||
config.RepositoryWorkers < 1 || config.DiscoveryWorkers > 32 ||
config.RepositoryWorkers > 32 || config.PollInterval < time.Second ||
config.Discovery.MaxPagesPerLease < 1 || config.Repository.MaxWorkspaceBytes < 1 {
return errors.New("invalid GitHub sync worker pool configuration")
}
instance := make([]byte, 6)
if _, err := rand.Read(instance); err != nil {
return err
}
instanceID := hex.EncodeToString(instance)
var running sync.WaitGroup
running.Add(1)
go func() {
defer running.Done()
for ctx.Err() == nil {
if err := control.HeartbeatWithCapabilities(ctx, "skillhub-"+instanceID, supportedCapabilities()); err != nil && ctx.Err() == nil {
slog.Warn("GitHub sync heartbeat failed", "code", safeErrorCode(err))
}
pause(ctx, 30*time.Second)
}
}()
start := func(kind string, count int) {
for index := range count {
running.Add(1)
workerID := fmt.Sprintf("skillhub-%s-%s-%d", instanceID, kind, index)
go func() {
defer running.Done()
claimLoop(ctx, control, config, counters, kind, workerID)
}()
}
}
start("DISCOVERY", config.DiscoveryWorkers)
start("REPOSITORY_SYNC", config.RepositoryWorkers)
running.Wait()
return ctx.Err()
}
func claimLoop(ctx context.Context, control *controlplane.Client, config PoolConfig,
counters *Counters, kind, workerID string) {
for ctx.Err() == nil {
task, err := control.ClaimWithCapabilities(ctx, workerID, kind, supportedCapabilities())
if err != nil {
setReady(counters, kind, false)
countError(counters, kind)
slog.Warn("GitHub sync claim failed", "kind", kind, "code", safeErrorCode(err))
pause(ctx, config.PollInterval)
continue
}
setReady(counters, kind, true)
if task == nil {
pause(ctx, config.PollInterval)
continue
}
countClaim(counters, kind)
countActive(counters, kind, 1)
if kind == "DISCOVERY" {
err = ExecuteDiscovery(ctx, control, task, config.Discovery)
} else {
err = ExecuteRepository(ctx, control, task, config.Repository)
}
countActive(counters, kind, -1)
if err != nil && ctx.Err() == nil {
countError(counters, kind)
slog.Warn("GitHub sync task did not finish in this lease",
"kind", kind, "task_id", task.TaskID, "code", safeErrorCode(err))
}
}
}
func pause(ctx context.Context, interval time.Duration) {
timer := time.NewTimer(interval)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func countClaim(counters *Counters, kind string) {
if kind == "DISCOVERY" {
counters.DiscoveryClaims.Add(1)
} else {
counters.RepositoryClaims.Add(1)
}
}
func countError(counters *Counters, kind string) {
if kind == "DISCOVERY" {
counters.DiscoveryErrors.Add(1)
} else {
counters.RepositoryErrors.Add(1)
}
}
func countActive(counters *Counters, kind string, delta int64) {
if kind == "DISCOVERY" {
counters.DiscoveryActive.Add(delta)
} else {
counters.RepositoryActive.Add(delta)
}
}
func setReady(counters *Counters, kind string, ready bool) {
if kind == "DISCOVERY" {
counters.DiscoveryReady.Store(ready)
} else {
counters.RepositoryReady.Store(ready)
}
}
func safeErrorCode(err error) string {
var apiError *controlplane.APIError
if errors.As(err, &apiError) {
return apiError.SafeCode()
}
return "WORKER_ERROR"
}

86
github-sync-go/internal/worker/pool_test.go Normal file
View file

@ -0,0 +1,86 @@
package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
)
// Run is the same entry point used by cmd/github-sync after loading its normal
// configuration. No test-only capability option is supplied to the pool.
func TestDefaultWorkerPoolAdvertisesOfficialLifecycle(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
observed := make(chan string, 16)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
WorkerID string `json:"workerId"`
Kind string `json:"kind"`
ProtocolMajor int `json:"protocolMajor"`
Capabilities []string `json:"capabilities"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
w.WriteHeader(400)
return
}
want := []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability, controlplane.OfficialOwnerCapability, controlplane.PlatformPauseCapability}
if len(body.Capabilities) != len(want) {
t.Errorf("capabilities=%v", body.Capabilities)
}
for _, capability := range want {
if !slices.Contains(body.Capabilities, capability) {
t.Errorf("missing %s", capability)
}
}
if !strings.HasPrefix(body.WorkerID, "skillhub-") {
t.Errorf("missing instance identity: %q", body.WorkerID)
}
if strings.HasSuffix(r.URL.Path, "/heartbeat") {
if body.ProtocolMajor != 1 {
t.Error("heartbeat omitted protocol major")
}
w.WriteHeader(204)
observed <- "heartbeat"
return
}
if !strings.HasSuffix(r.URL.Path, "/tasks/claim") {
t.Errorf("unexpected path %s", r.URL.Path)
}
w.WriteHeader(204)
observed <- body.Kind
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() {
done <- Run(ctx, control, PoolConfig{DiscoveryWorkers: 1, RepositoryWorkers: 1, PollInterval: time.Second, Discovery: DiscoveryLimits{MaxPagesPerLease: 10}, Repository: RepositoryLimits{MaxWorkspaceBytes: 1}}, &Counters{})
}()
seen := map[string]bool{}
for len(seen) < 3 {
select {
case kind := <-observed:
seen[kind] = true
case <-ctx.Done():
t.Fatal("default pool did not send heartbeat and both claims")
}
}
if !seen["heartbeat"] || !seen["DISCOVERY"] || !seen["REPOSITORY_SYNC"] {
t.Fatalf("unexpected pool requests %v", seen)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("pool shutdown: %v", err)
}
}

361
github-sync-go/internal/worker/repository.go Normal file
View file

@ -0,0 +1,361 @@
package worker
import (
"context"
"errors"
"fmt"
"sort"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
"github.com/iflytek/skillhub/github-sync-go/internal/repository"
)
type RepositoryLimits struct {
TempRoot string
AllowedS3Hosts []string
MaxTreeRequests int
MaxTreeEntries int
MaxWorkspaceBytes int64
MinFreeBytes int64
}
type repositoryGitHub interface {
repository.TreeReader
SetQuotaObserver(func(context.Context, github.Quota))
GetRepository(context.Context, string, string) (github.Repository, github.Quota, error)
GetHead(context.Context, github.Repository) (string, github.Quota, error)
GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error)
}
// ProcessRepository keeps all business progress in Java. A returned error means
// the caller should classify and report task failure while its lease is current.
func ProcessRepository(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits RepositoryLimits) error {
if task == nil {
return errors.New("invalid repository task or worker limits")
}
return processRepository(ctx, control, task, limits, github.NewClient(credentialToken(task)))
}
func processRepository(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, limits RepositoryLimits, upstream repositoryGitHub) error {
if task == nil || task.Kind != "REPOSITORY_SYNC" || task.Repository == nil ||
control == nil || upstream == nil ||
limits.MaxTreeRequests < 1 || limits.MaxTreeEntries < 1 ||
limits.MaxWorkspaceBytes < 1 || limits.MinFreeBytes < 0 || limits.TempRoot == "" {
return errors.New("invalid repository task or worker limits")
}
snapshot := task.Repository
completion := controlplane.RepositoryCompletion{
CommitSHA: snapshot.FixedCommitSHA, PathCheckpoint: snapshot.PathCheckpoint,
IncompleteSubtrees: append([]string(nil), snapshot.IncompleteSubtrees...),
}
if state, ok := ctx.Value(platformContextKey{}).(*platformExecution); ok {
state.completion = &completion
}
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
var quotaGuard *officialQuotaGuard
if repositoryNeedsOfficialControl(task) {
guard, err := installOfficialQuotaGuard(control, task, upstream)
if err != nil {
return err
}
quotaGuard = guard
ctx = repository.WithGitRequestPolicy(ctx, guard.beforeGit, guard.afterGit)
} else {
upstream.SetQuotaObserver(observeCredentialQuota(control, task))
}
metadata, _, err := upstream.GetRepository(ctx, snapshot.Owner, snapshot.Name)
if err != nil {
return err
}
if metadata.ID != snapshot.RepositoryID {
return errors.New("GitHub repository ID changed for task source")
}
if err := control.UpdateRepositoryMetadata(ctx, task.TaskID, task.Lease,
controlplane.RepositoryMetadata{RepositoryID: metadata.ID, OwnerID: metadata.Owner.ID,
Owner: metadata.Owner.Login, Name: metadata.Name,
DefaultBranch: metadata.DefaultBranch}); err != nil {
return err
}
if task.Mode == "HEAD_ONLY" {
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
head, _, err := upstream.GetHead(ctx, metadata)
if err != nil {
return err
}
if completion.CommitSHA == "" {
completion.CommitSHA = head
}
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
return control.CompleteHeadOnly(ctx, task.TaskID, task.Lease, head)
}
if task.Mode != "FULL" || task.MaxFilesPerPackage < 1 || task.MaxPackageBytes < 1 ||
task.MaxSingleFileBytes < 1 || task.MaxUploadBytes < 1 {
return errors.New("repository task has no valid package policy")
}
commitSHA := snapshot.FixedCommitSHA
if commitSHA == "" {
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
commitSHA, _, err = upstream.GetHead(ctx, metadata)
if err != nil {
return err
}
}
completion.CommitSHA = commitSHA
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
treeSHA, _, err := upstream.GetCommitTreeSHA(ctx, metadata.Owner.Login, metadata.Name, commitSHA)
if err != nil {
return err
}
var treeReader repository.TreeReader = upstream
treeContext := ctx
var guardedTree *officialRepositoryTreeReader
if repositoryNeedsOfficialControl(task) {
var cancelTree context.CancelFunc
treeContext, cancelTree = context.WithCancel(ctx)
defer cancelTree()
// An interrupted walk is replayed on resume; it is never complete coverage.
completion.IncompleteSubtrees = []string{"."}
guardedTree = &officialRepositoryTreeReader{reader: upstream, stop: cancelTree,
before: func(ctx context.Context) (bool, error) {
return pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion)
}}
treeReader = guardedTree
}
tree, treeErr := repository.EnumerateTree(treeContext, treeReader, metadata.Owner.Login,
metadata.Name, treeSHA, repository.TreeLimits{
MaxTreeRequests: limits.MaxTreeRequests, MaxEntries: limits.MaxTreeEntries,
})
if guardedTree != nil && guardedTree.stopped {
return guardedTree.err
}
if ctx.Err() != nil {
return ctx.Err()
}
if treeErr != nil && len(tree.Entries) == 0 && len(tree.IncompleteSubtrees) == 0 {
return treeErr
}
roots := repository.RootsFromSnapshot(tree, snapshot.IgnorePatterns)
preflight, err := repository.PreflightPackages(tree, roots.Paths, repository.PackageLimits{
MaxFiles: task.MaxFilesPerPackage, MaxSingleFileBytes: task.MaxSingleFileBytes,
MaxPackageBytes: task.MaxPackageBytes,
})
if err != nil {
return err
}
plans := make(map[string]repository.PackagePlan, len(preflight.Plans))
failures := make(map[string]repository.PackageFailure, len(preflight.Failures))
paths := make([]string, 0, len(preflight.Plans)+len(preflight.Failures))
for _, plan := range preflight.Plans {
plans[plan.Root] = plan
paths = append(paths, plan.Root)
}
for _, failure := range preflight.Failures {
failures[failure.Root] = failure
paths = append(paths, failure.Root)
}
retryPaths := make(map[string]struct{}, len(snapshot.RetryPaths))
acceptedPaths := make(map[string]struct{}, len(snapshot.AcceptedPaths))
for _, path := range snapshot.AcceptedPaths {
acceptedPaths[path] = struct{}{}
}
for _, path := range snapshot.RetryPaths {
retryPaths[path] = struct{}{}
if _, planned := plans[path]; !planned {
if _, failed := failures[path]; !failed && len(roots.IncompleteSubtrees) == 0 {
failures[path] = repository.PackageFailure{Root: path, Code: "SKILL_ROOT_NOT_FOUND"}
paths = append(paths, path)
}
}
}
sort.Strings(paths)
completion.IncompleteSubtrees = append([]string(nil), roots.IncompleteSubtrees...)
for _, path := range paths {
if ctx.Err() != nil {
return ctx.Err()
}
if skipPersistedPath(path, snapshot.PathCheckpoint, acceptedPaths, retryPaths) {
continue
}
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
if quotaGuard != nil {
if err := quotaGuard.before(ctx); err != nil {
return completeDeferred(ctx, control, task, &completion, "RATE_LIMIT", time.Now().Add(time.Minute))
}
}
if failure, failed := failures[path]; failed {
outcome := "TERMINAL_FAILURE"
if failure.Retryable {
outcome = "RETRYABLE_FAILURE"
}
completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{
SkillPath: path, Outcome: outcome, Reason: failure.Code,
})
advanceCheckpoint(&completion, path)
continue
}
admission, err := control.AdmitPackage(ctx, task.TaskID, task.Lease, commitSHA, path)
if err != nil {
var apiError *controlplane.APIError
if errors.As(err, &apiError) && apiError.Code == "BACKPRESSURE_ACTIVE" {
return completeDeferred(ctx, control, task, &completion, "BACKPRESSURE", time.Now().Add(15*time.Minute))
}
if errors.As(err, &apiError) && apiError.Code == "PATH_IGNORED" {
completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{
SkillPath: path, Outcome: "IGNORED", Reason: "SKILL_PATH_IGNORED",
})
advanceCheckpoint(&completion, path)
continue
}
return err
}
artifact, err := repository.AcquirePackage(ctx, metadata.Owner.Login, metadata.Name,
commitSHA, limits.TempRoot, plans[path], repository.AcquisitionLimits{
MaxWorkspaceBytes: limits.MaxWorkspaceBytes, MaxArchiveBytes: task.MaxUploadBytes,
MinFreeBytes: limits.MinFreeBytes,
})
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if errors.Is(err, repository.ErrCapacity) {
return completeDeferred(ctx, control, task, &completion, "CAPACITY", time.Now().Add(24*time.Hour))
}
if errors.Is(err, repository.ErrLocalDiskBudget) {
return completeDeferred(ctx, control, task, &completion, "LOCAL_DISK_BUDGET", time.Now().Add(15*time.Minute))
}
outcome := "RETRYABLE_FAILURE"
reason := "PACKAGE_ACQUISITION_FAILED"
if errors.Is(err, repository.ErrArchiveTooLarge) {
outcome, reason = "TERMINAL_FAILURE", "ARCHIVE_TOO_LARGE"
}
completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{
SkillPath: path, Outcome: outcome, Reason: reason,
})
advanceCheckpoint(&completion, path)
continue
}
if err := beginPlatformDelivery(ctx); err != nil {
if closeErr := artifact.Close(); closeErr != nil {
return fmt.Errorf("remove temporary package: %w", closeErr)
}
return err
}
err = deliverPackage(ctx, control, task, admission, commitSHA, path,
artifact, limits.AllowedS3Hosts)
closeErr := artifact.Close()
endPlatformDelivery(ctx)
if ctx.Err() != nil {
return ctx.Err()
}
if closeErr != nil {
return fmt.Errorf("remove temporary package: %w", closeErr)
}
if err != nil {
completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{
SkillPath: path, Outcome: "RETRYABLE_FAILURE", Reason: "PACKAGE_DELIVERY_FAILED",
})
}
advanceCheckpoint(&completion, path)
}
if treeErr != nil {
var apiError *github.APIError
if errors.As(treeErr, &apiError) && apiError.Code == "GITHUB_RATE_LIMITED" {
if reportErr := reportCredentialProblem(ctx, control, task, treeErr); reportErr != nil {
return reportErr
}
return completeDeferred(ctx, control, task, &completion, "RATE_LIMIT", futureRetry(apiError.RetryAt))
}
if len(completion.IncompleteSubtrees) == 0 {
completion.IncompleteSubtrees = []string{"."}
}
}
if len(completion.IncompleteSubtrees) > 0 || hasRetryable(completion.PathResults) {
return completeDeferred(ctx, control, task, &completion, "BUDGET", time.Now().Add(30*time.Minute))
}
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused {
return err
}
_, err = control.CompleteRepository(ctx, task.TaskID, task.Lease, completion)
return err
}
func skipPersistedPath(path, checkpoint string, acceptedPaths, retryPaths map[string]struct{}) bool {
if _, accepted := acceptedPaths[path]; accepted {
return true
}
_, retry := retryPaths[path]
return checkpoint != "" && path <= checkpoint && !retry
}
func deliverPackage(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, admission controlplane.PackageAdmission,
commitSHA, path string, artifact *repository.Artifact, allowedS3Hosts []string) error {
authorization, err := control.AuthorizeUpload(ctx, task.TaskID, task.Lease,
admission.AdmissionID, commitSHA, path, artifact.Size, artifact.SHA256)
if err != nil {
return err
}
if err := control.UploadPackage(ctx, authorization, artifact.Path, allowedS3Hosts); err != nil {
return err
}
_, err = control.ReportPackage(ctx, task.TaskID, task.Lease, admission.AdmissionID,
commitSHA, path, authorization, artifact.Size, artifact.SHA256)
return err
}
func advanceCheckpoint(completion *controlplane.RepositoryCompletion, path string) {
if completion.PathCheckpoint == "" || path > completion.PathCheckpoint {
completion.PathCheckpoint = path
}
}
func hasRetryable(results []controlplane.RepositoryPathResult) bool {
for _, result := range results {
if result.Outcome == "RETRYABLE_FAILURE" {
return true
}
}
return false
}
func completeDeferred(ctx context.Context, control *controlplane.Client,
task *controlplane.ClaimedTask, completion *controlplane.RepositoryCompletion,
reason string, retryAt time.Time) error {
if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, completion); err != nil || paused {
return err
}
completion.DeferralReason = reason
completion.RetryAt = futureRetry(retryAt)
_, err := control.CompleteRepository(ctx, task.TaskID, task.Lease, *completion)
return err
}
func futureRetry(retryAt time.Time) time.Time {
if retryAt.After(time.Now().Add(time.Minute)) {
return retryAt
}
return time.Now().Add(15 * time.Minute)
}
func credentialToken(task *controlplane.ClaimedTask) string {
if task.Credential == nil {
return ""
}
return task.Credential.Token
}

567
github-sync-go/internal/worker/repository_process_resume_test.go Normal file
View file

@ -0,0 +1,567 @@
package worker
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"os/signal"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type repositoryTreeFixture struct {
entries []github.TreeEntry
id int64
owner string
name string
}
type headOnlyRepositoryFixture struct {
*repositoryTreeFixture
treeCalls int
}
type selectedLiveRepositoryTree struct {
*github.Client
treeCalls int
rawEntries int
selectedEntries int
selectAll bool
}
func (reader *selectedLiveRepositoryTree) GetTree(ctx context.Context, owner, name, sha string,
recursive bool) (github.Tree, github.Quota, error) {
reader.treeCalls++
tree, quota, err := reader.Client.GetTree(ctx, owner, name, sha, recursive)
if err != nil || !recursive || tree.Truncated {
return tree, quota, err
}
reader.rawEntries = len(tree.Entries)
if reader.selectAll {
reader.selectedEntries = len(tree.Entries)
return tree, quota, nil
}
selected := tree.Entries[:0]
for _, entry := range tree.Entries {
if strings.HasPrefix(entry.Path, "skills/.system/skill-creator/") ||
strings.HasPrefix(entry.Path, "skills/.curated/define-goal/") {
selected = append(selected, entry)
}
}
tree.Entries = selected
reader.selectedEntries = len(selected)
return tree, quota, nil
}
func (fixture *headOnlyRepositoryFixture) GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error) {
fixture.treeCalls++
return "", github.Quota{}, errors.New("HEAD_ONLY must not inspect a commit tree")
}
func (fixture *headOnlyRepositoryFixture) GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error) {
fixture.treeCalls++
return github.Tree{}, github.Quota{}, errors.New("HEAD_ONLY must not enumerate a tree")
}
func (*repositoryTreeFixture) SetQuotaObserver(func(context.Context, github.Quota)) {}
func (fixture *repositoryTreeFixture) GetRepository(context.Context, string, string) (github.Repository, github.Quota, error) {
id, owner, name := fixture.id, fixture.owner, fixture.name
if id == 0 {
id, owner, name = 42, "fixture", "repo"
}
repo := github.Repository{ID: id, Name: name, DefaultBranch: "main"}
repo.Owner.Login = owner
return repo, github.Quota{}, nil
}
func (*repositoryTreeFixture) GetHead(context.Context, github.Repository) (string, github.Quota, error) {
return strings.Repeat("c", 40), github.Quota{}, nil
}
func (*repositoryTreeFixture) GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error) {
return strings.Repeat("b", 40), github.Quota{}, nil
}
func (fixture *repositoryTreeFixture) GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error) {
return github.Tree{Entries: fixture.entries}, github.Quota{}, nil
}
func TestProcessRepositorySkipsReceivedPackageAndRevisitsFailedPathBeforeCheckpoint(t *testing.T) {
var reports []map[string]any
var admissions int
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if strings.HasSuffix(request.URL.Path, "/repository-metadata") {
writer.WriteHeader(http.StatusOK)
return
}
if strings.HasSuffix(request.URL.Path, "/package-admissions") {
admissions++
writer.WriteHeader(http.StatusInternalServerError)
return
}
if !strings.HasSuffix(request.URL.Path, "/complete") {
writer.WriteHeader(http.StatusNotFound)
return
}
var report map[string]any
if err := json.NewDecoder(request.Body).Decode(&report); err != nil {
writer.WriteHeader(http.StatusBadRequest)
return
}
reports = append(reports, report)
writer.Header().Set("Content-Type", "application/json")
if report["coverage"] == "INCOMPLETE" {
_, _ = writer.Write([]byte(`{"status":"RETRY_WAIT"}`))
} else {
_, _ = writer.Write([]byte(`{"status":"PARTIAL"}`))
}
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
commit := strings.Repeat("c", 40)
task := &controlplane.ClaimedTask{
TaskID: "fixture-task", Kind: "REPOSITORY_SYNC", Mode: "FULL",
Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)},
Repository: &controlplane.RepositorySnapshot{
RepositoryID: 42, Owner: "fixture", Name: "repo", FixedCommitSHA: commit,
PathCheckpoint: "skills/retry", AcceptedPaths: []string{"skills/accepted"},
RetryPaths: []string{"skills/retry"},
},
MaxFilesPerPackage: 10, MaxPackageBytes: 1024,
MaxSingleFileBytes: 1024, MaxUploadBytes: 2048,
}
limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20}
accepted := github.TreeEntry{Path: "skills/accepted/SKILL.md", Type: "blob",
Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10}
retry := github.TreeEntry{Path: "skills/retry/SKILL.md", Type: "blob",
Mode: "100644", SHA: strings.Repeat("e", 40), Size: -1}
if err := processRepository(context.Background(), control, task, limits,
&repositoryTreeFixture{entries: []github.TreeEntry{accepted, retry}}); err != nil {
t.Fatal(err)
}
if admissions != 0 || len(reports) != 1 || reports[0]["coverage"] != "INCOMPLETE" {
t.Fatalf("received package was reacquired or retry was not deferred: admissions=%d reports=%v",
admissions, reports)
}
retryAt, err := time.Parse(time.RFC3339Nano, reports[0]["retryAt"].(string))
if err != nil || !retryAt.After(time.Now()) {
t.Fatalf("retryAt was not a future instant: %v, %v", reports[0]["retryAt"], err)
}
assertOnlyPathResult(t, reports[0], "skills/retry", "RETRYABLE_FAILURE")
// On the next lease the failing path has disappeared from the fixed tree.
// It must still be revisited despite being at the persisted checkpoint.
if err := processRepository(context.Background(), control, task, limits,
&repositoryTreeFixture{entries: []github.TreeEntry{accepted}}); err != nil {
t.Fatal(err)
}
if admissions != 0 || len(reports) != 2 || reports[1]["coverage"] != "COMPLETE" {
t.Fatalf("received package was reacquired or failed path was not resolved: admissions=%d reports=%v",
admissions, reports)
}
assertOnlyPathResult(t, reports[1], "skills/retry", "TERMINAL_FAILURE")
}
func TestProcessRepositoryDefersAtFixedCheckpointWhenOnePackageCannotFit(t *testing.T) {
var completion map[string]any
admissions := 0
uploads := 0
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
switch {
case strings.HasSuffix(request.URL.Path, "/repository-metadata"):
writer.WriteHeader(http.StatusOK)
case strings.HasSuffix(request.URL.Path, "/package-admissions"):
admissions++
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"admissionId":"capacity-fixture"}`))
case strings.HasSuffix(request.URL.Path, "/complete"):
if err := json.NewDecoder(request.Body).Decode(&completion); err != nil {
writer.WriteHeader(http.StatusBadRequest)
return
}
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"status":"RETRY_WAIT"}`))
default:
uploads++
writer.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
commit := strings.Repeat("c", 40)
task := &controlplane.ClaimedTask{
TaskID: "capacity-task", Kind: "REPOSITORY_SYNC", Mode: "FULL",
Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)},
Repository: &controlplane.RepositorySnapshot{
RepositoryID: 42, Owner: "fixture", Name: "repo", FixedCommitSHA: commit,
PathCheckpoint: "skills/a-previous", AcceptedPaths: []string{"skills/a-previous"},
},
MaxFilesPerPackage: 10, MaxPackageBytes: 1024,
MaxSingleFileBytes: 1024, MaxUploadBytes: 2048,
}
tempRoot := t.TempDir()
limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20}
entries := []github.TreeEntry{
{Path: "skills/a-previous/SKILL.md", Type: "blob", Mode: "100644",
SHA: strings.Repeat("d", 40), Size: 10},
{Path: "skills/z-next/SKILL.md", Type: "blob", Mode: "100644",
SHA: strings.Repeat("e", 40), Size: 10},
}
if err := processRepository(context.Background(), control, task, limits,
&repositoryTreeFixture{entries: entries}); err != nil {
t.Fatal(err)
}
if admissions != 1 || uploads != 0 || completion == nil ||
completion["coverage"] != "INCOMPLETE" || completion["deferralReason"] != "CAPACITY" ||
completion["pathCheckpoint"] != "skills/a-previous" {
t.Fatalf("capacity deferral lost the fixed checkpoint: admissions=%d uploads=%d completion=%v",
admissions, uploads, completion)
}
retryAt, err := time.Parse(time.RFC3339Nano, completion["retryAt"].(string))
if err != nil || retryAt.Before(time.Now().Add(23*time.Hour)) {
t.Fatalf("insufficient package capacity retried too soon: %v, %v", completion["retryAt"], err)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("capacity deferral left temporary files: %v, %v", remaining, err)
}
}
func assertOnlyPathResult(t *testing.T, report map[string]any, path, outcome string) {
t.Helper()
results, ok := report["pathResults"].([]any)
if !ok || len(results) != 1 {
t.Fatalf("unexpected path results: %v", report["pathResults"])
}
result, ok := results[0].(map[string]any)
if !ok || result["skillPath"] != path || result["outcome"] != outcome {
t.Fatalf("unexpected path result: %v", results[0])
}
}
// Opt-in cross-service check. Java owns the task and retry clock; this test runs
// the actual repository worker with a controlled GitHub tree in a separate Go process.
func TestInteropProcessRepositoryRecovery(t *testing.T) {
phase := os.Getenv("SKILLHUB_INTEROP_WORKER_PHASE")
if phase == "" {
t.Skip("requires an HTTP interoperability fixture")
}
if phase != "retry" && phase != "resolve" {
t.Fatal("invalid HTTP interoperability worker phase")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-recovery-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Repository == nil || task.Repository.RepositoryID != 42 ||
task.Repository.FixedCommitSHA != strings.Repeat("c", 40) ||
len(task.Repository.AcceptedPaths) != 1 ||
task.Repository.AcceptedPaths[0] != "skills/.system/skill-creator" ||
len(task.Repository.RetryPaths) != 1 || task.Repository.RetryPaths[0] != "skills/retry" {
t.Fatal("Java did not return the expected persisted recovery paths")
}
entries := []github.TreeEntry{{Path: "skills/.system/skill-creator/SKILL.md",
Type: "blob", Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10}}
if phase == "retry" {
entries = append(entries, github.TreeEntry{Path: "skills/retry/SKILL.md",
Type: "blob", Mode: "100644", SHA: strings.Repeat("e", 40), Size: -1})
}
upstream := &repositoryTreeFixture{entries: entries, id: 42, owner: "openai", name: "skills"}
limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20}
if err := processRepository(context.Background(), control, task, limits, upstream); err != nil {
t.Fatal(err)
}
}
// The real Java API grants admission; Go must return a delayed capacity result
// before Git can fetch any blob, and leave no package directory behind.
func TestInteropProcessRepositoryCapacity(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_WORKER_CAPACITY") != "1" {
t.Skip("requires a Java HTTP capacity fixture")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-capacity-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Repository == nil || task.Repository.RepositoryID != 987654330 ||
task.Repository.FixedCommitSHA != strings.Repeat("c", 40) || task.Mode != "FULL" {
t.Fatal("Java did not lease the expected fixed-commit repository task")
}
tempRoot := t.TempDir()
entries := []github.TreeEntry{{Path: "skills/fixture/SKILL.md", Type: "blob",
Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10}}
upstream := &repositoryTreeFixture{entries: entries, id: 987654330,
owner: "openai", name: "skills"}
limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processRepository(ctx, control, task, limits, upstream)
}); err != nil {
t.Fatal(err)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("capacity deferral left temporary files: %v, %v", remaining, err)
}
}
// Java supplies the lease and package admission; the Tree fixture isolates the
// REST quota while Git fetches two real Skill directories from one fixed commit.
func TestInteropProcessRepositoryPublicMultiSkill(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_WORKER_PUBLIC_MULTI") != "1" {
t.Skip("requires a Java HTTP lease and public Git access")
}
const commitSHA = "49f948faa9258a0c61caceaf225e179651397431"
realTree := os.Getenv("SKILLHUB_INTEROP_WORKER_REAL_TREE") == "1"
allTree := os.Getenv("SKILLHUB_INTEROP_WORKER_ALL_TREE") == "1"
interruptAfterReceipt := os.Getenv("SKILLHUB_INTEROP_WORKER_INTERRUPT_AFTER_RECEIPT") == "1"
if allTree && !realTree {
t.Fatal("full Tree interop requires the real GitHub Tree API")
}
repositoryID := int64(987654350)
if realTree {
repositoryID = 1103643573
}
javaURL := os.Getenv("SKILLHUB_INTEROP_JAVA_URL")
workContext := context.Background()
var interrupted atomic.Bool
if interruptAfterReceipt {
var stop context.CancelFunc
workContext, stop = signal.NotifyContext(workContext, syscall.SIGTERM)
defer stop()
proxy := interruptAfterPackageReceipt(t, javaURL, &interrupted)
defer proxy.Close()
javaURL = proxy.URL + "/internal/github-sync/v1"
}
control, err := controlplane.NewClient(javaURL,
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-public-multi-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Repository == nil || task.Repository.RepositoryID != repositoryID ||
task.Repository.FixedCommitSHA != commitSHA || task.Mode != "FULL" {
t.Fatal("Java did not lease the expected fixed-commit repository task")
}
if allTree && (task.MaxFilesPerPackage != 100 || task.MaxPackageBytes != 100<<20 ||
task.MaxUploadBytes != 100<<20) {
t.Fatal("full Tree lease did not use the current production package policy")
}
files := []struct {
path string
sha string
size int64
}{
{"skills/.system/skill-creator/LICENSE.txt", "d645695673349e3947e8e5ae42332d0ac3164cd7", 11358},
{"skills/.system/skill-creator/SKILL.md", "72bc0b97e7a6476254a9d5c424c9971748402ec3", 18664},
{"skills/.system/skill-creator/agents/openai.yaml", "f5ce09ed118b72908b7308fcfc1619f59e89f6ea", 192},
{"skills/.system/skill-creator/references/openai_yaml.md", "da5629f8de500ec6f81291d239a2d9c68d1d9044", 2130},
{"skills/.system/skill-creator/scripts/generate_openai_yaml.py", "1a9d784f8704999daf55a2b9036094949d9af3f8", 6614},
{"skills/.system/skill-creator/scripts/init_skill.py", "f90703eca8159a21a9684ed1df974c702d92f814", 14483},
{"skills/.system/skill-creator/scripts/quick_validate.py", "0547b4041a5f58fa19892079a114a1df98286406", 3293},
{"skills/.curated/define-goal/LICENSE.txt", "13e25df86ce06eb6488e6a6bc5c5847f5dedc352", 10776},
{"skills/.curated/define-goal/SKILL.md", "87f111bd700e0d993465f7ac741847b5daee57d6", 5647},
{"skills/.curated/define-goal/agents/openai.yaml", "28c7e7a644d7ae3f5616c7ddd408c573139999f0", 208},
}
entries := make([]github.TreeEntry, 0, len(files))
for _, file := range files {
entries = append(entries, github.TreeEntry{Path: file.path, Type: "blob",
Mode: "100644", SHA: file.sha, Size: file.size})
}
tempRoot := os.Getenv("SKILLHUB_INTEROP_WORKER_TEMP_ROOT")
if tempRoot == "" {
tempRoot = t.TempDir()
}
limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5,
MaxTreeEntries: 100, MaxWorkspaceBytes: 64 << 20, MinFreeBytes: 16 << 20}
if allTree {
limits.MaxTreeEntries = 5000
}
if host := os.Getenv("SKILLHUB_INTEROP_ALLOWED_S3_HOST"); host != "" {
limits.AllowedS3Hosts = []string{host}
}
var upstream repositoryGitHub = &repositoryTreeFixture{entries: entries, id: repositoryID,
owner: "openai", name: "skills"}
var liveReader *selectedLiveRepositoryTree
if realTree {
token := os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TOKEN")
if token == "" {
lookupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
output, lookupErr := exec.CommandContext(lookupCtx, "gh", "auth", "token").Output()
cancel()
if lookupErr != nil {
t.Fatal("real Tree mode requires a configured GitHub test token or local gh authentication")
}
token = strings.TrimSpace(string(output))
}
if token == "" {
t.Fatal("real Tree mode has no GitHub test token")
}
liveReader = &selectedLiveRepositoryTree{Client: github.NewClient(token), selectAll: allTree}
upstream = liveReader
}
err = executeLeasedTask(workContext, control, task, func(ctx context.Context) error {
return processRepository(ctx, control, task, limits, upstream)
})
if interruptAfterReceipt {
if !interrupted.Load() || !errors.Is(err, context.Canceled) {
t.Fatalf("worker did not stop after Java accepted the first package: interrupted=%v error=%v",
interrupted.Load(), err)
}
t.Log("interrupted after Java package receipt")
} else if err != nil {
t.Fatal(err)
}
if realTree {
if liveReader.treeCalls != 1 || liveReader.rawEntries < 1000 ||
liveReader.selectedEntries < len(entries) {
t.Fatalf("real Tree API did not provide the expected complete metadata: calls=%d raw=%d selected=%d",
liveReader.treeCalls, liveReader.rawEntries, liveReader.selectedEntries)
}
t.Logf("real Tree API: tree_calls=%d raw_entries=%d selected_entries=%d",
liveReader.treeCalls, liveReader.rawEntries, liveReader.selectedEntries)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 {
t.Fatalf("multi-Skill acquisition left temporary files: %v, %v", remaining, err)
}
}
func interruptAfterPackageReceipt(t *testing.T, javaURL string, interrupted *atomic.Bool) *httptest.Server {
t.Helper()
target, err := url.Parse(javaURL)
if err != nil || target.Scheme != "http" || target.Host == "" {
t.Fatal("interruption fixture requires a local Java HTTP endpoint")
}
client := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{Proxy: nil}}
return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
forward := request.Clone(request.Context())
forward.URL.Scheme = target.Scheme
forward.URL.Host = target.Host
forward.Host = target.Host
forward.RequestURI = ""
response, err := client.Do(forward)
if err != nil {
writer.WriteHeader(http.StatusBadGateway)
return
}
defer response.Body.Close()
for key, values := range response.Header {
writer.Header()[key] = values
}
writer.WriteHeader(response.StatusCode)
if strings.HasSuffix(request.URL.Path, "/packages/report") &&
response.StatusCode >= 200 && response.StatusCode < 300 &&
interrupted.CompareAndSwap(false, true) {
_ = syscall.Kill(os.Getpid(), syscall.SIGTERM)
}
_, _ = io.Copy(writer, response.Body)
}))
}
func TestInteropProcessRepositoryHeadOnly(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_WORKER_HEAD_ONLY") != "1" {
t.Skip("requires a Java HTTP HEAD_ONLY fixture")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-head-only-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Repository == nil || task.Repository.RepositoryID != 987654332 ||
task.Repository.Owner != "old-owner" || task.Repository.Name != "old-name" ||
task.Mode != "HEAD_ONLY" {
t.Fatal("Java did not downgrade the registered source to HEAD_ONLY")
}
fixture := &headOnlyRepositoryFixture{repositoryTreeFixture: &repositoryTreeFixture{
id: 987654332, owner: "new-owner", name: "new-name"}}
tempRoot := t.TempDir()
limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processRepository(ctx, control, task, limits, fixture)
}); err != nil {
t.Fatal(err)
}
remaining, err := os.ReadDir(tempRoot)
if err != nil || len(remaining) != 0 || fixture.treeCalls != 0 {
t.Fatalf("HEAD_ONLY inspected files or left temporary state: files=%v treeCalls=%d error=%v",
remaining, fixture.treeCalls, err)
}
}
func TestInteropProcessRepositoryRenamedCandidateWithoutSkill(t *testing.T) {
if os.Getenv("SKILLHUB_INTEROP_WORKER_EMPTY_CANDIDATE") != "1" {
t.Skip("requires a Java HTTP candidate validation lease")
}
control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"),
os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN"))
if err != nil {
t.Fatal(err)
}
task, err := control.ClaimWithCapabilities(context.Background(), "interop-empty-candidate-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability})
if err != nil {
t.Fatal(err)
}
if task == nil || task.PlatformControlVersion != nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") ||
task.Repository == nil || task.Repository.RepositoryID != 987654333 ||
task.Repository.Owner != "old-owner" || task.Repository.Name != "old-name" ||
task.Mode != "FULL" {
t.Fatal("Java did not lease the expected candidate repository")
}
fixture := &repositoryTreeFixture{
id: 987654333, owner: "new-owner", name: "new-name",
entries: []github.TreeEntry{{Path: "README.md", Type: "blob", Mode: "100644",
SHA: strings.Repeat("a", 40), Size: 10}},
}
limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5,
MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20}
if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error {
return processRepository(ctx, control, task, limits, fixture)
}); err != nil {
t.Fatal(err)
}
}

25
github-sync-go/internal/worker/repository_resume_test.go Normal file
View file

@ -0,0 +1,25 @@
package worker
import "testing"
func TestSkipPersistedPath(t *testing.T) {
accepted := map[string]struct{}{"skills/received": {}}
retry := map[string]struct{}{"skills/retry": {}}
cases := []struct {
path string
checkpoint string
skip bool
}{
{"skills/received", "", true},
{"skills/received", "skills/earlier", true},
{"skills/earlier", "skills/retry", true},
{"skills/retry", "skills/retry", false},
{"skills/new", "skills/earlier", false},
}
for _, test := range cases {
if got := skipPersistedPath(test.path, test.checkpoint, accepted, retry); got != test.skip {
t.Errorf("path %q, checkpoint %q: skip = %t, want %t",
test.path, test.checkpoint, got, test.skip)
}
}
}

79
github-sync-go/internal/worker/repository_revision_test.go Normal file
View file

@ -0,0 +1,79 @@
package worker
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/iflytek/skillhub/github-sync-go/internal/controlplane"
"github.com/iflytek/skillhub/github-sync-go/internal/github"
)
type revisionFixture struct {
repositoryTreeFixture
headCalls int
treeCommit string
}
func (f *revisionFixture) GetHead(context.Context, github.Repository) (string, github.Quota, error) {
f.headCalls++
return strings.Repeat("e", 40), github.Quota{}, nil
}
func (f *revisionFixture) GetCommitTreeSHA(_ context.Context, _, _, commit string) (string, github.Quota, error) {
f.treeCommit = commit
return strings.Repeat("b", 40), github.Quota{}, nil
}
func TestFullRepositoryResolvesCurrentHeadUnlessResumingPinnedCommit(t *testing.T) {
for _, pinned := range []bool{false, true} {
name := "new-task-current-head"
if pinned {
name = "resume-fixed-commit"
}
t.Run(name, func(t *testing.T) {
var reported string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/repository-metadata") {
w.WriteHeader(http.StatusOK)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") {
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
reported, _ = body["observedCommitSha"].(string)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`))
return
}
t.Errorf("unexpected request %s", r.URL.Path)
w.WriteHeader(500)
}))
defer server.Close()
control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32))
if err != nil {
t.Fatal(err)
}
task := &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", Mode: "FULL", Lease: controlplane.LeaseProof{Generation: 1, Token: strings.Repeat("a", 64)}, Repository: &controlplane.RepositorySnapshot{RepositoryID: 42, Owner: "fixture", Name: "repo"}, MaxFilesPerPackage: 100, MaxPackageBytes: 100 << 20, MaxSingleFileBytes: 10 << 20, MaxUploadBytes: 100 << 20}
want := strings.Repeat("e", 40)
wantHead := 1
if pinned {
want = strings.Repeat("c", 40)
task.Repository.FixedCommitSHA = want
wantHead = 0
}
fixture := &revisionFixture{}
err = processRepository(context.Background(), control, task, RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5, MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20}, fixture)
if err != nil {
t.Fatal(err)
}
if fixture.headCalls != wantHead || fixture.treeCommit != want || reported != want {
t.Fatalf("head calls=%d tree commit=%s reported=%s want=%s", fixture.headCalls, fixture.treeCommit, reported, want)
}
})
}
}

2
openspec/changes/add-github-skill-sync/.openspec.yaml Normal file
View file

@ -0,0 +1,2 @@
schema: spec-driven
created: 2026-09-28

30
openspec/changes/add-github-skill-sync/README.md Normal file
View file

@ -0,0 +1,30 @@
# add-github-skill-sync
服务端 GitHub 公共 Skill 发现、采集、审核与同步。
## 文档入口
- [技能同步统一管理入口](unified-admin.md)
- [统一管理实施合同](unified-admin-contract.md)
- [验证入口与边界](verification.md)
- [同步领域术语](glossary.md)
- [官方深度探索需求](deep-discovery.md)、[设计与验收矩阵](deep-discovery-design.md)、[持久化设计](deep-discovery-storage.md)
- [Go 服务产品需求](go-service-prd.md)
- [总体架构](design.md)与[状态／检查点设计](state-model.md)
- [Java–Go 内部 OpenAPI](../../../docs/api/github-sync-internal.openapi.yaml)与[接口说明](../../../docs/api/github-sync-internal.md)
- [官方探索管理 OpenAPI](../../../docs/api/github-deep-discovery.openapi.yaml)与[接口说明](../../../docs/api/github-deep-discovery.md)
- [实施任务与剩余事项](tasks.md)
- [验证说明](verification.md)
## 当前交付边界
Java 持久配置、任务和恢复状态，Go 独立执行公开仓库发现与逐包采集，包经上传和 Java 复核进入同步审核。平台和 TEAM 公开来源、管理页面、开源技能展示、审核页与同步修订号已有实现；官方深度探索纳入同一控制面和管理入口。
交付完成状态以任务清单为准；验证入口、必须观察的业务行为和延期边界见验证说明。构建和历史本地测试不能替代最终提交的集成验收。
## 启用边界
官方探索默认关闭。全平台和 namespace 水位初始为空，管理员配置前 Java 拒绝新候选树检查与逐包准入；已登记来源仍可执行 HEAD 检查。Go 在线、GitHub 凭证／额度可用、预算和容量允许是不同条件，启用开关不等于任务已执行。配置与部署步骤见[运行手册](../../../docs/github-sync-operations.md)。
OpenSpec 保存正式需求、设计和验收标准；Trellis 仅保留本地执行记录，不作为第二套需求，也不提交本地任务与凭证。

Some files were not shown because too many files have changed in this diff Show more