diff --git a/.env.release.draft b/.env.release.draft index ea870991a..2174977ef 100644 --- a/.env.release.draft +++ b/.env.release.draft @@ -78,6 +78,7 @@ SKILLHUB_STORAGE_S3_ENDPOINT=https://oss-cn-shanghai.aliyuncs.com SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT= SKILLHUB_STORAGE_S3_BUCKET=skillhub-prod +# Set both static keys, or leave both empty to use the deployment's IAM credentials. SKILLHUB_STORAGE_S3_ACCESS_KEY=TODO_fill_real_access_key SKILLHUB_STORAGE_S3_SECRET_KEY=TODO_fill_real_secret_key SKILLHUB_STORAGE_S3_REGION=cn-shanghai @@ -90,6 +91,25 @@ SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=false SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY=PT10M +# Set true for storage providers such as Aliyun OSS that reject aws-chunked uploads. +SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=false + +# Optional public GitHub Skill sync. Keep the github-sync Compose profile disabled +# until Java and Go have the same random service token (at least 32 characters). +# For S3, set a positive staging prefix budget and allow the internal presigned PUT host. +SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=0 +SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN= +SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS= +SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS= +# Optional system GitHub credential pool: v1=. +# Leave both blank for anonymous public API access. +SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS= +SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION= +# Outbound GitHub proxy only; Java control-plane and staging PUT remain direct. +SKILLHUB_GITHUB_SYNC_HTTP_PROXY= +SKILLHUB_GITHUB_SYNC_HTTPS_PROXY= +SKILLHUB_GITHUB_SYNC_NO_PROXY= + # Bootstrap admin is only for first login / first bootstrap. BOOTSTRAP_ADMIN_ENABLED=true BOOTSTRAP_ADMIN_USER_ID=docker-admin diff --git a/.env.release.example b/.env.release.example index 505c594ad..daac9786f 100644 --- a/.env.release.example +++ b/.env.release.example @@ -94,6 +94,20 @@ SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=false SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY=PT10M +# When enabling GitHub sync with S3, set the staging prefix byte budget from real storage capacity. +SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=0 +# Enable the optional github-sync profile only after filling the shared Java/Go token. +SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN= +SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS= +# S3/MinIO: comma-separated internal hosts (host:port) used by presigned PUT URLs. +SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS= +# Optional system credential pool; both fields stay blank for anonymous public API access. +SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS= +SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION= +# Optional GitHub outbound proxy; Java internal and staging PUT requests stay direct. +SKILLHUB_GITHUB_SYNC_HTTP_PROXY= +SKILLHUB_GITHUB_SYNC_HTTPS_PROXY= +SKILLHUB_GITHUB_SYNC_NO_PROXY= # Bootstrap local admin account for first login. Rotate or disable after initial setup. BOOTSTRAP_ADMIN_ENABLED=true diff --git a/.github/workflows/pr-github-sync.yml b/.github/workflows/pr-github-sync.yml new file mode 100644 index 000000000..ccd64993a --- /dev/null +++ b/.github/workflows/pr-github-sync.yml @@ -0,0 +1,57 @@ +name: PR GitHub Sync + +on: + pull_request: + paths: + - 'github-sync-go/**' + - '.github/workflows/pr-github-sync.yml' + +permissions: + contents: read + +jobs: + go: + name: Go checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-go@v5 + with: + go-version-file: github-sync-go/go.mod + cache-dependency-path: github-sync-go/go.mod + - name: Test and vet + working-directory: github-sync-go + run: | + go test -count=1 ./... + go vet ./... + + image: + name: Image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: docker/setup-buildx-action@v3 + - name: Build image + uses: docker/build-push-action@v6 + with: + context: github-sync-go + platforms: linux/${{ matrix.arch }} + tags: skillhub-github-sync-pr:${{ matrix.arch }}-${{ github.sha }} + load: true + - name: Check runtime tools and user + run: | + docker run --rm --platform linux/${{ matrix.arch }} \ + --entrypoint sh skillhub-github-sync-pr:${{ matrix.arch }}-${{ github.sha }} \ + -ec 'test "$(id -u)" = 10001; git --version >/dev/null; tini --version >/dev/null; test -x /usr/local/bin/github-sync' diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 031842a27..c75b2dd9f 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -45,17 +45,23 @@ jobs: platforms: linux/amd64,linux/arm64 image: ghcr.io/${{ github.repository_owner }}/skillhub-scanner mirror_image: skillhub-scanner + - name: github-sync + context: ./github-sync-go + dockerfile: ./github-sync-go/Dockerfile + platforms: linux/amd64,linux/arm64 + image: ghcr.io/${{ github.repository_owner }}/skillhub-github-sync + mirror_image: skillhub-github-sync steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GHCR uses: docker/login-action@v3 with: diff --git a/Makefile b/Makefile index 42941a61e..2fa6214cc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-backend build-backend-app build-builtin-skills build-cli build-frontend build-web check clean cli-install db-reset dev dev-all dev-all-down dev-all-reset dev-down dev-logs dev-server dev-server-restart dev-status dev-web docs-build docs-dev docs-preview generate-api help lint-cli lint-web namespace-smoke suite-smoke suite-bundle-smoke parallel-down parallel-init parallel-sync parallel-up pr publish-cli publish-cli-major publish-cli-minor staging staging-down staging-logs test test-backend test-backend-app test-builtin-skills test-cli test-e2e-frontend test-e2e-smoke-frontend test-frontend test-redis-cluster test-web typecheck-cli typecheck-web validate-release-config web-deps web-install web-install-ci +.PHONY: build build-backend build-backend-app build-builtin-skills build-cli build-frontend build-web check clean cli-install db-reset dev dev-all dev-all-down dev-all-reset dev-down dev-github-sync dev-logs dev-server dev-server-restart dev-status dev-web docs-build docs-dev docs-preview generate-api help lint-cli lint-web namespace-smoke suite-smoke suite-bundle-smoke parallel-down parallel-init parallel-sync parallel-up pr publish-cli publish-cli-major publish-cli-minor staging staging-down staging-logs test test-backend test-backend-app test-builtin-skills test-cli test-e2e-frontend test-e2e-smoke-frontend test-frontend test-redis-cluster test-web typecheck-cli typecheck-web validate-release-config web-deps web-install web-install-ci DEV_DIR := .dev DEV_SERVER_PID := $(DEV_DIR)/server.pid @@ -129,6 +129,17 @@ dev-all: ## 一键启动本地开发环境（依赖 + scanner + 后端 + 前端 dev-server: ## 启动后端开发服务器 cd server && bash -lc '$(DEV_SERVER_PREPARE) && exec env $(DEV_SERVER_SCANNER_ENV) $(DEV_SERVER_CMD)' +dev-github-sync: ## 前台运行可选的本地 GitHub 采集服务（需先给 Java 和 Go 配置同一服务密钥） + @test -n "$${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-}" || { echo "SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN is required" >&2; exit 1; } + @mkdir -p $(DEV_DIR)/github-sync-tmp + cd github-sync-go && go build -o ../$(DEV_DIR)/github-sync ./cmd/github-sync + SKILLHUB_GITHUB_SYNC_JAVA_URL=$(DEV_API_URL)/internal/github-sync/v1 \ + SKILLHUB_GITHUB_SYNC_TEMP_ROOT=$(abspath $(DEV_DIR))/github-sync-tmp \ + SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES=$${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912} \ + SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES=$${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912} \ + SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES=$${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728} \ + $(DEV_DIR)/github-sync + dev-server-restart: ## 重启后端开发服务器 @mkdir -p $(DEV_DIR) @$(DEV_PROCESS) stop --pid-file $(DEV_SERVER_PID) diff --git a/charts/skillhub/README.md b/charts/skillhub/README.md index 3a6cdcaa6..e7eb76fe6 100644 --- a/charts/skillhub/README.md +++ b/charts/skillhub/README.md @@ -57,6 +57,34 @@ helm -n skillhub upgrade -i skillhub ./charts/skillhub \ `secrets.initialRoleGrantsJson`，或在 `existingSecret` 中提供 `auth-initial-role-grants-json`。这些值仅用于数据库首次初始化；之后在后台 “系统配置”中管理。规则格式及管理入口见 [登录开关与外部账号首次授权](../../docs/923-auth-system-settings.md)。 +### 可选的公开 GitHub 采集服务 + +Go 采集 Pod 默认不部署。需要启用时，在受保护的 values 文件中配置： + +```yaml +githubSync: + enabled: true + # S3/MinIO 模式按暂存前缀容量填写正数字节预算 + stagingCapacityLimitBytes: 1073741824 + # S3/MinIO 模式填写预签名 PUT 的内部主机及端口，例如 minio:9000 + allowedS3Hosts: "" +secrets: + githubSyncServiceToken: "<至少32字符的随机服务密钥>" +``` + +Chart 把同一服务密钥注入 Java 与 Go。使用 `existingSecret` 时，该 Secret 必须包含 +`github-sync-service-token` 键，不再在 values 中填写明文。Go 固定单副本，使用 +768 MiB 内存型临时卷；默认单仓库预算 512 MiB，预留 128 MiB，容器内存上限 +1.5 GiB。需要 GitHub 出站代理时设置 `githubSync.httpProxy`、 +`githubSync.httpsProxy` 和 `githubSync.noProxy`；Go 到 Java 的内部请求始终直连。 +运行字段和当前实现边界见 [Go 服务说明](../../github-sync-go/README.md)。 + +系统级公开 GitHub Token 由 Java 加密保存。需同时向 Java 提供 +`github-sync-credential-keys`（`v1=<32 字节随机密钥的 Base64>`，可逗号分隔保留旧版本） +和 `github-sync-active-credential-key-version` 两个 Secret 键。使用 `existingSecret` +时直接写入该 Secret；Chart 自建 Secret 时对应 `secrets.githubSyncCredentialKeys` 与 +`secrets.githubSyncActiveCredentialKeyVersion`，不要将真实值提交到版本控制。不配置时 +Go 使用匿名公开 API 额度，后台创建凭证会明确拒绝。 > **Ingress values 迁移：** 当前版本只支持结构化的 `ingress.hosts[]` 和 > `ingress.tls[]`。旧的 `ingress.host`、`ingress.tls.enabled` 与 diff --git a/charts/skillhub/templates/_helpers.tpl b/charts/skillhub/templates/_helpers.tpl index 460f67f7c..b16b51c3f 100644 --- a/charts/skillhub/templates/_helpers.tpl +++ b/charts/skillhub/templates/_helpers.tpl @@ -70,6 +70,15 @@ app.kubernetes.io/component: scanner app.kubernetes.io/component: scanner {{- end }} +{{- define "skillhub.githubSync.labels" -}} +{{ include "skillhub.labels" . }} +app.kubernetes.io/component: github-sync +{{- end }} +{{- define "skillhub.githubSync.selectorLabels" -}} +{{ include "skillhub.selectorLabels" . }} +app.kubernetes.io/component: github-sync +{{- end }} + {{- /* Bitnami PostgreSQL subchart 完整名称 */}} {{- define "skillhub.postgresql.fullname" -}} {{- if .Values.postgresql.fullnameOverride -}} diff --git a/charts/skillhub/templates/github-sync-deployment.yaml b/charts/skillhub/templates/github-sync-deployment.yaml new file mode 100644 index 000000000..389b9ffd1 --- /dev/null +++ b/charts/skillhub/templates/github-sync-deployment.yaml @@ -0,0 +1,121 @@ +{{- if .Values.githubSync.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "skillhub.fullname" . }}-github-sync + labels: + {{- include "skillhub.githubSync.labels" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "skillhub.githubSync.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "skillhub.githubSync.selectorLabels" . | nindent 8 }} + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- range $key, $val := .Values.githubSync.podAnnotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + {{- with .Values.githubSync.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: github-sync + image: {{ .Values.githubSync.image.registry | default .Values.images.registry }}/skillhub-github-sync:{{ .Values.githubSync.image.tag | default .Values.images.tag | default (printf "v%s" .Chart.AppVersion) }} + imagePullPolicy: {{ .Values.images.pullPolicy }} + ports: + - name: health + containerPort: 8090 + env: + - name: SKILLHUB_GITHUB_SYNC_JAVA_URL + value: {{ printf "http://%s-server:%v/internal/github-sync/v1" (include "skillhub.fullname" .) .Values.server.service.port | quote }} + - name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "skillhub.secretName" . }} + key: github-sync-service-token + - name: SKILLHUB_GITHUB_SYNC_TEMP_ROOT + value: /var/lib/skillhub-github-sync + - name: SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS + value: {{ .Values.githubSync.discoveryWorkers | quote }} + - name: SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS + value: {{ .Values.githubSync.repositoryWorkers | quote }} + - name: SKILLHUB_GITHUB_SYNC_POLL_INTERVAL + value: {{ .Values.githubSync.pollInterval | quote }} + - name: SKILLHUB_GITHUB_SYNC_MAX_PAGES_PER_LEASE + value: {{ .Values.githubSync.maxPagesPerLease | quote }} + - name: SKILLHUB_GITHUB_SYNC_MAX_TREE_REQUESTS + value: {{ .Values.githubSync.maxTreeRequests | quote }} + - name: SKILLHUB_GITHUB_SYNC_MAX_TREE_ENTRIES + value: {{ .Values.githubSync.maxTreeEntries | quote }} + - name: SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES + value: {{ .Values.githubSync.maxWorkspaceBytes | quote }} + - name: SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES + value: {{ .Values.githubSync.totalWorkspaceBytes | quote }} + - name: SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES + value: {{ .Values.githubSync.minFreeBytes | quote }} + - name: SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS + value: {{ .Values.githubSync.allowedS3Hosts | quote }} + {{- if .Values.githubSync.httpProxy }} + - name: HTTP_PROXY + value: {{ .Values.githubSync.httpProxy | quote }} + {{- end }} + {{- if .Values.githubSync.httpsProxy }} + - name: HTTPS_PROXY + value: {{ .Values.githubSync.httpsProxy | quote }} + {{- end }} + {{- if .Values.githubSync.noProxy }} + - name: NO_PROXY + value: {{ .Values.githubSync.noProxy | quote }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: /var/lib/skillhub-github-sync + resources: + {{- toYaml .Values.githubSync.resources | nindent 12 }} + startupProbe: + httpGet: + path: /health/live + port: health + periodSeconds: 5 + failureThreshold: 12 + readinessProbe: + httpGet: + path: /health/ready + port: health + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health/live + port: health + periodSeconds: 15 + volumes: + - name: workspace + emptyDir: + medium: Memory + sizeLimit: {{ .Values.githubSync.tmpfsSizeLimit | quote }} + {{- with .Values.githubSync.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.githubSync.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.githubSync.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/skillhub/templates/secret.yaml b/charts/skillhub/templates/secret.yaml index 4cabd6d88..0c8266b13 100644 --- a/charts/skillhub/templates/secret.yaml +++ b/charts/skillhub/templates/secret.yaml @@ -55,6 +55,14 @@ stringData: {{- end }} skillhub-download-anon-cookie-secret: {{ $downloadSecret | quote }} + {{- if .Values.githubSync.enabled }} + github-sync-service-token: {{ .Values.secrets.githubSyncServiceToken | quote }} + {{- end }} + {{- if .Values.secrets.githubSyncCredentialKeys }} + github-sync-credential-keys: {{ .Values.secrets.githubSyncCredentialKeys | quote }} + github-sync-active-credential-key-version: {{ .Values.secrets.githubSyncActiveCredentialKeyVersion | quote }} + {{- end }} + # OAuth2 GitHub (optional) {{- if .Values.secrets.oauth2GithubClientId }} oauth2-github-client-id: {{ .Values.secrets.oauth2GithubClientId | quote }} diff --git a/charts/skillhub/templates/server-deployment.yaml b/charts/skillhub/templates/server-deployment.yaml index 564e2836c..1b8e503b9 100644 --- a/charts/skillhub/templates/server-deployment.yaml +++ b/charts/skillhub/templates/server-deployment.yaml @@ -79,6 +79,27 @@ spec: value: {{ $profiles | quote }} - name: SKILLHUB_SUITE_REVIEW_WRITES_ENABLED value: {{ .Values.server.suiteReviewWritesEnabled | quote }} + - name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES + value: {{ .Values.githubSync.stagingCapacityLimitBytes | quote }} + {{- if .Values.githubSync.enabled }} + - name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS + valueFrom: + secretKeyRef: + name: {{ include "skillhub.secretName" . }} + key: github-sync-service-token + {{- end }} + - name: SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS + valueFrom: + secretKeyRef: + name: {{ include "skillhub.secretName" . }} + key: github-sync-credential-keys + optional: true + - name: SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION + valueFrom: + secretKeyRef: + name: {{ include "skillhub.secretName" . }} + key: github-sync-active-credential-key-version + optional: true # Database - name: SPRING_DATASOURCE_URL diff --git a/charts/skillhub/templates/validate.yaml b/charts/skillhub/templates/validate.yaml index 667a59e90..f8f9a29c6 100644 --- a/charts/skillhub/templates/validate.yaml +++ b/charts/skillhub/templates/validate.yaml @@ -121,6 +121,24 @@ {{- fail "s3.publicEndpoint must be an absolute HTTP(S) URL" -}} {{- end -}} +{{- if .Values.githubSync.enabled -}} +{{- if not .Values.server.enabled -}} +{{- fail "githubSync.enabled requires server.enabled" -}} +{{- end -}} +{{- if and (not .Values.existingSecret) (lt (len .Values.secrets.githubSyncServiceToken) 32) -}} +{{- fail "secrets.githubSyncServiceToken must contain at least 32 characters, or use existingSecret with github-sync-service-token" -}} +{{- end -}} +{{- if and .Values.s3.enabled (not .Values.githubSync.allowedS3Hosts) -}} +{{- fail "githubSync.allowedS3Hosts is required when s3.enabled=true" -}} +{{- end -}} +{{- if or (lt (.Values.githubSync.discoveryWorkers | int) 1) (lt (.Values.githubSync.repositoryWorkers | int) 1) -}} +{{- fail "githubSync worker counts must be positive" -}} +{{- end -}} +{{- if lt (.Values.githubSync.totalWorkspaceBytes | int64) (mul (.Values.githubSync.repositoryWorkers | int64) (.Values.githubSync.maxWorkspaceBytes | int64) | int64) -}} +{{- fail "githubSync.totalWorkspaceBytes must cover all repository workers" -}} +{{- end -}} +{{- end -}} + {{- if not .Values.secrets.allowAutoGenerated -}} {{- if not .Values.existingSecret -}} {{- if not (or .Values.secrets.bootstrapAdminPassword .Values.bootstrapAdmin.password) -}} diff --git a/charts/skillhub/tests/configuration-contracts.sh b/charts/skillhub/tests/configuration-contracts.sh index 4297cf66d..d9dd8b140 100755 --- a/charts/skillhub/tests/configuration-contracts.sh +++ b/charts/skillhub/tests/configuration-contracts.sh @@ -75,6 +75,23 @@ assert_rejected web-dns-empty-label --set-string web.clusterDomain=cluster..loca assert_rejected web-dns-invalid-label --set-string web.clusterDomain=-cluster.local render verify "$CHART_DIR" >"$TMP_DIR/default.yaml" +if grep -Fq 'name: verify-skillhub-github-sync' "$TMP_DIR/default.yaml"; then + fail "GitHub sync deployment must stay disabled by default" +fi +grep -A1 -F 'name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES' "$TMP_DIR/default.yaml" \ + | grep -Fq 'value: "0"' \ + || fail "server must receive the default GitHub sync staging capacity budget" +render github-sync "$CHART_DIR" \ + --set githubSync.enabled=true \ + --set githubSync.stagingCapacityLimitBytes=1073741824 \ + --set-string secrets.githubSyncServiceToken=github-sync-service-token-at-least-32-characters \ + >"$TMP_DIR/github-sync.yaml" +grep -Fq 'name: github-sync-skillhub-github-sync' "$TMP_DIR/github-sync.yaml" \ + || fail "enabled GitHub sync must render its deployment" +grep -A1 -F 'name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES' "$TMP_DIR/github-sync.yaml" \ + | grep -Fq 'value: "1073741824"' \ + || fail "server must receive the configured GitHub sync staging capacity budget" +assert_rejected github-sync-negative-capacity --set githubSync.stagingCapacityLimitBytes=-1 grep -Fq 'name: POSTGRESQL_MAX_CONNECTIONS' "$TMP_DIR/default.yaml" grep -Fq 'value: "verify-postgresql"' "$TMP_DIR/default.yaml" grep -Fq 'value: "verify-redis-master"' "$TMP_DIR/default.yaml" diff --git a/charts/skillhub/values.schema.json b/charts/skillhub/values.schema.json index 24431eb68..40bba2ab7 100644 --- a/charts/skillhub/values.schema.json +++ b/charts/skillhub/values.schema.json @@ -202,7 +202,10 @@ "oauth2DingtalkClientSecret": { "type": "string" }, "scannerLlmApiKey": { "type": "string" }, "scannerLlmBaseUrl": { "type": "string" }, - "scannerLlmModel": { "type": "string" } + "scannerLlmModel": { "type": "string" }, + "githubSyncServiceToken": { "type": "string" }, + "githubSyncCredentialKeys": { "type": "string" }, + "githubSyncActiveCredentialKeyVersion": { "type": "string" } } }, "postgresql": { @@ -302,10 +305,40 @@ } }, "server": { "$ref": "#/definitions/serverComponent" }, + "githubSync": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "image", "discoveryWorkers", "repositoryWorkers", "pollInterval", "maxPagesPerLease", "maxTreeRequests", "maxTreeEntries", "maxWorkspaceBytes", "totalWorkspaceBytes", "minFreeBytes", "tmpfsSizeLimit", "allowedS3Hosts", "httpProxy", "httpsProxy", "noProxy", "resources", "imagePullSecrets", "podAnnotations", "nodeSelector", "tolerations", "affinity"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/definitions/image" }, + "stagingCapacityLimitBytes": { "type": "integer", "minimum": 0 }, + "discoveryWorkers": { "type": "integer", "minimum": 1, "maximum": 32 }, + "repositoryWorkers": { "type": "integer", "minimum": 1, "maximum": 32 }, + "pollInterval": { "type": "string", "pattern": "^[1-9][0-9]*(s|m|h)$" }, + "maxPagesPerLease": { "type": "integer", "minimum": 1 }, + "maxTreeRequests": { "type": "integer", "minimum": 1 }, + "maxTreeEntries": { "type": "integer", "minimum": 1 }, + "maxWorkspaceBytes": { "type": "integer", "minimum": 1 }, + "totalWorkspaceBytes": { "type": "integer", "minimum": 1 }, + "minFreeBytes": { "type": "integer", "minimum": 0 }, + "tmpfsSizeLimit": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" }, + "allowedS3Hosts": { "type": "string" }, + "httpProxy": { "type": "string" }, + "httpsProxy": { "type": "string" }, + "noProxy": { "type": "string" }, + "resources": { "$ref": "#/definitions/resources" }, + "imagePullSecrets": { "type": "array", "items": { "type": "object" } }, + "podAnnotations": { "$ref": "#/definitions/stringMap" }, + "nodeSelector": { "$ref": "#/definitions/stringMap" }, + "tolerations": { "type": "array", "items": { "type": "object" } }, + "affinity": { "type": "object" } + } + }, "web": { "$ref": "#/definitions/webComponent" }, "scanner": { "$ref": "#/definitions/scannerComponent" } }, - "required": ["images", "auth", "builtinSkills", "ingress", "s3", "session", "bootstrapAdmin", "springProfilesActive", "secrets", "postgresql", "externalDatabase", "redis", "externalRedis", "server", "web", "scanner"], + "required": ["images", "auth", "builtinSkills", "ingress", "s3", "session", "bootstrapAdmin", "springProfilesActive", "secrets", "postgresql", "externalDatabase", "redis", "externalRedis", "server", "githubSync", "web", "scanner"], "definitions": { "dnsLabel": { "type": "string", diff --git a/charts/skillhub/values.yaml b/charts/skillhub/values.yaml index 6137a4396..f548f2fe6 100644 --- a/charts/skillhub/values.yaml +++ b/charts/skillhub/values.yaml @@ -118,6 +118,10 @@ secrets: scannerLlmApiKey: "" scannerLlmBaseUrl: "" scannerLlmModel: "" + githubSyncServiceToken: "" + # 格式：v1=<32 字节密钥的 Base64>。轮换时保留旧版本并追加新版本。 + githubSyncCredentialKeys: "" + githubSyncActiveCredentialKeyVersion: "" # ============================================================================ # PostgreSQL 配置（Bitnami） @@ -386,6 +390,44 @@ server: enabled: false minAvailable: 1 +# 公开 GitHub Skill 采集服务；默认不创建任务执行 Pod。 +githubSync: + enabled: false + # S3/MinIO 暂存前缀容量预算（字节）；启用同步前按部署存储能力设置正值。 + stagingCapacityLimitBytes: 0 + image: + registry: "" + tag: "" + # 与 Java 的 SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS 使用同一服务密钥。 + # 在 secrets.githubSyncServiceToken 或 existingSecret 中配置。 + discoveryWorkers: 1 + repositoryWorkers: 1 + pollInterval: 5s + maxPagesPerLease: 20 + maxTreeRequests: 100 + maxTreeEntries: 100000 + maxWorkspaceBytes: 536870912 + totalWorkspaceBytes: 536870912 + minFreeBytes: 134217728 + tmpfsSizeLimit: 768Mi + # S3 模式需填写上传票据指向的内部存储主机及端口，例如 minio:9000。 + allowedS3Hosts: "" + httpProxy: "" + httpsProxy: "" + noProxy: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1000m + memory: 1536Mi + imagePullSecrets: [] + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + # ============================================================================ # Web 配置 # ============================================================================ diff --git a/compose.release.yml b/compose.release.yml index 30214c51b..876655749 100644 --- a/compose.release.yml +++ b/compose.release.yml @@ -46,6 +46,19 @@ services: timeout: 5s retries: 10 + # A named volume masks the app-owned directory created in server/Dockerfile. + storage-permissions: + image: ${SKILLHUB_SERVER_IMAGE:-ghcr.io/iflytek/skillhub-server}:${SKILLHUB_VERSION:-latest} + user: "0:0" + entrypoint: ["chown", "100:101", "/var/lib/skillhub/storage"] + restart: "no" + network_mode: none + read_only: true + cap_drop: [ALL] + cap_add: [CHOWN] + volumes: + - skillhub_storage:/var/lib/skillhub/storage + server: image: ${SKILLHUB_SERVER_IMAGE:-ghcr.io/iflytek/skillhub-server}:${SKILLHUB_VERSION:-latest} restart: unless-stopped @@ -91,6 +104,10 @@ services: SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING: ${SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING:-false} SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: ${SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET:-false} SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY: ${SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY:-PT10M} + SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS:-} + SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS: ${SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS:-} + SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION: ${SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION:-} + SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES:-0} SKILLHUB_SECURITY_SCANNER_ENABLED: ${SKILLHUB_SECURITY_SCANNER_ENABLED:-true} SKILLHUB_SECURITY_SCANNER_URL: http://skill-scanner:8000 SKILLHUB_SECURITY_SCANNER_MODE: upload @@ -149,6 +166,8 @@ services: volumes: - skillhub_storage:/var/lib/skillhub/storage depends_on: + storage-permissions: + condition: service_completed_successfully postgres: condition: service_healthy redis: @@ -186,6 +205,36 @@ services: retries: 12 start_period: 10s + github-sync: + profiles: [github-sync] + image: ${SKILLHUB_GITHUB_SYNC_IMAGE:-ghcr.io/iflytek/skillhub-github-sync}:${SKILLHUB_VERSION:-latest} + restart: unless-stopped + environment: + SKILLHUB_GITHUB_SYNC_JAVA_URL: http://server:8080/internal/github-sync/v1 + SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-} + SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS: ${SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS:-1} + SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS: ${SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS:-1} + SKILLHUB_GITHUB_SYNC_POLL_INTERVAL: ${SKILLHUB_GITHUB_SYNC_POLL_INTERVAL:-5s} + SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912} + SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912} + SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES: ${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728} + SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS: ${SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS:-} + HTTP_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTP_PROXY:-} + HTTPS_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTPS_PROXY:-} + NO_PROXY: ${SKILLHUB_GITHUB_SYNC_NO_PROXY:-} + tmpfs: + - /var/lib/skillhub-github-sync:size=${SKILLHUB_GITHUB_SYNC_TMPFS_BYTES:-805306368},mode=1777 + mem_limit: ${SKILLHUB_GITHUB_SYNC_MEMORY_LIMIT_BYTES:-1610612736} + depends_on: + server: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-Y", "off", "-qO-", "http://127.0.0.1:8090/health/ready"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + volumes: postgres_data: redis_data: diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 0ed6db14f..bb7ce2509 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -62,6 +62,9 @@ cp secret.yaml.example secret.yaml | redis-password | Redis 数据节点密码 | 否 | | redis-sentinel-password | Redis Sentinel 独立密码 | 否 | | bootstrap-admin-password | 管理员密码 | 是 | +| github-sync-service-token | 可选 Go 采集服务与 Java 之间的共享服务密钥，启用时至少 32 字符 | 仅启用采集时必填 | +| github-sync-credential-keys | 系统级 GitHub Token 加密密钥环，格式 `v1=<32 字节随机密钥的 Base64>` | 后台配置 Token 时必填 | +| github-sync-active-credential-key-version | 写入新密文时使用的密钥版本，例如 `v1` | 配置加密密钥环时必填 | | oauth2-github-client-id | GitHub OAuth ID | 否 | | oauth2-github-client-secret | GitHub OAuth 密钥 | 否 | | skill-scanner-llm-api-key | LLM API 密钥 | 否 | @@ -166,6 +169,17 @@ kubectl apply -k overlays/with-infra/ # 或 overlays/external/ ## 部署架构 +### 可选的公开 GitHub 采集服务 + +`base/github-sync-deployment.yaml` 不在默认 Kustomize 清单中。先在 `skillhub-secret` 配置 `github-sync-service-token`，并更新 Java Pod 使其读取该 Secret；再按实际镜像版本与存储网络配置此 Deployment，然后单独运行： + +```bash +kubectl apply -n skillhub -f deploy/k8s/base/github-sync-deployment.yaml +kubectl rollout status -n skillhub deployment/skillhub-github-sync +``` + +Go Pod 通过 `skillhub-server:8080` 领取任务。S3/MinIO 模式还要把 `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` 设为预签名 PUT 的内部主机及端口；需要 GitHub 出站代理时，在可选 Deployment 中填写 `HTTP_PROXY`、`HTTPS_PROXY`、`NO_PROXY`，默认空值不启用。代理地址如包含凭证，应改由 Kubernetes Secret 注入，勿写入清单或 ConfigMap。Go 对内部 Java 和暂存对象上传使用直连客户端。工作目录为 768 MiB 内存型 `emptyDir`，并按 512 MiB 单仓库预算和 128 MiB 预留空间配置。完整变量说明见 [Go 服务 README](../../github-sync-go/README.md)。 + ``` ┌─────────────────────────────────────────────────────────────┐ │ skillhub namespace │ diff --git a/deploy/k8s/base/backend-deployment.yaml b/deploy/k8s/base/backend-deployment.yaml index a7a396482..95f13a0a7 100644 --- a/deploy/k8s/base/backend-deployment.yaml +++ b/deploy/k8s/base/backend-deployment.yaml @@ -14,6 +14,9 @@ spec: labels: app.kubernetes.io/name: skillhub-server spec: + securityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch containers: - name: server image: ghcr.io/iflytek/skillhub-server:edge @@ -24,6 +27,29 @@ spec: env: - name: SPRING_PROFILES_ACTIVE value: docker + - name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS + valueFrom: + secretKeyRef: + name: skillhub-secret + key: github-sync-service-token + optional: true + - name: SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS + valueFrom: + secretKeyRef: + name: skillhub-secret + key: github-sync-credential-keys + optional: true + - name: SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION + valueFrom: + secretKeyRef: + name: skillhub-secret + key: github-sync-active-credential-key-version + optional: true + - name: SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES + valueFrom: + configMapKeyRef: + name: skillhub-config + key: github-sync-staging-capacity-limit-bytes # Database - name: SPRING_DATASOURCE_URL diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml index d7daeb327..55a3dc1f5 100644 --- a/deploy/k8s/base/configmap.yaml +++ b/deploy/k8s/base/configmap.yaml @@ -30,6 +30,8 @@ data: # 存储配置 # local: 本地存储（默认）, s3: S3/OSS 对象存储 skillhub-storage-provider: local + # S3/MinIO 模式启用 GitHub 同步前，按隔离暂存区容量预算填写正数字节数。 + github-sync-staging-capacity-limit-bytes: "0" # 技能扫描器配置 skill-scanner-enabled: "true" diff --git a/deploy/k8s/base/github-sync-deployment.yaml b/deploy/k8s/base/github-sync-deployment.yaml new file mode 100644 index 000000000..518f32504 --- /dev/null +++ b/deploy/k8s/base/github-sync-deployment.yaml @@ -0,0 +1,94 @@ +# 可选部署：先在 skillhub-secret 配置 github-sync-service-token，再单独 apply 此文件。 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skillhub-github-sync + labels: + app.kubernetes.io/name: skillhub-github-sync +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: skillhub-github-sync + template: + metadata: + labels: + app.kubernetes.io/name: skillhub-github-sync + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: github-sync + image: ghcr.io/iflytek/skillhub-github-sync:edge + imagePullPolicy: IfNotPresent + ports: + - name: health + containerPort: 8090 + env: + - name: SKILLHUB_GITHUB_SYNC_JAVA_URL + value: http://skillhub-server:8080/internal/github-sync/v1 + - name: SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: skillhub-secret + key: github-sync-service-token + - name: SKILLHUB_GITHUB_SYNC_TEMP_ROOT + value: /var/lib/skillhub-github-sync + - name: SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS + value: "1" + - name: SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS + value: "1" + - name: SKILLHUB_GITHUB_SYNC_POLL_INTERVAL + value: 5s + - name: SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES + value: "536870912" + - name: SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES + value: "536870912" + - name: SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES + value: "134217728" + # S3/MinIO 模式必须填入内部上传主机及端口，例如 minio:9000。 + - name: SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS + value: "" + # 按部署网络填写 GitHub 出站代理；Java 和对象存储由 Go 客户端直连。 + - name: HTTP_PROXY + value: "" + - name: HTTPS_PROXY + value: "" + - name: NO_PROXY + value: "" + volumeMounts: + - name: workspace + mountPath: /var/lib/skillhub-github-sync + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1536Mi + startupProbe: + httpGet: + path: /health/live + port: health + periodSeconds: 5 + failureThreshold: 12 + readinessProbe: + httpGet: + path: /health/ready + port: health + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health/live + port: health + periodSeconds: 15 + volumes: + - name: workspace + emptyDir: + medium: Memory + sizeLimit: 768Mi diff --git a/deploy/k8s/base/secret.yaml.example b/deploy/k8s/base/secret.yaml.example index f983fa196..1853b27ea 100644 --- a/deploy/k8s/base/secret.yaml.example +++ b/deploy/k8s/base/secret.yaml.example @@ -23,6 +23,12 @@ stringData: # Bootstrap 管理员密码（敏感） bootstrap-admin-password: ChangeMe!2026 + # 公开 GitHub 采集服务身份；启用独立 Go Deployment 时填写至少 32 字符的随机值 + github-sync-service-token: "" + # AES-256-GCM 密钥环：v1=<32 字节随机密钥的 Base64>，可逗号分隔保留旧版本。 + github-sync-credential-keys: "" + github-sync-active-credential-key-version: "" + # GitHub OAuth（可选，用于 GitHub 登录） oauth2-github-client-id: "" oauth2-github-client-secret: "" diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 05109d076..826c65434 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -34,6 +34,10 @@ services: SKILLHUB_STORAGE_S3_REGION: us-east-1 SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true" SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "true" + SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS:-} + SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS: ${SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS:-} + SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION: ${SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION:-} + SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES:-0} BOOTSTRAP_ADMIN_ENABLED: "true" BOOTSTRAP_ADMIN_USER_ID: staging-admin BOOTSTRAP_ADMIN_USERNAME: admin @@ -84,3 +88,32 @@ services: timeout: 5s retries: 6 start_period: 10s + + github-sync: + profiles: [github-sync] + build: ./github-sync-go + image: skillhub-github-sync:staging + environment: + SKILLHUB_GITHUB_SYNC_JAVA_URL: http://server:8080/internal/github-sync/v1 + SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN:-} + SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES:-536870912} + SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES: ${SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES:-536870912} + SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES: ${SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES:-134217728} + SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS: minio:9000 + HTTP_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTP_PROXY:-} + HTTPS_PROXY: ${SKILLHUB_GITHUB_SYNC_HTTPS_PROXY:-} + NO_PROXY: ${SKILLHUB_GITHUB_SYNC_NO_PROXY:-} + tmpfs: + - /var/lib/skillhub-github-sync:size=${SKILLHUB_GITHUB_SYNC_TMPFS_BYTES:-805306368},mode=1777 + mem_limit: ${SKILLHUB_GITHUB_SYNC_MEMORY_LIMIT_BYTES:-1610612736} + depends_on: + server: + condition: service_healthy + minio: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-Y", "off", "-qO-", "http://127.0.0.1:8090/health/ready"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s diff --git a/docs/01-system-architecture.md b/docs/01-system-architecture.md index fc587602e..72e11319a 100644 --- a/docs/01-system-architecture.md +++ b/docs/01-system-architecture.md @@ -24,7 +24,8 @@ server/ ├── skillhub-auth # OAuth2 认证 + RBAC + 授权判定 ├── skillhub-search # 搜索 SPI + PostgreSQL 全文实现 ├── skillhub-storage # 对象存储抽象 + LocalFile/S3 双实现 -└── skillhub-infra # JPA、通用工具、配置基础 +├── skillhub-infra # JPA、通用工具、配置基础 +└── skillhub-github-sync # GitHub 来源与同步控制面模型；仍在 Java 应用进程内 ``` ## 4. 模块依赖方向（依赖倒置，禁止领域层依赖基础设施） @@ -35,6 +36,7 @@ infra → domain # infra 实现 domain 定义的 Repository 接口 auth → domain # auth 引用 UserAccount 等领域实体 search → domain # search 引用 SkillSearchDocument 等领域模型 storage → (独立抽象) # 纯 SPI，不依赖 domain +app, infra → github-sync → domain, storage ``` 核心原则： @@ -115,6 +117,7 @@ web/ skillhub/ ├── server/ # Maven 多模块 Java 后端 │ └── Dockerfile # 后端多阶段构建 +├── github-sync-go/ # 独立 Go 采集服务（实施中） ├── web/ # React 前端 │ ├── Dockerfile # 前端多阶段构建 │ ├── nginx.conf.template # Nginx 运行时模板 diff --git a/docs/02-domain-model.md b/docs/02-domain-model.md index 1236b5264..7040e55fe 100644 --- a/docs/02-domain-model.md +++ b/docs/02-domain-model.md @@ -16,7 +16,7 @@ | id | bigint | 主键 | | slug | varchar(64) | URL 友好标识 | | display_name | varchar(128) | 展示名 | -| type | enum | `GLOBAL` / `TEAM` | +| type | enum | `GLOBAL` / `CATALOG` / `TEAM` | | description | text | 描述 | | avatar_url | varchar(512) | 头像 | | status | enum | `ACTIVE` / `FROZEN` / `ARCHIVED` | @@ -25,11 +25,13 @@ | updated_at | datetime | | - `GLOBAL` 类型全局唯一（只有一个 `@global`），由平台管理员管理 +- `CATALOG` 为系统管理的来源目录；首个 `@github` 通过迁移创建，普通用户不能修改或删除 +- `@github` 迁移同时创建不可登录的系统 actor `github-sync-publisher`；若 namespace slug 或 actor ID 已被占用，迁移明确失败而不接管已有记录 - `TEAM` 类型对应部门/团队，可创建多个 - 技能完整寻址：`@{namespace_slug}/{skill_slug}` - slug 唯一约束：`slug` - slug 格式校验：`[a-z0-9]([a-z0-9-]*[a-z0-9])?`，长度 2-64，且不得包含连续两个以上的连字符 `--`（为兼容层坐标映射保留） -- slug 保留词列表（用户创建 namespace 时不可使用）：`admin`, `api`, `dashboard`, `search`, `auth`, `me`, `global`, `system`, `static`, `assets`, `health` +- slug 保留词列表（用户创建 namespace 时不可使用）：`admin`, `api`, `dashboard`, `search`, `auth`, `me`, `global`, `github`, `system`, `static`, `assets`, `health`；`github` 仅在 namespace 创建入口保留，不禁止同名 Skill - 系统内置 namespace（`@global`）在数据库初始化时由 Flyway 脚本预置，绕过 slug 校验规则。保留词校验仅作用于用户创建 namespace 的接口 - 状态语义： - `ACTIVE`：正常使用 diff --git a/docs/09-deployment.md b/docs/09-deployment.md index a3bd287a1..ae2958002 100644 --- a/docs/09-deployment.md +++ b/docs/09-deployment.md @@ -93,6 +93,15 @@ make dev-all-down make dev-all-reset ``` +可选的 GitHub 公开 Skill 采集服务在宿主机前台运行。先为 Java 和 Go 设置同一枚本地服务密钥，再启动 `make dev-all` 与 `make dev-github-sync`；后者按 `Ctrl-C` 停止，不影响 Java 和其他开发依赖。默认使用本地存储，Go 临时目录为 `.dev/github-sync-tmp`。如需使用 S3/MinIO，应按[同步运行手册](github-sync-operations.md)配置两侧的存储地址和允许上传主机。 + +```bash +export SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS="<本机生成的至少 32 字符密钥>" +export SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN="$SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS" +make dev-all +make dev-github-sync +``` + ## 5 单机交付环境 ### 5.1 启动 @@ -103,6 +112,22 @@ make validate-release-config docker compose --env-file .env.release -f compose.release.yml up -d ``` +公开 GitHub Skill 采集服务使用独立的 `github-sync` profile，默认不启动。设置同一条至少 +32 字符的服务密钥到 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN` 与 Java 认可的 +`SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 后，才能显式启动；S3/MinIO 模式还需设置 +`SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` 为内部上传主机及端口。配置和当前实现边界见 +[Go 服务说明](../github-sync-go/README.md)，启停、后台操作与排障见 +[管理员运行手册](github-sync-operations.md)。 + +使用 S3/MinIO 时，给 Java 服务设置正值的 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES`， +表示 GitHub 同步隔离暂存前缀可使用的字节预算。未设置且暂存区已有对象时容量告警为 `1`； +扫描前缀不完整或读取失败时同样告警。预算应结合对象存储容量与试运行数据确定， +具体指标和扫描边界见 [内部协议说明](api/github-sync-internal.md#暂存容量监控)。 + +如需在后台配置公开读取 GitHub Token，Java 服务还需设置 +`SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS`（例如 `v1=<32 字节随机密钥的 Base64>`）和 +`SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION`（例如 `v1`）。两项缺省时保留匿名读取能力，后台不能保存 Token。轮换时同时保留旧、新版本密钥，待旧密文被替换后再移除旧版本；不要把密钥或 Token 写入 Compose 文件、仓库或命令历史。管理接口和状态见 [GitHub 同步管理接口](api/github-sync-admin.md)。 + 默认访问地址： - Web UI: `SKILLHUB_PUBLIC_BASE_URL` @@ -214,13 +239,15 @@ docker compose --env-file .env.release -f compose.release.yml up -d --force-recr 1. 检出代码 2. 登录 GHCR -3. 分别构建 `server/Dockerfile` 与 `web/Dockerfile` +3. 分别构建 `server`、`web`、`scanner` 与 `github-sync-go` 的 Dockerfile 4. 推送镜像： - `ghcr.io/iflytek/skillhub-server` - `ghcr.io/iflytek/skillhub-web` + - `ghcr.io/iflytek/skillhub-scanner` + - `ghcr.io/iflytek/skillhub-github-sync` 5. 写入 `edge` / `vX.Y.Z` / `latest` / `sha-*` 标签 6. 同时发布多架构 manifest：`server` / `web` 覆盖 `linux/amd64`、`linux/arm64` 与 - `linux/riscv64`，`scanner` 暂保持 `linux/amd64` 与 `linux/arm64` + `linux/riscv64`，`scanner` 与 `github-sync` 覆盖 `linux/amd64`、`linux/arm64` ## 7 配置管理 @@ -743,3 +770,8 @@ Flyway 仍是唯一 schema 变更入口： - 路径：`server/skillhub-app/src/main/resources/db/migration/` - 命名：`V{version}__{description}.sql` - 启动策略：应用容器启动时自动执行迁移 + +GitHub 同步的 V76 迁移会创建系统管理的 `@github` 目录和 `github-sync-publisher` 系统账户。若升级前已有同名 namespace 或账户，迁移会明确失败并回滚，不会接管或改写原数据。部署前应检查这两个标识是否被占用；遇到冲突时先处理现有数据的归属，再重新启动迁移。 + + +本次 GitHub 同步从 main 的 V75 schema 继续追加 V76–V119；main 已发行 V1–V75 不改名、不改 checksum。升级前备份数据库和对象存储，先在副本验证迁移与业务。早期未发布开发分支使用过 V66–V109 的不同编号，它们与 main 历史冲突，不能直接套用本次迁移，也不应通过 Flyway repair 掩盖 checksum/版本冲突。一次性开发库应使用新的隔离库重新建立；需要保留早期分支数据时须另行设计并验收数据迁移。回退镜像不等于数据库降级。 diff --git a/docs/api/github-deep-discovery.md b/docs/api/github-deep-discovery.md new file mode 100644 index 000000000..70997fd4c --- /dev/null +++ b/docs/api/github-deep-discovery.md @@ -0,0 +1,193 @@ +# 官方 GitHub 深度探索接口增量（D2） + +> V119统一管理补充：平台全部任务现在还要求 `platform-pause/v1`。当前Go声明四项官方能力加平台暂停能力，缺少后者的旧实例不能领取 `@github` 任务，也不显示为兼容实例。TEAM规则不变。管理员入口合并至“技能同步 → 采集配置”，详见[统一管理接口](github-sync-admin.md#统一技能同步管理)。 +管理端机器可读合同见 [管理 OpenAPI](github-deep-discovery.openapi.yaml)，Java–Go见 [内部 OpenAPI](github-sync-internal.openapi.yaml)。本文固定当前协议；验证入口和限制见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。 + +正式行为以 [需求](../../openspec/changes/add-github-skill-sync/deep-discovery.md)、[设计](../../openspec/changes/add-github-skill-sync/deep-discovery-design.md) 为准。本文固定字段与交互合同，物理表、索引与迁移见同一 OpenSpec 下的[持久化设计](../../openspec/changes/add-github-skill-sync/deep-discovery-storage.md)。 + +## 1. 已核对的实现基线 + +| 代码 | 当前事实 | D2 增量 | +| --- | --- | --- | +| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncController.java` | `/api/v1/admin/github-sync`，`SUPER_ADMIN`，已有来源/运行/水位/周期接口 | 独立AdminGithubDeepDiscoveryController已实现配置GET/PUT与六个读模型接口，并完成受控HTTP验证 | +| `server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ApiResponse.java`、`PageResponse.java` | `code/msg/data/timestamp/requestId`，成功 code=0；分页 `items/total/page/size` | 保持包装，分页增加 `asOf` | +| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalController.java` | claim已扩展可选capabilities并按持久能力过滤；官方响应与legacy互斥；官方批次与仓库控制已接线 | 增加能力过滤、独立官方快照和出处；202不变 | +| `github-sync-go/internal/controlplane/client.go` | Go生产工作池声明四项官方能力，执行官方客户端与循环 | 启用前全部Java/Go升级；实例能力心跳已持久化并验证 | +| `server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncWorkerHeartbeatController.java`、`repository/GithubSyncWorkerPresenceRepository.java` | 保留legacy全局联系时间，同时保存实例ID、协议与能力 | 官方在线判定使用实例能力和最近联系；心跳不证明业务正在推进 | + +Web SDK 已从当前 Java 运行时 `/v3/api-docs` 生成。管理页面使用生成类型；Go wire 类型与内部合同保持同步。受控集成验证不替代真实 GitHub 搜索效果对照，完整边界见[验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。 + +## 2. 管理端路径与授权 + +共同前缀：`/api/v1/admin/github-sync/deep-discovery`。所有读写仅 `SUPER_ADMIN`，范围固定为公共 `@github`，不接受客户端指定 namespace。写入沿用会话、CSRF 与审计；记录用户字符串 ID、配置版本及规范化差异。响应 `Cache-Control: no-store`，不包含凭证、租约、上传票据、原始查询响应或异常正文。 + +| 方法 | 相对路径 | data及实施状态 | +| --- | --- | --- | +| GET | `/configuration` | 当前启停、预算、排除项及version；已实现，隔离HTTP验证通过 | +| PUT | `/configuration` | 整体保存后的配置；请求含expectedVersion；已实现，隔离HTTP验证通过 | +| GET | `/summary?from=...&until=...` | 近期成果、当前积压、联系状态、等待原因、最多5个近期runId | +| GET | `/progress` | 当前策略各渠道的持久分片计数、最近业务进度和额度观测 | +| GET | `/repositories` | 仓库摘要分页 | +| GET | `/repositories/{repositoryId}` | 按需详情及关联sourceId/runId | +| GET | `/repositories/{repositoryId}/provenance` | 出处分页 | +| GET | `/repositories/{repositoryId}/paths` | 最近逐路径结果及Skill/version/review ID分页 | + +没有“执行一次”接口。既有 `/runs/{runId}` 及其 `/tasks`、`/paths` 继续承担运行详情；本次不另造运行对象。历史清理后的关联runId可失效，详情返回404时页面提示历史已清理。导航使用现有权限校验，不能拼造未经实现的前端路由。 + +错误沿用 `ApiResponse` 数值 code，HTTP 400 为参数不合法，401未登录，403权限或CSRF不满足，404目标不在可见官方范围，409配置版本冲突。当前配置接口沿用GlobalExceptionHandler，以HTTP状态码作为数值code；配置冲突使用DomainConflictException及本地化消息。前端不得匹配翻译后的msg。 + +## 3. 配置并发、排除与预算 + +V108迁移持久初始化 `version=1, enabled=false, budgetProfile=BALANCED, exclusions=[], strategyVersion=null, updatedBy=null`；strategyVersion在官方目录调度首次事务对账后设为已交付目录的真实版本；这表示目录已装载，不表示结果接收/执行链路完成。三个预算档位是 `ECONOMY/BALANCED/ACCELERATED`。这些值仅命名档位，当前预约节奏以预算设计为准，部署方仍须结合实际额度校准，不能把加速理解为绕过水位或删除探索渠道。 + +PUT 请求例（示意数据）： + +```json +{"expectedVersion":1,"enabled":true,"budgetProfile":"BALANCED","exclusions":[{"kind":"OWNER","value":"example"},{"kind":"REPOSITORY","value":"sample/skills"}]} +``` + +在同一事务中比较 expectedVersion 与当前 version；相同且有效则保存并递增 version，过期则409且不写任何字段。即便重复相同内容也先比较版本，不静默覆盖其他管理员操作。客户端重新读取并让管理员确认合并，不自动重试旧表单。官方策略升级只更新 strategyVersion，不能覆盖启停、预算和排除项；管理员配置的实际保存才递增version，策略版本单独演进；运行任务同时记录两者。 + +排除项最多1000条，OWNER 为登录名，REPOSITORY 为 `owner/name`；按合法 GitHub 坐标格式校验、统一大小写并去重，拒绝URL、通配符与查询语法。规范化后整体替换；返回规范化列表。排除按本次已核实的当前坐标匹配，不承诺旧名字跟随改名或转移。 + +调度语义：关闭/排除停止官方新发现及官方独占未登记候选的后续采集，保留候选、出处、进度。已登记来源按自身设置继续；其他独立启用入口需要的候选继续。取消排除或重开自动续排，不能复活已失效的租约。 + +## 4. 有界列表与按需详情 + +- 沿用 `page=0,size=20`；size为1–100，page为0–1000，且 `page*size<=100000`。超出返回400，提示缩小筛选；上限是合同防护，不是深分页性能已达标的证明。默认全部状态，服务端支持 q（owner/name，最长200）、status、channel、paused，多个条件取交集。 +- 仓库按 `(firstDiscoveredAt DESC, candidateId DESC)` 稳定排序，firstDiscoveredAt为候选全局首次发现时间；candidateId为本地候选ID，详情仍按稳定repositoryId定位。首次缺省asOf使用服务器时间，后续沿用响应asOf，仅纳入 `firstDiscoveredAt<=asOf` 的候选。出处按 `(lastSeenAt DESC,id DESC)`、路径按 `(skillPath ASC)`，均继续分页。 +- asOf是插入截止时间，**不是历史一致性快照**：已有记录的状态、排序时间与过滤条件仍可能变化，跨页可发生移动。客户端固定当前显示页，提示有新数据后手动刷新，禁止轮询自动重排；一致性快照或游标迁移要依据D3执行计划/规模测量另行收敛，不能暗示当前设计消除了页间漂移。 +- total仅统计同过滤范围，不在每页重新计算业务成果聚合；实施时采用匹配索引和受限计数/缓存策略，缓存窗口须公开在观测时间中。所有列表最多100项，详情只含摘要；出处、路径、正文、日志不一起打包。 +- progress最多3个默认渠道、100项额度观测；若主体超过上限，`quotaObservationsTruncated=true`，不得把截断后列表当完整额度池。等待项每组最多32项；同原因/工作类别合并。 +- 关闭抽屉保留筛选与位置；页面离开或隐藏时停止轮询，请求不叠加。每次返回observedAt，asOf和observedAt作用不同。 + +## 5. 成果、积压与等待口径 + +近期区间 `[from,until)`，UTC时间，最长90天，from`，多个版本用逗号分隔）和 `SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION`（如 `v1`）。不配置时只能匿名运行，创建凭证会被拒绝。轮换密钥时先保留旧版本并追加新版本为 active，旧密文仍可解；替换 Token 后采用新版本加密，确认所有旧密文都更新后才能移除旧密钥。不要把这些值提交到仓库或输出到日志。部署方应把它们放入 Compose 环境文件或 Kubernetes Secret，并确保 Go 到 Java 的内部连接采用可信网络与 TLS 边界。管理员页面已提供凭证表单及已观测额度；Go 会在有效 GitHub API 响应后按当前任务租约回报额度。额度回报失败不会阻断采集，页面数据可能延迟。 + +## 积压准入水位 + +| 方法 | 路径 | 说明 | +| --- | --- | --- | +| `GET` | `/api/v1/admin/github-sync/admission-policy` | 读取全平台和单 namespace 的高低水位、候选预算、全平台暂停状态、当前未解决项与 `@github` 候选数量及准入状态。 | +| `GET` | `/api/v1/admin/github-sync/worker-status` | 读取 Go 采集服务的最后在线时间和在线判断；从未连接时 `lastSeenAt` 为空。 | +| `PUT` | `/api/v1/admin/github-sync/admission-policy` | 同时设置两组水位和候选预算；写入操作者和新旧值的审计记录。 | +| `GET` | `/api/v1/admin/github-sync/schedule-policy` | 读取仓库更新、精选用户或组织、广泛搜索三类目标周期。 | +| `PUT` | `/api/v1/admin/github-sync/schedule-policy` | 以 `repositoryHours`、`curatedOwnerHours`、`searchRuleHours`（均为 1–720）更新三类周期与已有入口。 | +| `GET` | `/api/v1/admin/github-sync/sources?page=0&size=20` | 分页查看平台已解析来源、最近任务和逐包摄入状态计数，包含失败包数 `failedPackages`。 | +| `GET` | `/api/v1/admin/github-sync/sources/{sourceId}/skills?page=0&size=20` | 查看来源各 Skill 路径最近一次采集、摄入与审核状态。 | +| `POST` | `/api/v1/admin/github-sync/sources/{sourceId}/rescan` | 将平台来源提前纳入正常调度；已有开放任务时合并请求。 | +| `POST` | `/api/v1/admin/github-sync/sources/{sourceId}/retry-rejected-skill` | 对一个最新审核结果为拒绝的 Skill 路径请求重新采集并送审。请求体：`{"skillPath":"skills/example"}`。 | +| `POST` | `/api/v1/admin/github-sync/conflicts/{ingestionId}/bind-skill` | 平台同名冲突确认绑定到已有 Skill ID，并重试该同步包。 | +| `GET` | `/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns` | 查看平台来源追加的候选目录忽略规则。 | +| `PUT` | `/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns` | 用 `{"ignorePatterns":["examples/legacy"]}` 整体替换规则。 | +| `GET` | `/api/v1/admin/github-sync/runs?page=0&size=20` | 分页查看平台发现及仓库采集运行与后续包处理计数。 | +| `GET` | `/api/v1/admin/github-sync/runs/{runId}` | 查看一轮平台运行的类型、状态、时间、任务数、路径数、包处理阶段计数、最近任务错误码和下次重试时间。 | +| `GET` | `/api/v1/admin/github-sync/runs/{runId}/tasks?page=0&size=20` | 分页查看该轮发现或仓库任务的仓库、模式、状态和错误码。 | +| `GET` | `/api/v1/admin/github-sync/runs/{runId}/paths?page=0&size=20` | 分页查看该轮实际产生的 Skill 路径结果、采集结果、摄入错误码及关联审核状态。 | + +运行详情按 `runId` 与平台 namespace 定位，发现运行可能没有 Skill 路径；预览运行以及已按保留策略清理的运行详情返回 404。路径结果来自这轮任务，不把来源的最新路径状态当成历史运行结果。任务和路径分别分页，`page` 从 0 开始，`size` 为 1–100；凭证、任务快照及原始 GitHub 响应不在这些响应中。 + +逐路径响应增加 `latestBindingIngestionStatus`、`retryRequested` 与 `retryInProgress`。最新绑定项为 `REJECTED` 时，管理员可请求逐 Skill 重试；当前忽略的路径及暂停来源不可重试。请求会合并或提前安排来源任务，必要时把排队中的仅 HEAD 任务升级为完整扫描。每次请求只自动安排一次完整采集；任务终止且未产生新待审项时，管理员可以再次请求，不会因目录缺失形成无限重扫。只有请求后的新包才可用本次重试意图重新送审；审核历史和原拒绝记录保留，成功建立新待审目标后清除重试标记。高水位准入仍生效，重试不保证立即执行。 + +来源列表附带最近任务的 `latestTaskErrorCode`、`latestTaskErrorDetail` 和 `latestTaskRetryAt`。诊断详情限于 Worker 或 GitHub HTTP 的已定义代码，不含原始 GitHub 响应、Token 或异常文本。隔离环境已验证有效租约的失败回报会同时持久化诊断码和系统凭证的 SEARCH 额度，含 URL 的详情请求返回 400，旧租约重放返回 409。 + +调度配置初始为仓库 24 小时、精选用户或组织 24 小时、广泛搜索 72 小时。升级部署若仍设置 `SKILLHUB_GITHUB_SYNC_REPOSITORY_CHECK_INTERVAL`，在管理员首次保存后台周期前，仓库完成后的检查时间继续按原环境值计算；`GET` 的 `repositoryHours` 此时为 `null`，`effectiveRepositoryInterval` 返回实际生效的 ISO 8601 时长。保存后后台配置接管仓库周期，并更新已存在的精选仓库、精选用户或组织、搜索规则的目标周期，包括先前在单条入口上设置的周期；已有更早的到期时间不会被延后，已领取/等待重试任务不会因此跳过额度与积压保护。以后新增入口使用类别默认值，单条搜索规则和精选用户仍可另填周期。 + +`PUT` 请求体： + +```json +{ + "globalHigh": 1000, + "globalLow": 500, + "namespaceHigh": 200, + "namespaceLow": 100, + "candidatePendingLimit": 1000, + "candidatePerLeaseLimit": 100 +} +``` + +水位数值仅展示字段格式，不是建议默认值。新部署时两组水位为 `null`，Java 拒绝新的逐包采集准入；管理员应结合试运行的待审数量、包大小、对象存储和审核能力决定实际水位。水位要求 `0 ≤ low < high`；候选预算要求 `1 ≤ candidatePerLeaseLimit ≤ candidatePendingLimit`，并限制为 10000/100000 以内。数据库初始候选预算分别为 100 和 1000，试运行后可调整。`candidatePendingLimit` 限制单 namespace 中排队或正在验证的候选仓库总数，`candidatePerLeaseLimit` 是单次发现租约的新增上限。读取与修改均使用项目统一的 `ApiResponse` 包装；`globalPaused` 是执行状态，不由此接口直接写入。 +当前响应还包含 `globalUnresolvedCount`、`platformUnresolvedCount`、`platformPendingCandidateCount` 和 `platformPaused`。前两项沿用新包准入使用的去重口径，按最新未解决 `(namespace, repository ID, Skill 路径)` 计数，并包含仍持有效租约的包准入占位；候选数量只统计 `@github` 下的 `PENDING`、`VALIDATING`，与候选水位口径一致。`platformPaused` 是 `@github` namespace 的准入暂停状态，与 `globalPaused` 分开显示。平台页面每 30 秒重读这些运行值；它们是观察时刻的快照，不保证多个计数在并发写入下完全同时。 + +## 自动发现配置 + +以下接口要求平台 `SUPER_ADMIN`，所有写入均记审计；返回体为项目统一 `ApiResponse`。 + +| 方法 | 路径 | 说明 | +| --- | --- | --- | +| `GET` | `/api/v1/admin/github-sync/discovery-entries` | 列出平台搜索规则、精选用户或组织、精选仓库入口。 | +| `POST` | `/api/v1/admin/github-sync/discovery-entries/search-rules` | 创建搜索规则，初始停用。 | +| `POST` | `/api/v1/admin/github-sync/discovery-entries/search-rules/preview` | 按同样的结构化规则请求排队预览，返回预览任务 ID；不保存规则或登记来源。 | +| `GET` | `/api/v1/admin/github-sync/discovery-entries/search-rules/previews/{id}` | 轮询状态、各条件的仓库命中估计、最多五个样例及结果不完整信号。 | +| `PUT` | `/api/v1/admin/github-sync/discovery-entries/search-rules/{id}` | 编辑规则并递增版本；进行中的任务继续使用旧快照。 | +| `POST` | `/api/v1/admin/github-sync/discovery-entries/curated-owners` | 登记精选公开用户或组织并启用枚举。 | +| `POST` | `/api/v1/admin/github-sync/discovery-entries/curated-repositories` | 登记精选公开仓库并启用探测；接受 `ownerLogin` 和 `repositoryName`。 | +| `PUT` | `/api/v1/admin/github-sync/discovery-entries/{id}/enabled` | 启用或停用发现入口；重复设置不改变版本。 | + +搜索规则请求示例： + +```json +{ + "title": "Agent Skill 仓库", + "targetHours": 72, + "selectors": [ + {"kind": "KEYWORD", "value": "agent skills", "priority": "CORE"}, + {"kind": "TOPIC", "value": "agent-skills", "priority": "EXTENSION"} + ] +} +``` + +精选用户或组织请求体为 `{"ownerLogin":"example","targetHours":24}`；精选仓库请求体为 `{"ownerLogin":"example","repositoryName":"skills"}`；启停请求体为 `{"enabled":true}`。关键词和 topic 只能通过结构化条件提交，不能传入原始 GitHub 查询语句。首次部署预置一条停用规则，不会自行执行广泛搜索；精选用户入口目前会登记仓库线索。精选仓库由 Go 解析公开仓库及稳定 repository ID，Java 首次登记为来源并进入周期采集；重复解析只能指向首次固定的 repository ID。入口固定 ID 后，后续发现任务优先使用同 namespace 来源绑定的当前仓库坐标，避免改名或转移后持续访问最初填写的地址；来源详情中的坐标反映最近一次已核对的上游元数据。达到积压水位时仍可登记来源，但新包采集受准入水位阻止。来源创建可能晚于配置提交，读取入口的 `repositoryId` 可区分待首次解析与已解析。发现任务每分钟检查到期入口，同一入口若已有排队、运行或等待重试的任务，不会重复排队。 + +发现任务按规则版本和固定搜索窗口执行，核心条件先于扩展条件；同一仓库跨条件命中时按 repository ID 去重。搜索结果超过 1,000 条或 GitHub 返回不完整标志时，Go 拆分原时间窗继续查找，不能将截断页算作完整发现。精选用户和组织的公开仓库先作为带入口出处的候选线索保存，只有后续仓库树与有效 Skill 校验通过才成为正式来源。上述规则、截断处理和两类精选账号的 Java→Go→Java 合同已用受控仓库元数据验证；公开 GitHub 的多仓库规模验收见 6.3。 + +目前已实现入口配置、发现调度、候选仓库任务排队、有效包校验后的来源绑定、精选仓库直接来源登记、已登记来源的周期任务，以及已校验包的 Skill 版本与同步审核任务创建。高水位或尚未配置水位时，新候选不会进入仓库树检查；已登记来源仍可检查 HEAD，新 commit 在容量恢复后进入完整采集。没有有效 Skill 的候选仓库 7 天后重查；读取失败按 1/3/7 天退避，连续三次标记为不可达并继续低频重试。平台管理页首版提供水位配置、精选仓库与账号登记、搜索规则新建、编辑和入口启停，以及来源、运行、逐路径状态查询和来源忽略路径。系统凭证配置和 TEAM 公开仓库来源页面已有首版。无副作用规则预览已有 Java/Go/Web 首版；审核详情页已接入来源、首次文件树、逐文件变更摘要及受限文本差异预览。公开 API 变更须从实际 Java Springdoc 生成 Web 类型。 + +预览沿用 Java 任务租约，由 Go 对每个条件只搜索第一页并回写命中估计和公开仓库样例；搜索显式限定 `is:public`，各条件命中可能重叠，不能相加作为去重仓库数。GitHub 元数据搜索可能漏召回。预览不创建候选、来源或待审 Skill，预览运行从正式采集运行列表排除。隔离 PostgreSQL/Redis 环境中，Java→Go→Java 的真实公网双条件预览已完成，候选、来源和发现批次均为 0；受控响应还覆盖核心优先、样例上限、后续条件限流时不提交部分结果和持久重试。原始 GitHub 查询语法被拒绝，预览租约不能写发现批次，完成后的旧租约不能重复回写。精选仓库的跨进程来源登记及周期任务排队由受控 GitHub 元数据验证；可选的公网精选仓库演练遇到限流并进入 `RETRY_WAIT`，待在 6.3 的小范围验收中重试。 + +来源忽略规则以仓库相对目录前缀表示，最多 64 条，每条最长 1024 字符，不接受 `.`、`..`、空段、反斜杠或控制字符。只有已解析为来源绑定的仓库才能配置；平台默认忽略的 10 类目录始终有效。修改规则会重新安排当前 commit 的完整扫描；旧任务的快照不能把新规则下的来源标记为完整处理。Go 按快照跳过候选，Java 在准入、摄入和发布前按当前规则复核。被忽略的已有 Skill 不会自动删除或撤回。 + +## 同步审核队列 + +`GET /api/v1/reviews/sync?status=PENDING` 单独查询 GitHub 同步审核；每项包含 `review`、`ingestionId`、`runId`、`repositoryId`、仓库 owner/name、`skillPath` 和 `commitSha`。可选 `namespaceId`、`page`、`size` 与 `sortDirection`，权限沿用当前审核规则；来源摘要只对当前页通过权限过滤的审核项批量查询。普通 `GET /api/v1/reviews` 和 `/pending` 只查询普通提交。旧审核记录在迁移后标记为 `USER`；同步审核记录使用 `GITHUB_SYNC` 并关联包摄入 ID。后台交接可以创建新同步审核项；同一 Skill 的新内容会把旧项标记为 `SUPERSEDED`，不再允许批准。来源与许可证证据接口、审核详情页只读展示及变更文件的文本差异预览已提供；平台审核区和 TEAM namespace 审核区已增加独立同步队列，并在当前分页内按仓库与运行归组；每个 Skill 从详情页单独批准或拒绝。队列可查看待审、已通过、已拒绝和已被新版本替代的记录。验收应同时覆盖 Java 队列映射、既有审核服务、Web 组件以及含真实同步审核数据的浏览器流程。 + +同一来源路径的后台发布按数据库事务串行处理，并在事务内重新读取当前绑定。若较新的采集任务已形成待审目标，迟到的旧包会保留为 `SUPERSEDED` 摄入历史，不再创建版本或改写当前绑定。新内容待审期间，上一已发布版本继续可用；旧待审项保留历史但不能再批准。 + +是否升版依据规范化包文件和许可证证据的内容指纹，不依据 GitHub commit 字符串。不同 commit 的内容相同则记为 `NO_CHANGE`；包内许可证文件变化会产生新指纹并重新进入同步审核。内容指纹只用于同一来源绑定的内容判断，不能证明许可证法律有效性。 + +每个 Skill 另有持久化的 `syncRevision` 内容修订号。首次同步为 1；与最近同步目标内容不同则加 1；已拒绝内容显式重试时沿用原号；内容回退仍加 1。`GET /api/v1/reviews/sync/{id}/source`、市场技能详情的 `githubSource`、技能版本列表及市场列表的 `publishedVersion` 均返回该字段；市场列表优先显示“第 N 次修订”。原 `version` 字符串继续作为安装与精确定位用的技术版本，不改接口含义。 + +`GET /api/v1/reviews/sync/{id}/source` 返回单条同步审核的仓库 ID、任务当时的 owner/name、固定 commit、Skill 路径、内容指纹、摄入状态、许可证/NOTICE 证据，以及 `baseVersionId` 和 `fileChanges`。基线为审核提交前最近的已发布版本；首次导入的基线为空，文件列表显示全部新增文件；更新仅列出新增、修改、删除的路径及旧/新 SHA-256、大小，不传文件内容。只有按现有审核规则可查看该审核项的人员能读取；普通审核 ID 返回未找到。证据状态为 `MISSING`、`PRESENT_UNCLASSIFIED` 或 `PRESENT_UNPREVIEWABLE`：`MISSING` 指没有找到许可证类文件，即使存在 NOTICE 也仍为缺失；其余状态只表示取得证据的可预览程度，不推断法律有效性。单文件证据预览受 64 KiB 限制；上游仓库根文件的 Git blob 摘要与 Skill 包文件的 SHA-256 分开标注。同步审核详情页以只读方式展示来源、许可证证据及文件变化；当前版本文件仍使用已有审核预览接口；基线文件由 `GET /api/v1/reviews/sync/{id}/base-file?path=...` 读取。该接口沿用审核查看权限，只允许读取该审核差异中的旧版文件，且单文件不得超过 1 MiB。前端仅对已有文本预览支持的文件类型开放差异按钮。 + +审核详情首屏使用 `GET /api/v1/reviews/{id}/skill-detail?includeDocumentation=false` 读取技能、版本与文件元数据，进入“技能内容”页签后再读取说明正文。省略参数时默认包含正文，保持已有调用行为。逐文件在线预览调用 `GET /api/v1/reviews/{id}/file?path=...&preview=true` 和 `GET /api/v1/reviews/sync/{id}/base-file?path=...&preview=true`；预览最多允许 256 KiB，服务端同时检查记录大小与实际读取字节数，超限返回错误而非截断正文。完整文件下载不带 `preview` 参数，仍沿用原权限与大小规则。首次导入没有旧文件，不请求 `base-file`。历史记录只在打开“审核记录”页签时加载；来源响应不嵌入差异正文。 + +`@github` 目录的许可证目前没有自动识别结果；审核者批准这类同步项时，必须在审核意见中写明处理依据。缺少意见的批准请求返回业务错误，不能因为具有超级管理员权限而跳过。TEAM 来源仍展示同样的证据和警告，并沿用人工审核权限。 + +同步审核来源详情也返回采集 `runId`，详情页显示该 ID，以便从单条审核追溯到采集运行；审核记录自身展示提交者和审核者。首次导入的实际文件树及预览位于详情侧栏，来源卡片的文件列表列出新增路径。安全扫描已有结果时展示扫描卡片；尚在扫描、暂无结果或结果读取失败时，详情页分别显示对应状态。 + +## TEAM 公开仓库来源 + +TEAM namespace 的 OWNER、ADMIN 可使用以下接口；成员和其他 namespace 的管理员不能跨范围读取或修改。冻结或归档的 namespace 可读取配置，但不能新增或修改。请求沿用会话及 CSRF 保护，写入留审计。 + +| 方法 | 路径 | 说明 | +| --- | --- | --- | +| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources` | 列出本团队已配置的公开仓库入口。 | +| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources` | 登记 `ownerLogin`、`repositoryName` 与可选 `visibility`；默认 `NAMESPACE_ONLY`。 | +| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/enabled` | 以 `{"enabled":false}` 暂停、以 `true` 恢复入口和已解析来源。 | +| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/visibility` | 以 `{"visibility":"PUBLIC"}` 或 `NAMESPACE_ONLY` 修改后续导入的可见范围。 | +| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved?page=0&size=20` | 分页查看本团队已解析来源、最近任务及逐包摄入状态计数，包含 `failedPackages`。 | +| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills?page=0&size=20` | 查看本团队来源的逐 Skill 路径状态。 | +| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/rescan` | TEAM 管理员将已解析来源提前纳入正常调度；已有开放任务时合并请求。 | +| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/retry-rejected-skill` | TEAM OWNER/ADMIN 对本团队已拒绝的 Skill 路径请求重新采集并送审；请求体同平台接口。 | +| `POST` | `/api/v1/namespaces/{namespaceSlug}/github-sources/conflicts/{ingestionId}/bind-skill` | TEAM 管理员确认同名冲突或路径变化，并绑定到本 namespace 的已有 Skill ID。 | +| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns` | 查看本团队已解析来源的追加忽略规则。 | +| `PUT` | `/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns` | 在本团队来源上整体替换追加忽略规则。 | +| `GET` | `/api/v1/namespaces/{namespaceSlug}/github-sources/runs?page=0&size=20` | 分页查看本团队的发现与仓库采集运行。 | + +重扫接口不接收包或 commit，也不直接创建待审 Skill。返回 `sourceId`、`status`（`SCHEDULED` 或 `MERGED`）、可选 `taskId`、`taskStatus` 与 `nextCheckAt`。已有 `QUEUED` 任务会提前排序；`LEASED` 任务照常运行；`RETRY_WAIT` 保留原重试时间。没有开放任务时，仅把来源的下次检查时间提前到当前，最多等待正常调度周期。`PAUSED` 来源返回业务错误。正常任务仍按积压准入、GitHub 额度和 HEAD/FULL 判定执行；相同 commit 的完整内容不重新导入。 + +Go 只解析公开 GitHub 仓库；Java 将首次解析的 repository ID 固定到该 TEAM 入口，在 TEAM namespace 中建立独立来源绑定。页面可用返回体的 `repositoryId` 区分待首次解析与已解析。暂停不撤回既有 Skill；已批准入的在途包可结算，之后不再发放新包准入。可见范围变更不追改已发布 Skill。TEAM 管理页提供公开仓库登记、启停、可见范围、逐 Skill 状态及本 namespace 的运行记录。 + +来源、运行和逐 Skill 路径列表的 `page` 从 0 开始，`size` 为 1–100，缺省 20。来源返回最近仓库任务的 `latestTaskStatus`、`latestTaskMode`、错误码与 `latestTaskRetryAt`，另按每个 Skill 路径的最新摄入记录状态分别计数 `receivedPackages`、`processingPackages`、`pendingReviewPackages`、`blockedPackages`、`failedPackages`；旧失败在同一路径重新摄入后不再计入来源当前状态。运行列表的阶段计数仍按该次运行的摄入记录统计，因此保留历史运行结果。`trackedSkills` 是已绑定 Skill 数，包含待审核 Skill，**不是已发布数**。运行的 `ACQUISITION_COMPLETE` 只表示采集结束；`receivedPackages` 等字段用于识别 Java 摄入和审核仍在进行。逐路径接口按 Skill 路径返回最近一次采集 outcome、受控原因码、摄入状态和审核状态；来源 ID 必须属于当前平台或 TEAM namespace，否则返回未找到。本期列表不返回包正文或凭证。 + +平台和 TEAM 管理页面在前台查看时每 30 秒重取来源、运行与已展开的逐 Skill 列表；平台凭证页的已观测额度也同频刷新。手动重扫成功后会立即刷新相关查询；离开页面后不持续轮询。 + +运行列表的 `nextRetryAt` 是该运行中所有 `RETRY_WAIT` 任务最早的重试时间；没有等待重试的任务时为空。它表示采集任务的下一次尝试，不表示 Java 摄入或审核会在此时间完成。 + +运行列表的 `lastErrorCode` 是当前处于等待重试、部分完成或失败的采集任务，以及等待重试、阻塞或失败的包处理中，最近更新的一条错误码；没有这类错误时为空。发现运行没有仓库来源详情，仍可通过此字段查看失败原因。错误码只用于定位当前异常，不包含仓库文件内容或凭证。 + +Go 服务独立于工作池，每 30 秒向内部 `/heartbeat` 发送一次受机器身份保护的心跳。Java 只保存全平台最近一次成功心跳，不保存 Go 主机名或凭证；最近 90 秒内有心跳时 `online=true`。后台每 30 秒刷新，停止或失联后约 90–120 秒显示离线。该状态与高水位造成的“新包准入暂停”是两件事，后台分别显示；`online` 只表示 Go 与 Java 最近可通信，不保证 GitHub 出站访问成功。 + +## 同名冲突与路径变化人工确认 + +发布时若来源 Skill 的规范化 slug 已被同 namespace 的 Skill 占用，该摄入项进入 `BLOCKED`，`ingestionErrorCode` 为 `SLUG_CONFLICT`，逐路径接口同时返回 `targetSkillSlug` 和 `ingestionId`。平台 `SUPER_ADMIN` 可处理 `@github` 中的冲突；TEAM 的 OWNER/ADMIN 只能处理本团队 namespace 的冲突。其他状态不能调用绑定接口，跨 namespace 的摄入 ID 返回未找到。 + +两个绑定接口使用相同请求体： + +```json +{"skillId":123,"reason":"已核对为同一技能，保留现有历史版本"} +``` + +`skillId` 必须是本 namespace 中 slug 相同、未归档且没有普通用户待审版本的 Skill；已经由其他来源管理的 Skill 或已经占用的 GitHub 路径不能再次绑定。`reason` 去除首尾空白后至少 10 字符，最多 1000 字符。成功后返回 `QUEUED_FOR_RETRY`，记录操作者、仓库 ID、来源 ID、路径和处理依据；后台任务会对该固定 commit 的包重新执行发布与审核流程。预留绑定不把当前包指纹当作已发布指纹，因而不会错误地判为无变化。创建同步待审版本时，Skill 的技术所有者转为保留的系统 actor，由 namespace 管理；若创建失败，原所有权保持不变。此操作不会直接批准或公开新内容。 + +完整仓库扫描结束后，若旧路径消失，且新路径的完整包内容指纹只匹配本仓库一个已绑定 Skill，Java 自动更新路径绑定、沿用 Skill ID，并记录系统审计；相同内容不创建新版本。旧路径仍出现在本次扫描结果时不会被认定为移动。多个候选或内容变化导致无法唯一匹配时，摄入项标记 `PATH_MOVE_AMBIGUOUS`，不得自动新建 Skill；进行中的扫描等待完成；终止的部分扫描不据此推断目录消失或自动移动。若部分状态仅由其他包的确定性失败造成，新路径已接收且所有既有绑定路径都在本次结果中，可作为独立 Skill 进入同步审核；旧路径未得到确认时仍标记 `PATH_MOVE_SCAN_INCOMPLETE`。 + +平台 `SUPER_ADMIN` 或 TEAM OWNER/ADMIN 可用上述绑定接口处理 `PATH_MOVE_AMBIGUOUS`。逐路径列表显示已绑定的 Skill ID；管理员应核对旧路径及目标 Skill 后提交其 ID 和处理依据。服务端再次验证本次任务为完整扫描、新路径已采集、旧路径在该快照中确实不存在、目标 Skill 未归档且已由同 namespace、同仓库来源管理。成功后保留 Skill ID 与历史版本，记录操作者及旧新路径，并重新处理固定 commit 的包；内容变化才创建待审版本，相同内容只更新来源路径。接口本身不批准内容。`PATH_MOVE_SCAN_INCOMPLETE` 不能人工确认为移动，须等后续完整重扫。被当前忽略规则覆盖的旧路径或新路径也不能通过人工确认。 + +来源绑定记录上游 `UNKNOWN/PRESENT/MISSING`、最近核对 commit 和首次缺失时间，并通过逐路径响应的 `upstreamPresence`、`presenceCommitSha`、`missingSince` 展示。`UNKNOWN` 表示尚无有效完整核对；`PRESENT` 和 `MISSING` 都只说明最近一次有效完整扫描的结果，不是实时 GitHub 状态。仅完整扫描且忽略规则未变化时对账；部分扫描、暂停来源和当前忽略的路径不触发缺失推断，后续完整重扫看到旧路径会恢复 `PRESENT`。不可达来源成功完成扫描后恢复为 `ACTIVE`，旧快照不能覆盖较新的状态；过期扫描或规则变化的路径移动摄入项标记 `PATH_MOVE_STALE_SNAPSHOT`。目录消失本身不会删除、撤回或隐藏已发布 Skill。 + + +## 统一技能同步管理 + +产品方案见[统一管理入口](../../openspec/changes/add-github-skill-sync/unified-admin.md)，状态与一致性约束见[实施合同](../../openspec/changes/add-github-skill-sync/unified-admin-contract.md)。下列接口固定管理平台 `@github`，需要 `SUPER_ADMIN` 会话；写请求沿用 CSRF，返回既有 `ApiResponse` 包装并设置 `Cache-Control: no-store`。TEAM 继续使用 namespace 来源接口。 + +### 仓库列表与详情 + +根路径为 `/api/v1/admin/github-sync`： + +| 方法与路径 | 查询参数与行为 | +| --- | --- | +| `GET /repositories` | `page=0`、`size=20`（1..100）、可选 `asOf`、`q`（最长200字符）、`originType`、`state`、`sourceStatus`。按 namespace 与 GitHub 稳定仓库 ID 合并候选和来源，筛选后分页。 | +| `GET /repositories/{repositoryId}` | 返回仓库摘要及最近检查/运行；仅其他 namespace 存在的仓库同样返回404。 | +| `GET /repositories/{repositoryId}/origins` | `page`、`size`、`asOf`；分页展示各发现方式的持久出处，按类型与来源键去重。 | +| `GET /repositories/{repositoryId}/paths` | `page`、`size`、`asOf`；复用逐 Skill 路径及审核安全投影，按需读取。 | + +分页返回 `items / total / page / size / asOf`。非法枚举或分页参数返回400。`asOf` 是新增记录的时间截止点，防止新发现不断挤动当前分页，不代表跨请求冻结全部状态；同次列表请求的总数与列表使用一致事务快照。 + +仓库摘要字段：`repositoryId`、可空 `sourceId / candidateId`、`owner / name`、`firstSeenAt`、`state`、可空 `sourceStatus / candidateStatus`、`originTypes`。`state` 取 `REGISTERED / PENDING / VALIDATING / NO_VALID_SKILL / VALIDATION_FAILED / UNREACHABLE / BOUND_WITHOUT_SOURCE`，只表达登记阶段，不表示审核通过或市场可见。已登记来源状态另用 `ACTIVE / PAUSED / UNREACHABLE`。出处类型为 `CURATED_REPOSITORY / CURATED_OWNER / SEARCH_RULE / OFFICIAL_DISCOVERY / TEAM_CONFIG`，不会因此扩大到 TEAM 数据。 + +详情返回 `repository` 及可空 `latestRunId / lastCheckedAt / nextCheckAt / lastErrorCode`。出处返回 `originType / originKey / firstSeenAt / lastSeenAt / entryId / channel / queryId / runId / seedRepositoryId`；没有实际观测的字段为空，已清理详情的运行不提供失效链接。旧 `/sources` 和官方探索专用接口保留兼容。 + +### 指定仓库与运行 + +指定仓库重复添加返回同一 namespace、同类型、同目标的已有入口；不恢复其停用状态，不修改既有到期时间或重复排任务。尚未解析稳定仓库 ID 的入口在采集配置中展示，不计入统一仓库数量。 + +入口响应增加可空 `latestExecution`：`taskId / runId / status / errorCode / retryAt`，按持久任务的规则 ID 关联，不凭名称猜测。未知字段为空，不返回原始任务快照或外部异常正文。 + +既有 `GET /runs` 增加可选 `kind / status`，保留原分页调用。`kind` 为 `DISCOVERY / REPOSITORY_SYNC`，状态为 `QUEUED / LEASED / RETRY_WAIT / PARTIAL / ACQUISITION_COMPLETE / FAILED`。列表排除预览和已清理详情；`createdAt` 是创建时间，不是实际开始时间。 + +每项增加 `targetCount` 及可空 `target: {type, value}`：单目标显示摘要，多目标显示数量；同仓库优先按稳定 ID 去重，缺失时才按规范化坐标区分。HEAD_ONLY/FULL 仍在仓库任务详情中区分，不拆成虚构运行类型。 + +### 平台暂停与恢复 + +`GET /platform-control` 返回： + +| 字段 | 说明 | +| --- | --- | +| `namespaceId`、`version` | 平台 namespace 与单调控制版本。 | +| `desiredPaused` | 管理员保存的暂停意图。 | +| `state` | `RUNNING / PAUSING / PAUSED`；RUNNING只代表没有暂停，不证明Go在线。 | +| `pendingStopCount` | 当前暂停版本仍未确认停止的在途任务数。 | +| `expiredUnconfirmedCount` | 上述任务中租约已过期但仍无停止确认的数量。 | +| `updatedAt` | 持久状态更新时间。 | + +`PUT /platform-control` 请求示例： + +```json +{"expectedVersion": 3, "paused": true} +``` + +请求使用最近读取的版本，响应结构同 GET。并发版本冲突返回409，重新读取后再决定操作；相同版本、相同意图不反复推进版本。变更记录管理员审计。 + +暂停独立于自动探索和容量策略，阻止平台的新发现、检查和下载，TEAM不受影响。Go执行线程到安全点后确认；过期或失联不能被当作已停止。Java已经接收的包及人工审核继续，恢复不覆盖各来源和配置独立启停。部署须配套支持 `platform-pause/v1` 的Go版本，详细边界见[内部协议](github-sync-internal.md#9-平台github采集安全暂停)。 diff --git a/docs/api/github-sync-internal.md b/docs/api/github-sync-internal.md new file mode 100644 index 000000000..c1db95461 --- /dev/null +++ b/docs/api/github-sync-internal.md @@ -0,0 +1,322 @@ +# GitHub 同步内部接口说明 + +> **用途：当前 Java–Go 内部协议说明。** Java 已实现按类型 `claim`、`renew`、系统凭证临时下发与当前租约的额度/认证失效/限流回报、发现候选批次接收与结算、候选及已登记来源的仓库任务调度、仓库 `HEAD_ONLY` / `FULL` 结算、分类 `fail` 与受控诊断码、逐包准入、上传授权、摄入回报、后台归档校验，以及已校验包到同步 Skill 版本和独立审核任务的持久交接；Go 已实现常驻双工作池、发现执行器、选择性包获取和仓库任务处理器，并接入可选的 Compose、Helm 与 K8s 部署入口。规则预览不创建来源或待审项；具体集成验收入口和边界见 [验证说明](../../openspec/changes/add-github-skill-sync/verification.md)。`fail` 可在有效租约内同时回报系统凭证额度与受控诊断码；不持久原始异常文本。接口目标合同以同目录的 [OpenAPI YAML](github-sync-internal.openapi.yaml) 为准；产品行为、来源归属和审核规则以 [OpenSpec change](../../openspec/changes/add-github-skill-sync/README.md) 为准。本文解释流程，不另建一套协议。 + +## 1. 调用边界 + +- **调用方向：** Go 主动请求 Java。Java 保存配置、持久任务、租约、路径结果、包摄入与审核状态；Go 不连接 Java 数据库或 Redis，也不调用普通用户发布接口。 +- **基础路径：** `/internal/github-sync/v1`，只供内部网络访问，不暴露为面向浏览器的公开 API，也不加入 Web SDK。 +- **认证：** 所有接口使用独立服务身份 `Authorization: Bearer `。浏览器会话、普通用户/API Token、GitHub Token 均不能代替服务身份。本地流式上传还须提供 `X-Upload-Ticket`。 +- **当前 Java 配置：** `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 接受逗号分隔的新旧服务密钥，每枚至少 32 个字符；不配置时内部接口拒绝所有请求。`SKILLHUB_GITHUB_SYNC_LEASE_DURATION` 初值为 `PT5M`，需在小范围联调中校准。 +- **上传授权：** `SKILLHUB_GITHUB_SYNC_UPLOAD_TICKET_TTL` 默认 `PT5M`，最长一小时；S3/MinIO 的暂存 PUT 优先以 `SKILLHUB_STORAGE_S3_ENDPOINT` 签名，公开 GET 继续使用 `SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT`，本地存储使用 Java 同源相对路径与独立上传票据。Go 的 S3 目标主机须由部署方显式允许。回报前 Java 检查对象存在及实际大小；SHA-256 和包内容由后续摄入工作流再验证。 +- **暂存清理：** `SKILLHUB_GITHUB_SYNC_SIGNED_UPLOAD_CLEANUP_GRACE` 默认 `PT1H`，`SKILLHUB_GITHUB_SYNC_UPLOAD_CLEANUP_INTERVAL_MS` 默认 `300000`。未被摄入接受且已失效的上传，以及已完成版本复制、无变化或校验终止失败的对象可以清理；S3 票据须先经过宽限期，删除失败保留领取状态供后续重试。仍在校验、等待版本绑定或因冲突阻断的对象不会按固定 TTL 清理。 +- **包上限变化：** Java 在逐包回报时先核对已持久接收的幂等结果，再按当前包上限复核尚未接受的票据与实际对象大小。票据签发后上限调低或 S3/MinIO 实际对象超限时，回报 `TOO_LARGE`；仅在票据从未接收状态成功转为失败后尝试删除对象，避免并发回报删除已接收包。删除失败由暂存清理任务补偿，本地已上传对象也可立即进入补偿清理。 +- **后台校验：** `SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_LEASE` 默认 `PT30M`，`SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_INTERVAL_MS` 默认 `5000`。Java 从 `RECEIVED` 领取单包，复核归档大小、SHA-256、ZIP 符号链接及条目边界，并复用现有 Skill 包校验器；通过后标记 `VALIDATED`，仍未创建版本或审核目标。失败留错误码，未接受或确定失败的对象由清理任务按状态处理。 +- **后台发布交接：** `SKILLHUB_GITHUB_SYNC_PUBLICATION_LEASE` 默认 `PT4H`，`SKILLHUB_GITHUB_SYNC_PUBLICATION_INTERVAL_MS` 默认 `5000`。Java 独立领取 `VALIDATED` 包，锁定摄入记录并重新核对归档与内容指纹；同内容转 `NO_CHANGE`，新内容在同一数据库事务中创建 Skill 版本、来源路径绑定与 `GITHUB_SYNC` 审核任务，再转 `READY_FOR_REVIEW`。审核批准、拒绝或被更新目标取代后，摄入状态结算；待审前不会公开发布。归档按受限临时目录处理，真实容量与对象存储行为仍须联调。 +- **许可证证据：** Go 从固定 commit 的 Tree 元数据挑选仓库根目录的 LICENSE、LICENCE、COPYING、NOTICE、COPYRIGHT 及常见后缀文件，最多 16 个；单文件最多 64 KiB、预览内容合计最多 256 KiB。超过预览限制的文件仍记录 Git blob 摘要与大小，不下载全文。Go 在 ZIP 中使用 `__skillhub_sync_evidence__.json` 清单及 `__skillhub_sync_evidence__/` 文件区传递证据；Java 重新校验大小和已携带文件的 Git blob 摘要，把仓库根证据纳入正式内容指纹并单独保存证据快照。这些内部条目不进入 Skill 安装包。缺失或无法预览的许可证信息保留为审核警告；当前不自动判定 SPDX 许可证，也不自动批准。 +- **仓库观测周期：** `SKILLHUB_GITHUB_SYNC_REPOSITORY_CHECK_INTERVAL` 初值为 `PT24H`，只在后台尚未保存仓库目标周期时作为兼容回退。平台管理员通过 `/api/v1/admin/github-sync/schedule-policy` 配置仓库更新、精选用户或组织及广泛搜索的类别目标周期；`HEAD_ONLY` 和完整仓库任务完成后，Java 按当前后台值设置来源下次检查时间。 +- **候选忽略：** 已登记来源的仓库任务快照携带 `ignorePatterns`，Go 在固定 commit 的树上只过滤候选 Skill 根，不删除父包内的普通文件。Java 对当前来源规则复核：新包准入返回 `PATH_IGNORED`，已接收包在摄入或发布前转为忽略终态。规则变化后旧任务不能推进来源的完整处理检查点；已发布内容不自动撤回。 +- **调度周期：** `SKILLHUB_GITHUB_SYNC_DISCOVERY_SCHEDULER_INTERVAL_MS`、`SKILLHUB_GITHUB_SYNC_CANDIDATE_SCHEDULER_INTERVAL_MS`、`SKILLHUB_GITHUB_SYNC_SOURCE_SCHEDULER_INTERVAL_MS` 初值均为 `60000`。已登记来源的排队批次按 namespace 轮流选取到期来源，仓库任务领取优先已登记来源，并参考各 namespace 最近成功领取时间，避免单一 namespace 持续占用领取名额；候选来源仍按精选用户优先于搜索候选。Java 只在配置了水位且低于恢复边界时启动新候选树检查；已登记来源在反压期间按到期计划改做 `HEAD_ONLY`。仓库完整处理检查点要等待完整任务及其相关包校验结束，不随单次包回报提前推进。 +- **仓库改名与转移：** Go 在仓库元数据请求中仅跟随 `https://api.github.com` 内的有限次数跳转，取得响应后先核对稳定的 repository ID，再通过当前任务租约调用 `repository-metadata`。Java 再核对 ID 并更新来源或候选坐标与任务快照；后续 HEAD、树和包获取使用 GitHub 返回的新坐标，市场链接优先使用来源的当前坐标。跨主机跳转不会携带系统凭证继续请求。旧名称被别的仓库占用时，ID 不匹配会阻断处理；旧地址的跳转失效后，当前方案不能仅凭旧坐标自动找回新地址，需由后续发现命中同一 ID 或管理员处理。跳转处理依据 [GitHub REST API 最佳实践](https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api#follow-redirects)；GitHub 也说明[转移后的旧地址可能被复用而失去跳转](https://docs.github.com/en/repositories/creating-and-managing-repositories/transferring-a-repository)。 +- **历史清理：** `SKILLHUB_GITHUB_SYNC_COMPLETED_DETAIL_RETENTION` 默认 `P90D`，`SKILLHUB_GITHUB_SYNC_HISTORY_CLEANUP_INTERVAL_MS` 默认 `3600000`。调度器每轮限量清理已完成且无恢复引用的发现任务及空运行；候选入口归属独立保存，不因任务清理而丢失精选优先级或来源归属。已完成的仓库任务在包摄入均已结算且不再承担候选恢复后缩减快照和检查点；被更新结果取代的逐路径明细删除，来源仍需展示的最新路径状态保留。仓库任务/运行 ID 继续支撑已发布版本和审核追溯，过期运行不再出现在后台运行列表。未完成任务、待审或冲突项及其恢复数据不按期限清理；容器日志由部署环境管理。 +- **逐包续跑：** 仓库任务领取响应的 `acceptedPaths` 来自同一任务、固定 commit 下已被 Java 持久接收的包；即使检查点尚未提交，Go 也跳过这些路径。`retryPaths` 标识需要重试的失败路径，即使位于检查点之前仍要处理。两者均由 Java 的逐路径持久状态计算，不由 Go 本地文件推断。两个独立 Go HTTP 客户端进程验证已接收包不重复上传；实际仓库 worker 再用受控 GitHub Tree 跨进程验证失败路径重访、Java 持久 `retryAt`、到期重领和终态结算。两条链路均在本地和 MinIO 模式通过；GitHub Tree 网络请求仍由受控 fixture 代替。 +- **请求关联：** 每次调用均带独立 `X-Request-Id`；它只用于日志关联，不能充当幂等键。Go 到 Java 的 HTTP 客户端应与 GitHub 出站代理客户端分开。 +- **协议版本：** 路径主版本为 `v1`，`claim` 请求还传 `protocolMajor: 1`。不兼容主版本返回 `426 PROTOCOL_UNSUPPORTED`；兼容的可选字段增加不得改变已有状态含义。 +- **敏感数据：** `claim` 响应中的可选公开读取 GitHub Token、租约令牌、上传 URL/票据只在执行时使用，不能进入持久任务、日志、异常、指标标签或仓库 URL。服务身份与 GitHub 凭证是两种独立密钥。 + +以下示例中的 ID、SHA、路径和占位符只用于说明格式，不代表现有环境数据或可直接调用的服务。 + +## 2. 一次任务的调用顺序 + +```mermaid +sequenceDiagram + participant Go as Go 采集服务 + participant Java as SkillHub Java + participant Stage as 临时对象存储 + Go->>Java: POST /tasks/claim + Java-->>Go: 200 任务快照 + 当前租约 / 204 无任务 + loop 任务仍在执行 + Go->>Java: POST /tasks/{id}/renew + Java-->>Go: 新的 leaseExpiresAt + end + alt 发现任务 + Go->>Java: POST /tasks/{id}/discovered-repositories + Java-->>Go: 202 候选和游标已持久化 + else 仓库 FULL 任务 + Go->>Java: POST /tasks/{id}/package-admissions（每个新包下载前） + Java-->>Go: 200 准入 ID / 409 积压暂停 + alt 已获准 + Go->>Java: POST /tasks/{id}/upload-authorizations（逐包） + Java-->>Go: 临时对象上传授权 + Go->>Stage: PUT 单个 Skill 包 + Go->>Java: POST /tasks/{id}/packages/report（逐包） + Java-->>Go: 202 摄入记录已持久化 + else 积压暂停 + Note over Go,Java: 保存路径进度，不开始下载下一个包 + end + else 仓库 HEAD_ONLY 任务 + Note over Go,Java: 只观测当前 commit，不上传包 + end + Go->>Java: POST /tasks/{id}/complete 或 /fail + Java-->>Go: 任务采集状态已持久化 + Note over Java: 后台另行复核、扫描、建待审目标；人工审核后才发布 +``` + +`claim` 和 `renew` 建立当前租约代际。Go 的发现池与仓库池分别用 `kind` 领取；只在租约有效时进行上传授权、批次回报、逐包回报和任务结算，过期后须停止，等待 Java 重派。发现任务和仓库任务共用接口，但 `discovery`、`repository` 与 `mode` 由任务类型决定。仓库第一次领取时 `fixedCommitSha` 可为空，由 Go 固定本轮 commit；续跑时 Java 下发原固定 commit、`pathCheckpoint`、`incompleteSubtrees`、`retryPaths` 和 `acceptedPaths`。 + +发现批次须携带 `previousCheckpoint` 与新的 `checkpoint`。首批的前置值为 `null`；Java 在有效租约内原子保存候选、任务命中和新游标，相同批次重放不重复计数。任务快照中的 `candidateBudget` 限制当前租约新增线索数；Java 另以管理策略中的 `candidatePendingLimit` 限制该 namespace 中排队或正在验证的候选总数。前置游标已过期、同游标内容冲突或超出任一预算时拒绝整批，不能把搜索命中直接视为正式来源。重领任务时，`discovery.checkpoint` 由 Java 已持久游标覆盖原始快照；待验证候选未被消费前不会无限扩张。 + +发现任务调用 `complete` 时必须引用最后一批已确认的 `discoveryCheckpoint`。覆盖完整返回 `ACQUISITION_COMPLETE`；尚有未查页面或需延期的窗口时使用 `INCOMPLETE` 和未来的 `retryAt`，任务进入 `RETRY_WAIT`。这只结算搜索采集，不代表候选仓库已验证或 Skill 已发布。 + +`SEARCH_PREVIEW` 是发现池的独立任务快照。Go 对各结构化条件最多查询 GitHub 搜索第一页，经 `POST /tasks/{taskId}/preview-results` 回传每条件总量估计、不完整/千条上限信号与最多五个公开仓库样例；Java 在有效租约内原子写入任务快照并结算。预览不调用发现候选批次接口，不创建候选、来源、包或待审 Skill。Java→Go→Java 的两条件完成与第二条件限流场景已用受控搜索页联调：限流时只保存未来重试时间，不留下部分预览结果。各条件可能命中同一仓库，命中数不能直接相加；元数据搜索也可能漏召回。 + +`CURATED_REPOSITORY` 任务携带固定的 `owner` 与 `repositoryName`。Go 通过公开 GitHub API 解析 repository ID，最多回报一个仓库；Java 将首次解析的 ID 固定到精选入口，并直接登记平台来源。后续任务若解析为另一个 ID，Java 拒绝该批次。此类明确配置的来源不等待 Skill 候选验证；积压水位仍控制后续包准入。 + +`SEARCH_RULE` 的 `discovery.criteria.selectors` 是固定规则版本：每项为 `KEYWORD` 或 `TOPIC`，并标记 `CORE` 或 `EXTENSION`；`searchWindow` 固定本轮增量或全量重查时间范围。`KEYWORD` 仅搜索仓库名称、描述和 topics，`TOPIC` 限定 topic；每项单独查询并按 repository ID 合并。Go 不接受原始 GitHub 查询语法，不搜索 README 或代码；分页、GitHub 每查询最多 1,000 条的上限和 `incomplete_results` 必须纳入检查点与不完整结果处理。[GitHub 仓库搜索语法](https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories)、[REST 搜索限制](https://docs.github.com/en/rest/search/search)。 + +高水位期间，发现任务仍可在 GitHub 额度与 `candidateBudget` 内回报去重的仓库线索和游标；Java 暂不把这些线索转换为已验证来源或安排新候选仓库树检查。预算满时保存游标并延后。已登记仓库只执行 `HEAD_ONLY`；Go 不领取新的 `FULL` 任务，已领取的 `FULL` 任务在下一个包前须重新申请准入。低水位恢复后，Java 重新核对候选仓库与来源的当前上游状态，再安排包采集。 + +## 3. 接口一览 + +以下路径均相对于 `/internal/github-sync/v1`。所有请求都要服务身份和 `X-Request-Id`；`{taskId}`、`{uploadId}` 在 YAML 中定义为 UUID。 + +| 方法与路径 | Go 提交 | Java 的成功确认 | 主要边界 | +| --- | --- | --- | --- | +| `POST /tasks/claim` | `workerId`、`protocolMajor`、`kind` | `200 ClaimedTask`；无任务 `204` | Java 按发现/仓库类型原子领取，返回任务快照、租约和可选临时 GitHub 凭证。 | +| `POST /tasks/{taskId}/renew` | `lease`（代际、令牌） | `200 leaseExpiresAt` | 仅当前代际可续租；旧租约 `409`。 | +| `POST /tasks/{taskId}/preview-results` | 当前 `lease`、各条件命中估计与样例 | `200 ACQUISITION_COMPLETE` | 只允许预览任务，不写发现候选。 | +| `POST /tasks/{taskId}/credential-status` | 当前租约、凭证 ID、`OBSERVED`、`AUTH_INVALID` 或 `RATE_LIMITED`；`OBSERVED` 必须带额度，`RATE_LIMITED` 必须带 `retryAt` | `204` | 只有该租约领取时绑定的凭证可更新；认证失效和限流分别处理，旧 Token 不能误伤已替换的新 Token。 | +| `POST /tasks/{taskId}/discovered-repositories` | 当前租约、公开仓库候选批次、续跑 `checkpoint`、延后统计 | `202`，批次与游标已持久化 | 仅发现任务使用；搜索命中不是正式来源，更不是已发布 Skill。 | +| `POST /tasks/{taskId}/package-admissions` | 当前租约、固定 commit、Skill 路径 | `200 admissionId`；高水位 `409 BACKPRESSURE_ACTIVE` | 仅仓库 `FULL` 任务使用；Go 在获取文件内容前申请，Java 原子预留容量。 | +| `POST /tasks/{taskId}/upload-authorizations` | 当前租约、`admissionId`、固定 commit、Skill 路径、包大小与 SHA-256 | `200 UploadAuthorization` | 仅仓库 `FULL` 任务使用；授权绑定已准入包和临时对象。 | +| `PUT /uploads/{uploadId}/content` | 二进制包、上传票据 | `204`，本地临时对象已写入 | 仅 `LOCAL_STREAM` 模式使用；接收时执行大小硬上限。 | +| `POST /tasks/{taskId}/packages/report` | 当前租约、`admissionId`、固定 commit、Skill 路径、上传 ID、包大小与 SHA-256 | `202 ingestionId`、`RECEIVED` | Java 已持久接收摄入记录；准入占位转为未解决摄入项，不表示内容校验、审核或发布成功。 | +| `POST /tasks/{taskId}/complete` | 租约、覆盖情况、逐路径结果、观测 commit、可选额度/重试时间 | `200`，Java 权威任务状态 | `COMPLETE`、`INCOMPLETE`、`HEAD_ONLY` 的检查点推进规则不同。 | +| `POST /tasks/{taskId}/fail` | 租约、分类原因、可选 `retryAt`、额度与受控诊断码 | `200`，失败/重试状态已持久化 | 仅接受 `WORKER_TIMEOUT`、`WORKER_ERROR` 或 OpenAPI 所列的 GitHub HTTP 诊断码，不保存原始异常文本。 | + +### 领取任务示例 + +Go 发送： + +```json +{ + "workerId": "github-sync-worker-1", + "protocolMajor": 1, + "kind": "REPOSITORY_SYNC" +} +``` + +Java 对仓库 `FULL` 任务的响应示意（省略可选的 `credential`）： + +```json +{ + "taskId": "11111111-1111-4111-8111-111111111111", + "runId": "22222222-2222-4222-8222-222222222222", + "kind": "REPOSITORY_SYNC", + "mode": "FULL", + "namespaceId": 33, + "sourceId": 44, + "lease": { "generation": 2, "token": "" }, + "leaseExpiresAt": "2026-09-28T12:15:00Z", + "repository": { "repositoryId": 123456, "owner": "example", "name": "skills" }, + "maxPackageBytes": 104857600, + "maxSingleFileBytes": 10485760, + "maxFilesPerPackage": 100, + "maxUploadBytes": 104857600 +} +``` + +发现任务的 `discovery` 快照示意： + +```json +{ + "entryType": "SEARCH_RULE", + "ruleVersion": 3, + "criteria": { + "selectors": [ + { "kind": "KEYWORD", "value": "agent skills", "priority": "CORE" }, + { "kind": "TOPIC", "value": "agent-skills", "priority": "EXTENSION" } + ] + }, + "searchWindow": { + "mode": "INCREMENTAL", + "since": "2026-09-27T00:00:00Z", + "until": "2026-09-28T00:00:00Z" + }, + "candidateBudget": 100 +} +``` + +若响应为 `204`，Go 按运行配置退避后再轮询；不能把空队列记为同步成功。示例展示当前发布策略的默认值，部署的有效配置可改变下发上限。 + +### 逐包上传与回报 + +包下载前的准入请求示意： + +```json +{ + "lease": { "generation": 2, "token": "" }, + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "skillPath": "skills/example" +} +``` + +Java 准入后返回 `{"admissionId":"66666666-6666-4666-8666-666666666666"}`；该 ID 仅随当前租约、commit 和路径有效。若返回 `409 BACKPRESSURE_ACTIVE`，Go 不获取该包的文件内容。 + +1. Go 从固定 commit 的树元数据枚举到一个 `SKILL.md` 根目录，完成路径及大小预检后，先调用 `package-admissions`。Java 原子检查全局与 namespace 积压及未完成准入占位，返回绑定当前任务、commit 和路径的 `admissionId`；高水位返回 `409 BACKPRESSURE_ACTIVE`。Go 在准入被拒时不下载文件，保存路径进度并结算部分任务。已获准且正在处理的包可以完成；租约失效后不得继续下载。 +2. 准入成功后，Go 只把所需文件下载到受限本地临时目录。按当前有效发布政策预检，当前 `application.yml` 配置为最多 100 个文件、单文件 10 MiB、包总量 100 MiB；逐包打包并计算**上传对象**的 SHA-256 与字节数。此摘要用于传输完整性；正式 Skill 内容指纹由 Java 解包后按规范化文件计算。 +3. Go 携带 `admissionId` 调用 `upload-authorizations`，取得 `uploadId`、`mode`、`uploadUrl`、到期时间和必要标头。S3/MinIO 模式使用短时预签名 PUT；本地模式调用受服务身份与上传票据保护的 Java 流式 PUT。Go 仅向部署配置允许的存储主机上传，不跟随重定向。 +4. PUT 成功后，Go 携带 `admissionId` 调用 `packages/report`，只交 `uploadId` 与完整性信息，不提供任意下载 URL。Java 在短事务中记录任务、commit、规范化路径、对象引用和摘要，返回同一 `ingestionId`，并将准入占位转换为未解决摄入项。Java 用自己的存储适配器由 `uploadId` 定位临时对象，再核对实际对象、解包规则、许可证、正式内容哈希、安全扫描和审核目标。未使用的准入在任务结算或租约失效后释放。 +5. Go 可以继续处理同仓库其他包，但每个包都要重新申请准入。单包失败记录在路径结果中，不能让其他包失去已完成进度。 + +取得文件之前，Go 先通过固定 commit 的 GitHub Tree 元数据找到候选 `SKILL.md`，按子树处理截断响应，并依据 blob 大小和类型做预检。`blob:none` 仍可能在本地保存整个仓库的树元数据，稀疏检出会按需获取所选文件；因此工作目录预算要包含 `.git`、检出文件和归档，不能只算最终 Skill 包。逐包回报 `202` 后清理可释放的本地文件；接近预算时用 `pathCheckpoint`、`deferralReason: LOCAL_DISK_BUDGET` 和 `retryAt` 回报阶段结果，清理本批目录，从同一固定 commit 续跑。不完整范围不触发上游删除判断。[GitHub Tree API](https://docs.github.com/en/rest/git/trees)和[Git partial clone 文档](https://git-scm.com/docs/partial-clone)分别说明了树响应截断与延迟获取对象的边界。 + +`LOCAL_DISK_BUDGET` 延后只适用于清理工作目录后仍能处理下一批的情形。若当前容量连固定 commit 元数据或一个合规包都无法处理，Go 以 `coverage: INCOMPLETE`、`deferralReason: CAPACITY` 回报容量阻塞，Java 等待容量或规则调整后的管理员重试；不能给同一条件下的任务设置极短 `retryAt` 形成领取循环。预检须包含文件落地、归档及安全余量，且工作区应有部署可验证的独立容量边界；Kubernetes 临时存储限制可能通过驱逐 Pod 生效，不能视作对每次写入的精确拦截。[Kubernetes 临时存储资源说明](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)。 + +逐包回报请求示意： + +```json +{ + "lease": { "generation": 2, "token": "" }, + "admissionId": "66666666-6666-4666-8666-666666666666", + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "skillPath": "skills/example", + "uploadId": "55555555-5555-4555-8555-555555555555", + "sizeBytes": 4096, + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +} +``` + +S3/MinIO 预签名 PUT 的 `maxBytes` 是 Go 应遵守、Java 随后复核的边界，不能仅凭 PUT 签名宣称对象存储接收时已强制最大字节数。临时对象须存放在隔离区域并有容量告警、超额清理；本地流式入口须在接收时执行硬限制。已被 Java 接收但尚未处理的对象不能只按固定 TTL 删除；失败对象需有清理与重采集记录。 + +### 暂存容量监控 + +Java 每隔 `skillhub.github-sync.upload-cleanup-interval-ms` 采样一次暂存账面预留量与本地磁盘容量。`skillhub.github.sync.staging.reserved.bytes` 汇总尚未清理的上传票据所声明的字节数；它不能反映绕过客户端限制的超额 PUT。使用本地存储时，`skillhub.github.sync.staging.local.available.bytes` 是存储目录所在文件系统的可用字节数；低于一个允许的最大 Skill 包大小或读取失败时，`skillhub.github.sync.staging.local.capacity.warning` 为 `1`，并输出警告日志。 + +S3/MinIO 模式下，Java 默认每小时以最多 100 次列举请求、10,000 个对象的上限扫描 `github-sync/staging/` 前缀。完整扫描时，`skillhub.github.sync.staging.s3.used.bytes` 与 `.s3.objects` 记录实际字节数和对象数，`.s3.scan.complete` 为 `1`；扫描不完整或读取失败时，实际字节数为 `-1`，`.s3.capacity.warning` 为 `1`，防止把截断结果误判为有余量。部署方须在启用 S3 同步前配置 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES` 为该暂存前缀可使用的正数字节预算；达到预算或未配置预算且暂存区已有对象时也告警。没有同步对象时不会仅因预算未配置而告警。`SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_SAMPLE_INTERVAL_MS` 可调整采样周期，默认 `3600000`。这项预算只约束告警判断，不代表对象存储服务的真实剩余空间或上传硬配额；仍需按存储服务的总容量/配额监控其余对象。指标是否通过 Actuator 暴露取决于部署的端点配置。 + +### 完成与失败 + +`complete` 的 `coverage` 明确三种情况： + +| 值 | 允许的效果 | +| --- | --- | +| `COMPLETE` | 固定 commit 的树枚举完整，逐路径结果已持久化；仅在所有需处理路径均有持久采集结论且无待重试范围时，Java 可推进 `lastFullyProcessedCommit`。这仍不表示审核通过。 | +| `INCOMPLETE` | 树枚举或包采集未覆盖全部目标时，保存已完成路径、失败子树或 `pathCheckpoint`、延后原因和重试时间；不得据此判断上游删除，也不能推进完整处理检查点。 | +| `HEAD_ONLY` | 只保存 `lastObservedCommit`，用于审核积压时的低成本观测；不得上传包或推进 `lastFullyProcessedCommit`。 | + +`FULL` 仓库任务的 `pathCheckpoint` 是按字典序处理过的最后一个规范化 Skill 根路径；Java 拒绝倒退。`INCOMPLETE` 保存失败子树和待重试路径，并在重新领取时返回固定 commit、检查点与重试目标。`COMPLETE` 若仍有活动包准入或待重试路径，返回 `409 COVERAGE_CONFLICT`；已持久接收的包可用 `ingestionId` 核对，但不能仅靠结算请求伪造 `ACCEPTED`。存在终态失败路径时任务状态为 `PARTIAL`，该路径保留错误记录。 + +发现任务的 `complete` 还应保存发现游标和额度情况。任何 `retryAt` 均由 Java 持久调度；Go 不长时间占租约等待 GitHub 额度恢复。若来源在扫描期间变为不可读，使用 `fail` 分类报告，不以 `complete` 掩盖失败。 + +## 4. 幂等与恢复 + +| 场景 | Java/Go 约定 | +| --- | --- | +| Go 停机或租约到期 | Java 按持久检查点重派，生成新租约代际；旧执行者收到 `409 LEASE_EXPIRED` 或 `LEASE_STALE` 后停止。 | +| 发现任务提交批次后退出 | Java 先持久保存候选和发现检查点；新 Go 进程在下一代租约中读取检查点，重放同一批次仍只保留一条批次及候选命中，然后完成任务。应使用 Java HTTP、PostgreSQL 与两个独立 Go 进程验证。 | +| 同一个包回报重复送达 | 同任务、commit、规范化路径和相同摘要返回原 `ingestionId`；不同摘要返回 `409 RESULT_CONFLICT`。 | +| Java 已返回 `202` 后后台处理失败 | Go 不因审核耗时重复上传；Java 从摄入记录恢复。对象损坏或丢失时，Java 明确安排该路径重采集。 | +| Java 校验或发布进程在领取后退出 | `VALIDATING` 与 `VALIDATED` 分别保留持久租约及尝试次数；租约过期后其他 Java 实例可重新领取同一摄入项。过期校验令牌不能回写结果，发布仍以同一 `ingestionId` 幂等交接。应分别验证独立 JVM 退出与完整应用容器重启恢复。 | +| 同内容在新轮次出现 | Java 计算正式内容/合规指纹后决定无变化，不重复创建版本或待审项。Go 不自行判定 `UNCHANGED`。 | +| 未完整枚举目录 | 已完整部分继续处理，失败子树单独续跑；绝不据此删除既有 Skill。 | + +`ACQUISITION_COMPLETE` 是仓库采集任务状态；`RECEIVED` 是包摄入状态；审核与发布属于现有 Skill 生命周期。详细状态转换见 [状态与检查点设计](../../openspec/changes/add-github-skill-sync/state-model.md)。 + +## 5. 错误处理 + +错误体包含 `code`、可脱敏展示的 `message`，可重试时可包含 `retryAt`。Go 必须按 `code` 和当前任务状态决定停止、重领、延期或报告，不能把所有非 2xx 统一重试。 + +| HTTP / code | 处理原则 | +| --- | --- | +| `401 AUTH_INVALID` | 服务身份失效；停止领取并告警，不尝试普通用户或 GitHub Token。 | +| `426 PROTOCOL_UNSUPPORTED` | 协议主版本不兼容；停止该实例领取，按同版本镜像和合同排查。 | +| `409 LEASE_EXPIRED`、`LEASE_STALE` | 当前执行者失去任务所有权；停止回报，等待重新领取。 | +| `409 COMMIT_MISMATCH`、`RESULT_CONFLICT` | 不覆盖已有结果；保留诊断信息，重新读取 Java 任务快照或交人工处理。 | +| `409 BACKPRESSURE_ACTIVE` | 停止启动后续包下载，持久回报部分路径进度；不当作包失败。Java 到低水位后重新安排当前上游版本。 | +| `413 PACKAGE_TOO_LARGE` | 记录该包终态失败，其他包继续；不能上传截断包。 | +| `422 PATH_INVALID`、`OBJECT_INVALID`、`DIGEST_MISMATCH` | 拒绝该对象或路径；核对打包/传输原因，不创建正式版本。 | +| `GITHUB_RATE_LIMITED`、`RETRYABLE_FAILURE` | 保存 GitHub 指示的 `retryAt` 或可恢复失败信息，由 Java 延后任务；不占租约空等。 | +| `SOURCE_UNREADABLE` | 与限流区分；按来源不可达复查策略运行，已发布 Skill 保持原样。 | + +GitHub 自身的 HTTP 状态并不直接等于内部接口 HTTP 状态；Go 应通过 `/fail` 回报分类、额度和可重试时间。`403/429` 可能涉及权限或限流，需结合 GitHub 响应信息分类；不能通过不断轮换凭证规避限流。 + +## 6. 联调边界与待定参数 + +首轮联调应覆盖：无任务 `204`、领取后停机重派、旧租约回报、同包重复回报与摘要冲突、完整/不完整树、`HEAD_ONLY`、本地与 S3/MinIO 上传、包超限、Go 回报 `202` 后 Java 重启、限流和来源不可读。可观察结果应分别落在任务、包摄入、审核记录中；不能只看任务接口返回 `200` 就断言 Skill 已发布。 + +已确认首期自动发现只用仓库搜索、精选入口和后续仓库树验证，不做全站代码搜索；`criteria` 已锁定为独立的关键词/topic 条件与核心/扩展优先级，搜索窗口随任务固定。包限制沿用当前有效发布政策、广泛搜索默认每 7 天全量重查一次、已完成运行/任务详情默认保留 90 天；未完成任务与恢复数据不按此期限清理，容器日志由部署环境管理。租约和轮询值以当前绑定配置为准；全平台/单 namespace 未解决同步项的高低水位由管理员配置，候选预算初值见管理接口。首期 S3/MinIO 采用上传前 Go 预检和 Java 摄入前复核；若将来要求对象存储接收时硬拒绝超限对象，再验证条件上传方案。字段变更须先更新 OpenAPI YAML，再同步修改本文示例和对应实现。 +## 7. 采集服务在线报告 + +Go 服务启动后每 30 秒调用 `POST /internal/github-sync/v1/heartbeat`，包含现有机器身份 Bearer 凭证，不传 GitHub Token、主机信息或任务内容。Java 返回 `204`，仅记录最近一次成功心跳。心跳独立于发现和仓库工作池，长任务运行期间仍会发送；调用失败不会中断在途任务。平台管理员通过 `GET /api/v1/admin/github-sync/worker-status` 查看最近在线时间。该时间表示 Go 与 Java 最近连通，不代表 GitHub 出站访问成功。 + +## 8. 官方深度探索合同与当前接线状态 + +[深度探索增量说明](github-deep-discovery.md) 定义新管理接口、能力协商、固定策略快照、多渠道出处、检查点和暂停边界。同目录 OpenAPI 标有 `x-implementation-status: proposed-not-implemented` 的字段/路径是待实现设计，不能视为当前 Java/Go 已支持。 + +保留现有任务 kind、legacy discovery.entryType、状态/错误枚举和批次202空响应。新 claim 可选 capabilities 缺省为空；只有完整声明官方所需能力的实例才能领取含 officialDiscovery 的任务。V119起平台任务还统一要求 `platform-pause/v1`，因此空能力请求不再领取平台任务，TEAM规则不变。旧空心跳继续有效，但不能证明官方能力在线；实例/能力心跳是新增持久化工作。配置在领取和关键接收时按最新状态复核；暂停不影响已登记来源与其他独立启用入口，已接收进度不回滚。 + +发布时先实现 Java 的能力过滤再启用官方任务；回滚到旧 Java 前须隔离或排空官方任务，不能只关管理页面。旧协议的联调结果不覆盖这些新增行为；实现后再分别验证 Java、Go、管理页面与跨进程恢复。 + +官方独占候选的仓库暂停也使用新增 `official-pause`，仅要求 `official-pause/v1` 控制能力，不要求官方发现快照。新端点支持尚未取得commit时的空SHA/空路径进度，以及已固定commit的路径与已接收包进度；原 `complete` 对commit和未来retryAt的要求保持不变。任务途中成为正式来源或另有独立启用入口时返回新端点专用CONTROL_CHANGED，重新查询控制状态后继续。详见增量说明第8节。 + +### 官方执行与分片 + +Java已接通配置、版本化目录、持久query、能力领取、官方批次/检查点、发现与仓库任务control/pause/recovery，以及顺序时间分片。仓库临时隔离已由最新enabled/排除/独立入口资格替换。来源自身PAUSED返回SOURCE_PAUSED，已绑定来源不受官方开关影响。 + +SPLIT原子登记两子范围，预算为剩余深度0..32、root剩余累计child数0..126；全叶COMPLETE才推进根覆盖。官方GAP以INCOMPLETE请求结算为PARTIAL并轮转兄弟，普通IN_PROGRESS延期仍为RETRY_WAIT。GAP ledger保留原checkpoint、retryAt和租约代数；到期复用终结任务，或在历史已清理时从ledger恢复。旧自定义完成语义不变。详见增量说明第14节。 + +生产Go工作池现声明四项官方能力，Java配置仍默认关闭。首次启用前必须全部Java与Go升级并完成迁移和集成验收；三个v1能力名不能区分旧Go的16-child上限和新126预算，不能混跑旧显式opt-in worker。策略generation也不能保护旧Java接收器。作者扩展见增量说明第15节；统计口径见该说明第5节，协议文档本身不代表运行验收通过。 + +### 作者扩展（V113） + +metadata可选ownerId；OWNER快照另要求official-owner/v1，只由实际验证且具有独立非OWNER官方出处的seed触发一层扩展。OWNER_RECHECK不使用时间分片和pushed覆盖。其GAP完成保持RETRY_WAIT，页码为正int32、按租约预算续页；10页是单租约上限。CORE请求预算包含身份解析与末页身份复核，完整结果只记录作者复查时间。详见[增量说明第15节](github-deep-discovery.md#15-有效作者扩展v113)。 + +### 官方实例心跳 + +带 `workerId`、`protocolMajor=1`、`capabilities` 的心跳按实例持久记录服务端接收时间；空 body 或 `{}` 保留旧全局联系行为，不证明具备官方能力。后台根据完整官方能力集合（含作者扩展）与最近 90 秒联系统计兼容实例；配置启用、实例可联系、业务最近进度是三个不同事实。没有兼容实例但近期有旧式联系时为 `INCOMPATIBLE`；全部联系超时为 `UNREACHABLE`；从未联系为 `NEVER_SEEN`。 + +实例记录按既有 `completed-detail-retention` 和 `history-cleanup-interval-ms` 清理，每次最多 10 批、每批 500 行；不删除任务、检查点或旧全局联系记录。实例心跳本身不证明多实例并发业务正确。 + +### 额度观察与共享冷却（V116） + +`POST /tasks/{taskId}/official-quota`以有效官方租约接收`{lease, observation}`，无credentialId亦可用于匿名请求；旧credential-status必填保持不变，附加官方观察已接线。PRIMARY按资源冷却，SECONDARY部署级共享退避；未知主体不聚合余额。control新增QUOTA_COOLDOWN及retryAt，pause根据当前门控延期，既有准入可结算。Go持久回报失败停止后续GitHub请求并进程共享退避。具体时间/状态边界见[增量说明第16节](github-deep-discovery.md#16-额度观察与共享冷却v116)。跨进程并发限流需使用真实依赖独立验证。 + + +## 9. 平台@github采集安全暂停 + +合同见[统一管理实施合同](../../openspec/changes/add-github-skill-sync/unified-admin-contract.md)。新增能力 `platform-pause/v1` 覆盖所有平台采集任务，包括普通发现、搜索预览、官方探索和仓库检查采集；TEAM 不受该平台开关控制。新 Java 不向缺少此能力的旧 Go 领取者发放平台任务；受控任务的 claim 返回 `platformControlVersion`，TEAM 此字段为空。 + +### 读取任务控制 + +`POST /internal/github-sync/v1/tasks/{taskId}/platform-control`： + +```json +{"lease":{"generation":2,"token":"<领取时取得的租约令牌>"}} +``` + +返回 `{"action":"CONTINUE","platformVersion":3}` 或 `{"action":"PAUSE","platformVersion":4}`。Go 在网络请求和包处理安全点检查，并以5秒为目标间隔观察进行中的任务。仅有效租约或当前暂停版本捕获的原租约证明可以读取；过期的当前暂停证明只用于安全停止，不允许继续采集。 + +### 安全停止确认 + +`POST /internal/github-sync/v1/tasks/{taskId}/platform-pause`： + +| 字段 | 说明 | +| --- | --- | +| `lease` | 原领取的代际和令牌。 | +| `observedVersion` | 最近读取到的暂停版本。 | +| `kind` | `DISCOVERY` 或 `REPOSITORY_SYNC`，必须与任务匹配。 | +| `discoveryCheckpoint` | 发现任务最后已被 Java 确认的游标；从未保存批次时为空。 | +| `repositoryProgress` | 仓库任务的 `observedCommitSha / pathCheckpoint / incompleteSubtrees / pathResults`；复用仓库进度验证，不能推进完整扫描边界。 | + +发现任务不得携带仓库进度。仓库任务必须携带进度对象；尚未取得 commit 时 SHA 和路径检查点为空，数组为空，并且服务端也必须尚未绑定 commit 或准入。已取得 commit 的 FULL 任务复用现有结果与摄入校验；HEAD_ONLY 不接受包路径结果。成功返回 `{"status":"PAUSED"}`，相同确认可以幂等重试。 + +只有执行线程停止外部工作、完成必要结算与本地清理后才能确认，不能让控制轮询协程携带旧检查点代为确认。有效租约可原子保存已确认进度并释放任务；过期或已结算任务只能以当前暂停捕获的证明确认停止，不能写新进度，也不能影响新一代租约。恢复后旧版本返回 `409 CONTROL_CHANGED`，需重新读取；证明失效返回 `409 LEASE_STALE`，进度或类型冲突沿用 `RESULT_CONFLICT / CHECKPOINT_CONFLICT / COMMIT_MISMATCH`。范围不匹配返回 `409 PLATFORM_SCOPE_MISMATCH`；缺少 lease 返回 `422 LEASE_REQUIRED`，非法JSON或参数返回 `400 INVALID_REQUEST`。 + +### 状态与接收边界 + +Java持久保存平台暂停意图，claim和新包准入与暂停共享事务门控。租约过期或心跳丢失不等于已停止，当前暂停版本尚有未确认任务时管理端保持 `PAUSING`；保留期清理也不能删除这些记录来伪造 `PAUSED`。恢复递增控制版本，不强制等待失联 worker，但不允许旧代际回写新工作。 + +Java已接受摄入的边界是 `acceptReport` 事务成功，不是对象存储 PUT 完成。暂停前已完整打包且有有效准入/租约的包允许结算；不完整本地包清理并释放准入。已接受包后续校验、安全扫描、待审创建与人工审核继续运行。平台暂停不修改各发现入口的独立启停，恢复时按既有状态及持久进度继续。 + +运行证据与未完成范围见 [OpenSpec 验收记录](../../openspec/changes/add-github-skill-sync/verification.md)，接口定义本身不等同端到端验收通过。 diff --git a/docs/api/github-sync-internal.openapi.yaml b/docs/api/github-sync-internal.openapi.yaml new file mode 100644 index 000000000..38bab2ae8 --- /dev/null +++ b/docs/api/github-sync-internal.openapi.yaml @@ -0,0 +1,1508 @@ +openapi: 3.0.3 +info: + title: SkillHub GitHub Sync Internal API + version: 1.0.0 + description: | + Java 控制面供 Go 采集服务使用的设计合同；任务领取、续租、发现批次、规则预览、凭证下发与额度/失效回报、逐包准入、上传、摄入回报、失败诊断及仓库结算已分阶段实现。 + 仅机器身份可调用。响应中的 GitHub token、租约令牌和上传 URL 均是敏感数据，不得写入日志或持久任务。 + 官方深度探索扩展已接入 Java/Go 运行时；验证入口与限制见 openspec/changes/add-github-skill-sync/verification.md，接口定义本身不代表部署验收通过。 + 1.x 允许向响应新增可选字段，不改变既有字段、状态和错误含义；未知主版本由 Java 拒绝。 +servers: + - url: /internal/github-sync/v1 +security: + - ServiceBearer: [] +paths: + /heartbeat: + post: + operationId: heartbeatGithubSyncWorker + summary: 采集服务定时报告在线状态 + requestBody: + required: false + description: 已接入实例与能力记录；原无body或{}心跳仍有效，仅更新legacy联系状态。 + content: + application/json: + schema: + oneOf: + - type: object + maxProperties: 0 + - $ref: '#/components/schemas/OfficialWorkerHeartbeat' + x-implementation-status: implemented-controlled-runtime-partially-verified + description: Go 服务启动后和运行期间每 30 秒调用一次；即使所有工作池正在处理任务也继续报告。 + parameters: + - $ref: '#/components/parameters/RequestId' + responses: + '204': + description: 已记录最近在线时间。 + '401': { $ref: '#/components/responses/Unauthorized' } + /tasks/claim: + post: + operationId: claimGithubSyncTask + summary: 原子领取一个发现或仓库任务 + x-official-claim-status: implemented-controlled-runtime-partially-verified + description: 官方能力、结果接收、暂停及时间分片已接线并完成受控联调；生产Go声明四项官方能力及platform-pause/v1。平台任务统一要求暂停能力，TEAM规则不变。 + parameters: + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [workerId, protocolMajor, kind] + properties: + capabilities: + type: array + maxItems: 32 + items: + type: string + minLength: 1 + maxLength: 128 + x-implementation-status: implemented-controlled-runtime-partially-verified + description: 缺省为空；平台任务必须声明platform-pause/v1，官方任务还须具备全部requiredCapabilities。TEAM沿用原任务能力要求。未知能力不报错、不视为支持。 + workerId: + type: string + maxLength: 128 + description: 进程实例标识，仅供诊断；不得使用凭证或主机密钥。 + protocolMajor: + type: integer + enum: [1] + kind: + type: string + enum: [DISCOVERY, REPOSITORY_SYNC] + description: 发现池与仓库池分别领取，避免已领取任务在另一类工作池内等待。 + responses: + '200': + description: 已领取任务；GitHub 凭证仅出现在本次响应中。 + content: + application/json: + schema: + $ref: '#/components/schemas/ClaimedTask' + '204': + description: 暂无可领取任务，可按服务配置退避后再轮询。 + '401': { $ref: '#/components/responses/Unauthorized' } + '400': { $ref: '#/components/responses/BadRequest' } + '426': { $ref: '#/components/responses/UpgradeRequired' } + /tasks/{taskId}/renew: + post: + operationId: renewGithubSyncLease + summary: 当前代际续租 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: { $ref: '#/components/requestBodies/Lease' } + responses: + '200': + description: 续租成功。 + content: + application/json: + schema: + type: object + required: [leaseExpiresAt] + properties: + leaseExpiresAt: { type: string, format: date-time } + '401': { $ref: '#/components/responses/Unauthorized' } + '400': { $ref: '#/components/responses/BadRequest' } + '409': { $ref: '#/components/responses/Conflict' } + /tasks/{taskId}/repository-metadata: + post: + operationId: updateGithubRepositoryMetadata + summary: 按稳定仓库 ID 对账当前公开仓库坐标 + description: Go 从 GitHub 获取公开仓库元数据并核对任务中的 repository ID 后调用；Java 再核对当前租约与 ID，更新来源或待验证候选及任务快照。改名或转移不创建新的来源身份。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, repositoryId, owner, name, defaultBranch] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + repositoryId: { type: integer, format: int64, minimum: 1 } + owner: { type: string, minLength: 1, maxLength: 39 } + ownerId: { type: integer, format: int64, minimum: 1, description: 可选稳定GitHub账号ID；只有带此身份的实际Java验证成功记录可成为作者seed。省略保持旧客户端兼容，不从login猜身份。 } + name: { type: string, minLength: 1, maxLength: 100 } + defaultBranch: { type: string, minLength: 1, maxLength: 256 } + responses: + '204': { description: 已按原 repository ID 更新来源或候选坐标及当前任务快照。 } + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + /tasks/{taskId}/credential-status: + post: + operationId: reportGithubCredentialStatus + summary: 回报当前租约使用的系统凭证额度、认证失效或限流 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, credentialId, status] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + credentialId: { type: string, format: uuid } + status: { type: string, enum: [OBSERVED, AUTH_INVALID, RATE_LIMITED] } + retryAt: + type: string + format: date-time + description: RATE_LIMITED 必填，其余状态不得填写。 + githubQuota: { $ref: '#/components/schemas/GithubQuota' } + officialQuotaObservation: + allOf: + - $ref: '#/components/schemas/OfficialQuotaObservation' + x-implementation-status: implemented-controlled-runtime-partially-verified + responses: + '204': { description: 凭证状态已更新，响应不含凭证内容。 } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/discovered-repositories: + post: + operationId: reportGithubDiscoveryBatch + summary: 回报一批发现候选及可恢复游标 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, repositories, previousCheckpoint, checkpoint] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + repositories: + type: array + maxItems: 100 + items: { $ref: '#/components/schemas/RepositoryCandidate' } + previousCheckpoint: + type: string + nullable: true + maxLength: 4096 + description: 本批次开始前 Java 已确认的游标；首批传 null。服务端据此原子防止旧批次倒退游标。 + checkpoint: + type: string + minLength: 1 + maxLength: 4096 + description: Java 原样持久化的规则/分页续跑游标，不得包含 token。 + officialBatch: + $ref: '#/components/schemas/OfficialDiscoveryBatch' + x-implementation-status: implemented-controlled-runtime-partially-verified + deferredCount: { type: integer, minimum: 0 } + deferralReason: { $ref: '#/components/schemas/DeferralReason' } + responses: + '202': + description: 批次及游标已持久接收；正式来源仍需验证存在有效 Skill。 + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/preview-results: + post: + operationId: completeGithubSearchPreview + summary: 回报无副作用搜索预览并完成当前租约 + description: 只接受 entryType=SEARCH_PREVIEW 的发现任务；原子保存每个条件的第一页搜索估计与最多五个公开仓库样例，不写入发现候选或来源。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, selectors] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + selectors: + type: array + minItems: 1 + maxItems: 20 + items: + type: object + required: [kind, value, priority, totalCount, incompleteResults, limitReached, samples] + properties: + kind: { type: string, enum: [KEYWORD, TOPIC] } + value: { type: string } + priority: { type: string, enum: [CORE, EXTENSION] } + totalCount: { type: integer, minimum: 0 } + incompleteResults: { type: boolean } + limitReached: { type: boolean } + samples: + type: array + maxItems: 5 + items: { $ref: '#/components/schemas/RepositoryCandidate' } + responses: + '200': + description: 预览结果已保存，任务状态为 ACQUISITION_COMPLETE。 + content: + application/json: + schema: + type: object + required: [status] + properties: + status: { type: string, enum: [ACQUISITION_COMPLETE] } + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + /tasks/{taskId}/official-quota: + post: + operationId: reportGithubOfficialQuota + summary: 按当前官方租约回报真实额度或共享退避（含匿名） + x-implementation-status: implemented-controlled-runtime-partially-verified + description: 仅requiredCapabilities含official-pause/v1的有效租约；凭证ID和主体由Java租约关联，不接受Go自报。无凭证租约使用匿名观察键。与旧credential-status必填credentialId/status合同分开。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, observation] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + observation: { $ref: '#/components/schemas/OfficialQuotaObservation' } + responses: + '204': { description: 观察和适用的共享冷却已持久化。 } + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + /tasks/{taskId}/package-admissions: + post: + operationId: admitGithubPackageAcquisition + summary: 下载一个 Skill 包前原子申请采集准入 + description: Java 按全局和 namespace 积压及未完成准入占位判断；高水位返回 409 BACKPRESSURE_ACTIVE，当前来源规则忽略该路径时返回 409 PATH_IGNORED。相同租约、commit 和路径重复申请返回同一准入。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, commitSha, skillPath] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + commitSha: { $ref: '#/components/schemas/CommitSha' } + skillPath: { $ref: '#/components/schemas/SkillPath' } + responses: + '200': + description: 已预留一个包的采集准入；准入随当前租约续期，结算或租约失效时释放。 + content: + application/json: + schema: { $ref: '#/components/schemas/PackageAdmission' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/upload-authorizations: + post: + operationId: authorizeGithubPackageUpload + summary: 为一个固定 commit 的 Skill 路径签发临时上传授权 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, admissionId, commitSha, skillPath, sizeBytes, sha256] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + admissionId: { type: string, format: uuid, description: 下载前 Java 授予的当前包采集准入。 } + commitSha: { $ref: '#/components/schemas/CommitSha' } + skillPath: { $ref: '#/components/schemas/SkillPath' } + sizeBytes: { type: integer, format: int64, minimum: 1 } + sha256: { $ref: '#/components/schemas/Sha256' } + responses: + '200': + description: 短时授权。uploadUrl 是 bearer URL，不得记录或在客户端追随重定向。 + content: + application/json: + schema: { $ref: '#/components/schemas/UploadAuthorization' } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '413': { $ref: '#/components/responses/TooLarge' } + '422': { $ref: '#/components/responses/Unprocessable' } + /uploads/{uploadId}/content: + put: + operationId: streamGithubPackageToLocalStorage + summary: 本地存储模式的受限流式上传 + description: 仅 LOCAL_STREAM 票据使用；服务身份和上传票据双重校验，接收时限制实际字节数。S3/MinIO 直接使用授权 URL。 + parameters: + - name: uploadId + in: path + required: true + schema: { type: string, format: uuid } + - name: X-Upload-Ticket + in: header + required: true + schema: { type: string } + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/octet-stream: + schema: { type: string, format: binary } + responses: + '204': { description: 对象已写入临时隔离存储；仍须逐包回报。 } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '413': { $ref: '#/components/responses/TooLarge' } + /tasks/{taskId}/packages/report: + post: + operationId: reportGithubPackage + summary: 持久接收一个包的摄入请求 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, admissionId, commitSha, skillPath, uploadId, sizeBytes, sha256] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + admissionId: { type: string, format: uuid, description: 必须与当前任务、固定 commit 和路径对应；接收后准入占位转为未解决摄入项。 } + commitSha: { $ref: '#/components/schemas/CommitSha' } + skillPath: { $ref: '#/components/schemas/SkillPath' } + uploadId: { type: string, format: uuid, description: Java 签发的暂存对象标识；Go 不回报下载 URL。 } + sizeBytes: { type: integer, format: int64, minimum: 1 } + sha256: { $ref: '#/components/schemas/Sha256' } + responses: + '202': + description: 摄入记录已持久化；并非内容校验、审核或发布成功。相同业务键与摘要重放返回原 ingestionId。 + content: + application/json: + schema: + type: object + required: [ingestionId, status] + properties: + ingestionId: { type: string, format: uuid } + status: { type: string, enum: [RECEIVED] } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '413': { $ref: '#/components/responses/TooLarge' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/complete: + post: + operationId: completeGithubSyncTask + summary: 结算一次发现或仓库采集任务 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/TaskCompletion' } + responses: + '200': + description: 结算已持久化，返回 Java 权威任务状态。 + content: + application/json: + schema: + type: object + required: [status] + properties: + status: { type: string, enum: [ACQUISITION_COMPLETE, PARTIAL, RETRY_WAIT] } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/fail: + post: + operationId: failGithubSyncTask + summary: 分类报告无法完成的任务 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [lease, reason] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + reason: { $ref: '#/components/schemas/FailureReason' } + retryAt: { type: string, format: date-time } + githubQuota: { $ref: '#/components/schemas/GithubQuota' } + detail: + type: string + maxLength: 100 + pattern: '^(?:WORKER_(?:TIMEOUT|ERROR)|GITHUB_HTTP_[1-5][0-9]{2}:(?:RETRYABLE_FAILURE|AUTH_INVALID|SOURCE_UNREADABLE|GITHUB_RATE_LIMITED))$' + description: 仅接受已定义的 Worker 或 GitHub HTTP 诊断码，不接收原始异常、token、上传 URL 或采集正文。 + responses: + '200': + description: 失败类别与重试时间已持久化。 + content: + application/json: + schema: + type: object + required: [status, retryAt] + properties: + status: { type: string, enum: [RETRY_WAIT] } + retryAt: { type: string, format: date-time } + '401': { $ref: '#/components/responses/Unauthorized' } + '409': { $ref: '#/components/responses/Conflict' } + '422': { $ref: '#/components/responses/Unprocessable' } + /tasks/{taskId}/official-control: + post: + operationId: getOfficialGithubTaskControl + x-implementation-status: implemented-controlled-runtime-partially-verified + summary: 查询官方任务是否应在安全边界暂停；适用于官方发现及官方独占候选仓库任务。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + $ref: '#/components/requestBodies/Lease' + responses: + '200': + description: DISCOVERY与要求official-pause/v1的REPOSITORY_SYNC均已接线，尚未运行时联调。 + content: + application/json: + schema: + $ref: '#/components/schemas/OfficialTaskControl' + '401': + $ref: '#/components/responses/Unauthorized' + '409': + $ref: '#/components/responses/Conflict' + '422': + $ref: '#/components/responses/Unprocessable' + /tasks/{taskId}/official-pause: + post: + operationId: pauseOfficialGithubTask + x-implementation-status: implemented-controlled-runtime-partially-verified + summary: 确认官方发现或候选仓库任务安全暂停并释放租约；保存可恢复进度。 + description: 仅requiredCapabilities含official-pause/v1的任务可调用。事务锁定租约并复核最新配置/有效归属，当前已BOUND成为非暂停来源或其他独立入口需要继续时返回CONTROL_CHANGED，不写暂停进度、不释放租约；客户端重新查control。来源自身PAUSED优先，使用SOURCE_PAUSED暂停原因。成功时进度保存和释放租约原子完成，无需commit尚未取得时虚构SHA或retryAt。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/OfficialPauseRequest' + responses: + '200': + description: DISCOVERY与要求official-pause/v1的REPOSITORY_SYNC均已接线，尚未运行时联调。 + content: + application/json: + schema: + type: object + required: + - status + properties: + status: + type: string + enum: + - PAUSED + '401': + $ref: '#/components/responses/Unauthorized' + '409': + description: 租约或检查点冲突；最新配置已允许继续时返回CONTROL_CHANGED。 + content: + application/json: + schema: + $ref: '#/components/schemas/OfficialControlError' + '422': + $ref: '#/components/responses/Unprocessable' + /tasks/{taskId}/official-checkpoint-recovery: + post: + operationId: recoverOfficialGithubCheckpoint + x-implementation-status: implemented-controlled-runtime-partially-verified + summary: 回报不可续跑的官方检查点并安排安全重放 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/OfficialCheckpointRecovery' + responses: + '200': + description: 旧进度已停止，替代任务已原子安排；不表示重放已经执行。 + content: + application/json: + schema: + type: object + required: + - status + - replacementTaskId + properties: + status: + type: string + enum: + - REPLAY_SCHEDULED + replacementTaskId: + type: string + format: uuid + '401': + $ref: '#/components/responses/Unauthorized' + '409': + $ref: '#/components/responses/Conflict' + '422': + $ref: '#/components/responses/Unprocessable' + /tasks/{taskId}/platform-control: + post: + operationId: getGithubPlatformTaskControl + summary: 读取平台采集暂停意图 + x-implementation-status: implemented + description: 仅受platform-pause/v1控制的平台任务；TEAM不受此开关控制。lease过期不代表已停止，旧epoch不得覆盖当前状态。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformControlRequest' + responses: + '200': + description: 控制状态或安全停止确认；不代表Go心跳在线。 + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformTaskControl' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '409': + $ref: '#/components/responses/Conflict' + '422': + $ref: '#/components/responses/Unprocessable' + /tasks/{taskId}/platform-pause: + post: + operationId: acknowledgeGithubPlatformTaskPause + summary: 安全停止后确认平台采集暂停 + x-implementation-status: implemented + description: 仅受platform-pause/v1控制的平台任务；TEAM不受此开关控制。lease过期不代表已停止，旧epoch不得覆盖当前状态。 + parameters: + - $ref: '#/components/parameters/TaskId' + - $ref: '#/components/parameters/RequestId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformPauseRequest' + responses: + '200': + description: 控制状态或安全停止确认；不代表Go心跳在线。 + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformPauseResult' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '409': + $ref: '#/components/responses/Conflict' + '422': + $ref: '#/components/responses/Unprocessable' +components: + securitySchemes: + ServiceBearer: + type: http + scheme: bearer + description: 独立采集服务密钥，不接受用户会话或 GitHub token；部署经 Secret 注入，支持短期双密钥轮换。 + parameters: + TaskId: + name: taskId + in: path + required: true + schema: { type: string, format: uuid } + RequestId: + name: X-Request-Id + in: header + required: true + description: 每请求独立的日志关联 ID，不是业务幂等键。 + schema: { type: string, maxLength: 128 } + requestBodies: + Lease: + required: true + content: + application/json: + schema: + type: object + required: [lease] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + responses: + BadRequest: + description: 内部请求格式或字段无效。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + Unauthorized: + description: 服务身份无效；不泄露任务、凭证或对象信息。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + Conflict: + description: 租约失效、代际冲突、固定 commit 不符、同业务键摘要冲突或积压阻止新包准入，按 code 判断。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + Unprocessable: + description: 路径、对象或摘要不合法；不是可盲重试的采集错误。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + TooLarge: + description: 包或实际上传超出平台上限。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + UpgradeRequired: + description: 协议主版本不兼容。 + content: + application/json: + schema: { $ref: '#/components/schemas/ApiError' } + schemas: + PlatformControlRequest: + type: object + required: + - lease + properties: + lease: + $ref: '#/components/schemas/LeaseProof' + PlatformTaskControl: + type: object + required: + - action + - platformVersion + properties: + action: + type: string + enum: + - CONTINUE + - PAUSE + platformVersion: + type: integer + format: int64 + minimum: 1 + PlatformPauseRequest: + type: object + required: + - lease + - observedVersion + - kind + properties: + lease: + $ref: '#/components/schemas/LeaseProof' + observedVersion: + type: integer + format: int64 + minimum: 1 + kind: + type: string + enum: + - DISCOVERY + - REPOSITORY_SYNC + discoveryCheckpoint: + type: string + nullable: true + maxLength: 4096 + description: 发现任务最后持久确认游标；从未接收批次时为null。 + repositoryProgress: + $ref: '#/components/schemas/PlatformRepositoryProgress' + description: 仓库任务提交repositoryProgress，发现任务提交discoveryCheckpoint。过期租约仅可凭捕获proof确认停止，不写新进度；具体校验见配套MD。 + PlatformRepositoryProgress: + type: object + additionalProperties: false + required: + - observedCommitSha + - pathCheckpoint + - incompleteSubtrees + - pathResults + properties: + observedCommitSha: + type: string + pattern: ^[0-9a-f]{40}$ + nullable: true + pathCheckpoint: + type: string + maxLength: 1024 + nullable: true + incompleteSubtrees: + type: array + maxItems: 1000 + items: + $ref: '#/components/schemas/SkillPath' + pathResults: + type: array + maxItems: 10000 + items: + type: object + required: + - skillPath + - outcome + properties: + skillPath: + $ref: '#/components/schemas/SkillPath' + outcome: + type: string + enum: + - ACCEPTED + - IGNORED + - RETRYABLE_FAILURE + - TERMINAL_FAILURE + ingestionId: + type: string + format: uuid + reason: + type: string + maxLength: 500 + description: ACCEPTED必须有已持久接收且属于同任务/commit/路径的ingestionId，其余outcome禁止ingestionId；路径去重。 + description: 尚未取得commit时observedCommitSha=null，且pathCheckpoint必须null、两个数组必须空；仅当服务端也无固定commit/已接收包/绑定commit的准入时合法。否则须报有效SHA并与已固定commit一致。可在此原子固定首次观测SHA并保存合法路径进度；不推进来源完整处理边界。已接收包以Java摄入记录为准，不要求重复上传。 + PlatformPauseResult: + type: object + required: + - status + properties: + status: + type: string + enum: + - PAUSED + ApiError: + type: object + required: [code, message] + properties: + code: + type: string + enum: [AUTH_INVALID, INVALID_REQUEST, PROTOCOL_UNSUPPORTED, LEASE_EXPIRED, LEASE_STALE, COMMIT_MISMATCH, PATH_INVALID, PATH_IGNORED, OBJECT_INVALID, DIGEST_MISMATCH, PACKAGE_TOO_LARGE, RESULT_CONFLICT, CHECKPOINT_CONFLICT, CANDIDATE_BUDGET_EXCEEDED, COVERAGE_UNSUPPORTED, REPORT_UNSUPPORTED, BACKPRESSURE_ACTIVE, RETRYABLE_FAILURE, GITHUB_RATE_LIMITED, SOURCE_UNREADABLE] + message: { type: string } + retryAt: { type: string, format: date-time } + LeaseProof: + type: object + required: [generation, token] + properties: + generation: { type: integer, format: int64, minimum: 1 } + token: { type: string, minLength: 1, description: 敏感租约令牌，不得记录。 } + ClaimedTask: + type: object + required: [taskId, runId, kind, lease, leaseExpiresAt, namespaceId] + properties: + taskId: { type: string, format: uuid } + runId: { type: string, format: uuid } + kind: { type: string, enum: [DISCOVERY, REPOSITORY_SYNC] } + mode: { type: string, enum: [FULL, HEAD_ONLY], description: 仅仓库任务使用。 } + namespaceId: { type: integer, format: int64, minimum: 1 } + sourceId: { type: integer, format: int64, minimum: 1, description: 已登记仓库任务使用。 } + lease: { $ref: '#/components/schemas/LeaseProof' } + leaseExpiresAt: { type: string, format: date-time } + platformControlVersion: { type: integer, format: int64, minimum: 1, nullable: true, description: 仅受平台采集控制的任务返回，须协商platform-pause/v1。 } + officialDiscovery: + $ref: '#/components/schemas/OfficialDiscoverySnapshot' + x-implementation-status: implemented-controlled-runtime-partially-verified + requiredCapabilities: + type: array + maxItems: 32 + items: + type: string + maxLength: 128 + x-implementation-status: implemented-controlled-runtime-partially-verified + discovery: { $ref: '#/components/schemas/DiscoverySnapshot' } + repository: { $ref: '#/components/schemas/RepositorySnapshot' } + credential: + type: object + description: 可选的系统级公开 API 凭证，仅在 claim 响应临时下发；无凭证则匿名。 + required: [token] + properties: + credentialId: { type: string, format: uuid } + token: { type: string, minLength: 1, description: 敏感值，不得写入任务、URL、日志或指标。 } + maxPackageBytes: { type: integer, format: int64, minimum: 1, description: 当前有效发布政策的解包总量上限，默认 100 MiB。 } + maxSingleFileBytes: { type: integer, format: int64, minimum: 1, description: 当前有效发布政策的单文件上限，默认 10 MiB。 } + maxFilesPerPackage: { type: integer, minimum: 1, description: 当前有效发布政策的文件数上限，当前 application.yml 配置为 100。 } + maxUploadBytes: { type: integer, format: int64, minimum: 1, description: 临时归档对象的上传上限，默认沿用当前 100 MiB 上传限制；Go 在申请票据前预检。 } + DiscoverySnapshot: + type: object + required: [entryType, ruleVersion, candidateBudget] + description: SEARCH_RULE 和 SEARCH_PREVIEW 必须含 criteria 与 searchWindow；CURATED_OWNER 必须含 owner；CURATED_REPOSITORY 必须含 owner 与 repositoryName。预览每个条件只搜索第一页，不登记来源。Java 与 Go 均须按 entryType 校验，不接受条件缺失的任务。 + properties: + entryType: { type: string, enum: [SEARCH_RULE, SEARCH_PREVIEW, CURATED_OWNER, CURATED_REPOSITORY] } + ruleId: { type: string, format: uuid } + ruleVersion: { type: integer, format: int64, minimum: 1 } + criteria: + $ref: '#/components/schemas/RepositorySearchCriteria' + searchWindow: + $ref: '#/components/schemas/RepositorySearchWindow' + owner: { type: string, description: 精选用户或组织名称。 } + repositoryName: { type: string, description: 精选公开仓库名称，仅 CURATED_REPOSITORY 使用。 } + checkpoint: { type: string, maxLength: 4096, description: Java 已持久确认的最近游标；首次领取时可缺省。 } + candidateBudget: { type: integer, minimum: 1 } + RepositorySearchCriteria: + type: object + required: [selectors] + additionalProperties: false + description: 仅 SEARCH_RULE 任务使用。Java 下发已启用规则的不可变版本快照；Go 编译安全的仓库搜索查询，不接受管理员填写的原始 GitHub 查询表达式。 + properties: + selectors: + type: array + minItems: 1 + description: 每项单独执行仓库搜索，结果按 GitHub repository ID 合并；CORE 先于 EXTENSION 消耗发现预算。 + items: { $ref: '#/components/schemas/RepositorySearchSelector' } + RepositorySearchSelector: + type: object + required: [kind, value, priority] + additionalProperties: false + properties: + kind: + type: string + enum: [KEYWORD, TOPIC] + description: KEYWORD 搜索仓库名称、描述与 topics；TOPIC 只匹配指定 topic。均不搜索代码或 README。 + value: + type: string + minLength: 1 + description: 后台校验后的单个词或短语；Go 必须转义并编译为 GitHub 仓库搜索查询，不能直接拼接为原始表达式。 + priority: + type: string + enum: [CORE, EXTENSION] + description: 仅决定候选处理顺序，不影响 Skill 审核和发布。 + RepositorySearchWindow: + type: object + required: [mode, until] + additionalProperties: false + description: 仅 SEARCH_RULE 任务使用；INCREMENTAL 必须含 since，FULL_RECHECK 省略 since。窗口由 Java 固定并随任务版本持久化，分页重试时不得移动。 + properties: + mode: { type: string, enum: [INCREMENTAL, FULL_RECHECK] } + since: { type: string, format: date-time, description: 增量轮次使用的含重叠起点；全量重查时省略。 } + until: { type: string, format: date-time, description: 本轮固定的搜索终点。 } + RepositorySnapshot: + type: object + required: [repositoryId, owner, name] + properties: + repositoryId: { type: integer, format: int64, minimum: 1 } + owner: { type: string } + name: { type: string } + fixedCommitSha: { $ref: '#/components/schemas/CommitSha' } + pathCheckpoint: + $ref: '#/components/schemas/SkillPath' + description: Java 已确认的固定 commit 下最后一个按字典序处理的 Skill 根路径。 + incompleteSubtrees: + type: array + items: { $ref: '#/components/schemas/SkillPath' } + retryPaths: + type: array + items: { $ref: '#/components/schemas/SkillPath' } + description: 固定 commit 下需要重新采集的失败路径，即使它不大于路径检查点也须处理。 + acceptedPaths: + type: array + items: { $ref: '#/components/schemas/SkillPath' } + description: 固定 commit 下已被 Java 持久接收的包路径；续跑时不得再次下载或上传。 + ignorePatterns: + type: array + maxItems: 64 + items: { type: string, maxLength: 1024 } + description: 任务创建时的来源级追加目录前缀，仅供 Go 采集优化；Java 按当前规则复核。候选忽略不剔除父包文件。 + RepositoryCandidate: + type: object + required: [repositoryId, owner, name] + properties: + repositoryId: { type: integer, format: int64, minimum: 1 } + owner: { type: string } + name: { type: string } + defaultBranch: { type: string } + officialProvenance: + type: array + maxItems: 16 + items: + $ref: '#/components/schemas/OfficialProvenance' + x-implementation-status: implemented-controlled-runtime-partially-verified + CommitSha: + type: string + pattern: '^[0-9a-f]{40}$' + Sha256: + type: string + pattern: '^[0-9a-f]{64}$' + SkillPath: + type: string + maxLength: 1024 + description: 仓库内规范化 POSIX 相对目录；根目录用单个点表示，Java 必须再验证。 + PackageAdmission: + type: object + required: [admissionId] + description: 准入仅在当前租约有效时可用于上传和回报；租约失效或任务结算时释放未使用占位。 + properties: + admissionId: { type: string, format: uuid } + UploadAuthorization: + type: object + required: [uploadId, method, mode, uploadUrl, expiresAt, maxBytes] + properties: + uploadId: { type: string, format: uuid } + method: { type: string, enum: [PUT] } + mode: { type: string, enum: [S3_PRESIGNED_PUT, LOCAL_STREAM] } + uploadUrl: { type: string, format: uri-reference, description: S3 为敏感预签名绝对 URL；LOCAL_STREAM 为同源相对路径，另需 uploadTicket。 } + uploadTicket: { type: string, description: 仅 LOCAL_STREAM 模式需要，作为 X-Upload-Ticket 发送。 } + expiresAt: { type: string, format: date-time } + maxBytes: { type: integer, format: int64, minimum: 1, description: Go 必须遵守；S3 PUT 后仍需 Java 复核实际对象大小。 } + headers: + type: object + additionalProperties: { type: string } + description: 签名要求的上传标头，不得记录其敏感值。 + TaskCompletion: + type: object + required: [lease, coverage] + properties: + lease: { $ref: '#/components/schemas/LeaseProof' } + observedCommitSha: { $ref: '#/components/schemas/CommitSha' } + coverage: { type: string, enum: [COMPLETE, INCOMPLETE, HEAD_ONLY] } + incompleteSubtrees: + type: array + items: { $ref: '#/components/schemas/SkillPath' } + pathCheckpoint: + $ref: '#/components/schemas/SkillPath' + description: 固定 commit 下最后一个按字典序处理的 Skill 根路径；磁盘预算分批时由 Java 持久化，不得包含凭证。 + pathResults: + type: array + items: + type: object + required: [skillPath, outcome] + properties: + skillPath: { $ref: '#/components/schemas/SkillPath' } + outcome: { type: string, enum: [ACCEPTED, IGNORED, RETRYABLE_FAILURE, TERMINAL_FAILURE] } + ingestionId: { type: string, format: uuid } + reason: { type: string, maxLength: 500 } + discoveryCheckpoint: + type: string + minLength: 1 + maxLength: 4096 + description: 发现任务结算必须指向最后一批已持久确认的游标；COMPLETE结束当前采集。官方搜索GAP的INCOMPLETE携带未来retryAt，返回PARTIAL并轮转兄弟，不推进根覆盖；OWNER GAP、普通IN_PROGRESS及legacy延期仍返回RETRY_WAIT。 + deferralReason: { $ref: '#/components/schemas/DeferralReason' } + githubQuota: { $ref: '#/components/schemas/GithubQuota' } + retryAt: { type: string, format: date-time } + GithubQuota: + type: object + required: [resource, remaining, resetAt] + properties: + credentialId: { type: string, format: uuid } + resource: { type: string, enum: [CORE, SEARCH, GRAPHQL, OTHER] } + remaining: { type: integer, minimum: 0 } + resetAt: { type: string, format: date-time } + DeferralReason: + type: string + enum: [BUDGET, RATE_LIMIT, BACKPRESSURE, CAPACITY, LOCAL_DISK_BUDGET] + description: LOCAL_DISK_BUDGET 表示清理工作目录后可分批续跑，通常携带 retryAt；CAPACITY 表示当前部署容量下无进展，需调整容量或规则后再重试，不应立即重新领取。 + FailureReason: + type: string + enum: [RETRYABLE_ACQUISITION, GITHUB_RATE_LIMITED, SOURCE_UNREADABLE, INCOMPLETE_TREE, INTERNAL_FAILURE] + OfficialWorkerHeartbeat: + type: object + required: + - workerId + - protocolMajor + - capabilities + properties: + workerId: + type: string + minLength: 1 + maxLength: 128 + protocolMajor: + type: integer + enum: + - 1 + capabilities: + type: array + maxItems: 32 + items: + type: string + minLength: 1 + maxLength: 128 + x-implementation-status: implemented-controlled-runtime-partially-verified + description: 最多32项能力；未知能力可保存但不视为支持既有官方能力。空列表不证明能执行官方探索；V119后平台兼容实例还须声明platform-pause/v1。 + description: 带body时三个字段一起必填；workerId为进程实例随机标识，重启换ID。服务端记receivedAt，不能相信客户端时钟；心跳独立于业务进度。 + x-implementation-status: implemented-controlled-runtime-partially-verified + OfficialProvenance: + type: object + required: + - channel + - queryId + - queryVersion + - strategyVersion + - shardId + properties: + channel: + type: string + enum: + - REPOSITORY_METADATA + - README + - OWNER_EXPANSION + queryId: + type: string + format: uuid + minLength: 1 + maxLength: 128 + queryVersion: + type: integer + format: int64 + minimum: 1 + strategyVersion: + type: string + minLength: 1 + maxLength: 128 + shardId: + type: string + format: uuid + seedRepositoryId: + type: integer + format: int64 + minimum: 1 + description: queryId为持久查询UUID。Java按候选+queryId合并长期出处；shard/seed仅保留有界最近摘要。Java从任务核对并补入口/run/task身份和接收时间；OWNER_EXPANSION必须有种子仓库，其余渠道禁止伪造种子；一次命中最多16条，其余可续批。 + x-implementation-status: implemented-controlled-runtime-partially-verified + OfficialQueryRange: + type: object + required: + - since + - until + properties: + since: + type: string + format: date-time + pattern: ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|\+00:00)$ + description: Java生成的UTC整秒含起点，仓库搜索按pushed维度；Go拒绝小数秒，不静默截断。 + until: + type: string + format: date-time + pattern: ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|\+00:00)$ + description: Java生成的UTC整秒不含终点；since 本文保留 2026-03-22 的原始设计记录。2026-10-10 审核详情页改版以 [审核指南](../skillhub/guide/review.md) 和 `openspec/changes/add-github-skill-sync/specs/github-sync-review/spec.md` 为准：顶部常显安全状态摘要，完整扫描卡片与发现项放在“来源与安全”页签。下文关于卡片插入位置和空结果不渲染的描述仅为历史方案；无结果不得解释为安全通过，扫描中禁止通过的规则不变。 + **版本**: v1.0 **日期**: 2026-03-22 **状态**: Draft diff --git a/docs/prds/skill-file-browser-sidebar-v1.0-prd.md b/docs/prds/skill-file-browser-sidebar-v1.0-prd.md index 5ebd741d3..c3a8c050e 100644 --- a/docs/prds/skill-file-browser-sidebar-v1.0-prd.md +++ b/docs/prds/skill-file-browser-sidebar-v1.0-prd.md @@ -1,5 +1,7 @@ # 技能详情与审核页文件浏览侧边栏 - 产品需求文档 (PRD) +> 2026-10-09 更新：本文件记录早期侧栏方案。**审核详情页**现采用顶部摘要和页级 Tab，文件树移入“内容与文件”Tab；下文有关审核页常驻右侧栏及其三个内层 Tab 的条款不再适用。技能市场详情页的侧栏需求仍按本文件执行。审核页最新行为见 [审核指南](../skillhub/guide/review.md)。 + ## 需求说明 ### 背景 diff --git a/docs/skillhub/guide/review.md b/docs/skillhub/guide/review.md index 57374f3cf..194316bd3 100644 --- a/docs/skillhub/guide/review.md +++ b/docs/skillhub/guide/review.md @@ -69,15 +69,16 @@ SkillHub 提供了完整的审核工作流，确保发布到注册中心的技 1. 访问 `/dashboard/reviews` 2. 查看待审核列表 3. 点击技能包查看详情： - - 查看元数据（名称、描述、版本） - - 浏览文件列表 - - 在线查看文件内容 - - 下载完整包进行本地测试 - - 查看合规声明快照和相对上一发布版本的差异 + - 顶部核对技能名称、描述、状态、提交者和版本；GitHub 同步项显示独立的内容修订号、来源仓库与短 commit，完整安装用技术版本可在技术信息中查看。顶部的安全与许可证状态用于定位问题，点击后在“来源与安全”查看依据；没有结果、读取失败或许可证未识别都不能理解为通过 + - 在“技能内容”查看待审版本说明文档和文件树。说明文档在进入此页签后读取；选中单个文件后才读取其存储内容。如果存储对象已丢失，预览会提示失败 + - GitHub 更新项默认打开“变更对比”，逐文件核对新增、修改和删除；首次导入和普通技能默认打开“技能内容”。超过 64 KiB 或 1000 行时跳过逐行比较，超过 256 KiB 时不在线加载内容。当前版本文件可按原权限下载；已删除的旧文件还受基线接口 1 MiB 上限约束，超限时页面会说明无法从审核页下载 + - 在“来源与安全”核对固定 commit、许可证证据、完整扫描卡片和发现项；普通技能也可查看安全信息。扫描尚未完成时不能批准；GitHub 来源许可证依据缺失时按页面提示处理 + - 在“审核记录”查看提交历史和已完成的审核意见；该页签打开时才请求历史 + - 套件审核在内容区核对待审快照。快照加载失败时可重试，加载成功前“通过”按钮不可用 ![流程图](/diagrams/review-flow.png) -4. 做出决策： +4. 在页面底部固定的审核栏填写意见并做出决策；切换页签保留意见，切换到另一个审核对象时清空意见、驳回表单和未确认弹窗： - **批准**：技能包正式发布，开发者收到通知 - **拒绝**：填写拒绝原因，开发者可修改后重新提交 @@ -186,8 +187,11 @@ GET /api/v1/reviews/{id} **获取审核中的技能包详情**： ```bash GET /api/v1/reviews/{id}/skill-detail +GET /api/v1/reviews/{id}/skill-detail?includeDocumentation=false ``` +`includeDocumentation` 缺省为 `true`，保持原响应行为。设为 `false` 时仍返回技能、版本、文件元数据与说明文件路径，`documentationContent` 为空且不读取对象存储正文；审核详情首屏使用此模式，进入“技能内容”后再取正文。 + **下载审核包**： ```bash GET /api/v1/reviews/{id}/download diff --git a/github-sync-go/.dockerignore b/github-sync-go/.dockerignore new file mode 100644 index 000000000..a425b913b --- /dev/null +++ b/github-sync-go/.dockerignore @@ -0,0 +1,6 @@ +* +!go.mod +!cmd/ +!cmd/** +!internal/ +!internal/** diff --git a/github-sync-go/Dockerfile b/github-sync-go/Dockerfile new file mode 100644 index 000000000..f6981526b --- /dev/null +++ b/github-sync-go/Dockerfile @@ -0,0 +1,27 @@ +FROM --platform=$BUILDPLATFORM golang:1.27.1-alpine3.24 AS build + +WORKDIR /src +COPY go.mod ./ +COPY cmd/ ./cmd/ +COPY internal/ ./internal/ + +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ + go build -trimpath -ldflags="-s -w" -o /github-sync ./cmd/github-sync + +FROM alpine:3.24.2 +RUN apk add --no-cache ca-certificates git tini && \ + addgroup -S -g 10001 skillhub && \ + adduser -S -D -u 10001 -G skillhub skillhub && \ + mkdir -p /var/lib/skillhub-github-sync && \ + chown skillhub:skillhub /var/lib/skillhub-github-sync + +COPY --from=build /github-sync /usr/local/bin/github-sync + +USER 10001:10001 +ENV SKILLHUB_GITHUB_SYNC_TEMP_ROOT=/var/lib/skillhub-github-sync +EXPOSE 8090 +HEALTHCHECK --interval=30s --timeout=3s --retries=3 \ + CMD wget -Y off -qO- http://127.0.0.1:8090/health/live >/dev/null || exit 1 +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/github-sync"] diff --git a/github-sync-go/README.md b/github-sync-go/README.md new file mode 100644 index 000000000..d8b3804f9 --- /dev/null +++ b/github-sync-go/README.md @@ -0,0 +1,100 @@ +# GitHub 公开 Skill 采集服务 + +此目录是独立 Go 服务。它从 SkillHub Java 的内部接口领取发现或仓库任务，向 GitHub 读取公开仓库，逐个 Skill 包上传到 Java 授权的临时对象，再回报采集结果。来源配置、任务状态、审核和发布归 Java 管理。首期不读取私有仓库。 + +## 当前接入状态 + +Go 的发现、仓库树枚举、逐包采集、续租和回报代码已接入单进程双工作池，Java 的配置、来源转换、Skill 版本与审核流程也已接入。本地存储与 MinIO 应分别验收；ARM64 整镜像与长期容量须独立验证。启动容器本身不会让未经审核的 Skill 出现在公开市场。 + +官方深度探索已有独立搜索执行路径：metadata、README 和 topic 查询按固定 UTC 整秒半开窗口执行，逐页检查控制状态，保存分页/页内游标和出处，预算耗尽后保留 IN_PROGRESS。截断或不完整页面中已返回的仓库仍按候选预算保存；结果上限内可访问的页面继续采集，缺口原因跨页保留，最终按预算提交时间子分片或记录 GAP。GAP 按持久游标递增退避（暂为 5 分钟至 6 小时），普通工作预算续排暂为 1 秒；这些值仍需调度校准。批次响应丢失可重放相同请求，坏检查点请求服务端安全重放。非空二级分片暂不支持；SPLIT 由 Java 快照中的剩余深度与根范围子任务额度控制，额度为 0 或 1 时不能二分。父分片交接后不代表全部覆盖；恢复到已确认的 SPLIT 检查点只重试父任务结算，不重查父范围。 + +带 `official-pause/v1` 标记的候选仓库任务已接入控制检查：metadata/HEAD、每个树请求、下一个包准入及结算前查询；已获准包完成上传和回报后再暂停。暂停保存固定 commit、路径进度和未完成子树，尚未取得 commit 时使用空值；已接收包由 Java 摄入记录保留。候选途中绑定来源或获得独立入口时按最新控制结果继续，来源自身暂停以 `SOURCE_PAUSED` 区分。 + +作者扩展使用独立 `official-owner/v1` 能力：按稳定账号 ID 获取当前 login，复用 public owner 仓库分页并逐仓库核对 owner ID，终页再次核验账号身份。身份查询与分页均计 CORE 请求预算，终页核验预留请求额度；检查点页码支持正 int32，达到上限仍有下一页时记录 GAP。作者列表不按 pushed 时间过滤、不做时间二分，GAP 保持 RETRY_WAIT；已确认检查点支持暂停与跨租约续跑。GitHub 元数据上报可选 ownerId，供 Java 的有效 seed 事务使用。 + +官方配额观察通过租约绑定的 `official-quota` 接口持久化，包含匿名请求；REST 的 metadata、HEAD、commit、tree、search、owner 请求以及可能联网的 Git fetch / sparse checkout / checkout 都有请求守门。主配额耗尽按资源退避；二级限流与观察持久化失败触发进程共享退避，不能通过更换官方任务 token 绕过。回报失败锁住当前租约后续请求，进程失败退避暂为 1、2、4、8、15 分钟；Git 无 REST 响应头只报 UNKNOWN，失败附短退避，成功不虚构余额。已生成 artifact 的上传与结果回报不受 Git 请求守门影响。进程退避不是跨实例持久状态；跨实例冷却由 Java 的官方配额状态和控制 / 领取门控负责。 + +生产工作池通过 `ClaimWithCapabilities` 和 `HeartbeatWithCapabilities` 声明 `official-discovery/v1`、`official-checkpoint/v1`、`official-pause/v1`、`official-owner/v1` 和 `platform-pause/v1`；旧客户端方法仍可调用，但V119之后缺少平台暂停能力的实例不能领取平台任务，TEAM规则不变。Java 官方探索配置默认关闭，管理员启用后才允许领取官方任务。首次启用前必须整体升级 Java、数据库和 Go；不混跑早期显式 opt-in 的旧执行器。真实 GitHub 召回、预算档位校准和并发生产验收需要独立验证。策略目录及版本由 Java 维护。 + +平台 `@github` 的全部采集还受独立 `platform-pause/v1` 控制：Java 保存暂停意图，Go 在请求安全点及目标5秒观察间隔检查，停止外部工作并完成清理后由执行线程确认。暂停中不是已停止，离线与租约过期不能代替确认；TEAM不受平台开关控制。Java已接收的包可继续校验和审核，恢复不重置检查点或各入口独立启停。部署需配套升级Java/Go，操作见[管理员手册](../docs/github-sync-operations.md#平台暂停与恢复)，协议见[内部接口第9节](../docs/api/github-sync-internal.md#9-平台github采集安全暂停)。 + +## 构建与启动 + +```bash +cd github-sync-go +go build ./... +``` + +先在仓库根目录运行 `make staging` 准备预发布服务，再用同一个 Compose 项目启动 Go 容器： + +```bash +SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN='<至少 32 字符的随机服务密钥>' \ +SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS='<同一服务密钥>' \ +docker compose -p skillhub-staging -f docker-compose.yml -f docker-compose.staging.yml \ + --profile github-sync up -d github-sync +``` + +上述命令是参数示例；正式环境应通过受保护的环境文件或容器 Secret 提供密钥，不把真实值写进命令历史、仓库或日志。发布 Compose 使用 `compose.release.yml` 和相同的 `github-sync` profile。默认不启动 Go 服务；Helm 对应 `githubSync.enabled: false`。 + +| 环境变量 | 用途 | 默认值或要求 | +| --- | --- | --- | +| `SKILLHUB_GITHUB_SYNC_JAVA_URL` | Java 内部合同根地址 | 必填；Compose 已设置。 | +| `SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN` | Go 调用 Java 的服务身份 | 必填，至少 32 字符；Java 的 `SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS` 须包含它。 | +| `SKILLHUB_GITHUB_SYNC_TEMP_ROOT` | 受限临时目录 | 必填绝对路径；镜像默认 `/var/lib/skillhub-github-sync`。同一目录一次只能供一个 Go 进程使用。 | +| `SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS` / `SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS` | 两类工作池并发数 | 各 1。 | +| `SKILLHUB_GITHUB_SYNC_POLL_INTERVAL` | 无任务或领取失败后的轮询间隔 | `5s`。 | +| `SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES` | 单仓库工作目录预算 | Compose/Helm 默认 512 MiB。 | +| `SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES` | 所有仓库 worker 的总预算 | Compose/Helm 默认 512 MiB。 | +| `SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES` | 临时目录预留空间 | Compose/Helm 默认 128 MiB。 | +| `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS` | 允许上传的对象存储主机和端口，逗号分隔 | S3/MinIO 模式必填；应与 Java 返回的内部预签名 PUT 主机一致。 | +| `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` | GitHub 出站代理 | 按网络环境设置；Go→Java 内部请求与暂存对象上传直连，不走该代理。 | + +Compose 从 `.env.release` 中的 `SKILLHUB_GITHUB_SYNC_HTTP_PROXY`、`SKILLHUB_GITHUB_SYNC_HTTPS_PROXY`、`SKILLHUB_GITHUB_SYNC_NO_PROXY` 显式注入对应标准代理变量；staging 覆盖文件也使用这三个宿主变量。留空时不会自动继承宿主机的代理配置。Helm 使用 `githubSync.httpProxy`、`githubSync.httpsProxy`、`githubSync.noProxy`。 + +镜像提供 `GET /health/live`、`GET /health/ready` 和 `GET /metrics`，端口为 8090。就绪状态要求发现池和仓库池都曾成功访问 Java 领取接口。`/metrics` 只暴露固定种类的低基数计数，不包含服务密钥或 Skill 正文。 + +运行镜像使用 `tini` 作为 PID 1，以回收 Git 子进程并转发停止信号；应避免直接由 Go 充当 PID 1 时无法完整回收 Git 子进程。停止 Go 后，Java 保存的租约和逐包检查点供下一实例续跑。 + +临时目录使用独立的 768 MiB tmpfs，内存限制默认 1.5 GiB；tmpfs 占用计入容器内存。采集会预检并逐包释放目录，仍应监控容器内存与容量阻塞状态。S3/MinIO 预签名 PUT 的大小由 Go 上传前和 Java 摄入前复核；不能把 URL 票据字段当作对象存储的接收时硬限制。 + +Go 启动时会独占临时目录，并只清理该目录下本服务命名的遗留包工作区（`skillhub-github-package-*`）。进程被强制终止后，重启可先释放这些本地文件，再按 Java 保存的固定 commit 和检查点续跑；同一临时目录已有运行中的 Go 进程时，新进程启动失败。多实例部署须为每个实例分配独立临时目录。 + +S3/MinIO 开启桶自动创建时，Java 会在签发首张预签名 PUT 票据前建桶。对象存储仍可能接收超过票据声明大小的直接 PUT；Java 在接收采集回报前按真实对象大小复核，超限对象标失败并清理。 + +Go 在下载包文件前，为 Git 对象、检出文件、ZIP 和至少 16 MiB 基础 Git 元数据预留工作区；预留不足的仓库任务回报容量延后，不启动该包下载。仓库树元数据的实际大小无法事先精确获知，因此 Git 每一步会复查真实目录占用和最低剩余空间，写 ZIP 前也会复查剩余空间。下载前估算不是磁盘硬配额。 + +若单包在当前配置下根本无法容纳，任务以 `CAPACITY` 延后约 24 小时，保留固定 commit 和已完成路径检查点；Java 释放本次包准入，不生成摄入包。若是运行时暂存空间逼近阈值，使用 `LOCAL_DISK_BUDGET` 延后约 15 分钟。两者都不会在同一租约内继续下载后续包；先检查临时目录容量、`MAX_WORKSPACE_BYTES`、`TOTAL_WORKSPACE_BYTES` 和 `MIN_FREE_BYTES` 的组合，再决定是否调整配置。 + +归档写入时如果先触及工作区剩余预算，或文件系统返回空间/配额不足，也按 `LOCAL_DISK_BUDGET` 延后；只有归档达到任务下发的上传大小上限才记为 `ARCHIVE_TOO_LARGE`。 + +需要复核公开 GitHub 的实际受限 Git 获取时，可在 `github-sync-go/` 运行 `SKILLHUB_GITHUB_SYNC_LIVE=1 go test ./internal/repository -run TestLiveAcquireFixedPublicSkill -count=1 -v`。它使用固定公开 commit，检查 ZIP 文件、摘要和临时目录清理；普通 `go test ./...` 不访问 GitHub。 + +测试专用的 `SKILLHUB_GITHUB_SYNC_LIVE_SMALL_VOLUME=1` 将上述用例的单包工作区预算设为 32 MiB、保留空闲设为 8 MiB，便于在小 tmpfs 中复核。容器复核时应使用非 root 身份、只读根文件系统和有硬容量限制的 tmpfs，按网络配置显式提供 GitHub 出站代理。该用例只处理一个小 Skill，不代表大仓库峰值或完整 worker 停机续跑。 + +同一固定 commit 的双 Skill 顺序采集可用 `SKILLHUB_GITHUB_SYNC_LIVE=1 go test ./internal/repository -run TestLiveAcquireTwoFixedPublicSkillsSequentially -count=1 -v` 复核。测试在首包关闭后确认临时目录为空，再采集第二个 Skill；输出分别给出归档和采集结束时的工作目录字节数。这是单仓库小范围演练，不是并发或大规模容量基线。 + +较多文件的公开仓库可用 `SKILLHUB_GITHUB_SYNC_LIVE_LARGE_REPO=1 go test ./internal/repository -run '^TestLiveAcquireLargerPublicRepositorySkill$' -count=1 -v` 复核一个固定 commit 的单 Skill 稀疏获取。此用例要求临时文件系统至少容纳 64 MiB 工作区及 16 MiB 保留空间；建议在有硬容量限制的临时卷中运行。它不覆盖完整 GitHub Tree API 枚举、多包并发或进程瞬时峰值。 + +要手工复核 Go/Java 包格式互通，给上述 Go 测试额外设置 `SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE` 为一个尚不存在的临时 ZIP 路径；随后在 `server/` 给 `GithubSyncArchiveValidatorInteropTest` 设置同一路径的 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE` 并运行 Maven 定向测试，最后删除临时 ZIP。两个变量仅供测试使用，不参与服务启动。 + +要继续复核实际 Go→Java HTTP 合同，将同一个 ZIP 路径传给 `GithubSyncHttpIngestionInteropTest`：在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest' -Dsurefire.failIfNoSpecifiedTests=false test`。此测试需要本机 Go 与 Docker；它在隔离的 PostgreSQL、Redis 容器和随机 HTTP 端口上让真正的 Go 控制面客户端完成领取、准入、本地暂存上传、重复与冲突回报、仓库完成，再验证 Java 摄入校验、后台发布、Scanner outbox、独立同步待审任务及审核前市场版本指针为空。Scanner 消费者默认调用隔离的 HTTP 测试桩取得安全结果后，超级管理员批准同步审核，才将版本发布为市场最新版本。第二次同内容完整摄入必须落为 `NO_CHANGE`，不能新增版本或审核项；TEAM 来源另验证 `NAMESPACE_ONLY` Skill 被拒后不发布，同内容重采也不重复审核。显式请求逐 Skill 重试后，即使 commit 和包内容相同，下一次完整采集也必须建立新的同步待审项，并清除重试标记。测试还让 Go 对第一完整 Java 应用进程回报包，停掉该进程后由第二进程从同一数据库与存储自动续跑，检查唯一版本和同步待审项。加上 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 后，同一测试会启动隔离的 MinIO，验证改写预签名 URL 的对象 key 被拒且未生成对象、原签名正常上传、首次建桶、对象摄入以及超额对象被 Java 拒绝并清理。测试结束后删除临时 ZIP。新增用例还会在 Go 持有仓库任务租约时关闭 Java，确认续租连接失败，再在原端口启动 Java，由第二个 Go 进程接管并完成已接收路径；测试提前设置租约到期时间，避免等待生产环境的完整租约周期。 + +如需验证实际仓库 worker 的双 Skill 逐包路径，先用上面的 `SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE` 从固定公开 commit 生成第一个 Skill 的参考 ZIP，再在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_LIVE_WORKER=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerAcquiresTwoPublicSkillsThroughJava' -Dsurefire.failIfNoSpecifiedTests=false test`。Go 使用受控 Tree 元数据绕开 GitHub REST 额度，从公开固定 commit 实际稀疏获取两个 Skill；Java 完成领取、准入、上传、校验并生成两条独立同步待审项，第一个摄入包的 SHA-256 与参考 ZIP 一致。隔离 Scanner HTTP 测试桩将两包判为安全后，测试逐条批准：首条批准时另一条仍未发布，两条批准后均为已发布版本；TEAM 的两条 Skill 对匿名访问均返回 403。额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可复核隔离 MinIO 模式。此用例不验证真实 GitHub REST Tree 枚举或真实 Scanner 判定；运行后删除参考 ZIP。 + +同一固定 commit 的在途恢复用例可在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_LIVE_INTERRUPT_INTEROP=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerResumesAfterSigtermBetweenReceiptAndAcknowledgement' -Dsurefire.failIfNoSpecifiedTests=false test`；加 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可复核 MinIO。Go 测试进程在首包已被 Java 持久接收、确认尚未送达时收到 SIGTERM；新进程领取到期租约并只采集剩余包。测试检查首包摄入 ID、路径结果和准入次数不变，及中断后工作区为空。Tree 元数据受控，此用例仍不能代替完整服务容器在小临时卷中的停启演练。 + +实际服务镜像的在途续跑可在仓库根目录先执行 `docker build -t skillhub-github-sync:interop github-sync-go`，再在 `server/` 执行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/captured.zip SKILLHUB_GITHUB_SYNC_INTEROP_IMAGE=skillhub-github-sync:interop ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#serviceContainerResumesPublicRepositoryOnSmallTmpfs' -Dsurefire.failIfNoSpecifiedTests=false test`；额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 可验证隔离 MinIO。参考 ZIP 依上文单包采集命令生成，测试从 `SKILLHUB_GITHUB_SYNC_LIVE_TOKEN` 或本机 `gh auth token` 取得公开读取凭证；容器需要可用的 GitHub 出站代理。此显式公网用例在 96 MiB tmpfs 中首包持久接收后停止容器，由新容器按租约续跑真实 Tree 的 43 个合规包，并检查首包不重复摄入、容器进程表无僵尸。完成后删除参考 ZIP。 + +要把同一双 Skill 用例改为真实 GitHub REST Tree 元数据，在上述 Maven 命令额外设置 `SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE=1`。测试进程从 `SKILLHUB_GITHUB_SYNC_LIVE_TOKEN` 或本机 `gh auth token` 读取凭证，仅用于公开 API 的 GET 请求；测试代码不写入文件或日志，也不影响正式服务的 Java 凭证池。Go 核对真实 repository ID、固定 commit 和完整递归 Tree 响应后，仅选择上述两个 Skill 的元数据，以控制此次联调的包数。可以先在 `github-sync-go/` 用 `SKILLHUB_GITHUB_SYNC_LIVE_TREE=1 go test ./internal/repository -run '^TestLiveEnumerateFixedPublicGitHubTree$' -count=1 -v` 验证完整 Tree 枚举及所有候选 Skill 的元数据预检；预检采用当前 `application.yml` 的 100 文件、单文件 10 MiB、包总量 100 MiB 限制，不下载其余包。真实 Tree 双包联调已在本地存储与 MinIO 模式通过，但不能据此认定全部 44 个 Skill、其他仓库或高并发容量已验收。 + +如需在同一固定公开仓库中顺序获取所有通过预检的包，可在 `github-sync-go/` 运行 `SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES=1 go test ./internal/repository -run '^TestLiveAcquireAllFixedPublicSkillsSequentially$' -count=1 -v -timeout=16m`。此显式公网用例从真实 Tree 生成包计划，在单包 64 MiB 工作区预算和 16 MiB 保留空间下逐包执行 Git 获取，核对 ZIP 条目、摘要，并在下一包开始前确认工作区为空。当前固定 commit 的 43 个可采包全部通过，另 1 个因 312 个文件超过 100 文件政策而被拒。用例不经过 Java 摄入或 Scanner；完成时的目录占用不等于 Git 执行期间的严格瞬时峰值。 + +如需检查整仓的 Go→Java 接收与后台处理，可先按上文生成单包参考 ZIP，再在 `server/` 运行 `SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE=/path/to/creator.zip SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE=1 SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES_INTEROP=1 ./mvnw -q -pl skillhub-app -am '-Dtest=GithubSyncHttpIngestionInteropTest#repositoryWorkerReportsEveryEligiblePublicSkillThroughJava' -Dsurefire.failIfNoSpecifiedTests=false test`；加 `SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE=s3` 验证隔离 MinIO。此显式用例把包上限恢复为当前 100 文件、100 MiB 政策，约需数分钟，验证 43 包接收后有 42 个同步待审项、1 个同 namespace slug 冲突，另一个超文件数目录不下载；待审前无市场最新版本。用例不自动批准待审项，不代表跨仓库并发或长期容量验收。 + +需要验证真实 Scanner 时，先从当前 `scanner/Dockerfile` 构建并启动独立 Scanner，确认其 `/health` 可访问；然后为上述定向测试额外设置 `SKILLHUB_GITHUB_SYNC_INTEROP_SCANNER_URL=http://127.0.0.1:`。单包测试可用 `GithubSyncHttpIngestionInteropTest#goArchiveCanTravelThroughHttpReceiptAndPersistedIngestion`，双包 worker 测试可用 `GithubSyncHttpIngestionInteropTest#repositoryWorkerAcquiresTwoPublicSkillsThroughJava`。该模式使用正式 Compose 相同的 `upload` 扫描方式；不设置此变量时仍使用 HTTP 测试桩。真实 Scanner 的结论取决于测试包内容和当前规则，不应固定假设所有包都安全。 + +内部接口字段、错误码和恢复语义见 [合同说明](../docs/api/github-sync-internal.md) 与 [OpenAPI](../docs/api/github-sync-internal.openapi.yaml)。 + +上述互操作测试还从同一公开采集包生成四个受控内容 revision，用不同固定 commit 模拟上游连续更新。它验证旧审核被新目标替代、旧目标不能批准、已发布版本在新审核期间保持可用，以及较旧包迟到时不能覆盖较新的来源绑定。这是协议和发布事务测试，不代表已对四个真实 GitHub commit 做公网采集。 +同一测试继续以另一个固定 commit 回报完全相同的包，验证 `NO_CHANGE`；再只调整包内 `LICENSE.txt`，验证内容指纹和同步待审目标发生变化。这两步也使用受控 revision，不代表对应 commit 曾在 GitHub 公网真实存在。 +摄入后还会启动独立 Java JVM 领取校验租约并退出，再由测试中的 Java 应用实例在租约过期后完成校验；发布阶段重复相同的进程丢失与恢复检查。这验证持久租约和后台续跑，不替代完整应用容器的停止、重新启动及部署配置验收。 diff --git a/github-sync-go/cmd/github-sync/main.go b/github-sync-go/cmd/github-sync/main.go new file mode 100644 index 000000000..281f30e79 --- /dev/null +++ b/github-sync-go/cmd/github-sync/main.go @@ -0,0 +1,225 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/worker" +) + +type config struct { + JavaURL string + Token string + HealthAddr string + Pool worker.PoolConfig +} + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + settings, err := loadConfig() + if err != nil { + slog.Error("GitHub sync configuration invalid", "code", "INVALID_CONFIGURATION") + os.Exit(1) + } + workspaceGuard, err := prepareTempWorkspace(settings.Pool.Repository.TempRoot) + if err != nil { + slog.Error("GitHub sync temporary workspace unavailable", "code", "TEMP_WORKSPACE_UNAVAILABLE") + os.Exit(1) + } + defer workspaceGuard.Close() + control, err := controlplane.NewClient(settings.JavaURL, settings.Token) + if err != nil { + slog.Error("GitHub sync control plane unavailable", "code", "INVALID_CONTROL_PLANE") + os.Exit(1) + } + listener, err := net.Listen("tcp", settings.HealthAddr) + if err != nil { + slog.Error("GitHub sync health listener unavailable", "code", "HEALTH_BIND_FAILED") + os.Exit(1) + } + counters := &worker.Counters{} + server := &http.Server{Handler: healthHandler(counters), ReadHeaderTimeout: 5 * time.Second} + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + serveDone := make(chan error, 1) + go func() { serveDone <- server.Serve(listener) }() + poolDone := make(chan error, 1) + go func() { poolDone <- worker.Run(ctx, control, settings.Pool, counters) }() + poolStopped := false + failed := false + select { + case <-ctx.Done(): + case err := <-serveDone: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("GitHub sync health server stopped", "code", "HEALTH_SERVER_FAILED") + failed = true + } + stop() + case err := <-poolDone: + poolStopped = true + if err != nil && !errors.Is(err, context.Canceled) { + slog.Error("GitHub sync worker stopped", "code", "WORKER_STOPPED") + failed = true + } + stop() + } + counters.DiscoveryReady.Store(false) + counters.RepositoryReady.Store(false) + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = server.Shutdown(shutdown) + if !poolStopped { + select { + case <-poolDone: + case <-time.After(20 * time.Second): + slog.Warn("GitHub sync shutdown timed out", "code", "SHUTDOWN_TIMEOUT") + failed = true + } + } + if failed { + os.Exit(1) + } +} + +func healthHandler(counters *worker.Counters) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusNoContent) + }) + mux.HandleFunc("GET /health/ready", func(response http.ResponseWriter, _ *http.Request) { + if !counters.DiscoveryReady.Load() || !counters.RepositoryReady.Load() { + response.WriteHeader(http.StatusServiceUnavailable) + return + } + response.WriteHeader(http.StatusNoContent) + }) + mux.HandleFunc("GET /metrics", func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + fmt.Fprintf(response, "skillhub_github_sync_claims_total{kind=\"discovery\"} %d\n", counters.DiscoveryClaims.Load()) + fmt.Fprintf(response, "skillhub_github_sync_claims_total{kind=\"repository\"} %d\n", counters.RepositoryClaims.Load()) + fmt.Fprintf(response, "skillhub_github_sync_errors_total{kind=\"discovery\"} %d\n", counters.DiscoveryErrors.Load()) + fmt.Fprintf(response, "skillhub_github_sync_errors_total{kind=\"repository\"} %d\n", counters.RepositoryErrors.Load()) + fmt.Fprintf(response, "skillhub_github_sync_active_tasks{kind=\"discovery\"} %d\n", counters.DiscoveryActive.Load()) + fmt.Fprintf(response, "skillhub_github_sync_active_tasks{kind=\"repository\"} %d\n", counters.RepositoryActive.Load()) + }) + return mux +} + +func loadConfig() (config, error) { + var settings config + settings.JavaURL = os.Getenv("SKILLHUB_GITHUB_SYNC_JAVA_URL") + settings.Token = os.Getenv("SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN") + settings.HealthAddr = valueOrDefault("SKILLHUB_GITHUB_SYNC_HEALTH_ADDR", ":8090") + var err error + if settings.Pool.DiscoveryWorkers, err = positiveInt("SKILLHUB_GITHUB_SYNC_DISCOVERY_WORKERS", 1); err != nil { + return config{}, err + } + if settings.Pool.RepositoryWorkers, err = positiveInt("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", 1); err != nil { + return config{}, err + } + if settings.Pool.Discovery.MaxPagesPerLease, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_PAGES_PER_LEASE", 20); err != nil { + return config{}, err + } + if settings.Pool.Repository.MaxTreeRequests, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_TREE_REQUESTS", 100); err != nil { + return config{}, err + } + if settings.Pool.Repository.MaxTreeEntries, err = positiveInt("SKILLHUB_GITHUB_SYNC_MAX_TREE_ENTRIES", 100000); err != nil { + return config{}, err + } + settings.Pool.PollInterval, err = time.ParseDuration(valueOrDefault("SKILLHUB_GITHUB_SYNC_POLL_INTERVAL", "5s")) + if err != nil || settings.Pool.PollInterval < time.Second { + return config{}, errors.New("invalid poll interval") + } + settings.Pool.Repository.TempRoot = os.Getenv("SKILLHUB_GITHUB_SYNC_TEMP_ROOT") + settings.Pool.Repository.AllowedS3Hosts = splitHosts(os.Getenv("SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS")) + settings.Pool.Repository.MaxWorkspaceBytes, err = positiveBytes("SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES") + if err != nil { + return config{}, err + } + settings.Pool.Repository.MinFreeBytes, err = nonnegativeBytes("SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES") + if err != nil { + return config{}, err + } + totalWorkspaceBytes, err := positiveBytes("SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES") + if err != nil || int64(settings.Pool.RepositoryWorkers) > + totalWorkspaceBytes/settings.Pool.Repository.MaxWorkspaceBytes { + return config{}, errors.New("repository workers exceed total workspace budget") + } + if !filepath.IsAbs(settings.Pool.Repository.TempRoot) || settings.JavaURL == "" || settings.Token == "" { + return config{}, errors.New("missing GitHub sync service configuration") + } + info, err := os.Stat(settings.Pool.Repository.TempRoot) + if err != nil || !info.IsDir() { + return config{}, errors.New("GitHub sync temporary directory is unavailable") + } + var filesystem syscall.Statfs_t + if err := syscall.Statfs(settings.Pool.Repository.TempRoot, &filesystem); err != nil || + uint64(totalWorkspaceBytes)+uint64(settings.Pool.Repository.MinFreeBytes) > + uint64(filesystem.Blocks)*uint64(filesystem.Bsize) { + return config{}, errors.New("temporary filesystem is smaller than configured workspace budget") + } + probe, err := os.MkdirTemp(settings.Pool.Repository.TempRoot, "write-check-") + if err != nil { + return config{}, errors.New("GitHub sync temporary directory is not writable") + } + if err := os.Remove(probe); err != nil { + return config{}, errors.New("GitHub sync temporary directory cannot be cleaned") + } + return settings, nil +} + +func valueOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func positiveInt(name string, fallback int) (int, error) { + value, err := strconv.Atoi(valueOrDefault(name, strconv.Itoa(fallback))) + if err != nil || value < 1 { + return 0, errors.New("invalid positive worker configuration") + } + return value, nil +} + +func positiveBytes(name string) (int64, error) { + value, err := strconv.ParseInt(os.Getenv(name), 10, 64) + if err != nil || value < 1 { + return 0, errors.New("invalid workspace byte budget") + } + return value, nil +} + +func nonnegativeBytes(name string) (int64, error) { + value, err := strconv.ParseInt(os.Getenv(name), 10, 64) + if err != nil || value < 0 { + return 0, errors.New("invalid minimum free byte budget") + } + return value, nil +} + +func splitHosts(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + var hosts []string + for _, item := range strings.Split(value, ",") { + if host := strings.TrimSpace(item); host != "" { + hosts = append(hosts, host) + } + } + return hosts +} diff --git a/github-sync-go/cmd/github-sync/main_test.go b/github-sync-go/cmd/github-sync/main_test.go new file mode 100644 index 000000000..71ca71edd --- /dev/null +++ b/github-sync-go/cmd/github-sync/main_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "strings" + "testing" +) + +func TestLoadConfigBoundsRepositoryWorkersByTotalWorkspace(t *testing.T) { + t.Setenv("SKILLHUB_GITHUB_SYNC_JAVA_URL", "http://127.0.0.1:8080/internal/github-sync/v1") + t.Setenv("SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN", strings.Repeat("t", 32)) + t.Setenv("SKILLHUB_GITHUB_SYNC_TEMP_ROOT", t.TempDir()) + t.Setenv("SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES", "33554432") + t.Setenv("SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES", "67108864") + t.Setenv("SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES", "8388608") + t.Setenv("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", "2") + + settings, err := loadConfig() + if err != nil || settings.Pool.RepositoryWorkers != 2 { + t.Fatalf("two workers within the total budget should start: workers=%d, err=%v", + settings.Pool.RepositoryWorkers, err) + } + + t.Setenv("SKILLHUB_GITHUB_SYNC_REPOSITORY_WORKERS", "3") + if _, err := loadConfig(); err == nil { + t.Fatal("workers exceeding the total workspace budget must be rejected") + } +} diff --git a/github-sync-go/cmd/github-sync/workspace.go b/github-sync-go/cmd/github-sync/workspace.go new file mode 100644 index 000000000..e78facc9d --- /dev/null +++ b/github-sync-go/cmd/github-sync/workspace.go @@ -0,0 +1,40 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "syscall" +) + +const packageWorkspacePrefix = "skillhub-github-package-" + +// Keep the directory locked while the worker runs so another process cannot +// remove active packages from a shared local workspace during startup. +func prepareTempWorkspace(root string) (*os.File, error) { + guard, err := os.Open(root) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(guard.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = guard.Close() + return nil, errors.New("GitHub sync temporary workspace is already in use") + } + entries, err := os.ReadDir(root) + if err != nil { + _ = guard.Close() + return nil, err + } + for _, entry := range entries { + if !strings.HasPrefix(entry.Name(), packageWorkspacePrefix) || + len(entry.Name()) == len(packageWorkspacePrefix) { + continue + } + if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil { + _ = guard.Close() + return nil, err + } + } + return guard, nil +} diff --git a/github-sync-go/cmd/github-sync/workspace_test.go b/github-sync-go/cmd/github-sync/workspace_test.go new file mode 100644 index 000000000..16009323d --- /dev/null +++ b/github-sync-go/cmd/github-sync/workspace_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPrepareTempWorkspaceCleansOnlyAbandonedPackages(t *testing.T) { + root := t.TempDir() + abandoned := filepath.Join(root, packageWorkspacePrefix+"abandoned") + if err := os.Mkdir(abandoned, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(abandoned, "object"), []byte("old"), 0600); err != nil { + t.Fatal(err) + } + unrelated := filepath.Join(root, "operator-note") + if err := os.WriteFile(unrelated, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + guard, err := prepareTempWorkspace(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = guard.Close() }) + if _, err := os.Stat(abandoned); !os.IsNotExist(err) { + t.Fatalf("abandoned package remains: %v", err) + } + if content, err := os.ReadFile(unrelated); err != nil || string(content) != "keep" { + t.Fatalf("unrelated file changed: %q, %v", content, err) + } + if _, err := prepareTempWorkspace(root); err == nil { + t.Fatal("a second worker must not clean an active workspace") + } +} + +func TestPrepareTempWorkspaceCanRecoverAfterPreviousOwnerExits(t *testing.T) { + root := t.TempDir() + guard, err := prepareTempWorkspace(root) + if err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, packageWorkspacePrefix+"interrupted"), 0700); err != nil { + t.Fatal(err) + } + if err := guard.Close(); err != nil { + t.Fatal(err) + } + guard, err = prepareTempWorkspace(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = guard.Close() }) + if entries, err := os.ReadDir(root); err != nil || len(entries) != 0 { + t.Fatalf("restarted worker did not clear its old package: %v, %v", entries, err) + } +} diff --git a/github-sync-go/go.mod b/github-sync-go/go.mod new file mode 100644 index 000000000..3cf6a381c --- /dev/null +++ b/github-sync-go/go.mod @@ -0,0 +1,3 @@ +module github.com/iflytek/skillhub/github-sync-go + +go 1.27.1 diff --git a/github-sync-go/internal/controlplane/client.go b/github-sync-go/internal/controlplane/client.go new file mode 100644 index 000000000..34ff0b97d --- /dev/null +++ b/github-sync-go/internal/controlplane/client.go @@ -0,0 +1,592 @@ +package controlplane + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const maxResponseBytes = 16 << 20 + +// Client speaks the versioned internal API. It never uses the GitHub proxy +// transport and never sends a service token to a redirect target. +type Client struct { + baseURL string + token string + http *http.Client +} + +func NewClient(baseURL, serviceToken string) (*Client, error) { + parsed, err := url.Parse(baseURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || + parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || + len(strings.TrimSpace(serviceToken)) < 32 { + return nil, errors.New("invalid internal API URL or missing service token") + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + token: strings.TrimSpace(serviceToken), + http: &http.Client{ + Timeout: 20 * time.Second, + Transport: &http.Transport{ + Proxy: nil, + MaxIdleConnsPerHost: 4, + ResponseHeaderTimeout: 15 * time.Second, + }, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +func (c *Client) Heartbeat(ctx context.Context) error { + _, err := c.post(ctx, "/heartbeat", struct{}{}, nil) + return err +} + +type LeaseProof struct { + Generation int64 `json:"generation"` + Token string `json:"token"` +} + +type PackageAdmission struct { + AdmissionID string `json:"admissionId"` +} + +type ClaimedTask struct { + PlatformControlVersion *int64 `json:"platformControlVersion,omitempty"` + TaskID string `json:"taskId"` + RunID string `json:"runId"` + Kind string `json:"kind"` + Mode string `json:"mode,omitempty"` + NamespaceID int64 `json:"namespaceId"` + SourceID int64 `json:"sourceId,omitempty"` + Lease LeaseProof `json:"lease"` + LeaseExpiresAt time.Time `json:"leaseExpiresAt"` + Discovery *DiscoverySnapshot `json:"discovery,omitempty"` + OfficialDiscovery *OfficialDiscoverySnapshot `json:"officialDiscovery,omitempty"` + RequiredCapabilities []string `json:"requiredCapabilities,omitempty"` + Repository *RepositorySnapshot `json:"repository,omitempty"` + Credential *GithubCredential `json:"credential,omitempty"` + MaxPackageBytes int64 `json:"maxPackageBytes,omitempty"` + MaxSingleFileBytes int64 `json:"maxSingleFileBytes,omitempty"` + MaxFilesPerPackage int `json:"maxFilesPerPackage,omitempty"` + MaxUploadBytes int64 `json:"maxUploadBytes,omitempty"` +} + +type GithubCredential struct { + CredentialID string `json:"credentialId"` + Token string `json:"token"` +} + +type GithubQuota struct { + Resource string `json:"resource"` + Remaining int `json:"remaining"` + ResetAt time.Time `json:"resetAt"` +} + +type DiscoverySnapshot struct { + EntryType string `json:"entryType"` + RuleID string `json:"ruleId,omitempty"` + RuleVersion int64 `json:"ruleVersion"` + Criteria *SearchCriteria `json:"criteria,omitempty"` + SearchWindow *SearchWindow `json:"searchWindow,omitempty"` + Owner string `json:"owner,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` + Checkpoint string `json:"checkpoint,omitempty"` + CandidateBudget int `json:"candidateBudget,omitempty"` +} + +type SearchCriteria struct { + Selectors []SearchSelector `json:"selectors"` +} + +type SearchSelector struct { + Kind string `json:"kind"` + Value string `json:"value"` + Priority string `json:"priority"` +} + +type SearchWindow struct { + Mode string `json:"mode"` + Since time.Time `json:"since,omitempty"` + Until time.Time `json:"until"` +} + +type RepositorySnapshot struct { + RepositoryID int64 `json:"repositoryId"` + Owner string `json:"owner"` + Name string `json:"name"` + FixedCommitSHA string `json:"fixedCommitSha,omitempty"` + PathCheckpoint string `json:"pathCheckpoint,omitempty"` + IncompleteSubtrees []string `json:"incompleteSubtrees,omitempty"` + RetryPaths []string `json:"retryPaths,omitempty"` + AcceptedPaths []string `json:"acceptedPaths,omitempty"` + IgnorePatterns []string `json:"ignorePatterns,omitempty"` +} + +type RepositoryPathResult struct { + SkillPath string `json:"skillPath"` + Outcome string `json:"outcome"` + IngestionID string `json:"ingestionId,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type RepositoryCompletion struct { + CommitSHA string + PathCheckpoint string + IncompleteSubtrees []string + PathResults []RepositoryPathResult + RetryAt time.Time + DeferralReason string +} + +type RepositoryCandidate struct { + RepositoryID int64 `json:"repositoryId"` + Owner string `json:"owner"` + Name string `json:"name"` + DefaultBranch string `json:"defaultBranch,omitempty"` + OfficialProvenance []OfficialProvenance `json:"officialProvenance,omitempty"` +} + +// RepositoryMetadata is the canonical public identity returned by GitHub for +// the stable repository ID in a leased task. +type RepositoryMetadata struct { + OwnerID int64 `json:"ownerId,omitempty"` + RepositoryID int64 `json:"repositoryId"` + Owner string `json:"owner"` + Name string `json:"name"` + DefaultBranch string `json:"defaultBranch"` +} + +func (c *Client) UpdateRepositoryMetadata(ctx context.Context, taskID string, + lease LeaseProof, metadata RepositoryMetadata) error { + if metadata.RepositoryID <= 0 || metadata.Owner == "" || metadata.Name == "" || + metadata.DefaultBranch == "" || metadata.OwnerID < 0 { + return errors.New("invalid GitHub repository metadata") + } + request := map[string]any{ + "lease": lease, "repositoryId": metadata.RepositoryID, "owner": metadata.Owner, + "name": metadata.Name, "defaultBranch": metadata.DefaultBranch, + } + if metadata.OwnerID > 0 { + request["ownerId"] = metadata.OwnerID + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/repository-metadata", request, nil) + return err +} + +type DiscoveryBatch struct { + PreviousCheckpoint *string `json:"previousCheckpoint"` + Checkpoint string `json:"checkpoint"` + Repositories []RepositoryCandidate `json:"repositories"` + DeferredCount int `json:"deferredCount,omitempty"` + DeferralReason string `json:"deferralReason,omitempty"` + OfficialBatch *OfficialDiscoveryBatch `json:"officialBatch,omitempty"` +} + +type PreviewSelectorResult struct { + Kind string `json:"kind"` + Value string `json:"value"` + Priority string `json:"priority"` + TotalCount int `json:"totalCount"` + IncompleteResults bool `json:"incompleteResults"` + LimitReached bool `json:"limitReached"` + Samples []RepositoryCandidate `json:"samples"` +} + +func (c *Client) CompletePreview(ctx context.Context, taskID string, lease LeaseProof, + results []PreviewSelectorResult) error { + if len(results) < 1 || len(results) > 20 { + return errors.New("invalid preview result count") + } + var response struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/preview-results", map[string]any{ + "lease": lease, "selectors": results, + }, &response) + if err != nil { + return err + } + if response.Status != "ACQUISITION_COMPLETE" { + return errors.New("internal API did not complete preview") + } + return nil +} + +// APIError intentionally omits the server's free-form message, which can +// contain sensitive upstream details. Callers can act on Code and RetryAt. +type APIError struct { + Status int + Code string + RetryAt time.Time +} + +func (e *APIError) Error() string { + return fmt.Sprintf("internal API returned HTTP %d (%s)", e.Status, e.SafeCode()) +} + +// SafeCode is suitable for logs. The response code is untrusted HTTP content. +func (e *APIError) SafeCode() string { + switch e.Code { + case "ADMISSION_STALE", "BACKPRESSURE_ACTIVE", "CANDIDATE_BUDGET_EXCEEDED", + "CHECKPOINT_CONFLICT", "COMMIT_MISMATCH", "CONTROL_CHANGED", "PLATFORM_SCOPE_MISMATCH", "COVERAGE_CONFLICT", + "COVERAGE_UNSUPPORTED", "INVALID_REQUEST", "LEASE_STALE", + "PACKAGE_TOO_LARGE", "PATH_IGNORED", "PROTOCOL_UNSUPPORTED", + "REPOSITORY_MISMATCH", "RESULT_CONFLICT", "UPLOAD_MISMATCH", + "UPLOAD_NOT_READY", "UPLOAD_TICKET_INVALID", "SERVICE_UNAVAILABLE": + return e.Code + default: + return "CONTROL_PLANE_ERROR" + } +} + +func (c *Client) Claim(ctx context.Context, workerID, kind string) (*ClaimedTask, error) { + return c.ClaimWithCapabilities(ctx, workerID, kind, nil) +} + +// ClaimWithCapabilities is opt-in. Capability declarations must only be supplied +// by an executor that implements their full lifecycle, including safe pause. +func (c *Client) ClaimWithCapabilities(ctx context.Context, workerID, kind string, capabilities []string) (*ClaimedTask, error) { + if workerID == "" || len(workerID) > 128 || + (kind != "DISCOVERY" && kind != "REPOSITORY_SYNC") { + return nil, errors.New("invalid worker ID or task kind") + } + if err := validateCapabilities(capabilities); err != nil { + return nil, err + } + var task ClaimedTask + request := map[string]any{ + "workerId": workerID, "protocolMajor": 1, "kind": kind, + } + if len(capabilities) > 0 { + request["capabilities"] = capabilities + } + status, err := c.post(ctx, "/tasks/claim", request, &task) + if err != nil { + return nil, err + } + if status == http.StatusNoContent { + return nil, nil + } + if task.TaskID == "" || task.RunID == "" || task.NamespaceID <= 0 || + task.Lease.Generation < 1 || task.Lease.Token == "" || task.LeaseExpiresAt.IsZero() || + task.Kind != kind { + return nil, errors.New("internal API returned an incomplete task") + } + if err := validateOfficialTask(task, capabilities); err != nil { + return nil, err + } + if task.OfficialDiscovery != nil { + return &task, nil + } + if task.Kind == "DISCOVERY" && (task.Discovery == nil || task.Discovery.RuleVersion < 1 || + task.Discovery.CandidateBudget < 1 || + (task.Discovery.EntryType == "SEARCH_RULE" && + (task.Discovery.Criteria == nil || task.Discovery.SearchWindow == nil)) || + (task.Discovery.EntryType == "CURATED_OWNER" && task.Discovery.Owner == "")) { + return nil, errors.New("internal API returned an incomplete discovery task") + } + if task.Kind == "DISCOVERY" && task.Discovery.EntryType == "CURATED_REPOSITORY" && + (task.Discovery.Owner == "" || task.Discovery.RepositoryName == "") { + return nil, errors.New("internal API returned an incomplete discovery task") + } + if task.Kind == "REPOSITORY_SYNC" && (task.Repository == nil || task.Repository.RepositoryID <= 0 || + task.Repository.Owner == "" || task.Repository.Name == "" || + (task.Mode != "FULL" && task.Mode != "HEAD_ONLY")) { + return nil, errors.New("internal API returned an incomplete repository task") + } + if task.Kind == "REPOSITORY_SYNC" && task.Mode == "FULL" && + (task.MaxPackageBytes < 1 || task.MaxSingleFileBytes < 1 || + task.MaxFilesPerPackage < 1 || task.MaxUploadBytes < 1) { + return nil, errors.New("internal API returned no package limits") + } + return &task, nil +} + +func (c *Client) Renew(ctx context.Context, taskID string, lease LeaseProof) (time.Time, error) { + var response struct { + LeaseExpiresAt time.Time `json:"leaseExpiresAt"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/renew", map[string]any{ + "lease": lease, + }, &response) + if err != nil { + return time.Time{}, err + } + if response.LeaseExpiresAt.IsZero() { + return time.Time{}, errors.New("internal API returned no lease expiration") + } + return response.LeaseExpiresAt, nil +} + +// AdmitPackage reserves capacity before the worker requests any file content for this Skill. +func (c *Client) AdmitPackage(ctx context.Context, taskID string, lease LeaseProof, + commitSHA, skillPath string) (PackageAdmission, error) { + var admission PackageAdmission + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/package-admissions", map[string]any{ + "lease": lease, "commitSha": commitSHA, "skillPath": skillPath, + }, &admission) + if err != nil { + return PackageAdmission{}, err + } + if admission.AdmissionID == "" { + return PackageAdmission{}, errors.New("internal API returned no package admission ID") + } + return admission, nil +} + +// ReportDiscoveryBatch persists leads and advances an opaque checkpoint atomically. +// A replay with the same batch succeeds; a stale previous checkpoint is rejected. +func (c *Client) ReportDiscoveryBatch(ctx context.Context, taskID string, + lease LeaseProof, batch DiscoveryBatch) error { + if err := validateOfficialBatch(batch); err != nil { + return err + } + if batch.Checkpoint == "" || len(batch.Checkpoint) > 4096 || len(batch.Repositories) > 100 || + batch.DeferredCount < 0 || (batch.DeferredCount > 0 && batch.DeferralReason == "") { + return errors.New("invalid discovery batch") + } + for _, repository := range batch.Repositories { + if repository.RepositoryID <= 0 || repository.Owner == "" || repository.Name == "" { + return errors.New("invalid discovery repository candidate") + } + } + if batch.Repositories == nil { + batch.Repositories = []RepositoryCandidate{} + } + status, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/discovered-repositories", + struct { + Lease LeaseProof `json:"lease"` + DiscoveryBatch + }{Lease: lease, DiscoveryBatch: batch}, nil) + if err != nil { + return err + } + if status != http.StatusAccepted { + return errors.New("internal API did not accept discovery batch") + } + if observer, ok := ctx.Value(checkpointObserverKey{}).(func(string)); ok { + observer(batch.Checkpoint) + } + return nil +} + +// CompleteDiscovery closes a fully covered search or releases its lease for a +// delayed continuation. The checkpoint must already have been acknowledged. +func (c *Client) CompleteDiscovery(ctx context.Context, taskID string, lease LeaseProof, + checkpoint string, retryAt time.Time, deferralReason string) (string, error) { + return c.completeDiscovery(ctx, taskID, lease, checkpoint, retryAt, deferralReason, "RETRY_WAIT") +} + +func (c *Client) completeDiscovery(ctx context.Context, taskID string, lease LeaseProof, + checkpoint string, retryAt time.Time, deferralReason, deferredStatus string) (string, error) { + if checkpoint == "" || len(checkpoint) > 4096 || + (retryAt.IsZero() && deferralReason != "") || + (!retryAt.IsZero() && !retryAt.After(time.Now())) { + return "", errors.New("invalid discovery completion") + } + coverage := "COMPLETE" + if !retryAt.IsZero() { + coverage = "INCOMPLETE" + } + request := map[string]any{ + "lease": lease, "coverage": coverage, "discoveryCheckpoint": checkpoint, + } + if !retryAt.IsZero() { + request["retryAt"] = retryAt + if deferralReason != "" { + request["deferralReason"] = deferralReason + } + } + var response struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", request, &response) + if err != nil { + return "", err + } + if (coverage == "COMPLETE" && response.Status != "ACQUISITION_COMPLETE") || + (coverage == "INCOMPLETE" && response.Status != deferredStatus) { + return "", errors.New("internal API returned inconsistent discovery status") + } + return response.Status, nil +} + +// CompleteHeadOnly records an observed commit without claiming package acquisition finished. +func (c *Client) CompleteHeadOnly(ctx context.Context, taskID string, lease LeaseProof, observedCommitSHA string) error { + if len(observedCommitSHA) != 40 { + return errors.New("observed commit SHA must have 40 characters") + } + var response struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", map[string]any{ + "lease": lease, "coverage": "HEAD_ONLY", "observedCommitSha": observedCommitSHA, + }, &response) + if err != nil { + return err + } + if response.Status != "ACQUISITION_COMPLETE" { + return errors.New("internal API did not complete HEAD_ONLY task") + } + return nil +} + +// CompleteRepository closes a fixed-revision acquisition or records a delayed +// continuation. Accepted paths must already have been acknowledged by Java. +func (c *Client) CompleteRepository(ctx context.Context, taskID string, lease LeaseProof, + completion RepositoryCompletion) (string, error) { + if len(completion.CommitSHA) != 40 || len(completion.PathCheckpoint) > 1024 || + len(completion.IncompleteSubtrees) > 1000 || len(completion.PathResults) > 10000 { + return "", errors.New("invalid repository completion") + } + for _, digit := range completion.CommitSHA { + if digit < '0' || digit > '9' && (digit < 'a' || digit > 'f') { + return "", errors.New("invalid repository commit SHA") + } + } + coverage := "COMPLETE" + if !completion.RetryAt.IsZero() { + if !completion.RetryAt.After(time.Now()) { + return "", errors.New("repository retry time must be in the future") + } + coverage = "INCOMPLETE" + } else if completion.DeferralReason != "" || len(completion.IncompleteSubtrees) > 0 { + return "", errors.New("complete repository has deferred coverage") + } + for _, result := range completion.PathResults { + if result.SkillPath == "" || (result.Outcome == "ACCEPTED") != (result.IngestionID != "") { + return "", errors.New("invalid repository path result") + } + switch result.Outcome { + case "ACCEPTED", "IGNORED", "TERMINAL_FAILURE": + case "RETRYABLE_FAILURE": + if coverage == "COMPLETE" { + return "", errors.New("complete repository has retryable path") + } + default: + return "", errors.New("invalid repository path outcome") + } + } + request := map[string]any{ + "lease": lease, "coverage": coverage, "observedCommitSha": completion.CommitSHA, + "pathResults": completion.PathResults, "incompleteSubtrees": completion.IncompleteSubtrees, + } + if completion.PathCheckpoint != "" { + request["pathCheckpoint"] = completion.PathCheckpoint + } + if coverage == "INCOMPLETE" { + request["retryAt"] = completion.RetryAt + if completion.DeferralReason != "" { + request["deferralReason"] = completion.DeferralReason + } + } + var response struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/complete", request, &response) + if err != nil { + return "", err + } + if coverage == "COMPLETE" && response.Status != "ACQUISITION_COMPLETE" && response.Status != "PARTIAL" || + coverage == "INCOMPLETE" && response.Status != "RETRY_WAIT" { + return "", errors.New("internal API returned inconsistent repository status") + } + return response.Status, nil +} + +// Fail releases the lease and returns the Java-owned retry time. +func (c *Client) Fail(ctx context.Context, taskID string, lease LeaseProof, + reason, detail string, retryAt time.Time, quota *GithubQuota) (time.Time, error) { + request := map[string]any{"lease": lease, "reason": reason} + if detail != "" { + request["detail"] = detail + } + if quota != nil { + request["githubQuota"] = quota + } + if !retryAt.IsZero() { + request["retryAt"] = retryAt + } + var response struct { + Status string `json:"status"` + RetryAt time.Time `json:"retryAt"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/fail", request, &response) + if err != nil { + return time.Time{}, err + } + if response.Status != "RETRY_WAIT" || response.RetryAt.IsZero() { + return time.Time{}, errors.New("internal API returned no retry schedule") + } + return response.RetryAt, nil +} + +// ReportCredentialStatus updates only the credential bound to this current task lease. +func (c *Client) ReportCredentialStatus(ctx context.Context, taskID string, lease LeaseProof, + credentialID, status string, retryAt time.Time, quota *GithubQuota) error { + request := map[string]any{"lease": lease, "credentialId": credentialID, "status": status} + if !retryAt.IsZero() { + request["retryAt"] = retryAt + } + if quota != nil { + request["githubQuota"] = quota + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/credential-status", request, nil) + return err +} + +func (c *Client) post(ctx context.Context, path string, body any, target any) (int, error) { + encoded, err := json.Marshal(body) + if err != nil { + return 0, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(encoded)) + if err != nil { + return 0, err + } + requestID := make([]byte, 16) + if _, err := rand.Read(requestID); err != nil { + return 0, err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("X-Request-Id", hex.EncodeToString(requestID)) + request.Header.Set("Content-Type", "application/json") + response, err := c.http.Do(request) + if err != nil { + return 0, err + } + defer response.Body.Close() + limited := io.LimitReader(response.Body, maxResponseBytes+1) + if response.StatusCode < 200 || response.StatusCode >= 300 { + var apiError struct { + Code string `json:"code"` + RetryAt time.Time `json:"retryAt"` + } + _ = json.NewDecoder(limited).Decode(&apiError) + return response.StatusCode, &APIError{Status: response.StatusCode, Code: apiError.Code, RetryAt: apiError.RetryAt} + } + if response.StatusCode == http.StatusNoContent || target == nil { + return response.StatusCode, nil + } + contents, err := io.ReadAll(limited) + if err != nil || len(contents) > maxResponseBytes { + return 0, errors.New("internal API response exceeds size limit") + } + if err := json.Unmarshal(contents, target); err != nil { + return 0, fmt.Errorf("decode internal API response: %w", err) + } + return response.StatusCode, nil +} diff --git a/github-sync-go/internal/controlplane/official.go b/github-sync-go/internal/controlplane/official.go new file mode 100644 index 000000000..5f826d35f --- /dev/null +++ b/github-sync-go/internal/controlplane/official.go @@ -0,0 +1,259 @@ +package controlplane + +import ( + "errors" + "slices" + "time" + "unicode/utf8" +) + +const ( + OfficialDiscoveryCapability = "official-discovery/v1" + OfficialCheckpointCapability = "official-checkpoint/v1" + OfficialPauseCapability = "official-pause/v1" + OfficialOwnerCapability = "official-owner/v1" +) + +// OfficialDiscoverySnapshot is an immutable Java-owned policy and shard snapshot. +// Decoding it does not imply that this worker implements the official executor. +type OfficialDiscoverySnapshot struct { + FormatVersion int `json:"formatVersion"` + ConfigurationVersion int64 `json:"configurationVersion"` + StrategyVersion string `json:"strategyVersion"` + QueryID string `json:"queryId"` + QueryVersion int64 `json:"queryVersion"` + ShardID string `json:"shardId"` + Channel string `json:"channel"` + Mode string `json:"mode"` + Query OfficialQuery `json:"query"` + Range OfficialQueryRange `json:"range"` + FixedCutoffAt time.Time `json:"fixedCutoffAt"` + SeedRepositoryID int64 `json:"seedRepositoryId,omitempty"` + OwnerID int64 `json:"ownerId,omitempty"` + OwnerLogin string `json:"ownerLogin,omitempty"` + ExpansionDepth int `json:"expansionDepth,omitempty"` + Checkpoint string `json:"checkpoint,omitempty"` + Budget OfficialBudget `json:"budget"` +} + +type OfficialQuery struct { + Kind string `json:"kind"` + Value string `json:"value"` + Scope string `json:"scope"` +} + +type OfficialQueryRange struct { + Since time.Time `json:"since"` + Until time.Time `json:"until"` + Partition string `json:"partition,omitempty"` +} + +type OfficialBudget struct { + Profile string `json:"profile"` + MaxRequests int64 `json:"maxRequests"` + MaxPages int64 `json:"maxPages"` + CandidateBudget int64 `json:"candidateBudget"` + MaxSplitDepth int64 `json:"maxSplitDepth"` + MaxChildShards int64 `json:"maxChildShards"` +} + +type OfficialCheckpoint struct { + FormatVersion int `json:"formatVersion"` + StrategyVersion string `json:"strategyVersion"` + QueryID string `json:"queryId"` + QueryVersion int64 `json:"queryVersion"` + ShardID string `json:"shardId"` + NextPage int64 `json:"nextPage"` + Cursor string `json:"cursor,omitempty"` + Coverage string `json:"coverage"` +} + +type OfficialProvenance struct { + Channel string `json:"channel"` + QueryID string `json:"queryId"` + QueryVersion int64 `json:"queryVersion"` + StrategyVersion string `json:"strategyVersion"` + ShardID string `json:"shardId"` + SeedRepositoryID int64 `json:"seedRepositoryId,omitempty"` +} + +type OfficialDiscoveryBatch struct { + OwnerLogin string `json:"ownerLogin,omitempty"` + ShardID string `json:"shardId"` + Coverage string `json:"coverage"` + RequestsUsed int64 `json:"requestsUsed"` + ChildRanges []OfficialQueryRange `json:"childRanges,omitempty"` + GapReason string `json:"gapReason,omitempty"` +} + +type OfficialTaskControl struct { + Action string `json:"action"` + ConfigurationVersion int64 `json:"configurationVersion"` + Reason string `json:"reason,omitempty"` + RetryAt *time.Time `json:"retryAt,omitempty"` +} + +// Null commit/checkpoint values must stay explicit on the wire. No commit is +// required when pausing before the first repository read has completed. +type OfficialRepositoryProgress struct { + ObservedCommitSHA *string `json:"observedCommitSha"` + PathCheckpoint *string `json:"pathCheckpoint"` + IncompleteSubtrees []string `json:"incompleteSubtrees"` + PathResults []RepositoryPathResult `json:"pathResults"` +} + +type OfficialCheckpointRecovery struct { + Lease LeaseProof `json:"lease"` + Reason string `json:"reason"` + RejectedCheckpoint string `json:"rejectedCheckpoint"` +} + +type OfficialQuotaObservation struct { + Resource string `json:"resource"` + ObservedAt time.Time `json:"observedAt"` + State string `json:"state"` + Remaining *int `json:"remaining,omitempty"` + ResetAt *time.Time `json:"resetAt,omitempty"` + RetryAfterSeconds *int `json:"retryAfterSeconds,omitempty"` +} + +func validateCapabilities(capabilities []string) error { + if len(capabilities) > 32 { + return errors.New("too many worker capabilities") + } + for _, capability := range capabilities { + if !boundedOfficialString(capability, 128) { + return errors.New("invalid worker capability") + } + } + return nil +} + +func boundedOfficialString(value string, maximum int) bool { + return value != "" && utf8.ValidString(value) && utf8.RuneCountInString(value) <= maximum +} + +func validOfficialTime(value time.Time) bool { + _, offset := value.Zone() + return !value.IsZero() && value.Nanosecond() == 0 && offset == 0 +} + +func validOfficialRange(window OfficialQueryRange) bool { + return validOfficialTime(window.Since) && validOfficialTime(window.Until) && + window.Since.Before(window.Until) && window.Partition == "" +} + +func validateOfficialTask(task ClaimedTask, capabilities []string) error { + if err := validateCapabilities(task.RequiredCapabilities); err != nil { + return err + } + for _, required := range task.RequiredCapabilities { + if !slices.Contains(capabilities, required) { + return errors.New("internal API returned a task requiring an undeclared capability") + } + } + if task.PlatformControlVersion != nil && (*task.PlatformControlVersion < 1 || !slices.Contains(task.RequiredCapabilities, PlatformPauseCapability)) { + return errors.New("internal API returned invalid platform control capability") + } + if slices.Contains(task.RequiredCapabilities, PlatformPauseCapability) && task.PlatformControlVersion == nil { + return errors.New("internal API omitted platform control version") + } + snapshot := task.OfficialDiscovery + if snapshot == nil { + if task.Kind == "DISCOVERY" { + for _, required := range task.RequiredCapabilities { + if required != PlatformPauseCapability { + return errors.New("internal API returned capabilities without an official discovery snapshot") + } + } + } + return nil + } + if task.Kind != "DISCOVERY" || task.Discovery != nil || task.Repository != nil { + return errors.New("internal API returned conflicting discovery snapshots") + } + for _, required := range []string{OfficialDiscoveryCapability, OfficialCheckpointCapability, OfficialPauseCapability} { + if !slices.Contains(task.RequiredCapabilities, required) { + return errors.New("internal API omitted a required official discovery capability") + } + } + if snapshot.FormatVersion != 1 || snapshot.ConfigurationVersion < 1 || snapshot.QueryVersion < 1 || + !boundedOfficialString(snapshot.StrategyVersion, 128) || !boundedOfficialString(snapshot.QueryID, 128) || + snapshot.ShardID == "" || !boundedOfficialString(snapshot.Query.Value, 256) || + !validOfficialRange(snapshot.Range) || !validOfficialTime(snapshot.FixedCutoffAt) || + snapshot.Range.Until.After(snapshot.FixedCutoffAt) || len(snapshot.Checkpoint) > 4096 || + !utf8.ValidString(snapshot.Checkpoint) { + return errors.New("internal API returned an invalid official discovery snapshot") + } + if !slices.Contains([]string{"HISTORICAL", "INCREMENTAL", "GAP_RECHECK", "OWNER_RECHECK"}, snapshot.Mode) { + return errors.New("internal API returned an unsupported official discovery mode") + } + budget := snapshot.Budget + if !slices.Contains([]string{"ECONOMY", "BALANCED", "ACCELERATED"}, budget.Profile) || + budget.MaxRequests < 1 || budget.MaxPages < 1 || budget.CandidateBudget < 1 || + budget.MaxSplitDepth < 0 || budget.MaxSplitDepth > 32 || budget.MaxChildShards < 0 || budget.MaxChildShards > 126 { + return errors.New("internal API returned an invalid official discovery budget") + } + switch snapshot.Channel { + case "REPOSITORY_METADATA": + if snapshot.Query.Scope != "NAME_DESCRIPTION_TOPICS" || + (snapshot.Query.Kind != "KEYWORD" && snapshot.Query.Kind != "TOPIC") { + return errors.New("invalid official metadata query") + } + case "README": + if snapshot.Query.Scope != "README" || snapshot.Query.Kind != "KEYWORD" { + return errors.New("invalid official README query") + } + case "OWNER_EXPANSION": + if !slices.Contains(task.RequiredCapabilities, OfficialOwnerCapability) || snapshot.Mode != "OWNER_RECHECK" || budget.MaxRequests < 3 || budget.MaxSplitDepth != 0 || budget.MaxChildShards != 0 || snapshot.Range.Partition != "" || snapshot.Query.Scope != "OWNER_REPOSITORIES" || snapshot.Query.Kind != "OWNER" || + snapshot.SeedRepositoryID <= 0 || snapshot.OwnerID <= 0 || + !boundedOfficialString(snapshot.OwnerLogin, 39) || snapshot.ExpansionDepth != 1 || + snapshot.Query.Value != snapshot.OwnerLogin { + return errors.New("invalid official owner expansion") + } + default: + return errors.New("unsupported official discovery channel") + } + if snapshot.Channel != "OWNER_EXPANSION" && + (snapshot.SeedRepositoryID != 0 || snapshot.OwnerID != 0 || snapshot.OwnerLogin != "" || snapshot.ExpansionDepth != 0) { + return errors.New("official search contains owner expansion metadata") + } + return nil +} + +func validateOfficialBatch(batch DiscoveryBatch) error { + official := batch.OfficialBatch + for _, repository := range batch.Repositories { + if len(repository.OfficialProvenance) > 16 || + (official == nil) != (len(repository.OfficialProvenance) == 0) { + return errors.New("invalid official provenance count or batch") + } + for _, provenance := range repository.OfficialProvenance { + if (official.OwnerLogin != "" && provenance.Channel != "OWNER_EXPANSION") || !boundedOfficialString(provenance.QueryID, 128) || provenance.QueryVersion < 1 || + !boundedOfficialString(provenance.StrategyVersion, 128) || provenance.ShardID != official.ShardID || + !slices.Contains([]string{"REPOSITORY_METADATA", "README", "OWNER_EXPANSION"}, provenance.Channel) || + (provenance.Channel == "OWNER_EXPANSION" && provenance.SeedRepositoryID < 1) || + (provenance.Channel != "OWNER_EXPANSION" && provenance.SeedRepositoryID != 0) { + return errors.New("invalid official repository provenance") + } + } + } + if official == nil { + return nil + } + if (official.OwnerLogin != "" && !boundedOfficialString(official.OwnerLogin, 39)) || official.ShardID == "" || official.RequestsUsed < 0 || len(official.ChildRanges) > 16 || + !slices.Contains([]string{"IN_PROGRESS", "COMPLETE", "SPLIT", "GAP"}, official.Coverage) || + (official.Coverage == "SPLIT") != (len(official.ChildRanges) > 0) || + (official.Coverage == "GAP") != (official.GapReason != "") { + return errors.New("invalid official discovery batch") + } + if official.GapReason != "" && !slices.Contains([]string{"RESULT_LIMIT", "INCOMPLETE_RESULTS", "UNSPLITTABLE", "WORK_BUDGET"}, official.GapReason) { + return errors.New("invalid official discovery gap reason") + } + for _, child := range official.ChildRanges { + if !validOfficialRange(child) { + return errors.New("invalid official child range") + } + } + return nil +} diff --git a/github-sync-go/internal/controlplane/official_client.go b/github-sync-go/internal/controlplane/official_client.go new file mode 100644 index 000000000..d4f6e05c4 --- /dev/null +++ b/github-sync-go/internal/controlplane/official_client.go @@ -0,0 +1,246 @@ +package controlplane + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "slices" + "time" + "unicode/utf8" +) + +// CompleteOfficialDiscovery releases a search GAP as PARTIAL for its siblings; +// owner GAPs retain RETRY_WAIT. The acknowledged checkpoint is sent unchanged. +func (c *Client) CompleteOfficialDiscovery(ctx context.Context, taskID string, lease LeaseProof, + checkpoint string, retryAt time.Time, deferralReason, channel string) (string, error) { + var progress OfficialCheckpoint + if err := json.Unmarshal([]byte(checkpoint), &progress); err != nil || progress.FormatVersion != 1 || + !slices.Contains([]string{"REPOSITORY_METADATA", "README", "OWNER_EXPANSION"}, channel) || + (channel == "OWNER_EXPANSION" && progress.Coverage == "SPLIT") { + return "", errors.New("invalid official discovery completion checkpoint") + } + deferredStatus := "RETRY_WAIT" + switch progress.Coverage { + case "GAP": + if retryAt.IsZero() { + return "", errors.New("official GAP completion requires a retry time") + } + if channel != "OWNER_EXPANSION" { + deferredStatus = "PARTIAL" + } + case "IN_PROGRESS": + if retryAt.IsZero() { + return "", errors.New("unfinished official discovery requires a retry time") + } + case "COMPLETE", "SPLIT": + if !retryAt.IsZero() { + return "", errors.New("finished official shard cannot be deferred") + } + default: + return "", errors.New("unsupported official discovery completion coverage") + } + return c.completeDiscovery(ctx, taskID, lease, checkpoint, retryAt, deferralReason, deferredStatus) +} + +// HeartbeatWithCapabilities is separate from the legacy empty heartbeat so +// protocol scaffolding cannot accidentally advertise an executable capability. +func (c *Client) HeartbeatWithCapabilities(ctx context.Context, workerID string, capabilities []string) error { + if !boundedOfficialString(workerID, 128) { + return errors.New("invalid worker ID") + } + if err := validateCapabilities(capabilities); err != nil { + return err + } + if capabilities == nil { + capabilities = []string{} + } + status, err := c.post(ctx, "/heartbeat", map[string]any{ + "workerId": workerID, "protocolMajor": 1, "capabilities": capabilities, + }, nil) + if err != nil { + return err + } + if status != http.StatusNoContent { + return errors.New("internal API did not accept worker heartbeat") + } + return nil +} + +func (c *Client) GetOfficialTaskControl(ctx context.Context, taskID string, lease LeaseProof) (OfficialTaskControl, error) { + var control OfficialTaskControl + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-control", map[string]any{"lease": lease}, &control) + if err != nil { + return OfficialTaskControl{}, err + } + if control.ConfigurationVersion < 1 || (control.Action != "CONTINUE" && control.Action != "PAUSE") || + (control.Action == "PAUSE" && !slices.Contains([]string{"DISABLED", "SOURCE_PAUSED", "EXCLUDED", "CANDIDATE_CAPACITY", "REVIEW_BACKPRESSURE", "QUOTA_COOLDOWN"}, control.Reason)) { + return OfficialTaskControl{}, errors.New("internal API returned invalid official task control") + } + if control.Reason == "QUOTA_COOLDOWN" && (control.RetryAt == nil || control.RetryAt.IsZero()) { + return OfficialTaskControl{}, errors.New("quota cooldown requires retry time") + } + return control, nil +} + +// PauseOfficialDiscovery saves only the last acknowledged checkpoint. A nil +// checkpoint is sent as null when no batch has been accepted yet. +func (c *Client) PauseOfficialDiscovery(ctx context.Context, taskID string, lease LeaseProof, checkpoint *string) error { + if checkpoint != nil && (len(*checkpoint) > 4096 || !utf8.ValidString(*checkpoint)) { + return errors.New("invalid official discovery pause checkpoint") + } + return c.pauseOfficial(ctx, taskID, map[string]any{ + "lease": lease, "kind": "DISCOVERY", "discoveryCheckpoint": checkpoint, + }) +} + +// PauseOfficialRepository also supports pausing before a commit is known. The +// server atomically checks the latest ownership, accepted paths and fixed commit. +func (c *Client) PauseOfficialRepository(ctx context.Context, taskID string, lease LeaseProof, progress OfficialRepositoryProgress) error { + if progress.ObservedCommitSHA == nil { + if progress.PathCheckpoint != nil || len(progress.IncompleteSubtrees) != 0 || len(progress.PathResults) != 0 { + return errors.New("official repository pause has progress without a commit") + } + } else if !validOfficialCommit(*progress.ObservedCommitSHA) { + return errors.New("invalid official repository pause commit") + } + if (progress.PathCheckpoint != nil && (!utf8.ValidString(*progress.PathCheckpoint) || utf8.RuneCountInString(*progress.PathCheckpoint) > 1024)) || + len(progress.IncompleteSubtrees) > 1000 || len(progress.PathResults) > 10000 { + return errors.New("official repository pause exceeds progress limits") + } + for _, path := range progress.IncompleteSubtrees { + if !boundedOfficialString(path, 1024) { + return errors.New("invalid official incomplete subtree") + } + } + seen := make(map[string]bool, len(progress.PathResults)) + for _, result := range progress.PathResults { + if !boundedOfficialString(result.SkillPath, 1024) || seen[result.SkillPath] || + !utf8.ValidString(result.Reason) || utf8.RuneCountInString(result.Reason) > 500 || + !slices.Contains([]string{"ACCEPTED", "IGNORED", "RETRYABLE_FAILURE", "TERMINAL_FAILURE"}, result.Outcome) || + (result.Outcome == "ACCEPTED") != (result.IngestionID != "") { + return errors.New("invalid official repository pause path result") + } + seen[result.SkillPath] = true + } + if progress.IncompleteSubtrees == nil { + progress.IncompleteSubtrees = []string{} + } + if progress.PathResults == nil { + progress.PathResults = []RepositoryPathResult{} + } + return c.pauseOfficial(ctx, taskID, map[string]any{ + "lease": lease, "kind": "REPOSITORY_SYNC", "repositoryProgress": progress, + }) +} + +func validOfficialCommit(commit string) bool { + if len(commit) != 40 { + return false + } + for _, digit := range commit { + if (digit < '0' || digit > '9') && (digit < 'a' || digit > 'f') { + return false + } + } + return true +} + +// On APIError.Code CONTROL_CHANGED, callers must re-read control and retain +// their lease. This endpoint-specific code does not extend legacy enums. +func (c *Client) pauseOfficial(ctx context.Context, taskID string, request any) error { + var response struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-pause", request, &response) + if err != nil { + return err + } + if response.Status != "PAUSED" { + return errors.New("internal API did not pause official task") + } + return nil +} + +// RecoverOfficialCheckpoint returns the replacement task ID, not a new lease. +// The server has retired the old task; the caller must stop writing to it. +func (c *Client) RecoverOfficialCheckpoint(ctx context.Context, taskID string, recovery OfficialCheckpointRecovery) (string, error) { + if !slices.Contains([]string{"MALFORMED", "FORMAT_UNSUPPORTED", "IDENTITY_MISMATCH"}, recovery.Reason) || + len(recovery.RejectedCheckpoint) > 4096 || !utf8.ValidString(recovery.RejectedCheckpoint) { + return "", errors.New("invalid official checkpoint recovery") + } + var response struct { + Status string `json:"status"` + ReplacementTaskID string `json:"replacementTaskId"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-checkpoint-recovery", recovery, &response) + if err != nil { + return "", err + } + if response.Status != "REPLAY_SCHEDULED" || response.ReplacementTaskID == "" { + return "", errors.New("internal API did not schedule checkpoint replay") + } + return response.ReplacementTaskID, nil +} + +// ReportOfficialCredentialStatus carries the optional observation while leaving +// the legacy credential-status enum and payload unchanged for existing callers. +func (c *Client) ReportOfficialCredentialStatus(ctx context.Context, taskID string, lease LeaseProof, + credentialID, status string, retryAt time.Time, quota *GithubQuota, observation OfficialQuotaObservation) error { + if !slices.Contains([]string{"OBSERVED", "AUTH_INVALID", "RATE_LIMITED"}, status) || + (status == "RATE_LIMITED") != !retryAt.IsZero() || credentialID == "" || + (status == "OBSERVED" && quota == nil) || + (observation.State == "UNKNOWN" && quota != nil) || + !slices.Contains([]string{"CORE", "SEARCH", "GRAPHQL", "OTHER"}, observation.Resource) || observation.ObservedAt.IsZero() || + !slices.Contains([]string{"OBSERVED", "UNKNOWN", "PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN", "AUTH_INVALID"}, observation.State) || + (observation.Remaining != nil && *observation.Remaining < 0) || + (observation.ResetAt != nil && observation.ResetAt.IsZero()) || + (observation.RetryAfterSeconds != nil && *observation.RetryAfterSeconds < 0) || + (observation.State == "UNKNOWN" && observation.Remaining != nil) || + ((observation.State == "OBSERVED" || observation.State == "PRIMARY_EXHAUSTED") && (observation.Remaining == nil || observation.ResetAt == nil)) { + return errors.New("invalid official credential observation") + } + request := map[string]any{ + "lease": lease, "credentialId": credentialID, "status": status, "officialQuotaObservation": observation, + } + if quota != nil { + request["githubQuota"] = quota + } + if !retryAt.IsZero() { + request["retryAt"] = retryAt + } + responseStatus, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/credential-status", request, nil) + if err != nil { + return err + } + if responseStatus != http.StatusNoContent { + return errors.New("internal API did not accept official credential observation") + } + return nil +} + +// ReportOfficialQuota lets Java derive the quota subject from the lease, including +// anonymous requests. It never accepts a caller-selected credential or subject. +func (c *Client) ReportOfficialQuota(ctx context.Context, taskID string, lease LeaseProof, observation OfficialQuotaObservation) error { + now := time.Now() + if !slices.Contains([]string{"CORE", "SEARCH", "GRAPHQL", "OTHER"}, observation.Resource) || observation.ObservedAt.IsZero() || + observation.ObservedAt.Before(now.Add(-5*time.Minute)) || observation.ObservedAt.After(now.Add(5*time.Minute)) || + !slices.Contains([]string{"OBSERVED", "UNKNOWN", "PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN", "AUTH_INVALID"}, observation.State) || + (observation.Remaining != nil && *observation.Remaining < 0) || + (observation.ResetAt != nil && (observation.ResetAt.IsZero() || observation.ResetAt.After(now.Add(24*time.Hour)))) || + (observation.RetryAfterSeconds != nil && (*observation.RetryAfterSeconds < 0 || *observation.RetryAfterSeconds > 86400)) || + (observation.State == "UNKNOWN" && (observation.Remaining != nil || observation.ResetAt != nil)) || + (observation.State == "PRIMARY_EXHAUSTED" && (observation.Remaining == nil || *observation.Remaining != 0 || observation.ResetAt == nil || !observation.ResetAt.After(now))) || + ((observation.State == "OBSERVED" || observation.State == "PRIMARY_EXHAUSTED") && (observation.Remaining == nil || observation.ResetAt == nil)) { + return errors.New("invalid official quota observation") + } + status, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/official-quota", map[string]any{"lease": lease, "observation": observation}, nil) + if err != nil { + return err + } + if status != http.StatusNoContent { + return errors.New("internal API did not accept official quota observation") + } + return nil +} diff --git a/github-sync-go/internal/controlplane/platform.go b/github-sync-go/internal/controlplane/platform.go new file mode 100644 index 000000000..a7309cf50 --- /dev/null +++ b/github-sync-go/internal/controlplane/platform.go @@ -0,0 +1,48 @@ +package controlplane + +import ( + "context" + "errors" + "net/url" +) + +const PlatformPauseCapability = "platform-pause/v1" + +type PlatformControl struct { + Action string `json:"action"` + PlatformVersion int64 `json:"platformVersion"` +} +type PlatformPause struct { + Lease LeaseProof `json:"lease"` + ObservedVersion int64 `json:"observedVersion"` + Kind string `json:"kind"` + DiscoveryCheckpoint *string `json:"discoveryCheckpoint,omitempty"` + RepositoryProgress *OfficialRepositoryProgress `json:"repositoryProgress,omitempty"` +} + +func (c *Client) GetPlatformControl(ctx context.Context, task string, lease LeaseProof) (PlatformControl, error) { + var result PlatformControl + _, err := c.post(ctx, "/tasks/"+url.PathEscape(task)+"/platform-control", struct { + Lease LeaseProof `json:"lease"` + }{lease}, &result) + if err == nil && (result.PlatformVersion < 1 || result.Action != "CONTINUE" && result.Action != "PAUSE") { + err = errors.New("invalid platform control") + } + return result, err +} +func (c *Client) PausePlatform(ctx context.Context, task string, request PlatformPause) error { + var result struct { + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(task)+"/platform-pause", request, &result) + if err == nil && result.Status != "PAUSED" { + err = errors.New("platform pause not acknowledged") + } + return err +} + +type checkpointObserverKey struct{} + +func WithCheckpointObserver(ctx context.Context, observer func(string)) context.Context { + return context.WithValue(ctx, checkpointObserverKey{}, observer) +} diff --git a/github-sync-go/internal/controlplane/platform_test.go b/github-sync-go/internal/controlplane/platform_test.go new file mode 100644 index 000000000..c9788d57a --- /dev/null +++ b/github-sync-go/internal/controlplane/platform_test.go @@ -0,0 +1,21 @@ +package controlplane + +import "testing" + +func TestPlatformCapabilityIsValidForLegacyDiscoveryAndRequiresControlVersion(t *testing.T) { + version := int64(1) + for _, entry := range []string{"CURATED_REPOSITORY", "SEARCH_RULE", "SEARCH_PREVIEW", "CURATED_OWNER"} { + task := ClaimedTask{Kind: "DISCOVERY", Discovery: &DiscoverySnapshot{EntryType: entry}, RequiredCapabilities: []string{PlatformPauseCapability}, PlatformControlVersion: &version} + if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err != nil { + t.Fatalf("%s: %v", entry, err) + } + task.PlatformControlVersion = nil + if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err == nil { + t.Fatal("accepted missing platform version") + } + } + task := ClaimedTask{Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version} + if err := validateOfficialTask(task, []string{PlatformPauseCapability}); err == nil { + t.Fatal("accepted undeclared platform capability") + } +} diff --git a/github-sync-go/internal/controlplane/testdata/discoveryinterop/main.go b/github-sync-go/internal/controlplane/testdata/discoveryinterop/main.go new file mode 100644 index 000000000..ea4a459f6 --- /dev/null +++ b/github-sync-go/internal/controlplane/testdata/discoveryinterop/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +const checkpoint = "interop-discovery-checkpoint" + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + phase := os.Getenv("SKILLHUB_INTEROP_DISCOVERY_PHASE") + if phase != "report" && phase != "resume" { + return errors.New("invalid discovery interoperability phase") + } + client, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + task, err := client.ClaimWithCapabilities(ctx, "interop-discovery-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + return err + } + if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Discovery == nil || task.Discovery.EntryType != "CURATED_OWNER" || + task.Discovery.Owner != "interop-owner" { + return errors.New("Java did not lease the expected discovery task") + } + if expected := os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID"); expected != "" && + (task.Credential == nil || task.Credential.CredentialID != expected || + task.Credential.Token != os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN")) { + return errors.New("Java did not lease the expected fixture credential") + } + if phase == "report" && (task.Lease.Generation != 1 || task.Discovery.Checkpoint != "") || + phase == "resume" && (task.Lease.Generation != 2 || task.Discovery.Checkpoint != checkpoint) { + return errors.New("Java did not preserve the expected discovery lease and checkpoint") + } + batch := controlplane.DiscoveryBatch{ + Checkpoint: checkpoint, + Repositories: []controlplane.RepositoryCandidate{{ + RepositoryID: 424242, Owner: "interop-owner", Name: "public-skills", + DefaultBranch: "main", + }}, + } + if err := client.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch); err != nil { + return err + } + if phase == "report" { + fmt.Println("REPORTED") + return nil + } + status, err := client.CompleteDiscovery(ctx, task.TaskID, task.Lease, + checkpoint, time.Time{}, "") + if err != nil || status != "ACQUISITION_COMPLETE" { + return errors.New("reclaimed discovery lease did not complete the saved checkpoint") + } + fmt.Println("COMPLETED") + return nil +} diff --git a/github-sync-go/internal/controlplane/testdata/httpinterop/main.go b/github-sync-go/internal/controlplane/testdata/httpinterop/main.go new file mode 100644 index 000000000..4efbb5340 --- /dev/null +++ b/github-sync-go/internal/controlplane/testdata/httpinterop/main.go @@ -0,0 +1,301 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + javaURL := os.Getenv("SKILLHUB_INTEROP_JAVA_URL") + serviceToken := os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN") + taskID := os.Getenv("SKILLHUB_INTEROP_TASK_ID") + commitSHA := os.Getenv("SKILLHUB_INTEROP_COMMIT_SHA") + skillPath := os.Getenv("SKILLHUB_INTEROP_SKILL_PATH") + archivePath := os.Getenv("SKILLHUB_INTEROP_ARCHIVE_PATH") + if taskID == "" || commitSHA == "" || skillPath == "" || archivePath == "" { + return errors.New("missing HTTP interoperability fixture") + } + client, err := controlplane.NewClient(javaURL, serviceToken) + if err != nil { + return err + } + archive, err := os.Open(archivePath) + if err != nil { + return err + } + defer archive.Close() + info, err := archive.Stat() + if err != nil || !info.Mode().IsRegular() || info.Size() < 1 { + return errors.New("invalid interoperability archive") + } + digest := sha256.New() + if _, err := io.Copy(digest, archive); err != nil { + return err + } + digestHex := hex.EncodeToString(digest.Sum(nil)) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + task, err := client.ClaimWithCapabilities(ctx, "interop-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + return err + } + if task == nil || task.TaskID != taskID || task.Mode != "FULL" || + task.Repository == nil || task.Repository.FixedCommitSHA != commitSHA { + if task == nil { + return errors.New("Java did not lease a repository task") + } + return fmt.Errorf("Java leased task %s in mode %s, expected task %s in FULL mode", + task.TaskID, task.Mode, taskID) + } + mode := os.Getenv("SKILLHUB_INTEROP_MODE") + if mode == "resume-only" { + expectedIngestionID := os.Getenv("SKILLHUB_INTEROP_EXPECT_INGESTION_ID") + if expectedIngestionID == "" || len(task.Repository.AcceptedPaths) != 1 || + task.Repository.AcceptedPaths[0] != skillPath || task.Lease.Generation < 2 { + return errors.New("reclaimed lease did not return the persisted package path") + } + status, err := client.CompleteRepository(ctx, taskID, task.Lease, + controlplane.RepositoryCompletion{CommitSHA: commitSHA, + PathResults: []controlplane.RepositoryPathResult{{ + SkillPath: skillPath, Outcome: "ACCEPTED", IngestionID: expectedIngestionID, + }}}) + if err != nil || status != "ACQUISITION_COMPLETE" { + return errors.New("reclaimed lease could not complete the persisted package") + } + fmt.Println(expectedIngestionID) + return nil + } + if mode != "" && mode != "report-only" && mode != "wait-for-java-outage" { + return errors.New("invalid HTTP interoperability mode") + } + admission, err := client.AdmitPackage(ctx, taskID, task.Lease, commitSHA, skillPath) + if err != nil { + return err + } + authorization, err := client.AuthorizeUpload(ctx, taskID, task.Lease, + admission.AdmissionID, commitSHA, skillPath, info.Size(), digestHex) + if err != nil { + return err + } + expectedMode := os.Getenv("SKILLHUB_INTEROP_UPLOAD_MODE") + if expectedMode == "" { + expectedMode = "LOCAL_STREAM" + } + if authorization.Mode != expectedMode { + return errors.New("Java returned an unexpected staging upload mode") + } + var allowedS3Hosts []string + if expectedMode == "S3_PRESIGNED_PUT" { + allowedHost := os.Getenv("SKILLHUB_INTEROP_ALLOWED_S3_HOST") + if allowedHost == "" { + return errors.New("missing allowed S3 fixture host") + } + allowedS3Hosts = []string{allowedHost} + if err := verifySignedUploadCannotWriteAnotherKey(ctx, authorization); err != nil { + return err + } + } + if err := client.UploadPackage(ctx, authorization, archivePath, allowedS3Hosts); err != nil { + return err + } + ingestionID, err := client.ReportPackage(ctx, taskID, task.Lease, + admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), digestHex) + if err != nil { + return err + } + replayedID, err := client.ReportPackage(ctx, taskID, task.Lease, + admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), digestHex) + if err != nil || replayedID != ingestionID { + return errors.New("Java did not replay the same package receipt") + } + forgedDigest := digestHex[:63] + "0" + if digestHex[63] == '0' { + forgedDigest = digestHex[:63] + "1" + } + _, err = client.ReportPackage(ctx, taskID, task.Lease, + admission.AdmissionID, commitSHA, skillPath, authorization, info.Size(), forgedDigest) + var apiError *controlplane.APIError + if !errors.As(err, &apiError) || apiError.Code != "RESULT_CONFLICT" { + return errors.New("Java accepted a conflicting package report") + } + if mode == "report-only" { + fmt.Println(ingestionID) + return nil + } + if mode == "wait-for-java-outage" { + if err := awaitJavaOutage(ctx, client, task, ingestionID); err != nil { + return err + } + fmt.Println(ingestionID) + return nil + } + status, err := client.CompleteRepository(ctx, taskID, task.Lease, + controlplane.RepositoryCompletion{ + CommitSHA: commitSHA, + PathResults: []controlplane.RepositoryPathResult{{ + SkillPath: skillPath, Outcome: "ACCEPTED", IngestionID: ingestionID, + }}, + }) + if err != nil { + return err + } + if status != "ACQUISITION_COMPLETE" { + return errors.New("Java did not complete the repository acquisition") + } + if oversizedTaskID := os.Getenv("SKILLHUB_INTEROP_OVERSIZED_TASK_ID"); oversizedTaskID != "" { + if err := verifyOversizedS3Object(ctx, client, oversizedTaskID, commitSHA, + skillPath, info.Size(), digestHex, allowedS3Hosts); err != nil { + return err + } + } + fmt.Println(ingestionID) + return nil +} + +func awaitJavaOutage(ctx context.Context, client *controlplane.Client, + task *controlplane.ClaimedTask, ingestionID string) error { + readyPath := os.Getenv("SKILLHUB_INTEROP_READY_MARKER") + outagePath := os.Getenv("SKILLHUB_INTEROP_OUTAGE_MARKER") + if readyPath == "" || outagePath == "" { + return errors.New("missing Java outage fixture markers") + } + if err := os.WriteFile(readyPath, []byte(ingestionID), 0600); err != nil { + return err + } + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + if _, err := os.Stat(outagePath); err == nil { + break + } else if !os.IsNotExist(err) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + renewContext, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _, err := client.Renew(renewContext, task.TaskID, task.Lease) + if err == nil { + return errors.New("Java accepted renewal while the service was stopped") + } + if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + return errors.New("Java outage check expired before observing a connection failure") + } + var transportError *url.Error + if !errors.As(err, &transportError) { + return errors.New("Java outage check did not observe a transport failure") + } + return nil +} + +func verifySignedUploadCannotWriteAnotherKey(ctx context.Context, + authorization controlplane.UploadAuthorization) error { + target, err := url.Parse(authorization.UploadURL) + if err != nil || target.Scheme == "" || target.Host == "" { + return errors.New("invalid signed upload target") + } + target.Path += ".forbidden" + target.RawPath = "" + request, err := http.NewRequestWithContext(ctx, http.MethodPut, + target.String(), bytes.NewReader([]byte("forbidden"))) + if err != nil { + return errors.New("could not create changed-key upload probe") + } + for name, value := range authorization.Headers { + request.Header.Set(name, value) + } + probeClient := &http.Client{Transport: &http.Transport{Proxy: nil}, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := probeClient.Do(request) + if err != nil { + return errors.New("changed-key upload probe did not reach object storage") + } + response.Body.Close() + if response.StatusCode != http.StatusForbidden { + return fmt.Errorf("changed-key upload unexpectedly returned HTTP %d", response.StatusCode) + } + return nil +} + +// This deliberately bypasses UploadPackage's client-side size guard to prove +// Java rejects and removes an oversized object accepted by a signed PUT URL. +func verifyOversizedS3Object(ctx context.Context, client *controlplane.Client, + taskID, commitSHA, skillPath string, size int64, digest string, + allowedHosts []string) error { + if len(allowedHosts) != 1 { + return errors.New("oversized object fixture requires S3 staging") + } + task, err := client.ClaimWithCapabilities(ctx, "interop-oversize-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + return err + } + if task == nil || task.TaskID != taskID { + if task == nil { + return errors.New("Java had no repository task for the oversized object fixture") + } + return fmt.Errorf("Java leased repository task %s instead of oversized fixture task %s", + task.TaskID, taskID) + } + admission, err := client.AdmitPackage(ctx, taskID, task.Lease, commitSHA, skillPath) + if err != nil { + return err + } + authorization, err := client.AuthorizeUpload(ctx, taskID, task.Lease, + admission.AdmissionID, commitSHA, skillPath, size, digest) + if err != nil { + return err + } + if authorization.Mode != "S3_PRESIGNED_PUT" || authorization.MaxBytes < size || + authorization.MaxBytes > 1<<20 { + return errors.New("invalid oversized object fixture authorization") + } + maliciousBody := bytes.NewReader(make([]byte, authorization.MaxBytes+1)) + request, err := http.NewRequestWithContext(ctx, http.MethodPut, authorization.UploadURL, maliciousBody) + if err != nil { + return err + } + for name, value := range authorization.Headers { + request.Header.Set(name, value) + } + uploadClient := &http.Client{Transport: &http.Transport{Proxy: nil}, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := uploadClient.Do(request) + if err != nil { + return err + } + response.Body.Close() + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent { + return errors.New("S3 did not accept the intentionally oversized object") + } + _, err = client.ReportPackage(ctx, taskID, task.Lease, admission.AdmissionID, + commitSHA, skillPath, authorization, size, digest) + var apiError *controlplane.APIError + if !errors.As(err, &apiError) || apiError.Status != http.StatusRequestEntityTooLarge || + apiError.Code != "PACKAGE_TOO_LARGE" { + return errors.New("Java did not reject the oversized S3 object") + } + return nil +} diff --git a/github-sync-go/internal/controlplane/upload.go b/github-sync-go/internal/controlplane/upload.go new file mode 100644 index 000000000..6d83288b3 --- /dev/null +++ b/github-sync-go/internal/controlplane/upload.go @@ -0,0 +1,168 @@ +package controlplane + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type UploadAuthorization struct { + UploadID string `json:"uploadId"` + Method string `json:"method"` + Mode string `json:"mode"` + UploadURL string `json:"uploadUrl"` + UploadTicket string `json:"uploadTicket,omitempty"` + ExpiresAt time.Time `json:"expiresAt"` + MaxBytes int64 `json:"maxBytes"` + Headers map[string]string `json:"headers,omitempty"` +} + +func (c *Client) AuthorizeUpload(ctx context.Context, taskID string, lease LeaseProof, + admissionID, commitSHA, skillPath string, sizeBytes int64, digest string) (UploadAuthorization, error) { + if taskID == "" || admissionID == "" || sizeBytes < 1 || len(digest) != 64 { + return UploadAuthorization{}, errors.New("invalid package upload request") + } + var authorization UploadAuthorization + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/upload-authorizations", map[string]any{ + "lease": lease, "admissionId": admissionID, "commitSha": commitSHA, + "skillPath": skillPath, "sizeBytes": sizeBytes, "sha256": digest, + }, &authorization) + if err != nil { + return UploadAuthorization{}, err + } + if authorization.UploadID == "" || authorization.Method != http.MethodPut || + (authorization.Mode != "LOCAL_STREAM" && authorization.Mode != "S3_PRESIGNED_PUT") || + authorization.UploadURL == "" || authorization.ExpiresAt.IsZero() || + authorization.MaxBytes < sizeBytes || authorization.ExpiresAt.Before(time.Now()) { + return UploadAuthorization{}, errors.New("internal API returned an invalid upload authorization") + } + return authorization, nil +} + +func (c *Client) ReportPackage(ctx context.Context, taskID string, lease LeaseProof, + admissionID, commitSHA, skillPath string, authorization UploadAuthorization, + sizeBytes int64, digest string) (string, error) { + if taskID == "" || admissionID == "" || authorization.UploadID == "" || + sizeBytes < 1 || len(digest) != 64 { + return "", errors.New("invalid package report") + } + var response struct { + IngestionID string `json:"ingestionId"` + Status string `json:"status"` + } + _, err := c.post(ctx, "/tasks/"+url.PathEscape(taskID)+"/packages/report", map[string]any{ + "lease": lease, "admissionId": admissionID, "commitSha": commitSHA, + "skillPath": skillPath, "uploadId": authorization.UploadID, + "sizeBytes": sizeBytes, "sha256": digest, + }, &response) + if err != nil { + return "", err + } + if response.IngestionID == "" || response.Status != "RECEIVED" { + return "", errors.New("internal API returned an invalid package receipt") + } + return response.IngestionID, nil +} + +// UploadPackage never sends the Java service token to an object-storage URL. +// S3 hosts must be explicitly configured by the deployment, including their port. +func (c *Client) UploadPackage(ctx context.Context, authorization UploadAuthorization, + archivePath string, allowedS3Hosts []string) error { + archive, err := os.Open(archivePath) + if err != nil { + return err + } + defer archive.Close() + info, err := archive.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() < 1 || info.Size() > authorization.MaxBytes { + return errors.New("archive does not match upload size policy") + } + parsed, err := url.Parse(authorization.UploadURL) + if err != nil || parsed.User != nil || parsed.Fragment != "" { + return errors.New("invalid upload URL") + } + var target string + var client *http.Client + local := authorization.Mode == "LOCAL_STREAM" + if local { + expected := "/internal/github-sync/v1/uploads/" + authorization.UploadID + "/content" + if parsed.IsAbs() || parsed.Host != "" || parsed.Path != expected || parsed.RawQuery != "" || + authorization.UploadTicket == "" { + return errors.New("invalid local upload target") + } + base, err := url.Parse(c.baseURL) + if err != nil { + return err + } + target = base.Scheme + "://" + base.Host + expected + client = directUploadClient() + } else { + if authorization.Mode != "S3_PRESIGNED_PUT" || (parsed.Scheme != "https" && parsed.Scheme != "http") || + parsed.Host == "" || len(allowedS3Hosts) == 0 { + return errors.New("invalid object-storage upload target") + } + allowed := false + for _, host := range allowedS3Hosts { + allowed = allowed || strings.EqualFold(parsed.Host, host) + } + if !allowed { + return errors.New("object-storage host is not allowed") + } + target = authorization.UploadURL + client = directUploadClient() + } + request, err := http.NewRequestWithContext(ctx, http.MethodPut, target, archive) + if err != nil { + return errors.New("cannot prepare package upload request") + } + request.ContentLength = info.Size() + for name, value := range authorization.Headers { + if strings.EqualFold(name, "authorization") || strings.EqualFold(name, "host") || + strings.EqualFold(name, "x-upload-ticket") { + return errors.New("upload authorization contains a forbidden header") + } + request.Header.Set(name, value) + } + if local { + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("X-Upload-Ticket", authorization.UploadTicket) + request.Header.Set("Content-Type", "application/octet-stream") + requestID := make([]byte, 16) + if _, err := rand.Read(requestID); err != nil { + return err + } + request.Header.Set("X-Request-Id", hex.EncodeToString(requestID)) + } + response, err := client.Do(request) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("package upload transport failed") + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + if response.StatusCode != http.StatusNoContent && response.StatusCode != http.StatusOK { + return fmt.Errorf("package upload returned HTTP %d", response.StatusCode) + } + return nil +} + +func directUploadClient() *http.Client { + return &http.Client{ + Timeout: 10 * time.Minute, + Transport: &http.Transport{Proxy: nil, MaxIdleConnsPerHost: 2}, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } +} diff --git a/github-sync-go/internal/controlplane/upload_test.go b/github-sync-go/internal/controlplane/upload_test.go new file mode 100644 index 000000000..a9f23029c --- /dev/null +++ b/github-sync-go/internal/controlplane/upload_test.go @@ -0,0 +1,60 @@ +package controlplane + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" +) + +func TestStagingUploadUsesDirectTransport(t *testing.T) { + transport, ok := directUploadClient().Transport.(*http.Transport) + if !ok || transport.Proxy != nil { + t.Fatal("staging upload must not use the GitHub proxy") + } +} + +func TestS3UploadUsesOnlySignedHeaders(t *testing.T) { + const contents = "Skill package fixture" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPut || request.URL.Path != "/staging/package.zip" { + t.Errorf("unexpected upload target: %s %s", request.Method, request.URL.Path) + } + if request.Header.Get("Authorization") != "" || request.Header.Get("X-Upload-Ticket") != "" { + t.Error("service credentials reached object storage") + } + if request.Header.Get("Content-Type") != "application/zip" { + t.Errorf("signed content type was not forwarded: %q", request.Header.Get("Content-Type")) + } + body, err := io.ReadAll(request.Body) + if err != nil || string(body) != contents { + t.Errorf("unexpected uploaded package: %q, %v", body, err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + archive := filepath.Join(t.TempDir(), "package.zip") + if err := os.WriteFile(archive, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + client, err := NewClient(server.URL+"/internal/github-sync/v1", "a-32-character-internal-service-token") + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + authorization := UploadAuthorization{ + Mode: "S3_PRESIGNED_PUT", UploadURL: server.URL + "/staging/package.zip", + MaxBytes: int64(len(contents)), Headers: map[string]string{"Content-Type": "application/zip"}, + } + if err := client.UploadPackage(context.Background(), authorization, archive, []string{parsed.Host}); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/github/client.go b/github-sync-go/internal/github/client.go new file mode 100644 index 000000000..6e60e7ed9 --- /dev/null +++ b/github-sync-go/internal/github/client.go @@ -0,0 +1,319 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + apiBaseURL = "https://api.github.com" + apiVersion = "2026-03-10" + maxResponseBytes = 9 << 20 +) + +var ErrResponseTooLarge = errors.New("GitHub API response exceeds size limit") + +var ( + ownerPattern = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$`) + repoPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,100}$`) + shaPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +type Client struct { + http *http.Client + token string + quotaObserver func(context.Context, Quota) + beforeRequest func(context.Context) error + afterRequest func(context.Context, Quota, error) error +} + +func (c *Client) SetQuotaObserver(observer func(context.Context, Quota)) { + c.quotaObserver = observer +} + +// SetRequestPolicy installs fail-closed hooks for official acquisition only. +func (c *Client) SetRequestPolicy(before func(context.Context) error, after func(context.Context, Quota, error) error) { + c.beforeRequest, c.afterRequest = before, after +} + +func NewClient(token string) *Client { + return &Client{ + http: &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 4, + ResponseHeaderTimeout: 20 * time.Second, + }, + CheckRedirect: func(request *http.Request, previous []*http.Request) error { + if len(previous) >= 4 || request.URL.Scheme != "https" || + request.URL.Host != "api.github.com" || request.URL.User != nil || + request.URL.Fragment != "" { + return errors.New("unsafe GitHub API redirect") + } + return nil + }, + }, + token: token, + } +} + +type Repository struct { + ID int64 `json:"id"` + Name string `json:"name"` + Private bool `json:"private"` + DefaultBranch string `json:"default_branch"` + Owner struct { + Login string `json:"login"` + ID int64 `json:"id"` + } `json:"owner"` +} + +type Reference struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +type Commit struct { + SHA string `json:"sha"` + Tree struct { + SHA string `json:"sha"` + } `json:"tree"` +} + +type TreeEntry struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + Size int64 `json:"size"` + SHA string `json:"sha"` +} + +type Tree struct { + SHA string `json:"sha"` + Truncated bool `json:"truncated"` + Entries []TreeEntry `json:"tree"` +} + +type Quota struct { + Resource string + Remaining int + ResetAt time.Time +} + +type APIError struct { + Status int + Code string + RetryAt time.Time + Quota Quota + SecondaryRateLimit bool +} + +func (e *APIError) Error() string { + return fmt.Sprintf("GitHub API returned HTTP %d (%s)", e.Status, e.Code) +} + +func (c *Client) GetRepository(ctx context.Context, owner, name string) (Repository, Quota, error) { + if err := validateRepository(owner, name); err != nil { + return Repository{}, Quota{}, err + } + var repository Repository + quota, err := c.get(ctx, "/repos/"+owner+"/"+name, &repository) + if err != nil { + return Repository{}, quota, err + } + if repository.Private || repository.ID <= 0 || repository.Owner.Login == "" || repository.Name == "" { + return Repository{}, quota, errors.New("repository is private or has incomplete metadata") + } + return repository, quota, nil +} + +func (c *Client) GetHead(ctx context.Context, repository Repository) (string, Quota, error) { + if err := validateRepository(repository.Owner.Login, repository.Name); err != nil { + return "", Quota{}, err + } + if repository.DefaultBranch == "" { + return "", Quota{}, errors.New("repository has no default branch") + } + // Branch names may contain '/', so escape each segment while retaining the separator. + segments := strings.Split(repository.DefaultBranch, "/") + for i, segment := range segments { + if segment == "" || segment == "." || segment == ".." { + return "", Quota{}, errors.New("invalid default branch") + } + segments[i] = url.PathEscape(segment) + } + var reference Reference + quota, err := c.get(ctx, "/repos/"+repository.Owner.Login+"/"+repository.Name+ + "/git/ref/heads/"+strings.Join(segments, "/"), &reference) + if err != nil { + return "", quota, err + } + if reference.Object.Type != "commit" || !shaPattern.MatchString(reference.Object.SHA) { + return "", quota, errors.New("default branch did not resolve to a commit") + } + return reference.Object.SHA, quota, nil +} + +func (c *Client) GetCommitTreeSHA(ctx context.Context, owner, name, commitSHA string) (string, Quota, error) { + if err := validateRepository(owner, name); err != nil { + return "", Quota{}, err + } + if !shaPattern.MatchString(commitSHA) { + return "", Quota{}, errors.New("invalid commit SHA") + } + var commit Commit + quota, err := c.get(ctx, "/repos/"+owner+"/"+name+"/git/commits/"+commitSHA, &commit) + if err != nil { + return "", quota, err + } + if commit.SHA != commitSHA || !shaPattern.MatchString(commit.Tree.SHA) { + return "", quota, errors.New("GitHub commit does not match the fixed revision") + } + return commit.Tree.SHA, quota, nil +} + +func (c *Client) GetTree(ctx context.Context, owner, name, treeSHA string, recursive bool) (Tree, Quota, error) { + if err := validateRepository(owner, name); err != nil { + return Tree{}, Quota{}, err + } + if !shaPattern.MatchString(treeSHA) { + return Tree{}, Quota{}, errors.New("invalid tree SHA") + } + path := "/repos/" + owner + "/" + name + "/git/trees/" + treeSHA + if recursive { + path += "?recursive=1" + } + var tree Tree + quota, err := c.get(ctx, path, &tree) + if err != nil { + return Tree{}, quota, err + } + if tree.SHA != treeSHA { + return Tree{}, quota, errors.New("GitHub tree does not match the fixed revision") + } + return tree, quota, nil +} + +func validateRepository(owner, name string) error { + if !ownerPattern.MatchString(owner) || !repoPattern.MatchString(name) || name == "." || name == ".." { + return errors.New("invalid GitHub repository coordinate") + } + return nil +} + +// PublicCloneURL never embeds a credential in the remote URL. +func PublicCloneURL(owner, name string) (string, error) { + if err := validateRepository(owner, name); err != nil { + return "", err + } + return "https://github.com/" + owner + "/" + name + ".git", nil +} + +func (c *Client) get(ctx context.Context, path string, target any) (quota Quota, resultErr error) { + quota.Remaining = -1 + if err := CheckAcquisitionControl(ctx); err != nil { + return quota, err + } + if c.beforeRequest != nil { + if err := c.beforeRequest(ctx); err != nil { + return quota, err + } + } + if c.afterRequest != nil { + defer func() { + if err := c.afterRequest(ctx, quota, resultErr); err != nil { + resultErr = err + } + }() + } + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, apiBaseURL+path, nil) + if err != nil { + return Quota{}, err + } + request.Header.Set("Accept", "application/vnd.github+json") + request.Header.Set("X-GitHub-Api-Version", apiVersion) + request.Header.Set("User-Agent", "SkillHub-GitHub-Sync") + if c.token != "" { + request.Header.Set("Authorization", "Bearer "+c.token) + } + response, err := c.http.Do(request) + if err != nil { + return quota, err + } + defer response.Body.Close() + quota = parseQuota(response.Header) + if c.quotaObserver != nil && quota.Remaining >= 0 && !quota.ResetAt.IsZero() { + c.quotaObserver(ctx, quota) + } + if response.StatusCode != http.StatusOK { + return quota, classifyError(response, quota) + } + contents, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1)) + if err != nil { + return quota, err + } + if len(contents) > maxResponseBytes { + return quota, ErrResponseTooLarge + } + if err := json.Unmarshal(contents, target); err != nil { + return quota, fmt.Errorf("decode GitHub API response: %w", err) + } + return quota, nil +} + +func parseQuota(header http.Header) Quota { + quota := Quota{Resource: header.Get("X-RateLimit-Resource"), Remaining: -1} + if remaining, err := strconv.Atoi(header.Get("X-RateLimit-Remaining")); err == nil { + quota.Remaining = remaining + } + if reset, err := strconv.ParseInt(header.Get("X-RateLimit-Reset"), 10, 64); err == nil { + quota.ResetAt = time.Unix(reset, 0).UTC() + } + return quota +} + +func classifyError(response *http.Response, quota Quota) error { + status := response.StatusCode + classified := &APIError{Status: status, Quota: quota, Code: "RETRYABLE_FAILURE"} + var payload struct { + Message string `json:"message"` + } + _ = json.NewDecoder(io.LimitReader(response.Body, 8192)).Decode(&payload) + if status == http.StatusUnauthorized { + classified.Code = "AUTH_INVALID" + } else if status == http.StatusNotFound || status == http.StatusForbidden { + classified.Code = "SOURCE_UNREADABLE" + } + if status == http.StatusTooManyRequests || + (status == http.StatusForbidden && (response.Header.Get("X-RateLimit-Remaining") == "0" || + response.Header.Get("Retry-After") != "" || strings.Contains(strings.ToLower(payload.Message), "rate limit"))) { + classified.Code = "GITHUB_RATE_LIMITED" + classified.SecondaryRateLimit = response.Header.Get("Retry-After") != "" || + strings.Contains(strings.ToLower(payload.Message), "secondary rate limit") || quota.Remaining > 0 + classified.RetryAt = rateLimitRetryAt(response.Header, quota) + } + return classified +} + +func rateLimitRetryAt(header http.Header, quota Quota) time.Time { + if seconds, err := strconv.Atoi(header.Get("Retry-After")); err == nil && seconds > 0 { + return time.Now().Add(time.Duration(seconds) * time.Second) + } + if !quota.ResetAt.IsZero() && quota.Remaining == 0 { + return quota.ResetAt + } + return time.Now().Add(time.Minute) +} diff --git a/github-sync-go/internal/github/client_redirect_test.go b/github-sync-go/internal/github/client_redirect_test.go new file mode 100644 index 000000000..67f1cf063 --- /dev/null +++ b/github-sync-go/internal/github/client_redirect_test.go @@ -0,0 +1,62 @@ +package github + +import ( + "context" + "io" + "net/http" + "strings" + "testing" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return fn(request) +} + +func TestGetRepositoryFollowsSameHostRenameRedirect(t *testing.T) { + client := NewClient("fixture-token") + requests := 0 + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests++ + if request.Header.Get("Authorization") != "Bearer fixture-token" { + t.Fatal("GitHub credential missing from same-host request") + } + switch request.URL.Path { + case "/repos/old-owner/old-name": + return &http.Response{StatusCode: http.StatusMovedPermanently, + Header: http.Header{"Location": {"https://api.github.com/repositories/42"}}, + Body: io.NopCloser(strings.NewReader("")), Request: request}, nil + case "/repositories/42": + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"id":42,"name":"new-name","owner":{"login":"new-owner"},"default_branch":"main","private":false}`)), + Request: request}, nil + default: + t.Fatalf("unexpected GitHub path %s", request.URL.Path) + return nil, nil + } + }) + repository, _, err := client.GetRepository(context.Background(), "old-owner", "old-name") + if err != nil || repository.ID != 42 || repository.Owner.Login != "new-owner" || + repository.Name != "new-name" || requests != 2 { + t.Fatalf("unexpected renamed repository result: %+v, requests=%d, error=%v", repository, requests, err) + } +} + +func TestGetRepositoryRejectsCrossHostRedirect(t *testing.T) { + client := NewClient("fixture-token") + requests := 0 + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests++ + if request.URL.Host != "api.github.com" { + t.Fatal("GitHub credential request escaped API host") + } + return &http.Response{StatusCode: http.StatusMovedPermanently, + Header: http.Header{"Location": {"https://example.invalid/repos/new-owner/new-name"}}, + Body: io.NopCloser(strings.NewReader("")), Request: request}, nil + }) + _, _, err := client.GetRepository(context.Background(), "old-owner", "old-name") + if err == nil || requests != 1 { + t.Fatalf("cross-host redirect must fail before a second request: requests=%d, error=%v", requests, err) + } +} diff --git a/github-sync-go/internal/github/client_status_test.go b/github-sync-go/internal/github/client_status_test.go new file mode 100644 index 000000000..d4b140a9c --- /dev/null +++ b/github-sync-go/internal/github/client_status_test.go @@ -0,0 +1,93 @@ +package github + +import ( + "context" + "errors" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" +) + +func TestRepositoryStatusClassificationAndQuota(t *testing.T) { + reset := time.Now().Add(5 * time.Minute).Truncate(time.Second) + tests := []struct { + name string + status int + message string + headers http.Header + code string + wantRetryTime time.Time + }{ + {"invalid credential", http.StatusUnauthorized, `{"message":"Bad credentials"}`, + nil, "AUTH_INVALID", time.Time{}}, + {"private or missing", http.StatusNotFound, `{"message":"Not Found"}`, + nil, "SOURCE_UNREADABLE", time.Time{}}, + {"forbidden without rate limit", http.StatusForbidden, `{"message":"Resource not accessible"}`, + nil, "SOURCE_UNREADABLE", time.Time{}}, + {"primary rate limit", http.StatusForbidden, `{"message":"API rate limit exceeded"}`, + http.Header{"X-Ratelimit-Remaining": {"0"}, + "X-Ratelimit-Reset": {strconv.FormatInt(reset.Unix(), 10)}, + "X-Ratelimit-Resource": {"core"}}, "GITHUB_RATE_LIMITED", reset}, + {"server failure", http.StatusInternalServerError, `{"message":"Server error"}`, + nil, "RETRYABLE_FAILURE", time.Time{}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := NewClient("fixture-token") + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.Header.Get("Authorization") != "Bearer fixture-token" { + t.Fatal("GitHub credential was not sent") + } + return &http.Response{StatusCode: test.status, Header: test.headers, + Body: io.NopCloser(strings.NewReader(test.message)), Request: request}, nil + }) + _, quota, err := client.GetRepository(context.Background(), "public-owner", "skills") + var apiError *APIError + if !errors.As(err, &apiError) || apiError.Code != test.code { + t.Fatalf("status %d: expected %s, got %v", test.status, test.code, err) + } + if !test.wantRetryTime.IsZero() && + (!apiError.RetryAt.Equal(test.wantRetryTime) || quota.Remaining != 0 || + quota.Resource != "core") { + t.Fatalf("rate limit headers were not preserved: error=%+v quota=%+v", apiError, quota) + } + }) + } +} + +func TestSecondaryRateLimitRetryAfter(t *testing.T) { + client := NewClient("fixture-token") + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": {"120"}}, + Body: io.NopCloser(strings.NewReader(`{"message":"You have exceeded a secondary rate limit"}`)), + Request: request}, nil + }) + before := time.Now() + _, _, err := client.GetRepository(context.Background(), "public-owner", "skills") + var apiError *APIError + if !errors.As(err, &apiError) || apiError.Code != "GITHUB_RATE_LIMITED" || + apiError.RetryAt.Before(before.Add(120*time.Second)) || + apiError.RetryAt.After(time.Now().Add(121*time.Second)) { + t.Fatalf("secondary rate limit did not honor Retry-After: %v", err) + } +} + +func TestAnonymousRepositoryRequestHasNoAuthorizationHeader(t *testing.T) { + client := NewClient("") + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.Header.Get("Authorization") != "" { + t.Fatal("anonymous GitHub request contained an authorization header") + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"id":42,"name":"skills","owner":{"login":"public-owner"},"default_branch":"main","private":false}`)), + Request: request}, nil + }) + repository, _, err := client.GetRepository(context.Background(), "public-owner", "skills") + if err != nil || repository.ID != 42 { + t.Fatalf("anonymous public repository lookup failed: repository=%+v error=%v", repository, err) + } +} diff --git a/github-sync-go/internal/github/discovery.go b/github-sync-go/internal/github/discovery.go new file mode 100644 index 000000000..b931f3a6b --- /dev/null +++ b/github-sync-go/internal/github/discovery.go @@ -0,0 +1,193 @@ +package github + +import ( + "context" + "errors" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const repositoryPageSize = 100 + +var topicPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`) + +type SearchPage struct { + Repositories []Repository + TotalCount int + IncompleteResults bool + LimitReached bool + HasMore bool +} + +type OwnerType string + +const ( + UserOwner OwnerType = "User" + OrganizationOwner OwnerType = "Organization" +) + +type OwnerPage struct { + Repositories []Repository + HasMore bool +} + +// SearchRepositories accepts only structured selectors. Callers must retain the +// incomplete/limit signals and split the fixed window instead of marking it done. +func (c *Client) SearchRepositories(ctx context.Context, kind, value string, + since *time.Time, until time.Time, page int) (SearchPage, Quota, error) { + if page < 1 || page > 10 || until.IsZero() || (since != nil && since.After(until)) { + return SearchPage{}, Quota{}, errors.New("invalid search page or window") + } + selector, err := searchSelector(kind, value) + if err != nil { + return SearchPage{}, Quota{}, err + } + windowEnd := until.UTC().Format("2006-01-02T15:04:05+00:00") + window := "pushed:*.." + windowEnd + if since != nil { + window = "pushed:" + since.UTC().Format("2006-01-02T15:04:05+00:00") + ".." + windowEnd + } + return c.searchRepositoryPage(ctx, selector+" is:public "+window, page) +} + +// searchRepositoryPage preserves GitHub's coverage signals for both custom and +// official discovery. Its callers compile and validate the query before use. +func (c *Client) searchRepositoryPage(ctx context.Context, compiled string, page int) (SearchPage, Quota, error) { + query := url.Values{} + query.Set("q", compiled) + query.Set("sort", "updated") + query.Set("order", "desc") + query.Set("per_page", strconv.Itoa(repositoryPageSize)) + query.Set("page", strconv.Itoa(page)) + var response struct { + TotalCount *int `json:"total_count"` + IncompleteResults *bool `json:"incomplete_results"` + Items []Repository `json:"items"` + } + quota, err := c.get(ctx, "/search/repositories?"+query.Encode(), &response) + if err != nil { + return SearchPage{}, quota, err + } + if response.TotalCount == nil || response.IncompleteResults == nil || *response.TotalCount < 0 { + return SearchPage{}, quota, errors.New("GitHub search returned incomplete metadata") + } + accessible := min(*response.TotalCount, 1000) + return SearchPage{ + Repositories: publicRepositories(response.Items), + TotalCount: *response.TotalCount, + IncompleteResults: *response.IncompleteResults, + LimitReached: *response.TotalCount > 1000, + HasMore: page*repositoryPageSize < accessible, + }, quota, nil +} + +func (c *Client) GetOwnerType(ctx context.Context, owner string) (OwnerType, Quota, error) { + if !ownerPattern.MatchString(owner) { + return "", Quota{}, errors.New("invalid GitHub owner") + } + var response struct { + Type string `json:"type"` + } + quota, err := c.get(ctx, "/users/"+owner, &response) + if err != nil { + return "", quota, err + } + ownerType := OwnerType(response.Type) + if ownerType != UserOwner && ownerType != OrganizationOwner { + return "", quota, errors.New("GitHub owner is not a user or organization") + } + return ownerType, quota, nil +} + +// OwnerIdentity is resolved through GitHub's durable account ID endpoint. +type OwnerIdentity struct { + ID int64 `json:"id"` + Login string `json:"login"` + Type OwnerType `json:"type"` +} + +func (c *Client) GetOwnerByID(ctx context.Context, id int64) (OwnerIdentity, Quota, error) { + if id < 1 { + return OwnerIdentity{}, Quota{}, errors.New("invalid GitHub owner ID") + } + var owner OwnerIdentity + quota, err := c.get(ctx, "/user/"+strconv.FormatInt(id, 10), &owner) + if err != nil { + return OwnerIdentity{}, quota, err + } + if owner.ID != id || !ownerPattern.MatchString(owner.Login) || (owner.Type != UserOwner && owner.Type != OrganizationOwner) { + return OwnerIdentity{}, quota, errors.New("GitHub returned inconsistent owner identity") + } + return owner, quota, nil +} + +// ListPublicOwnerRepositories uses the public-only endpoint for the account type +// resolved by GetOwnerType. Candidate budgeting and page checkpoints belong to the worker. +func (c *Client) ListPublicOwnerRepositories(ctx context.Context, owner string, + ownerType OwnerType, page int) (OwnerPage, Quota, error) { + if !ownerPattern.MatchString(owner) || page < 1 || + (ownerType != UserOwner && ownerType != OrganizationOwner) { + return OwnerPage{}, Quota{}, errors.New("invalid GitHub owner or page") + } + path := "/users/" + owner + "/repos" + query := url.Values{} + if ownerType == OrganizationOwner { + path = "/orgs/" + owner + "/repos" + query.Set("type", "public") + } else { + query.Set("type", "owner") + } + query.Set("sort", "full_name") + query.Set("direction", "asc") + query.Set("per_page", strconv.Itoa(repositoryPageSize)) + query.Set("page", strconv.Itoa(page)) + var repositories []Repository + quota, err := c.get(ctx, path+"?"+query.Encode(), &repositories) + if err != nil { + return OwnerPage{}, quota, err + } + return OwnerPage{ + Repositories: publicRepositories(repositories), + HasMore: len(repositories) == repositoryPageSize, + }, quota, nil +} + +func searchSelector(kind, value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || utf8.RuneCountInString(value) > 100 { + return "", errors.New("invalid search selector") + } + switch kind { + case "KEYWORD": + for _, char := range value { + if !unicode.IsLetter(char) && !unicode.IsDigit(char) && char != ' ' && + char != '-' && char != '_' { + return "", errors.New("keyword contains unsupported search syntax") + } + } + return `"` + value + `" in:name,description,topics`, nil + case "TOPIC": + if !topicPattern.MatchString(value) { + return "", errors.New("invalid GitHub topic selector") + } + return "topic:" + strings.ToLower(value), nil + default: + return "", errors.New("unsupported search selector kind") + } +} + +func publicRepositories(repositories []Repository) []Repository { + public := make([]Repository, 0, len(repositories)) + for _, repository := range repositories { + if !repository.Private && repository.ID > 0 && + validateRepository(repository.Owner.Login, repository.Name) == nil { + public = append(public, repository) + } + } + return public +} diff --git a/github-sync-go/internal/github/discovery_live_test.go b/github-sync-go/internal/github/discovery_live_test.go new file mode 100644 index 000000000..dbf1f8c3c --- /dev/null +++ b/github-sync-go/internal/github/discovery_live_test.go @@ -0,0 +1,33 @@ +package github + +import ( + "context" + "os" + "testing" + "time" +) + +// Opt-in check of the structured search contract against the public GitHub API. +func TestLiveStructuredRepositorySearch(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_SEARCH") != "1" { + t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_SEARCH=1 to use the public GitHub API") + } + client := NewClient("") + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + until := time.Now().UTC() + for _, selector := range []struct{ kind, value string }{ + {"KEYWORD", "agent skills"}, + {"TOPIC", "agent-skills"}, + } { + page, _, err := client.SearchRepositories(ctx, selector.kind, selector.value, + nil, until, 1) + if err != nil { + t.Fatalf("%s search failed: %v", selector.kind, err) + } + if page.TotalCount < 0 || len(page.Repositories) > repositoryPageSize { + t.Fatalf("%s search returned invalid metadata: total=%d page=%d", + selector.kind, page.TotalCount, len(page.Repositories)) + } + } +} diff --git a/github-sync-go/internal/github/discovery_test.go b/github-sync-go/internal/github/discovery_test.go new file mode 100644 index 000000000..32741de79 --- /dev/null +++ b/github-sync-go/internal/github/discovery_test.go @@ -0,0 +1,78 @@ +package github + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" +) + +func TestSearchRepositoriesReportsLimitAndIncompleteCoverage(t *testing.T) { + client := NewClient("") + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + query := request.URL.Query() + if request.URL.Path != "/search/repositories" || query.Get("page") != "1" || + query.Get("q") != `"agent skills" in:name,description,topics is:public pushed:2026-09-01T00:00:00+00:00..2026-09-02T00:00:00+00:00` { + t.Fatalf("unexpected search request: %s", request.URL.String()) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"total_count":1001,"incomplete_results":true,"items":[{"id":1,"name":"skills","owner":{"login":"owner"},"private":false}]}`)), + Request: request}, nil + }) + from := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + until := from.Add(24 * time.Hour) + page, _, err := client.SearchRepositories(context.Background(), "KEYWORD", "agent skills", &from, until, 1) + if err != nil || !page.LimitReached || !page.IncompleteResults || !page.HasMore || + len(page.Repositories) != 1 || page.Repositories[0].ID != 1 { + t.Fatalf("search coverage = %+v, error = %v", page, err) + } +} + +func TestCuratedUserAndOrganizationEnumerateOnlyPublicRepositories(t *testing.T) { + for _, testcase := range []struct { + name string + ownerType OwnerType + listPath string + filterType string + }{ + {name: "user", ownerType: UserOwner, listPath: "/users/public-owner/repos", filterType: "owner"}, + {name: "organization", ownerType: OrganizationOwner, + listPath: "/orgs/public-owner/repos", filterType: "public"}, + } { + t.Run(testcase.name, func(t *testing.T) { + client := NewClient("") + calls := 0 + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + calls++ + body := `{"type":"` + string(testcase.ownerType) + `"}` + if calls == 1 { + if request.URL.Path != "/users/public-owner" { + t.Fatalf("unexpected owner lookup: %s", request.URL.Path) + } + } else { + query := request.URL.Query() + if request.URL.Path != testcase.listPath || query.Get("type") != testcase.filterType || + query.Get("page") != "1" || query.Get("per_page") != "100" { + t.Fatalf("unexpected repository listing: %s", request.URL.String()) + } + body = `[{"id":7,"name":"skills","private":false,"owner":{"login":"public-owner"}},` + + `{"id":8,"name":"hidden","private":true,"owner":{"login":"public-owner"}}]` + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), Request: request}, nil + }) + ownerType, _, err := client.GetOwnerType(context.Background(), "public-owner") + if err != nil || ownerType != testcase.ownerType { + t.Fatalf("owner type = %s, error = %v", ownerType, err) + } + page, _, err := client.ListPublicOwnerRepositories(context.Background(), + "public-owner", ownerType, 1) + if err != nil || calls != 2 || page.HasMore || len(page.Repositories) != 1 || + page.Repositories[0].ID != 7 { + t.Fatalf("public repository page = %+v, error = %v", page, err) + } + }) + } +} diff --git a/github-sync-go/internal/github/official_discovery.go b/github-sync-go/internal/github/official_discovery.go new file mode 100644 index 000000000..ef1149ba8 --- /dev/null +++ b/github-sync-go/internal/github/official_discovery.go @@ -0,0 +1,102 @@ +package github + +import ( + "context" + "errors" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +type OfficialQueryKind string +type OfficialQueryScope string + +const ( + OfficialKeyword OfficialQueryKind = "KEYWORD" + OfficialTopic OfficialQueryKind = "TOPIC" + + OfficialMetadataScope OfficialQueryScope = "NAME_DESCRIPTION_TOPICS" + OfficialReadmeScope OfficialQueryScope = "README" +) + +// OfficialRepositoryQuery is the executable search portion of a Java-owned +// policy snapshot. Policy versions and query catalogs remain in the control plane. +type OfficialRepositoryQuery struct { + Kind OfficialQueryKind + Value string + Scope OfficialQueryScope +} + +// OfficialRepositoryRange is a half-open pushed-time interval. Partition values +// require an agreed compiler and are currently rejected rather than ignored. +type OfficialRepositoryRange struct { + Since time.Time + Until time.Time + Partition string +} + +// CompileOfficialRepositoryQuery only accepts structured repository searches. +// It does not compile code search or owner expansion. Whole-second boundaries +// avoid silently changing coverage when translating to GitHub's inclusive range. +func CompileOfficialRepositoryQuery(query OfficialRepositoryQuery, window OfficialRepositoryRange, + fixedCutoffAt time.Time) (string, error) { + _, sinceOffset := window.Since.Zone() + _, untilOffset := window.Until.Zone() + _, cutoffOffset := fixedCutoffAt.Zone() + if window.Since.IsZero() || window.Until.IsZero() || fixedCutoffAt.IsZero() || + !window.Since.Before(window.Until) || window.Until.After(fixedCutoffAt) || + window.Since.Nanosecond() != 0 || window.Until.Nanosecond() != 0 || fixedCutoffAt.Nanosecond() != 0 || + sinceOffset != 0 || untilOffset != 0 || cutoffOffset != 0 { + return "", errors.New("invalid official search window or cutoff") + } + if window.Partition != "" { + return "", errors.New("unsupported official search partition") + } + value := strings.TrimSpace(query.Value) + if !utf8.ValidString(value) || value == "" || utf8.RuneCountInString(value) > 256 { + return "", errors.New("invalid official search value") + } + var selector string + switch query.Kind { + case OfficialKeyword: + for _, char := range value { + if !unicode.IsLetter(char) && !unicode.IsDigit(char) && char != ' ' && + char != '-' && char != '_' && char != '.' && char != '/' { + return "", errors.New("official keyword contains unsupported search syntax") + } + } + switch query.Scope { + case OfficialMetadataScope: + selector = `"` + value + `" in:name,description,topics` + case OfficialReadmeScope: + selector = `"` + value + `" in:readme` + default: + return "", errors.New("unsupported official keyword scope") + } + case OfficialTopic: + if query.Scope != OfficialMetadataScope || !topicPattern.MatchString(value) { + return "", errors.New("invalid official topic or scope") + } + selector = "topic:" + strings.ToLower(value) + default: + return "", errors.New("unsupported official repository query kind") + } + const timestampFormat = "2006-01-02T15:04:05+00:00" + return selector + " is:public pushed:" + window.Since.UTC().Format(timestampFormat) + + ".." + window.Until.Add(-time.Second).UTC().Format(timestampFormat), nil +} + +// SearchOfficialRepositories executes one page; the caller retains incomplete +// coverage and limit signals and owns durable checkpoints and shard splitting. +func (c *Client) SearchOfficialRepositories(ctx context.Context, query OfficialRepositoryQuery, + window OfficialRepositoryRange, fixedCutoffAt time.Time, page int) (SearchPage, Quota, error) { + if page < 1 || page > 10 { + return SearchPage{}, Quota{}, errors.New("invalid official search page") + } + compiled, err := CompileOfficialRepositoryQuery(query, window, fixedCutoffAt) + if err != nil { + return SearchPage{}, Quota{}, err + } + return c.searchRepositoryPage(ctx, compiled, page) +} diff --git a/github-sync-go/internal/github/official_discovery_test.go b/github-sync-go/internal/github/official_discovery_test.go new file mode 100644 index 000000000..1ddaf2040 --- /dev/null +++ b/github-sync-go/internal/github/official_discovery_test.go @@ -0,0 +1,94 @@ +package github + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +func TestOfficialQueryHalfOpenUTCAndScopes(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, scope := range []OfficialQueryScope{OfficialQueryScope("NAME_DESCRIPTION_TOPICS"), OfficialQueryScope("README")} { + t.Run(string(scope), func(t *testing.T) { + q, err := CompileOfficialRepositoryQuery(OfficialRepositoryQuery{Kind: "KEYWORD", Value: "agent skills", Scope: scope}, OfficialRepositoryRange{Since: start, Until: start.Add(2 * time.Second)}, start.Add(time.Hour)) + if err != nil || !strings.Contains(q, "00:00:01+00:00") || strings.Contains(q, "00:00:02+00:00") { + t.Fatalf("half-open query %q, %v", q, err) + } + if scope == "README" && !strings.Contains(q, "in:readme") { + t.Fatalf("missing README qualifier %q", q) + } + }) + } + for _, kind := range []string{"fractional", "partition", "cutoff", "injection"} { + t.Run(kind, func(t *testing.T) { + window := OfficialRepositoryRange{Since: start, Until: start.Add(time.Second)} + cutoff := start.Add(time.Hour) + value := "skills" + switch kind { + case "fractional": + window.Since = start.Add(time.Nanosecond) + case "partition": + window.Partition = "stars:>1" + case "cutoff": + cutoff = start + case "injection": + value = "skills user:someone" + } + if _, err := CompileOfficialRepositoryQuery(OfficialRepositoryQuery{Kind: "KEYWORD", Value: value, Scope: "NAME_DESCRIPTION_TOPICS"}, window, cutoff); err == nil { + t.Fatal("invalid official query accepted") + } + }) + } +} +func TestOfficialOwnerStableIDResponseValidation(t *testing.T) { + for _, body := range []string{`{"id":42,"login":"renamed","type":"User"}`, `{"id":42,"login":"renamed","type":"Organization"}`, `{"id":43,"login":"renamed","type":"User"}`, `{"id":42,"login":"bad/name","type":"User"}`, `{"id":42,"login":"renamed","type":"Bot"}`} { + t.Run(body, func(t *testing.T) { + c := NewClient("") + c.http.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/user/42" { + t.Errorf("lookup not by stable ID: %s", r.URL.Path) + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) + owner, _, err := c.GetOwnerByID(context.Background(), 42) + valid := strings.Contains(body, `"id":42`) && !strings.Contains(body, "bad/name") && !strings.Contains(body, "Bot") + if (err == nil) != valid { + t.Fatalf("owner %+v err %v", owner, err) + } + }) + } +} +func TestOfficialRESTPolicyStopsRequestsAfterObservationFailure(t *testing.T) { + c := NewClient("") + calls, observations := 0, 0 + stopped := false + c.http.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + calls++ + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"id":42,"login":"owner","type":"User"}`)), Request: r}, nil + }) + c.SetRequestPolicy(func(context.Context) error { + if stopped { + return errors.New("cooldown") + } + return nil + }, func(_ context.Context, q Quota, _ error) error { + observations++ + if q.Remaining != -1 { + t.Error("missing headers produced known remaining") + } + stopped = true + return errors.New("persistence failed") + }) + for i := 0; i < 2; i++ { + if _, _, err := c.GetOwnerByID(context.Background(), 42); err == nil { + t.Fatal("policy error was ignored") + } + } + if calls != 1 || observations != 1 { + t.Fatalf("calls=%d observations=%d", calls, observations) + } +} diff --git a/github-sync-go/internal/github/request_control.go b/github-sync-go/internal/github/request_control.go new file mode 100644 index 000000000..00b09543c --- /dev/null +++ b/github-sync-go/internal/github/request_control.go @@ -0,0 +1,16 @@ +package github + +import "context" + +type acquisitionControlKey struct{} + +// WithAcquisitionControl composes platform stopping with the existing quota policy. +func WithAcquisitionControl(ctx context.Context, before func(context.Context) error) context.Context { + return context.WithValue(ctx, acquisitionControlKey{}, before) +} +func CheckAcquisitionControl(ctx context.Context) error { + if before, ok := ctx.Value(acquisitionControlKey{}).(func(context.Context) error); ok { + return before(ctx) + } + return ctx.Err() +} diff --git a/github-sync-go/internal/github/request_control_test.go b/github-sync-go/internal/github/request_control_test.go new file mode 100644 index 000000000..a8759ffb1 --- /dev/null +++ b/github-sync-go/internal/github/request_control_test.go @@ -0,0 +1,22 @@ +package github + +import ( + "context" + "errors" + "testing" +) + +func TestPlatformControlFencesEveryGithubReadBeforeQuotaAndNetwork(t *testing.T) { + stop := errors.New("platform paused") + ctx := WithAcquisitionControl(context.Background(), func(context.Context) error { return stop }) + client := NewClient("") + client.SetRequestPolicy(func(context.Context) error { t.Fatal("quota hook ran after platform stop"); return nil }, nil) + // All acquisition channels (search, owner listing, metadata, HEAD and Tree) + // pass through get; no external request may occur after the control denial. + for _, path := range []string{"/search/repositories", "/search/code", "/users/fixture/repos", "/repos/fixture/skills", "/repos/fixture/skills/git/ref/heads/main", "/repos/fixture/skills/git/trees/fixture"} { + var payload any + if _, err := client.get(ctx, path, &payload); !errors.Is(err, stop) { + t.Fatalf("%s: %v", path, err) + } + } +} diff --git a/github-sync-go/internal/github/tree_test.go b/github-sync-go/internal/github/tree_test.go new file mode 100644 index 000000000..d8f7458a7 --- /dev/null +++ b/github-sync-go/internal/github/tree_test.go @@ -0,0 +1,34 @@ +package github + +import ( + "context" + "io" + "net/http" + "strings" + "testing" +) + +func TestFixedCommitTreeRejectsMismatchedResponses(t *testing.T) { + const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const treeSHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + client := NewClient("") + client.http.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + var body string + switch request.URL.Path { + case "/repos/owner/repo/git/commits/" + commitSHA: + body = `{"sha":"cccccccccccccccccccccccccccccccccccccccc","tree":{"sha":"` + treeSHA + `"}}` + case "/repos/owner/repo/git/trees/" + treeSHA: + body = `{"sha":"dddddddddddddddddddddddddddddddddddddddd","truncated":false,"tree":[]}` + default: + t.Fatalf("unexpected GitHub tree request: %s", request.URL.String()) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), Request: request}, nil + }) + if _, _, err := client.GetCommitTreeSHA(context.Background(), "owner", "repo", commitSHA); err == nil { + t.Fatal("commit response with a different SHA must be rejected") + } + if _, _, err := client.GetTree(context.Background(), "owner", "repo", treeSHA, false); err == nil { + t.Fatal("tree response with a different SHA must be rejected") + } +} diff --git a/github-sync-go/internal/repository/acquire.go b/github-sync-go/internal/repository/acquire.go new file mode 100644 index 000000000..22e3edace --- /dev/null +++ b/github-sync-go/internal/repository/acquire.go @@ -0,0 +1,576 @@ +package repository + +import ( + "archive/zip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +var ( + ErrLocalDiskBudget = errors.New("local Git workspace disk budget exceeded") + ErrCapacity = errors.New("local disk cannot hold a compliant package") + ErrArchiveTooLarge = errors.New("Skill archive exceeds upload limit") +) + +const ( + evidenceManifestPath = "__skillhub_sync_evidence__.json" + evidenceEntryPrefix = "__skillhub_sync_evidence__/" + gitMetadataReserve = 16 << 20 + archiveOverheadReserve = 1 << 20 +) + +type rootEvidenceManifest struct { + FormatVersion int `json:"formatVersion"` + RootFiles []RootEvidenceFile `json:"rootFiles"` +} + +type AcquisitionLimits struct { + MaxWorkspaceBytes int64 + MaxArchiveBytes int64 + MinFreeBytes int64 +} + +// Artifact owns its isolated temporary directory until Close is called. +type Artifact struct { + Path string + Size int64 + SHA256 string + dir string + once sync.Once + closeErr error +} + +func (artifact *Artifact) Close() error { + artifact.once.Do(func() { artifact.closeErr = os.RemoveAll(artifact.dir) }) + return artifact.closeErr +} + +// AcquirePackage fetches one fixed public commit without repository blobs, +// checks out only the preflighted files, then builds one bounded ZIP. The +// caller must obtain Java package admission before invoking this function. +func AcquirePackage(ctx context.Context, owner, name, commitSHA, tempRoot string, + plan PackagePlan, limits AcquisitionLimits) (*Artifact, error) { + if !validGitSHA(commitSHA) || (plan.Root != "." && !validPath(plan.Root)) || + len(plan.Files) == 0 || + plan.TotalBytes < 0 || limits.MaxWorkspaceBytes < 1 || limits.MaxArchiveBytes < 1 || + limits.MinFreeBytes < 0 || limits.MaxWorkspaceBytes > math.MaxInt64-limits.MinFreeBytes { + return nil, errors.New("invalid fixed revision, package plan, or acquisition limits") + } + remote, err := github.PublicCloneURL(owner, name) + if err != nil { + return nil, err + } + var evidenceBytes int64 + for _, file := range plan.Evidence { + if !validPath(file.RepositoryPath) || strings.Contains(file.RepositoryPath, "/") || + !rootEvidenceName.MatchString(file.RepositoryPath) || !validGitSHA(file.SHA) || + file.Size < 0 { + return nil, errors.New("invalid root license or notice evidence") + } + if file.Bundled { + if file.Size > maxEvidenceFileBytes || + file.Size > maxEvidenceTotalBytes-evidenceBytes { + return nil, errors.New("root evidence exceeds acquisition limit") + } + evidenceBytes += file.Size + } + } + if len(plan.Evidence) > maxRootEvidenceFiles || + !canReserveWorkspace(plan.TotalBytes, evidenceBytes, limits) { + return nil, ErrCapacity + } + patterns, allowed, err := prepareSparseCheckout(plan) + if err != nil { + return nil, err + } + if err := ensureFreeSpace(tempRoot, limits.MaxWorkspaceBytes+limits.MinFreeBytes); err != nil { + return nil, err + } + workDir, err := os.MkdirTemp(tempRoot, "skillhub-github-package-") + if err != nil { + return nil, localDiskWriteError(err) + } + keep := false + defer func() { + if !keep { + _ = os.RemoveAll(workDir) + } + }() + repoDir := filepath.Join(workDir, "repo") + if err := os.Mkdir(repoDir, 0700); err != nil { + return nil, localDiskWriteError(err) + } + for _, step := range []struct { + name string + args []string + }{ + {"init", []string{"init", "--quiet"}}, + {"remote", []string{"remote", "add", "origin", remote}}, + {"fetch", []string{"fetch", "--quiet", "--no-tags", "--filter=blob:none", "--depth=1", "origin", commitSHA}}, + } { + if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes, + limits.MinFreeBytes, step.args...); err != nil { + return nil, fmt.Errorf("Git %s: %w", step.name, err) + } + } + if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes, + limits.MinFreeBytes, patterns...); err != nil { + return nil, fmt.Errorf("Git sparse checkout: %w", err) + } + if err := runBoundedGit(ctx, repoDir, workDir, limits.MaxWorkspaceBytes, + limits.MinFreeBytes, + "checkout", "--quiet", "--detach", "--force", "FETCH_HEAD"); err != nil { + return nil, fmt.Errorf("Git fixed checkout: %w", err) + } + head, err := os.ReadFile(filepath.Join(repoDir, ".git", "HEAD")) + if err != nil || strings.TrimSpace(string(head)) != commitSHA { + return nil, errors.New("checked-out commit differs from fixed revision") + } + if err := verifyCheckout(repoDir, allowed); err != nil { + return nil, err + } + archivePath := filepath.Join(workDir, "package.zip") + usedBytes, err := directoryBytes(workDir) + if err != nil { + return nil, err + } + remainingBytes := limits.MaxWorkspaceBytes - usedBytes + if remainingBytes < 1 { + return nil, ErrLocalDiskBudget + } + archiveLimit := min(limits.MaxArchiveBytes, remainingBytes) + if err := requireMinimumFree(workDir, archiveLimit+limits.MinFreeBytes); err != nil { + return nil, err + } + size, digest, err := writePackageArchive(ctx, repoDir, archivePath, plan, archiveLimit) + if err != nil { + return nil, archiveWriteError(err, archiveLimit, limits.MaxArchiveBytes) + } + if size+usedBytes > limits.MaxWorkspaceBytes { + return nil, ErrLocalDiskBudget + } + keep = true + return &Artifact{Path: archivePath, Size: size, SHA256: digest, dir: workDir}, nil +} + +// Validate the complete file plan before creating a workspace or fetching Git objects. +func prepareSparseCheckout(plan PackagePlan) ([]string, map[string]PackageFile, error) { + patterns := make([]string, 0, len(plan.Files)+len(plan.Evidence)+4) + patterns = append(patterns, "sparse-checkout", "set", "--no-cone", "--") + allowed := make(map[string]PackageFile, len(plan.Files)) + hasSkillMD := false + var expectedTotal int64 + for _, file := range plan.Files { + if !validPath(file.RepositoryPath) || !validPath(file.RelativePath) || + !validGitSHA(file.SHA) || file.Size < 0 { + return nil, nil, errors.New("invalid preflighted file") + } + if plan.Root != "." && !strings.HasPrefix(file.RepositoryPath, plan.Root+"/") { + return nil, nil, errors.New("preflighted file is outside Skill root") + } + relativePath := file.RepositoryPath + if plan.Root != "." { + relativePath = strings.TrimPrefix(file.RepositoryPath, plan.Root+"/") + } + if file.RelativePath != relativePath || file.Size > plan.TotalBytes-expectedTotal { + return nil, nil, errors.New("preflighted file does not match package plan") + } + expectedTotal += file.Size + if file.RelativePath == "SKILL.md" { + hasSkillMD = true + } + if _, duplicate := allowed[file.RepositoryPath]; duplicate { + return nil, nil, errors.New("duplicate preflighted file") + } + allowed[file.RepositoryPath] = file + patterns = append(patterns, sparseFilePattern(file.RepositoryPath)) + } + for _, evidence := range plan.Evidence { + if !evidence.Bundled { + continue + } + if selected, exists := allowed[evidence.RepositoryPath]; exists { + if selected.SHA != evidence.SHA || selected.Size != evidence.Size { + return nil, nil, errors.New("root evidence conflicts with package file") + } + continue + } + allowed[evidence.RepositoryPath] = PackageFile{ + RepositoryPath: evidence.RepositoryPath, + SHA: evidence.SHA, Size: evidence.Size, + } + patterns = append(patterns, sparseFilePattern(evidence.RepositoryPath)) + } + if !hasSkillMD || expectedTotal != plan.TotalBytes { + return nil, nil, errors.New("package plan has no SKILL.md or mismatched total size") + } + return patterns, allowed, nil +} + +// Reserve room for Git objects, checked-out files, a ZIP, and baseline Git +// metadata before any network fetch. Actual usage is still checked at each Git +// step because repository tree metadata has no predictable size. +func canReserveWorkspace(packageBytes, evidenceBytes int64, limits AcquisitionLimits) bool { + if packageBytes < 0 || evidenceBytes < 0 || packageBytes > math.MaxInt64-evidenceBytes || + limits.MaxWorkspaceBytes <= gitMetadataReserve || limits.MaxArchiveBytes < 1 { + return false + } + rawBytes := packageBytes + evidenceBytes + available := limits.MaxWorkspaceBytes - gitMetadataReserve + if rawBytes > available/2 { + return false + } + available -= 2 * rawBytes + archiveBytes := limits.MaxArchiveBytes + if rawBytes <= limits.MaxArchiveBytes-archiveOverheadReserve { + archiveBytes = rawBytes + archiveOverheadReserve + } + return archiveBytes <= available +} + +func archiveWriteError(err error, workspaceLimit, uploadLimit int64) error { + if errors.Is(err, ErrArchiveTooLarge) && workspaceLimit < uploadLimit { + return ErrLocalDiskBudget + } + return localDiskWriteError(err) +} + +func localDiskWriteError(err error) error { + if errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) { + return ErrLocalDiskBudget + } + return err +} + +func sparseFilePattern(path string) string { + var pattern strings.Builder + pattern.WriteByte('/') + for _, char := range path { + if char == '*' || char == '?' || char == '[' || char == ']' || char == '\\' { + pattern.WriteByte('\\') + } + pattern.WriteRune(char) + } + return pattern.String() +} + +func gitEnvironment() []string { + environment := make([]string, 0, len(os.Environ())+5) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, "GIT_") || strings.HasPrefix(entry, "HOME=") { + continue + } + environment = append(environment, entry) + } + return append(environment, + "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_TERMINAL_PROMPT=0", "GIT_ALLOW_PROTOCOL=https", "GIT_LFS_SKIP_SMUDGE=1") +} + +type gitRequestPolicyKey struct{} +type gitRequestPolicy struct { + before func(context.Context) error + after func(context.Context, error) error +} + +// WithGitRequestPolicy guards commands that may fetch lazy partial-clone blobs. +// Archive creation and artifact delivery do not perform GitHub requests. +func WithGitRequestPolicy(ctx context.Context, before func(context.Context) error, after func(context.Context, error) error) context.Context { + return context.WithValue(ctx, gitRequestPolicyKey{}, gitRequestPolicy{before, after}) +} + +func runBoundedGit(ctx context.Context, repoDir, workDir string, + maxBytes, minFreeBytes int64, args ...string) (resultErr error) { + if err := github.CheckAcquisitionControl(ctx); err != nil { + return err + } + var afterRequest func(context.Context, error) error + if policy, ok := ctx.Value(gitRequestPolicyKey{}).(gitRequestPolicy); ok { + if err := policy.before(ctx); err != nil { + return err + } + if len(args) > 0 && (args[0] == "fetch" || args[0] == "checkout" || args[0] == "sparse-checkout") { + afterRequest = policy.after + } + } + + if err := checkWorkspace(workDir, maxBytes, minFreeBytes); err != nil { + return err + } + command := exec.CommandContext(ctx, "git", append([]string{ + "-c", "credential.helper=", "-c", "core.hooksPath=/dev/null", + "-c", "http.followRedirects=false", + }, args...)...) + command.Dir = repoDir + command.Env = gitEnvironment() + command.Stdout = io.Discard + command.Stderr = io.Discard + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + return localDiskWriteError(err) + } + if afterRequest != nil { + defer func() { + if err := afterRequest(ctx, resultErr); err != nil { + resultErr = err + } + }() + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + select { + case err := <-done: + if checkErr := checkWorkspace(workDir, maxBytes, minFreeBytes); checkErr != nil { + return checkErr + } + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(requireMinimumFree(workDir, 1), ErrLocalDiskBudget) { + return ErrLocalDiskBudget + } + return errors.New("Git command failed") + } + return nil + case <-ticker.C: + err := checkWorkspace(workDir, maxBytes, minFreeBytes) + if err != nil { + _ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + <-done + return err + } + case <-ctx.Done(): + _ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + <-done + return ctx.Err() + } + } +} + +func checkWorkspace(workDir string, maxBytes, minFreeBytes int64) error { + size, err := directoryBytes(workDir) + if err != nil { + return err + } + if size > maxBytes { + return ErrLocalDiskBudget + } + return requireMinimumFree(workDir, minFreeBytes) +} + +func requireMinimumFree(path string, bytes int64) error { + err := ensureFreeSpace(path, bytes) + if errors.Is(err, ErrCapacity) { + return ErrLocalDiskBudget + } + return err +} + +func directoryBytes(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + if path != root && errors.Is(walkErr, os.ErrNotExist) { + return nil + } + return walkErr + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + total += info.Size() + return nil + }) + return total, err +} + +func ensureFreeSpace(path string, required int64) error { + var stats syscall.Statfs_t + if err := syscall.Statfs(path, &stats); err != nil { + return err + } + blockSize := uint64(stats.Bsize) + if blockSize == 0 { + return ErrCapacity + } + requiredBlocks := uint64(required) / blockSize + if uint64(required)%blockSize != 0 { + requiredBlocks++ + } + if stats.Bavail < requiredBlocks { + return ErrCapacity + } + return nil +} + +func verifyCheckout(repoDir string, allowed map[string]PackageFile) error { + seen := make(map[string]struct{}, len(allowed)) + err := filepath.WalkDir(repoDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(repoDir, path) + if err != nil { + return err + } + if relative == ".git" { + return filepath.SkipDir + } + if entry.IsDir() { + return nil + } + file, exists := allowed[filepath.ToSlash(relative)] + if !exists || entry.Type()&os.ModeSymlink != 0 { + return errors.New("checkout contains an unexpected file") + } + info, err := entry.Info() + if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size { + return errors.New("checked-out file differs from tree metadata") + } + seen[file.RepositoryPath] = struct{}{} + return nil + }) + if err != nil { + return err + } + if len(seen) != len(allowed) { + return errors.New("checkout omitted preflighted files") + } + return nil +} + +type limitedArchiveWriter struct { + writer io.Writer + limit int64 + written int64 +} + +func (writer *limitedArchiveWriter) Write(data []byte) (int, error) { + if int64(len(data)) > writer.limit-writer.written { + return 0, ErrArchiveTooLarge + } + n, err := writer.writer.Write(data) + writer.written += int64(n) + return n, err +} + +func writePackageArchive(ctx context.Context, repoDir, archivePath string, + plan PackagePlan, maxBytes int64) (int64, string, error) { + if err := ctx.Err(); err != nil { + return 0, "", err + } + archive, err := os.OpenFile(archivePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return 0, "", err + } + defer archive.Close() + digest := sha256.New() + limited := &limitedArchiveWriter{writer: io.MultiWriter(archive, digest), limit: maxBytes} + zipWriter := zip.NewWriter(limited) + for _, file := range plan.Files { + if err := appendArchiveFile(ctx, zipWriter, repoDir, file.RepositoryPath, + file.RelativePath, file.Size); err != nil { + return 0, "", err + } + } + for _, evidence := range plan.Evidence { + if evidence.Bundled { + if err := appendArchiveFile(ctx, zipWriter, repoDir, evidence.RepositoryPath, + evidenceEntryPrefix+evidence.RepositoryPath, evidence.Size); err != nil { + return 0, "", err + } + } + } + manifest, err := json.Marshal(rootEvidenceManifest{ + FormatVersion: 1, RootFiles: append([]RootEvidenceFile{}, plan.Evidence...), + }) + if err != nil { + return 0, "", err + } + entry, err := zipWriter.Create(evidenceManifestPath) + if err != nil { + return 0, "", err + } + if _, err := entry.Write(manifest); err != nil { + return 0, "", err + } + if err := ctx.Err(); err != nil { + return 0, "", err + } + if err := zipWriter.Close(); err != nil { + return 0, "", err + } + if err := archive.Sync(); err != nil { + return 0, "", err + } + if err := ctx.Err(); err != nil { + return 0, "", err + } + return limited.written, hex.EncodeToString(digest.Sum(nil)), nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (reader contextReader) Read(data []byte) (int, error) { + if err := reader.ctx.Err(); err != nil { + return 0, err + } + return reader.reader.Read(data) +} + +func appendArchiveFile(ctx context.Context, zipWriter *zip.Writer, repoDir, repositoryPath, + archivePath string, expectedSize int64) error { + if err := ctx.Err(); err != nil { + return err + } + input, err := os.Open(filepath.Join(repoDir, filepath.FromSlash(repositoryPath))) + if err != nil { + return err + } + entry, err := zipWriter.Create(archivePath) + if err == nil { + var copied int64 + if expectedSize == math.MaxInt64 { + err = errors.New("preflighted file is too large") + } else { + copied, err = io.Copy(entry, io.LimitReader(contextReader{ctx, input}, expectedSize+1)) + } + if err == nil && copied != expectedSize { + err = errors.New("file changed while creating package") + } + } + closeErr := input.Close() + if err != nil { + return err + } + return closeErr +} diff --git a/github-sync-go/internal/repository/acquire_test.go b/github-sync-go/internal/repository/acquire_test.go new file mode 100644 index 000000000..206a851b1 --- /dev/null +++ b/github-sync-go/internal/repository/acquire_test.go @@ -0,0 +1,804 @@ +package repository + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "math" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +type cancelAfterFirstRead struct { + cancel context.CancelFunc + reads int +} + +func (reader *cancelAfterFirstRead) Read(data []byte) (int, error) { + reader.reads++ + if reader.reads > 1 { + return 0, io.EOF + } + data[0] = 'x' + reader.cancel() + return 1, nil +} + +func TestArchiveReaderStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + source := &cancelAfterFirstRead{cancel: cancel} + _, err := io.Copy(io.Discard, contextReader{ctx: ctx, reader: source}) + if !errors.Is(err, context.Canceled) || source.reads != 1 { + t.Fatalf("archive reader continued after cancellation: reads=%d error=%v", source.reads, err) + } +} + +func TestWritePackageArchiveDoesNotStartAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + path := filepath.Join(t.TempDir(), "package.zip") + _, _, err := writePackageArchive(ctx, t.TempDir(), path, PackagePlan{}, 1024) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled archive returned %v", err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled archive created a file: %v", err) + } +} + +func TestCanReserveWorkspaceIncludesGitCheckoutAndArchive(t *testing.T) { + const mib = int64(1 << 20) + limits := AcquisitionLimits{MaxWorkspaceBytes: 64 * mib, MaxArchiveBytes: 24 * mib} + if canReserveWorkspace(20*mib, 0, limits) { + t.Fatal("a workspace smaller than the combined Git, checkout, archive and metadata reserve was accepted") + } + limits.MaxWorkspaceBytes = 80 * mib + if !canReserveWorkspace(20*mib, 0, limits) { + t.Fatal("a workspace with enough reserved capacity was rejected") + } + if canReserveWorkspace(math.MaxInt64, 1, limits) { + t.Fatal("overflowing package and evidence sizes were accepted") + } +} + +func TestArchiveWriteErrorDistinguishesWorkspaceAndUploadLimits(t *testing.T) { + if err := archiveWriteError(ErrArchiveTooLarge, 1024, 2048); !errors.Is(err, ErrLocalDiskBudget) { + t.Fatalf("workspace-limited archive was not deferred as disk budget: %v", err) + } + if err := archiveWriteError(ErrArchiveTooLarge, 2048, 2048); !errors.Is(err, ErrArchiveTooLarge) { + t.Fatalf("upload-limited archive was not rejected as too large: %v", err) + } + for _, diskError := range []error{syscall.ENOSPC, syscall.EDQUOT} { + writeError := &os.PathError{Op: "write", Path: "package.zip", Err: diskError} + if err := archiveWriteError(writeError, 2048, 2048); !errors.Is(err, ErrLocalDiskBudget) { + t.Fatalf("filesystem write limit %v was not deferred: %v", diskError, err) + } + } +} + +func TestAcquirePackageDefersWhenArchiveExceedsRemainingWorkspace(t *testing.T) { + const workspaceMax = int64(20 << 20) + const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fakeBin := t.TempDir() + git := filepath.Join(fakeBin, "git") + script := `#!/bin/sh +case " $* " in + *" checkout "*) + mkdir -p .git + printf '%s\n' "$SKILLHUB_TEST_COMMIT" > .git/HEAD + head -c 10000 /dev/urandom > SKILL.md + truncate -s "$((SKILLHUB_TEST_WORKSPACE_MAX - 20000))" .git/filler + ;; +esac +` + if err := os.WriteFile(git, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("SKILLHUB_TEST_COMMIT", commitSHA) + t.Setenv("SKILLHUB_TEST_WORKSPACE_MAX", strconv.FormatInt(workspaceMax, 10)) + root := t.TempDir() + artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA, root, + PackagePlan{Root: ".", TotalBytes: 10000, Files: []PackageFile{{ + RepositoryPath: "SKILL.md", RelativePath: "SKILL.md", + SHA: strings.Repeat("b", 40), Size: 10000, + }}}, AcquisitionLimits{ + MaxWorkspaceBytes: workspaceMax, MaxArchiveBytes: 1 << 20, + }) + if artifact != nil || !errors.Is(err, ErrLocalDiskBudget) || errors.Is(err, ErrArchiveTooLarge) { + t.Fatalf("workspace-limited archive returned artifact=%v, error=%v", artifact, err) + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 0 { + t.Fatalf("capacity deferral left temporary files: %v, %v", entries, err) + } +} + +func TestAcquirePackageCancellationRemovesGitWorkspace(t *testing.T) { + fakeBin := t.TempDir() + started := filepath.Join(fakeBin, "git-started") + git := filepath.Join(fakeBin, "git") + if err := os.WriteFile(git, []byte("#!/bin/sh\n: > \"$SKILLHUB_TEST_GIT_STARTED\"\nexec sleep 60\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("SKILLHUB_TEST_GIT_STARTED", started) + root := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { + artifact, err := AcquirePackage(ctx, "openai", "skills", strings.Repeat("a", 40), root, + PackagePlan{Root: "skills/demo", TotalBytes: 1, Files: []PackageFile{{ + RepositoryPath: "skills/demo/SKILL.md", RelativePath: "SKILL.md", + SHA: strings.Repeat("b", 40), Size: 1, + }}}, AcquisitionLimits{MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 1 << 20}) + if artifact != nil { + _ = artifact.Close() + } + finished <- err + }() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(started); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if time.Now().After(deadline) { + cancel() + t.Fatal("Git command did not start") + } + time.Sleep(20 * time.Millisecond) + } + cancel() + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled acquisition returned %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("canceled Git command did not stop") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 0 { + t.Fatalf("canceled acquisition left temporary files: %v, %v", entries, err) + } +} + +func TestAcquirePackageStopsOnSIGTERM(t *testing.T) { + if os.Getenv("SKILLHUB_TEST_SIGNAL_CHILD") == "1" { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM) + defer stop() + _, err := AcquirePackage(ctx, "openai", "skills", strings.Repeat("a", 40), + os.Getenv("SKILLHUB_TEST_TEMP_ROOT"), PackagePlan{ + Root: "skills/demo", TotalBytes: 1, Files: []PackageFile{{ + RepositoryPath: "skills/demo/SKILL.md", RelativePath: "SKILL.md", + SHA: strings.Repeat("b", 40), Size: 1, + }}, + }, AcquisitionLimits{MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 1 << 20}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("SIGTERM did not cancel Git acquisition: %v", err) + } + return + } + fakeBin := t.TempDir() + started := filepath.Join(fakeBin, "git-started") + git := filepath.Join(fakeBin, "git") + if err := os.WriteFile(git, []byte("#!/bin/sh\nprintf '%s' \"$$\" > \"$SKILLHUB_TEST_GIT_STARTED\"\nexec sleep 60\n"), 0700); err != nil { + t.Fatal(err) + } + root := t.TempDir() + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + command := exec.Command(binary, "-test.run=^TestAcquirePackageStopsOnSIGTERM$", "-test.v") + command.Env = append(os.Environ(), + "PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"), + "SKILLHUB_TEST_GIT_STARTED="+started, + "SKILLHUB_TEST_SIGNAL_CHILD=1", "SKILLHUB_TEST_TEMP_ROOT="+root) + var output bytes.Buffer + command.Stdout, command.Stderr = &output, &output + if err := command.Start(); err != nil { + t.Fatal(err) + } + defer func() { + if command.ProcessState == nil { + _ = command.Process.Kill() + if data, err := os.ReadFile(started); err == nil { + if pid, err := strconv.Atoi(string(data)); err == nil { + _ = syscall.Kill(-pid, syscall.SIGKILL) + } + } + } + }() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(started); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if time.Now().After(deadline) { + t.Fatalf("child Git command did not start: %s", output.String()) + } + time.Sleep(20 * time.Millisecond) + } + if err := command.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + finished := make(chan error, 1) + go func() { finished <- command.Wait() }() + select { + case err := <-finished: + if err != nil { + t.Fatalf("SIGTERM child did not exit cleanly: %v, %s", err, output.String()) + } + case <-time.After(10 * time.Second): + t.Fatalf("SIGTERM child did not stop: %s", output.String()) + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 0 { + t.Fatalf("SIGTERM left temporary files: %v, %v", entries, err) + } +} + +// This immutable public commit exercises the real HTTPS partial-clone path. +// Run manually with SKILLHUB_GITHUB_SYNC_LIVE=1; regular tests remain offline. +func TestLiveAcquireFixedPublicSkill(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE") != "1" { + t.Skip("requires an explicit public GitHub smoke test") + } + const commitSHA = "49f948faa9258a0c61caceaf225e179651397431" + plan, want := liveSkillCreatorPlan() + tempRoot := t.TempDir() + limits := AcquisitionLimits{ + MaxWorkspaceBytes: 512 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 128 << 20, + } + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_SMALL_VOLUME") == "1" { + limits.MaxWorkspaceBytes = 32 << 20 + limits.MinFreeBytes = 8 << 20 + } + stopSampling := make(chan struct{}) + peakSample := make(chan int64, 1) + go func() { + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var peak int64 + for { + select { + case <-ticker.C: + if size, err := directoryBytes(tempRoot); err == nil && size > peak { + peak = size + } + case <-stopSampling: + peakSample <- peak + return + } + } + }() + artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA, + tempRoot, plan, limits) + finalSample, finalSampleErr := directoryBytes(tempRoot) + close(stopSampling) + sampledPeakBytes := <-peakSample + if finalSampleErr == nil && finalSample > sampledPeakBytes { + sampledPeakBytes = finalSample + } + if err != nil { + t.Fatal(err) + } + archiveBytes, err := os.ReadFile(artifact.Path) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(archiveBytes) + if artifact.Size != int64(len(archiveBytes)) || artifact.SHA256 != hex.EncodeToString(digest[:]) { + t.Fatal("archive size or digest does not match the generated artifact") + } + if capture := os.Getenv("SKILLHUB_GITHUB_SYNC_CAPTURE_ARCHIVE"); capture != "" { + output, err := os.OpenFile(capture, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + _, writeErr := output.Write(archiveBytes) + closeErr := output.Close() + if writeErr != nil || closeErr != nil { + t.Fatalf("write captured archive: %v, close: %v", writeErr, closeErr) + } + } + workspaceBytes, err := directoryBytes(artifact.dir) + if err != nil { + t.Fatal(err) + } + t.Logf("fixed commit %s: archive=%d bytes, workspace after acquisition=%d bytes, sampled peak=%d bytes", + commitSHA, artifact.Size, workspaceBytes, sampledPeakBytes) + archive, err := zip.OpenReader(artifact.Path) + if err != nil { + t.Fatal(err) + } + got := make([]string, 0, len(archive.File)) + for _, entry := range archive.File { + got = append(got, entry.Name) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + sort.Strings(got) + if len(got) != len(want) { + t.Fatalf("archive has %d entries, want %d: %v", len(got), len(want), got) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("archive entries = %v, want %v", got, want) + } + } + dir := artifact.dir + if err := artifact.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary package directory remains after Close: %v", err) + } +} + +// This repository has thousands of tracked paths at the fixed commit. The +// opt-in smoke test checks that a single Skill stays within a small workspace +// even when the remote repository tree is substantially larger. +func TestLiveAcquireLargerPublicRepositorySkill(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_LARGE_REPO") != "1" { + t.Skip("requires an explicit public GitHub smoke test") + } + const commitSHA = "e325b0ee302fdf66abb1990ad7957e1a1c7aba5a" + plan := liveLargerPublicRepositoryPlan() + tempRoot := t.TempDir() + artifact, err := AcquirePackage(context.Background(), "SangHynh", "agent-skills", + commitSHA, tempRoot, plan, AcquisitionLimits{ + MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 8 << 20, MinFreeBytes: 16 << 20, + }) + if err != nil { + t.Fatal(err) + } + archive, err := zip.OpenReader(artifact.Path) + if err != nil { + _ = artifact.Close() + t.Fatal(err) + } + if len(archive.File) != 2 || archive.File[0].Name != "SKILL.md" || + archive.File[1].Name != evidenceManifestPath { + _ = archive.Close() + _ = artifact.Close() + t.Fatalf("unexpected Skill archive entries: %+v", archive.File) + } + if err := archive.Close(); err != nil { + _ = artifact.Close() + t.Fatal(err) + } + workspaceBytes, err := directoryBytes(tempRoot) + if err != nil || workspaceBytes >= 64<<20 { + _ = artifact.Close() + t.Fatalf("workspace exceeded configured limit: %d bytes, error %v", workspaceBytes, err) + } + t.Logf("fixed commit %s: archive=%d bytes, workspace after acquisition=%d bytes", + commitSHA, artifact.Size, workspaceBytes) + if err := artifact.Close(); err != nil { + t.Fatal(err) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("temporary files remain after package cleanup: %v, %v", remaining, err) + } +} + +func liveLargerPublicRepositoryPlan() PackagePlan { + const root = "skills/00-andruia-consultant" + return PackagePlan{Root: root, TotalBytes: 2894, Files: []PackageFile{{ + RepositoryPath: root + "/SKILL.md", RelativePath: "SKILL.md", + SHA: "e173357610b8204ce3d676ca9172b3463b02ce97", Size: 2894, + }}} +} + +func liveSkillCreatorPlan() (PackagePlan, []string) { + const skillRoot = "skills/.system/skill-creator" + files := []struct { + path string + sha string + size int64 + }{ + {"LICENSE.txt", "d645695673349e3947e8e5ae42332d0ac3164cd7", 11358}, + {"SKILL.md", "72bc0b97e7a6476254a9d5c424c9971748402ec3", 18664}, + {"agents/openai.yaml", "f5ce09ed118b72908b7308fcfc1619f59e89f6ea", 192}, + {"references/openai_yaml.md", "da5629f8de500ec6f81291d239a2d9c68d1d9044", 2130}, + {"scripts/generate_openai_yaml.py", "1a9d784f8704999daf55a2b9036094949d9af3f8", 6614}, + {"scripts/init_skill.py", "f90703eca8159a21a9684ed1df974c702d92f814", 14483}, + {"scripts/quick_validate.py", "0547b4041a5f58fa19892079a114a1df98286406", 3293}, + } + plan := PackagePlan{Root: skillRoot} + want := make([]string, 0, len(files)+1) + for _, file := range files { + plan.Files = append(plan.Files, PackageFile{ + RepositoryPath: skillRoot + "/" + file.path, RelativePath: file.path, + SHA: file.sha, Size: file.size, + }) + plan.TotalBytes += file.size + want = append(want, file.path) + } + want = append(want, evidenceManifestPath) + sort.Strings(want) + return plan, want +} + +// Two Skill roots in one fixed repository revision must use isolated package +// workspaces: closing the first package releases its Git objects before the +// second package starts. +func TestLiveAcquireTwoFixedPublicSkillsSequentially(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE") != "1" { + t.Skip("requires an explicit public GitHub smoke test") + } + const commitSHA = "49f948faa9258a0c61caceaf225e179651397431" + creator, _ := liveSkillCreatorPlan() + goal := liveDefineGoalPlan() + tempRoot := t.TempDir() + for _, plan := range []PackagePlan{creator, goal} { + artifact, err := AcquirePackage(context.Background(), "openai", "skills", commitSHA, + tempRoot, plan, AcquisitionLimits{ + MaxWorkspaceBytes: 512 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 128 << 20, + }) + if err != nil { + t.Fatalf("acquire %s: %v", plan.Root, err) + } + archive, err := zip.OpenReader(artifact.Path) + if err != nil { + t.Fatal(err) + } + if len(archive.File) != len(plan.Files)+1 { + t.Fatalf("%s: archive entries = %d, want %d", + plan.Root, len(archive.File), len(plan.Files)+1) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + workspaceBytes, err := directoryBytes(tempRoot) + if err != nil || workspaceBytes >= 32<<20 { + t.Fatalf("%s: unexpected workspace bytes %d: %v", plan.Root, workspaceBytes, err) + } + t.Logf("%s: archive=%d bytes, workspace after acquisition=%d bytes", + plan.Root, artifact.Size, workspaceBytes) + if err := artifact.Close(); err != nil { + t.Fatal(err) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("%s: temporary files remain after package cleanup: %v, %v", + plan.Root, remaining, err) + } + } +} + +func liveDefineGoalPlan() PackagePlan { + return PackagePlan{ + Root: "skills/.curated/define-goal", + Files: []PackageFile{ + {RepositoryPath: "skills/.curated/define-goal/LICENSE.txt", RelativePath: "LICENSE.txt", + SHA: "13e25df86ce06eb6488e6a6bc5c5847f5dedc352", Size: 10776}, + {RepositoryPath: "skills/.curated/define-goal/SKILL.md", RelativePath: "SKILL.md", + SHA: "87f111bd700e0d993465f7ac741847b5daee57d6", Size: 5647}, + {RepositoryPath: "skills/.curated/define-goal/agents/openai.yaml", RelativePath: "agents/openai.yaml", + SHA: "28c7e7a644d7ae3f5616c7ddd408c573139999f0", Size: 208}, + }, + TotalBytes: 16631, + } +} + +// Run in the production image with a 72 MiB /tmp tmpfs. Both packages use +// independent Git workspaces while the volume provides a hard physical limit. +func TestLiveAcquireTwoFixedPublicSkillsConcurrentlyOnSmallVolume(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CONCURRENT") != "1" { + t.Skip("requires an explicit concurrent public GitHub smoke test") + } + const commitSHA = "49f948faa9258a0c61caceaf225e179651397431" + tempRoot := t.TempDir() + var filesystem syscall.Statfs_t + if err := syscall.Statfs(tempRoot, &filesystem); err != nil { + t.Fatal(err) + } + volumeBytes := int64(filesystem.Blocks) * int64(filesystem.Bsize) + if volumeBytes > 80<<20 { + t.Fatalf("concurrent smoke test requires a small tmpfs: volume=%d bytes", volumeBytes) + } + creator, creatorFiles := liveSkillCreatorPlan() + goal := liveDefineGoalPlan() + type request struct { + owner, name, commit string + plan PackagePlan + } + requests := []request{ + {owner: "openai", name: "skills", commit: commitSHA, plan: creator}, + {owner: "openai", name: "skills", commit: commitSHA, plan: goal}, + } + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CROSS_REPOSITORY") == "1" { + goal = liveLargerPublicRepositoryPlan() + requests[1] = request{owner: "SangHynh", name: "agent-skills", + commit: "e325b0ee302fdf66abb1990ad7957e1a1c7aba5a", plan: goal} + } + expectedEntries := map[string]int{ + creator.Root: len(creatorFiles), + goal.Root: len(goal.Files) + 1, + } + limits := AcquisitionLimits{ + MaxWorkspaceBytes: 32 << 20, MaxArchiveBytes: 8 << 20, MinFreeBytes: 8 << 20, + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + type result struct { + root string + artifact *Artifact + err error + } + results := make(chan result, len(requests)) + start := make(chan struct{}) + for _, requested := range requests { + go func(requested request) { + <-start + artifact, err := AcquirePackage(ctx, requested.owner, requested.name, + requested.commit, tempRoot, requested.plan, limits) + results <- result{root: requested.plan.Root, artifact: artifact, err: err} + }(requested) + } + close(start) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + var sampledPeak int64 + var overlappingWorkspaces bool + artifacts := make([]*Artifact, 0, len(requests)) + for completed := 0; completed < len(requests); { + select { + case outcome := <-results: + completed++ + if outcome.err != nil { + t.Errorf("acquire %s: %v", outcome.root, outcome.err) + continue + } + artifacts = append(artifacts, outcome.artifact) + archive, err := zip.OpenReader(outcome.artifact.Path) + if err != nil { + t.Errorf("open %s archive: %v", outcome.root, err) + continue + } + if len(archive.File) != expectedEntries[outcome.root] { + t.Errorf("%s archive entries=%d, want=%d", outcome.root, + len(archive.File), expectedEntries[outcome.root]) + } + if err := archive.Close(); err != nil { + t.Errorf("close %s archive: %v", outcome.root, err) + } + case <-ticker.C: + if completed == 0 && len(results) == 0 { + if entries, err := os.ReadDir(tempRoot); err == nil && len(entries) == len(requests) { + overlappingWorkspaces = true + } + } + if used, err := directoryBytes(tempRoot); err == nil { + sampledPeak = max(sampledPeak, used) + } else { + t.Errorf("measure concurrent workspaces: %v", err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + if len(artifacts) != len(requests) { + t.Errorf("completed packages=%d, want=%d", len(artifacts), len(requests)) + } + if !overlappingWorkspaces { + t.Error("the two package workspaces did not overlap during acquisition") + } + if used, err := directoryBytes(tempRoot); err != nil || used > 64<<20 { + t.Errorf("concurrent workspaces exceeded total budget: bytes=%d error=%v", used, err) + } else { + sampledPeak = max(sampledPeak, used) + } + for _, artifact := range artifacts { + if err := artifact.Close(); err != nil { + t.Errorf("remove %s: %v", artifact.Path, err) + } + } + if remaining, err := os.ReadDir(tempRoot); err != nil || len(remaining) != 0 { + t.Errorf("concurrent workspaces were not cleared: entries=%d error=%v", len(remaining), err) + } + t.Logf("concurrent fixed commit packages=%d cross_repository=%t volume_bytes=%d sampled_peak_bytes=%d overlapping_workspaces=%t", + len(artifacts), os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_CROSS_REPOSITORY") == "1", + volumeBytes, sampledPeak, overlappingWorkspaces) +} + +// Fill both Git metadata directories near the configured aggregate budget. +// The fake Git executable avoids network variance while retaining the real +// workspace accounting and ZIP creation path. +func TestConcurrentAcquisitionNearSmallVolumeBudget(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_CONCURRENT_CAPACITY_TEST") != "1" { + t.Skip("requires an explicit small-volume capacity test") + } + const commitSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const mib = int64(1 << 20) + root := t.TempDir() + var filesystem syscall.Statfs_t + if err := syscall.Statfs(root, &filesystem); err != nil { + t.Fatal(err) + } + if volume := int64(filesystem.Blocks) * int64(filesystem.Bsize); volume > 80*mib { + t.Fatalf("capacity test requires a small tmpfs: volume=%d bytes", volume) + } + fakeBin := t.TempDir() + script := `#!/bin/sh +case " $* " in + *" checkout "*) + mkdir -p .git + printf '%s\n' "$SKILLHUB_TEST_COMMIT" > .git/HEAD + head -c 10000 /dev/zero > SKILL.md + dd if=/dev/zero of=.git/filler bs=1048576 count=28 2>/dev/null + sleep 0.2 + ;; +esac +` + if err := os.WriteFile(filepath.Join(fakeBin, "git"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("SKILLHUB_TEST_COMMIT", commitSHA) + if resolved, err := exec.LookPath("git"); err != nil || resolved != filepath.Join(fakeBin, "git") { + t.Fatalf("fake Git executable was not selected: %q, %v", resolved, err) + } + plan := PackagePlan{Root: ".", TotalBytes: 10000, Files: []PackageFile{{ + RepositoryPath: "SKILL.md", RelativePath: "SKILL.md", SHA: strings.Repeat("b", 40), Size: 10000, + }}} + limits := AcquisitionLimits{MaxWorkspaceBytes: 32 * mib, + MaxArchiveBytes: mib, MinFreeBytes: 8 * mib} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + results := make(chan struct { + artifact *Artifact + err error + }, 2) + start := make(chan struct{}) + for range 2 { + go func() { + <-start + artifact, err := AcquirePackage(ctx, "openai", "skills", commitSHA, root, plan, limits) + results <- struct { + artifact *Artifact + err error + }{artifact, err} + }() + } + close(start) + artifacts := make([]*Artifact, 0, 2) + for range 2 { + select { + case outcome := <-results: + if outcome.err != nil { + t.Errorf("near-budget acquisition failed: %v", outcome.err) + } else { + artifacts = append(artifacts, outcome.artifact) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + used, err := directoryBytes(root) + if err != nil || used < 56*mib || used > 64*mib { + t.Errorf("unexpected aggregate workspace occupancy: bytes=%d error=%v", used, err) + } + if err := syscall.Statfs(root, &filesystem); err != nil || + int64(filesystem.Bavail)*int64(filesystem.Bsize) < 8*mib { + t.Errorf("temporary filesystem lost its 8 MiB reserve: %v", err) + } + for _, artifact := range artifacts { + if err := artifact.Close(); err != nil { + t.Errorf("clean near-budget package: %v", err) + } + } + if remaining, err := os.ReadDir(root); err != nil || len(remaining) != 0 { + t.Errorf("near-budget workspaces remain: entries=%d error=%v", len(remaining), err) + } + t.Logf("near-budget concurrent packages=%d aggregate_workspace_bytes=%d", + len(artifacts), used) +} + +func TestWorkspaceChecksDirectorySizeAndFreeSpace(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "object"), make([]byte, 32), 0600); err != nil { + t.Fatal(err) + } + if err := checkWorkspace(root, 31, 0); !errors.Is(err, ErrLocalDiskBudget) { + t.Fatalf("workspace over its byte limit: %v", err) + } + if err := checkWorkspace(root, 32, 0); err != nil { + t.Fatalf("workspace at its byte limit: %v", err) + } + var stats syscall.Statfs_t + if err := syscall.Statfs(root, &stats); err != nil { + t.Fatal(err) + } + available := uint64(stats.Bavail) * uint64(stats.Bsize) + if available > math.MaxInt64-1 { + t.Skip("filesystem is too large for an int64 free-space boundary") + } + if err := requireMinimumFree(root, int64(available)+1); !errors.Is(err, ErrLocalDiskBudget) { + t.Fatalf("minimum free space was not protected: %v", err) + } +} + +func TestDirectoryBytesDoesNotIgnoreMissingWorkspace(t *testing.T) { + _, err := directoryBytes(filepath.Join(t.TempDir(), "missing-workspace")) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing workspace must fail accounting: %v", err) + } +} + +func TestAcquirePackageRejectsCapacityBeforeNetworkFetch(t *testing.T) { + const mib = int64(1 << 20) + plan := PackagePlan{ + Root: ".", TotalBytes: 20 * mib, + Files: []PackageFile{{RepositoryPath: "SKILL.md", RelativePath: "SKILL.md", + SHA: skillBlobSHA, Size: 20 * mib}}, + } + artifact, err := AcquirePackage(context.Background(), "owner", "repo", rootTreeSHA, + t.TempDir(), plan, AcquisitionLimits{MaxWorkspaceBytes: 64 * mib, + MaxArchiveBytes: 24 * mib, MinFreeBytes: 0}) + if artifact != nil || !errors.Is(err, ErrCapacity) { + t.Fatalf("expected pre-fetch capacity block, got artifact=%v, error=%v", artifact, err) + } +} + +func TestSparseCheckoutKeepsOnlySelectedSkillFile(t *testing.T) { + root := t.TempDir() + repoDir := filepath.Join(root, "repo") + for _, directory := range []string{filepath.Join(repoDir, "selected"), filepath.Join(repoDir, "sibling")} { + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + for _, path := range []string{"selected/SKILL.md", "sibling/SKILL.md"} { + if err := os.WriteFile(filepath.Join(repoDir, path), []byte("name: fixture\n"), 0600); err != nil { + t.Fatal(err) + } + } + for _, arguments := range [][]string{ + {"init", "--quiet"}, {"add", "."}, + {"-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "-m", "fixture"}, + } { + command := exec.Command("git", arguments...) + command.Dir = repoDir + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git fixture setup failed: %v: %s", err, output) + } + } + if err := runBoundedGit(context.Background(), repoDir, root, 64<<20, 0, + "sparse-checkout", "set", "--no-cone", "--", sparseFilePattern("selected/SKILL.md")); err != nil { + t.Fatal(err) + } + if err := runBoundedGit(context.Background(), repoDir, root, 64<<20, 0, + "checkout", "--quiet", "--detach", "--force", "HEAD"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(repoDir, "selected/SKILL.md")); err != nil { + t.Fatalf("planned Skill file missing: %v", err) + } + if _, err := os.Stat(filepath.Join(repoDir, "sibling/SKILL.md")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unplanned sibling file was checked out: %v", err) + } +} diff --git a/github-sync-go/internal/repository/capacity_test.go b/github-sync-go/internal/repository/capacity_test.go new file mode 100644 index 000000000..4eff3ce9e --- /dev/null +++ b/github-sync-go/internal/repository/capacity_test.go @@ -0,0 +1,69 @@ +package repository + +import ( + "context" + "fmt" + "runtime" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type capacityTreeReader struct { + tree github.Tree + calls int +} + +func (reader *capacityTreeReader) GetTree(_ context.Context, _, _, sha string, + _ bool) (github.Tree, github.Quota, error) { + reader.calls++ + if sha != reader.tree.SHA { + return github.Tree{}, github.Quota{}, fmt.Errorf("unexpected tree SHA") + } + return reader.tree, github.Quota{}, nil +} + +func TestSyntheticRepositoryMetadataCapacity(t *testing.T) { + const repositories = 1000 + const skillsPerRepository = 10 + const expectedSkills = repositories * skillsPerRepository + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + started := time.Now() + requests := 0 + plans := 0 + for repositoryIndex := range repositories { + sha := fmt.Sprintf("%040x", repositoryIndex+1) + entries := make([]github.TreeEntry, 0, skillsPerRepository) + for skillIndex := range skillsPerRepository { + entries = append(entries, github.TreeEntry{ + Path: fmt.Sprintf("skills/skill-%02d/SKILL.md", skillIndex), + Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 64, + }) + } + reader := &capacityTreeReader{tree: github.Tree{SHA: sha, Entries: entries}} + snapshot, err := EnumerateTree(context.Background(), reader, "fixture", "repo", sha, + TreeLimits{MaxTreeRequests: 8, MaxEntries: 100_000}) + if err != nil || len(snapshot.IncompleteSubtrees) != 0 { + t.Fatalf("repository %d metadata failed: snapshot=%+v, error=%v", repositoryIndex, snapshot, err) + } + roots := RootsFromSnapshot(snapshot, nil) + preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{ + MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + }) + if err != nil || len(roots.Paths) != skillsPerRepository || + len(preflight.Plans) != skillsPerRepository || len(preflight.Failures) != 0 { + t.Fatalf("repository %d preflight failed: roots=%d, plans=%d, failures=%d, error=%v", + repositoryIndex, len(roots.Paths), len(preflight.Plans), len(preflight.Failures), err) + } + requests += reader.calls + plans += len(preflight.Plans) + } + runtime.ReadMemStats(&after) + if requests != repositories || plans != expectedSkills { + t.Fatalf("unexpected synthetic coverage: requests=%d, plans=%d", requests, plans) + } + t.Logf("synthetic metadata baseline: repositories=%d, skill paths=%d, tree requests=%d, elapsed=%s, allocated=%d bytes", + repositories, plans, requests, time.Since(started), after.TotalAlloc-before.TotalAlloc) +} diff --git a/github-sync-go/internal/repository/official_quota_test.go b/github-sync-go/internal/repository/official_quota_test.go new file mode 100644 index 000000000..36591980e --- /dev/null +++ b/github-sync-go/internal/repository/official_quota_test.go @@ -0,0 +1,58 @@ +package repository + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestOfficialGitQuotaPolicy(t *testing.T) { + for _, kind := range []string{"blocked-before-start", "fetch-report-failure", "checkout-report-failure", "local-init"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "git"), []byte("#!/bin/sh\nexit 0\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + before, after := 0, 0 + denied := errors.New("quota policy denied") + ctx := WithGitRequestPolicy(context.Background(), func(context.Context) error { + before++ + if kind == "blocked-before-start" { + return denied + } + return nil + }, func(context.Context, error) error { after++; return denied }) + command := "fetch" + if kind == "checkout-report-failure" { + command = "checkout" + } + if kind == "local-init" { + command = "init" + } + err := runBoundedGit(ctx, dir, dir, 1<<20, 0, command) + if before != 1 { + t.Fatalf("before=%d", before) + } + if kind == "local-init" { + if err != nil || after != 0 { + t.Fatal("local init fabricated quota observation") + } + return + } + if !errors.Is(err, denied) { + t.Fatalf("policy failure swallowed: %v", err) + } + wantAfter := 1 + if kind == "blocked-before-start" { + wantAfter = 0 + } + if after != wantAfter { + t.Fatalf("after=%d want=%d", after, wantAfter) + } + }) + } +} diff --git a/github-sync-go/internal/repository/package_boundaries_test.go b/github-sync-go/internal/repository/package_boundaries_test.go new file mode 100644 index 000000000..6df5f3389 --- /dev/null +++ b/github-sync-go/internal/repository/package_boundaries_test.go @@ -0,0 +1,74 @@ +package repository + +import ( + "fmt" + "reflect" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +func TestSourceIgnoreRemovalRestoresOnlySelectedSkill(t *testing.T) { + snapshot := TreeSnapshot{Entries: []github.TreeEntry{}} + for _, path := range []string{"skills/custom/SKILL.md", "skills/good/SKILL.md", "docs/demo/SKILL.md", "examples/demo/SKILL.md", "tests/demo/SKILL.md"} { + snapshot.Entries = append(snapshot.Entries, github.TreeEntry{Path: path, Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}) + } + for _, testcase := range []struct { + name string + ignores []string + want []string + }{ + {"ignored", []string{"skills/custom"}, []string{"docs/demo", "examples/demo", "skills/good"}}, + {"restored", nil, []string{"docs/demo", "examples/demo", "skills/custom", "skills/good"}}, + } { + t.Run(testcase.name, func(t *testing.T) { + roots := RootsFromSnapshot(snapshot, testcase.ignores) + if !reflect.DeepEqual(roots.Paths, testcase.want) { + t.Fatalf("roots=%v want=%v", roots.Paths, testcase.want) + } + }) + } +} + +func TestPackagePolicyExactBoundariesAndSubmoduleIsolation(t *testing.T) { + limits := PackageLimits{MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20} + for _, tc := range []struct { + name string + files int + size int64 + submodule bool + wantFailure string + }{ + {"100-files", 100, 1, false, ""}, {"101-files", 101, 1, false, "PACKAGE_TOO_LARGE"}, + {"10MiB-file", 1, 10 << 20, false, ""}, {"10MiB-plus-one", 1, (10 << 20) + 1, false, "PACKAGE_TOO_LARGE"}, + {"100MiB-package", 10, 10 << 20, false, ""}, {"110MiB-package", 11, 10 << 20, false, "PACKAGE_TOO_LARGE"}, + {"submodule", 1, 1, true, "SUBMODULE"}, + } { + t.Run(tc.name, func(t *testing.T) { + entries := []github.TreeEntry{{Path: "good/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}} + for i := 0; i < tc.files; i++ { + name := fmt.Sprintf("file-%03d.txt", i) + if i == 0 { + name = "SKILL.md" + } + entries = append(entries, github.TreeEntry{Path: "selected/" + name, Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: tc.size}) + } + if tc.submodule { + entries = append(entries, github.TreeEntry{Path: "selected/vendor", Type: "commit", Mode: "160000", SHA: skillBlobSHA}) + } + result, err := PreflightPackages(TreeSnapshot{Entries: entries}, []string{"good", "selected"}, limits) + if err != nil { + t.Fatal(err) + } + if tc.wantFailure == "" { + if len(result.Plans) != 2 || len(result.Failures) != 0 { + t.Fatalf("boundary rejected: %+v", result) + } + } else { + if len(result.Plans) != 1 || result.Plans[0].Root != "good" || len(result.Failures) != 1 || result.Failures[0].Code != tc.wantFailure || result.Failures[0].Retryable { + t.Fatalf("failure not isolated: %+v", result) + } + } + }) + } +} diff --git a/github-sync-go/internal/repository/package_plan.go b/github-sync-go/internal/repository/package_plan.go new file mode 100644 index 000000000..be6b4f8ca --- /dev/null +++ b/github-sync-go/internal/repository/package_plan.go @@ -0,0 +1,226 @@ +package repository + +import ( + "errors" + "regexp" + "sort" + "strings" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type PackageLimits struct { + MaxFiles int + MaxSingleFileBytes int64 + MaxPackageBytes int64 +} + +type PackageFile struct { + RepositoryPath string + RelativePath string + SHA string + Size int64 +} + +type PackagePlan struct { + Root string + Files []PackageFile + Evidence []RootEvidenceFile + TotalBytes int64 +} + +// RootEvidenceFile is repository-level license or notice metadata at the fixed commit. +// Large files keep their Git blob digest in the manifest without being downloaded. +type RootEvidenceFile struct { + RepositoryPath string `json:"path"` + SHA string `json:"gitSha"` + Size int64 `json:"size"` + Bundled bool `json:"bundled"` +} + +const ( + maxRootEvidenceFiles = 16 + maxEvidenceFileBytes = 64 << 10 + maxEvidenceTotalBytes = 256 << 10 +) + +var rootEvidenceName = regexp.MustCompile(`(?i)^(license|licence|copying|notice|copyright)(?:[-_.][a-z0-9][a-z0-9._-]*)?$`) + +type PackageFailure struct { + Root string + Code string + Retryable bool +} + +type PackagePreflight struct { + Plans []PackagePlan + Failures []PackageFailure +} + +type packageAccumulator struct { + plan PackagePlan + seen map[string]struct{} + hasSkillMD bool + failure *PackageFailure +} + +// PreflightPackages assigns every blob to its nearest Skill root. Ignored +// candidate directories are still counted when they belong to a parent Skill. +// An incomplete range never produces a package plan for an affected root. +func PreflightPackages(snapshot TreeSnapshot, roots []string, limits PackageLimits) (PackagePreflight, error) { + if limits.MaxFiles < 1 || limits.MaxSingleFileBytes < 1 || limits.MaxPackageBytes < 1 { + return PackagePreflight{}, errors.New("package limits must be positive") + } + packages := make(map[string]*packageAccumulator, len(roots)) + for _, root := range roots { + if root != "." && !validPath(root) { + return PackagePreflight{}, errors.New("invalid Skill root") + } + if _, exists := packages[root]; exists { + return PackagePreflight{}, errors.New("duplicate Skill root") + } + packages[root] = &packageAccumulator{ + plan: PackagePlan{Root: root}, seen: make(map[string]struct{}), + } + } + for root, accumulator := range packages { + for _, subtree := range snapshot.IncompleteSubtrees { + if subtree != "." && !validPath(subtree) { + return PackagePreflight{}, errors.New("invalid incomplete subtree") + } + if subtree == "." || root == "." || subtree == root || + strings.HasPrefix(subtree, root+"/") || strings.HasPrefix(root, subtree+"/") { + accumulator.failure = &PackageFailure{Root: root, Code: "TREE_INCOMPLETE", Retryable: true} + break + } + } + } + entries := append([]github.TreeEntry(nil), snapshot.Entries...) + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + evidence, evidenceFailure := planRootEvidence(entries) + for _, entry := range entries { + if !validPath(entry.Path) { + return PackagePreflight{}, errors.New("invalid repository tree path") + } + root := nearestSkillRoot(entry.Path, packages) + if root == "" { + continue + } + accumulator := packages[root] + if accumulator.failure != nil || entry.Type == "tree" { + continue + } + relativePath := entry.Path + if root != "." { + relativePath = strings.TrimPrefix(entry.Path, root+"/") + } + if _, duplicate := accumulator.seen[relativePath]; duplicate { + accumulator.failure = &PackageFailure{Root: root, Code: "DUPLICATE_PATH"} + continue + } + accumulator.seen[relativePath] = struct{}{} + if relativePath == evidenceManifestPath || strings.HasPrefix(relativePath, evidenceEntryPrefix) { + accumulator.failure = &PackageFailure{Root: root, Code: "RESERVED_EVIDENCE_PATH"} + continue + } + switch { + case entry.Type == "commit" || entry.Mode == "160000": + accumulator.failure = &PackageFailure{Root: root, Code: "SUBMODULE"} + case entry.Type == "blob" && entry.Mode == "120000": + accumulator.failure = &PackageFailure{Root: root, Code: "SYMLINK"} + case entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755"): + accumulator.failure = &PackageFailure{Root: root, Code: "UNSUPPORTED_ENTRY"} + case entry.Size < 0 || !validGitSHA(entry.SHA): + accumulator.failure = &PackageFailure{Root: root, Code: "INVALID_BLOB_METADATA", Retryable: true} + case entry.Size > limits.MaxSingleFileBytes || + entry.Size > limits.MaxPackageBytes-accumulator.plan.TotalBytes || + len(accumulator.plan.Files) >= limits.MaxFiles: + accumulator.failure = &PackageFailure{Root: root, Code: "PACKAGE_TOO_LARGE"} + default: + accumulator.plan.Files = append(accumulator.plan.Files, PackageFile{ + RepositoryPath: entry.Path, RelativePath: relativePath, + SHA: entry.SHA, Size: entry.Size, + }) + accumulator.plan.TotalBytes += entry.Size + if relativePath == "SKILL.md" { + accumulator.hasSkillMD = true + } + } + } + result := PackagePreflight{} + for _, root := range sortedAccumulatorRoots(packages) { + accumulator := packages[root] + if accumulator.failure != nil { + result.Failures = append(result.Failures, *accumulator.failure) + } else if evidenceFailure != "" { + result.Failures = append(result.Failures, + PackageFailure{Root: root, Code: evidenceFailure}) + } else if !accumulator.hasSkillMD { + result.Failures = append(result.Failures, + PackageFailure{Root: root, Code: "MISSING_SKILL_MD"}) + } else { + accumulator.plan.Evidence = append([]RootEvidenceFile(nil), evidence...) + result.Plans = append(result.Plans, accumulator.plan) + } + } + return result, nil +} + +func planRootEvidence(entries []github.TreeEntry) ([]RootEvidenceFile, string) { + result := make([]RootEvidenceFile, 0) + var bundledBytes int64 + for _, entry := range entries { + if strings.Contains(entry.Path, "/") || !rootEvidenceName.MatchString(entry.Path) { + continue + } + if len(result) >= maxRootEvidenceFiles { + return nil, "TOO_MANY_ROOT_EVIDENCE_FILES" + } + if entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755") || + entry.Size < 0 || !validGitSHA(entry.SHA) { + return nil, "UNSAFE_ROOT_EVIDENCE" + } + bundled := entry.Size <= maxEvidenceFileBytes && + entry.Size <= maxEvidenceTotalBytes-bundledBytes + if bundled { + bundledBytes += entry.Size + } + result = append(result, RootEvidenceFile{ + RepositoryPath: entry.Path, SHA: entry.SHA, + Size: entry.Size, Bundled: bundled, + }) + } + return result, "" +} + +func nearestSkillRoot(path string, packages map[string]*packageAccumulator) string { + for directory := parent(path); ; directory = parent(directory) { + if _, exists := packages[directory]; exists { + return directory + } + if directory == "." { + return "" + } + } +} + +func sortedAccumulatorRoots(packages map[string]*packageAccumulator) []string { + roots := make([]string, 0, len(packages)) + for root := range packages { + roots = append(roots, root) + } + sort.Strings(roots) + return roots +} + +func validGitSHA(value string) bool { + if len(value) != 40 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} diff --git a/github-sync-go/internal/repository/package_plan_test.go b/github-sync-go/internal/repository/package_plan_test.go new file mode 100644 index 000000000..cdd6eb39a --- /dev/null +++ b/github-sync-go/internal/repository/package_plan_test.go @@ -0,0 +1,85 @@ +package repository + +import ( + "reflect" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +func TestIgnoredCandidateStaysInParentPackageAndNestedSkillIsSeparate(t *testing.T) { + snapshot := TreeSnapshot{Entries: []github.TreeEntry{ + {Path: "skills/parent/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/parent/test/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/parent/test/example.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4}, + {Path: "skills/parent/child/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/parent/child/example.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4}, + }} + roots := RootsFromSnapshot(snapshot, nil) + if !reflect.DeepEqual(roots.Paths, []string{"skills/parent", "skills/parent/child"}) { + t.Fatalf("unexpected Skill roots: %v", roots.Paths) + } + preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{ + MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + }) + if err != nil || len(preflight.Failures) != 0 || len(preflight.Plans) != 2 { + t.Fatalf("unexpected package preflight: %+v, error=%v", preflight, err) + } + if got := packagePaths(preflight.Plans[0]); !reflect.DeepEqual(got, + []string{"SKILL.md", "test/SKILL.md", "test/example.txt"}) { + t.Fatalf("parent package paths = %v", got) + } + if got := packagePaths(preflight.Plans[1]); !reflect.DeepEqual(got, + []string{"SKILL.md", "example.txt"}) { + t.Fatalf("child package paths = %v", got) + } +} + +func TestDefaultAndSourceIgnoredDirectoriesDoNotBecomeSkillRoots(t *testing.T) { + ignored := []string{ + ".git", "node_modules", ".venv", "venv", "dist", "build", + "target", "__pycache__", "test", "tests", + } + if len(defaultIgnoredSegments) != len(ignored) { + t.Fatalf("default ignored directory count = %d", len(defaultIgnoredSegments)) + } + snapshot := TreeSnapshot{Entries: []github.TreeEntry{ + {Path: "skills/good/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/custom/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + }} + for _, segment := range ignored { + snapshot.Entries = append(snapshot.Entries, github.TreeEntry{ + Path: "skills/" + segment + "/SKILL.md", Type: "blob", + Mode: "100644", SHA: skillBlobSHA, Size: 12, + }) + } + roots := RootsFromSnapshot(snapshot, []string{"skills/custom"}) + if !reflect.DeepEqual(roots.Paths, []string{"skills/good"}) { + t.Fatalf("ignored directories became Skill roots: %v", roots.Paths) + } +} + +func TestUnsafeChildPackageDoesNotBlockParent(t *testing.T) { + snapshot := TreeSnapshot{Entries: []github.TreeEntry{ + {Path: "skills/parent/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/parent/child/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + {Path: "skills/parent/child/link", Type: "blob", Mode: "120000", SHA: skillBlobSHA, Size: 5}, + }} + roots := RootsFromSnapshot(snapshot, nil) + preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{ + MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + }) + if err != nil || len(preflight.Plans) != 1 || preflight.Plans[0].Root != "skills/parent" || + len(preflight.Failures) != 1 || preflight.Failures[0].Root != "skills/parent/child" || + preflight.Failures[0].Code != "SYMLINK" { + t.Fatalf("unsafe child isolation failed: %+v, error=%v", preflight, err) + } +} + +func packagePaths(plan PackagePlan) []string { + paths := make([]string, 0, len(plan.Files)) + for _, file := range plan.Files { + paths = append(paths, file.RelativePath) + } + return paths +} diff --git a/github-sync-go/internal/repository/platform_control_test.go b/github-sync-go/internal/repository/platform_control_test.go new file mode 100644 index 000000000..d128b1ea5 --- /dev/null +++ b/github-sync-go/internal/repository/platform_control_test.go @@ -0,0 +1,20 @@ +package repository + +import ( + "context" + "errors" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +func TestPlatformControlFencesGitStepsWithoutStartingSubprocess(t *testing.T) { + stopped := errors.New("platform paused") + ctx := github.WithAcquisitionControl(context.Background(), func(context.Context) error { return stopped }) + ctx = WithGitRequestPolicy(ctx, func(context.Context) error { t.Fatal("quota policy ran after platform stopped"); return nil }, nil) + for _, step := range []string{"fetch", "sparse-checkout", "checkout"} { + if err := runBoundedGit(ctx, "does-not-exist", "does-not-exist", 1, 0, step); !errors.Is(err, stopped) { + t.Fatalf("%s: %v", step, err) + } + } +} diff --git a/github-sync-go/internal/repository/root_evidence_boundaries_test.go b/github-sync-go/internal/repository/root_evidence_boundaries_test.go new file mode 100644 index 000000000..60218cac3 --- /dev/null +++ b/github-sync-go/internal/repository/root_evidence_boundaries_test.go @@ -0,0 +1,80 @@ +package repository + +import ( + "fmt" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +func TestRootEvidenceBoundariesPreserveDigestWithoutUnboundedBundling(t *testing.T) { + for _, tc := range []struct { + name string + sizes []int64 + wantBundled int + wantFailure string + }{ + {"single-at-64KiB", []int64{64 << 10}, 1, ""}, + {"single-over-64KiB", []int64{(64 << 10) + 1}, 0, ""}, + {"total-at-256KiB", []int64{64 << 10, 64 << 10, 64 << 10, 64 << 10}, 4, ""}, + {"total-over-256KiB", []int64{64 << 10, 64 << 10, 64 << 10, 64 << 10, 1}, 4, ""}, + {"16-evidence-files", make([]int64, 16), 16, ""}, + {"17-evidence-files", make([]int64, 17), 0, "TOO_MANY_ROOT_EVIDENCE_FILES"}, + } { + t.Run(tc.name, func(t *testing.T) { + entries := []github.TreeEntry{{Path: "skills/demo/SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}} + for i, size := range tc.sizes { + entries = append(entries, github.TreeEntry{Path: fmt.Sprintf("LICENSE-%02d", i), Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: size}) + } + // Unrelated root files are not license evidence and do not consume its budget. + entries = append(entries, github.TreeEntry{Path: "README.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 1}) + result, err := PreflightPackages(TreeSnapshot{Entries: entries}, []string{"skills/demo"}, PackageLimits{MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20}) + if err != nil { + t.Fatal(err) + } + if tc.wantFailure != "" { + if len(result.Plans) != 0 || len(result.Failures) != 1 || result.Failures[0].Code != tc.wantFailure { + t.Fatalf("evidence limit must block plan: %+v", result) + } + return + } + if len(result.Plans) != 1 || len(result.Failures) != 0 { + t.Fatalf("unexpected plan: %+v", result) + } + plan := result.Plans[0] + if len(plan.Evidence) != len(tc.sizes) || len(plan.Files) != 1 || plan.TotalBytes != 1 { + t.Fatalf("evidence metadata lost or conflated with Skill files: %+v", plan) + } + bundled := 0 + var bundledBytes int64 + for i, evidence := range plan.Evidence { + if evidence.RepositoryPath != fmt.Sprintf("LICENSE-%02d", i) || evidence.SHA != skillBlobSHA || evidence.Size != tc.sizes[i] { + t.Fatalf("unbundled evidence must retain path, digest and size: %+v", evidence) + } + if evidence.Bundled { + bundled++ + bundledBytes += evidence.Size + } + } + if bundled != tc.wantBundled || bundledBytes > 256<<10 { + t.Fatalf("bundled=%d bytes=%d want=%d", bundled, bundledBytes, tc.wantBundled) + } + }) + } +} + +func TestRootLicenseAndNoticeAreBothRetained(t *testing.T) { + entries := []github.TreeEntry{ + {Path: "LICENSE", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 4}, + {Path: "NOTICE.txt", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 5}, + } + evidence, failure := planRootEvidence(entries) + if failure != "" || len(evidence) != 2 { + t.Fatalf("license and notice not retained: %v %s", evidence, failure) + } + for i, file := range evidence { + if file.RepositoryPath != entries[i].Path || file.SHA != skillBlobSHA || !file.Bundled || file.Size != entries[i].Size { + t.Fatalf("invalid evidence: %+v", file) + } + } +} diff --git a/github-sync-go/internal/repository/tree.go b/github-sync-go/internal/repository/tree.go new file mode 100644 index 000000000..b57077d2c --- /dev/null +++ b/github-sync-go/internal/repository/tree.go @@ -0,0 +1,208 @@ +package repository + +import ( + "context" + "errors" + "sort" + "strings" + "unicode" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +var defaultIgnoredSegments = map[string]struct{}{ + ".git": {}, "node_modules": {}, ".venv": {}, "venv": {}, "dist": {}, + "build": {}, "target": {}, "__pycache__": {}, "test": {}, "tests": {}, +} + +type TreeReader interface { + GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error) +} + +type TreeLimits struct { + MaxTreeRequests int + MaxEntries int +} + +type SkillRoots struct { + Paths []string + IncompleteSubtrees []string + EntriesExamined int +} + +type TreeSnapshot struct { + Entries []github.TreeEntry + IncompleteSubtrees []string + EntriesExamined int +} + +type pendingTree struct { + SHA string + Prefix string +} + +// EnumerateTree retains file metadata for package preflight. Truncated recursive +// trees are rewalked one subtree at a time; failed siblings are reported explicitly. +func EnumerateTree(ctx context.Context, reader TreeReader, owner, name, rootTreeSHA string, + limits TreeLimits) (TreeSnapshot, error) { + if limits.MaxTreeRequests < 1 || limits.MaxEntries < 1 { + return TreeSnapshot{}, errors.New("tree request and entry limits must be positive") + } + root, _, err := reader.GetTree(ctx, owner, name, rootTreeSHA, true) + if err != nil && !errors.Is(err, github.ErrResponseTooLarge) { + return TreeSnapshot{}, err + } + if err == nil && !root.Truncated && len(root.Entries) <= limits.MaxEntries { + result := TreeSnapshot{EntriesExamined: len(root.Entries)} + for _, entry := range root.Entries { + if !validPath(entry.Path) { + result.IncompleteSubtrees = []string{"."} + continue + } + result.Entries = append(result.Entries, entry) + } + return result, nil + } + result := TreeSnapshot{} + incomplete := make(map[string]struct{}) + queue := []pendingTree{{SHA: rootTreeSHA, Prefix: "."}} + requests := 1 + for len(queue) > 0 { + if err := ctx.Err(); err != nil { + for _, left := range queue { + incomplete[left.Prefix] = struct{}{} + } + result.IncompleteSubtrees = sortedKeys(incomplete) + return result, err + } + current := queue[0] + queue = queue[1:] + if requests >= limits.MaxTreeRequests || result.EntriesExamined >= limits.MaxEntries { + incomplete[current.Prefix] = struct{}{} + for _, left := range queue { + incomplete[left.Prefix] = struct{}{} + } + break + } + tree, _, treeErr := reader.GetTree(ctx, owner, name, current.SHA, false) + requests++ + if apiError, ok := treeErr.(*github.APIError); ok && + (apiError.Code == "GITHUB_RATE_LIMITED" || apiError.Code == "AUTH_INVALID") { + incomplete[current.Prefix] = struct{}{} + for _, left := range queue { + incomplete[left.Prefix] = struct{}{} + } + result.IncompleteSubtrees = sortedKeys(incomplete) + return result, treeErr + } + if treeErr != nil || tree.Truncated { + incomplete[current.Prefix] = struct{}{} + continue + } + if len(tree.Entries) > limits.MaxEntries-result.EntriesExamined { + incomplete[current.Prefix] = struct{}{} + continue + } + for _, entry := range tree.Entries { + path := entry.Path + if current.Prefix != "." { + path = current.Prefix + "/" + path + } + result.EntriesExamined++ + if !validPath(path) { + incomplete[current.Prefix] = struct{}{} + continue + } + entry.Path = path + result.Entries = append(result.Entries, entry) + if entry.Type == "tree" { + queue = append(queue, pendingTree{SHA: entry.SHA, Prefix: path}) + } + } + } + result.IncompleteSubtrees = sortedKeys(incomplete) + sort.Slice(result.Entries, func(i, j int) bool { + return result.Entries[i].Path < result.Entries[j].Path + }) + return result, nil +} + +// DiscoverSkillRoots treats incomplete ranges as unknown. A candidate inside an +// ignored directory is skipped, but that directory's files remain available to +// a parent Skill package. +func DiscoverSkillRoots(ctx context.Context, reader TreeReader, owner, name, rootTreeSHA string, + limits TreeLimits, addedIgnoredPrefixes []string) (SkillRoots, error) { + for _, prefix := range addedIgnoredPrefixes { + if !validPath(prefix) || prefix == "." { + return SkillRoots{}, errors.New("invalid additional ignored path prefix") + } + } + snapshot, err := EnumerateTree(ctx, reader, owner, name, rootTreeSHA, limits) + return RootsFromSnapshot(snapshot, addedIgnoredPrefixes), err +} + +func RootsFromSnapshot(snapshot TreeSnapshot, addedIgnoredPrefixes []string) SkillRoots { + result := SkillRoots{ + IncompleteSubtrees: snapshot.IncompleteSubtrees, + EntriesExamined: snapshot.EntriesExamined, + } + roots := make(map[string]struct{}) + for _, entry := range snapshot.Entries { + if candidateRoot(entry.Path, entry, addedIgnoredPrefixes) { + roots[parent(entry.Path)] = struct{}{} + } + } + result.Paths = sortedKeys(roots) + return result +} + +func candidateRoot(path string, entry github.TreeEntry, addedIgnoredPrefixes []string) bool { + return entry.Type == "blob" && (path == "SKILL.md" || strings.HasSuffix(path, "/SKILL.md")) && + !ignoredDirectory(parent(path), addedIgnoredPrefixes) +} + +func ignoredDirectory(path string, addedIgnoredPrefixes []string) bool { + if path == "." { + return false + } + for _, segment := range strings.Split(path, "/") { + if _, ignored := defaultIgnoredSegments[segment]; ignored { + return true + } + } + for _, prefix := range addedIgnoredPrefixes { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} + +func validPath(path string) bool { + if path == "" || path == "." || strings.HasPrefix(path, "/") || strings.HasSuffix(path, "/") || + strings.Contains(path, "\\") || strings.IndexFunc(path, unicode.IsControl) >= 0 { + return false + } + for _, segment := range strings.Split(path, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + +func parent(path string) string { + if cut := strings.LastIndexByte(path, '/'); cut >= 0 { + return path[:cut] + } + return "." +} + +func sortedKeys(values map[string]struct{}) []string { + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Strings(result) + return result +} diff --git a/github-sync-go/internal/repository/tree_live_test.go b/github-sync-go/internal/repository/tree_live_test.go new file mode 100644 index 000000000..0a0bda5cc --- /dev/null +++ b/github-sync-go/internal/repository/tree_live_test.go @@ -0,0 +1,261 @@ +package repository + +import ( + "archive/zip" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "os/exec" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type countedLiveTreeReader struct { + client *github.Client + calls int +} + +const fixedPublicCommitSHA = "49f948faa9258a0c61caceaf225e179651397431" + +func loadFixedPublicTree(t *testing.T, ctx context.Context) (TreeSnapshot, int) { + t.Helper() + anonymous := os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_ANONYMOUS") == "1" + token := "" + if !anonymous { + token = os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TOKEN") + if token == "" { + output, err := exec.CommandContext(ctx, "gh", "auth", "token").Output() + if err != nil { + t.Fatal("no configured GitHub test token or usable local gh authentication") + } + token = strings.TrimSpace(string(output)) + } + if token == "" { + t.Fatal("GitHub test token is empty") + } + } + client := github.NewClient(token) + metadata, _, err := client.GetRepository(ctx, "openai", "skills") + if err != nil { + t.Fatal(err) + } + if metadata.ID != 1103643573 || metadata.Private || metadata.Owner.Login != "openai" { + t.Fatal("public repository metadata differs from the fixed fixture") + } + treeSHA, _, err := client.GetCommitTreeSHA(ctx, metadata.Owner.Login, + metadata.Name, fixedPublicCommitSHA) + if err != nil { + t.Fatal(err) + } + if treeSHA != "224b363928698e143876aef008232ed73d8e6f81" { + t.Fatal("fixed commit resolved to an unexpected tree") + } + reader := &countedLiveTreeReader{client: client} + snapshot, err := EnumerateTree(ctx, reader, metadata.Owner.Login, metadata.Name, + treeSHA, TreeLimits{MaxTreeRequests: 10, MaxEntries: 5000}) + if err != nil { + t.Fatal(err) + } + return snapshot, 2 + reader.calls +} + +func (reader *countedLiveTreeReader) GetTree(ctx context.Context, owner, name, sha string, + recursive bool) (github.Tree, github.Quota, error) { + reader.calls++ + return reader.client.GetTree(ctx, owner, name, sha, recursive) +} + +// This opt-in test checks the real GitHub REST metadata path for a fixed public +// commit. SKILLHUB_GITHUB_SYNC_LIVE_ANONYMOUS=1 avoids local gh credentials; +// otherwise any credential stays in process memory and is never included in output. +func TestLiveEnumerateFixedPublicGitHubTree(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TREE") != "1" { + t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_TREE=1 to query the public GitHub Tree API") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + snapshot, apiOperations := loadFixedPublicTree(t, ctx) + roots := RootsFromSnapshot(snapshot, nil) + for _, expected := range []string{ + "skills/.system/skill-creator", "skills/.curated/define-goal", + } { + if !slices.Contains(roots.Paths, expected) { + t.Fatalf("real GitHub tree omitted expected Skill root %q", expected) + } + } + if len(snapshot.IncompleteSubtrees) != 0 || len(snapshot.Entries) < 1000 || + apiOperations != 3 { + t.Fatalf("real tree was incomplete: entries=%d incomplete=%d treeCalls=%d", + len(snapshot.Entries), len(snapshot.IncompleteSubtrees), apiOperations-2) + } + if len(roots.Paths) != 44 { + t.Fatalf("fixed public tree changed Skill root count: got=%d want=44", len(roots.Paths)) + } + // Match the effective publish policy in Java's application.yml. This reads only + // Tree metadata; it never downloads the 44 candidate packages. + policy := PackageLimits{ + MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + } + preflight, err := PreflightPackages(snapshot, roots.Paths, policy) + if err != nil { + t.Fatal(err) + } + if len(preflight.Plans)+len(preflight.Failures) != len(roots.Paths) { + t.Fatalf("preflight lost Skill roots: roots=%d plans=%d failures=%d", + len(roots.Paths), len(preflight.Plans), len(preflight.Failures)) + } + const rejectedRoot = "skills/.curated/cloudflare-deploy" + if len(preflight.Plans) != 43 || len(preflight.Failures) != 1 || + preflight.Failures[0].Root != rejectedRoot || + preflight.Failures[0].Code != "PACKAGE_TOO_LARGE" { + t.Fatalf("unexpected fixed-tree package policy result: plans=%d failures=%+v", + len(preflight.Plans), preflight.Failures) + } + rejectedFiles := 0 + for _, entry := range snapshot.Entries { + if strings.HasPrefix(entry.Path, rejectedRoot+"/") && entry.Type == "blob" { + rejectedFiles++ + } + } + if rejectedFiles <= policy.MaxFiles { + t.Fatalf("rejected root does not exceed file-count policy: files=%d limit=%d", + rejectedFiles, policy.MaxFiles) + } + failureCounts := make(map[string]int) + for _, failure := range preflight.Failures { + failureCounts[failure.Code]++ + t.Logf("package preflight rejected root: path=%s code=%s files=%d limit=%d", + failure.Root, failure.Code, rejectedFiles, policy.MaxFiles) + } + failureCodes := make([]string, 0, len(failureCounts)) + for code := range failureCounts { + failureCodes = append(failureCodes, code) + } + sort.Strings(failureCodes) + var totalBytes, largestPackageBytes int64 + maxPackageFiles := 0 + for _, plan := range preflight.Plans { + totalBytes += plan.TotalBytes + largestPackageBytes = max(largestPackageBytes, plan.TotalBytes) + maxPackageFiles = max(maxPackageFiles, len(plan.Files)) + } + t.Logf("fixed public tree: api_operations=%d entries=%d skill_roots=%d incomplete=%d", + apiOperations, len(snapshot.Entries), len(roots.Paths), len(snapshot.IncompleteSubtrees)) + t.Logf("package preflight: eligible=%d rejected=%d declared_bytes=%d largest_package_bytes=%d max_package_files=%d", + len(preflight.Plans), len(preflight.Failures), totalBytes, largestPackageBytes, maxPackageFiles) + for _, code := range failureCodes { + t.Logf("package preflight rejection: code=%s count=%d", code, failureCounts[code]) + } +} + +// Opt in separately from the Tree-only test: this makes one bounded Git +// acquisition per eligible Skill and closes each workspace before the next. +func TestLiveAcquireAllFixedPublicSkillsSequentially(t *testing.T) { + if os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES") != "1" { + t.Skip("set SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES=1 for full public repository acquisition") + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + started := time.Now() + snapshot, apiOperations := loadFixedPublicTree(t, ctx) + if apiOperations != 3 || len(snapshot.IncompleteSubtrees) != 0 { + t.Fatalf("fixed public Tree is incomplete: operations=%d incomplete=%d", + apiOperations, len(snapshot.IncompleteSubtrees)) + } + roots := RootsFromSnapshot(snapshot, nil) + preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{ + MaxFiles: 100, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + }) + if err != nil { + t.Fatal(err) + } + if len(preflight.Plans) != 43 || len(preflight.Failures) != 1 || + preflight.Failures[0].Code != "PACKAGE_TOO_LARGE" { + t.Fatalf("unexpected package preflight: plans=%d failures=%+v", + len(preflight.Plans), preflight.Failures) + } + tempRoot := t.TempDir() + limits := AcquisitionLimits{ + MaxWorkspaceBytes: 64 << 20, MaxArchiveBytes: 100 << 20, MinFreeBytes: 16 << 20, + } + var totalArchiveBytes, largestWorkspaceAfterAcquisition int64 + for index, plan := range preflight.Plans { + func() { + artifact, err := AcquirePackage(ctx, "openai", "skills", fixedPublicCommitSHA, + tempRoot, plan, limits) + if err != nil { + t.Fatalf("acquire package %d/%d %s: %v", + index+1, len(preflight.Plans), plan.Root, err) + } + defer func() { + if err := artifact.Close(); err != nil { + t.Fatalf("clean package %s: %v", plan.Root, err) + } + }() + archive, err := zip.OpenReader(artifact.Path) + if err != nil { + t.Fatalf("open package %s: %v", plan.Root, err) + } + defer archive.Close() + want := map[string]struct{}{evidenceManifestPath: {}} + for _, file := range plan.Files { + want[file.RelativePath] = struct{}{} + } + for _, evidence := range plan.Evidence { + if evidence.Bundled { + want[evidenceEntryPrefix+evidence.RepositoryPath] = struct{}{} + } + } + if len(archive.File) != len(want) { + t.Fatalf("package %s has %d ZIP entries, expected %d", + plan.Root, len(archive.File), len(want)) + } + for _, file := range archive.File { + if _, expected := want[file.Name]; !expected { + t.Fatalf("package %s contains unexpected ZIP path %s", plan.Root, file.Name) + } + delete(want, file.Name) + } + if len(want) != 0 { + t.Fatalf("package %s omitted %d planned ZIP entries", plan.Root, len(want)) + } + stored, err := os.Open(artifact.Path) + if err != nil { + t.Fatalf("read package %s: %v", plan.Root, err) + } + digest := sha256.New() + storedBytes, copyErr := io.Copy(digest, stored) + closeErr := stored.Close() + if copyErr != nil || closeErr != nil || storedBytes != artifact.Size || + hex.EncodeToString(digest.Sum(nil)) != artifact.SHA256 { + t.Fatalf("package %s archive size or digest mismatch: copy=%v close=%v", + plan.Root, copyErr, closeErr) + } + workspaceBytes, err := directoryBytes(tempRoot) + if err != nil || workspaceBytes > limits.MaxWorkspaceBytes { + t.Fatalf("package %s exceeded workspace budget: bytes=%d error=%v", + plan.Root, workspaceBytes, err) + } + largestWorkspaceAfterAcquisition = max(largestWorkspaceAfterAcquisition, workspaceBytes) + totalArchiveBytes += artifact.Size + }() + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("package %d/%d left a workspace: entries=%d error=%v", + index+1, len(preflight.Plans), len(remaining), err) + } + if (index+1)%10 == 0 { + t.Logf("sequential packages completed=%d/%d", index+1, len(preflight.Plans)) + } + } + t.Logf("fixed public repository acquisition: api_operations=%d eligible=%d rejected=%d git_acquisitions=%d total_archive_bytes=%d largest_workspace_after_acquisition=%d elapsed=%s", + apiOperations, len(preflight.Plans), len(preflight.Failures), len(preflight.Plans), + totalArchiveBytes, largestWorkspaceAfterAcquisition, time.Since(started)) +} diff --git a/github-sync-go/internal/repository/tree_test.go b/github-sync-go/internal/repository/tree_test.go new file mode 100644 index 000000000..56c4027f6 --- /dev/null +++ b/github-sync-go/internal/repository/tree_test.go @@ -0,0 +1,95 @@ +package repository + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +const ( + rootTreeSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + skillsTreeSHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + goodTreeSHA = "cccccccccccccccccccccccccccccccccccccccc" + badTreeSHA = "dddddddddddddddddddddddddddddddddddddddd" + skillBlobSHA = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +) + +type fixtureTreeReader struct { + trees map[string]github.Tree + errors map[string]error + calls []string +} + +func (reader *fixtureTreeReader) GetTree(_ context.Context, _, _, sha string, + recursive bool) (github.Tree, github.Quota, error) { + key := sha + if recursive { + key += ":recursive" + } + reader.calls = append(reader.calls, key) + if err := reader.errors[key]; err != nil { + return github.Tree{}, github.Quota{}, err + } + return reader.trees[key], github.Quota{}, nil +} + +func TestEnumerateTreeContinuesCompleteSiblingAfterTruncation(t *testing.T) { + reader := &fixtureTreeReader{ + trees: map[string]github.Tree{ + rootTreeSHA + ":recursive": {SHA: rootTreeSHA, Truncated: true}, + rootTreeSHA: {SHA: rootTreeSHA, Entries: []github.TreeEntry{ + {Path: "skills", Type: "tree", SHA: skillsTreeSHA}, + }}, + skillsTreeSHA: {SHA: skillsTreeSHA, Entries: []github.TreeEntry{ + {Path: "good", Type: "tree", SHA: goodTreeSHA}, + {Path: "bad", Type: "tree", SHA: badTreeSHA}, + }}, + goodTreeSHA: {SHA: goodTreeSHA, Entries: []github.TreeEntry{ + {Path: "SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + }}, + }, + errors: map[string]error{badTreeSHA: errors.New("temporary tree failure")}, + } + snapshot, err := EnumerateTree(context.Background(), reader, "owner", "repo", rootTreeSHA, + TreeLimits{MaxTreeRequests: 8, MaxEntries: 20}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(snapshot.IncompleteSubtrees, []string{"skills/bad"}) || + !reflect.DeepEqual(reader.calls, []string{ + rootTreeSHA + ":recursive", rootTreeSHA, skillsTreeSHA, goodTreeSHA, badTreeSHA, + }) { + t.Fatalf("incorrect partial tree coverage: %+v, calls=%v", snapshot, reader.calls) + } + roots := RootsFromSnapshot(snapshot, nil) + if !reflect.DeepEqual(roots.Paths, []string{"skills/good"}) { + t.Fatalf("complete sibling root was lost: %+v", roots) + } + preflight, err := PreflightPackages(snapshot, roots.Paths, PackageLimits{ + MaxFiles: 500, MaxSingleFileBytes: 10 << 20, MaxPackageBytes: 100 << 20, + }) + if err != nil || len(preflight.Plans) != 1 || preflight.Plans[0].Root != "skills/good" || + len(preflight.Failures) != 0 { + t.Fatalf("complete sibling was blocked: %+v, error=%v", preflight, err) + } +} + +func TestEnumerateTreeFallsBackWhenRecursiveResponseIsTooLarge(t *testing.T) { + reader := &fixtureTreeReader{ + trees: map[string]github.Tree{rootTreeSHA: { + SHA: rootTreeSHA, Entries: []github.TreeEntry{ + {Path: "SKILL.md", Type: "blob", Mode: "100644", SHA: skillBlobSHA, Size: 12}, + }, + }}, + errors: map[string]error{rootTreeSHA + ":recursive": github.ErrResponseTooLarge}, + } + snapshot, err := EnumerateTree(context.Background(), reader, "owner", "repo", rootTreeSHA, + TreeLimits{MaxTreeRequests: 2, MaxEntries: 2}) + if err != nil || len(snapshot.IncompleteSubtrees) != 0 || + !reflect.DeepEqual(RootsFromSnapshot(snapshot, nil).Paths, []string{"."}) { + t.Fatalf("oversized recursive tree did not fall back: %+v, error=%v", snapshot, err) + } +} diff --git a/github-sync-go/internal/worker/discovery.go b/github-sync-go/internal/worker/discovery.go new file mode 100644 index 000000000..f488bce56 --- /dev/null +++ b/github-sync-go/internal/worker/discovery.go @@ -0,0 +1,401 @@ +package worker + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type DiscoveryLimits struct { + MaxPagesPerLease int +} + +type discoveryGitHub interface { + SetQuotaObserver(func(context.Context, github.Quota)) + GetRepository(context.Context, string, string) (github.Repository, github.Quota, error) + GetOwnerType(context.Context, string) (github.OwnerType, github.Quota, error) + ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error) + SearchRepositories(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error) +} + +type discoveryCursor struct { + Version int `json:"v"` + Entry string `json:"entry"` + Selector int `json:"selector,omitempty"` + Page int `json:"page"` + Offset int `json:"offset,omitempty"` + Path string `json:"path,omitempty"` + Revisit int `json:"revisit,omitempty"` +} + +// ProcessDiscovery persists each cursor only after its candidate batch is +// accepted. The cursor contains no token or process-local business state. +func ProcessDiscovery(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits DiscoveryLimits) error { + if task == nil { + return errors.New("invalid discovery task or worker limits") + } + if task.OfficialDiscovery != nil { + return processOfficialDiscovery(ctx, control, task, limits, github.NewClient(credentialToken(task))) + } + return processDiscoveryWithGitHub(ctx, control, task, limits, github.NewClient(credentialToken(task))) +} + +func processDiscoveryWithGitHub(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits DiscoveryLimits, upstream discoveryGitHub) error { + if task == nil || task.Kind != "DISCOVERY" || task.Discovery == nil || task.OfficialDiscovery != nil || + limits.MaxPagesPerLease < 1 || task.Discovery.CandidateBudget < 1 || upstream == nil { + return errors.New("invalid discovery task or worker limits") + } + snapshot := task.Discovery + if snapshot.EntryType == "SEARCH_PREVIEW" { + return processSearchPreview(ctx, control, task) + } + upstream.SetQuotaObserver(observeCredentialQuota(control, task)) + checkpoint := snapshot.Checkpoint + var cursor discoveryCursor + if checkpoint == "" { + cursor = discoveryCursor{Version: 1, Entry: snapshot.EntryType, Page: 1} + encoded, err := encodeCursor(cursor) + if err != nil { + return err + } + if err := control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, + controlplane.DiscoveryBatch{Checkpoint: encoded, Repositories: []controlplane.RepositoryCandidate{}}); err != nil { + return err + } + checkpoint = encoded + } else if err := json.Unmarshal([]byte(checkpoint), &cursor); err != nil { + return errors.New("invalid persisted discovery cursor") + } + if cursor.Version != 1 || cursor.Entry != snapshot.EntryType || cursor.Page < 1 || + cursor.Offset < 0 || cursor.Selector < 0 || cursor.Revisit < 0 || len(cursor.Path) > 64 || + strings.Trim(cursor.Path, "LR") != "" { + return errors.New("persisted discovery cursor does not match task") + } + if snapshot.EntryType == "CURATED_REPOSITORY" { + if snapshot.Owner == "" || snapshot.RepositoryName == "" { + return errors.New("curated repository task has no coordinate") + } + if cursor.Page == 1 { + repository, _, err := upstream.GetRepository(ctx, snapshot.Owner, snapshot.RepositoryName) + if err != nil { + return completeDiscoveryError(ctx, control, task, checkpoint, err) + } + next := cursor + next.Page = 2 + checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next, + toCandidates([]github.Repository{repository}), 0) + if err != nil { + return err + } + } else if cursor.Page != 2 { + return errors.New("invalid curated repository cursor") + } + _, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "") + return err + } + spent := 0 + pages := 0 + seen := make(map[int64]struct{}) + var ownerType github.OwnerType + if snapshot.EntryType == "CURATED_OWNER" { + var err error + ownerType, _, err = upstream.GetOwnerType(ctx, snapshot.Owner) + if err != nil { + return completeDiscoveryError(ctx, control, task, checkpoint, err) + } + } else if snapshot.EntryType != "SEARCH_RULE" || snapshot.Criteria == nil || + snapshot.SearchWindow == nil || len(snapshot.Criteria.Selectors) == 0 { + return errors.New("discovery task has no valid entry configuration") + } + var selectors []controlplane.SearchSelector + if snapshot.EntryType == "SEARCH_RULE" { + selectors = selectorsFor(snapshot) + } + for { + if ctx.Err() != nil { + return ctx.Err() + } + if snapshot.EntryType == "SEARCH_RULE" && cursor.Selector >= len(selectors) { + _, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "") + return err + } + if spent >= snapshot.CandidateBudget || pages >= limits.MaxPagesPerLease { + return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(30*time.Minute)) + } + var candidates []controlplane.RepositoryCandidate + var hasMore bool + var next discoveryCursor + if snapshot.EntryType == "CURATED_OWNER" { + page, _, err := upstream.ListPublicOwnerRepositories(ctx, snapshot.Owner, ownerType, cursor.Page) + if err != nil { + return completeDiscoveryError(ctx, control, task, checkpoint, err) + } + pages++ + candidates = toCandidates(page.Repositories) + hasMore = page.HasMore + next = cursor + } else { + from, until, err := searchBounds(snapshot.SearchWindow, cursor.Path) + if err != nil { + return err + } + selector := selectors[cursor.Selector] + page, _, err := upstream.SearchRepositories(ctx, selector.Kind, selector.Value, + &from, until, cursor.Page) + if err != nil { + return completeDiscoveryError(ctx, control, task, checkpoint, err) + } + pages++ + if page.LimitReached || page.IncompleteResults { + if !from.Before(until) { + return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(time.Hour)) + } + next = cursor + next.Path += "L" + next.Page, next.Offset = 1, 0 + var err error + checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next, nil, 0) + if err != nil { + return err + } + cursor = next + continue + } + candidates = toCandidates(page.Repositories) + hasMore = page.HasMore + next = cursor + } + if cursor.Offset > len(candidates) { + // GitHub search pages can shrink between leases. Revisit the current + // page and let Java deduplicate previously reported repository IDs. + // Bump the cursor generation so replay cannot reuse an older checkpoint. + if cursor.Revisit == int(^uint(0)>>1) { + return errors.New("discovery page revisit limit reached") + } + cursor.Revisit++ + cursor.Offset = 0 + next = cursor + } + remaining := snapshot.CandidateBudget - spent + batch, end := selectDiscoveryBatch(candidates, cursor.Offset, remaining, seen) + deferred := len(candidates) - end + if end < len(candidates) { + next.Offset = end + } else if hasMore { + next.Page++ + next.Offset = 0 + } else if snapshot.EntryType == "CURATED_OWNER" { + next.Page++ + next.Offset = 0 + } else { + next = nextSearchCursor(cursor) + } + var err error + checkpoint, err = reportDiscoveryProgress(ctx, control, task, checkpoint, next, batch, deferred) + if err != nil { + var apiError *controlplane.APIError + if errors.As(err, &apiError) && apiError.Code == "CANDIDATE_BUDGET_EXCEEDED" { + return deferDiscovery(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(30*time.Minute)) + } + return err + } + for _, candidate := range batch { + seen[candidate.RepositoryID] = struct{}{} + } + spent += len(batch) + cursor = next + if snapshot.EntryType == "CURATED_OWNER" && !hasMore && deferred == 0 { + _, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "") + return err + } + } +} + +// The offset tracks raw page positions so a resumed lease does not depend on +// the in-memory seen set. Java also deduplicates repository IDs across leases. +func selectDiscoveryBatch(candidates []controlplane.RepositoryCandidate, offset, budget int, + seen map[int64]struct{}) ([]controlplane.RepositoryCandidate, int) { + batch := make([]controlplane.RepositoryCandidate, 0, min(budget, len(candidates)-offset)) + selected := make(map[int64]struct{}) + end := offset + for end < len(candidates) && len(batch) < budget { + candidate := candidates[end] + end++ + if _, exists := seen[candidate.RepositoryID]; exists { + continue + } + if _, exists := selected[candidate.RepositoryID]; exists { + continue + } + selected[candidate.RepositoryID] = struct{}{} + batch = append(batch, candidate) + } + return batch, end +} + +// A preview only reads one search page per selector. Counts are estimates from +// repository metadata and can overlap; no discovery candidates are reported. +func processSearchPreview(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask) error { + upstream := github.NewClient(credentialToken(task)) + upstream.SetQuotaObserver(observeCredentialQuota(control, task)) + return processSearchPreviewWithSearch(ctx, control, task, upstream.SearchRepositories) +} + +func processSearchPreviewWithSearch(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, + search func(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error)) error { + results, err := previewSearchResults(ctx, task.Discovery, search) + if err != nil { + return err + } + return control.CompletePreview(ctx, task.TaskID, task.Lease, results) +} + +func previewSearchResults(ctx context.Context, snapshot *controlplane.DiscoverySnapshot, + search func(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error)) ([]controlplane.PreviewSelectorResult, error) { + if snapshot == nil || snapshot.Criteria == nil || snapshot.SearchWindow == nil || + len(snapshot.Criteria.Selectors) < 1 || len(snapshot.Criteria.Selectors) > 20 || + snapshot.SearchWindow.Until.IsZero() || snapshot.Checkpoint != "" { + return nil, errors.New("invalid search preview snapshot") + } + results := make([]controlplane.PreviewSelectorResult, 0, len(snapshot.Criteria.Selectors)) + for _, selector := range selectorsFor(snapshot) { + page, _, err := search(ctx, selector.Kind, selector.Value, + nil, snapshot.SearchWindow.Until, 1) + if err != nil { + return nil, err + } + samples := toCandidates(page.Repositories) + if len(samples) > 5 { + samples = samples[:5] + } + results = append(results, controlplane.PreviewSelectorResult{ + Kind: selector.Kind, Value: selector.Value, Priority: selector.Priority, + TotalCount: page.TotalCount, IncompleteResults: page.IncompleteResults, + LimitReached: page.LimitReached, Samples: samples, + }) + } + return results, nil +} + +func selectorsFor(snapshot *controlplane.DiscoverySnapshot) []controlplane.SearchSelector { + selectors := append([]controlplane.SearchSelector(nil), snapshot.Criteria.Selectors...) + sort.SliceStable(selectors, func(i, j int) bool { + return selectors[i].Priority == "CORE" && selectors[j].Priority != "CORE" + }) + return selectors +} + +func toCandidates(repositories []github.Repository) []controlplane.RepositoryCandidate { + result := make([]controlplane.RepositoryCandidate, 0, len(repositories)) + for _, candidate := range repositories { + result = append(result, controlplane.RepositoryCandidate{ + RepositoryID: candidate.ID, Owner: candidate.Owner.Login, + Name: candidate.Name, DefaultBranch: candidate.DefaultBranch, + }) + } + return result +} + +func reportDiscoveryProgress(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, previous string, next discoveryCursor, + repositories []controlplane.RepositoryCandidate, deferred int) (string, error) { + encoded, err := encodeCursor(next) + if err != nil { + return "", err + } + if encoded == previous { + return "", errors.New("discovery cursor did not advance") + } + reason := "" + if deferred > 0 { + reason = "BUDGET" + } + err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, controlplane.DiscoveryBatch{ + PreviousCheckpoint: &previous, Checkpoint: encoded, Repositories: repositories, + DeferredCount: deferred, DeferralReason: reason, + }) + if err != nil { + return "", err + } + return encoded, nil +} + +func encodeCursor(cursor discoveryCursor) (string, error) { + encoded, err := json.Marshal(cursor) + if err != nil || len(encoded) > 4096 { + return "", errors.New("discovery cursor exceeds protocol limit") + } + return string(encoded), nil +} + +func searchBounds(window *controlplane.SearchWindow, path string) (time.Time, time.Time, error) { + if window.Until.IsZero() || window.Mode != "FULL_RECHECK" && window.Mode != "INCREMENTAL" || + window.Mode == "INCREMENTAL" && window.Since.IsZero() { + return time.Time{}, time.Time{}, errors.New("invalid fixed search window") + } + lower := int64(0) + if window.Mode == "INCREMENTAL" { + lower = window.Since.Unix() + } + upper := window.Until.Unix() + if lower > upper { + return time.Time{}, time.Time{}, errors.New("invalid fixed search bounds") + } + for _, step := range path { + if lower >= upper { + return time.Time{}, time.Time{}, errors.New("discovery split cursor exceeds search window") + } + middle := lower + (upper-lower)/2 + if step == 'L' { + upper = middle + } else if step == 'R' { + lower = middle + 1 + } else { + return time.Time{}, time.Time{}, errors.New("invalid discovery split cursor") + } + } + return time.Unix(lower, 0).UTC(), time.Unix(upper, 0).UTC(), nil +} + +func nextSearchCursor(current discoveryCursor) discoveryCursor { + next := current + next.Page, next.Offset = 1, 0 + for len(next.Path) > 0 { + last := next.Path[len(next.Path)-1] + next.Path = next.Path[:len(next.Path)-1] + if last == 'L' { + next.Path += "R" + return next + } + } + next.Selector++ + return next +} + +func deferDiscovery(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, checkpoint, reason string, retryAt time.Time) error { + _, err := control.CompleteDiscovery(ctx, task.TaskID, task.Lease, checkpoint, futureRetry(retryAt), reason) + return err +} + +func completeDiscoveryError(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, checkpoint string, err error) error { + var upstream *github.APIError + if errors.As(err, &upstream) && upstream.Code == "GITHUB_RATE_LIMITED" { + if reportErr := reportCredentialProblem(ctx, control, task, err); reportErr != nil { + return reportErr + } + return deferDiscovery(ctx, control, task, checkpoint, "RATE_LIMIT", upstream.RetryAt) + } + return err +} diff --git a/github-sync-go/internal/worker/discovery_curated_interop_test.go b/github-sync-go/internal/worker/discovery_curated_interop_test.go new file mode 100644 index 000000000..56c7cba3b --- /dev/null +++ b/github-sync-go/internal/worker/discovery_curated_interop_test.go @@ -0,0 +1,73 @@ +package worker + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type curatedRepositoryFixture struct{} + +func (curatedRepositoryFixture) SetQuotaObserver(func(context.Context, github.Quota)) {} + +func (curatedRepositoryFixture) GetRepository(_ context.Context, owner, name string) (github.Repository, github.Quota, error) { + if owner != "openai" || name != "skills" { + return github.Repository{}, github.Quota{}, errors.New("unexpected curated repository coordinate") + } + repository := github.Repository{ID: 987654321, Name: name, DefaultBranch: "main"} + repository.Owner.Login = owner + return repository, github.Quota{}, nil +} + +func (curatedRepositoryFixture) GetOwnerType(context.Context, string) (github.OwnerType, github.Quota, error) { + return "", github.Quota{}, errors.New("unexpected owner lookup") +} + +func (curatedRepositoryFixture) ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error) { + return github.OwnerPage{}, github.Quota{}, errors.New("unexpected owner repository listing") +} + +func (curatedRepositoryFixture) SearchRepositories(context.Context, string, string, *time.Time, time.Time, int) (github.SearchPage, github.Quota, error) { + return github.SearchPage{}, github.Quota{}, errors.New("unexpected repository search") +} + +// Opt-in cross-process check of Java's curated source binding contract. +// The optional live mode also probes GitHub but depends on its public quota. +func TestInteropCuratedPublicRepository(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_CURATED_REPOSITORY") != "1" { + t.Skip("requires a Java HTTP curated repository fixture") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-curated-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") { + t.Fatal("Java did not lease the queued curated repository task") + } + if task.Discovery == nil || task.Discovery.EntryType != "CURATED_REPOSITORY" || + task.Discovery.Owner != "openai" || task.Discovery.RepositoryName != "skills" { + t.Fatalf("Java returned unexpected curated repository coordinates: %+v", task.Discovery) + } + if task.Credential != nil { + t.Fatal("Java should have leased an anonymous public repository task") + } + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + limits := DiscoveryLimits{MaxPagesPerLease: 2} + if os.Getenv("SKILLHUB_INTEROP_CURATED_MODE") == "live" { + return ProcessDiscovery(ctx, control, task, limits) + } + return processDiscoveryWithGitHub(ctx, control, task, limits, curatedRepositoryFixture{}) + }); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/worker/discovery_owner_interop_test.go b/github-sync-go/internal/worker/discovery_owner_interop_test.go new file mode 100644 index 000000000..353fce3ea --- /dev/null +++ b/github-sync-go/internal/worker/discovery_owner_interop_test.go @@ -0,0 +1,68 @@ +package worker + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type curatedOwnerFixture struct { + curatedRepositoryFixture + owner string + ownerType github.OwnerType +} + +func (fixture curatedOwnerFixture) GetOwnerType(_ context.Context, owner string) (github.OwnerType, github.Quota, error) { + if owner != fixture.owner { + return "", github.Quota{}, errors.New("unexpected curated owner") + } + return fixture.ownerType, github.Quota{}, nil +} + +func (fixture curatedOwnerFixture) ListPublicOwnerRepositories(_ context.Context, owner string, + ownerType github.OwnerType, page int) (github.OwnerPage, github.Quota, error) { + if owner != fixture.owner || ownerType != fixture.ownerType || page != 1 { + return github.OwnerPage{}, github.Quota{}, errors.New("unexpected public owner repository page") + } + repository := github.Repository{ID: 987654324, Name: "public-skills", DefaultBranch: "main"} + if fixture.ownerType == github.OrganizationOwner { + repository.ID++ + } + repository.Owner.Login = owner + return github.OwnerPage{Repositories: []github.Repository{repository}}, github.Quota{}, nil +} + +func TestInteropCuratedOwnerDiscovery(t *testing.T) { + ownerType := github.OwnerType(os.Getenv("SKILLHUB_INTEROP_OWNER_TYPE")) + if ownerType == "" { + t.Skip("requires a Java HTTP curated owner fixture") + } + if ownerType != github.UserOwner && ownerType != github.OrganizationOwner { + t.Fatal("invalid curated owner fixture type") + } + owner := os.Getenv("SKILLHUB_INTEROP_OWNER_LOGIN") + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-owner-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Discovery == nil || task.Discovery.EntryType != "CURATED_OWNER" || + task.Discovery.Owner != owner || task.Credential != nil { + t.Fatal("Java did not lease the expected anonymous curated owner task") + } + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processDiscoveryWithGitHub(ctx, control, task, + DiscoveryLimits{MaxPagesPerLease: 2}, curatedOwnerFixture{owner: owner, ownerType: ownerType}) + }); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/worker/discovery_pagination_interop_test.go b/github-sync-go/internal/worker/discovery_pagination_interop_test.go new file mode 100644 index 000000000..c41c34c6e --- /dev/null +++ b/github-sync-go/internal/worker/discovery_pagination_interop_test.go @@ -0,0 +1,73 @@ +package worker + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type changingSearchPageFixture struct { + curatedRepositoryFixture + phase string +} + +func (fixture changingSearchPageFixture) SearchRepositories(_ context.Context, kind, _ string, + _ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) { + if page != 1 { + return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page") + } + if kind == "TOPIC" && fixture.phase == "shrunk" { + return github.SearchPage{}, github.Quota{}, nil + } + if kind != "KEYWORD" { + return github.SearchPage{}, github.Quota{}, errors.New("unexpected search selector") + } + ids := []int64{9400001, 9400002, 9400003, 9400004, 9400005} + if fixture.phase == "shrunk" { + ids = []int64{9400002, 9400006} + } + repositories := make([]github.Repository, 0, len(ids)) + for _, id := range ids { + repository := github.Repository{ID: id, Name: "public-skills", DefaultBranch: "main"} + repository.Owner.Login = "fixture-owner" + repositories = append(repositories, repository) + } + return github.SearchPage{Repositories: repositories}, github.Quota{}, nil +} + +// Java starts the same discovery task in two Go processes. The second lease +// reads a shorter GitHub page than the first and must not lose the new ID. +func TestInteropChangingSearchPage(t *testing.T) { + phase := os.Getenv("SKILLHUB_INTEROP_SEARCH_PAGE_PHASE") + if phase == "" { + t.Skip("requires a Java HTTP changing search page fixture") + } + if phase != "initial" && phase != "shrunk" { + t.Fatal("invalid search page fixture phase") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-changing-page-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Discovery == nil || task.Discovery.EntryType != "SEARCH_RULE" || + task.Discovery.CandidateBudget != 3 { + t.Fatal("Java did not lease the expected bounded search task") + } + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processDiscoveryWithGitHub(ctx, control, task, + DiscoveryLimits{MaxPagesPerLease: 5}, changingSearchPageFixture{phase: phase}) + }); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/worker/discovery_preview_interop_test.go b/github-sync-go/internal/worker/discovery_preview_interop_test.go new file mode 100644 index 000000000..90ebd7bb3 --- /dev/null +++ b/github-sync-go/internal/worker/discovery_preview_interop_test.go @@ -0,0 +1,98 @@ +package worker + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +// Opt-in wire check: Java creates the preview task and persists the result; +// this process exercises the real worker completion/failure path with fixed search pages. +func TestInteropSearchPreview(t *testing.T) { + phase := os.Getenv("SKILLHUB_INTEROP_PREVIEW_PHASE") + if phase == "" { + t.Skip("requires a Java HTTP preview fixture") + } + if phase != "success" && phase != "limit" && phase != "live" { + t.Fatal("invalid preview fixture phase") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-preview-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Discovery == nil || task.Discovery.EntryType != "SEARCH_PREVIEW" || + task.Discovery.Criteria == nil || len(task.Discovery.Criteria.Selectors) != 2 { + t.Fatal("Java did not lease the expected multi-selector preview") + } + if expected := os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID"); expected != "" { + if task.Credential == nil || task.Credential.CredentialID != expected || + task.Credential.Token != os.Getenv("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN") { + t.Fatal("Java did not lease the expected fixture credential") + } + } else if os.Getenv("SKILLHUB_INTEROP_EXPECT_ANONYMOUS") == "1" && task.Credential != nil { + t.Fatal("Java should have leased an anonymous public request") + } + if phase == "live" { + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processSearchPreview(ctx, control, task) + }); err != nil { + t.Fatal(err) + } + return + } + var calls []string + search := func(_ context.Context, kind, value string, since *time.Time, + until time.Time, page int) (github.SearchPage, github.Quota, error) { + if since != nil || until.IsZero() || page != 1 { + t.Fatal("preview did not use the first page of its fixed window") + } + calls = append(calls, kind+":"+value) + if kind == "TOPIC" && phase == "limit" { + return github.SearchPage{}, github.Quota{}, &github.APIError{ + Status: 429, Code: "GITHUB_RATE_LIMITED", RetryAt: time.Now().Add(10 * time.Minute), + } + } + if kind == "KEYWORD" { + repositories := make([]github.Repository, 6) + for index := range repositories { + repositories[index].ID = int64(900001 + index) + repositories[index].Name = "public-skill" + repositories[index].Owner.Login = "fixture-owner" + } + return github.SearchPage{Repositories: repositories, TotalCount: 1001, + IncompleteResults: true, LimitReached: true}, github.Quota{}, nil + } + repository := github.Repository{ID: 900101, Name: "topic-skill"} + repository.Owner.Login = "fixture-owner" + return github.SearchPage{Repositories: []github.Repository{repository}, + TotalCount: 4}, github.Quota{}, nil + } + // The worker's lease wrapper reports upstream errors to Java, including retryAt. + err = executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processSearchPreviewWithSearch(ctx, control, task, search) + }) + if phase == "success" && err != nil || phase == "limit" && err == nil { + t.Fatalf("unexpected worker outcome: phase=%s err=%v", phase, err) + } + if phase == "limit" { + var upstream *github.APIError + if !errors.As(err, &upstream) || upstream.Code != "GITHUB_RATE_LIMITED" { + t.Fatalf("rate limit was not preserved: %v", err) + } + } + if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" || + calls[1] != "TOPIC:agent-skills" { + t.Fatalf("unexpected preview search order: %v", calls) + } +} diff --git a/github-sync-go/internal/worker/discovery_rule_interop_test.go b/github-sync-go/internal/worker/discovery_rule_interop_test.go new file mode 100644 index 000000000..0df0833de --- /dev/null +++ b/github-sync-go/internal/worker/discovery_rule_interop_test.go @@ -0,0 +1,99 @@ +package worker + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type searchRuleFixture struct { + curatedRepositoryFixture + calls []string + windows []searchRuleWindow + saturate bool + reportedLimit bool +} + +type searchRuleWindow struct { + from time.Time + until time.Time +} + +func (fixture *searchRuleFixture) SearchRepositories(_ context.Context, kind, value string, + since *time.Time, until time.Time, page int) (github.SearchPage, github.Quota, error) { + if since == nil || until.IsZero() || page != 1 { + return github.SearchPage{}, github.Quota{}, errors.New("search did not use its fixed first page") + } + fixture.calls = append(fixture.calls, kind+":"+value) + fixture.windows = append(fixture.windows, searchRuleWindow{from: *since, until: until}) + if fixture.saturate && kind == "KEYWORD" && !fixture.reportedLimit { + fixture.reportedLimit = true + return github.SearchPage{TotalCount: 1001, LimitReached: true, + IncompleteResults: true}, github.Quota{}, nil + } + first := github.Repository{ID: 987654322, Name: "skill-one", DefaultBranch: "main"} + first.Owner.Login = "fixture-owner" + if kind == "KEYWORD" && value == "agent skills" { + return github.SearchPage{Repositories: []github.Repository{first}, TotalCount: 1}, github.Quota{}, nil + } + if kind == "TOPIC" && value == "agent-skills" { + second := github.Repository{ID: 987654323, Name: "skill-two", DefaultBranch: "main"} + second.Owner.Login = "fixture-owner" + return github.SearchPage{Repositories: []github.Repository{first, second}, TotalCount: 2}, github.Quota{}, nil + } + return github.SearchPage{}, github.Quota{}, errors.New("unexpected search selector") +} + +// Java supplies a versioned rule and fixed window; Go reports deduplicated leads. +func TestInteropSearchRuleDiscovery(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_SEARCH_RULE") != "1" { + t.Skip("requires a Java HTTP search rule fixture") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-search-rule-worker", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Discovery == nil || task.Discovery.EntryType != "SEARCH_RULE" || + task.Discovery.RuleVersion != 2 || task.Discovery.SearchWindow == nil || + task.Discovery.SearchWindow.Mode != "FULL_RECHECK" || task.Credential != nil { + t.Fatal("Java did not lease the expected anonymous versioned search rule") + } + fixture := &searchRuleFixture{saturate: os.Getenv("SKILLHUB_INTEROP_SEARCH_SATURATED") == "1"} + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processDiscoveryWithGitHub(ctx, control, task, + DiscoveryLimits{MaxPagesPerLease: 5}, fixture) + }); err != nil { + t.Fatal(err) + } + if len(fixture.windows) == 0 || !fixture.windows[0].from.Equal(time.Unix(0, 0)) || + fixture.windows[0].until.Unix() != task.Discovery.SearchWindow.Until.Unix() { + t.Fatal("discovery changed the Java-provided full search window") + } + if fixture.saturate { + if len(fixture.calls) != 4 || fixture.calls[0] != "KEYWORD:agent skills" || + fixture.calls[1] != "KEYWORD:agent skills" || + fixture.calls[2] != "KEYWORD:agent skills" || + fixture.calls[3] != "TOPIC:agent-skills" || + !fixture.windows[1].from.Equal(fixture.windows[0].from) || + !fixture.windows[2].from.Equal(fixture.windows[1].until.Add(time.Second)) || + !fixture.windows[2].until.Equal(fixture.windows[0].until) || + !fixture.windows[3].from.Equal(fixture.windows[0].from) || + !fixture.windows[3].until.Equal(fixture.windows[0].until) { + t.Fatalf("incomplete search did not split and cover the fixed window: %v", fixture.calls) + } + } else if len(fixture.calls) != 2 || fixture.calls[0] != "KEYWORD:agent skills" || + fixture.calls[1] != "TOPIC:agent-skills" { + t.Fatalf("discovery did not execute core before extension: %v", fixture.calls) + } +} diff --git a/github-sync-go/internal/worker/discovery_test.go b/github-sync-go/internal/worker/discovery_test.go new file mode 100644 index 000000000..8ce5f429d --- /dev/null +++ b/github-sync-go/internal/worker/discovery_test.go @@ -0,0 +1,205 @@ +package worker + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type shrinkingSearchFixture struct{ curatedRepositoryFixture } + +func (shrinkingSearchFixture) SearchRepositories(_ context.Context, _, _ string, + _ *time.Time, _ time.Time, page int) (github.SearchPage, github.Quota, error) { + if page != 1 { + return github.SearchPage{}, github.Quota{}, errors.New("unexpected search page") + } + repositories := []github.Repository{{ID: 11, Name: "first"}, {ID: 12, Name: "second"}} + for i := range repositories { + repositories[i].Owner.Login = "public-owner" + } + return github.SearchPage{Repositories: repositories}, github.Quota{}, nil +} + +func TestProcessDiscoveryRevisitsShrinkingPage(t *testing.T) { + previous := `{"v":1,"entry":"SEARCH_RULE","page":1,"offset":3}` + var reported controlplane.DiscoveryBatch + var completed bool + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch { + case strings.HasSuffix(request.URL.Path, "/discovered-repositories"): + if err := json.NewDecoder(request.Body).Decode(&reported); err != nil || + reported.PreviousCheckpoint == nil || *reported.PreviousCheckpoint != previous { + writer.WriteHeader(http.StatusBadRequest) + return + } + writer.WriteHeader(http.StatusAccepted) + case strings.HasSuffix(request.URL.Path, "/complete"): + var body struct { + DiscoveryCheckpoint string `json:"discoveryCheckpoint"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil || + body.DiscoveryCheckpoint != reported.Checkpoint { + writer.WriteHeader(http.StatusBadRequest) + return + } + completed = true + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`)) + default: + writer.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + task := &controlplane.ClaimedTask{ + TaskID: "shrinking-page", Kind: "DISCOVERY", + Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)}, + Discovery: &controlplane.DiscoverySnapshot{ + EntryType: "SEARCH_RULE", Checkpoint: previous, CandidateBudget: 3, + Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{ + {Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"}, + }}, + SearchWindow: &controlplane.SearchWindow{ + Mode: "FULL_RECHECK", Until: time.Now(), + }, + }, + } + if err := processDiscoveryWithGitHub(context.Background(), control, task, + DiscoveryLimits{MaxPagesPerLease: 2}, shrinkingSearchFixture{}); err != nil { + t.Fatal(err) + } + var next discoveryCursor + if err := json.Unmarshal([]byte(reported.Checkpoint), &next); err != nil { + t.Fatal(err) + } + if !completed || len(reported.Repositories) != 2 || + reported.Repositories[0].RepositoryID != 11 || reported.Repositories[1].RepositoryID != 12 || + next.Revisit != 1 || next.Selector != 1 { + t.Fatalf("shrinking page was not revisited: completed=%v batch=%+v cursor=%+v", + completed, reported.Repositories, next) + } +} + +func TestPreviewSearchResultsUsesOnePagePerSelectorAndPreservesWarnings(t *testing.T) { + until := time.Date(2026, 9, 29, 0, 0, 0, 0, time.UTC) + snapshot := &controlplane.DiscoverySnapshot{ + Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{ + {Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"}, + {Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"}, + }}, + SearchWindow: &controlplane.SearchWindow{Mode: "FULL_RECHECK", Until: until}, + } + var calls []string + search := func(_ context.Context, kind, value string, since *time.Time, + end time.Time, page int) (github.SearchPage, github.Quota, error) { + if since != nil || !end.Equal(until) || page != 1 { + t.Fatalf("unexpected preview search window or page: since=%v end=%s page=%d", + since, end, page) + } + calls = append(calls, kind+":"+value) + repositories := make([]github.Repository, 6) + for i := range repositories { + repositories[i].ID = int64(i + 1) + repositories[i].Name = "skill" + repositories[i].Owner.Login = "public-owner" + } + return github.SearchPage{ + Repositories: repositories, TotalCount: 1001, + IncompleteResults: true, LimitReached: true, + }, github.Quota{}, nil + } + + results, err := previewSearchResults(context.Background(), snapshot, search) + if err != nil { + t.Fatal(err) + } + if len(calls) != 2 || calls[0] != "KEYWORD:agent skills" || + calls[1] != "TOPIC:agent-skills" || len(results) != 2 { + t.Fatalf("unexpected selectors: calls=%v results=%+v", calls, results) + } + for _, result := range results { + if result.TotalCount != 1001 || !result.IncompleteResults || + !result.LimitReached || len(result.Samples) != 5 { + t.Fatalf("preview lost estimates or warnings: %+v", result) + } + } +} + +func TestPreviewSearchResultsDoesNotReturnPartialResultsAfterFailure(t *testing.T) { + snapshot := &controlplane.DiscoverySnapshot{ + Criteria: &controlplane.SearchCriteria{Selectors: []controlplane.SearchSelector{ + {Kind: "KEYWORD", Value: "agent skills", Priority: "CORE"}, + {Kind: "TOPIC", Value: "agent-skills", Priority: "EXTENSION"}, + }}, + SearchWindow: &controlplane.SearchWindow{Until: time.Now()}, + } + calls := 0 + results, err := previewSearchResults(context.Background(), snapshot, + func(context.Context, string, string, *time.Time, time.Time, int) ( + github.SearchPage, github.Quota, error) { + calls++ + if calls == 2 { + return github.SearchPage{}, github.Quota{}, errors.New("rate limited") + } + return github.SearchPage{TotalCount: 1}, github.Quota{}, nil + }) + if err == nil || len(results) != 0 || calls != 2 { + t.Fatalf("preview should fail atomically: results=%+v err=%v calls=%d", + results, err, calls) + } +} + +func TestSelectDiscoveryBatchCountsUniqueRepositoriesAndKeepsRawOffset(t *testing.T) { + candidates := []controlplane.RepositoryCandidate{ + {RepositoryID: 10}, {RepositoryID: 10}, {RepositoryID: 20}, + {RepositoryID: 20}, {RepositoryID: 30}, + } + batch, offset := selectDiscoveryBatch(candidates, 0, 2, map[int64]struct{}{}) + if offset != 3 || len(batch) != 2 || batch[0].RepositoryID != 10 || batch[1].RepositoryID != 20 { + t.Fatalf("first batch = %+v, offset = %d", batch, offset) + } + batch, offset = selectDiscoveryBatch(candidates, offset, 1, + map[int64]struct{}{10: {}, 20: {}}) + if offset != len(candidates) || len(batch) != 1 || batch[0].RepositoryID != 30 { + t.Fatalf("second batch = %+v, offset = %d", batch, offset) + } +} + +func TestSearchBoundsPartitionsFixedWindowWithoutOverlap(t *testing.T) { + since := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + window := &controlplane.SearchWindow{ + Mode: "INCREMENTAL", Since: since, Until: since.Add(7 * time.Second), + } + leftFrom, leftUntil, err := searchBounds(window, "L") + if err != nil { + t.Fatal(err) + } + rightFrom, rightUntil, err := searchBounds(window, "R") + if err != nil { + t.Fatal(err) + } + if !leftFrom.Equal(since) || !leftUntil.Add(time.Second).Equal(rightFrom) || + !rightUntil.Equal(window.Until) || nextSearchCursor(discoveryCursor{Path: "L"}).Path != "R" { + t.Fatalf("split windows are not contiguous: [%s, %s], [%s, %s]", + leftFrom, leftUntil, rightFrom, rightUntil) + } +} + +func TestSelectDiscoveryBatchCanAdvancePastOnlyDuplicates(t *testing.T) { + candidates := []controlplane.RepositoryCandidate{{RepositoryID: 10}, {RepositoryID: 10}} + batch, offset := selectDiscoveryBatch(candidates, 0, 1, map[int64]struct{}{10: {}}) + if len(batch) != 0 || offset != len(candidates) { + t.Fatalf("batch = %+v, offset = %d", batch, offset) + } +} diff --git a/github-sync-go/internal/worker/heartbeat_interop_test.go b/github-sync-go/internal/worker/heartbeat_interop_test.go new file mode 100644 index 000000000..c64f26615 --- /dev/null +++ b/github-sync-go/internal/worker/heartbeat_interop_test.go @@ -0,0 +1,23 @@ +package worker + +import ( + "context" + "os" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +func TestInteropWorkerHeartbeat(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_HEARTBEAT") != "1" { + t.Skip("requires a Java HTTP worker heartbeat fixture") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + if err := control.Heartbeat(context.Background()); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/worker/lease.go b/github-sync-go/internal/worker/lease.go new file mode 100644 index 000000000..421a5ffcc --- /dev/null +++ b/github-sync-go/internal/worker/lease.go @@ -0,0 +1,202 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +// ExecuteRepository renews the Java lease while processing and stops all +// downstream work as soon as renewal fails or shutdown begins. +func ExecuteRepository(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits RepositoryLimits) error { + if task == nil || task.Kind != "REPOSITORY_SYNC" || task.LeaseExpiresAt.IsZero() { + return errors.New("invalid leased repository task") + } + return executeLeasedTask(ctx, control, task, func(workContext context.Context) error { + return ProcessRepository(workContext, control, task, limits) + }) +} + +func ExecuteDiscovery(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits DiscoveryLimits) error { + if task == nil || task.Kind != "DISCOVERY" || task.LeaseExpiresAt.IsZero() { + return errors.New("invalid leased discovery task") + } + return executeLeasedTask(ctx, control, task, func(workContext context.Context) error { + return ProcessDiscovery(workContext, control, task, limits) + }) +} + +func executeLeasedTask(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, process func(context.Context) error) error { + workContext, cancel := context.WithCancel(ctx) + defer cancel() + renewDone := make(chan error, 1) + go func() { + renewDone <- keepLease(workContext, cancel, control, task) + }() + workErr := runPlatformTask(workContext, control, task, process) + cancel() + renewErr := <-renewDone + if ctx.Err() != nil { + return ctx.Err() + } + if renewErr != nil { + return renewErr + } + if workErr == nil { + return nil + } + if errors.Is(workErr, context.Canceled) { + return workErr + } + failureReason, retryAt := classifyFailure(workErr, task.SourceID != 0) + reportContext, reportCancel := context.WithTimeout(ctx, 20*time.Second) + defer reportCancel() + credentialErr := reportCredentialProblem(reportContext, control, task, workErr) + var upstream *github.APIError + var quota *controlplane.GithubQuota + if errors.As(workErr, &upstream) { + quota = credentialQuota(upstream.Quota) + } + _, reportErr := control.Fail(reportContext, task.TaskID, task.Lease, + failureReason, safeFailureDetail(workErr), retryAt, quota) + if reportErr != nil { + return reportErr + } + if task.PlatformControlVersion != nil { + // Fail has settled all acquisition; acknowledge a pause racing that report. + _, ackErr := (&platformExecution{control: control, task: task}).acknowledge(reportContext) + if ackErr != nil { + return ackErr + } + } + if credentialErr != nil { + return credentialErr + } + return workErr +} + +func safeFailureDetail(err error) string { + var upstream *github.APIError + if errors.As(err, &upstream) { + return fmt.Sprintf("GITHUB_HTTP_%d:%s", upstream.Status, upstream.Code) + } + if errors.Is(err, context.DeadlineExceeded) { + return "WORKER_TIMEOUT" + } + return "WORKER_ERROR" +} + +func reportCredentialProblem(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, err error) error { + // Official requests already persisted their lease-derived observation. The + // legacy RATE_LIMITED status cools the entire credential, including CORE + // after a SEARCH-only primary limit, so it must not be reported again. + if task.OfficialDiscovery != nil || repositoryNeedsOfficialControl(task) { + return nil + } + if task.Credential == nil || task.Credential.CredentialID == "" { + return nil + } + var upstream *github.APIError + if !errors.As(err, &upstream) { + return nil + } + quota := credentialQuota(upstream.Quota) + switch upstream.Code { + case "AUTH_INVALID": + return control.ReportCredentialStatus(ctx, task.TaskID, task.Lease, + task.Credential.CredentialID, "AUTH_INVALID", time.Time{}, quota) + case "GITHUB_RATE_LIMITED": + return control.ReportCredentialStatus(ctx, task.TaskID, task.Lease, + task.Credential.CredentialID, "RATE_LIMITED", futureRetry(upstream.RetryAt), quota) + default: + return nil + } +} + +func credentialQuota(observed github.Quota) *controlplane.GithubQuota { + if observed.Remaining < 0 || observed.ResetAt.IsZero() { + return nil + } + resource := strings.ToUpper(observed.Resource) + switch resource { + case "CORE", "SEARCH", "GRAPHQL": + default: + resource = "OTHER" + } + return &controlplane.GithubQuota{ + Resource: resource, Remaining: observed.Remaining, ResetAt: observed.ResetAt, + } +} + +func observeCredentialQuota(control *controlplane.Client, + task *controlplane.ClaimedTask) func(context.Context, github.Quota) { + if task.Credential == nil || task.Credential.CredentialID == "" { + return nil + } + return func(ctx context.Context, observed github.Quota) { + quota := credentialQuota(observed) + if quota == nil { + return + } + reportCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + _ = control.ReportCredentialStatus(reportCtx, task.TaskID, task.Lease, + task.Credential.CredentialID, "OBSERVED", time.Time{}, quota) + } +} + +func keepLease(ctx context.Context, cancel context.CancelFunc, control *controlplane.Client, + task *controlplane.ClaimedTask) error { + expiresAt := task.LeaseExpiresAt + for { + remaining := time.Until(expiresAt) + if remaining <= 0 { + cancel() + return errors.New("GitHub sync lease expired") + } + wait := remaining / 3 + if wait > time.Minute { + wait = time.Minute + } + if wait < time.Second { + wait = time.Second + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return nil + case <-timer.C: + } + var err error + expiresAt, err = control.Renew(ctx, task.TaskID, task.Lease) + if err != nil { + cancel() + return err + } + } +} + +func classifyFailure(err error, registeredSource bool) (string, time.Time) { + var upstream *github.APIError + if errors.As(err, &upstream) { + switch upstream.Code { + case "GITHUB_RATE_LIMITED": + return "GITHUB_RATE_LIMITED", futureRetry(upstream.RetryAt) + case "SOURCE_UNREADABLE": + if registeredSource { + return "SOURCE_UNREADABLE", time.Time{} + } + } + } + return "RETRYABLE_ACQUISITION", time.Time{} +} diff --git a/github-sync-go/internal/worker/lease_test.go b/github-sync-go/internal/worker/lease_test.go new file mode 100644 index 000000000..f208dccf4 --- /dev/null +++ b/github-sync-go/internal/worker/lease_test.go @@ -0,0 +1,54 @@ +package worker + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +func TestLeasedTaskStopsWorkWhenJavaRenewalFails(t *testing.T) { + var renewCalls atomic.Int32 + var otherCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/internal/github-sync/v1/tasks/fixture/renew" { + renewCalls.Add(1) + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusServiceUnavailable) + _, _ = response.Write([]byte(`{"code":"SERVICE_UNAVAILABLE"}`)) + return + } + otherCalls.Add(1) + response.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + client, err := controlplane.NewClient(server.URL+"/internal/github-sync/v1", + strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + task := &controlplane.ClaimedTask{ + TaskID: "fixture", Lease: controlplane.LeaseProof{Generation: 1, Token: "lease"}, + LeaseExpiresAt: time.Now().Add(3 * time.Second), + } + workReleased := false + err = executeLeasedTask(context.Background(), client, task, func(ctx context.Context) error { + defer func() { workReleased = true }() + <-ctx.Done() + return ctx.Err() + }) + var apiError *controlplane.APIError + if !errors.As(err, &apiError) || apiError.Code != "SERVICE_UNAVAILABLE" { + t.Fatalf("renewal failure was not returned: %v", err) + } + if !workReleased || renewCalls.Load() != 1 || otherCalls.Load() != 0 { + t.Fatalf("worker did not release work after failed renewal: released=%t renew=%d other=%d", + workReleased, renewCalls.Load(), otherCalls.Load()) + } +} diff --git a/github-sync-go/internal/worker/official_discovery.go b/github-sync-go/internal/worker/official_discovery.go new file mode 100644 index 000000000..c70e57dce --- /dev/null +++ b/github-sync-go/internal/worker/official_discovery.go @@ -0,0 +1,341 @@ +package worker + +import ( + "context" + "encoding/json" + "errors" + "math" + "slices" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +// These are continuation delays, not calibrated budget profile allocations. +const officialBudgetRetryDelay = time.Second + +type officialDiscoveryGitHub interface { + SearchOfficialRepositories(context.Context, github.OfficialRepositoryQuery, github.OfficialRepositoryRange, + time.Time, int) (github.SearchPage, github.Quota, error) +} + +// The outer checkpoint identifies immutable Java-owned scope; cursor holds only +// bounded page-local progress. Revisit makes replay of a changed page unambiguous. +type officialPageCursor struct { + Offset int `json:"offset"` + Revisit int64 `json:"revisit"` + GapReason string `json:"gapReason,omitempty"` +} + +func processOfficialDiscovery(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + limits DiscoveryLimits, upstream officialDiscoveryGitHub) error { + if task == nil || task.Kind != "DISCOVERY" || task.OfficialDiscovery == nil || task.Discovery != nil || + control == nil || upstream == nil || limits.MaxPagesPerLease < 1 { + return errors.New("invalid official discovery task") + } + if _, err := installOfficialQuotaGuard(control, task, upstream); err != nil { + return err + } + snapshot := task.OfficialDiscovery + for _, capability := range []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability} { + if !slices.Contains(task.RequiredCapabilities, capability) { + return errors.New("official discovery task omits required lifecycle capabilities") + } + } + if snapshot.Channel == "OWNER_EXPANSION" { + ownerAPI, ok := upstream.(officialOwnerGitHub) + if !ok { + return errors.New("GitHub client lacks owner enumeration") + } + return processOfficialOwner(ctx, control, task, limits, ownerAPI) + } + if snapshot.FormatVersion != 1 || snapshot.QueryID == "" || snapshot.QueryVersion < 1 || snapshot.ShardID == "" || + snapshot.StrategyVersion == "" || snapshot.Budget.CandidateBudget < 1 || snapshot.Budget.MaxPages < 1 || snapshot.Budget.MaxRequests < 1 || + (snapshot.Channel != "REPOSITORY_METADATA" && snapshot.Channel != "README") || + (snapshot.Channel == "README" && snapshot.Query.Scope != "README") || + (snapshot.Channel == "REPOSITORY_METADATA" && snapshot.Query.Scope != "NAME_DESCRIPTION_TOPICS") { + return errors.New("unsupported official discovery snapshot or channel") + } + query := github.OfficialRepositoryQuery{Kind: github.OfficialQueryKind(snapshot.Query.Kind), Value: snapshot.Query.Value, + Scope: github.OfficialQueryScope(snapshot.Query.Scope)} + window := github.OfficialRepositoryRange{Since: snapshot.Range.Since, Until: snapshot.Range.Until, Partition: snapshot.Range.Partition} + if _, err := github.CompileOfficialRepositoryQuery(query, window, snapshot.FixedCutoffAt); err != nil { + return err + } + checkpoint := snapshot.Checkpoint + paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if err != nil || paused { + return err + } + progress, cursor, recoveryReason := decodeOfficialProgress(snapshot) + if recoveryReason != "" { + _, err := control.RecoverOfficialCheckpoint(ctx, task.TaskID, controlplane.OfficialCheckpointRecovery{ + Lease: task.Lease, Reason: recoveryReason, RejectedCheckpoint: checkpoint, + }) + return err + } + if progress.Coverage == "COMPLETE" || progress.Coverage == "SPLIT" { + _, err := control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", task.OfficialDiscovery.Channel) + return err + } + if checkpoint == "" || progress.Coverage == "GAP" { + if progress.Coverage == "GAP" { + cursor.Revisit++ + } + progress.Coverage, progress.NextPage, cursor.Offset = "IN_PROGRESS", 1, 0 + cursor.GapReason = "" + checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 0, nil, "", "") + if err != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, err) + } + } + var requests, spent int64 + seen := make(map[int64]struct{}) + for { + if ctx.Err() != nil { + return ctx.Err() + } + paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if err != nil || paused { + return err + } + if requests >= min(snapshot.Budget.MaxPages, snapshot.Budget.MaxRequests, int64(limits.MaxPagesPerLease)) || spent >= snapshot.Budget.CandidateBudget { + return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialBudgetRetryDelay)) + } + page, _, searchErr := upstream.SearchOfficialRepositories(ctx, query, window, snapshot.FixedCutoffAt, int(progress.NextPage)) + requests++ + if searchErr != nil { + // Account for the attempted request without advancing the page/offset. + cursor.Revisit++ + checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 1, nil, "", "") + if err != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, err) + } + var upstreamError *github.APIError + if errors.As(searchErr, &upstreamError) && upstreamError.Code == "GITHUB_RATE_LIMITED" { + retryAt := upstreamError.RetryAt + if !retryAt.After(time.Now()) { + retryAt = time.Now().Add(time.Minute) + } + return completeOfficialDeferred(ctx, control, task, checkpoint, "RATE_LIMIT", retryAt) + } + return searchErr + } + if page.LimitReached || (page.HasMore && progress.NextPage >= 10) { + if cursor.GapReason == "" { + cursor.GapReason = "RESULT_LIMIT" + } + } + if page.IncompleteResults { + cursor.GapReason = "INCOMPLETE_RESULTS" + } + candidates := toCandidates(page.Repositories) + if cursor.Offset > len(candidates) { + cursor.Offset = 0 + cursor.Revisit++ + } + batch, end := selectDiscoveryBatch(candidates, cursor.Offset, int(min(snapshot.Budget.CandidateBudget-spent, 100)), seen) + provenance := controlplane.OfficialProvenance{Channel: snapshot.Channel, QueryID: snapshot.QueryID, + QueryVersion: snapshot.QueryVersion, StrategyVersion: snapshot.StrategyVersion, ShardID: snapshot.ShardID} + for index := range batch { + batch[index].OfficialProvenance = []controlplane.OfficialProvenance{provenance} + } + next, nextCursor := progress, cursor + var children []controlplane.OfficialQueryRange + gapReason := "" + if end < len(candidates) { + // Even an incomplete page may contain useful public repositories. + // Drain it across leases before handing off or recording a gap. + nextCursor.Offset = end + } else { + nextCursor.Offset = end + if cursor.GapReason != "" { + children = officialSplitRanges(snapshot) + } + if len(children) > 0 { + next.Coverage = "SPLIT" + } else if page.HasMore && progress.NextPage < 10 && cursor.GapReason != "INCOMPLETE_RESULTS" { + // A result cap still allows the accessible pages to be harvested. + // Retain the gap signal so page ten can never claim full coverage. + next.NextPage++ + nextCursor.Offset = 0 + } else if cursor.GapReason != "" { + next.Coverage, gapReason = "GAP", cursor.GapReason + if snapshot.Range.Until.Unix()-snapshot.Range.Since.Unix() < 2 { + gapReason = "UNSPLITTABLE" + } + } else { + next.Coverage = "COMPLETE" + } + } + acceptedCheckpoint, reportErr := reportOfficialProgress(ctx, control, task, checkpoint, next, nextCursor, batch, 1, children, gapReason, "") + if reportErr != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, reportErr) + } + checkpoint, progress, cursor = acceptedCheckpoint, next, nextCursor + for _, candidate := range batch { + seen[candidate.RepositoryID] = struct{}{} + } + spent += int64(len(batch)) + if progress.Coverage == "GAP" { + return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialGapBackoff(cursor.Revisit))) + } + if progress.Coverage == "COMPLETE" || progress.Coverage == "SPLIT" { + _, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", task.OfficialDiscovery.Channel) + return err + } + } +} + +func decodeOfficialProgress(snapshot *controlplane.OfficialDiscoverySnapshot) (controlplane.OfficialCheckpoint, officialPageCursor, string) { + progress := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: snapshot.StrategyVersion, + QueryID: snapshot.QueryID, QueryVersion: snapshot.QueryVersion, ShardID: snapshot.ShardID, NextPage: 1, Coverage: "IN_PROGRESS"} + cursor := officialPageCursor{} + if snapshot.Checkpoint == "" { + return progress, cursor, "" + } + progress = controlplane.OfficialCheckpoint{} + if len(snapshot.Checkpoint) > 4096 || json.Unmarshal([]byte(snapshot.Checkpoint), &progress) != nil { + return progress, cursor, "MALFORMED" + } + if progress.FormatVersion != 1 { + return progress, cursor, "FORMAT_UNSUPPORTED" + } + if progress.StrategyVersion != snapshot.StrategyVersion || progress.QueryID != snapshot.QueryID || + progress.QueryVersion != snapshot.QueryVersion || progress.ShardID != snapshot.ShardID { + return progress, cursor, "IDENTITY_MISMATCH" + } + ownerChannel := snapshot.Channel == "OWNER_EXPANSION" + if progress.NextPage < 1 || progress.NextPage > math.MaxInt32 || + (!ownerChannel && (progress.NextPage > 11 || (progress.NextPage == 11 && progress.Coverage != "COMPLETE" && progress.Coverage != "SPLIT"))) || + (ownerChannel && progress.Coverage == "SPLIT") || + !slices.Contains([]string{"IN_PROGRESS", "COMPLETE", "SPLIT", "GAP"}, progress.Coverage) || + len(progress.Cursor) > 1024 || (progress.Cursor != "" && json.Unmarshal([]byte(progress.Cursor), &cursor) != nil) || + cursor.Offset < 0 || cursor.Offset > 100 || cursor.Revisit < 0 || cursor.Revisit >= math.MaxInt64-16 { + return progress, cursor, "MALFORMED" + } + if progress.Cursor != "" { + var fields map[string]json.RawMessage + if json.Unmarshal([]byte(progress.Cursor), &fields) != nil || (len(fields) != 2 && len(fields) != 3) || + len(fields["offset"]) == 0 || len(fields["revisit"]) == 0 || + string(fields["offset"]) == "null" || string(fields["revisit"]) == "null" || + (len(fields) == 3 && len(fields["gapReason"]) == 0) || + (cursor.GapReason != "" && !slices.Contains([]string{"RESULT_LIMIT", "INCOMPLETE_RESULTS"}, cursor.GapReason)) { + return progress, cursor, "MALFORMED" + } + } + return progress, cursor, "" +} + +func reportOfficialProgress(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + previous string, progress controlplane.OfficialCheckpoint, cursor officialPageCursor, + repositories []controlplane.RepositoryCandidate, requests int64, children []controlplane.OfficialQueryRange, gapReason, ownerLogin string) (string, error) { + if ownerLogin != "" && task.OfficialDiscovery.Channel != "OWNER_EXPANSION" { + return previous, errors.New("only owner expansion can report canonical owner login") + } + encodedCursor, err := json.Marshal(cursor) + if err != nil { + return previous, err + } + progress.Cursor = string(encodedCursor) + encoded, err := json.Marshal(progress) + if err != nil || len(encoded) > 4096 || string(encoded) == previous { + return previous, errors.New("official discovery checkpoint did not advance or exceeds limit") + } + var previousCheckpoint *string + if previous != "" { + previousCheckpoint = &previous + } + batch := controlplane.DiscoveryBatch{ + PreviousCheckpoint: previousCheckpoint, Checkpoint: string(encoded), Repositories: repositories, + OfficialBatch: &controlplane.OfficialDiscoveryBatch{ShardID: progress.ShardID, Coverage: progress.Coverage, + RequestsUsed: requests, ChildRanges: children, GapReason: gapReason, OwnerLogin: ownerLogin}, + } + err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch) + if err != nil && ctx.Err() == nil { + var apiError *controlplane.APIError + if !errors.As(err, &apiError) || apiError.Status >= 500 { + // The first response may have been lost after commit. Replay the + // identical payload; Java's batch digest prevents double accounting. + err = control.ReportDiscoveryBatch(ctx, task.TaskID, task.Lease, batch) + } + } + if err != nil { + return previous, err + } + return string(encoded), nil +} + +// Child tasks are registered atomically by Java. The snapshot carries remaining +// depth and root-wide child capacity; zero/one remaining child cannot bisect. +// Go never scans unacknowledged children; Java rechecks the root budget on write. +func officialSplitRanges(snapshot *controlplane.OfficialDiscoverySnapshot) []controlplane.OfficialQueryRange { + start, end := snapshot.Range.Since.Unix(), snapshot.Range.Until.Unix() + if snapshot.Budget.MaxSplitDepth < 1 || snapshot.Budget.MaxChildShards < 2 || end-start < 2 { + return nil + } + middle := time.Unix(start+(end-start)/2, 0).UTC() + return []controlplane.OfficialQueryRange{ + {Since: snapshot.Range.Since, Until: middle}, + {Since: middle, Until: snapshot.Range.Until}, + } +} + +func pauseOfficialDiscoveryIfNeeded(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, checkpoint string) (bool, error) { + for attempts := 0; attempts < 3; attempts++ { + state, err := control.GetOfficialTaskControl(ctx, task.TaskID, task.Lease) + if err != nil || state.Action == "CONTINUE" { + return false, err + } + var acknowledged *string + if checkpoint != "" { + acknowledged = &checkpoint + } + err = control.PauseOfficialDiscovery(ctx, task.TaskID, task.Lease, acknowledged) + if err == nil { + return true, nil + } + var apiError *controlplane.APIError + if !errors.As(err, &apiError) || apiError.Code != "CONTROL_CHANGED" { + return false, err + } + } + return false, errors.New("official discovery control changed repeatedly") +} + +// A refused batch never advances the acknowledged checkpoint. Control is +// re-read before deciding whether to pause or release the lease for capacity. +func handleOfficialBatchFailure(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + checkpoint string, reportErr error) error { + var apiError *controlplane.APIError + if !errors.As(reportErr, &apiError) || + (apiError.Code != "BACKPRESSURE_ACTIVE" && apiError.Code != "CANDIDATE_BUDGET_EXCEEDED") { + return reportErr + } + paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if err != nil || paused { + return err + } + retryAt := time.Now().Add(time.Minute) + if checkpoint == "" { + // Complete requires a persisted checkpoint. If the very first batch + // was refused, release through fail without inventing any progress. + _, err := control.Fail(ctx, task.TaskID, task.Lease, "RETRYABLE_ACQUISITION", + "", retryAt, nil) + return err + } + return completeOfficialDeferred(ctx, control, task, checkpoint, "CAPACITY", retryAt) +} + +func completeOfficialDeferred(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + checkpoint, reason string, retryAt time.Time) error { + _, err := control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, retryAt, reason, task.OfficialDiscovery.Channel) + return err +} + +// A repeatedly unresolved fixed range retains its retry generation across +// leases. Normal work-budget exhaustion does not use this gap backoff. +func officialGapBackoff(revisit int64) time.Duration { + return min(5*time.Minute*time.Duration(int64(1)< 0 { + h.pause = true + w.WriteHeader(409) + _, _ = w.Write([]byte(`{"code":"BACKPRESSURE_ACTIVE"}`)) + return + } + if b.Checkpoint != h.accepted { + previous := "" + if b.PreviousCheckpoint != nil { + previous = *b.PreviousCheckpoint + } + if previous != h.accepted { + t.Errorf("CAS previous %q != accepted %q", previous, h.accepted) + } + h.accepted = b.Checkpoint + } + if h.loseCandidateResponse && len(b.Repositories) > 0 { + h.loseCandidateResponse = false + w.WriteHeader(503) + return + } + w.WriteHeader(202) + case strings.HasSuffix(r.URL.Path, "/official-pause"): + var body struct{ DiscoveryCheckpoint *string } + _ = json.NewDecoder(r.Body).Decode(&body) + h.paused = body.DiscoveryCheckpoint + _, _ = w.Write([]byte(`{"status":"PAUSED"}`)) + case strings.HasSuffix(r.URL.Path, "/complete"): + var body struct { + Coverage string + DiscoveryCheckpoint string + RetryAt time.Time + } + _ = json.NewDecoder(r.Body).Decode(&body) + h.completions = append(h.completions, body) + if body.DiscoveryCheckpoint != h.accepted { + t.Error("completion did not retain accepted checkpoint") + } + status := "ACQUISITION_COMPLETE" + if body.Coverage == "INCOMPLETE" { + status = "RETRY_WAIT" + var cp controlplane.OfficialCheckpoint + _ = json.Unmarshal([]byte(h.accepted), &cp) + if cp.Coverage == "GAP" && h.channel != "OWNER_EXPANSION" { + status = "PARTIAL" + } + } + _ = json.NewEncoder(w).Encode(map[string]string{"status": status}) + default: + t.Errorf("unexpected control path %s", r.URL.Path) + w.WriteHeader(404) + } + })) + t.Cleanup(server.Close) + var err error + h.control, err = controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + return h +} +func officialTask(channel string) *controlplane.ClaimedTask { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + task := &controlplane.ClaimedTask{TaskID: "official-task", Kind: "DISCOVERY", Lease: controlplane.LeaseProof{Generation: 1, Token: strings.Repeat("a", 64)}, RequiredCapabilities: []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability}, OfficialDiscovery: &controlplane.OfficialDiscoverySnapshot{FormatVersion: 1, ConfigurationVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", Channel: channel, Mode: "HISTORICAL", Query: controlplane.OfficialQuery{Kind: "KEYWORD", Value: "skills", Scope: "NAME_DESCRIPTION_TOPICS"}, Range: controlplane.OfficialQueryRange{Since: start, Until: start.Add(8 * time.Second)}, FixedCutoffAt: start.Add(time.Minute), Budget: controlplane.OfficialBudget{Profile: "BALANCED", MaxRequests: 10, MaxPages: 10, CandidateBudget: 100}}} + if channel == "OWNER_EXPANSION" { + s := task.OfficialDiscovery + s.Mode = "OWNER_RECHECK" + s.OwnerID = 42 + s.OwnerLogin = "old-name" + s.SeedRepositoryID = 11 + s.ExpansionDepth = 1 + s.Query = controlplane.OfficialQuery{Kind: "OWNER", Value: "old-name", Scope: "OWNER_REPOSITORIES"} + task.RequiredCapabilities = append(task.RequiredCapabilities, controlplane.OfficialOwnerCapability) + } + return task +} +func officialRepo(id, ownerID int64) github.Repository { + r := github.Repository{ID: id, Name: "skills"} + r.Owner.Login = "new-name" + r.Owner.ID = ownerID + return r +} +func officialRun(t *testing.T, h *officialHarness, task *controlplane.ClaimedTask, f *officialFixture) { + t.Helper() + if err := processOfficialDiscovery(context.Background(), h.control, task, DiscoveryLimits{MaxPagesPerLease: 10}, f); err != nil { + t.Fatal(err) + } +} +func decodedCP(t *testing.T, s string) controlplane.OfficialCheckpoint { + t.Helper() + var cp controlplane.OfficialCheckpoint + if err := json.Unmarshal([]byte(s), &cp); err != nil { + t.Fatal(err) + } + return cp +} + +func TestOfficialSearchPaginationAndReplay(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + h.loseCandidateResponse = true + f := &officialFixture{search: func(page int) (github.SearchPage, error) { + return github.SearchPage{Repositories: []github.Repository{officialRepo(int64(page), 42)}, HasMore: page < 2}, nil + }} + officialRun(t, h, officialTask(h.channel), f) + if f.searches != 2 || len(h.attempts) != 4 || len(h.completions) != 1 { + t.Fatalf("searches=%d batches=%d completions=%d", f.searches, len(h.attempts), len(h.completions)) + } + first, _ := json.Marshal(h.attempts[1]) + replay, _ := json.Marshal(h.attempts[2]) + if string(first) != string(replay) { + t.Fatal("response-loss replay changed payload") + } + if cp := decodedCP(t, h.accepted); cp.Coverage != "COMPLETE" || cp.NextPage != 2 { + t.Fatalf("checkpoint %+v", cp) + } +} +func TestOfficialRejectedBatchPausesLastAcknowledgedCheckpoint(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + h.rejectCandidate = true + f := &officialFixture{search: func(int) (github.SearchPage, error) { + return github.SearchPage{Repositories: []github.Repository{officialRepo(1, 42)}}, nil + }} + officialRun(t, h, officialTask(h.channel), f) + if h.paused == nil || *h.paused != h.attempts[0].Checkpoint || *h.paused == h.attempts[1].Checkpoint || len(h.completions) != 0 { + t.Fatal("rejected candidate advanced checkpoint or completed") + } +} +func TestOfficialSearchCoverageBoundaries(t *testing.T) { + for _, kind := range []string{"split", "gap", "unsplittable"} { + t.Run(kind, func(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + task := officialTask(h.channel) + want := "GAP" + if kind == "split" { + task.OfficialDiscovery.Budget.MaxSplitDepth = 1 + task.OfficialDiscovery.Budget.MaxChildShards = 2 + want = "SPLIT" + } + if kind == "unsplittable" { + task.OfficialDiscovery.Range.Until = task.OfficialDiscovery.Range.Since.Add(time.Second) + } + f := &officialFixture{search: func(int) (github.SearchPage, error) { + return github.SearchPage{Repositories: []github.Repository{officialRepo(1, 42)}, IncompleteResults: true}, nil + }} + officialRun(t, h, task, f) + b := h.attempts[len(h.attempts)-1] + if b.OfficialBatch.Coverage != want || len(b.Repositories) != 1 { + t.Fatalf("lost coverage/candidates: %+v", b) + } + if want == "SPLIT" { + ranges := b.OfficialBatch.ChildRanges + if len(ranges) != 2 || !ranges[0].Since.Equal(task.OfficialDiscovery.Range.Since) || !ranges[0].Until.Equal(ranges[1].Since) || !ranges[1].Until.Equal(task.OfficialDiscovery.Range.Until) { + t.Fatal("split does not partition parent") + } + } else if !h.completions[0].RetryAt.After(time.Now().Add(4 * time.Minute)) { + t.Fatal("GAP lacks backoff") + } + if kind == "unsplittable" && b.OfficialBatch.GapReason != "UNSPLITTABLE" { + t.Fatal("missing unsplittable reason") + } + }) + } +} +func TestOfficialSplitResumeDoesNotSearch(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + task := officialTask(h.channel) + cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 1, Coverage: "SPLIT", Cursor: `{"offset":0,"revisit":0}`} + b, _ := json.Marshal(cp) + task.OfficialDiscovery.Checkpoint = string(b) + h.accepted = string(b) + f := &officialFixture{search: func(int) (github.SearchPage, error) { + t.Fatal("resumed parent was searched") + return github.SearchPage{}, nil + }} + officialRun(t, h, task, f) + if f.searches != 0 || len(h.completions) != 1 { + t.Fatal("split resume did not only finish parent") + } +} +func TestOfficialOwnerIdentityAndDepth(t *testing.T) { + for _, kind := range []string{"renamed", "wrong-repo-owner", "rename-at-empty-tail", "recursive"} { + t.Run(kind, func(t *testing.T) { + h := newOfficialHarness(t, "OWNER_EXPANSION") + task := officialTask(h.channel) + f := &officialFixture{owner: func(call int) github.OwnerIdentity { + login := "new-name" + if kind == "rename-at-empty-tail" && call == 2 { + login = "newer-name" + } + return github.OwnerIdentity{ID: 42, Login: login, Type: github.UserOwner} + }, list: func(login string, page int) github.OwnerPage { + if login != "new-name" { + t.Error("used stale snapshot login") + } + if kind == "rename-at-empty-tail" { + return github.OwnerPage{} + } + id := int64(42) + if kind == "wrong-repo-owner" { + id = 99 + } + return github.OwnerPage{Repositories: []github.Repository{officialRepo(9, id)}} + }} + if kind == "recursive" { + task.OfficialDiscovery.ExpansionDepth = 2 + if err := processOfficialDiscovery(context.Background(), h.control, task, DiscoveryLimits{MaxPagesPerLease: 10}, f); err == nil || f.owners != 0 { + t.Fatal("recursive expansion made requests") + } + return + } + officialRun(t, h, task, f) + if h.attempts[0].OfficialBatch.OwnerLogin != "new-name" { + t.Fatal("canonical login not persisted before enumeration") + } + cp := decodedCP(t, h.accepted) + if kind == "renamed" { + if cp.Coverage != "COMPLETE" || f.owners != 2 || h.attempts[len(h.attempts)-1].Repositories[0].OfficialProvenance[0].SeedRepositoryID != 11 { + t.Fatal("valid owner expansion incomplete") + } + } else if cp.Coverage != "IN_PROGRESS" { + t.Fatal("identity race falsely completed") + } + }) + } +} +func TestOfficialOwnerResumeBeyondSearchLimit(t *testing.T) { + h := newOfficialHarness(t, "OWNER_EXPANSION") + task := officialTask(h.channel) + cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 12, Coverage: "IN_PROGRESS", Cursor: `{"offset":0,"revisit":0}`} + b, _ := json.Marshal(cp) + task.OfficialDiscovery.Checkpoint = string(b) + h.accepted = string(b) + f := &officialFixture{owner: func(int) github.OwnerIdentity { + return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner} + }, list: func(_ string, page int) github.OwnerPage { + if page != 12 { + t.Error("lost resumed page") + } + return github.OwnerPage{} + }} + officialRun(t, h, task, f) + if decodedCP(t, h.accepted).Coverage != "COMPLETE" { + t.Fatal("owner page beyond ten rejected") + } +} +func TestOfficialQuotaPersistenceFailureStopsLeaseAndOtherTokens(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + h.quotaFailure = true + f := &officialFixture{search: func(int) (github.SearchPage, error) { return github.SearchPage{HasMore: true}, nil }} + err := processOfficialDiscovery(context.Background(), h.control, officialTask(h.channel), DiscoveryLimits{MaxPagesPerLease: 10}, f) + if err == nil || f.searches != 1 || len(h.observations) != 1 { + t.Fatalf("did not fail closed: %v searches=%d", err, f.searches) + } + if err := (&officialQuotaGuard{resource: "CORE"}).before(context.Background()); err == nil { + t.Fatal("different resource/token bypassed persistence cooldown") + } +} +func TestOfficialQuotaResourceAndAnonymousSemantics(t *testing.T) { + for _, kind := range []string{"primary", "secondary", "anonymous-unknown", "git-failure", "auth"} { + t.Run(kind, func(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + g := &officialQuotaGuard{control: h.control, task: officialTask(h.channel), resource: "SEARCH"} + quota := github.Quota{Resource: "SEARCH", Remaining: 0, ResetAt: time.Now().Add(time.Hour)} + var err error + switch kind { + case "primary": + err = g.after(context.Background(), quota, nil) + case "secondary": + quota.Remaining = 5 + err = g.after(context.Background(), quota, &github.APIError{Code: "GITHUB_RATE_LIMITED", SecondaryRateLimit: true, RetryAt: time.Now().Add(time.Minute)}) + case "anonymous-unknown": + err = g.after(context.Background(), github.Quota{Remaining: -1}, nil) + case "git-failure": + err = g.afterGit(context.Background(), errors.New("git failed")) + case "auth": + err = g.after(context.Background(), github.Quota{Remaining: -1}, &github.APIError{Code: "AUTH_INVALID"}) + } + obs := h.observations[0] + if kind == "anonymous-unknown" { + if err != nil || obs.State != "UNKNOWN" || obs.Remaining != nil || obs.ResetAt != nil { + t.Fatal("unknown manufactured quota") + } + return + } + if err == nil { + t.Fatal("limit/error did not stop current lease") + } + coreErr := (&officialQuotaGuard{resource: "CORE"}).before(context.Background()) + if (kind == "primary") != (coreErr == nil) { + t.Fatal("primary resource isolation or secondary global sharing broken") + } + if kind == "git-failure" && (obs.Resource != "OTHER" || obs.State != "UNKNOWN" || obs.RetryAfterSeconds == nil) { + t.Fatal("git invented REST quota") + } + }) + } +} + +func TestOfficialSearchLimitHarvestsAccessiblePagesBeforeGap(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + f := &officialFixture{search: func(page int) (github.SearchPage, error) { + return github.SearchPage{Repositories: []github.Repository{officialRepo(int64(page), 42)}, HasMore: true, LimitReached: true}, nil + }} + officialRun(t, h, officialTask(h.channel), f) + last := h.attempts[len(h.attempts)-1] + if f.searches != 10 || last.OfficialBatch.Coverage != "GAP" || last.OfficialBatch.GapReason != "RESULT_LIMIT" { + t.Fatal("accessible search pages discarded or falsely complete") + } + total := 0 + for _, b := range h.attempts { + total += len(b.Repositories) + } + if total != 10 { + t.Fatalf("stored %d candidates", total) + } +} +func TestOfficialOwnerBudgetResumeAndPause(t *testing.T) { + h := newOfficialHarness(t, "OWNER_EXPANSION") + task := officialTask(h.channel) + task.OfficialDiscovery.Budget.MaxRequests = 3 + task.OfficialDiscovery.Budget.CandidateBudget = 1 + f := &officialFixture{owner: func(int) github.OwnerIdentity { + return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner} + }, list: func(_ string, page int) github.OwnerPage { + return github.OwnerPage{Repositories: []github.Repository{officialRepo(1, 42), officialRepo(2, 42)}} + }} + officialRun(t, h, task, f) + cp := decodedCP(t, h.accepted) + var cursor officialPageCursor + _ = json.Unmarshal([]byte(cp.Cursor), &cursor) + if cp.Coverage != "IN_PROGRESS" || cp.NextPage != 1 || cursor.Offset != 1 || f.owners != 1 { + t.Fatal("candidate budget lost page offset") + } + task.OfficialDiscovery.Checkpoint = h.accepted + task.Lease.Generation++ + h.pause = true + officialRun(t, h, task, f) + if h.paused == nil || *h.paused != task.OfficialDiscovery.Checkpoint || f.lists != 1 { + t.Fatal("paused resume performed upstream request") + } + h.pause = false + officialRun(t, h, task, f) + last := h.attempts[len(h.attempts)-1] + if len(last.Repositories) != 1 || last.Repositories[0].RepositoryID != 2 || last.OfficialBatch.Coverage != "COMPLETE" || last.OfficialBatch.RequestsUsed != 2 { + t.Fatal("resume duplicated candidate or skipped tail identity cost") + } +} +func TestOfficialSearchFailurePersistsAttemptWithoutAdvancingPage(t *testing.T) { + h := newOfficialHarness(t, "REPOSITORY_METADATA") + f := &officialFixture{search: func(int) (github.SearchPage, error) { + return github.SearchPage{}, errors.New("controlled upstream error") + }} + if err := processOfficialDiscovery(context.Background(), h.control, officialTask(h.channel), DiscoveryLimits{MaxPagesPerLease: 10}, f); err == nil { + t.Fatal("upstream error swallowed") + } + last := h.attempts[len(h.attempts)-1] + cp := decodedCP(t, h.accepted) + if len(h.attempts) != 2 || last.OfficialBatch.RequestsUsed != 1 || cp.NextPage != 1 || cp.Coverage != "IN_PROGRESS" || len(h.completions) != 0 { + t.Fatal("failed search advanced or lost request accounting") + } +} + +func TestOfficialOwnerPageOverflowIsGap(t *testing.T) { + h := newOfficialHarness(t, "OWNER_EXPANSION") + task := officialTask(h.channel) + cp := controlplane.OfficialCheckpoint{FormatVersion: 1, StrategyVersion: "v1", QueryID: "query", QueryVersion: 1, ShardID: "shard", NextPage: 2147483647, Coverage: "IN_PROGRESS", Cursor: `{"offset":0,"revisit":0}`} + encoded, _ := json.Marshal(cp) + task.OfficialDiscovery.Checkpoint = string(encoded) + h.accepted = string(encoded) + f := &officialFixture{owner: func(int) github.OwnerIdentity { + return github.OwnerIdentity{ID: 42, Login: "new-name", Type: github.UserOwner} + }, list: func(_ string, page int) github.OwnerPage { + if page != 2147483647 { + t.Fatal("page overflow before request") + } + return github.OwnerPage{HasMore: true} + }} + officialRun(t, h, task, f) + last := h.attempts[len(h.attempts)-1] + if decodedCP(t, h.accepted).NextPage != 2147483647 || last.OfficialBatch.Coverage != "GAP" || last.OfficialBatch.GapReason != "RESULT_LIMIT" || h.completions[0].Coverage != "INCOMPLETE" { + t.Fatal("owner overflow claimed completion") + } +} diff --git a/github-sync-go/internal/worker/official_owner.go b/github-sync-go/internal/worker/official_owner.go new file mode 100644 index 000000000..cb26741ca --- /dev/null +++ b/github-sync-go/internal/worker/official_owner.go @@ -0,0 +1,180 @@ +package worker + +import ( + "context" + "errors" + "math" + "slices" + "strings" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type officialOwnerGitHub interface { + GetOwnerByID(context.Context, int64) (github.OwnerIdentity, github.Quota, error) + ListPublicOwnerRepositories(context.Context, string, github.OwnerType, int) (github.OwnerPage, github.Quota, error) +} + +// Owner rechecks enumerate a changing public collection, not a pushed-time window. +// Stable IDs fence account renames and login reuse between API requests. +func processOfficialOwner(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + limits DiscoveryLimits, upstream officialOwnerGitHub) error { + snapshot := task.OfficialDiscovery + if !slices.Contains(task.RequiredCapabilities, controlplane.OfficialOwnerCapability) || + snapshot.FormatVersion != 1 || snapshot.QueryID == "" || snapshot.QueryVersion < 1 || snapshot.ShardID == "" || snapshot.StrategyVersion == "" || + snapshot.OwnerID < 1 || snapshot.SeedRepositoryID < 1 || snapshot.ExpansionDepth != 1 || + snapshot.OwnerLogin == "" || snapshot.Query.Value != snapshot.OwnerLogin || snapshot.Range.Partition != "" || + snapshot.Mode != "OWNER_RECHECK" || snapshot.Query.Kind != "OWNER" || snapshot.Query.Scope != "OWNER_REPOSITORIES" || + snapshot.Budget.MaxRequests < 3 || snapshot.Budget.MaxPages < 1 || snapshot.Budget.CandidateBudget < 1 || + snapshot.Budget.MaxSplitDepth != 0 || snapshot.Budget.MaxChildShards != 0 { + return errors.New("invalid official owner expansion snapshot") + } + checkpoint := snapshot.Checkpoint + paused, err := pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if paused || err != nil { + return err + } + progress, cursor, recoveryReason := decodeOfficialProgress(snapshot) + if recoveryReason != "" { + _, err = control.RecoverOfficialCheckpoint(ctx, task.TaskID, controlplane.OfficialCheckpointRecovery{ + Lease: task.Lease, Reason: recoveryReason, RejectedCheckpoint: checkpoint, + }) + return err + } + if progress.Coverage == "COMPLETE" { + _, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", snapshot.Channel) + return err + } + if progress.Coverage == "GAP" { + cursor.Revisit++ + progress.Coverage, progress.NextPage, cursor.Offset, cursor.GapReason = "IN_PROGRESS", 1, 0, "" + } + owner, _, lookupErr := upstream.GetOwnerByID(ctx, snapshot.OwnerID) + canonicalLogin := "" + if lookupErr == nil { + if owner.ID != snapshot.OwnerID || owner.Login == "" { + lookupErr = errors.New("owner identity changed") + } else { + canonicalLogin = owner.Login + } + } + cursor.Revisit++ + checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, 1, nil, "", canonicalLogin) + if err != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, err) + } + if lookupErr != nil { + return deferOfficialOwnerError(ctx, control, task, checkpoint, lookupErr) + } + requests, pages, spent := int64(1), int64(0), int64(0) + seen := make(map[int64]struct{}) + for { + paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if paused || err != nil { + return err + } + // Reserve a second request to verify even an empty last page. With a minimum + // budget of three, a fresh lease can always resolve identity and finish a page. + if requests+2 > snapshot.Budget.MaxRequests || pages >= min(snapshot.Budget.MaxPages, int64(limits.MaxPagesPerLease)) || spent >= snapshot.Budget.CandidateBudget { + return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialBudgetRetryDelay)) + } + page, _, pageErr := upstream.ListPublicOwnerRepositories(ctx, owner.Login, owner.Type, int(progress.NextPage)) + requests++ + pages++ + used := int64(1) + if pageErr == nil { + for _, repository := range page.Repositories { + if repository.Owner.ID != snapshot.OwnerID { + pageErr = errors.New("repository owner identity changed during enumeration") + break + } + } + } + if pageErr != nil { + cursor.Revisit++ + checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, used, nil, "", canonicalLogin) + if err != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, err) + } + return deferOfficialOwnerError(ctx, control, task, checkpoint, pageErr) + } + candidates := toCandidates(page.Repositories) + if cursor.Offset > len(candidates) { + cursor.Offset = 0 + cursor.Revisit++ + } + batch, end := selectDiscoveryBatch(candidates, cursor.Offset, int(min(snapshot.Budget.CandidateBudget-spent, 100)), seen) + next, nextCursor := progress, cursor + nextCursor.Offset = end + gapReason := "" + if end == len(candidates) { + if page.HasMore { + if progress.NextPage == math.MaxInt32 { + next.Coverage, gapReason = "GAP", "RESULT_LIMIT" + } else { + next.NextPage++ + nextCursor.Offset = 0 + } + } else { + paused, err = pauseOfficialDiscoveryIfNeeded(ctx, control, task, checkpoint) + if paused || err != nil { + return err + } + verified, _, verifyErr := upstream.GetOwnerByID(ctx, snapshot.OwnerID) + requests++ + used++ + if verifyErr == nil && (verified.ID != owner.ID || verified.Type != owner.Type || !strings.EqualFold(verified.Login, owner.Login)) { + verifyErr = errors.New("owner identity changed before final page confirmation") + } + if verifyErr != nil { + // Retain the page; no candidate or completion from an unconfirmed tail is + // committed. Retry resolves the canonical login again using the stable ID. + cursor.Revisit++ + checkpoint, err = reportOfficialProgress(ctx, control, task, checkpoint, progress, cursor, nil, used, nil, "", canonicalLogin) + if err != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, err) + } + return deferOfficialOwnerError(ctx, control, task, checkpoint, verifyErr) + } + next.Coverage = "COMPLETE" + } + } + provenance := controlplane.OfficialProvenance{Channel: snapshot.Channel, QueryID: snapshot.QueryID, + QueryVersion: snapshot.QueryVersion, StrategyVersion: snapshot.StrategyVersion, ShardID: snapshot.ShardID, + SeedRepositoryID: snapshot.SeedRepositoryID} + for i := range batch { + batch[i].OfficialProvenance = []controlplane.OfficialProvenance{provenance} + } + accepted, reportErr := reportOfficialProgress(ctx, control, task, checkpoint, next, nextCursor, batch, used, nil, gapReason, canonicalLogin) + if reportErr != nil { + return handleOfficialBatchFailure(ctx, control, task, checkpoint, reportErr) + } + checkpoint, progress, cursor = accepted, next, nextCursor + for _, candidate := range batch { + seen[candidate.RepositoryID] = struct{}{} + } + spent += int64(len(batch)) + if progress.Coverage == "COMPLETE" { + _, err = control.CompleteOfficialDiscovery(ctx, task.TaskID, task.Lease, checkpoint, time.Time{}, "", snapshot.Channel) + return err + } + if progress.Coverage == "GAP" { + return completeOfficialDeferred(ctx, control, task, checkpoint, "BUDGET", time.Now().Add(officialGapBackoff(cursor.Revisit))) + } + } +} + +func deferOfficialOwnerError(ctx context.Context, control *controlplane.Client, task *controlplane.ClaimedTask, + checkpoint string, acquisitionErr error) error { + reason, retryAt := "BUDGET", time.Now().Add(time.Minute) + var apiError *github.APIError + if errors.As(acquisitionErr, &apiError) && apiError.Code == "GITHUB_RATE_LIMITED" { + reason = "RATE_LIMIT" + if apiError.RetryAt.After(retryAt) { + retryAt = apiError.RetryAt + } + } + return completeOfficialDeferred(ctx, control, task, checkpoint, reason, retryAt) +} diff --git a/github-sync-go/internal/worker/official_process_interop_test.go b/github-sync-go/internal/worker/official_process_interop_test.go new file mode 100644 index 000000000..1bc238401 --- /dev/null +++ b/github-sync-go/internal/worker/official_process_interop_test.go @@ -0,0 +1,68 @@ +package worker + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +// This test is compiled and launched as a separate OS process by the Java +// PostgreSQL interop test. Only GitHub is a fixture; control-plane HTTP is real. +func TestInteropOfficialDiscoveryProcess(t *testing.T) { + phase := os.Getenv("SKILLHUB_INTEROP_OFFICIAL_PROCESS") + if phase == "" { + t.Skip("requires Java official discovery process fixture") + } + if phase != "before-kill" && phase != "resume" { + t.Fatal("invalid interop phase") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + if err := control.HeartbeatWithCapabilities(context.Background(), "interop-official-"+phase, supportedCapabilities()); err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-official-"+phase, "DISCOVERY", supportedCapabilities()) + if err != nil { + t.Fatal(err) + } + if task == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || task.OfficialDiscovery == nil || task.Credential != nil { + t.Fatal("Java did not return expected anonymous official task") + } + if phase == "resume" { + var cp controlplane.OfficialCheckpoint + if json.Unmarshal([]byte(task.OfficialDiscovery.Checkpoint), &cp) != nil || cp.NextPage != 2 || cp.Coverage != "IN_PROGRESS" || task.Lease.Generation < 2 { + t.Fatal("claim lost persisted checkpoint or generation") + } + } else if task.OfficialDiscovery.Checkpoint != "" { + t.Fatal("first process received unexpected prior progress") + } + fixture := &officialFixture{search: func(page int) (github.SearchPage, error) { + if phase == "before-kill" && page == 2 { + if err := os.WriteFile(os.Getenv("SKILLHUB_INTEROP_BARRIER"), []byte("PAGE_TWO_AFTER_ACK"), 0600); err != nil { + t.Fatal(err) + } + select {} // Java kills this exact executable, including its renewal goroutine. + } + if (phase == "before-kill" && page != 1) || (phase == "resume" && page != 2) { + t.Fatalf("unexpected %s search page %d", phase, page) + } + repository := officialRepo(987654900+int64(page), 42) + repository.DefaultBranch = "main" + return github.SearchPage{Repositories: []github.Repository{repository}, HasMore: page == 1}, nil + }} + err = executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processOfficialDiscovery(ctx, control, task, DiscoveryLimits{MaxPagesPerLease: 10}, fixture) + }) + if err != nil { + t.Fatal(err) + } + if phase != "resume" || fixture.searches != 1 { + t.Fatal("process did not resume exclusively from second page") + } +} diff --git a/github-sync-go/internal/worker/official_quota.go b/github-sync-go/internal/worker/official_quota.go new file mode 100644 index 000000000..ca8379a9b --- /dev/null +++ b/github-sync-go/internal/worker/official_quota.go @@ -0,0 +1,192 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "sync" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +// A single bounded process gate prevents token rotation from bypassing secondary +// cooldown or a failed persistence attempt. Primary limits are scoped to the credential revision and resource. +var officialProcessQuota = struct { + sync.Mutex + globalUntil time.Time + resourceUntil map[string]time.Time + failures int + lastFailure time.Time +}{resourceUntil: make(map[string]time.Time)} + +type officialRequestPolicy interface { + SetRequestPolicy(func(context.Context) error, func(context.Context, github.Quota, error) error) +} + +type officialQuotaGuard struct { + control *controlplane.Client + task *controlplane.ClaimedTask + resource string + stopped error +} + +func installOfficialQuotaGuard(control *controlplane.Client, task *controlplane.ClaimedTask, upstream any) (*officialQuotaGuard, error) { + client, ok := upstream.(officialRequestPolicy) + if !ok { + return nil, errors.New("official GitHub client lacks quota request policy") + } + resource := "CORE" + if task.OfficialDiscovery != nil && task.OfficialDiscovery.Channel != "OWNER_EXPANSION" { + resource = "SEARCH" + } + guard := &officialQuotaGuard{control: control, task: task, resource: resource} + client.SetRequestPolicy(guard.before, guard.after) + return guard, nil +} + +func (g *officialQuotaGuard) before(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + if g.stopped != nil { + return g.stopped + } + officialProcessQuota.Lock() + blocked := time.Now().Before(officialProcessQuota.globalUntil) || time.Now().Before(officialProcessQuota.resourceUntil[g.quotaKey(g.resource)]) || time.Now().Before(officialProcessQuota.resourceUntil[g.quotaKey("*")]) + officialProcessQuota.Unlock() + if blocked { + g.stopped = errors.New("official GitHub requests are in process quota cooldown") + } + return g.stopped +} + +func (g *officialQuotaGuard) after(ctx context.Context, quota github.Quota, requestErr error) error { + return g.observe(ctx, quota, requestErr, g.resource) +} + +func (g *officialQuotaGuard) afterGit(ctx context.Context, requestErr error) error { + // Git HTTPS exposes no REST quota headers. UNKNOWN is not a fabricated CORE + // observation; a failed transport requests a conservative shared cooldown. + return g.observe(ctx, github.Quota{Remaining: -1}, requestErr, "OTHER") +} + +func (g *officialQuotaGuard) beforeGit(ctx context.Context) error { + if err := g.before(ctx); err != nil { + return err + } + // Another worker may have persisted a deployment-wide cooldown while this + // package was admitted. Finish local delivery, but do not fetch more blobs. + control, err := g.control.GetOfficialTaskControl(ctx, g.task.TaskID, g.task.Lease) + if err != nil { + g.stopped = err + } else if control.Action == "PAUSE" && control.Reason == "QUOTA_COOLDOWN" { + g.stopped = errors.New("official Git acquisition is in quota cooldown") + } + return g.stopped +} + +func (g *officialQuotaGuard) observe(ctx context.Context, observed github.Quota, requestErr error, resource string) error { + now := time.Now().UTC() + observation := controlplane.OfficialQuotaObservation{Resource: resource, ObservedAt: now, State: "UNKNOWN"} + quota := credentialQuota(observed) + if quota != nil && quota.ResetAt.Before(now.Add(24*time.Hour)) { + observation.Resource, observation.State = quota.Resource, "OBSERVED" + observation.Remaining, observation.ResetAt = "a.Remaining, "a.ResetAt + } + var upstreamError *github.APIError + var until time.Time + global := false + if errors.As(requestErr, &upstreamError) && upstreamError.Code == "AUTH_INVALID" { + observation.State, until = "AUTH_INVALID", now.Add(time.Minute) + global = g.task == nil || g.task.Credential == nil || g.task.Credential.Token == "" + } else if errors.As(requestErr, &upstreamError) && upstreamError.Code == "GITHUB_RATE_LIMITED" { + if quota != nil && quota.Remaining == 0 && quota.ResetAt.After(now) && quota.ResetAt.Before(now.Add(24*time.Hour)) && !upstreamError.SecondaryRateLimit { + observation.State, until = "PRIMARY_EXHAUSTED", quota.ResetAt + } else { + observation.State, until, global = "SECONDARY_COOLDOWN", upstreamError.RetryAt, true + if !until.After(now) { + until = now.Add(time.Minute) + } + if until.After(now.Add(24 * time.Hour)) { + until = now.Add(24 * time.Hour) + } + } + } else if quota != nil && observation.State == "OBSERVED" && quota.Remaining == 0 && quota.ResetAt.After(now) { + observation.State, until = "PRIMARY_EXHAUSTED", quota.ResetAt + } else if requestErr != nil && observation.State == "UNKNOWN" { + until, global = now.Add(time.Minute), true + } + if !until.IsZero() { + seconds := int(until.Sub(now).Seconds()) + if seconds < 1 { + seconds = 1 + } + observation.RetryAfterSeconds = &seconds + key := g.quotaKey(observation.Resource) + if observation.State == "AUTH_INVALID" { + key = g.quotaKey("*") + } + rememberOfficialCooldown(key, until, global) + } + // Missing headers always retain nil balances and reset time, including Git. + reportCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := g.control.ReportOfficialQuota(reportCtx, g.task.TaskID, g.task.Lease, observation); err != nil { + rememberOfficialPersistenceFailure(now) + g.stopped = errors.New("official quota persistence failed; GitHub requests stopped") + return g.stopped + } + if !until.IsZero() { + g.stopped = errors.New("official GitHub quota requires cooldown") + if requestErr != nil { + return requestErr + } + return g.stopped + } + return nil +} + +// Token digests are process-local keys, never persisted or logged. A rotated token +// must not inherit the previous token's primary exhaustion. Anonymous requests +// share one key; different tokens are not assumed to have independent balances. +func (g *officialQuotaGuard) quotaKey(resource string) string { + if g.task == nil || g.task.Credential == nil || g.task.Credential.Token == "" { + return "anonymous:" + resource + } + digest := sha256.Sum256([]byte(g.task.Credential.Token)) + return hex.EncodeToString(digest[:]) + ":" + resource +} + +func rememberOfficialCooldown(resource string, until time.Time, global bool) { + officialProcessQuota.Lock() + defer officialProcessQuota.Unlock() + for key, expiry := range officialProcessQuota.resourceUntil { + if !expiry.After(time.Now()) { + delete(officialProcessQuota.resourceUntil, key) + } + } + if global { + if until.After(officialProcessQuota.globalUntil) { + officialProcessQuota.globalUntil = until + } + } else if until.After(officialProcessQuota.resourceUntil[resource]) { + officialProcessQuota.resourceUntil[resource] = until + } +} + +func rememberOfficialPersistenceFailure(now time.Time) { + officialProcessQuota.Lock() + defer officialProcessQuota.Unlock() + if now.Sub(officialProcessQuota.lastFailure) > 15*time.Minute { + officialProcessQuota.failures = 0 + } + officialProcessQuota.failures = min(officialProcessQuota.failures+1, 5) + officialProcessQuota.lastFailure = now + until := now.Add(min(time.Minute*time.Duration(1< 1 { + action = "PAUSE" + } + _ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 2}) + })) + client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + version := int64(1) + err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { return result }) + server.Close() + if err != nil || acknowledgements.Load() != 1 { + t.Fatalf("terminal ack: %v", err) + } + } +} + +func TestPlatformChangedEpochNeverAcknowledgesNewRunningState(t *testing.T) { + var controls, acks atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/platform-pause") { + acks.Add(1) + w.WriteHeader(409) + _, _ = w.Write([]byte(`{"code":"CONTROL_CHANGED"}`)) + return + } + action := "PAUSE" + if controls.Add(1) > 2 { + action = "CONTINUE" + } + _ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 3}) + })) + defer server.Close() + client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + version := int64(1) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := runPlatformTask(ctx, client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { t.Fatal("paused work started"); return nil }) + if err == nil || acks.Load() != 1 { + t.Fatalf("epoch handling err=%v acks=%d", err, acks.Load()) + } +} + +// Runs against the real Java HTTP server launched by the platform-control PG test. +// No GitHub network or credentials are used; the acquisition waits until the +// production control monitor cancels it, then the production executor sends ACK. +func TestPlatformJavaInterop(t *testing.T) { + base := os.Getenv("SKILLHUB_PLATFORM_INTEROP_URL") + if base == "" { + t.Skip("Java interoperability fixture not supplied") + } + client, err := controlplane.NewClient(base, os.Getenv("SKILLHUB_PLATFORM_INTEROP_TOKEN")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + task, err := client.ClaimWithCapabilities(ctx, "platform-interop", "DISCOVERY", []string{controlplane.PlatformPauseCapability}) + if err != nil || task == nil { + t.Fatalf("real Java claim: %v", err) + } + if task.Discovery.Checkpoint != "confirmed-page" { + t.Fatal("Java claim lost persisted checkpoint") + } + if task.PlatformControlVersion == nil { + t.Fatal("platform claim missing control version") + } + err = executeLeasedTask(ctx, client, task, func(work context.Context) error { + if err := os.WriteFile(os.Getenv("SKILLHUB_PLATFORM_INTEROP_READY"), []byte("ready"), 0600); err != nil { + return err + } + <-work.Done() + return work.Err() + }) + if err != nil { + t.Fatal(err) + } +} + +func TestPlatformPauseBeforeRepositoryStartsUsesPersistedProgress(t *testing.T) { + var received controlplane.PlatformPause + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/platform-control") { + _, _ = w.Write([]byte(`{"action":"PAUSE","platformVersion":2}`)) + return + } + _ = json.NewDecoder(r.Body).Decode(&received) + _, _ = w.Write([]byte(`{"status":"PAUSED"}`)) + })) + defer server.Close() + client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + version := int64(1) + task := &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version, + Repository: &controlplane.RepositorySnapshot{FixedCommitSHA: strings.Repeat("a", 40), PathCheckpoint: "confirmed", IncompleteSubtrees: []string{"unknown"}}} + err := runPlatformTask(context.Background(), client, task, func(context.Context) error { t.Fatal("paused worker started"); return nil }) + if err != nil { + t.Fatal(err) + } + if received.RepositoryProgress == nil || received.RepositoryProgress.ObservedCommitSHA == nil || *received.RepositoryProgress.ObservedCommitSHA != strings.Repeat("a", 40) || *received.RepositoryProgress.PathCheckpoint != "confirmed" || len(received.RepositoryProgress.IncompleteSubtrees) != 1 { + t.Fatalf("lost persisted progress: %+v", received.RepositoryProgress) + } +} + +func TestPlatformPauseAllowsCompletedArtifactSettlementBeforeAck(t *testing.T) { + var paused, settled atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/platform-control") { + action := "CONTINUE" + if paused.Load() { + action = "PAUSE" + } + _ = json.NewEncoder(w).Encode(controlplane.PlatformControl{Action: action, PlatformVersion: 2}) + return + } + if !settled.Load() { + t.Error("ACK before completed artifact settlement") + } + _, _ = w.Write([]byte(`{"status":"PAUSED"}`)) + })) + defer server.Close() + client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + version := int64(1) + err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", PlatformControlVersion: &version}, func(ctx context.Context) error { + if err := beginPlatformDelivery(ctx); err != nil { + return err + } + paused.Store(true) + state := ctx.Value(platformContextKey{}).(*platformExecution) + if err := state.before(ctx); !errors.Is(err, errPlatformPaused) { + t.Fatalf("expected pause: %v", err) + } + if ctx.Err() != nil { + t.Fatal("platform pause cancelled completed artifact settlement") + } + settled.Store(true) + endPlatformDelivery(ctx) + if ctx.Err() == nil { + t.Fatal("acquisition was not cancelled after settlement") + } + return ctx.Err() + }) + if err != nil { + t.Fatal(err) + } +} + +func TestPlatformAckResponseLossRetriesIdempotently(t *testing.T) { + var acknowledgements atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/platform-control") { + _, _ = w.Write([]byte(`{"action":"PAUSE","platformVersion":2}`)) + return + } + if acknowledgements.Add(1) == 1 { + connection, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = connection.Close() + return + } + _, _ = w.Write([]byte(`{"status":"PAUSED"}`)) + })) + defer server.Close() + client, _ := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + version := int64(1) + err := runPlatformTask(context.Background(), client, &controlplane.ClaimedTask{TaskID: "fixture", Kind: "DISCOVERY", PlatformControlVersion: &version}, func(context.Context) error { t.Fatal("paused work started"); return nil }) + if err != nil || acknowledgements.Load() != 2 { + t.Fatalf("ack recovery=%v calls=%d", err, acknowledgements.Load()) + } +} diff --git a/github-sync-go/internal/worker/pool.go b/github-sync-go/internal/worker/pool.go new file mode 100644 index 000000000..dd5d611bd --- /dev/null +++ b/github-sync-go/internal/worker/pool.go @@ -0,0 +1,164 @@ +package worker + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +type PoolConfig struct { + DiscoveryWorkers int + RepositoryWorkers int + PollInterval time.Duration + Discovery DiscoveryLimits + Repository RepositoryLimits +} + +type Counters struct { + DiscoveryClaims atomic.Uint64 + RepositoryClaims atomic.Uint64 + DiscoveryErrors atomic.Uint64 + RepositoryErrors atomic.Uint64 + DiscoveryActive atomic.Int64 + RepositoryActive atomic.Int64 + DiscoveryReady atomic.Bool + RepositoryReady atomic.Bool +} + +// supportedCapabilities describes the lifecycle implemented by the executable. +// Java configuration still determines whether official work is scheduled. +func supportedCapabilities() []string { + return []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, + controlplane.OfficialPauseCapability, controlplane.OfficialOwnerCapability, controlplane.PlatformPauseCapability} +} + +// Run starts separate bounded claim loops. Each loop claims only its task kind, +// so no lease waits in a queue behind work from the other pool. +func Run(ctx context.Context, control *controlplane.Client, + config PoolConfig, counters *Counters) error { + if control == nil || counters == nil || config.DiscoveryWorkers < 1 || + config.RepositoryWorkers < 1 || config.DiscoveryWorkers > 32 || + config.RepositoryWorkers > 32 || config.PollInterval < time.Second || + config.Discovery.MaxPagesPerLease < 1 || config.Repository.MaxWorkspaceBytes < 1 { + return errors.New("invalid GitHub sync worker pool configuration") + } + instance := make([]byte, 6) + if _, err := rand.Read(instance); err != nil { + return err + } + instanceID := hex.EncodeToString(instance) + var running sync.WaitGroup + running.Add(1) + go func() { + defer running.Done() + for ctx.Err() == nil { + if err := control.HeartbeatWithCapabilities(ctx, "skillhub-"+instanceID, supportedCapabilities()); err != nil && ctx.Err() == nil { + slog.Warn("GitHub sync heartbeat failed", "code", safeErrorCode(err)) + } + pause(ctx, 30*time.Second) + } + }() + start := func(kind string, count int) { + for index := range count { + running.Add(1) + workerID := fmt.Sprintf("skillhub-%s-%s-%d", instanceID, kind, index) + go func() { + defer running.Done() + claimLoop(ctx, control, config, counters, kind, workerID) + }() + } + } + start("DISCOVERY", config.DiscoveryWorkers) + start("REPOSITORY_SYNC", config.RepositoryWorkers) + running.Wait() + return ctx.Err() +} + +func claimLoop(ctx context.Context, control *controlplane.Client, config PoolConfig, + counters *Counters, kind, workerID string) { + for ctx.Err() == nil { + task, err := control.ClaimWithCapabilities(ctx, workerID, kind, supportedCapabilities()) + if err != nil { + setReady(counters, kind, false) + countError(counters, kind) + slog.Warn("GitHub sync claim failed", "kind", kind, "code", safeErrorCode(err)) + pause(ctx, config.PollInterval) + continue + } + setReady(counters, kind, true) + if task == nil { + pause(ctx, config.PollInterval) + continue + } + countClaim(counters, kind) + countActive(counters, kind, 1) + if kind == "DISCOVERY" { + err = ExecuteDiscovery(ctx, control, task, config.Discovery) + } else { + err = ExecuteRepository(ctx, control, task, config.Repository) + } + countActive(counters, kind, -1) + if err != nil && ctx.Err() == nil { + countError(counters, kind) + slog.Warn("GitHub sync task did not finish in this lease", + "kind", kind, "task_id", task.TaskID, "code", safeErrorCode(err)) + } + } +} + +func pause(ctx context.Context, interval time.Duration) { + timer := time.NewTimer(interval) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func countClaim(counters *Counters, kind string) { + if kind == "DISCOVERY" { + counters.DiscoveryClaims.Add(1) + } else { + counters.RepositoryClaims.Add(1) + } +} + +func countError(counters *Counters, kind string) { + if kind == "DISCOVERY" { + counters.DiscoveryErrors.Add(1) + } else { + counters.RepositoryErrors.Add(1) + } +} + +func countActive(counters *Counters, kind string, delta int64) { + if kind == "DISCOVERY" { + counters.DiscoveryActive.Add(delta) + } else { + counters.RepositoryActive.Add(delta) + } +} + +func setReady(counters *Counters, kind string, ready bool) { + if kind == "DISCOVERY" { + counters.DiscoveryReady.Store(ready) + } else { + counters.RepositoryReady.Store(ready) + } +} + +func safeErrorCode(err error) string { + var apiError *controlplane.APIError + if errors.As(err, &apiError) { + return apiError.SafeCode() + } + return "WORKER_ERROR" +} diff --git a/github-sync-go/internal/worker/pool_test.go b/github-sync-go/internal/worker/pool_test.go new file mode 100644 index 000000000..35c43fca2 --- /dev/null +++ b/github-sync-go/internal/worker/pool_test.go @@ -0,0 +1,86 @@ +package worker + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" +) + +// Run is the same entry point used by cmd/github-sync after loading its normal +// configuration. No test-only capability option is supplied to the pool. +func TestDefaultWorkerPoolAdvertisesOfficialLifecycle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + observed := make(chan string, 16) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + WorkerID string `json:"workerId"` + Kind string `json:"kind"` + ProtocolMajor int `json:"protocolMajor"` + Capabilities []string `json:"capabilities"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + want := []string{controlplane.OfficialDiscoveryCapability, controlplane.OfficialCheckpointCapability, controlplane.OfficialPauseCapability, controlplane.OfficialOwnerCapability, controlplane.PlatformPauseCapability} + if len(body.Capabilities) != len(want) { + t.Errorf("capabilities=%v", body.Capabilities) + } + for _, capability := range want { + if !slices.Contains(body.Capabilities, capability) { + t.Errorf("missing %s", capability) + } + } + if !strings.HasPrefix(body.WorkerID, "skillhub-") { + t.Errorf("missing instance identity: %q", body.WorkerID) + } + if strings.HasSuffix(r.URL.Path, "/heartbeat") { + if body.ProtocolMajor != 1 { + t.Error("heartbeat omitted protocol major") + } + w.WriteHeader(204) + observed <- "heartbeat" + return + } + if !strings.HasSuffix(r.URL.Path, "/tasks/claim") { + t.Errorf("unexpected path %s", r.URL.Path) + } + w.WriteHeader(204) + observed <- body.Kind + })) + defer server.Close() + control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + done <- Run(ctx, control, PoolConfig{DiscoveryWorkers: 1, RepositoryWorkers: 1, PollInterval: time.Second, Discovery: DiscoveryLimits{MaxPagesPerLease: 10}, Repository: RepositoryLimits{MaxWorkspaceBytes: 1}}, &Counters{}) + }() + seen := map[string]bool{} + for len(seen) < 3 { + select { + case kind := <-observed: + seen[kind] = true + case <-ctx.Done(): + t.Fatal("default pool did not send heartbeat and both claims") + } + } + if !seen["heartbeat"] || !seen["DISCOVERY"] || !seen["REPOSITORY_SYNC"] { + t.Fatalf("unexpected pool requests %v", seen) + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("pool shutdown: %v", err) + } +} diff --git a/github-sync-go/internal/worker/repository.go b/github-sync-go/internal/worker/repository.go new file mode 100644 index 000000000..214fe482d --- /dev/null +++ b/github-sync-go/internal/worker/repository.go @@ -0,0 +1,361 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" + "github.com/iflytek/skillhub/github-sync-go/internal/repository" +) + +type RepositoryLimits struct { + TempRoot string + AllowedS3Hosts []string + MaxTreeRequests int + MaxTreeEntries int + MaxWorkspaceBytes int64 + MinFreeBytes int64 +} + +type repositoryGitHub interface { + repository.TreeReader + SetQuotaObserver(func(context.Context, github.Quota)) + GetRepository(context.Context, string, string) (github.Repository, github.Quota, error) + GetHead(context.Context, github.Repository) (string, github.Quota, error) + GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error) +} + +// ProcessRepository keeps all business progress in Java. A returned error means +// the caller should classify and report task failure while its lease is current. +func ProcessRepository(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits RepositoryLimits) error { + if task == nil { + return errors.New("invalid repository task or worker limits") + } + return processRepository(ctx, control, task, limits, github.NewClient(credentialToken(task))) +} + +func processRepository(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, limits RepositoryLimits, upstream repositoryGitHub) error { + if task == nil || task.Kind != "REPOSITORY_SYNC" || task.Repository == nil || + control == nil || upstream == nil || + limits.MaxTreeRequests < 1 || limits.MaxTreeEntries < 1 || + limits.MaxWorkspaceBytes < 1 || limits.MinFreeBytes < 0 || limits.TempRoot == "" { + return errors.New("invalid repository task or worker limits") + } + snapshot := task.Repository + completion := controlplane.RepositoryCompletion{ + CommitSHA: snapshot.FixedCommitSHA, PathCheckpoint: snapshot.PathCheckpoint, + IncompleteSubtrees: append([]string(nil), snapshot.IncompleteSubtrees...), + } + if state, ok := ctx.Value(platformContextKey{}).(*platformExecution); ok { + state.completion = &completion + } + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + var quotaGuard *officialQuotaGuard + if repositoryNeedsOfficialControl(task) { + guard, err := installOfficialQuotaGuard(control, task, upstream) + if err != nil { + return err + } + quotaGuard = guard + ctx = repository.WithGitRequestPolicy(ctx, guard.beforeGit, guard.afterGit) + } else { + upstream.SetQuotaObserver(observeCredentialQuota(control, task)) + } + metadata, _, err := upstream.GetRepository(ctx, snapshot.Owner, snapshot.Name) + if err != nil { + return err + } + if metadata.ID != snapshot.RepositoryID { + return errors.New("GitHub repository ID changed for task source") + } + if err := control.UpdateRepositoryMetadata(ctx, task.TaskID, task.Lease, + controlplane.RepositoryMetadata{RepositoryID: metadata.ID, OwnerID: metadata.Owner.ID, + Owner: metadata.Owner.Login, Name: metadata.Name, + DefaultBranch: metadata.DefaultBranch}); err != nil { + return err + } + if task.Mode == "HEAD_ONLY" { + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + head, _, err := upstream.GetHead(ctx, metadata) + if err != nil { + return err + } + if completion.CommitSHA == "" { + completion.CommitSHA = head + } + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + return control.CompleteHeadOnly(ctx, task.TaskID, task.Lease, head) + } + if task.Mode != "FULL" || task.MaxFilesPerPackage < 1 || task.MaxPackageBytes < 1 || + task.MaxSingleFileBytes < 1 || task.MaxUploadBytes < 1 { + return errors.New("repository task has no valid package policy") + } + commitSHA := snapshot.FixedCommitSHA + if commitSHA == "" { + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + commitSHA, _, err = upstream.GetHead(ctx, metadata) + if err != nil { + return err + } + } + completion.CommitSHA = commitSHA + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + treeSHA, _, err := upstream.GetCommitTreeSHA(ctx, metadata.Owner.Login, metadata.Name, commitSHA) + if err != nil { + return err + } + var treeReader repository.TreeReader = upstream + treeContext := ctx + var guardedTree *officialRepositoryTreeReader + if repositoryNeedsOfficialControl(task) { + var cancelTree context.CancelFunc + treeContext, cancelTree = context.WithCancel(ctx) + defer cancelTree() + // An interrupted walk is replayed on resume; it is never complete coverage. + completion.IncompleteSubtrees = []string{"."} + guardedTree = &officialRepositoryTreeReader{reader: upstream, stop: cancelTree, + before: func(ctx context.Context) (bool, error) { + return pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion) + }} + treeReader = guardedTree + } + tree, treeErr := repository.EnumerateTree(treeContext, treeReader, metadata.Owner.Login, + metadata.Name, treeSHA, repository.TreeLimits{ + MaxTreeRequests: limits.MaxTreeRequests, MaxEntries: limits.MaxTreeEntries, + }) + if guardedTree != nil && guardedTree.stopped { + return guardedTree.err + } + if ctx.Err() != nil { + return ctx.Err() + } + if treeErr != nil && len(tree.Entries) == 0 && len(tree.IncompleteSubtrees) == 0 { + return treeErr + } + roots := repository.RootsFromSnapshot(tree, snapshot.IgnorePatterns) + preflight, err := repository.PreflightPackages(tree, roots.Paths, repository.PackageLimits{ + MaxFiles: task.MaxFilesPerPackage, MaxSingleFileBytes: task.MaxSingleFileBytes, + MaxPackageBytes: task.MaxPackageBytes, + }) + if err != nil { + return err + } + plans := make(map[string]repository.PackagePlan, len(preflight.Plans)) + failures := make(map[string]repository.PackageFailure, len(preflight.Failures)) + paths := make([]string, 0, len(preflight.Plans)+len(preflight.Failures)) + for _, plan := range preflight.Plans { + plans[plan.Root] = plan + paths = append(paths, plan.Root) + } + for _, failure := range preflight.Failures { + failures[failure.Root] = failure + paths = append(paths, failure.Root) + } + retryPaths := make(map[string]struct{}, len(snapshot.RetryPaths)) + acceptedPaths := make(map[string]struct{}, len(snapshot.AcceptedPaths)) + for _, path := range snapshot.AcceptedPaths { + acceptedPaths[path] = struct{}{} + } + for _, path := range snapshot.RetryPaths { + retryPaths[path] = struct{}{} + if _, planned := plans[path]; !planned { + if _, failed := failures[path]; !failed && len(roots.IncompleteSubtrees) == 0 { + failures[path] = repository.PackageFailure{Root: path, Code: "SKILL_ROOT_NOT_FOUND"} + paths = append(paths, path) + } + } + } + sort.Strings(paths) + completion.IncompleteSubtrees = append([]string(nil), roots.IncompleteSubtrees...) + for _, path := range paths { + if ctx.Err() != nil { + return ctx.Err() + } + if skipPersistedPath(path, snapshot.PathCheckpoint, acceptedPaths, retryPaths) { + continue + } + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + if quotaGuard != nil { + if err := quotaGuard.before(ctx); err != nil { + return completeDeferred(ctx, control, task, &completion, "RATE_LIMIT", time.Now().Add(time.Minute)) + } + } + if failure, failed := failures[path]; failed { + outcome := "TERMINAL_FAILURE" + if failure.Retryable { + outcome = "RETRYABLE_FAILURE" + } + completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{ + SkillPath: path, Outcome: outcome, Reason: failure.Code, + }) + advanceCheckpoint(&completion, path) + continue + } + admission, err := control.AdmitPackage(ctx, task.TaskID, task.Lease, commitSHA, path) + if err != nil { + var apiError *controlplane.APIError + if errors.As(err, &apiError) && apiError.Code == "BACKPRESSURE_ACTIVE" { + return completeDeferred(ctx, control, task, &completion, "BACKPRESSURE", time.Now().Add(15*time.Minute)) + } + if errors.As(err, &apiError) && apiError.Code == "PATH_IGNORED" { + completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{ + SkillPath: path, Outcome: "IGNORED", Reason: "SKILL_PATH_IGNORED", + }) + advanceCheckpoint(&completion, path) + continue + } + return err + } + artifact, err := repository.AcquirePackage(ctx, metadata.Owner.Login, metadata.Name, + commitSHA, limits.TempRoot, plans[path], repository.AcquisitionLimits{ + MaxWorkspaceBytes: limits.MaxWorkspaceBytes, MaxArchiveBytes: task.MaxUploadBytes, + MinFreeBytes: limits.MinFreeBytes, + }) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, repository.ErrCapacity) { + return completeDeferred(ctx, control, task, &completion, "CAPACITY", time.Now().Add(24*time.Hour)) + } + if errors.Is(err, repository.ErrLocalDiskBudget) { + return completeDeferred(ctx, control, task, &completion, "LOCAL_DISK_BUDGET", time.Now().Add(15*time.Minute)) + } + outcome := "RETRYABLE_FAILURE" + reason := "PACKAGE_ACQUISITION_FAILED" + if errors.Is(err, repository.ErrArchiveTooLarge) { + outcome, reason = "TERMINAL_FAILURE", "ARCHIVE_TOO_LARGE" + } + completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{ + SkillPath: path, Outcome: outcome, Reason: reason, + }) + advanceCheckpoint(&completion, path) + continue + } + if err := beginPlatformDelivery(ctx); err != nil { + if closeErr := artifact.Close(); closeErr != nil { + return fmt.Errorf("remove temporary package: %w", closeErr) + } + return err + } + err = deliverPackage(ctx, control, task, admission, commitSHA, path, + artifact, limits.AllowedS3Hosts) + closeErr := artifact.Close() + endPlatformDelivery(ctx) + if ctx.Err() != nil { + return ctx.Err() + } + if closeErr != nil { + return fmt.Errorf("remove temporary package: %w", closeErr) + } + if err != nil { + completion.PathResults = append(completion.PathResults, controlplane.RepositoryPathResult{ + SkillPath: path, Outcome: "RETRYABLE_FAILURE", Reason: "PACKAGE_DELIVERY_FAILED", + }) + } + advanceCheckpoint(&completion, path) + } + if treeErr != nil { + var apiError *github.APIError + if errors.As(treeErr, &apiError) && apiError.Code == "GITHUB_RATE_LIMITED" { + if reportErr := reportCredentialProblem(ctx, control, task, treeErr); reportErr != nil { + return reportErr + } + return completeDeferred(ctx, control, task, &completion, "RATE_LIMIT", futureRetry(apiError.RetryAt)) + } + if len(completion.IncompleteSubtrees) == 0 { + completion.IncompleteSubtrees = []string{"."} + } + } + if len(completion.IncompleteSubtrees) > 0 || hasRetryable(completion.PathResults) { + return completeDeferred(ctx, control, task, &completion, "BUDGET", time.Now().Add(30*time.Minute)) + } + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, &completion); err != nil || paused { + return err + } + _, err = control.CompleteRepository(ctx, task.TaskID, task.Lease, completion) + return err +} + +func skipPersistedPath(path, checkpoint string, acceptedPaths, retryPaths map[string]struct{}) bool { + if _, accepted := acceptedPaths[path]; accepted { + return true + } + _, retry := retryPaths[path] + return checkpoint != "" && path <= checkpoint && !retry +} + +func deliverPackage(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, admission controlplane.PackageAdmission, + commitSHA, path string, artifact *repository.Artifact, allowedS3Hosts []string) error { + authorization, err := control.AuthorizeUpload(ctx, task.TaskID, task.Lease, + admission.AdmissionID, commitSHA, path, artifact.Size, artifact.SHA256) + if err != nil { + return err + } + if err := control.UploadPackage(ctx, authorization, artifact.Path, allowedS3Hosts); err != nil { + return err + } + _, err = control.ReportPackage(ctx, task.TaskID, task.Lease, admission.AdmissionID, + commitSHA, path, authorization, artifact.Size, artifact.SHA256) + return err +} + +func advanceCheckpoint(completion *controlplane.RepositoryCompletion, path string) { + if completion.PathCheckpoint == "" || path > completion.PathCheckpoint { + completion.PathCheckpoint = path + } +} + +func hasRetryable(results []controlplane.RepositoryPathResult) bool { + for _, result := range results { + if result.Outcome == "RETRYABLE_FAILURE" { + return true + } + } + return false +} + +func completeDeferred(ctx context.Context, control *controlplane.Client, + task *controlplane.ClaimedTask, completion *controlplane.RepositoryCompletion, + reason string, retryAt time.Time) error { + if paused, err := pauseOfficialRepositoryIfNeeded(ctx, control, task, completion); err != nil || paused { + return err + } + completion.DeferralReason = reason + completion.RetryAt = futureRetry(retryAt) + _, err := control.CompleteRepository(ctx, task.TaskID, task.Lease, *completion) + return err +} + +func futureRetry(retryAt time.Time) time.Time { + if retryAt.After(time.Now().Add(time.Minute)) { + return retryAt + } + return time.Now().Add(15 * time.Minute) +} + +func credentialToken(task *controlplane.ClaimedTask) string { + if task.Credential == nil { + return "" + } + return task.Credential.Token +} diff --git a/github-sync-go/internal/worker/repository_process_resume_test.go b/github-sync-go/internal/worker/repository_process_resume_test.go new file mode 100644 index 000000000..3fae8a20d --- /dev/null +++ b/github-sync-go/internal/worker/repository_process_resume_test.go @@ -0,0 +1,567 @@ +package worker + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type repositoryTreeFixture struct { + entries []github.TreeEntry + id int64 + owner string + name string +} + +type headOnlyRepositoryFixture struct { + *repositoryTreeFixture + treeCalls int +} + +type selectedLiveRepositoryTree struct { + *github.Client + treeCalls int + rawEntries int + selectedEntries int + selectAll bool +} + +func (reader *selectedLiveRepositoryTree) GetTree(ctx context.Context, owner, name, sha string, + recursive bool) (github.Tree, github.Quota, error) { + reader.treeCalls++ + tree, quota, err := reader.Client.GetTree(ctx, owner, name, sha, recursive) + if err != nil || !recursive || tree.Truncated { + return tree, quota, err + } + reader.rawEntries = len(tree.Entries) + if reader.selectAll { + reader.selectedEntries = len(tree.Entries) + return tree, quota, nil + } + selected := tree.Entries[:0] + for _, entry := range tree.Entries { + if strings.HasPrefix(entry.Path, "skills/.system/skill-creator/") || + strings.HasPrefix(entry.Path, "skills/.curated/define-goal/") { + selected = append(selected, entry) + } + } + tree.Entries = selected + reader.selectedEntries = len(selected) + return tree, quota, nil +} + +func (fixture *headOnlyRepositoryFixture) GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error) { + fixture.treeCalls++ + return "", github.Quota{}, errors.New("HEAD_ONLY must not inspect a commit tree") +} + +func (fixture *headOnlyRepositoryFixture) GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error) { + fixture.treeCalls++ + return github.Tree{}, github.Quota{}, errors.New("HEAD_ONLY must not enumerate a tree") +} + +func (*repositoryTreeFixture) SetQuotaObserver(func(context.Context, github.Quota)) {} + +func (fixture *repositoryTreeFixture) GetRepository(context.Context, string, string) (github.Repository, github.Quota, error) { + id, owner, name := fixture.id, fixture.owner, fixture.name + if id == 0 { + id, owner, name = 42, "fixture", "repo" + } + repo := github.Repository{ID: id, Name: name, DefaultBranch: "main"} + repo.Owner.Login = owner + return repo, github.Quota{}, nil +} + +func (*repositoryTreeFixture) GetHead(context.Context, github.Repository) (string, github.Quota, error) { + return strings.Repeat("c", 40), github.Quota{}, nil +} + +func (*repositoryTreeFixture) GetCommitTreeSHA(context.Context, string, string, string) (string, github.Quota, error) { + return strings.Repeat("b", 40), github.Quota{}, nil +} + +func (fixture *repositoryTreeFixture) GetTree(context.Context, string, string, string, bool) (github.Tree, github.Quota, error) { + return github.Tree{Entries: fixture.entries}, github.Quota{}, nil +} + +func TestProcessRepositorySkipsReceivedPackageAndRevisitsFailedPathBeforeCheckpoint(t *testing.T) { + var reports []map[string]any + var admissions int + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if strings.HasSuffix(request.URL.Path, "/repository-metadata") { + writer.WriteHeader(http.StatusOK) + return + } + if strings.HasSuffix(request.URL.Path, "/package-admissions") { + admissions++ + writer.WriteHeader(http.StatusInternalServerError) + return + } + if !strings.HasSuffix(request.URL.Path, "/complete") { + writer.WriteHeader(http.StatusNotFound) + return + } + var report map[string]any + if err := json.NewDecoder(request.Body).Decode(&report); err != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + reports = append(reports, report) + writer.Header().Set("Content-Type", "application/json") + if report["coverage"] == "INCOMPLETE" { + _, _ = writer.Write([]byte(`{"status":"RETRY_WAIT"}`)) + } else { + _, _ = writer.Write([]byte(`{"status":"PARTIAL"}`)) + } + })) + defer server.Close() + control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + commit := strings.Repeat("c", 40) + task := &controlplane.ClaimedTask{ + TaskID: "fixture-task", Kind: "REPOSITORY_SYNC", Mode: "FULL", + Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)}, + Repository: &controlplane.RepositorySnapshot{ + RepositoryID: 42, Owner: "fixture", Name: "repo", FixedCommitSHA: commit, + PathCheckpoint: "skills/retry", AcceptedPaths: []string{"skills/accepted"}, + RetryPaths: []string{"skills/retry"}, + }, + MaxFilesPerPackage: 10, MaxPackageBytes: 1024, + MaxSingleFileBytes: 1024, MaxUploadBytes: 2048, + } + limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20} + accepted := github.TreeEntry{Path: "skills/accepted/SKILL.md", Type: "blob", + Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10} + retry := github.TreeEntry{Path: "skills/retry/SKILL.md", Type: "blob", + Mode: "100644", SHA: strings.Repeat("e", 40), Size: -1} + if err := processRepository(context.Background(), control, task, limits, + &repositoryTreeFixture{entries: []github.TreeEntry{accepted, retry}}); err != nil { + t.Fatal(err) + } + if admissions != 0 || len(reports) != 1 || reports[0]["coverage"] != "INCOMPLETE" { + t.Fatalf("received package was reacquired or retry was not deferred: admissions=%d reports=%v", + admissions, reports) + } + retryAt, err := time.Parse(time.RFC3339Nano, reports[0]["retryAt"].(string)) + if err != nil || !retryAt.After(time.Now()) { + t.Fatalf("retryAt was not a future instant: %v, %v", reports[0]["retryAt"], err) + } + assertOnlyPathResult(t, reports[0], "skills/retry", "RETRYABLE_FAILURE") + + // On the next lease the failing path has disappeared from the fixed tree. + // It must still be revisited despite being at the persisted checkpoint. + if err := processRepository(context.Background(), control, task, limits, + &repositoryTreeFixture{entries: []github.TreeEntry{accepted}}); err != nil { + t.Fatal(err) + } + if admissions != 0 || len(reports) != 2 || reports[1]["coverage"] != "COMPLETE" { + t.Fatalf("received package was reacquired or failed path was not resolved: admissions=%d reports=%v", + admissions, reports) + } + assertOnlyPathResult(t, reports[1], "skills/retry", "TERMINAL_FAILURE") +} + +func TestProcessRepositoryDefersAtFixedCheckpointWhenOnePackageCannotFit(t *testing.T) { + var completion map[string]any + admissions := 0 + uploads := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch { + case strings.HasSuffix(request.URL.Path, "/repository-metadata"): + writer.WriteHeader(http.StatusOK) + case strings.HasSuffix(request.URL.Path, "/package-admissions"): + admissions++ + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"admissionId":"capacity-fixture"}`)) + case strings.HasSuffix(request.URL.Path, "/complete"): + if err := json.NewDecoder(request.Body).Decode(&completion); err != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"status":"RETRY_WAIT"}`)) + default: + uploads++ + writer.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + commit := strings.Repeat("c", 40) + task := &controlplane.ClaimedTask{ + TaskID: "capacity-task", Kind: "REPOSITORY_SYNC", Mode: "FULL", + Lease: controlplane.LeaseProof{Generation: 2, Token: strings.Repeat("a", 64)}, + Repository: &controlplane.RepositorySnapshot{ + RepositoryID: 42, Owner: "fixture", Name: "repo", FixedCommitSHA: commit, + PathCheckpoint: "skills/a-previous", AcceptedPaths: []string{"skills/a-previous"}, + }, + MaxFilesPerPackage: 10, MaxPackageBytes: 1024, + MaxSingleFileBytes: 1024, MaxUploadBytes: 2048, + } + tempRoot := t.TempDir() + limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20} + entries := []github.TreeEntry{ + {Path: "skills/a-previous/SKILL.md", Type: "blob", Mode: "100644", + SHA: strings.Repeat("d", 40), Size: 10}, + {Path: "skills/z-next/SKILL.md", Type: "blob", Mode: "100644", + SHA: strings.Repeat("e", 40), Size: 10}, + } + if err := processRepository(context.Background(), control, task, limits, + &repositoryTreeFixture{entries: entries}); err != nil { + t.Fatal(err) + } + if admissions != 1 || uploads != 0 || completion == nil || + completion["coverage"] != "INCOMPLETE" || completion["deferralReason"] != "CAPACITY" || + completion["pathCheckpoint"] != "skills/a-previous" { + t.Fatalf("capacity deferral lost the fixed checkpoint: admissions=%d uploads=%d completion=%v", + admissions, uploads, completion) + } + retryAt, err := time.Parse(time.RFC3339Nano, completion["retryAt"].(string)) + if err != nil || retryAt.Before(time.Now().Add(23*time.Hour)) { + t.Fatalf("insufficient package capacity retried too soon: %v, %v", completion["retryAt"], err) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("capacity deferral left temporary files: %v, %v", remaining, err) + } +} + +func assertOnlyPathResult(t *testing.T, report map[string]any, path, outcome string) { + t.Helper() + results, ok := report["pathResults"].([]any) + if !ok || len(results) != 1 { + t.Fatalf("unexpected path results: %v", report["pathResults"]) + } + result, ok := results[0].(map[string]any) + if !ok || result["skillPath"] != path || result["outcome"] != outcome { + t.Fatalf("unexpected path result: %v", results[0]) + } +} + +// Opt-in cross-service check. Java owns the task and retry clock; this test runs +// the actual repository worker with a controlled GitHub tree in a separate Go process. +func TestInteropProcessRepositoryRecovery(t *testing.T) { + phase := os.Getenv("SKILLHUB_INTEROP_WORKER_PHASE") + if phase == "" { + t.Skip("requires an HTTP interoperability fixture") + } + if phase != "retry" && phase != "resolve" { + t.Fatal("invalid HTTP interoperability worker phase") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-recovery-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Repository == nil || task.Repository.RepositoryID != 42 || + task.Repository.FixedCommitSHA != strings.Repeat("c", 40) || + len(task.Repository.AcceptedPaths) != 1 || + task.Repository.AcceptedPaths[0] != "skills/.system/skill-creator" || + len(task.Repository.RetryPaths) != 1 || task.Repository.RetryPaths[0] != "skills/retry" { + t.Fatal("Java did not return the expected persisted recovery paths") + } + entries := []github.TreeEntry{{Path: "skills/.system/skill-creator/SKILL.md", + Type: "blob", Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10}} + if phase == "retry" { + entries = append(entries, github.TreeEntry{Path: "skills/retry/SKILL.md", + Type: "blob", Mode: "100644", SHA: strings.Repeat("e", 40), Size: -1}) + } + upstream := &repositoryTreeFixture{entries: entries, id: 42, owner: "openai", name: "skills"} + limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20} + if err := processRepository(context.Background(), control, task, limits, upstream); err != nil { + t.Fatal(err) + } +} + +// The real Java API grants admission; Go must return a delayed capacity result +// before Git can fetch any blob, and leave no package directory behind. +func TestInteropProcessRepositoryCapacity(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_WORKER_CAPACITY") != "1" { + t.Skip("requires a Java HTTP capacity fixture") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-capacity-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Repository == nil || task.Repository.RepositoryID != 987654330 || + task.Repository.FixedCommitSHA != strings.Repeat("c", 40) || task.Mode != "FULL" { + t.Fatal("Java did not lease the expected fixed-commit repository task") + } + tempRoot := t.TempDir() + entries := []github.TreeEntry{{Path: "skills/fixture/SKILL.md", Type: "blob", + Mode: "100644", SHA: strings.Repeat("d", 40), Size: 10}} + upstream := &repositoryTreeFixture{entries: entries, id: 987654330, + owner: "openai", name: "skills"} + limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20} + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processRepository(ctx, control, task, limits, upstream) + }); err != nil { + t.Fatal(err) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("capacity deferral left temporary files: %v, %v", remaining, err) + } +} + +// Java supplies the lease and package admission; the Tree fixture isolates the +// REST quota while Git fetches two real Skill directories from one fixed commit. +func TestInteropProcessRepositoryPublicMultiSkill(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_WORKER_PUBLIC_MULTI") != "1" { + t.Skip("requires a Java HTTP lease and public Git access") + } + const commitSHA = "49f948faa9258a0c61caceaf225e179651397431" + realTree := os.Getenv("SKILLHUB_INTEROP_WORKER_REAL_TREE") == "1" + allTree := os.Getenv("SKILLHUB_INTEROP_WORKER_ALL_TREE") == "1" + interruptAfterReceipt := os.Getenv("SKILLHUB_INTEROP_WORKER_INTERRUPT_AFTER_RECEIPT") == "1" + if allTree && !realTree { + t.Fatal("full Tree interop requires the real GitHub Tree API") + } + repositoryID := int64(987654350) + if realTree { + repositoryID = 1103643573 + } + javaURL := os.Getenv("SKILLHUB_INTEROP_JAVA_URL") + workContext := context.Background() + var interrupted atomic.Bool + if interruptAfterReceipt { + var stop context.CancelFunc + workContext, stop = signal.NotifyContext(workContext, syscall.SIGTERM) + defer stop() + proxy := interruptAfterPackageReceipt(t, javaURL, &interrupted) + defer proxy.Close() + javaURL = proxy.URL + "/internal/github-sync/v1" + } + control, err := controlplane.NewClient(javaURL, + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-public-multi-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Repository == nil || task.Repository.RepositoryID != repositoryID || + task.Repository.FixedCommitSHA != commitSHA || task.Mode != "FULL" { + t.Fatal("Java did not lease the expected fixed-commit repository task") + } + if allTree && (task.MaxFilesPerPackage != 100 || task.MaxPackageBytes != 100<<20 || + task.MaxUploadBytes != 100<<20) { + t.Fatal("full Tree lease did not use the current production package policy") + } + files := []struct { + path string + sha string + size int64 + }{ + {"skills/.system/skill-creator/LICENSE.txt", "d645695673349e3947e8e5ae42332d0ac3164cd7", 11358}, + {"skills/.system/skill-creator/SKILL.md", "72bc0b97e7a6476254a9d5c424c9971748402ec3", 18664}, + {"skills/.system/skill-creator/agents/openai.yaml", "f5ce09ed118b72908b7308fcfc1619f59e89f6ea", 192}, + {"skills/.system/skill-creator/references/openai_yaml.md", "da5629f8de500ec6f81291d239a2d9c68d1d9044", 2130}, + {"skills/.system/skill-creator/scripts/generate_openai_yaml.py", "1a9d784f8704999daf55a2b9036094949d9af3f8", 6614}, + {"skills/.system/skill-creator/scripts/init_skill.py", "f90703eca8159a21a9684ed1df974c702d92f814", 14483}, + {"skills/.system/skill-creator/scripts/quick_validate.py", "0547b4041a5f58fa19892079a114a1df98286406", 3293}, + {"skills/.curated/define-goal/LICENSE.txt", "13e25df86ce06eb6488e6a6bc5c5847f5dedc352", 10776}, + {"skills/.curated/define-goal/SKILL.md", "87f111bd700e0d993465f7ac741847b5daee57d6", 5647}, + {"skills/.curated/define-goal/agents/openai.yaml", "28c7e7a644d7ae3f5616c7ddd408c573139999f0", 208}, + } + entries := make([]github.TreeEntry, 0, len(files)) + for _, file := range files { + entries = append(entries, github.TreeEntry{Path: file.path, Type: "blob", + Mode: "100644", SHA: file.sha, Size: file.size}) + } + tempRoot := os.Getenv("SKILLHUB_INTEROP_WORKER_TEMP_ROOT") + if tempRoot == "" { + tempRoot = t.TempDir() + } + limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5, + MaxTreeEntries: 100, MaxWorkspaceBytes: 64 << 20, MinFreeBytes: 16 << 20} + if allTree { + limits.MaxTreeEntries = 5000 + } + if host := os.Getenv("SKILLHUB_INTEROP_ALLOWED_S3_HOST"); host != "" { + limits.AllowedS3Hosts = []string{host} + } + var upstream repositoryGitHub = &repositoryTreeFixture{entries: entries, id: repositoryID, + owner: "openai", name: "skills"} + var liveReader *selectedLiveRepositoryTree + if realTree { + token := os.Getenv("SKILLHUB_GITHUB_SYNC_LIVE_TOKEN") + if token == "" { + lookupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + output, lookupErr := exec.CommandContext(lookupCtx, "gh", "auth", "token").Output() + cancel() + if lookupErr != nil { + t.Fatal("real Tree mode requires a configured GitHub test token or local gh authentication") + } + token = strings.TrimSpace(string(output)) + } + if token == "" { + t.Fatal("real Tree mode has no GitHub test token") + } + liveReader = &selectedLiveRepositoryTree{Client: github.NewClient(token), selectAll: allTree} + upstream = liveReader + } + err = executeLeasedTask(workContext, control, task, func(ctx context.Context) error { + return processRepository(ctx, control, task, limits, upstream) + }) + if interruptAfterReceipt { + if !interrupted.Load() || !errors.Is(err, context.Canceled) { + t.Fatalf("worker did not stop after Java accepted the first package: interrupted=%v error=%v", + interrupted.Load(), err) + } + t.Log("interrupted after Java package receipt") + } else if err != nil { + t.Fatal(err) + } + if realTree { + if liveReader.treeCalls != 1 || liveReader.rawEntries < 1000 || + liveReader.selectedEntries < len(entries) { + t.Fatalf("real Tree API did not provide the expected complete metadata: calls=%d raw=%d selected=%d", + liveReader.treeCalls, liveReader.rawEntries, liveReader.selectedEntries) + } + t.Logf("real Tree API: tree_calls=%d raw_entries=%d selected_entries=%d", + liveReader.treeCalls, liveReader.rawEntries, liveReader.selectedEntries) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 { + t.Fatalf("multi-Skill acquisition left temporary files: %v, %v", remaining, err) + } +} + +func interruptAfterPackageReceipt(t *testing.T, javaURL string, interrupted *atomic.Bool) *httptest.Server { + t.Helper() + target, err := url.Parse(javaURL) + if err != nil || target.Scheme != "http" || target.Host == "" { + t.Fatal("interruption fixture requires a local Java HTTP endpoint") + } + client := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{Proxy: nil}} + return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + forward := request.Clone(request.Context()) + forward.URL.Scheme = target.Scheme + forward.URL.Host = target.Host + forward.Host = target.Host + forward.RequestURI = "" + response, err := client.Do(forward) + if err != nil { + writer.WriteHeader(http.StatusBadGateway) + return + } + defer response.Body.Close() + for key, values := range response.Header { + writer.Header()[key] = values + } + writer.WriteHeader(response.StatusCode) + if strings.HasSuffix(request.URL.Path, "/packages/report") && + response.StatusCode >= 200 && response.StatusCode < 300 && + interrupted.CompareAndSwap(false, true) { + _ = syscall.Kill(os.Getpid(), syscall.SIGTERM) + } + _, _ = io.Copy(writer, response.Body) + })) +} + +func TestInteropProcessRepositoryHeadOnly(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_WORKER_HEAD_ONLY") != "1" { + t.Skip("requires a Java HTTP HEAD_ONLY fixture") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-head-only-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion == nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Repository == nil || task.Repository.RepositoryID != 987654332 || + task.Repository.Owner != "old-owner" || task.Repository.Name != "old-name" || + task.Mode != "HEAD_ONLY" { + t.Fatal("Java did not downgrade the registered source to HEAD_ONLY") + } + fixture := &headOnlyRepositoryFixture{repositoryTreeFixture: &repositoryTreeFixture{ + id: 987654332, owner: "new-owner", name: "new-name"}} + tempRoot := t.TempDir() + limits := RepositoryLimits{TempRoot: tempRoot, MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20} + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processRepository(ctx, control, task, limits, fixture) + }); err != nil { + t.Fatal(err) + } + remaining, err := os.ReadDir(tempRoot) + if err != nil || len(remaining) != 0 || fixture.treeCalls != 0 { + t.Fatalf("HEAD_ONLY inspected files or left temporary state: files=%v treeCalls=%d error=%v", + remaining, fixture.treeCalls, err) + } +} + +func TestInteropProcessRepositoryRenamedCandidateWithoutSkill(t *testing.T) { + if os.Getenv("SKILLHUB_INTEROP_WORKER_EMPTY_CANDIDATE") != "1" { + t.Skip("requires a Java HTTP candidate validation lease") + } + control, err := controlplane.NewClient(os.Getenv("SKILLHUB_INTEROP_JAVA_URL"), + os.Getenv("SKILLHUB_INTEROP_SERVICE_TOKEN")) + if err != nil { + t.Fatal(err) + } + task, err := control.ClaimWithCapabilities(context.Background(), "interop-empty-candidate-worker", "REPOSITORY_SYNC", []string{controlplane.PlatformPauseCapability}) + if err != nil { + t.Fatal(err) + } + if task == nil || task.PlatformControlVersion != nil || task.TaskID != os.Getenv("SKILLHUB_INTEROP_TASK_ID") || + task.Repository == nil || task.Repository.RepositoryID != 987654333 || + task.Repository.Owner != "old-owner" || task.Repository.Name != "old-name" || + task.Mode != "FULL" { + t.Fatal("Java did not lease the expected candidate repository") + } + fixture := &repositoryTreeFixture{ + id: 987654333, owner: "new-owner", name: "new-name", + entries: []github.TreeEntry{{Path: "README.md", Type: "blob", Mode: "100644", + SHA: strings.Repeat("a", 40), Size: 10}}, + } + limits := RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5, + MaxTreeEntries: 20, MaxWorkspaceBytes: 1 << 20} + if err := executeLeasedTask(context.Background(), control, task, func(ctx context.Context) error { + return processRepository(ctx, control, task, limits, fixture) + }); err != nil { + t.Fatal(err) + } +} diff --git a/github-sync-go/internal/worker/repository_resume_test.go b/github-sync-go/internal/worker/repository_resume_test.go new file mode 100644 index 000000000..858c4b6c5 --- /dev/null +++ b/github-sync-go/internal/worker/repository_resume_test.go @@ -0,0 +1,25 @@ +package worker + +import "testing" + +func TestSkipPersistedPath(t *testing.T) { + accepted := map[string]struct{}{"skills/received": {}} + retry := map[string]struct{}{"skills/retry": {}} + cases := []struct { + path string + checkpoint string + skip bool + }{ + {"skills/received", "", true}, + {"skills/received", "skills/earlier", true}, + {"skills/earlier", "skills/retry", true}, + {"skills/retry", "skills/retry", false}, + {"skills/new", "skills/earlier", false}, + } + for _, test := range cases { + if got := skipPersistedPath(test.path, test.checkpoint, accepted, retry); got != test.skip { + t.Errorf("path %q, checkpoint %q: skip = %t, want %t", + test.path, test.checkpoint, got, test.skip) + } + } +} diff --git a/github-sync-go/internal/worker/repository_revision_test.go b/github-sync-go/internal/worker/repository_revision_test.go new file mode 100644 index 000000000..596ca4e32 --- /dev/null +++ b/github-sync-go/internal/worker/repository_revision_test.go @@ -0,0 +1,79 @@ +package worker + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/iflytek/skillhub/github-sync-go/internal/controlplane" + "github.com/iflytek/skillhub/github-sync-go/internal/github" +) + +type revisionFixture struct { + repositoryTreeFixture + headCalls int + treeCommit string +} + +func (f *revisionFixture) GetHead(context.Context, github.Repository) (string, github.Quota, error) { + f.headCalls++ + return strings.Repeat("e", 40), github.Quota{}, nil +} +func (f *revisionFixture) GetCommitTreeSHA(_ context.Context, _, _, commit string) (string, github.Quota, error) { + f.treeCommit = commit + return strings.Repeat("b", 40), github.Quota{}, nil +} + +func TestFullRepositoryResolvesCurrentHeadUnlessResumingPinnedCommit(t *testing.T) { + for _, pinned := range []bool{false, true} { + name := "new-task-current-head" + if pinned { + name = "resume-fixed-commit" + } + t.Run(name, func(t *testing.T) { + var reported string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/repository-metadata") { + w.WriteHeader(http.StatusOK) + return + } + if strings.HasSuffix(r.URL.Path, "/complete") { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + reported, _ = body["observedCommitSha"].(string) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ACQUISITION_COMPLETE"}`)) + return + } + t.Errorf("unexpected request %s", r.URL.Path) + w.WriteHeader(500) + })) + defer server.Close() + control, err := controlplane.NewClient(server.URL, strings.Repeat("t", 32)) + if err != nil { + t.Fatal(err) + } + task := &controlplane.ClaimedTask{TaskID: "fixture", Kind: "REPOSITORY_SYNC", Mode: "FULL", Lease: controlplane.LeaseProof{Generation: 1, Token: strings.Repeat("a", 64)}, Repository: &controlplane.RepositorySnapshot{RepositoryID: 42, Owner: "fixture", Name: "repo"}, MaxFilesPerPackage: 100, MaxPackageBytes: 100 << 20, MaxSingleFileBytes: 10 << 20, MaxUploadBytes: 100 << 20} + want := strings.Repeat("e", 40) + wantHead := 1 + if pinned { + want = strings.Repeat("c", 40) + task.Repository.FixedCommitSHA = want + wantHead = 0 + } + fixture := &revisionFixture{} + err = processRepository(context.Background(), control, task, RepositoryLimits{TempRoot: t.TempDir(), MaxTreeRequests: 5, MaxTreeEntries: 20, MaxWorkspaceBytes: 32 << 20}, fixture) + if err != nil { + t.Fatal(err) + } + if fixture.headCalls != wantHead || fixture.treeCommit != want || reported != want { + t.Fatalf("head calls=%d tree commit=%s reported=%s want=%s", fixture.headCalls, fixture.treeCommit, reported, want) + } + }) + } +} diff --git a/openspec/changes/add-github-skill-sync/.openspec.yaml b/openspec/changes/add-github-skill-sync/.openspec.yaml new file mode 100644 index 000000000..ee7c54481 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-28 diff --git a/openspec/changes/add-github-skill-sync/README.md b/openspec/changes/add-github-skill-sync/README.md new file mode 100644 index 000000000..dfb67861f --- /dev/null +++ b/openspec/changes/add-github-skill-sync/README.md @@ -0,0 +1,30 @@ +# add-github-skill-sync + +服务端 GitHub 公共 Skill 发现、采集、审核与同步。 + +## 文档入口 + +- [技能同步统一管理入口](unified-admin.md) +- [统一管理实施合同](unified-admin-contract.md) +- [验证入口与边界](verification.md) +- [同步领域术语](glossary.md) + +- [官方深度探索需求](deep-discovery.md)、[设计与验收矩阵](deep-discovery-design.md)、[持久化设计](deep-discovery-storage.md) +- [Go 服务产品需求](go-service-prd.md) +- [总体架构](design.md)与[状态／检查点设计](state-model.md) +- [Java–Go 内部 OpenAPI](../../../docs/api/github-sync-internal.openapi.yaml)与[接口说明](../../../docs/api/github-sync-internal.md) +- [官方探索管理 OpenAPI](../../../docs/api/github-deep-discovery.openapi.yaml)与[接口说明](../../../docs/api/github-deep-discovery.md) +- [实施任务与剩余事项](tasks.md) +- [验证说明](verification.md) + +## 当前交付边界 + +Java 持久配置、任务和恢复状态，Go 独立执行公开仓库发现与逐包采集，包经上传和 Java 复核进入同步审核。平台和 TEAM 公开来源、管理页面、开源技能展示、审核页与同步修订号已有实现；官方深度探索纳入同一控制面和管理入口。 + +交付完成状态以任务清单为准；验证入口、必须观察的业务行为和延期边界见验证说明。构建和历史本地测试不能替代最终提交的集成验收。 + +## 启用边界 + +官方探索默认关闭。全平台和 namespace 水位初始为空，管理员配置前 Java 拒绝新候选树检查与逐包准入；已登记来源仍可执行 HEAD 检查。Go 在线、GitHub 凭证／额度可用、预算和容量允许是不同条件，启用开关不等于任务已执行。配置与部署步骤见[运行手册](../../../docs/github-sync-operations.md)。 + +OpenSpec 保存正式需求、设计和验收标准；Trellis 仅保留本地执行记录，不作为第二套需求，也不提交本地任务与凭证。 diff --git a/openspec/changes/add-github-skill-sync/deep-discovery-design.md b/openspec/changes/add-github-skill-sync/deep-discovery-design.md new file mode 100644 index 000000000..82a6f8aa8 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/deep-discovery-design.md @@ -0,0 +1,120 @@ +# 官方深度探索增量设计 + +状态：本增量实现与D1–D9限定验收已完成；产品决定见[需求](deep-discovery.md)，当前测试范围、资源边界与剩余事项见[验收追踪](verification.md)和[任务清单](tasks.md)。 + +## 1. 职责与持久状态 + +Java 保存平台策略配置、策略版本、任务快照、查询检查点、候选与出处、额度观测、租约和重试时间。Go 领取任务快照，执行查询、受限作者扩展、仓库检查与采集并上报。沿用现有任务体系，不新增轮次领域对象，不引入消息队列或 Go 私有任务库。 + +官方策略为随版本交付的受控查询集合，与管理员自定义规则隔离。平台只配置启停、预算档位、排除项。查询保留稳定标识及版本，记录类型、渠道、频率类别和来源；默认模板不得覆盖自定义条件。部署版本之间的策略格式须可识别，不兼容的 Go 不领取无法执行的任务。 + +## 2. 搜索执行与覆盖 + +- 精准仓库线索、生态线索、README 线索分别形成可跟踪查询；宽泛补漏低频执行，不以热门或最低星数作为全部渠道的硬门槛。 +- 首次启用、新增查询或扩大查询条件须补搜历史；日常增量保留重叠窗口，低频历史补漏用于索引延迟和仅元数据变化。 +- 查询使用固定截止时间与适用范围。每页候选持久化成功后推进检查点；不能因单页失败推进成功边界。 +- 单次仓库搜索超过可访问结果上限或返回不完整时，按时间拆分；时间无法继续拆分时尝试适用的次级范围。分片设置工作预算和终止边界，仍不完整则保存缺口并退避，不无限拆分或虚报完成。 +- 固定时间窗不等于 GitHub 索引快照；使用重叠与补漏减小翻页漂移，不能承诺全站穷尽。 +- 作者扩展只由确认含有效技能的仓库触发，按作者去重并记录复查进度，只自动扩展一层。扩展所得新作者不递归扩散；大型组织分页与其他工作公平分配资源。 +- 代码搜索不作为首期默认渠道；单独核实语法、索引边界与额度，不能套用仓库搜索的分片限定词。 + +### 时间子分片实施约束 + +Java 持久保存根范围、父子关系、固定策略快照和各子段完成状态；同一长期查询仍只运行一个活动任务，子段顺序领取，不增加管理员操作。Go 提交两个整秒、互不重叠且完整覆盖父段的半开时间范围，Java 在租约事务内核验并交接。父段拆分不推进长期覆盖边界，所有叶子完整完成后才推进根范围的覆盖；缺口、预算暂停和检查点恢复均保留原范围。 + +首个实现采用最大深度 32、每根最多 126 个子节点的有界初值，根节点加子节点至多 127 个；子任务只获得剩余深度和节点预算，不能递归重置预算。历史窗口横跨多年，过低的拆分深度会让最近的密集区间一直停在多年窗口内，因此深度允许按需细分到秒；节点总数和每任务请求预算分别限制持久工作量和单次执行量。它们是防止工作失控的未校准初值，不是召回保证。不能继续拆分时明确记录缺口；有缺口的叶子退避期间，应允许同根其他待处理叶子执行，不能由单个不可完成区间永久阻塞全部兄弟。根覆盖仍需所有叶子完成。每查询只保留一个未结束根，根完成后的诊断树可在下次新根创建时清理，长期覆盖和出处不删除。策略退休允许已存在根续完，但不新建根。 + +有缺口的当前执行以 `PARTIAL` 结算，叶子保存缺口原因、检查点与最早重试时间。待处理兄弟优先运行；无待处理兄弟时才重试到期缺口。重试优先复用已终结的原任务，租约代际继续增加，不重复增长任务/运行历史；若原诊断任务已按保留期清理，则从持久叶子恢复。固定查询、策略、范围和截止时间保持不变，仅在新尝试前按根剩余额度收紧分片预算。缺口退避计数不能因任务复用或清理而归零。 + +### 作者一层扩展 + +作者使用 GitHub 稳定账号 ID 标识；login 仅为可更新坐标。实际包校验成功后保存仓库与账号的有效种子证据，且必须具有非作者扩展的官方搜索出处才能触发动态作者查询。动态查询按 namespace 与账号 ID 去重，只扩展一层，不因扩展命中的仓库递归扩散；后来独立搜索命中且具真实验证证据的仓库可以成为普通种子。 + +Go 按稳定 ID 获取当前名称，枚举时逐仓核对 owner ID，末页再次确认名称没有在分页期间变化；若名称变动导致分页不可靠则保存缺口重查，不误报完整。规范名称经首批空回报送到 Java，最新作者排除项据此生效。旧采集器省略 owner ID 时清除无法证明的身份关联，不拼接旧 ID 与新名称。 + +作者枚举使用公共仓库列表及 CORE 资源，不施加搜索的 pushed 过滤、十页上限或时间二分。正 int32 页码配合单次请求/候选预算保存续页进度；预算至少容纳身份查询、列表页和末页身份复核。作者任务不进入时间分片账本；完成表示本次列表枚举结束，仅安排下一次复查，不推进搜索时间覆盖。查询快照的范围字段仅标识该次执行，不表示列表具有时间一致性快照。缺口按 RETRY_WAIT 保存检查点并退避。 + +动态作者查询不随静态关键词目录退休；任务额外要求 `official-owner/v1`，不由只声明搜索能力的采集器领取。生产Go工作池现声明完整能力；Java配置保持默认关闭，启用后才领取官方任务。 + +## 3. 预算、凭证与暂停 + +复用部署额度、并发、候选容量和包准入上限。节省／均衡／加速调节官方探索可占用的工作预算，不取消任何渠道。调度在保护既有来源更新的同时，给历史补搜和宽泛发现有限但持续的机会。正常预算用尽保存进度并公平续排，不统一延后固定长时间；真正限流按恢复依据延后，候选满与待审容量满分开处理。 + +凭证按资源类别和可确认的限额主体管理最近观测。不得汇总同主体 token 余额；主体不明时不能承诺额度独立。次级限流按响应退避并收紧共享并发。凭证失效、未知、普通额度耗尽和次级冷却分开记录。凭证池的资源额度观察与下述平台任务预算预约分别执行；平台预约不代表锁定了GitHub侧的可用配额。 + +主限流持久状态按服务端租约绑定的凭证与资源记录；匿名主限流单独保存，不阻断后来配置的认证凭证。Go进程内保护按token摘要与资源隔离，不保存明文键；同一配置槽更换token不能继承旧token的进程冷却。认证失败仅停用对应凭证，次级限流与额度回报持久化失败仍触发共享保护。不同token可能属于同一GitHub主体，未知主体只允许尝试未确认耗尽的凭证，不累加余额或承诺独立额度。 + +探索关闭与排除项变更由 Java 在领取和关键接收阶段检查；已登记来源及其他独立入口不被连带暂停。在途操作到可恢复边界结束，已确认上传或接收不删除、不回滚，后续工作暂停；旧租约写入被拒绝。 + +### 官方探索档位的执行约束（D7a 实施合同） + +Java 以一条持久的平台预算记录控制官方发现的领取节奏。首期采用固定份额预约，不新增队列或逐 HTTP 请求许可接口：每次成功领取官方任务，按该快照 `maxRequests` 预约逻辑 API 调用预算，并计算下次最早领取时间。多个 Java/Go 实例共用数据库状态；同一平台最多一个未过期的官方发现租约。旧任务续领、租约过期接管均重新预约，续租不重复预约。 + +| 档位 | 初始逻辑 API 调用预算/小时 | 10 次请求一份时的最短领取间隔 | +| --- | ---: | ---: | +| 节省 | 60 | 600 秒 | +| 均衡 | 180 | 200 秒 | +| 加速 | 360 | 100 秒 | + +这些是经小范围公网试运行后保留的本系统保守初值，不是 GitHub 的额度。12次三档请求成本观察均成功；另一次Java真实领取后的Go发现批次完成9页、816唯一候选，均衡档预留10请求并持久保留200秒间隔。其余档位的预约差异由PostgreSQL回归覆盖；不将短批次结果解释为长期吞吐或召回率保证。没有闲置累积或补发额度，未使用的份额不退款；崩溃和响应丢失不增加可用预算。预约明细按任务保留期随任务清理，平台最早领取时间独立保留，不随任务删除重置。数据库时间为预约依据，进程重启、启停和切换档位不重置既有最早领取时间。档位变更作用于下一份成功预约，在途任务保留原快照与预算；从节省改为加速也可能先等待上一份的剩余间隔。 + +领取事务按配置、平台预算、任务、凭证的固定锁顺序处理；只有实际可以向 worker 交付的租约才扣份额。凭证不可用、额度冷却或候选容量阻断不能被记成成功预约。官方预算受阻时，其他自定义发现入口仍能领取。预算时间到且无官方在途租约时，优先交付一份官方任务，随后在预算间隔内继续原自定义入口顺序。官方组内按最久未成功领取的查询安排，跨续领和任务替换保留公平性，不因低档位取消 README、宽泛发现或作者渠道。 + +后台沿用 `BUDGET` 等待原因，最早领取时间仅表示允许 Go 重新领任务；若仍有活跃租约、服务离线、额度或容量受限，则继续等待相应条件。不修改覆盖边界或把正常预算等待标成覆盖缺口。 + +预算计量范围为官方搜索和作者枚举的逻辑 API 调用，不包含其后仓库 Tree、文件下载、上传，也不精确计量 HTTP 重定向。它约束任务准入速率，不保证逐秒请求均匀；单份内仍可能连续发出最多 `maxRequests` 次调用。已有主额度、共享次级冷却、候选容量和包准入保护继续生效。首期不承诺既有来源更新的绝对 SLA。 + +## 4. 策略升级与恢复 + +任务固定策略版本、查询版本与窗口。新任务采用新策略；兼容查询保留检查点，新增或语义改变的查询建立独立历史补搜进度。无法迁移的检查点不能悄悄跳过范围：记录重扫原因，从可证明安全的边界重放并依赖候选去重。旧快照须保留到在途任务结束，升级不得自动开启已停用探索。 + +Java 存储到期时间，Go 轮询领取后才执行重试。离线时显示等待采集服务而非宣称正在请求 GitHub；租约过期后可接管，同一范围允许重放但不重复生成业务结果。心跳只说明最近联系，不证明任务处理正常；运行详情同时展示最近进度与等待原因。 + +## 5. 页面和查询性能 + +当前页面结构以[统一管理方案](unified-admin.md)为准：总览、仓库、运行记录三个页签；自动探索配置与折叠进度在采集配置抽屉中，旧独立探索链接重定向。近期成果与当前积压口径分别保留，最近运行复用现有详情。列表服务端过滤和稳定排序（发现时间加唯一标识），每页 20 条；沿用现有分页方式并实测深分页，必要时改为游标，不提供无界全量接口。抽屉按需查询，历史和路径结果独立分页。 + +聚合与列表解耦，明确刷新时间，禁止每次翻页重算全部历史。索引根据真实查询计划设计，不能仅靠前端虚拟滚动处理数据库扫描问题。新结果提示刷新，不自动重排；请求取消、超时和隐藏页面停止轮询遵循现有前端模式。 + +近期成果依据持久事件时间计数：首次发现、首次确认有效、首次成功采集新 Skill；内容更新单独计版本。当前积压按当前状态统计，不与近期时间过滤混用。官方探索带来的候选与其他入口重复命中时，保留归因但同范围去重，不宣称独占贡献。具体统计字段和历史数据缺失处理写入 API 合同。 + +## 6. 接口变更清单 + +1. 平台官方探索配置读取与更新，配置版本及权限校验。 +2. 概览、渠道覆盖、分页候选、按需候选详情、关联来源和审核导航。 +3. 任务快照携带策略／查询身份、兼容版本、查询范围与可恢复进度。 +4. 候选回报携带多渠道出处；幂等与进度确认保持一致。 +5. 等待反馈区分资源限流、容量、服务联系状态和任务进度。 + +具体路径、字段与实现状态以 `docs/api/github-deep-discovery.openapi.yaml`、对应 MD 及现有 Java–Go 内部合同为准；Web 类型已从运行时生成，验证入口和边界见 [verification.md](verification.md)。 + +## 7. 验证矩阵与判定 + +| 场景 | 必须观察到的结果 | +|---|---| +| 名称无特征、README 有线索 | 新渠道能补回有效仓库，并留下出处 | +| 零星、长期不活跃、多 Skill 仓库 | 不因统一星数门槛遗漏；逐包处理 | +| 作者仓库超过一页 | 完整续页、作者去重、一层扩展有界 | +| 搜索超过上限／结果不完整 | 分片或明确缺口，不虚报全量完成 | +| 多规则重复命中 | 单候选、多出处；不重复版本或待审项 | +| 限流和多 token | 只延后受影响工作，遵守共享主体和次级退避 | +| Go 强停、网络断开、多个实例接管 | 检查点恢复、旧租约拒绝、重复回报幂等 | +| 探索关闭／排除／重开 | 未登记候选暂停续跑，独立来源不受影响 | +| 策略升级 | 配置不被覆盖；新查询补历史；旧任务不混版本 | +| 排除整页命中及批次重放 | 不创建被排除新候选；原请求摘要不变；原ID仍计当前代际预算；成功确认页可前进 | +| 候选转正式来源后再次领取 | 来源自己的启停生效；探索停用/排除不误停来源，HEAD_ONLY可正常执行 | +| 暂停与包验证并发 | 已准入包可结算；新准入被阻止；task/摄入/路径/来源锁顺序无相互等待环 | +| 子分片回报响应丢失及租约接管 | 同一拆分幂等，不重复创建子段；恢复原范围；旧租约不能推进新任务 | +| 多叶子中一个缺口或暂停 | 根覆盖不前进；已完成叶子不重搜；重开后只续未完成工作 | +| 子分片达到深度/节点上限 | 保存明确缺口并有界退避；不递归重置预算、不虚报完成 | +| 审核积压／候选满／资源不足 | 保存进度、按各层保护线暂停，不无限排队 | +| 大候选列表与详情 | 有界响应、稳定分页、无全量前端加载、查询可解释 | +| 新旧策略对照 | 记录请求量、耗时、独有有效仓库与技能数，不以命中数冒充收益 | + +小范围试验先固定样本集、资源环境和观察窗口，覆盖上述边界；再采用相同预算对照查询效率、完成范围对照召回收益。规模测试使用分档候选数据量，记录数据库执行计划、接口延迟和浏览器交互；具体阈值需在基线测量后冻结，未达标不宣称性能验收通过。代码搜索需展示独有有效召回及其请求成本，不明显则不纳入默认策略。 + +建议先固定数据库/Java资源、1k/10k合成候选、多个出处与热仓路径样本，分别记录单请求和8并发的p95、错误率、分页total及查询计划；阈值在本次交付验收前冻结。此基线不构成生产SLA。 + +## 8. 发布与回退 + +先发布向后兼容的持久字段和协议，再发布 Go 能力及页面；默认不开启官方探索。试运行通过后管理员启用。出现问题先关闭官方探索保留进度，既有来源继续更新；不通过删除候选、版本或任务历史回退。V112 使用新的分片账本，策略执行代次随之升级，以阻止旧调度器重排新根；这不保证旧 Java 接收路径理解新分片。首次启用官方能力前必须整体完成 Java 与 Go 升级，不支持新旧 Java 混跑处理本次官方任务；早期显式 opt-in 的 Go 也不能继续领取包含 126 个累计子节点预算的新快照。旧客户端的空心跳仍可接收；V119之后平台领取另需 `platform-pause/v1`，无能力声明的legacy领取只沿用TEAM原规则；这与本次官方任务不支持新旧Java混跑是两个边界。官方能力过滤由`GithubOfficialDeepDiscoveryIntegrationTest`验证，策略代次与快照保留由`GithubOfficialRecoveryBoundaryIntegrationTest`验证；这些受控测试不等于旧二进制部署混跑验收。完整部署升级、历史数据兼容、混跑防护和回退操作须针对交付提交验收。 diff --git a/openspec/changes/add-github-skill-sync/deep-discovery-storage.md b/openspec/changes/add-github-skill-sync/deep-discovery-storage.md new file mode 100644 index 000000000..32e24b205 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/deep-discovery-storage.md @@ -0,0 +1,59 @@ +# 官方深度探索持久化增量 + +状态：当前迁移覆盖V108–V118；真实迁移、升级、并发及限定规模验证结果统一见[验收追踪](verification.md)。物理结构以SQL迁移为准，以下描述当前持久化边界。 + +## 复用与最小增量 + +| 结构（实际迁移命名） | 用途与边界 | +| --- | --- | +| 复用 github_sync_task | 继续使用 DISCOVERY/REPOSITORY_SYNC、租约代际、去重键、固定快照及检查点；新增 required_capabilities（有界能力集合）与 last_progress_at。旧任务能力集合为空，业务进度不能使用续租更新的 updated_at 代替。 | +| 新增 github_deep_discovery_config | 平台单例配置：namespace_id、enabled 默认 false、budget_profile 默认均衡、config_version、数量受限的 exclusions_json、updated_at。策略版本与管理员配置版本分开。 | +| 新增 github_deep_discovery_query | 长期查询/作者进度：query_key、query_version、strategy_version、definition_json（含channel与查询定义）、covered_until、checkpoint、active_task_id、next_due_at、last_progress_at、wait_reason、retired_at。唯一namespace_id/query_key/query_version；V113补query_source、github_owner_id、owner_login、seed_repository_id，作者只扩一层；V117补last_successful_claim_at。分片账本独立保存，不把无限分片栈放入4096字符checkpoint。 | +| 新增 github_official_discovery_shard | V112保存根/父节点、固定范围与快照、PENDING/ACTIVE/SPLIT/COMPLETE/GAP、可清理的task引用、缺口检查点/重试时间/租约代际。每查询一个未结束根；最多127节点，顺序执行。全部叶子完整才更新长期覆盖；已完成根在下次新根或保留期清理时删除，不删除出处和覆盖。 | +| 扩展 candidate/source origin | 在既有 origin_type CHECK 增加 OFFICIAL_DISCOVERY，origin_key 指向长期 query UUID 而非临时 task；候选出处按candidate/queryId合并，recent_shard_id/recent_run_id/recent_seed_repository_id/latest_strategy_version仅存在于github_discovery_candidate_origin，可随历史清理置空；github_source_origin不新增这些诊断列。两者保留既有SEARCH_RULE/CURATED_OWNER。同一候选仍以 namespace/repository_id 唯一。 | +| 首次有效事件（V115） | 使用github_discovery_outcome_event的FIRST_VALID、occurred_at、official_at_event和first_occurrence_known记录首次有效事实，不新增候选first_validated_at列。事件发生时是否已有官方出处决定归因；无法证明的旧历史由空时间或legacy_unknown_validation标记保留，不能从当前BOUND状态倒推首次有效。 | +| 新增轻量成果事件 | 记录真正新建 Skill、新建内容版本的事件时间、对象ID与当时归因，按事件类型/对象去重；不得把 NO_CHANGE 或关联已有 Skill 当新技能。与运行详情清理独立。 | +| 新增 github_discovery_outcome_event / github_discovery_metrics_epoch / github_discovery_legacy_unknown_validation | V115持久成果去重、可验证历史起点与无法证明首次有效的旧身份；与任务历史清理独立。 | +| 新增 github_official_quota_observation / github_official_resource_cooldown / github_official_secondary_cooldown | V116按资源观察与持久冷却；未知主体不累加余额，不维护GitHub配额预约账本。 | +| 新增 github_official_discovery_budget / github_official_discovery_reservation | V117保存单例平台最早领取时间与task/generation预约明细；这是本系统任务预算，不是GitHub配额余额。任务清理可删除明细，但不重置平台最早领取时间。 | +| 更新 github_candidate_discovery_eligibility | V118复用候选owner/name消除重复查询，保持最终11列和来源／排除／暂停规则；不增加缓存状态。 | +| 新增 github_sync_worker_instance | V114保存实例、协议主版本、能力集合与服务端最近联系时间；旧全局 presence保留，但不能证明支持官方探索。按完整官方能力集合及90秒窗口统计兼容实例；过期实例按既有保留期分批清理，不删除业务记录。 | + +## 额度持久化范围 + +保留现有凭证资源额度观测，不累加不同 token。主限流按服务端租约绑定的凭证及资源保存，匿名单独记录；次级限流采用共享冷却。未知限额主体不被强行识别或合并。未知主体不声称额度独立，不自动展示精确总额。调度仅做保守限速，不建设精确实时余额系统。 + +V117的平台任务预算预约已经实现，规则见设计第3节：成功领取按快照maxRequests预约，续租不重复预约，过期接管重新预约；未使用份额不退款，没有闲置累积。它限制本系统领取节奏，不提供逐HTTP请求许可，不锁定GitHub侧配额；API仍不承诺精确可用请求数。 + +## 查询与统计 + +候选列表按 first_seen_at DESC,id DESC 稳定排序，文案为首次发现；官方后续命中已有候选不重写全局首次时间。当前列表通过持久官方出处过滤。近期“新增候选”只统计事件发生时已有官方归因的对象，不将晚到归因回填成官方新发现。 + +候选索引按 namespace、first_seen_at、id 和必要状态组合；出处反查 origin_type/origin_key/candidate_id；长期查询索引 next_due_at/id（未退休）；成果事件按 namespace/type/occurred_at/subject。每项索引须经实际 SQL/EXPLAIN 选取，避免重复索引。子串搜索不能假定普通 B-tree 可加速。 + +默认20、最大页大小由 API 合同限制。沿用现有 page/size 时明确深页成本；摘要独立缓存/查询并提供生成时间。无法恢复的历史成果从统计可靠起点开始展示，不填伪造时间，不把未知显示为零。 + +## 暂停与竞态 + +候选 REPOSITORY_SYNC 的 source_id 为 NULL，因此在候选入队、领取和关键接收三个位置核对最新配置及有效出处，不能只依赖 DISCOVERY ruleId 门控。存在其他独立启用入口仍需要时允许继续；已登记来源按自身状态管理。已有持久接收处理可结算，未开始的新包停止准入，保留进度。 + +官方发现接收按最新配置过滤排除项：保留完整原请求摘要及原始仓库 ID 的代际去重预算，过滤后仍确认本页检查点；被排除的新仓库不创建官方候选或出处。已有候选保留并刷新坐标，采集资格动态判断；取消排除后已有候选可续排，从未入库的仓库由后续历史补搜重新发现。 + +配置修改与结果接收必须有一致事务顺序/版本复核，避免关闭提交后仍无条件推进新工作。所有写入保留 task ID/generation/token hash/有效期检查。暂停不是删除，也不通过无穷远 due_at 隐藏任务。 + +## 历史保留、升级与回退 + +现有终态发现任务会被清理，task_hit/batch 会级联删除；长期 query、origin 和成果事件不得依赖这些记录保持存在。完整分片结算时原子推进长期覆盖边界；运行详情可过期，但已确认覆盖、出处与统计不能随之消失。退休查询保留，不删除被出处或在途任务引用的快照。 + +坏检查点应报告明确不可续跑原因；Java 从可证明安全的边界建立重放任务并去重，不盲目反复执行同一个坏游标。升级先兼容 schema 与 Java 能力过滤，再 Go，再页面启用；回退必须先关闭并排空/隔离官方任务，旧 Java 无能力过滤，不能直接带着官方任务降级。 + +当前 main 整合迁移范围为 V76–V119；官方探索从 V108 起，平台暂停为 V119。所有已发行 main SQL 保持原 checksum，升级验收见 [验证说明](verification.md)。 + + +## V117：官方探索预约节奏 + +`github_official_discovery_budget` 为部署级单行状态，保存 `next_lease_eligible_at`，跨 Java/Go 实例、进程重启、配置启停与档位切换共享。它不是 GitHub 余额或按 Token 汇总额度。 + +`github_official_discovery_reservation` 以 `(task_id, lease_generation)` 去重，保存实际领取时的档位、预约逻辑请求数、授予时间和下次最早领取时间；随任务保留期级联清理。清理预约明细不删除平台预算行，也不提前最早领取时间。查询新增 `last_successful_claim_at`，用于官方查询间的持久公平排序，续租不更新它。 + +锁顺序、三档初值、预约和失败回滚语义以 [设计第3节](deep-discovery-design.md#3-预算凭证与暂停) 为准。对真实 PostgreSQL 的迁移、并发预约、回滚及清理行为必须由集成测试验证。 diff --git a/openspec/changes/add-github-skill-sync/deep-discovery.md b/openspec/changes/add-github-skill-sync/deep-discovery.md new file mode 100644 index 000000000..80dc097dd --- /dev/null +++ b/openspec/changes/add-github-skill-sync/deep-discovery.md @@ -0,0 +1,75 @@ +# 官方深度探索：需求澄清 + +状态：本页记录已确认的产品行为；官方深度探索实现及D1–D9限定验收已完成。当前证据与限制统一见[验收追踪](verification.md)，整个同步功能的剩余事项见[任务清单](tasks.md)。既有采集、更新、审核流程继续遵循原规格。 + +## 已确认的产品边界 + +- 提供官方内置的深度探索方式，部署后默认关闭，管理员首次启用后持续自动执行，不增加“执行一次”按钮。 +- 关闭深度探索停止该途径的新仓库发现；已登记的有效仓库继续按来源设置更新。精选仓库及自定义搜索规则由自己的启停设置控制。 +- 后台仅提供启停、资源预算档位和排除项；查询集合、渠道搭配、分页和补漏节奏由系统维护。自定义搜索规则保留为独立补充入口。 +- 深度探索开关不是所有 GitHub 同步的总开关。其预算仍受部署额度、候选容量和采集准入约束。 +- 首期仅平台管理员配置官方深度探索，目标为公共 `@github` namespace；TEAM 不单独配置官方探索，继续使用既有来源能力。 +- 关闭探索保留候选和检查点，暂停尚未成为正式来源的候选后续采集；在途工作到安全检查点暂停，重开后续跑。其他独立启用入口仍需要的候选不受影响。 +- 探索排除项只排除指定仓库或作者的新发现，不暂停已登记来源；既有来源暂停和 Skill 目录忽略继续使用各自配置。 +- 首期默认探索渠道采用仓库名称／描述／Topic、README 搜索以及有效仓库作者的一层扩展，配合历史补搜、分页与周期补漏。代码搜索先小范围验证，满足上线条件后再纳入默认策略；不阻塞主要渠道。 +- 资源预算提供“节省／均衡／加速”，默认均衡；档位只影响执行速度和资源分配，不删减探索渠道或覆盖范围。加速仍受额度、容量和既有仓库更新资源保护约束，当前保守初值及小范围校验结论见设计第3节。 +- 新增探索排除项也暂停命中的未登记候选后续采集，保留记录和进度；取消排除后可继续。其他独立启用入口仍需要的候选不受影响，已登记来源不受影响。 +- 官方策略随 SkillHub 版本升级自动采用，保留管理员的启停、预算与排除项；新增查询自动补搜历史，尽量复用既有进度，在途任务继续使用固定策略版本。 +- 官方深度探索采用独立后台页面，集中展示配置和结果；不采用仅嵌入“来源与发现”页面的方案。产品图用于布局评审，不代表已有实现或真实运行数据。 +- 代码搜索先做对照试验：确认能补回其他渠道遗漏的有效仓库，且请求成本、限流恢复与去重可控，再通过后续版本纳入默认策略；效果不明显则保留为备选。 +- 本次创建本地 Trellis 规划任务；OpenSpec 为正式需求来源，Trellis 只保留执行索引。 + +## 已选定的页面结构 + +- 采用独立探索页面搭配配置弹窗，主页面展示状态与成果，不铺开全部配置表单。 +- 顶部展示探索启停、预算摘要与当前状态；数据摘要区分仓库和技能，不以搜索命中数代替导入数量。 +- 中间通过“探索进展／发现仓库”切换，探索进展展示渠道工作及等待原因，页面提供最近运行与详情入口。 +- 配置弹窗集中编辑启停、节省／均衡／加速档位和仓库／作者排除项。 +- 暂停状态明确说明进度已保留、已登记仓库继续更新。 +- 采用紧凑按钮、表格、细边框与少量状态色；两张概念图采用统一主页面结构，图二仅作为配置弹窗及暂停状态参考。 +- 图片中的日期、数量、仓库和状态是示例，不是已验证事实；近期新增与当前待审数量须明确区分。统计口径见下文及 API 合同，已验证的性能样本和资源边界见验收追踪。 + +## 列表、抽屉与性能边界 + +已确认：仓库点击后打开详情抽屉，展示命中渠道、检查结果、未导入原因及后续安排；存在对应来源或审核记录时提供跳转，关闭抽屉保留列表筛选和位置。 + +已确认：主页面显示渠道当前工作、已完成查询分片、待处理工作及暂停原因；查询与分页细节放入运行详情。总量未知不显示完成百分比，不宣称覆盖全部 GitHub。 + +已确认：默认展示全部状态的可查询结果，按最近发现排序，每页 20 条；不是一次性加载全部记录。采用服务端搜索、筛选及分页，接口限制最大页大小；详情点击后按需加载。分页方式与既有接口风格协调，大数据下验证深分页成本，不能依赖浏览器全量过滤。 + +性能设计约束：列表只返回摘要，不附带全部命中历史、日志或技能内容；抽屉中的长列表继续分页。聚合统计独立查询或缓存，不随每页重复全表计算；查询采用匹配过滤和稳定排序的索引，具体方案需结合现有数据库验证。后台执行不应导致表格自动重排、跳页或丢失选中项；轮询在隐藏页面或离开后停止，避免请求叠加。实现和有限性能验收见验收追踪；这些约束不构成所有部署规模的性能保证。 + +## 已确认的统计和等待反馈 + +- 近期成果按所选时间段统计新增候选仓库、首次确认有效仓库及新采集技能；同仓库多渠道命中去重，已有技能内容变化单列更新版本。 +- 当前积压展示此刻待检查仓库与待审版本，不受近期成果的时间筛选影响。 +- 等待额度或容量时展示原因、受影响工作、仍可继续工作及相关配置或审核入口；有可靠时间才显示预计可重试时间，否则显示恢复条件。系统自动续跑，不要求反复手动重试。 + +### 多凭证额度反馈 + +额度不是实时保证值：基于 GitHub 响应头的最近观测，并展示观测时间；缺失或过期标为未知，不伪装成零余额。不开高频额度探测或实时推送系统。按请求资源类别区分仓库搜索与普通仓库读取，不能将单个凭证限流直接呈现为全部探索暂停。 + +多 token 不等于多份独立额度，同一身份可能共享限额，不累加各 token 剩余额度。调度选择当前请求资源可用的凭证；全部适用凭证不可用才延后对应工作。限额主体无法确认时按未知处理，不累加余额；平台预算预约与凭证额度观察分开，具体合同见设计第3节，不承诺其他工具共用凭证时的精确余额。 + +普通额度耗尽时隔离受影响资源与额度主体；遇到次级限流遵守 Retry-After 或退避，必要时收紧共享并发，不能不断切换 token 继续冲击同一限制。界面优先展示“最近观测／等待额度／冷却中／状态未知”，重置时间只表示预计可重试，不承诺届时一定恢复。 + +依据：https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api + +## 已确认的采集服务离线行为 + +- Go 不可用时不执行 GitHub 请求；Java 和数据库正常时保留任务、候选、重试时间和检查点。 +- 到达重试时间仅意味着允许再次领取，不意味着已执行。页面优先显示“等待采集服务恢复”，不继续承诺准时重试。 +- 执行中断后从已确认保存的检查点恢复，未确认的部分可能重做；租约到期允许其他实例接管，旧租约不得写入，业务接收保持幂等。 +- 探索配置启用与执行服务在线是两个维度；心跳超时表示联系中断，不能据此断言进程已关闭。多实例须按可执行能力汇总，不能因单实例离线宣布全部离线。 +- Go 恢复并满足凭证、准入和排队条件后自动领取续跑。Java 不负责拉起 Go 进程，进程恢复由部署工具或运维负责。 + +## 实施与验证入口 + +预算初值、小范围代码搜索对照、运行反馈以及策略升级／恢复的约定验收已记录在[任务清单D1–D9](tasks.md)与[验收追踪](verification.md)。代码搜索继续保留为备选；未来若启用，仍需验证完整包有效性、内容重复率及对应额度合同。长期公网稳定性和全站召回率未由小样本试验证明。 + +## 设计约束 + +- 不新增必须由用户理解或操作的“轮次”对象。 +- 不把更多命中等同于更多有效技能，不把搜索完成等同于 GitHub 全量覆盖。 +- 首期渠道、预算和调度参数按当前设计合同执行；代码搜索保持备选，不自动扩展首期范围。 +- 执行合同及实现状态见 deep-discovery-design.md、API 文档及 tasks.md；不能仅据需求描述推断任意部署环境已通过验收。 diff --git a/openspec/changes/add-github-skill-sync/design.md b/openspec/changes/add-github-skill-sync/design.md new file mode 100644 index 000000000..e3d8cf3e4 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/design.md @@ -0,0 +1,214 @@ +## Context + +> 已实现增量：官方深度探索扩展原有元数据发现范围，新增 README 与有界作者扩展，采用独立页面和独立探索开关。产品决定见[需求](deep-discovery.md)，技术合同见[增量设计](deep-discovery-design.md)，当前验收范围见[验收追踪](verification.md)。下文仅元数据搜索及只按自定义规则启停的原有入口边界不适用于官方探索；自定义规则继续兼容。 + +设计前基线（新增能力前），见 [proposal.md](proposal.md)：原开源版只有 `GLOBAL`、`TEAM` namespace；`ReviewTask` 是共享审核表，尚无同步来源标记；`ObjectStorageService` 只有预签名下载 URL，没有预签名上传能力。现有 `SkillPackageValidator`、发布领域逻辑、扫描任务和审核权限可复用，但不能让外部采集结果直接进入发布状态。现有 `SkillPublishService` 有超级管理员直接发布分支，且同步写文件与对象存储，不能原样用作 Go 回报入口。Trellis 访谈记录保留在本机，不作为第二份正式规格。 + +## Goals / Non-Goals + +**Goals:** + +- SkillHub Java 始终是来源、任务、凭证、Skill 版本和审核的事实源；独立 Go 服务承担 GitHub 外部访问与采集负载。 +- 一个仓库任务处理多个 Skill，同时做到逐包成功、失败隔离、固定 revision 和可恢复续跑。 +- 管理后台能回答“配置了什么、发现了什么、为何延后、哪些 Skill 待审、下一步能做什么”。 +- 无需 GitHub Actions、独立队列、独立 Redis 或额外证书管理组件即可私有化部署。 + +**Non-Goals:** + +- Go 服务不解析 SkillHub 发布权限、不写 SkillHub 数据库、不直接发布 Skill；本期不支持私有 Git 仓库、其他托管平台、自动批准审核或审核发布方式开关。 +- 不根据早期审核产出自动调节各搜索规则权重；先记录效果数据，等样本充分后单独评估。 + +## Decisions + +### 1. 控制面与单 Go 执行服务 + +Java 保存来源、规则、运行、仓库检查点、任务租约、重试时间和凭证密文。一个 Go 服务部署包含发现与仓库同步两类工作池，主动向 Java 领取任务；工作池有独立并发及 GitHub 额度控制。自动搜索任务负责规则/精选用户的仓库召回；已登记仓库任务在一个租约下固定 commit、枚举 Skill 路径并逐包采集。平台精选仓库和 TEAM 明确登记的仓库直接进入同一个仓库账本。Go 不保持跨重启的业务状态。 + +任务合同采用版本化 HTTP/JSON，并以 OpenAPI 描述，至少包含 `claim`、`renew`、逐包采集准入、`report`、`complete/fail` 和短时上传授权操作。领取结果携带任务 ID、运行 ID、类型、namespace/source、规则版本或固定 commit、租约令牌与过期时间、必要的过滤与安全边界；GitHub Token 只在受认证的领取响应中临时提供，不写入任务记录。进度回报携带仓库 ID、commit、枚举覆盖范围、逐 Skill 路径结果、临时对象引用、额度与可重试时间。Java 只接受当前租约令牌和单调检查点；重放通过 `(任务, commit, Skill 路径, 内容哈希)` 幂等。一次仓库任务失败后可携带原 commit 与失败路径重新领取，不重新采集成功包。逐包 `report` 仅在短事务持久接受摄入记录后确认接收，Java 后台再逐包复核和推进审核；仓库采集任务完成不表示所有包已审核或发布。每个内部 HTTP 请求使用独立的 `X-Request-Id` 关联日志，业务幂等仍以任务、租约和结果键判断，不复用 request ID 充当业务检查点。 + +**备选：Java 为每个 Skill 拆采集任务。** 不采用。它增加任务量与中间状态，也违背已确认的仓库任务模型。逐 Skill 的结果、对象和重试目标仍单独持久化。 + +### 2. 统一来源账本与 namespace + +新增系统管理的 `CATALOG` namespace 类型和保留 slug `github`，创建不可由普通用户删除的 `@github`。迁移发现现有 namespace 占用该 slug 时，停止初始化并给出明确迁移错误，不自动改名或删除用户空间。所有平台发现入口在 `@github` 以 GitHub repository ID 去重，`owner/repo` 只作展示与首次解析线索；同一仓库可在 TEAM namespace 形成独立来源绑定。来源记录保留发现入口集合、公开状态、同步开关、最近成功 revision、下次运行时间及不可达状态。 + +现有 `Skill.ownerId` 非空。同步 Skill 使用保留的系统 actor 作为技术所有者，业务归属以 namespace 和来源绑定为准；界面不将采集工作归属给配置管理员个人。TEAM OWNER/ADMIN 与平台审核角色仍按现有 namespace 权限执行操作，系统 actor 不成为可登录账号。自动搜索命中的仓库先保存为发现运行中的候选坐标，仓库任务验证有效 Skill 后才建立正式来源；管理员明确配置的精选仓库可以先建立来源并显示待首次扫描状态。 + +规则只决定新仓库召回；来源停用不会变成删除，搜索重命中不能自动恢复。TEAM 来源默认 `NAMESPACE_ONLY`，来源级可见范围只用于未来导入，已发布 Skill 的可见性遵守现有生命周期。 + +**备选：所有外部来源复用 `GLOBAL`。** 不采用。它无法表达 `@github` 的系统归属、不可删除规则和未来按平台区分的治理权限。 + +### 3. GitHub 发现、固定树与包采集 + +官方深度探索使用名称、描述、topic、README 线索以及有效仓库作者的一层扩展，配合历史补搜、分页和周期补漏；详细增量设计见 [deep-discovery-design.md](deep-discovery-design.md)。管理员自定义搜索继续使用元数据关键词和 topic 条件。首期默认策略不依赖全站代码搜索，后续需经过对照试验才可纳入。上述渠道仍受 GitHub 索引和查询范围限制，不能承诺全站穷尽。 + +现有自定义规则快照包含 `KEYWORD`、`TOPIC` 两类选择条件，以及 `CORE`、`EXTENSION` 处理优先级；每个条件编译成独立的 GitHub 仓库搜索查询，合并时按 repository ID 去重并保留命中出处。`KEYWORD` 限定仓库名称、描述和 topics，`TOPIC` 使用 GitHub 的 topic 过滤；不搜索 README 或代码。Go 转义条件值，拒绝原始查询表达式，Java 对后台输入与规则版本做权威校验。增量/七天全量的搜索窗口由 Java 固定在任务快照，Go 根据固定窗口和检查点续跑；核心条件先消耗发现预算，扩展条件在剩余额度内执行。GitHub 仓库搜索的每查询 1,000 条结果上限及 `incomplete_results` 标志必须显式处理：超过覆盖范围时按可行的时间窗进一步分片或标记未完成并续跑，不能把截断页记为全量成功。[仓库搜索限定条件](https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories)、[REST 搜索限制](https://docs.github.com/en/rest/search/search)。 + +发现包含结构化的核心/扩展查询、精选用户或组织公开仓库枚举和精选仓库。Go 先读取公开仓库元信息并解析稳定 repository ID；搜索返回只是仓库线索，确认有效 `SKILL.md` 后才登记。Java 保存发现入口启用状态、策略或规则版本、目标周期、增量窗口和运行检查点，只为已启用入口生成兼容的发现任务。官方探索有独立开关但不是全局同步开关，自定义规则继续各自启停；Go 领取固定快照执行。首次部署及现有部署升级时预置搜索规则初始停用，由平台管理员在同一规则页面启用；明确登记的精选仓库不受影响。正常轮次采用有重叠的增量窗口；默认每 7 天安排一次去除增量时间限制的全量搜索，以找回遗漏的仓库线索，不代表整仓下载。仓库更新和精选用户或组织发现的默认目标周期均约 24 小时，广泛搜索约 72 小时，可由后台按类别调整。目标周期不是限流或反压下的执行时限；全量搜索仍受额度、预算和可恢复检查点约束，逾期未完成须继续而非丢弃。 + +每次仓库任务先固定默认分支 commit。Go 使用 GitHub REST 取得仓库元数据、搜索结果和树；递归树响应截断时按子树继续，无法枚举的子树标记 `INCOMPLETE` 并续跑，其他完整子树继续。平台默认忽略候选路径中任一目录段为 `.git`、`node_modules`、`.venv`、`venv`、`dist`、`build`、`target`、`__pycache__`、`test`、`tests` 的 `SKILL.md`；`examples`、`docs` 不默认忽略。管理员只能追加来源级候选路径规则，不能放宽平台基线；忽略候选不剔除父 Skill 包内的合法测试文件。Go 可使用 Java 下发的有效规则作采集优化，Java 导入前重新执行权威判断。包以 `SKILL.md` 所在目录为根，排除嵌套 Skill 子树；仓库根级许可证/NOTICE 作为来源合规证据一并记录。包文件首期采用容器内受限 Git CLI 的 blobless 局部克隆和稀疏读取，把固定 commit 下所需文件取到有时间、磁盘和文件数边界的本地临时目录，并在读包前验证检出的 commit 等于任务固定 commit；若固定 commit 无法安全取得，则报告失败或重试，绝不混用最新分支内容。Git 命令只能访问由公开仓库 ID 解析出的 GitHub 地址，使用参数数组而非 shell 拼接，禁用全局配置、hooks、外部过滤器和 submodule，限制时间、磁盘与输出大小。不以每个文件一次 Contents API 请求工作，也不回退为无限制整仓下载。符号链接和 submodule 不跟随；文件数、单文件与总大小采用当前有效发布策略，当前 `application.yml` 配置为最多 100 个文件、单文件 10 MiB、包总量 100 MiB，在 Go 预检且在 Java 复核；临时归档上传大小同样受已有上传限制。 + +**本地磁盘边界：** Go 必须先用固定 commit 的 GitHub Tree 元数据发现候选 `SKILL.md`、文件类型和 blob 大小，按子树逐步枚举而非将整仓清单无限驻留内存；确认路径不被忽略、包的文件数和已知大小未超限后，才启动该包的内容获取。GitHub 递归 Tree API 在 100,000 项或 7 MB 响应处可能截断，须按非递归子树续查；树不完整时不能推断 Skill 消失。[GitHub Tree API](https://docs.github.com/en/rest/git/trees) 返回 blob 大小与截断标志。`blob:none` 初始获取仍可能包含大量 commit/tree 元数据，稀疏检出又会按需下载所选 blob，不能把“局部克隆”等同于磁盘有界；[Git partial clone](https://git-scm.com/docs/partial-clone) 与 [sparse checkout](https://git-scm.com/docs/sparse-checkout) 只提供选择性获取机制。Go 的每仓库工作目录、进程总临时磁盘、最低剩余空间和并发目录数必须受部署资源预算约束，统计包括 `.git` 对象、检出文件和归档；逐包上传并收到 Java 接收确认后删除可释放的本地文件。Git 对象缓存可能继续增长，因此达到预算前应保存已处理路径检查点、结束本批并清理工作目录，再从同一固定 commit 续跑剩余路径；低磁盘须报告容量延后而不是继续下载或将来源标为不可达。单个包超过业务限制按该包失败，其他包继续；元数据或固定 commit 无法在预算内安全取得时标记扫描不完整并显示资源不足，不能自动删除既有 Skill。工作目录容量与并发的初始数值在 Go 镜像/Compose/Helm 的实际临时存储限制下确定并做大仓库试验，不能只在代码里设置一个未验证的常量。 + +执行前须为本批预留 Git 元数据、选中包的落地文件、归档和安全余量；业务包上限 100 MiB 不等于单包峰值磁盘需求只有 100 MiB。Go 在各阶段检查实际目录占用与文件系统剩余空间，写入失败时停止该仓库、清理未确认的临时文件并保留 Java 已确认的检查点。应用层检查存在采样间隙，部署时还须给临时工作区独立容量边界，并验证写满时不会侵占 Java/宿主机关键存储；Kubernetes 的临时存储限制可能通过驱逐 Pod 生效，不能把它当作每次写入前的精确拦截。若连固定 commit 的基础 Git 元数据或一个合规包都无法在当前预算下完成，同一配置下反复领取不会产生进展：Java 应展示容量阻塞并等待管理员调整容量或规则，不能无限立即重试，也不能把该包误判为业务超限。容量边界与一次批处理后可继续处理下一个路径的关系须在大仓库试运行中记录峰值和清理后占用。 + +Java 以规范化路径序列及对应文件字节计算正式 SHA-256 内容指纹，并把继承的许可证/NOTICE 证据纳入合规指纹；上游许可证变化因此需要重新核对。固定 commit 只是获取一致快照的坐标，不代替内容比较。路径移动仅在同仓库完整包指纹唯一匹配时自动绑定，否则进入冲突处理。完整扫描确认目录消失仅更新来源状态，不删除已发布 Skill；扫描不完整不触发缺失推断。 + +同 namespace slug 冲突时，Java 将受控冲突码及规范化目标 slug 留在摄入记录中；平台 `SUPER_ADMIN` 或该 TEAM 的 OWNER/ADMIN 可提供既有 Skill ID 与处理依据进行人工确认。确认事务锁定摄入项和目标 Skill，核对 namespace、slug、非归档、无普通用户待审项及无其他权威来源绑定，先以空的“最近已发布内容”指针预留来源路径，然后把该固定 commit 包重新排入发布流程。成功确认须留下操作者和依据的审计记录。只有待审版本创建成功的事务才将技术所有者改为系统 actor；失败时不改变原 Skill 内容或所有者。自动路径移动仍须等待同仓库完整扫描后唯一指纹匹配，不能用人工同名确认代替该判断。 + +**备选：整仓 clone 或每文件 Contents API。** 不采用。前者在超大仓库消耗不可控，后者易耗尽 API 额度；两者都不能替代包级上限。 + +### 4. 临时对象与正式版本隔离 + +Java 为每个包签发短时、限对象 key 且记录预期大小的上传票据。Go 先在本地临时目录把一个 Skill 的固定 commit 文件打包，再上传到隔离的临时对象存储；S3/MinIO 首期采用预签名 PUT，本地文件存储部署使用受服务身份保护的 Java 流式入口。Go 向 Java 回报 `uploadId`、固定 commit、路径、归档大小与 SHA-256，由 Java 通过自己的存储适配器把 `uploadId` 解析为临时对象并读取；回报和持久任务不传、也不保存预签名下载 URL 或任意外部链接。Go 无对象存储管理凭证，也不能写正式版本存储。预签名 PUT 不能仅凭签名保证服务端拒收超大对象，Go 须在本地预检，Java 在摄入前复核实际对象大小并清理超额对象，隔离暂存区域配置容量告警与清理；本地流式入口在接收时执行硬上限。Java 再校验摘要、解包边界、SKILL.md、忽略规则、内容政策与许可证；成功后才写入正式版本存储，暂存对象在消费后或确认无需保留时清理，并为失败清理保留补偿记录。 + +摄入端不得直接复用普通上传 Controller 的整包驻留内存解包路径。Go 的逐包回报由 Java 校验服务身份、当前租约代际、固定 commit、路径及临时对象元数据，在短事务中持久保存摄入记录后返回接受；Java 可恢复后台任务再逐包读取、校验、创建待审目标并触发扫描，单包失败隔离重试。Java 应先根据临时对象元数据执行包总量上限，再以有界、流式方式读取条目；任何需要把单文件内容物化的校验都受既有单文件、文件数和总量政策约束。现有 Skill Suite Bundle 的暂存对象流式读取可作为实现参照，但同步摄入需要自己的状态与来源校验。 + +**备选：Go 直写正式对象或共享存储密钥。** 不采用。采集结果不可信，且会扩大 Go 服务的权限与泄露半径。 + +### 5. 调度、额度与恢复 + +Java 使用持久任务、租约和唯一约束保证多实例领取；Go 在任务内续租并在停机时停止领取。调度按已登记来源更新与新配置精选仓库、精选用户发现、广泛搜索分层；TEAM 来源有公平份额。namespace 和全局“未解决同步项”高低水位控制反压：按 `(namespace, repository ID, Skill 路径)` 对当前最新目标去重，计入已接收、复核/扫描中、待同步审核和待管理员处理冲突的项；已批准、拒绝或被新目标取代的历史项不计入。达到全局高水位后，所有 namespace 停止**新包采集准入**；仅某个 namespace 达到高水位时，只暂停该范围的新包采集。Go 在取得每个包的文件内容之前须向 Java 申请一次原子准入；Java 将未完成准入计入占位，避免并发仓库任务越过水位。高水位后不得发放新准入，已获准且正在处理的包可完成或失败并释放占位；正在执行的仓库任务在下一个包前回报部分进度，不能继续下载兄弟包。 + +高水位期间，已启用的仓库搜索与精选用户或组织枚举可在 GitHub 额度及候选预算内继续，结果只作为按 repository ID 去重的**待验证仓库线索**和可恢复游标保存；不读取新候选仓库树、不下载包、不建立已验证来源或待审目标。候选预算满时保存游标并延后，而不是无限堆积候选。已登记仓库在额度允许时继续执行仅查询最新 commit 的轻量任务。Java 分别保存 `lastObservedCommit` 与 `lastFullyProcessedCommit`，前者只表示看见上游变化，不得推进后者或创建待审项；回落到低水位后重新核对待验证线索的当前仓库状态与已登记来源的当前 HEAD，再采集最新包，不补采所有中间 revision。每轮发现预算根据 GitHub 资源额度、重置时间、执行容量和未解决同步项数计算；超出候选保留检查点与延后原因。类别级配置的是 24/24/72 小时默认目标周期，限流和反压可以延后实际执行。Go 不长时间睡眠占租约，回报 `retryAt` 给 Java。 + +配额按 GitHub 返回的资源类别与凭证分别跟踪。无效或被撤销的凭证标记不可用并通知平台管理员；任务可改用另一健康的系统凭证，没有健康凭证时仅对可公开读取的仓库使用匿名额度。认证失败、仓库访问失败与 GitHub 限流分别归因；限流遵守响应的重试时间，不以连续轮换凭证规避。规则命中、有效 Skill、延后、失败、配额与积压进入同一运行视图；短期产出只用于观察，不自动修改规则权重。来源不可达按已确认的 24 小时、3 天、7 天低频复查节奏恢复，限流与网络暂时故障不等同于不可达。 + +**备选：固定每规则采集数量或 Go 长时间等待 reset。** 不采用。固定数量无法适应部署规模与审核积压；长等待占用租约和工作池。 + +### 6. 凭证与服务身份 + +GitHub 凭证在 Java 用部署提供的密钥加密保存，密文带密钥版本与随机 nonce，以便轮换。新部署凭证密钥应通过现有 Compose/Helm/Kubernetes secret 入口配置，不引入 KMS。平台后台只展示脱敏标识、有效性和额度状态；更新后不可读取明文。认证失效的凭证需要明确的健康状态与更新提示，不能把配额耗尽误标为失效。Go 调用 Java 内部接口使用专用服务密钥，不复用用户 Token 或 GitHub 凭证；密钥通过现有部署 Secret 提供，支持新旧密钥短期重叠轮换，Java 仅对内部路由验证该服务身份。优先复用已有 TLS，首期不强制 mTLS。Token 与服务密钥均不进入持久任务、普通 API、URL、日志、异常或指标标签。Go 只访问公开仓库，并复核仓库公开状态。 + +### 7. 版本审核与管理界面 + +Java 复用现有包校验、Scanner、Skill 版本与审核权限，在审核记录增加来源标记与同步绑定关联。Web 在既有审核区提供独立的“同步审核”入口，按仓库/运行分组但逐 Skill 决策；普通审核列表不混入同步项。新/变更内容先扫描再待审；只有最新上游目标可批准，旧目标保留审计但不再可批准。相同已拒绝指纹不反复排队。许可证或 NOTICE 证据和文件差异在审核页展示，`@github` 未识别许可证先阻止公开批准。同步内容不能在 SkillHub 内直接编辑。 + +普通技能和 GitHub 同步技能的详情页共用顶部对象说明、中部页级 Tab、底部固定审核意见栏。顶部始终显示扫描与许可证状态摘要，完整扫描卡片、发现项和来源证据在“来源与安全”Tab，摘要可直达详情；差异 Tab 以全宽单栏方式逐文件预览，不保留常驻文件或证据侧栏。复用现有开源 React 差异组件及审核绑定文件接口，按选中文件请求正文，维持大文件和非文本降级。审核意见位于 Tab 外，切换内容不丢失，扫描中、许可证依据及退回理由的原有服务端约束不变。 + +审核首屏不应无条件读取可能达到单文件上限的说明文档正文或审核历史。现有审核详情响应可用向后兼容的 `includeDocumentation` 查询参数控制是否读取文档：旧调用缺省仍包含文档，页面首屏仅取元数据；进入内容 Tab 后用受 256 KiB 服务端限制的单文件预览接口取得说明正文，不再重复请求整份详情。审核历史同样按 Tab 加载。来源响应继续先提供文件变化元数据，旧/新文件正文只对当前选中项读取。是否进一步拆分许可证证据响应，取决于浏览器网络测量结果。 + +平台管理区新增 GitHub 同步入口，信息结构为总览、发现规则、精选入口、来源、运行与额度、凭证；TEAM 在自身 namespace 页面只看到其仓库来源和运行结果。来源详情以仓库为主，展开逐 Skill 的最近 revision、同步/审核状态和失败原因；操作包括暂停/恢复、重扫、忽略路径及来源可见范围。手动重扫只调整合格任务顺序，不穿透配额或积压反压。 + +平台页把总览与已登记入口、运行列表、当前设置值常驻显示；新增仓库/账号/搜索规则，以及容量、周期和凭证编辑使用按需打开的弹窗。搜索规则的编辑与预览共处一处，新增后仍须在入口列表手动启用。弹窗关闭后清空未提交的凭证输入；密集运行/来源表在窄屏内部横向滚动，避免整页溢出。页内操作按钮保持紧凑，状态和配置摘要仍应易于辨认。 + +规则预览由 Java 排队、Go 实际搜索；Go 离线或状态未知时平台页面禁用预览并解释原因，仍允许保存为停用规则。页面把排队、领取执行、延后重试、失败和完成分开呈现，不能用“正在搜索”概括所有未完成任务。规则表单使用与后台一致的平面输入框、原生下拉框和可见字段标签，不采用大面积色块或发光聚焦效果。 + +## Risks / Trade-offs + +- **GitHub 搜索遗漏或返回截断** → 增量窗口重叠、低频全量、规则与子树检查点；扫描不完整时禁止缺失推断。 +- **超大仓库耗尽 Go 临时磁盘** → 先基于 Tree 元数据定向选包，限制工作目录及总临时磁盘，逐包清理与分批续跑；资源不足要可观察，不得回退为整仓下载。 +- **审核队列积压** → namespace/全局高低水位停止新包采集准入；有预算地继续记录仓库线索和已登记仓库的低成本 HEAD 变化，候选满时保存游标延后；同步审核独立视图，暂不做未经验证的自动批准。 +- **Go 失联、重复回报或部分上传** → DB 租约代际、幂等键、逐 Skill 结果、暂存对象 TTL 与补偿清理。 +- **复用普通发布入口可能绕过同步审核** → 同步摄入使用独立编排，明确禁止超级管理员自动发布分支；正式版本仍复用可复用的校验与扫描能力。 +- **Token 泄露或扩大权限** → 加密静态存储、短时任务响应、服务身份、日志脱敏、公开状态复核；无 TLS 部署仍须在部署说明中标注传输风险。 +- **`@github` slug 已被占用** → 安装/迁移时阻断并给管理员明确处理路径，不自动移动现有 namespace。 +- **本地文件存储没有原生预签名 PUT** → 用同一上传票据语义的 Java 流式端点，不要求客户部署 S3。 + +## Migration Plan + +1. 增加来源、规则、运行、任务、凭证及来源绑定表/列；给 `ReviewTask` 增加同步来源判别，旧记录默认普通审核；创建系统 `@github` 时检查 slug 冲突。 +2. 先部署 Java 数据/API 与后台页面，再部署 Go 容器与服务身份，完成匿名/单凭证、对象上传和任务领取联调；预置广泛搜索规则初始停用，平台管理员可在同一规则页面启用，试运行前不对公网启动无控制搜索。 +3. 以少量精选仓库试运行，验证包级失败、限流、扫描、审核和停机恢复；逐步启用精选用户及广泛搜索。保留人工上传现有路径。 +4. 回滚时停止新任务领取并关闭发现，保留已发布 Skill、待审项和运行历史；卸载 Go 容器不回滚领域数据。 + +## Open Questions + +- 默认 24/24/72 小时目标周期、候选忽略目录、业务包限制、7 天全量搜索和运行记录保留边界已确定。首期继续人工同步审核，不提供自动发布或审核方式开关。长期无人审核时已确定继续有预算地发现仓库线索、暂停后续包采集；全平台 200/100、单 namespace 50/25 的高低水位建议偏低，不作为默认值。具体水位和候选预算须结合试运行的待审内容大小、存储与摄入能力确定；租约时长与轮询间隔由实施及小范围联调确定。 +- Go 临时磁盘预算必须低于部署实际提供的存储边界；每仓库和总预算、保留空闲量及并发目录数要在大仓库小范围验证后给出部署默认值。已完成的同步运行/任务详情默认保留 90 天；未完成任务与恢复数据不能按天数清理，来源与审核审计按现有生命周期保留；容器标准输出日志由部署环境决定保留期。 +- Java 后台摄入记录的具体状态名、重试上限和过期清理周期需在实施前与现有审核及存储政策对齐。 + +## Go 服务专项设计 + +已确认 Go 服务与 SkillHub 位于同一仓库，当前实现目录为 `github-sync-go/`，构建独立镜像并随同版本发布。采集能力可关闭，Go 离线不阻断人工上传、市场浏览与既有审核。首期默认部署一个 Go 副本，Java 的租约和幂等合同仍须允许后续增加副本，首期不交付自动扩缩容。Go 独立于工作池每 30 秒向 Java 报告一次在线状态；Java 只保存最近一次成功心跳，90 秒无心跳则后台显示离线。该状态与审核积压引起的新包准入暂停分别呈现。容量验收目标为每部署约 1000 个已登记仓库、10000 个 Skill 路径，不作为产品硬上限；先用少量公开仓库验证正确性，再用可控样本验证容量。接口字段、超时、续租时机和固定 commit 获取的异常路径仍需收敛并验证。 + +### 内部结构与数据流 + +单个 Go 进程划分任务客户端、发现执行器、仓库采集执行器、临时包上传器和运行观测五个边界。发现与仓库同步使用独立的有界工作池；两者共用按凭证和 GitHub 资源类别记录的额度状态，但不共享业务持久化。任务客户端只调用 Java 的内部版本化 HTTP/JSON 合同；不访问 SkillHub 数据库、Redis、普通用户 API 或发布接口。Java 仍决定任务优先级、租约、来源绑定、审核反压和版本状态。 + +仓库执行器先解析公开仓库稳定 ID 与固定 commit，再枚举树并形成逐 Skill 路径结果；包上传器仅使用 Java 签发的短时临时对象授权。已报告成功的路径由 Java 检查点记账；Go 进程重启后根据重新领取的任务与检查点恢复，不依赖本地工作目录。每个任务的工作目录有时间、磁盘和文件数上限，结束后清理。所有出站 GitHub 访问只能从解析后的 GitHub 仓库坐标构造目标，不能跟随仓库内容提供的任意 URL 或执行采集到的脚本。 + +Java 从 Go 的领取/心跳记录推导最近在线时间。Go 离线时，后台显示采集暂停与最近在线时间；相同来源的周期任务合并，保留下一次应执行时间与重扫意图，避免每个周期累积一份重复任务。Go 恢复后继续按来源优先级、额度和审核反压处理，不能把离线期间未执行的轮次当作成功同步。 + +### Java 与 Go 的模块落点 + +Java 新增一个 `skillhub-github-sync` Maven 模块，作为来源发现与同步控制面的功能模块，最终仍打包到现有 `skillhub-app` 进程，不额外部署 Java 服务。它包含来源、规则、任务、凭证引用和包摄入的模型、仓储接口、调度/摄入协调服务，负责来源绑定、最新可审目标和检查点推进。该模块可依赖现有 `skillhub-domain` 的 Skill/审核能力与独立的 `skillhub-storage` SPI，但不能依赖 `skillhub-app` 或 `skillhub-infra`；现有核心 `skillhub-domain` 不反向依赖新模块。`skillhub-infra` 依赖新模块并实现其 JPA 仓储、凭证加解密适配和原子领取；`skillhub-app` 依赖新模块，承载管理员/TEAM 配置入口、独立内部任务入口、运行查询以及与现有 Scanner/审核流程的装配；Flyway 迁移仍在 app。`skillhub-auth` 的机器身份校验不依赖新模块，由 app 给内部路由配置。`skillhub-storage` 扩展临时对象上传能力，S3/MinIO 使用限对象 key 的短时预签名 PUT，本地存储经 Java 受服务身份保护的流式入口接收。现有包校验、Scanner 和审核领域能力可复用；Go 回报不能直接调用普通发布入口，因为现有 `SkillPublishService` 包含超级管理员直发分支。新摄入编排必须始终先建立待扫描/待审目标。 + +这个模块应交付完整的同步状态机和少量面向 app 的操作，而不是为每个表各加一层透传服务。新增模块需要同步更新 `server/pom.xml`、`skillhub-app` 与 `skillhub-infra` 的依赖，以及 `server/Dockerfile` 的 Maven 缓存阶段；仓库现有 `skillhub-search` 和 `skillhub-notification` 已提供独立功能模块的先例。 + +Java 的定时调度只创建或合并到期任务；PostgreSQL 任务表是唯一持久队列，领取在事务中以行锁/`SKIP LOCKED` 或等价原子条件更新完成，并生成租约代际。所有 Java 实例均可参与调度与后台摄入，但唯一约束防止重复任务，租约令牌防止旧 Go 进程继续回报。逐包回报只在摄入记录与暂存对象引用持久化后确认；后台 worker 在独立短事务中读取受限临时对象、重新校验并推进 Scanner/审核。摄入队列积压期间须保留临时对象直到消费或明确失败，不能仅凭固定 TTL 清理尚待处理的已接收包。仓库树不完整或某个包仍需重试时，保留路径级进度；`lastFullyProcessedCommit` 只能在完整枚举且所有应处理路径已有终态后推进。 + +内部路由采用独立的机器身份校验，只赋予领取、续租、临时上传授权和回报权限；浏览器会话、普通用户/API Token 不可调用，内部路由也不能落入“仅需已登录”的默认规则。仅对这组内部路由采用无会话认证及相应的 CSRF 处理，不改变用户 API 的安全策略。现有 `IdempotencyInterceptor` 未在 MVC 配置中发现注册，因此 `X-Request-Id` 只用于日志关联；真正幂等由租约代际、逐包业务键和 PostgreSQL 唯一约束实现，不依赖该拦截器。 + +Go 建议保持一个独立 Go module 和单个二进制；`cmd/` 只做配置加载、依赖组装和优雅停机，`internal/controlplane` 封装版本化 Java HTTP 合同，`internal/github` 封装 REST、额度判断和受限 Git CLI，`internal/discovery` 做规则执行与仓库召回，`internal/repository` 做固定树、Skill 根枚举与包归档，`internal/worker` 管理两类有界工作池、领取、续租及恢复。每个内部模块只暴露少量完成一项工作所需的接口；Go 不持有数据库、Redis、发布业务对象或跨重启检查点。GitHub 出站客户端与 Java 内部客户端使用分开的 HTTP transport，避免部署级 GitHub 代理误代理内部请求。Go 关闭时停止领取、取消或限时收尾在途任务，让租约过期后由 Java 重派。 + +建议的代码布局如下，属于新建方案，当前仓库尚无这些 Go 文件： + +```text +github-sync/ + go.mod # module github.com/iflytek/skillhub/github-sync + cmd/skillhub-github-sync/ # 单个进程入口 + internal/controlplane/ # Java 任务合同客户端 + internal/github/ # REST、Git CLI 与额度 + internal/discovery/ # 规则执行和仓库召回 + internal/repository/ # 固定树、Skill 根和包归档 + internal/worker/ # 有界执行、续租、停止 + Dockerfile +``` + +Go 没有面向业务的入站接口或对外 Ingress；仅保留健康检查与可选的内部指标抓取入口。Java 不需要连接 Go，服务发现仅需 Go 能访问 Java 内部接口和临时对象上传目标。 + +```mermaid +flowchart LR + Admin[SkillHub 管理后台] --> Config[Java 配置与运行查询] + Config --> DB[(PostgreSQL 来源、任务、租约、摄入记录)] + Scheduler[Java 调度与反压] --> DB + GoWorker[Go 轮询与有界工作池] -->|机器身份、领取与续租| Internal[Java 内部任务接口] + Internal <--> DB + GoWorker --> Discovery[Go 搜索与仓库发现] + GoWorker --> Repository[Go 固定 revision 与逐包采集] + Discovery --> GitHub[公开 GitHub] + Repository --> GitHub + Repository -->|上传临时包| Stage[(临时对象)] + GoWorker -->|逐包回报与任务完成| Internal + DB --> Ingest[Java 可恢复摄入 worker] + Stage --> Ingest + Ingest --> Review[现有扫描与独立同步审核] + Review --> Market[已发布 Skill] +``` + +内部合同的逻辑顺序为 `claim → renew → package admission → upload authorization → package report → complete/fail`。`package report` 确认的是摄入记录已持久化；Java 后台处理与审核拥有独立状态。高水位时 Java 可下发仓库的“仅检查 HEAD”工作模式，Go 只回报当前 commit，不能上传包或推进“已完整处理”检查点。以下状态名是设计语义，不预设必须照字面增加数据库枚举： + +状态转换、检查点及异常恢复见 [state-model.md](state-model.md)。字段和 HTTP 错误语义以 [内部 OpenAPI 合同](../../../docs/api/github-sync-internal.openapi.yaml)为准；调用流程与联调边界见[接口说明](../../../docs/api/github-sync-internal.md)。此阶段合同是实现目标，尚无 Java/Go 端点。 + +版本化内部 OpenAPI 合同单独维护，计划放在现有 `docs/api/` 下的 `github-sync-internal.openapi.yaml`，与该目录已存在的认证合同并列；它只描述机器接口，不混入面向 Web 的公开 OpenAPI 或自动生成的 Web SDK。该文件是 Java/Go 协议字段、错误语义和兼容窗口的唯一合同，OpenSpec 则保留业务行为与验收要求，二者不复制同一份规格。Java 实现和 Go 客户端都需有合同样例与兼容性检查。`claim` 返回任务快照及租约代际，空队列时返回无任务；`renew` 只延长当前代际；上传授权只允许当前任务、commit、Skill 路径对应的临时对象；`package report` 重放同一业务键应返回已有摄入结果，内容冲突或旧租约应拒绝；`complete/fail` 回报完整枚举范围、逐路径终态、额度与可重试时间。服务端拒绝未知协议主版本，兼容的字段增加不得改变已存在状态含义。跨服务错误至少区分认证失效、租约失效、对象或摘要不符、可重试采集失败、GitHub 限流、来源不可读和不可恢复的包规则失败；Go 据此决定重试、停止或回报，而不是把所有非 2xx 当成可重试。 + +| 层次 | 应区分的状态 | 对管理员的含义 | +| --- | --- | --- | +| Go 采集任务 | 待领取、执行中、部分完成、已交付、延后/失败 | 是否已从 GitHub 获得并交付包，或仅检查到上游变化 | +| Java 包摄入 | 已接收、复核中、待同步审核、处理失败 | 暂存对象是否已验证并形成可审目标 | +| Skill 版本 | 已发布、待审、被拒、被新目标取代 | 市场当前可见版本与下一次发布决定 | + +### 首期技术栈基线与待定依赖 + +| 部位 | 首期建议 | 选择理由与验证点 | +| --- | --- | --- | +| Go 语言与服务主体 | 已选 Go 1.27 构建基线，首期固定受支持的最新补丁版、单进程；标准库 `context`、`net/http`、`encoding/json`、`log/slog`、`os/exec`、`archive/zip`、`crypto/sha256`；独立 `go.mod`，module 路径随仓库子目录 | 构建版本以 `github-sync-go/go.mod` 和 Dockerfile 为准，升级时核对[Go 官方发布记录](https://go.dev/doc/devel/release)；任务轮询只需少量健康/指标入站接口，不需要 Web 框架、ORM 或自己的数据库 | +| Java 协作 | 已选独立、版本化的内部 OpenAPI 文件，HTTP/JSON 实现；专用服务密钥；每请求独立 `X-Request-Id` 与 W3C trace context | 与现有 Java REST、request ID 和追踪方式对齐；领取、续租、逐包回报具备租约代际与业务幂等键；不混入公开 Web SDK | +| Java 控制面 | 现有 Java 21 / Spring Boot / JPA / PostgreSQL / Flyway / Spring Security，加一个 `skillhub-github-sync` Maven 模块，仍运行于 `skillhub-app` | 新模块封装同步状态机和控制面；Go 不直连数据库，也不新增队列、Redis 通道或独立 Java 服务 | +| GitHub 元数据 | 固定 REST API 版本的薄客户端，显式处理分页、ETag、搜索不完整、Tree 截断、403/429 和响应额度头 | GitHub Search 和 Tree 有明确返回上限；不能假设一次查询或一次递归树响应完整 | +| 包文件获取 | Go 镜像内置 Git CLI，受限的 blobless 局部克隆与稀疏读取固定 commit | 小样本已证明路径可运行；实施前仍需验证大仓库、分支变化、磁盘峰值与失败恢复，不默认增加第二套逐 Blob 采集引擎 | +| GitHub 出站代理 | 部署层提供标准 HTTP 代理环境配置，Go 的 GitHub HTTP 客户端与 Git CLI 共用受控设置；无逐来源代理管理页面 | 满足需经代理访问 GitHub 的私有化环境；内部 Java 通信与暂存对象访问不应误走 GitHub 代理，代理凭证不得进入日志 | +| 可观测性 | 标准库 `slog` JSON 日志；[Prometheus 官方 Go 客户端](https://prometheus.io/docs/guides/go-application/)暴露低基数指标；需要跨服务链路时接入[稳定的 OpenTelemetry Go traces](https://opentelemetry.io/docs/languages/go/)，未配置采集端时不影响采集 | 对齐 Java 现有 Actuator/Micrometer 与 OTel；任务 ID、运行 ID、来源 ID 只作日志/trace 关联，不作指标标签；Token 和完整 Skill 正文不进入观测数据 | +| 部署 | Go 多阶段构建，运行镜像含 Git、CA 证书且以非 root 运行；与 SkillHub Compose/Helm 同版本发布并进入现有镜像发布矩阵；默认一个副本；工作目录使用专用临时挂载并配置应用预算、容量监测和部署侧存储边界 | 必须实际提供 Git CLI、证书和可写的受限临时目录。Kubernetes 可配置 `emptyDir.sizeLimit` 与 `ephemeral-storage` 请求/限制，但达到限制可能导致 Pod 驱逐，不能替代 Go 在获取前预检和逐包清理；[Kubernetes 官方说明](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)。Compose 的具体磁盘挂载和容量约束需按目标环境验证，不假定普通 bind mount 自动有配额。 | + +默认不执行 GitHub 仓库中的代码，不跟随符号链接或 submodule。Git CLI 以固定参数调用、限制超时/资源、禁止 shell 拼接及凭证出现在 URL/命令行；只处理 GitHub 公开仓库。建议仅引入 Prometheus Go 客户端和按实际部署需要的 OpenTelemetry Go tracing 作为运行依赖；Git CLI 由运行镜像提供。依赖具体版本和多架构镜像以实施时受支持版本、漏洞扫描及实测结果锁定，不把本文的技术选型误认为现有代码。 + +### 分阶段验证 + +先选择少量结构不同的公开仓库，覆盖单 Skill、多 Skill、嵌套目录和较大仓库；只验证固定 revision、树完整性、包边界、请求数、传输量、磁盘峰值和单包失败隔离。随后用受控 GitHub fixture 与合成任务逐级增加到容量目标，测量任务延迟、暂存对象数量、Java 数据库写入和审核积压反压；不对 GitHub 公网发起 1000 仓库的无控制压测。容量目标与目标同步周期分别报告，不能用一次完成数量代替周期达标。 + +局部 Git 获取只用于受限包文件；必须分别验证大仓库、网络波动、额度、实际下载成本和临时目录容量，不能从单一小样本推断总体性能。 + + +Go 当前在 Git 子进程运行期间周期采样工作目录并限制 ZIP 写入字节数；采样间隔内 Git 仍可能短时超过应用预算，因此部署时还须提供独立临时卷容量边界。这个应用级预算不能单独证明大仓库不会耗尽宿主磁盘，需结合小容量卷和大仓库样本验证停止、清理及续跑。 diff --git a/openspec/changes/add-github-skill-sync/glossary.md b/openspec/changes/add-github-skill-sync/glossary.md new file mode 100644 index 000000000..59a6bf273 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/glossary.md @@ -0,0 +1,68 @@ +# SkillHub 社区 Skill 同步 + +本文统一描述社区仓库同步中的产品术语，避免把自动获取上游内容与自动公开混为一谈。 + +## 同步与发布 + +**自动同步**：服务端自动发现或检查上游仓库变化，固定变更内容，生成 SkillHub 新版本，并提交安全扫描和人工审核。 +_避免_：自动发布、自动替换已发布版本 + +**人工审核**：授权审核人对新 Skill 或内容变化后的新版本做出发布或拒绝决定。同步服务本身不能代替审核人。 +_避免_：机器审核（仅指自动化策略或扫描） + +**自动发布**：新版本未经过人工审核就对目标受众可见。本需求首期不允许。 +_避免_：自动同步 + +## 来源与同步状态 + +**发现候选**：搜索规则或精选 GitHub 账号返回、尚未确认含有效 Skill 的公开仓库线索；它还不是纳入持续同步的正式来源。 +_避免_：把搜索命中直接称为已导入来源 + +**正式来源**：已纳入某个 namespace 的持续同步账本的 GitHub 仓库；管理员明确登记的仓库可在首次扫描前进入该状态。 +_避免_：把正式来源等同于已发布 Skill + +**权威同步来源**：唯一允许为某个 SkillHub Skill 提供同步内容和后续版本的外部来源绑定；其他同名或重复内容来源不会自动成为更新源。 +_避免_：多个主来源 + +**来源绑定**：将 GitHub 仓库中的一个 Skill 路径关联到一个 SkillHub Skill，并记录上游身份与同步进度。 +_避免_：按 Skill 名称推断关联 + +**来源身份**：由 GitHub 仓库 ID 和 Skill 路径标识；内容 Hash 用来比较快照，不单独证明两个路径属于同一个 Skill。路径移动时，仅同仓库内唯一匹配的完整技能包 Hash 可自动作为改绑依据。 +_避免_：用 `SKILL.md.name` 或单个文件 Hash 作为唯一身份 + +**技能包内容 Hash**：对完整 Skill 包快照计算的内容指纹，用于发现内容变化和辅助识别路径移动；不替代来源身份。 +_避免_：只比较 `SKILL.md` 文件 + +**上游托管内容**：同步 Skill 的正文和文件由上游来源维护；SkillHub 负责扫描、审核、发布和可见性治理。需要定制时应创建独立副本。 +_避免_：直接编辑同步副本 + +**同步暂停**：可停止仓库的后续检查；单个 Skill 路径通过来源忽略规则排除，暂停不改变关联 Skill 的发布、归档、隐藏或可见性状态。 +_避免_：暂停发布、自动归档 + +**目录忽略规则**：用于排除仓库中不应发现或同步的 Skill 路径；平台默认规则包含测试目录，管理员还可在后台按来源添加或覆盖规则。命中已有绑定时停止该来源的后续同步并保留本地 Skill；取消忽略后重新对账。 +_避免_：删除已导入技能 + +**上游不可达**：来源连续无法通过公开接口读取，经 24 小时和 3 天复查仍失败后进入不可达状态；此状态不等于 SkillHub Skill 已删除或应停止分发，来源仍每 7 天复查并在恢复后重新对账。 +_避免_：自动归档、自动删除、永久停止检查 + +**已取代的待审目标**：某个上游快照因发现更新的 revision 而不再允许审核或批准；原快照保留审计记录，新快照成为当前审核目标。 +_避免_：覆盖原待审快照、过期版本仍可批准 + +## 官方探索 + +**技能同步**：统一管理公开仓库发现、技能采集和后续内容更新的业务能力；同步后的新版本仍按既有规则扫描和审核。 +_避免_：把技能同步与深度探索作为两套并列产品 + +**深度探索**：平台内置并维护发现策略的公开 GitHub 技能仓库发现方式；管理员以少量配置控制持续探索。 +_避免_：一次性抓取、自动发布、所有来源同步的总开关 + +**探索停用**：停止该探索途径的新仓库发现，不等于暂停已登记来源的后续更新。 +_避免_：来源停用、删除技能 + + +**指定仓库**：管理员明确指定要持续检查和采集的公开仓库，不代表技能已通过质量、安全或发布审核。 +_避免_：精选仓库（旧界面名称）、已认证仓库、已审核技能 + + +**平台采集暂停**：暂停平台@github的新发现、检查和下载，并让在途采集到安全检查点停止；不暂停TEAM采集、已接收包处理、人工审核或已发布技能使用。 +_避免_：暂停全部同步、全局暂停、自动探索停用 diff --git a/openspec/changes/add-github-skill-sync/go-service-prd.md b/openspec/changes/add-github-skill-sync/go-service-prd.md new file mode 100644 index 000000000..de484a7f2 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/go-service-prd.md @@ -0,0 +1,97 @@ +# GitHub Skill 采集 Go 服务：产品需求 + +> 已实现增量：官方深度探索扩展原有元数据发现范围，新增 README 与有界作者扩展，采用统一技能同步后台中的采集配置抽屉和独立探索开关。产品决定见[需求](deep-discovery.md)，技术合同见[增量设计](deep-discovery-design.md)，当前验收范围见[验收追踪](verification.md)。下文仅元数据搜索及只按自定义规则启停的原有入口边界不适用于官方探索；自定义规则继续兼容。 + +> Issue #516 的 Go 服务产品视角。跨服务的正式行为与验收以本变更的 [三份行为规格](specs/) 为准，技术取舍见 [design.md](design.md)；本文只描述服务要交付的运行能力、操作体验与衡量方式，不维护第二套 API 或任务清单。 + +## 1. 产品定位 + +Go 服务是随 SkillHub 私有化部署的**公开 GitHub 采集执行端**。它与 SkillHub 位于同一代码仓库，根目录有独立的 Go 服务目录和独立镜像，首期默认单副本。平台管理员仍在 SkillHub 后台配置来源、查看运行反馈；Java 保存配置、任务、凭证、来源账本、版本与审核，Go 主动领取任务，完成 GitHub 访问、固定 revision、Skill 包采集和结果回报。Go 服务不提供面向用户的独立后台，也不拥有 Skill 发布权限。 + +服务的价值是把搜索、仓库树枚举、文件下载和限流等待从 SkillHub 主服务移出，同时让平台能持续发现社区 Skill，而不要求管理员逐仓库逐文件操作。 + +## 2. 使用者与要完成的事 + +| 使用者 | 在 SkillHub 中做什么 | Go 服务要提供的结果 | +| --- | --- | --- | +| 平台管理员 | 配置搜索规则、精选仓库、精选 GitHub 用户或组织；查看运行与额度 | 按规则持续发现公开仓库，报告命中、有效 Skill、延后、错误和下次可运行时间 | +| TEAM 管理员 | 在本 TEAM namespace 登记公开仓库、暂停或重扫来源 | 仅处理本 TEAM 明确登记的仓库，逐 Skill 回报状态，保留 TEAM 归属 | +| 同步审核者 | 在独立队列审查新 Skill 和更新 | 获得固定 commit、完整包、逐文件变化、来源与许可证证据；Go 不决定审核结果 | +| 私有化部署运维者 | 部署、升级、排查采集问题 | 一个可独立启停的 Go 容器；健康与任务进度可观察，重启后继续，不要求 GitHub Actions 或新的队列组件 | + +## 3. 首期能力 + +### 3.1 多类来源入口，一条仓库采集流水线 + +官方深度探索、自定义搜索规则、精选仓库、精选用户或组织均只产生公开 GitHub 仓库坐标。TEAM 首期只登记具体公开仓库。发现入口的结果按 GitHub repository ID 去重；自动搜索或精选用户发现的仓库在确认存在有效 Skill 候选后才成为正式来源。管理员明确登记的仓库可先呈现“待首次扫描”。同一仓库在 `@github` 与 TEAM 下分别形成来源绑定，不共享审核决定。 + +官方深度探索从仓库名称、描述、topic、README 线索及有效仓库作者的一层扩展获取候选，再检查仓库树中的 `SKILL.md`；自定义规则继续保持元数据条件。全站代码搜索先进行独立试验，不作为首期默认渠道。界面说明索引和搜索范围仍存在覆盖边界。 + +后台自定义规则使用关键词/topic 两类结构化条件，分别标记核心或扩展优先级。Go 将每项编译为独立的仓库搜索查询，按仓库 ID 去重并记录命中条件；规则版本与增量/全量搜索窗口在领取时固定，分页续跑不能因管理员改规则而换查询。核心条件先使用预算，扩展条件在剩余额度内运行；查询结果截断或标记不完整时应分片或明确延后，不能把遗漏当作完成。 + +Go 服务使用一个部署，内部区分“发现”和“仓库同步”工作。Java 按后台配置与目标周期生成任务；Go 轮询领取。仓库任务在固定 commit 上一次枚举该仓库所有可见 `SKILL.md`，逐 Skill 包采集和回报，不要求 Java 再按 Skill 拆任务。 + +### 3.2 可恢复的逐包采集 + +每个 `SKILL.md` 所在目录是一个 Skill 包；嵌套 Skill 自成包，不并入父包。Go 在固定 commit 上只取得需要的文件到受限本地临时目录，按当前有效发布政策逐包检查文件数、单文件和包总大小，拒绝符号链接和 Git submodule；随后逐包打包上传到隔离临时对象存储，只把 `uploadId`、路径、commit、大小和摘要回报 Java。Java 用自己的存储适配器读取对象，不依赖 Go 提供的预签名下载链接。一个包失败时，其他包继续；下次从固定 commit 重试失败路径。Java 独立复核、计算正式内容指纹、创建待审版本，并在消费后清理临时对象。 + +树枚举被截断时，Go 按子树继续；部分子树仍无法读取时，已完整发现的 Skill 继续采集，未完成范围明确标记并续跑。扫描不完整、仓库临时不可达或 Go 重启都不能被解释为 Skill 已被上游删除。 + +大仓库先通过固定 commit 的树元数据定位 `SKILL.md`、预估各候选包文件数与大小，再针对合格目录下载文件；`blob:none` 仍会占用 Git 树元数据空间，因此每仓库目录和总临时磁盘受预算及部署工作区容量约束。Go 在每批开始前预留 Git 元数据、选中包文件、归档和安全余量，并在下载、归档及清理后检查实际占用。逐包上传并持久回报后清理本地文件；达到预算时保存路径游标、释放目录并从同一 commit 分批继续。若当前容量连固定 commit 元数据或一个合规包都处理不了，后台显示容量阻塞并等待配置调整，避免同一条件下立即重试；不能无限制整仓下载，也不能当作 Skill 已删除。 + +### 3.3 自动调度与反馈 + +广泛搜索默认每 7 天做一次去除增量时间窗口的全量搜索，以补找遗漏的仓库线索；仍遵守额度、处理预算与可恢复检查点，不代表整仓下载。 + +后台配置的是三类目标周期：已登记仓库更新默认约每 24 小时、精选用户或组织发现默认约每 24 小时、广泛搜索默认约每 72 小时；后台可按类别调整。预置搜索规则初始停用；管理员可在同一后台页面查看、修改和启用，明确添加的精选仓库可直接安排同步。官方探索独立启停，自定义规则各自启停；另外提供仅约束平台 `@github` 的安全暂停/恢复，不控制 TEAM、不重写独立入口开关，合同见 [统一管理](unified-admin-contract.md)；Java 只为已启用入口创建任务，Go 领取固定策略或规则快照执行。运行时结合 GitHub 额度、执行容量、审核积压与 namespace 公平性控制当轮工作量；已有来源更新和明确精选仓库优先，同时为官方探索的历史补搜和扩展查询保留有界的调度机会，避免永久饥饿。超出预算的候选留待续跑，不能静默丢弃。手动重扫会合并重复请求，也不能越过限流或审核反压。这些周期是目标时间，不保证在 GitHub 限流或审核积压时准点执行。 + +“积压”按 namespace、repository ID、Skill 路径统计当前最新且未解决的同步项，覆盖已接收、校验/扫描中、待审核和待管理员处理冲突；已批准、拒绝或被取代的历史项不计。全平台达到高水位时停止所有新包采集准入；只有某个 namespace 达到高水位时只暂停该范围。Go 在下载每个包前向 Java 申请准入，已开始的包可结算，下一个包不得继续。已启用的仓库搜索与精选用户或组织枚举仍可在额度及候选预算内继续，但只保存去重的仓库线索，不读取新候选仓库树或形成待审项；候选满时保存游标并延后。已登记仓库仍可低成本查看最新 commit，并在后台标记“发现更新、待采集”。积压回落到低水位后重新核对候选仓库与已登记来源的当前 revision，采集最新内容，不补齐每个中间 commit。 + +首期不使用历史审核结果自动调整各搜索规则份额。服务记录规则命中、有效 Skill、实际额度消耗和延后原因，供管理员判断规则效果及后续优化。 + +Go 服务停机或被关闭时，人工上传、市场浏览和既有审核继续可用。Java 合并同一来源的待执行任务，后台按兼容执行能力汇总实例联系状态，显示等待采集服务及最近联系时间；不能由单实例失联推断全部服务停机；恢复后按正常限流和审核反压续跑，不把未执行的轮次记为成功。 + +系统级 GitHub 凭证失效时，后台标记并提示，不再将其用于任务；可使用其他健康凭证，已配置凭证池仍无可用凭证时等待修复，不自动退回匿名；未配置任何凭证时可使用匿名额度。凭证失效与 GitHub 限流分别展示，限流时遵守 GitHub 返回的重试时间，不靠切换凭证掩盖限流。 + +## 4. Go 与 SkillHub 的产品边界 + +| 事项 | Go 服务 | SkillHub Java | +| --- | --- | --- | +| 来源/规则/凭证配置 | 领取当次所需的有效配置 | 保存、授权、加密、展示配置 | +| 调度与任务状态 | 主动领取、续租、回报和停止 | 创建任务、租约、检查点、重试与公平调度 | +| GitHub 访问 | 搜索、公开状态核对、固定 commit、树与包采集 | 核对结果与来源身份，不直接承担高负载采集 | +| 内容与对象 | 每个 Skill 上传一个临时包，回报完整性信息 | 签发短时上传票据，复核并写入正式版本存储 | +| Skill 业务 | 回报候选路径、采集状态与错误 | 忽略规则最终判断、冲突、内容哈希、扫描、审核和发布 | +| 管理反馈 | 回报进度、额度、截断、失败和可重试时间 | 在平台/TEAM 后台与独立审核队列呈现、审计 | + +Go 的输入与输出是**任务和采集事实**，不是“已经审核的 Skill”。Java 可以拒绝任何 Go 回报的包；Go 的成功只表示采集完成。 + +服务间只开放版本化 HTTP/JSON 内部任务合同，Go 使用独立服务身份调用 Java；GitHub Token 仅用于公开 GitHub 读取，不作为服务身份。GitHub 搜索与树枚举走 REST，包文件通过受限 Git 局部获取，固定 revision 不可取得时不混用新内容。 + +Go 上传并回报一个包后，Java 先持久记录摄入结果并确认接收，再由可恢复后台任务独立完成包校验、扫描与待审版本创建。因此后台需区分“已采集并交付”“处理中”“待同步审核”“处理失败”；不能把 Go 任务完成显示为已发布。Go 到 GitHub 的访问允许部署者统一配置出站 HTTP 代理，不提供逐来源代理设置。 + +## 5. 后台应看得到什么 + +Go 没有独立 UI；SkillHub 管理后台需要把 Go 的运行事实展示为以下四个问题： + +1. **目前在做什么**：任务类型、来源、固定 commit、开始时间、已处理/延后 Skill 数、当前阶段。 +2. **为什么没有继续**：GitHub 限流与 reset、审核积压、超出当轮预算、树不完整、包超限、来源不可达，均有明确原因和下次可运行时间。 +3. **能做什么**：按权限暂停/恢复来源或单个 Skill 路径、手动重扫、调整搜索条件与忽略路径；不可用操作说明限制原因。 +4. **采集后去了哪里**：来源详情逐 Skill 显示未变化、待扫描、待同步审核、已发布、失败或冲突，并链接到对应审核记录。 + +后台分平台管理视图和 TEAM 自身视图；TEAM 无法读取平台 Token 或其他 TEAM 的运行详情。同步审核与普通提交审核分开呈现。 + +## 6. 首期成功标准 + +- 配置一个精选公开仓库后，不依赖 CLI 或 GitHub Actions，Go 能通过 Java 任务接口发现其中多个 Skill，分别回报、上传和触发 Java 后续审核。 +- 上游未变化不产生冗余版本；上游变化不会在审核通过前替换已发布版本；一个包失败不阻断兄弟包。 +- GitHub 限流、部分树失败、Go 重启、重复回报和凭证缺失均有可观察状态与恢复路径；已发布 Skill 不因采集失败而消失。 +- 管理员能从后台解释每轮发现了什么、为何延后、下一次何时可运行、哪些包进入同步审核；Token 和 Skill 正文不出现在日志或任务历史。 +- 在本地文件存储及 S3/MinIO 模式下，Go 都无需正式对象存储管理凭证即可上传临时包；私有化部署只增加一个 Go 服务容器。 +- 先用少量公开仓库验证单 Skill、多 Skill、嵌套与较大仓库，再用受控样本逐级验证约 1000 个已登记仓库、10000 个 Skill 路径的容量目标；这是验收基线，不是产品硬上限，也不要求对 GitHub 公网做无控制压测。 +- 首次部署或升级后，预置搜索规则未启用时不产生广泛搜索任务；管理员在后台启用规则后生效，明确添加精选仓库无需等待规则启用。 +- 审核积压达到高水位时，后续包无法取得采集准入；已登记仓库只检查并记录最新 commit，已启用发现任务仅记录有预算的仓库线索。积压回落后重新固定当前 revision 处理，且另一未触发高水位的 TEAM 不被误暂停。 + +## 7. 后续候选 + +TEAM 自有公开 API 凭证与团队自动搜索、Gitee/私有仓库、MCP/DSH 来源、用户提交来源建议、外部目录种子，以及基于审核结果的搜索规则自动调权均不在首期。先保留来源与凭证的作用域扩展点，后续按独立需求评估。 diff --git a/openspec/changes/add-github-skill-sync/proposal.md b/openspec/changes/add-github-skill-sync/proposal.md new file mode 100644 index 000000000..ecbda6a7c --- /dev/null +++ b/openspec/changes/add-github-skill-sync/proposal.md @@ -0,0 +1,30 @@ +## Why + +SkillHub 开源版目前需要管理员逐个上传 Skill，无法从公开 GitHub 仓库持续发现和更新以 `SKILL.md` 为根的技能包。Issue #516 需要一套可私有化部署、可观察且受审核约束的服务端同步能力，让社区来源进入市场，同时保持 SkillHub 对版本、权限和发布的控制。 + +## What Changes + +- 在管理后台增加 GitHub 来源管理：平台管理员可配置结构化搜索规则、精选仓库及精选用户或组织；TEAM 管理员可为本团队登记公开仓库。自动发现的来源归不可删除的 `@github` 平台 namespace，TEAM 来源归自己的 namespace。 +- 增加独立 Go 采集服务，与 SkillHub Java 一起部署。Java 持有配置、凭证、任务、租约、来源账本、版本和审核；Go 轮询领取发现或仓库同步任务，访问公开 GitHub、固定 commit、枚举 `SKILL.md` 并逐 Skill 包上传临时采集对象，逐包回报结果。 +- 新 Skill 和内容变化通过现有安全扫描及独立的同步审核队列进入发布流程。同步内容保持上游只读；已发布版本、待审目标、路径冲突、上游不可达或目录消失都有明确且可审计的处理方式。 +- 管理后台展示规则、来源、运行进度、GitHub 额度、延后与失败原因；同步任务依据额度、执行容量、审核积压及 namespace 公平性控制节奏。管理员可手动触发或暂停，但手动操作不绕过限流和审核门禁。 +- 首期只处理 GitHub 公开仓库，支持匿名 API 或管理员配置的系统级公开读取凭证池。不纳入私有仓库、Gitee、CLI 导入、团队自有凭证池、团队自动搜索或自动审核通过；这些方向另行评估。 + +## Capabilities + +### New Capabilities + +- `github-source-management`：公开来源的登记、自动发现、归属、规则、权限及管理页面。 +- `github-sync-execution`：Java 与 Go 的任务协作、固定快照、增量更新、限流、断点续跑和可观察状态。 +- `github-sync-review`：同步内容的独立审核队列、版本更新、冲突处理及发布保护。 + +### Modified Capabilities + +无。当前主 OpenSpec 没有需要修改的已归档 capability；本变更通过新增能力复用现有 Skill 生命周期与审核实现。 + +## Impact + +- **后端与数据**：新增来源、规则、凭证引用、发现运行、仓库进度、逐 Skill 结果及租约记录；扩展 namespace 治理、包导入、扫描与审核编排。 +- **服务与部署**：新增单个 Go 服务容器，支持私有化部署；SkillHub Java 提供版本化的服务间任务接口和短时授权的临时对象上传入口。Go 不直连 SkillHub 数据库，也不具备发布权限。 +- **Web**：新增 GitHub 同步管理页、TEAM 范围内的来源管理视图、单独的同步审核列表及 GitHub 来源展示。 +- **安全与兼容**：公开 GitHub 仓库不自动获得 SkillHub 公开发布资格；保留上游许可证与 NOTICE，所有采集包由 Java 重新校验。现有人工上传、Skill 版本和普通审核路径继续可用。 diff --git a/openspec/changes/add-github-skill-sync/specs/github-source-management/spec.md b/openspec/changes/add-github-skill-sync/specs/github-source-management/spec.md new file mode 100644 index 000000000..2ce9d3ff3 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/specs/github-source-management/spec.md @@ -0,0 +1,241 @@ +## Purpose + +定义公开 GitHub 来源如何被发现、登记、归属和管理，使平台社区目录与团队来源拥有清晰的权限、可见性和持续同步边界。 + +## ADDED Requirements + +### Requirement: 平台 SHALL 提供官方探索与可配置的公开 GitHub 来源入口 + +官方深度探索 SHALL 使用仓库名称、描述、topic、README 线索与有效仓库作者的一层扩展，配合历史补搜和周期补漏，再检查候选仓库树中的 `SKILL.md`。自定义规则继续使用关键词和 topic 条件。首期默认策略 SHALL NOT 依赖全站代码搜索；代码搜索经独立对照试验确认增量有效召回、请求成本和限流恢复后，才可通过后续版本纳入。规则预览与运行反馈 SHALL 说明搜索索引与线索范围存在覆盖边界，不保证全站穷尽。 + +后台自定义搜索规则 SHALL 由可编辑的关键词与 topic 条件组成，每个条件区分核心或扩展优先级；管理员 SHALL NOT 被要求输入原始 GitHub 查询表达式。各条件独立搜索，候选按 GitHub repository ID 合并，核心条件优先使用发现预算；条件优先级 SHALL NOT 直接决定发布资格。规则变更 SHALL 生成新版本，已领取任务使用其固定快照与固定增量/全量搜索窗口，不因后台编辑改变本轮语义。 + +平台 SHALL 支持官方深度探索、结构化规则自动搜索、管理员指定精选仓库、管理员指定精选 GitHub 用户或组织。精选用户或组织 SHALL 持续检查其公开仓库；自动发现的仓库 SHALL 在确认存在有效 Skill 候选后，按 GitHub repository ID 去重并登记来源，无需仓库级人工确认。管理员明确指定的精选仓库 SHALL 可先登记为待首次扫描来源。发现分数 SHALL NOT 直接批准发布 Skill。 + +审核积压达到高水位期间，仓库搜索和精选用户或组织枚举的结果 SHALL 只成为有预算、可去重的待验证线索；此时不得因搜索命中创建正式来源。低水位恢复后 SHALL 重新检查仓库当前公开状态与 Skill 候选，再沿正常流程登记来源。 + +#### Scenario: 搜索发现有效公开仓库 +- **WHEN** 启用的搜索规则发现一个可公开读取且包含有效 `SKILL.md` 的仓库 +- **THEN** 系统自动登记或复用对应来源，并安排采集 +- **AND** 不因此发布任何 Skill + +#### Scenario: 多个搜索条件命中同一仓库 +- **WHEN** 核心关键词与扩展 topic 条件都命中同一 GitHub repository ID +- **THEN** 本轮只建立一个候选，保留两个条件的命中出处 +- **AND** 条件优先级只影响先后顺序，不直接批准或拒绝 Skill + +#### Scenario: 精选用户出现新仓库 +- **WHEN** 平台管理员配置的精选用户或组织新增公开仓库，且其中存在有效 Skill 候选 +- **THEN** 系统与搜索来源使用相同的仓库去重和采集流程 + +#### Scenario: 同一仓库被多个入口发现 +- **WHEN** 搜索规则、精选用户或精选仓库指向同一 GitHub repository ID +- **THEN** 同一目标 namespace 中只形成一条来源记录 +- **AND** 来源记录保留各发现入口的出处信息 + +### Requirement: 来源 SHALL 归属明确的 namespace + +平台自动发现和平台精选来源 SHALL 归不可由普通用户删除的 `@github` 平台 namespace，其审核通过的 Skill SHALL 全局可见。TEAM 管理员 SHALL 能为本团队登记公开 GitHub 仓库；TEAM 来源及 Skill SHALL 归该 TEAM namespace，不自动合并到 `@github`。同步 Skill 的业务归属 SHALL 是对应 namespace，不归配置管理员个人。同一仓库在不同 namespace 中 SHALL 保持独立的来源绑定与审核结果。 + +#### Scenario: 团队登记平台已采集的仓库 +- **WHEN** TEAM 管理员在本团队登记一个已存在于 `@github` 的公开仓库 +- **THEN** 系统在 TEAM 中建立独立来源绑定，且不复用平台的发布审核决定 + +#### Scenario: 非平台管理员修改平台发现规则 +- **WHEN** 无平台管理权限的用户尝试修改自动搜索规则、精选用户或组织 +- **THEN** 系统拒绝操作，并保持规则不变 + +#### Scenario: 配置管理员离开团队 +- **WHEN** 登记 TEAM 来源的管理员退出团队 +- **THEN** 来源及已同步 Skill 仍归 TEAM namespace，并由现有 TEAM 管理角色继续管理 + +### Requirement: TEAM 来源 SHALL 使用来源级可见性 + +TEAM 来源 SHALL 有一个适用现有 SkillHub 可见范围的来源级配置，默认 `NAMESPACE_ONLY`。来源可见范围的修改 SHALL 只作用于之后导入的 Skill 或版本，不得静默改变已发布 Skill 的可见性。GitHub 仓库公开状态 SHALL NOT 直接决定 SkillHub 可见范围。 + +#### Scenario: 团队来源首次导入 +- **WHEN** TEAM 管理员登记公开仓库但未设置来源可见范围 +- **THEN** 后续新 Skill 默认使用 `NAMESPACE_ONLY` + +#### Scenario: 修改来源可见范围 +- **WHEN** TEAM 管理员修改来源可见范围 +- **THEN** 系统明确说明该设置仅影响后续导入 +- **AND** 已发布 Skill 的可见范围保持不变 + +### Requirement: 来源与发现规则 SHALL 分别管理生命周期 + +规则变更或停用 SHALL 只影响未来发现，不自动停用已登记来源。来源 SHALL 可暂停同步；单个 Skill 路径 SHALL 能通过来源忽略路径规则停止后续采集，移除该忽略项后可恢复，未单独排除的路径继承来源状态；停用或移除来源 SHALL 停止后续同步，并保留已经创建的 Skill、版本、来源出处与运行历史。已经停用的来源 SHALL NOT 因再次被搜索命中而自动重新启用。 + +#### Scenario: 停用发现规则 +- **WHEN** 管理员停用一个搜索规则 +- **THEN** 该规则停止产生新来源，而此前登记的来源仍按自身状态同步 + +#### Scenario: 停用来源后再次命中 +- **WHEN** 已停用来源对应的仓库再次被任一发现入口命中 +- **THEN** 系统保留停用状态，不自动恢复同步或改变已发布 Skill + +#### Scenario: 只暂停一个 Skill 路径 +- **WHEN** 管理员暂停仓库中的某个 Skill 路径 +- **THEN** 该路径停止后续同步，同仓库其他路径继续处理 +- **AND** 已发布版本保持不变 + +### Requirement: 后台 SHALL 提供统一配置和反馈入口 + +平台后台 SHALL 在统一的 GitHub 同步管理区域呈现规则预览、启停、精选来源、仓库来源、运行结果、延后与失败原因。TEAM 管理员 SHALL 只能查看和操作有权限的 TEAM 来源。规则界面 SHALL 使用结构化条件，不要求管理员编写 GitHub 搜索表达式。自定义广泛搜索 SHALL 只执行后台已启用的规则及目标周期。官方深度探索 SHALL 作为内置方式提供独立启停，首次默认停用；其开关 SHALL 仅控制该探索途径，不作为所有 GitHub 同步的全局开关。关闭探索 SHALL 不暂停已登记有效仓库的更新，也不影响精选仓库及独立自定义规则。官方探索 SHALL 仅向管理员提供启停、资源预算档位与排除项，内部查询集合、渠道搭配、分页和补漏节奏由系统维护；具体范围参见 [深度探索需求澄清](../../deep-discovery.md)。首期预置规则初始停用，管理员可在同一配置区域查看、编辑和启用；明确添加精选仓库 SHALL 不受预置搜索规则启用状态影响。 + +平台运行详情 SHALL 按运行 ID 展示该轮任务及逐 Skill 路径结果，并分别分页；发现运行可以没有 Skill 路径。逐路径结果 SHALL 关联当轮任务、仓库、采集结果及对应摄入/审核状态，不得用来源的最新状态冒充历史运行结果。已按保留策略清理的运行详情不再作为可查看运行返回。 + +#### Scenario: 平台启用官方深度探索 +- **WHEN** 平台管理员首次启用内置深度探索 +- **THEN** 系统以公共 `@github` namespace 为目标持续发现公开仓库 +- **AND** 首期 TEAM 不提供独立官方探索配置，继续使用既有来源管理能力 + +#### Scenario: 停用和恢复官方深度探索 +- **WHEN** 管理员关闭官方深度探索 +- **THEN** 系统保留候选及检查点，暂停该途径尚未登记为正式来源的候选后续采集，在途操作到安全检查点暂停 +- **AND** 已登记来源继续按自身配置更新，其他独立启用入口仍需要的候选不受影响 +- **WHEN** 管理员重新开启官方深度探索 +- **THEN** 系统从保留的进度继续处理，不清空候选重新搜索 + +#### Scenario: 排除探索仓库或作者 +- **WHEN** 管理员在官方探索排除项中配置仓库或作者 +- **THEN** 系统排除该探索途径对其新仓库的发现，不据此暂停已登记来源 +- **AND** 对命中的未登记候选暂停后续采集并保留记录和进度，取消排除后可继续；其他独立启用入口仍需要的候选不受影响 +- **AND** 来源暂停与 Skill 目录忽略仍使用现有独立配置 + +#### Scenario: 调整官方探索预算档位 +- **WHEN** 管理员配置官方探索资源预算 +- **THEN** 系统提供节省、均衡、加速三个档位，默认均衡 +- **AND** 档位只调整执行速度和资源分配，不删减探索渠道或覆盖范围 +- **AND** 加速仍受部署额度、候选容量、采集准入与既有来源更新的资源保护约束 + +#### Scenario: 官方策略随产品升级 +- **WHEN** SkillHub 升级引入新版官方探索策略 +- **THEN** 系统自动采用新策略，保留管理员启停、预算与排除配置 +- **AND** 新增查询自动安排历史补搜并尽量复用兼容进度，停用状态不被升级恢复 +- **AND** 在途任务保持固定策略版本，不中途混用新旧策略 + +#### Scenario: 查看官方深度探索 +- **WHEN** 平台管理员进入官方深度探索 +- **THEN** 系统在统一技能同步后台集中展示配置及结果，与既有来源和运行详情保持关联；旧独立探索页面重定向到采集配置抽屉 +- **AND** 主页面通过总览／仓库／运行记录切换内容；启停、预算与排除项在采集配置抽屉集中编辑，探索进度按需展开 +- **AND** 仓库数、技能数、近期新增与当前待审数量明确区分，不用搜索命中数代替导入成果 + +#### Scenario: 分页查看发现仓库 +- **WHEN** 管理员打开发现仓库列表 +- **THEN** 默认包含所有状态的可查询结果，按最近发现排序，每页 20 条 +- **AND** 搜索、状态和渠道筛选及分页在服务端完成，不向浏览器返回完整候选集合进行筛选 +- **AND** 列表只加载摘要，详情按需加载；后台新增结果不自动跳页或丢失筛选与选中项 + +#### Scenario: 查看探索仓库详情 +- **WHEN** 管理员点击探索仓库 +- **THEN** 系统按需加载详情抽屉，展示命中渠道、检查结果、未导入原因及后续安排 +- **AND** 存在关联来源或审核记录时提供相应入口，关闭抽屉保留列表筛选与位置 + +#### Scenario: 展示探索覆盖进度 +- **WHEN** 管理员查看探索进展 +- **THEN** 系统展示渠道工作、已完成查询分片、待处理工作与暂停原因，查询和分页细节进入运行详情 +- **AND** 总量未知时不显示完成百分比，不将当前计划范围完成描述为 GitHub 全量覆盖 + +#### Scenario: 区分近期成果与当前积压 +- **WHEN** 管理员切换近期成果时间范围 +- **THEN** 新增候选、首次确认有效仓库及新采集技能按所选时间统计，同仓库多渠道命中去重，更新版本单列 +- **AND** 当前待检查仓库与待审版本独立展示，不受该时间筛选影响 + +#### Scenario: 反馈探索等待原因 +- **WHEN** 探索因额度或容量限制延后 +- **THEN** 系统展示原因、受影响工作、仍可继续的工作及相关配置或审核入口 +- **AND** 有可靠依据才显示预计可重试时间，否则展示恢复条件；满足条件后自动续跑 + +#### Scenario: 采集服务不可用时等待和接管 +- **WHEN** 官方探索已启用但无可用 Go 执行实例 +- **THEN** Java 保留任务与已确认检查点，页面显示等待采集服务恢复，不将到期重试时间描述为实际执行 +- **AND** 心跳超时只表示联系中断，单实例离线不代表全部实例离线 +- **WHEN** Go 恢复或其他实例领取过期租约任务 +- **THEN** 从已保存进度继续；未确认部分可重放，旧租约写入拒绝，业务接收幂等 + +#### Scenario: 预览规则 +- **WHEN** 平台管理员预览尚未启用或已编辑的搜索规则 +- **THEN** 后台展示可解释的预估命中结果和规则条件 +- **AND** 预览本身不登记来源或创建待审 Skill + +#### Scenario: 采集服务离线时打开规则预览 +- **WHEN** 平台后台检测到 Go 采集服务离线或无法确认状态 +- **THEN** 规则预览操作不可提交，并明确说明服务状态 +- **AND** 已排队、执行中、延后重试和失败的预览分别显示对应状态，不将排队误称为正在搜索 GitHub +- **AND** 管理员仍可将新规则保存为停用状态 + +#### Scenario: 后台启用预置规则 +- **WHEN** 管理员在 GitHub 同步管理区域启用一条预置搜索规则 +- **THEN** Java 依据该规则和配置的目标周期生成发现任务，Go 后续领取执行 +- **AND** 无需再启用官方探索；执行仍受平台采集安全暂停、额度及容量控制 + +#### Scenario: 首次部署或升级尚未启用规则 +- **WHEN** 系统已预置搜索规则且官方探索与所有自定义搜索规则均未启用 +- **THEN** 不产生广泛搜索任务 +- **AND** 管理员明确添加的精选公开仓库仍可进入同步流程 + +#### Scenario: 查看来源失败 +- **WHEN** 来源采集失败或被延期 +- **THEN** 有权限的管理员能看到当前状态、最近尝试、原因和下一次可运行时间 + +### Requirement: 市场 SHALL 展示可公开的 GitHub 来源 + +已发布同步 Skill 的详情 SHALL 展示来源仓库、Skill 路径、固定 commit 与可公开的许可证信息。无权访问 TEAM Skill 的用户 SHALL 不能通过来源展示接口获得其仓库绑定或运行状态。 + +市场 SHALL 提供单独的公开 GitHub 技能入口，仅检索已发布且当前用户可见的 `@github` Skill，并支持站内搜索、排序和分页；展示数量 SHALL 来自实际市场查询，不得把发现候选、待审包或仓库线索计入已发布 Skill 数量。 + +该入口的人类可见名称 SHALL 为“开源技能”，并明确标注首期内容仅来自 GitHub 公开仓库；现有 `/github` 路由和 `@github` 过滤范围保持不变，不暗示已接入其他代码托管平台。 + +#### Scenario: 查看平台同步 Skill +- **WHEN** 用户查看一个已发布的 `@github` Skill +- **THEN** 页面展示可核对的 GitHub 来源与许可证信息 + +#### Scenario: 无权访问团队来源 +- **WHEN** 无 TEAM 权限的用户请求仅团队可见 Skill 的来源详情 +- **THEN** 系统沿用现有 Skill 可见性规则拒绝或过滤该信息 + +### Requirement: 凭证配置 SHALL 只服务于公开读取 + +平台管理员 SHALL 可配置零个或多个系统级公开 GitHub API 凭证；无凭证时公开仓库同步 SHALL 仍可在匿名额度内运行。认证失效的凭证 SHALL 标记不可用并在后台提示，系统 SHALL 可选用另一健康系统凭证；已配置凭证池但无健康可用凭证时 SHALL 延后任务并提示修复，不自动降级为匿名；仅未配置任何凭证时使用匿名额度。凭证认证失效、仓库访问失败和 GitHub 限流 SHALL 分别展示。系统 SHALL 不接受本期来源的个人 OAuth 或私有仓库凭证；凭证值 SHALL 不出现在普通页面响应、任务历史、日志或指标中。 + +#### Scenario: 未配置凭证 +- **WHEN** 系统未配置 GitHub 凭证 +- **THEN** 公开来源发现与同步可使用匿名访问，并按实际额度延后任务 + +#### Scenario: 系统凭证失效 +- **WHEN** GitHub 确认当前系统凭证认证失效 +- **THEN** 系统不再把该凭证分配给任务，并在后台提示管理员更新 +- **AND** 后续公开读取可使用其他健康系统凭证；若池内均不可用（含全部被停用），则等待凭证修复，不自动切换为匿名 + +#### Scenario: 凭证达到额度限制 +- **WHEN** GitHub 对某凭证返回限流或重试时间，而非认证失效 +- **THEN** 系统保留该凭证的健康状态并按限流规则延后任务 +- **AND** 不通过连续切换凭证规避限流 + +#### Scenario: 仓库变为私有 +- **WHEN** 已登记公开仓库不再可公开读取 +- **THEN** 系统将来源标记为访问失败并进入恢复策略 +- **AND** 不使用其他凭证尝试私有访问 + + +### Requirement: 平台 SHALL 提供单一技能同步入口和统一仓库列表 + +平台管理员 SHALL 通过「技能同步」的概览、仓库、运行记录三个页签查看工作，采集配置与系统设置 SHALL 按需打开。自动探索 SHALL 是采集方式之一，指定仓库 SHALL 表达管理员指定而非质量认证。仓库列表 SHALL 覆盖平台适用的自动探索、指定仓库、指定账号和自定义搜索规则结果，以namespace和稳定repository ID去重，保留多出处，通过服务端筛选及分页，不在浏览器拼接独立分页结果。 + +首次准备 SHALL 根据真实配置和服务状态提示缺项，凭证可选，不强制向导或擅自启用探索。添加指定仓库 SHALL 只要求仓库URL或owner/repo，固定平台@github，沿用已配置的公共策略；重复添加 SHALL 复用仓库记录而不改变既有暂停意图。运行列表 SHALL 展示真实运行及其类型、目标、状态和结果，不虚构全站探索轮次。概览 SHALL 区分需要人工处理和可自动恢复的等待。 + +#### Scenario: 同一仓库由多个入口发现 +- **WHEN** 平台自动探索和指定仓库指向同一个稳定repository ID +- **THEN** 统一仓库列表仅显示一行，保留两种出处并返回准确分页总数 +- **AND** TEAM相同仓库不合并到平台仓库行 + +#### Scenario: 首次配置和旧链接兼容 +- **WHEN** 管理员打开未配置完成的技能同步页面 +- **THEN** 准备清单显示真实缺项并可定位配置，不把无凭证直接判为错误 +- **AND** 既有sourceId/runId深链与旧探索入口继续可达对应内容 + +#### Scenario: 已存在的指定仓库在平台暂停时再次添加 +- **WHEN** 管理员提交已经存在的仓库地址且平台采集已暂停 +- **THEN** 系统复用仓库、保留出处和已有启停状态，并显示等待平台采集恢复 +- **AND** 不产生重复采集来源或绕过暂停 diff --git a/openspec/changes/add-github-skill-sync/specs/github-sync-execution/spec.md b/openspec/changes/add-github-skill-sync/specs/github-sync-execution/spec.md new file mode 100644 index 000000000..cdbd9309d --- /dev/null +++ b/openspec/changes/add-github-skill-sync/specs/github-sync-execution/spec.md @@ -0,0 +1,217 @@ +## Purpose + +定义 GitHub 来源发现与仓库同步的可观察执行契约，使独立采集服务在限流、故障、大仓库和多实例条件下仍能安全地续跑与审计。 + +## ADDED Requirements + +### Requirement: 同步 SHALL 使用持久任务和可恢复领取 + +SkillHub SHALL 持久保存发现与仓库同步任务、运行历史、租约、重试时间和结果。采集执行端 SHALL 主动领取任务并回报进度；重复领取、超时重试或结果重放 SHALL NOT 创建重复的来源、版本或审核记录。单个仓库同步任务 SHALL 可独立记录每个 Skill 包的结果，不要求为每个包建立任务。 + +#### Scenario: 执行端中途退出 +- **WHEN** 执行端领取仓库任务后退出且租约到期 +- **THEN** 其他执行端可从已持久化进度继续处理 +- **AND** 已成功回报的 Skill 包不产生重复版本 + +#### Scenario: 单个 Skill 包采集失败 +- **WHEN** 同一仓库的一个 Skill 包失败、其他包成功 +- **THEN** 成功包继续进入后续处理，失败包有独立原因与重试状态 +- **AND** 后续重试只重新采集需要重试的路径及其固定 revision + +### Requirement: 采集服务离线 SHALL 不影响现有 SkillHub 业务 + +采集服务可单独关闭或重启。服务离线时，SkillHub SHALL 继续提供人工上传、市场浏览和既有审核；同一来源的到期同步工作 SHALL 合并而不是每轮重复堆积。管理后台 SHALL 显示采集暂停与最近在线时间；恢复后未执行工作 SHALL 按正常额度和审核反压续跑，不得记为已成功同步。 + +#### Scenario: Go 服务停机跨越多个调度周期 +- **WHEN** 采集服务离线期间同一来源多次到达目标同步时间 +- **THEN** 该来源保留一份待执行工作和可观察的延后状态 +- **AND** 人工上传与市场浏览继续可用，恢复后不会伪造已完成轮次 + +### Requirement: 内部任务接口 SHALL 只接受采集服务身份 + +领取、续租、结果回报、完成及临时对象授权接口 SHALL 验证独立采集服务身份，不得把普通用户 Token 或 GitHub API 凭证当作服务身份。协议版本不兼容或身份无效时 SHALL 拒绝请求，不得泄露任务中的 GitHub 凭证或临时对象授权。 + +#### Scenario: 普通用户访问内部领取接口 +- **WHEN** 请求携带普通用户 Token 或无效服务密钥 +- **THEN** 系统拒绝领取，且不返回任务内容或 GitHub 凭证 + +### Requirement: 每次仓库扫描 SHALL 固定可追溯的 revision + +系统 SHALL 按仓库默认分支发现最新 revision，记录本次使用的 commit SHA，并从该固定 revision 获取树和技能包内容。仓库改名或转移 SHALL 优先用 GitHub repository ID 保持来源身份。每个 Skill 来源身份 SHALL 由 repository ID 与 Skill 路径组成，内容哈希只用于比较内容或辅助唯一的路径移动判断。 + +#### Scenario: 扫描中默认分支更新 +- **WHEN** 仓库默认分支在一次扫描过程中更新 +- **THEN** 本次各 Skill 包仍来自扫描开始时固定的同一 commit + +#### Scenario: 仓库改名 +- **WHEN** GitHub repository ID 不变而 `owner/repo` 变化 +- **THEN** 系统更新展示名称和 URL，保留原来源与 Skill 绑定 + +### Requirement: 仓库 SHALL 递归发现独立 Skill 包 + +系统 SHALL 递归枚举 `SKILL.md`，以其所在目录为技能包根目录；嵌套 `SKILL.md` 所在子树 SHALL 属于独立技能包，不并入父包。平台默认忽略候选路径中目录段为 `.git/`、`node_modules/`、`.venv/`、`venv/`、`dist/`、`build/`、`target/`、`__pycache__/`、`test/`、`tests/` 的 `SKILL.md`；`examples/` 和 `docs/` 不默认忽略。管理员可追加来源级忽略路径，且不得放宽平台安全规则。忽略候选目录 SHALL NOT 删除父 Skill 包中合法的测试文件。 + +#### Scenario: 父子 Skill 包 +- **WHEN** `skills/parent/SKILL.md` 与 `skills/parent/child/SKILL.md` 同时存在 +- **THEN** 系统识别两个独立 Skill 包 +- **AND** 父包的内容指纹和上传内容不包含子包目录 + +#### Scenario: 测试目录中的 Skill 候选 +- **WHEN** `tests/sample/SKILL.md` 位于默认忽略目录 +- **THEN** 该路径不成为新 Skill 候选 +- **AND** 其他 Skill 包内部的普通测试文件仍可保留 + +### Requirement: 扫描不完整 SHALL 与上游删除区分 + +仓库树响应截断时，系统 SHALL 尝试按子目录继续枚举。无法完整枚举的子树 SHALL 标记为不完整并安排重试；已完整发现的 Skill SHALL 继续处理。系统 SHALL NOT 根据未枚举到的路径推断上游删除或停用本地 Skill。 + +#### Scenario: 部分子树不可枚举 +- **WHEN** 仓库中一个子树反复枚举失败，其他子树已完整枚举 +- **THEN** 已完整枚举的 Skill 继续采集 +- **AND** 失败子树被标记为不完整并重试，不触发删除判断 + +### Requirement: 单个技能包 SHALL 在资源边界内完整采集 + +系统 SHALL 在固定 commit 上仅取得需要的技能包文件到受限本地临时目录，不要求下载整个仓库；每个包 SHALL 使用当前有效的发布文件数、单文件大小和包总大小政策，当前 `application.yml` 分别配置为 100 个、10 MiB 和 100 MiB，Go 预检而 Java 复核。超过上限、包含不支持的符号链接或 Git submodule 的包 SHALL 作为该包失败，不得产生部分版本；其他包 SHALL 继续处理。 + +Go SHALL 先根据固定 commit 的仓库树元数据确定候选路径、文件类型和已知大小，完成候选忽略与包上限预检后才获取所需文件内容。每仓库工作目录、进程总临时磁盘、最低剩余空间与并发目录数 SHALL 有部署可验证的资源边界，计入 Git 对象、检出文件和归档；逐包上传并得到持久接收确认后 SHALL 清理可释放的本地文件。某仓库达到磁盘预算时 SHALL 保存已处理路径进度、释放工作目录并从相同固定 commit 分批续跑；资源不足 SHALL 可观察，不得回退为无界整仓下载，也不得据此推断上游 Skill 删除。 + +Go SHALL 在获取文件前预留所需 Git 元数据、包文件、归档和安全余量，并在各阶段检查工作目录占用与剩余空间。若当前工作区连固定 commit 的基础元数据或一个符合业务上限的包都无法容纳，系统 SHALL 把该来源标为容量阻塞并等待容量或规则变化，不得在相同条件下无限立即重试；容量阻塞不得计为业务包超限或完整扫描。部署侧 SHALL 验证临时工作区写满时不会占用关键服务存储。 + +#### Scenario: 某包超过上限 +- **WHEN** 一个仓库内的某个 Skill 包超过平台包总大小上限 +- **THEN** 系统记录该包失败原因，不导入不完整包 +- **AND** 同仓库内未超限的包继续采集、校验并分别进入同步审核 +- **AND** 任务因该包终态失败标记为 `PARTIAL` 时，不得仅凭任务状态阻断已接收的兄弟包 +- **AND** 只有确认既有路径也在本次结果中时，才可在该部分结果下把新路径作为独立 Skill；不得据此自动推断旧路径消失或移动 + +#### Scenario: 巨型仓库的 Git 元数据逼近工作目录预算 +- **WHEN** 已定位到多个 Skill 候选，但固定 commit 的 Git 元数据与已获取包使 Go 临时目录接近预算 +- **THEN** Go 停止继续下载，持久回报已完成包及剩余路径，清理本批目录后从同一 commit 续跑 +- **AND** 管理后台显示容量延后，不将未处理路径视为删除或来源不可达 + +#### Scenario: 一个合规包也无法装入临时工作区 +- **WHEN** 候选包符合发布限制，但预留落地文件、归档和 Git 元数据后的空间不足 +- **THEN** 系统不下载该包，记录容量阻塞及当前预算，并释放工作目录 +- **AND** 在容量或规则未改变前不立即重复领取同一工作，不把该包记录为业务超限 + +### Requirement: 采集对象 SHALL 临时隔离并由 SkillHub 复核 + +采集执行端 SHALL 在本地逐 Skill 打包固定 revision 的文件，分别上传到隔离的临时对象存储，并向 SkillHub 回报路径、revision、归档大小/摘要和 `uploadId`。SkillHub SHALL 通过自身存储适配器解析并读取临时对象，不要求 Go 回传预签名下载 URL 或任意外部链接；SkillHub SHALL 独立校验对象、解析包、执行忽略与安全规则、计算正式内容哈希，再决定是否创建版本。Go 在上传前遵守包与归档大小边界；S3/MinIO 的预签名 PUT 上传后仍由 Java 复核实际对象大小、拒绝并清理超额对象，临时存储 SHALL 有隔离、容量告警与清理；本地流式入口接收时执行硬上限。已被接收且仍待处理的对象 SHALL 保留到消费或明确失败，不能仅凭固定 TTL 清理。待审与已发布版本内容 SHALL 按正式版本生命周期持久保存。 + +逐包回报的接收确认 SHALL 表示 SkillHub 已持久记录该包的摄入请求，不表示包已通过校验、审核或发布。SkillHub SHALL 可恢复地逐包推进后续处理，摄入失败 SHALL 与同仓库其他包隔离；采集任务完成与内容审核状态 SHALL 分开显示。 + +#### Scenario: 上传结果被篡改或不完整 +- **WHEN** 回报的对象与完整性信息不匹配，或包含越界路径 +- **THEN** SkillHub 拒绝导入该包并记录失败 +- **AND** 不创建 Skill、版本或审核任务 + +#### Scenario: 内容未变化 +- **WHEN** 固定 revision 中某个 Skill 包的正式内容哈希等于已知目标内容 +- **THEN** 系统更新同步检查记录，但不创建冗余版本或审核任务 + +#### Scenario: 接收后 Java 处理进程退出 +- **WHEN** SkillHub 已确认接收逐包回报，但尚未完成暂存包校验时进程退出 +- **THEN** 后台处理从持久摄入记录恢复，或留下可重试的明确失败状态 +- **AND** 暂存对象仍有效时 Go 不需因 Java 后续处理耗时而重新上传；对象失效则记录明确原因并安排重采集 + +#### Scenario: Go 回报一个临时包 +- **WHEN** Go 在固定 commit 上取得所需文件、逐包打包并上传隔离临时对象 +- **THEN** 回报仅引用 Java 签发的 `uploadId` 与大小、摘要、路径、commit +- **AND** Java 用自身存储适配器读取并复核对象，不接受 Go 提供的任意下载链接 + +### Requirement: 同步 SHALL 尊重额度、积压和公平性 + +系统 SHALL 按已登记来源更新、精选仓库、精选用户或组织发现、广泛搜索的顺序分配受限执行容量，同时避免 TEAM 来源长期饥饿。已登记仓库更新与精选用户或组织发现的默认目标周期 SHALL 约为 24 小时，广泛搜索 SHALL 约为 72 小时；平台管理员 SHALL 可在后台按类别调整目标周期。广泛搜索 SHALL 默认每 7 天执行一次去除增量时间限制的全量搜索，受同一额度、预算和可恢复检查点约束；全量搜索不等于整仓下载。目标周期 SHALL NOT 被解释为在 GitHub 限流或积压下的准点执行保证。反压 SHALL 统计同一 `(namespace, repository ID, Skill 路径)` 当前最新的未解决同步项，包括已接收、校验/扫描中、待审核及待管理员处理冲突；已批准、拒绝和被新目标取代的历史项不计入。GitHub 限流与按 namespace、全平台分别计算的高低水位 SHALL 使任务延后或暂停，且在恢复条件满足后续跑；手动触发 SHALL NOT 绕过这些边界，重复触发 SHALL 合并待执行工作。每轮发现的动态预算耗尽时，未处理候选 SHALL 保留可恢复进度和延后原因。 + +全平台未解决同步项达到高水位时，所有 namespace 的**新包采集准入** SHALL 暂停；仅某个 namespace 达到高水位时，只暂停该 namespace 的新包采集。Go 在下载每个新包文件前 SHALL 向 Java 申请绑定当前租约、固定 commit 与 Skill 路径的准入；Java SHALL 原子计入未完成准入占位并在水位满足时拒绝新准入，避免并发任务越过水位。已获准且正在处理的包 MAY 完成或失败并释放占位；已领取的仓库任务 SHALL 在下一个包前停止采集并回报可恢复路径进度，不得继续下载兄弟包。新准入在积压回落至低水位后恢复。 + +高水位期间，已启用规则的仓库搜索与精选用户或组织公开仓库枚举 SHALL 在 GitHub 额度和有界候选预算允许时继续，结果 SHALL 只保存为按 repository ID 去重的待验证仓库线索与发现检查点；不得因此读取新候选仓库树、下载包、建立已验证来源或创建待审目标。每次领取的新增线索 SHALL 受任务快照的 `candidateBudget` 限制，namespace 中排队或正在验证的候选总数 SHALL 受管理策略的 `candidatePendingLimit` 限制；任一预算耗尽 SHALL 保存游标并延后，不能无限积累线索。已登记仓库 SHALL 在 GitHub 额度允许时继续低成本检查默认分支最新 commit 并记录“发现更新、待采集”，但不得因此创建新待审项或把已完整同步 revision 向前推进。积压回落至相应低水位后 SHALL 重新核对待验证仓库与已登记来源的当前上游 revision，再处理最新内容；其他未达到阈值的 namespace 仍可按其额度继续。 + +已完成的同步运行与任务详情 SHALL 默认保留 90 天；未完成任务及恢复检查点不得按此期限清理，来源、Skill 版本和审核审计遵守原生命周期。容器标准输出日志的保留由部署环境管理，不能把通知或扫描任务的既有清理期限当作同步记录保留期。 + +#### Scenario: GitHub 返回限流 +- **WHEN** GitHub 指示额度耗尽或要求稍后重试 +- **THEN** 系统记录对应可重试时间和受影响任务，不要求执行端长期占用租约等待 +- **AND** 管理后台可查看限流原因及下一次可运行时间 + +#### Scenario: 管理员调整类别目标周期 +- **WHEN** 管理员在后台调整已登记仓库、精选用户或组织、广泛搜索中某一类的目标周期 +- **THEN** Java 按该类别新周期生成或合并后续任务，Go 仍只领取已生成的任务 +- **AND** 已延后的任务继续受实际额度与审核积压约束 + +#### Scenario: 搜索超出当轮预算 +- **WHEN** 搜索发现的候选超过当轮可处理预算 +- **THEN** 系统只处理预算内候选，并保留其余候选的续跑位置或等价检查点 +- **AND** 管理后台显示纳入与延后数量 + +#### Scenario: 七天全量搜索被限流 +- **WHEN** 到期的全量搜索因 GitHub 额度或当轮预算不足未能完成 +- **THEN** 系统保留规则与分页检查点，额度恢复后续跑，不把本轮记为成功 +- **AND** 不要求 Go 下载所有候选仓库的完整内容 + +#### Scenario: 重复手动重扫 +- **WHEN** 管理员在同一来源已有待执行重扫任务时再次触发重扫 +- **THEN** 系统合并请求，并且仍遵守 GitHub 限流及审核积压阈值 + +#### Scenario: 完整枚举后个别 Skill 确定性失败 +- **WHEN** 仓库的固定提交已完整枚举，但个别 Skill 因包大小等确定性限制而使任务成为 `PARTIAL` +- **THEN** 周期检查仅查询仓库 HEAD，不因未取得完全处理标记而反复下载同一提交的已成功 Skill +- **AND** 上游提交变化、忽略规则变化、显式重扫或路径重试仍可触发受额度与积压限制的完整采集 + +#### Scenario: 某 TEAM 未解决同步项达到高水位 +- **WHEN** 该 TEAM 最新且未解决的同步项达到高水位，而另一 TEAM 未达到 +- **THEN** 前者的仓库可记录最新 commit，但不采集新包或创建待审项 +- **AND** 后者仍可按自身额度和公平调度继续采集 + +#### Scenario: 全平台积压达到高水位后仍有发现任务 +- **WHEN** 已启用规则在全平台高水位期间搜索到新的公开仓库 +- **THEN** 系统在当轮候选预算内去重记录仓库线索与游标，不读取该仓库树或下载 Skill 包 +- **AND** 候选预算耗尽时延后剩余搜索结果，不建立已验证来源或待审目标 + +#### Scenario: 已领取仓库任务遇到高水位 +- **WHEN** 某包已获准开始采集，随后积压达到高水位，同一仓库还有未处理的 Skill 路径 +- **THEN** 已获准包可完成或失败并释放准入占位；Java 拒绝新路径的采集准入 +- **AND** Go 保存后续路径进度并停止下载，积压回落到低水位后重新核对当前上游 revision + +#### Scenario: 已完成运行记录到期 +- **WHEN** 一条已完成的同步运行详情超过默认 90 天保留期，而另一任务尚待重试 +- **THEN** 已完成详情可按保留政策清理，未完成任务及恢复检查点仍保留 +- **AND** 来源、Skill 版本和审核审计不按运行详情期限清理 + +#### Scenario: 积压回落后上游已再次变化 +- **WHEN** 待采集来源在反压期间多次更新且待审积压回落至低水位 +- **THEN** 系统重新固定当前上游 commit 并采集最新内容 +- **AND** 不为每个未采集的中间 commit 补建待审项 + +### Requirement: 来源不可达 SHALL 有持续恢复路径 + +来源首次不可读后 SHALL 于约 24 小时复查，再失败则约 3 天后复查；该次仍失败后 SHALL 标记为“不可达”，此后约每 7 天继续复查。限流和短暂执行错误 SHALL 与来源不可达区分。来源恢复后 SHALL 继续对账；不可达期间不得自动撤回或删除已发布 Skill。 + +#### Scenario: 长期不可达后恢复 +- **WHEN** 已标记不可达的公开仓库再次可读 +- **THEN** 系统恢复来源同步并重新对账 +- **AND** 已发布 Skill 在整个不可达期间保持原发布状态 + + +### Requirement: 平台采集 SHALL 支持持久安全暂停与恢复 + +超级管理员 SHALL 能暂停和恢复平台@github的新发现、检查和下载。该控制 SHALL 不暂停TEAM任务，不重写各发现入口和来源的独立启停配置。控制状态 SHALL 持久保存；在途任务 SHALL 到安全检查点保存进度并停止，不能等待整个大仓库完成才响应。已上传并被Java接收的包 SHALL 继续校验、扫描及进入待审，人工审核及已发布技能使用不受影响。 + +界面 SHALL 区分暂停中与已暂停，基于任务停止的可靠证据显示状态；不得仅因配置写入成功或Go心跳过期便声称在途工作已停止。恢复 SHALL 继续可恢复工作，不自动启用原本独立停用的入口或来源。暂停及恢复权限、审计、任务领取和租约边界 SHALL 由服务端保证，不通过前端批量修改来源代替。 + +#### Scenario: 平台暂停但TEAM继续 +- **WHEN** 超级管理员暂停平台采集且存在平台与TEAM的待执行任务 +- **THEN** 新平台任务不能领取，TEAM任务仍按原有资格和资源规则领取 +- **AND** 已接收的平台包仍继续处理 + +#### Scenario: 暂停在途任务后恢复 +- **WHEN** 正在处理大仓库或分页发现的worker观察到暂停 +- **THEN** 它在安全检查点保存已确认进度并停止后续采集 +- **WHEN** 管理员恢复平台采集 +- **THEN** 可执行的工作从持久进度继续，已接收包不重复摄入，独立停用的来源不会被启用 + +#### Scenario: 暂停与领取竞争或worker离线 +- **WHEN** 暂停与新领取并发，或暂停期间worker失联 +- **THEN** 按服务端持久控制和租约规则阻止暂停生效后的新平台领取，并防止过期worker写入新进度 +- **AND** 未获得停止证据时不虚报已暂停 diff --git a/openspec/changes/add-github-skill-sync/specs/github-sync-review/spec.md b/openspec/changes/add-github-skill-sync/specs/github-sync-review/spec.md new file mode 100644 index 000000000..1de617669 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/specs/github-sync-review/spec.md @@ -0,0 +1,97 @@ +## Purpose + +定义 GitHub 同步内容如何进入 SkillHub 的版本、扫描、审核和发布流程，使上游变化可追溯，同时不冲击普通用户提交的审核队列。 + +## ADDED Requirements + +### Requirement: 同步内容 SHALL 使用独立审核队列 + +新同步 Skill 和内容变化形成的新版本 SHALL 进入安全扫描与人工同步审核，审核通过前不得发布或替换当前已发布版本。同步审核 SHALL 与普通用户提交审核分开呈现；可按仓库和运行批次归组，但每个 Skill SHALL 独立批准或拒绝。审核权限 SHALL 沿用现有 SkillHub 角色规则。 + +首期 SHALL NOT 提供切换人工审核与自动发布的配置项；无人审核时，待审内容仍不得自动进入市场。自动审核能力须在独立需求中明确检查与误判处理后再评估。 + +#### Scenario: 自动发现新 Skill +- **WHEN** 自动发现并采集到一个此前未发布的有效 Skill 包 +- **THEN** 系统创建待审内容并显示在同步审核队列 +- **AND** 在审核通过前，该 Skill 不作为已发布市场内容展示 + +#### Scenario: 上游更新待审 +- **WHEN** 已发布 Skill 的上游内容发生变化 +- **THEN** 当前已发布版本继续可用，新内容独立等待扫描及同步审核 + +### Requirement: 同一 Skill SHALL 只有一个当前可批准的同步目标 + +同一 Skill 已有待审同步目标时，新上游 revision SHALL 取代其可批准资格；系统 SHALL 保留旧目标及决策历史供审计，仅最新目标可被批准。已拒绝的相同内容指纹 SHALL 不因周期同步反复创建待审项；内容变化或显式重试后才可再次提交。 + +#### Scenario: 待审期间上游再更新 +- **WHEN** Skill 的新 revision 到达，而旧 revision 仍待审 +- **THEN** 旧目标不再可批准，最新目标成为唯一可批准对象 +- **AND** 旧快照和审核历史保持可查 + +#### Scenario: 相同拒绝内容再次出现 +- **WHEN** 周期同步再次得到此前已拒绝且内容指纹相同的包 +- **THEN** 系统不重复创建待审项 + +### Requirement: 同步审核 SHALL 提供可核对的内容差异 + +首次导入 SHALL 展示文件树与内容预览；更新 SHALL 展示相对上次已发布版本的文件新增、修改、删除及可预览文本差异。审核信息 SHALL 展示来源仓库、固定 commit、Skill 路径、安全扫描结果、许可证或 NOTICE 信息及发起者/运行记录。审核者 SHALL 不得直接编辑上游采集内容。同步审核与普通技能审核 SHALL 共用一致的详情页框架：顶部显示对象说明、来源摘要及始终可见的安全和许可证状态；中部通过“技能内容”“变更对比”“来源与安全”“审核记录”Tab 访问适用信息，完整安全卡片与发现项位于“来源与安全”Tab，顶部状态可直达该 Tab；底部审核意见和批准/退回操作在切换 Tab 与滚动时保持可见。没有扫描结果 SHALL 显示明确空态，不得暗示安全通过。同步项 SHALL 展示独立持久化的内容修订号和短 Git commit；完整内部版本标识须可追溯但不得作为溢出的主展示标签。 + +#### Scenario: 审核更新 +- **WHEN** 审核者打开一个上游更新的待审项 +- **THEN** 界面展示与上次已发布版本的逐文件变化和 GitHub commit 来源 +- **AND** 审核者可逐个 Skill 批准或拒绝，但不能改写采集文件 + +#### Scenario: 在审核页逐文件核对变化 +- **WHEN** 审核者进入“变更对比”并选择一个可预览的变更文件 +- **THEN** 页面以占据主内容宽度的单栏视图展示该文件差异，并保持文件切换入口和底部审核栏可用 +- **AND** 页面只按需读取所选文件内容；首次导入无基线时展示新增文件内容，二进制或超限文件展示明确的不可预览原因 + +#### Scenario: 查看安全发现项并作出决定 +- **WHEN** 审核者在顶部看到扫描状态并进入“来源与安全”Tab +- **THEN** 完整安全结论、发现项和许可证证据可核对，缺失或加载失败不会显示为通过 +- **AND** 审核意见在切换 Tab 后仍保留，原有扫描和许可证阻断规则继续生效 + +### Requirement: GitHub 同步内容 SHALL 使用独立的内容修订号 + +每个 Skill 的首次采集内容 SHALL 记为第 1 次修订。相对该 Skill 最近同步目标的内容 SHA-256 变化时，修订号 SHALL 递增；同内容的审核重试 SHALL 沿用原修订号，Git commit 变化但内容不变 SHALL 不产生新修订。内容回退到更早的 SHA-256，但与最近同步目标不同，SHALL 产生新修订。修订号 SHALL 随 Skill 版本持久化，审核页和市场来源信息展示该号。现有技术版本字段 SHALL 保留，继续用于安装和精确版本定位；上游自报版本仅作为来源元数据，不参与修订排序。 + +#### Scenario: 拒绝后重新审核相同内容 +- **WHEN** 管理员要求对已拒绝且内容 SHA-256 未变化的 GitHub Skill 重新审核 +- **THEN** 新审核目标沿用原内容修订号，技术版本仍保持唯一 + +#### Scenario: 内容回退 +- **WHEN** 某 Skill 的内容从 A 变为 B 后又回到 A +- **THEN** 回退后的目标获得更大的修订号 + +### Requirement: 同步 SHALL 保留来源权利与许可证信息 + +系统 SHALL 保留和展示仓库或技能包的许可证、NOTICE 与来源署名，不得将第三方内容标记为 SkillHub 自有许可证。平台 `@github` 来源的许可证缺失或无法识别时 SHALL 阻止自动公开发布，待具有现有审核权限的平台人员明确处理；TEAM 来源 SHALL 呈现相同警告和人工审核信息。 + +#### Scenario: 平台 Skill 缺少可识别许可证 +- **WHEN** `@github` 来源的待审 Skill 无可识别许可证 +- **THEN** 系统在审核页显示阻塞原因，并要求平台人员明确处理后才能公开发布 + +### Requirement: 同步冲突 SHALL 阻断自动绑定 + +同一 namespace 中 Skill 坐标冲突或多个来源指向同一 Skill 时，系统 SHALL 阻断自动导入并交给该 namespace 的管理员处理，不得覆盖现有 Skill。确认是同一 Skill 的情况下，具有权限的管理员 SHALL 能把来源绑定到既有 Skill ID，并留下审计记录。单个 Skill SHALL 只有一个权威同步来源。 + +#### Scenario: 同名冲突 +- **WHEN** 新来源 Skill 的目标坐标已属于另一个 Skill 或来源 +- **THEN** 系统将该 Skill 标记为冲突，其他 Skill 继续处理 +- **AND** namespace 管理员获得处理入口，现有内容保持不变 + +#### Scenario: 人工确认绑定 +- **WHEN** 有权限的管理员确认新来源应绑定到既有 Skill ID +- **THEN** 系统记录操作者和绑定依据，并从后续固定 revision 更新该 Skill + +### Requirement: 路径变化与消失 SHALL 不误删 Skill + +同一仓库旧 Skill 路径消失时，仅当新路径的完整技能包内容哈希存在唯一匹配，系统 SHALL 自动认定路径移动；否则进入人工确认。上游目录删除、部分扫描或来源停用 SHALL NOT 自动删除、撤回或隐藏已发布 Skill，也不得仅凭目录消失创建删除审核项。 + +#### Scenario: 唯一路径移动 +- **WHEN** 同仓库旧路径消失，且只有一个新路径的完整包哈希与旧包相同 +- **THEN** 系统更新来源路径绑定并保留 Skill ID 和历史版本 + +#### Scenario: 上游删除目录 +- **WHEN** 完整扫描确认上游某个 Skill 目录不再存在 +- **THEN** 系统记录上游缺失，但不删除已发布 Skill 或创建删除审核项 diff --git a/openspec/changes/add-github-skill-sync/state-model.md b/openspec/changes/add-github-skill-sync/state-model.md new file mode 100644 index 000000000..631409a8d --- /dev/null +++ b/openspec/changes/add-github-skill-sync/state-model.md @@ -0,0 +1,70 @@ +# GitHub 同步状态与检查点设计 + +本文件细化 [design.md](design.md) 的跨服务状态边界；业务行为以同一 change 下的 `specs/` 为准，传输字段以 `docs/api/github-sync-internal.openapi.yaml` 为准。以下名称是设计语义，实施时可映射到现有实体和枚举。 + +## 1. 分层状态 + +| 层 | 建议状态 | 独立语义 | +| --- | --- | --- | +| 来源绑定 | `ACTIVE`、`PAUSED`、`UNREACHABLE` | 决定是否安排后续采集；`UNREACHABLE` 仍按周期复查。最近一次访问错误单独记录，不能把限流当不可达。 | +| 任务 | `QUEUED`、`LEASED`、`RETRY_WAIT`、`PARTIAL`、`ACQUISITION_COMPLETE`、`FAILED` | 只表达发现或固定 revision 采集进度；`ACQUISITION_COMPLETE` 不表示摄入、扫描、审核或发布完成。 | +| Skill 路径 | `ACCEPTED`、`UNCHANGED`、`IGNORED`、`RETRYABLE_FAILURE`、`TERMINAL_FAILURE` | `ACCEPTED` 是 Java 已持久接收包摄入记录。不能由 Go 宣称 `UNCHANGED`：它只能报告已采集包，Java 比较正式内容指纹后确定。 | +| 包摄入 | `RECEIVED`、`VALIDATING`、`RETRY_WAIT`、`VALIDATED`、`READY_FOR_REVIEW`、`NO_CHANGE`、`BLOCKED`、`FAILED`、`APPROVED`、`REJECTED`、`SUPERSEDED` | 每包独立推进；`VALIDATED` 仅表示归档与内容政策复核通过，尚未创建审核目标。审核决策和目标取代后，摄入状态也结算，避免历史项长期占用积压容量。 | +| 审核目标 | `PENDING`、`APPROVED`、`REJECTED`、`SUPERSEDED` | 审核状态不复用任务状态；旧快照保留审计，只有最新目标可批准。 | + +`PAUSED` 由管理员显式操作；访问失败不覆盖管理员暂停。`UNREACHABLE` 的重查约为首次失败后 24 小时、再失败后 3 天、再次失败后每 7 天。来源恢复时重新对账，已发布 Skill 始终按原生命周期保留。仓库/路径忽略是来源绑定的策略状态，命中已有绑定时停止后续同步，但不删除或撤回内容。 + +反压计数按 `(namespace, repository ID, Skill 路径)` 去重，只计该坐标当前最新且未解决的摄入、复核/扫描、待审或待管理员处理冲突项；已批准、拒绝、被取代的历史目标不计。全平台计数是各 namespace 当前计数之和。具体高低水位数值单独确定，不把已完成历史记录数量误当积压。 + +已登记来源在审核未结束时仍按目标周期执行 `HEAD_ONLY`；观测到新 commit 且容量允许时，重新执行 `FULL`。现有待审项不阻止新目标取代；正在校验或发布的同仓库包则先完成当前处理，避免重复采集。 + +待验证仓库线索不计入待审积压，但有独立的候选预算与发现游标；未完成包准入占位也不冒充待审项，Java 在发放新准入时把两者合并判断剩余容量。后台分别显示待审积压、在途准入和延后候选数量。 + +新包下载前由 Java 原子授予当前租约、commit、路径的采集准入，并把未完成准入占位纳入容量判断；高水位拒绝新的准入，已获准包完成或失败后释放占位。积压期间发现结果仅进入有预算的待验证仓库线索，不建立正式来源或待审版本；已登记仓库的 `HEAD_ONLY` 观测不增加待审项。到低水位后重新核对当前上游状态，再恢复包准入。 + +## 2. 身份与持久记录 + +| 记录 | 主键或唯一边界 | 保存的事实 | +| --- | --- | --- | +| 来源绑定 | namespace、GitHub repository ID；TEAM 与 `@github` 可各有绑定 | 入口集合、管理范围、状态、下次运行时间、最近观测/完整处理 commit、可见性默认值 | +| 发现规则与运行 | 规则 ID、版本、运行 ID；运行检查点 | 结构化查询快照、启用状态、候选与延后原因；不把搜索命中直接视为有效 Skill | +| 仓库任务 | 任务 ID；同来源同类型只有一份未完成工作 | 固定 commit、工作模式、到期时间、租约代际、租约截止、路径进度、重试目标 | +| 路径结果 | 任务 ID、固定 commit、规范化 Skill 路径 | 枚举覆盖范围、结果、摘要或错误；同键重复结果应幂等，摘要冲突应拒绝 | +| 包摄入 | 来源绑定、固定 commit、Skill 路径与内容摘要；另存任务回报键 | `uploadId`、暂存对象引用、大小/摘要、接收时间、摄入状态；不保存预签名下载 URL，同内容不产生第二待审目标 | +| 同步目标 | 来源绑定与 Skill ID，目标内容/合规指纹 | 当前可批准资格、被取代关联、扫描和审核记录；已拒绝相同指纹不反复入队 | +| 凭证 | 凭证 ID、作用域、密钥版本 | 仅 Java 保存密文；任务表、日志、指标、URL 均不保存明文 | + +路径需统一规范化为仓库内相对 POSIX 路径，拒绝绝对路径、`..`、空段、反斜杠和非 `SKILL.md` 根；Java 不信任 Go 给出的路径。GitHub repository ID 是上游身份；owner/name 是可变展示坐标。Hash 用于内容比较及同仓库唯一路径移动判定，不取代来源身份。 + +## 3. 领取、续租与重复回报 + +1. Java 定时调度仅创建或合并到期任务；PostgreSQL 原子领取生成新的 `leaseGeneration`、随机 `leaseToken` 和到期时间。Go 只在当前租约内回报。 +2. `renew` 校验任务 ID、代际和令牌后延长截止时间。租约到期或代际变化时旧执行端必须停止上传和回报；Java 返回租约失效，不允许旧结果覆盖新进度。 +3. Go 在仓库任务内固定一个 commit，完整枚举后逐包采集。每次上传授权绑定当前任务、commit、规范化 Skill 路径和单个临时对象。相同路径同摘要的回报重放返回同一摄入 ID；同路径不同摘要是冲突。 +4. Java 在短事务中保存路径结果、摄入记录与对象引用，再确认包回报。对象内容与摘要的最终校验在后台异步进行，因此“已接收”不代表“可审核”。 +5. 任务 `complete` 仅在覆盖范围明确后结算。完整枚举且所有应处理路径已被持久接收或明确判为无需采集/终态失败，才可推进 `lastFullyProcessedCommit`；需重试的路径或不完整子树阻止推进。`TERMINAL_FAILURE` 保留可见错误，不自动视为成功同步该路径。 +6. `HEAD_ONLY` 任务只能推进 `lastObservedCommit`。积压下降后重新解析当前 HEAD 并重新建立采集任务，不补采未处理的每个中间 commit。 + +`lastFullyProcessedCommit` 表示该固定 revision 的仓库枚举完整且所有应处理路径已有**持久采集结论**，不表示 Java 校验成功或审核通过。若某个已接收对象在摄入前丢失/过期，Java 将该路径转为需重采集并使该 revision 的完整处理资格失效；在恢复前不得把检查点作为“已同步成功”对外展示。包内容和合规指纹仍由 Java 计算。 + +## 4. 失败与补偿 + +| 情况 | 处理 | +| --- | --- | +| Go 停机或续租失败 | 租约到期后重派，恢复已持久的路径检查点；未回报包重做。 | +| 单个包失败 | 继续其他路径的采集、Java 校验及独立同步审核；失败路径按可重试类别和原固定 commit 续跑。超过安全上限、symlink、submodule 是明确的包失败，不产生部分版本。终态失败造成的任务 `PARTIAL` 不单独阻断已接收兄弟包；旧路径未确认时仍禁止自动移动。 | +| GitHub 403/429 或额度耗尽 | 保存资源类别、额度和 `retryAt`，释放租约；不得通过持续轮换 Token 规避。 | +| 审核积压达到高水位 | 停止授予新包采集准入；已获准包可以结算，仓库任务保存未处理路径进度。发现任务只保存有预算的仓库线索与游标，已登记仓库只检查 HEAD。低水位后按当前 revision 恢复。 | +| 树截断或子树不可枚举 | 完整子树继续；不完整范围记为 `INCOMPLETE` 并重试，不能据此推断目录删除。 | +| Go 临时磁盘预算不足 | 能清理并分批续跑时保存路径检查点，回报 `LOCAL_DISK_BUDGET`；当前容量下一个合规包也无法处理时回报 `CAPACITY` 并等待管理员调整后重试，不立即循环领取，不把来源标为不可达。 | +| 上游仓库或路径消失 | 仓库不可达按复查节奏；完整树中路径消失仅记录来源变化，不删除已发布 Skill。 | +| 暂存对象摘要/大小不符或过期 | Java 隔离该包并记录原因；可重试对象缺失安排固定 commit 重采集；不创建版本。 | +| 内容未变化 | Java 更新检查记录，不重复创建版本或审核项；许可证/NOTICE 变化仍需重新核对合规指纹。 | +| 发现目标与现有 Skill 冲突 | 阻断该包自动绑定，交 namespace 管理员处理；同仓库其他包继续。 | +| 待审期间出现新内容 | 旧目标失去批准资格，最新目标成为唯一可批准项；已发布版本继续可见。 | + +已被 Java 接受且尚未消费的暂存对象不能单凭固定 TTL 清理；清理任务要先核对摄入状态，失败清理留补偿记录。任务与摄入、审核分别展示，避免将“采集完成”误读为“已发布”。 + +## 5. 上传大小边界 + +Go 预检当前有效发布限制，当前 `application.yml` 配置为最多 100 个文件、单文件 10 MiB、包总量 100 MiB，并限制归档上传大小。Go 在受限本地目录逐包打包，上传授权含预期大小、SHA-256、短有效期和唯一对象 key；回报仅包含 `uploadId` 与完整性信息。Java 用自身存储适配器解析对象并复核实际大小与摘要，不接收 Go 提供的任意外部链接。本地流式入口在接收时执行硬上限。S3/MinIO 预签名 PUT 对 key 与有效期有限制，但不能仅凭签名宣称服务端强制最大上传字节数；首期以 Go 预检、Java 后检、隔离暂存区容量告警及超额对象清理形成边界。若将来需要对象存储在接收前硬拒绝，应另行验证条件上传策略。 diff --git a/openspec/changes/add-github-skill-sync/tasks.md b/openspec/changes/add-github-skill-sync/tasks.md new file mode 100644 index 000000000..3d88dab14 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/tasks.md @@ -0,0 +1,93 @@ +## 1. 领域模型与迁移 + +- [x] 1.0 新增 `skillhub-github-sync` Maven 功能模块并接入 parent/app/infra 依赖与 server Dockerfile；验证依赖方向为 app、infra → github-sync → domain/storage，且不新增 Java 进程。 +- [x] 1.1 增加系统管理的 `CATALOG` namespace 与 `@github` 初始化、保留 slug 和系统 actor；验证已有同名 namespace 会明确阻断且不改动用户数据。 +- [x] 1.2 增加 GitHub 来源、发现入口、规则版本、仓库 ID、namespace 归属与来源级可见范围模型；验证同 namespace 去重和跨 namespace 独立绑定。 +- [x] 1.3 增加发现运行、仓库任务、租约、检查点、逐 Skill 结果和重试时间持久化；验证并发领取只有一个有效租约且重放不产生重复记录。 +- [x] 1.4 为审核记录增加普通/同步来源判别及同步目标关联，迁移旧数据为普通审核；验证普通审核查询结果不变。 +- [x] 1.5 增加系统级公开 GitHub 凭证密文、版本化密钥与健康状态支持；验证无密钥无法解密、轮换可读旧密文、认证失效可标记并提示、配额耗尽不误标失效、API/日志只返回脱敏信息。 + +## 2. Java 控制面与服务间合同 + +- [x] 2.1 实现平台规则、精选仓库、精选用户或组织的配置与无副作用预览 API；规则只接受关键词/topic 条件及核心/扩展优先级，变更生成新版本并固定增量/全量搜索窗口，预置规则初始停用。验证首次部署/升级不自动广泛搜索、规则启用后生效、精选仓库可直接同步、权限、规则版本、去重、预览不创建来源及原始 GitHub 查询被拒绝。 +- [x] 2.2 实现 TEAM 公开仓库来源配置、来源级可见范围和暂停/恢复 API；验证越权拒绝、默认可见性及已发布 Skill 不变。 +- [x] 2.3 实现默认约 24/24/72 小时、后台按类别可调的任务目标周期，以及广泛搜索默认 7 天全量重查、来源级待执行任务合并、人工触发合并、来源优先级、TEAM 公平性和未解决同步项高低水位；按最新 `(namespace, repository ID, Skill 路径)` 计数并原子预留未完成包准入，高水位停止新包采集，但已启用发现任务可有预算地保存仓库线索，已登记仓库只查 HEAD；分别记录已观察与已完整处理 commit，低水位后重新核对当前 revision；验证并发任务不越过准入线、候选预算有界、不同 namespace 反压隔离和 Go 离线不重复堆积。 +- [x] 2.4 在 `docs/api/github-sync-internal.openapi.yaml` 定义独立于 Web 公开 API 的版本化 HTTP/JSON 内部领取、续租、逐包采集准入、临时上传授权、逐包回报、完成/失败合同，并使 Java/Go 合同样例保持一致；在 Spring Security 为内部路由单独验证服务身份、限制权限并处理无会话请求，不依赖未注册的 `IdempotencyInterceptor`；逐包回报只在持久摄入记录成功后确认接收，验证过期租约、重复/冲突回报、准入占位释放、密钥轮换、非机器身份和不兼容版本。 +- [x] 2.5 实现有预算、按 GitHub repository ID 去重的待验证仓库线索、发现游标及到正式来源的转换，并处理仓库 rename/transfer 对账；验证高水位只积累有界线索、不检查新候选树，低水位后重新核对当前仓库状态，无有效 Skill 的搜索候选不进入正式来源。 +- [x] 2.6 为 S3/MinIO 与本地存储提供短时、限对象 key 的上传票据，Java 以 `uploadId` 定位暂存对象，不接收 Go 提供的任意下载链接；实现隔离暂存区容量告警与清理补偿。验证 Go 无管理凭证仍可逐包上传、越权 key 被拒绝、本地流式入口执行大小硬上限、S3/MinIO 超额对象在 Java 摄入前被拒绝并清理。 +- [x] 2.7 实现可恢复的运行检查点、同 commit 失败路径重试和 `retryAt`，验证执行端重启后不重做已完成 Skill 包。 +- [x] 2.8 已完成运行/任务详情默认保留 90 天，未完成任务与恢复数据豁免按期清理，来源/审核审计沿用原生命周期；验证清理不会破坏续跑或审核追溯，容器日志保留交部署环境管理。 + +## 3. Go 发现与采集服务 + +- [x] 3.1 在仓库根目录建立独立 Go module 与单二进制，按控制面客户端、GitHub 访问、发现、仓库包采集和 worker 分层；建立含 Git CLI/CA 证书、非 root 运行的镜像、健康/指标入口和两个有界工作池；验证默认单副本在本地 Compose 可启动、停机不再领取新任务并释放在途资源。 +- [x] 3.2 实现带匿名/系统凭证选择的 GitHub 客户端、资源类别额度跟踪、认证失效回报、限流退避与脱敏日志；用 HTTP fixture 验证认证失败、仓库不可读、403/429 限流、reset、健康凭证回退和匿名公开读取，确保不以轮换规避限流。 +- [x] 3.3 实现关键词/topic 条件的独立仓库搜索查询、核心/扩展预算顺序、精选用户或组织公开仓库枚举及精选仓库探测；首期只用仓库元数据召回线索，不做 README/全站代码搜索，再以仓库树验证 `SKILL.md`。验证固定窗口与规则版本、去重、增量与全量轮次、1,000 结果上限和 `incomplete_results` 不误记成功，并说明元数据缺少线索时可能漏召回。 +- [x] 3.4 实现固定 commit、递归树枚举与截断后子树续查；验证部分子树失败仍处理完整子树且结果标记为不完整。 +- [x] 3.5 实现独立 Skill 根识别、嵌套子树排除、已确认的 10 个默认候选忽略目录和来源级追加规则；Go 在固定 commit 上仅把所需包文件取到受限本地临时目录，按当前有效发布政策预检文件数、单文件与包总量，逐包打包上传；验证 `test/` 候选被忽略而父包测试文件保留，且不下载无限制整仓。 +- [x] 3.6 实现先 Tree 元数据枚举/预检、后受限 Git CLI 的 blobless 局部获取、固定 commit 校验、稀疏读取及逐 Skill 包归档，拒绝符号链接与 submodule；对 `.git`、检出文件和归档施加每仓库/进程总临时磁盘预算，逐包清理，逼近预算时保存 `pathCheckpoint`、释放目录并从同一 commit 续跑；若容量不足以处理一个合规包，回报容量阻塞且同配置下不立即重试。验证大仓库不会无界下载或占满本地磁盘，单包失败不阻断兄弟包。 +- [x] 3.7 实现领取、续租、逐包采集准入、短时上传、逐包结果与运行额度回报，并支持已登记仓库的仅 HEAD 检查模式；验证高水位后不再启动新包下载或创建待审项、已获准包可结算、未处理路径保存检查点，以及固定 commit 失败路径重试的幂等和临时对象摘要。 + +## 4. SkillHub 导入、版本与审核 + +- [x] 4.1 建立可恢复的逐包摄入记录与后台消费流程，对临时采集包以有界流式读取重新校验摘要、路径、文件数/大小、`SKILL.md`、忽略规则和许可证证据；验证已接收回报在 Java 重启后续跑，伪造回报不创建 Skill 或版本。 +- [x] 4.2 实现规范化包内容与许可证证据的内容指纹；验证同内容不升版、仅 commit 变化不升版、许可证变化可被发现。 +- [x] 4.3 以 namespace 和系统 actor 创建来源绑定及待审 Skill 版本，接入现有 Scanner；验证 `@github` 全局可见、TEAM 默认 `NAMESPACE_ONLY` 且审核前不发布。 +- [x] 4.4 实现最新待审目标取代旧目标资格、相同拒绝指纹抑制和审计保留；验证待审期间连续两次上游变化仅最新可批准。 +- [x] 4.5 实现同 namespace slug 冲突阻断、经授权的既有 Skill ID 绑定及唯一内容哈希路径移动；验证不覆盖他人 Skill、不误合并跨 namespace 来源。 +- [x] 4.6 实现目录消失、部分扫描、来源暂停及不可达恢复语义；验证所有场景均不自动删除或撤回已发布 Skill。 +- [x] 4.7 在同步审核中提供首次文件树、更新逐文件差异、来源 commit、许可证/NOTICE、扫描与操作者信息；验证审核者不能修改上游采集内容且许可证阻塞可见。 + +## 5. 管理后台与市场展示 + +- [x] 5.1 增加平台 GitHub 同步总览、规则编辑/预览、精选入口、来源列表、运行与额度、凭证管理；分别展示已采集交付、Java 处理中、待同步审核和处理失败，通过页面验证空态、延后、失败和手动触发反馈。 +- [x] 5.2 在 TEAM namespace 管理视图增加公开仓库来源及逐 Skill 状态，验证团队管理员只看到本团队数据和可用操作。 +- [x] 5.3 在审核区增加与普通审核分离的同步队列，按仓库/运行分组但逐 Skill 决策；验证现有普通审核列表仍仅显示普通项。 +- [x] 5.4 在市场 Skill 详情展示 GitHub 来源、固定 commit、许可证/NOTICE 与同步状态允许公开的信息；验证 TEAM 私有或仅 namespace 可见来源不会泄露给无权用户。 + +- [x] 5.5 按产品图重组平台 GitHub 同步总览、来源发现与运行详情，并提供公开 `@github` 技能专页；运行详情必须使用当轮任务和逐路径事实，公开页数量只统计市场可见已发布 Skill。 + +- [ ] 5.6 统一普通技能与 GitHub 同步技能审核详情布局：顶部技能说明及安全/许可证摘要，中部内容、变更、来源与安全、记录 Tab，底部固定审核意见与操作；GitHub 逐文件差异占满主内容宽度，完整安全发现项在对应 Tab 中，长技术版本仅在折叠信息中显示。 + +- [ ] 5.7 为 GitHub 同步技能持久化内容修订号，并在审核和市场展示；保留技术版本作为安装坐标。同内容显式重试沿用修订号，内容变化及回退递增。 + +## 6. 部署、文档与跨服务验收 + +- [ ] 6.1 为开发/发布 Compose、Helm、基础 K8s 和现有镜像发布矩阵接入默认单副本的 Go 服务、专用服务密钥、GitHub 凭证密钥、标准 GitHub 出站 HTTP 代理及专用受限临时目录配置；验证可关闭采集、内部 Java 通信不误走代理、镜像目标架构可构建，并在小容量临时磁盘下安全停止/续跑，且无需新增队列/KMS/mTLS 组件即可在本地存储和 S3 两种模式运行。 +- [x] 6.2 编写中文管理员配置与运行手册，明确公开仓库范围、Token 权限、无 TLS 部署风险、停用/恢复、限流与故障排查；验证文档命令和配置字段与实际部署一致。 +- [x] 6.3 先用少量真实公开仓库验证单/多 Skill、嵌套与较大仓库的采集边界，再用受控 fixture 和合成任务逐级验证约 1000 仓库、10000 Skill 路径的容量基线；同时覆盖搜索、精选用户、部分树、无变化、更新待审、路径冲突、不可达和恢复，分别记录公网请求数与本地容量证据。 +- [x] 6.4 检查日志、指标、普通 API 和任务记录的凭证与内容泄漏路径；验证 Token、Authorization 与完整 Skill 正文不出现于这些输出。 +- [x] 6.5 运行 OpenSpec 严格校验、相关后端/Go/Web 检查并建立规格到验收结果的追踪表；记录尚未完成的环境验证与原因。 + + +## 官方深度探索增量（已授权，实施与运行验收中） + +需求：[deep-discovery.md](deep-discovery.md)；设计与验收：[deep-discovery-design.md](deep-discovery-design.md)。既有任务勾选不代表本增量完成；具体实现与验证按下列任务独立追踪。 + +以下为既有功能任务完成记录。main 整合后的交付须按 [验证说明](verification.md) 重新验证，不能继承历史运行结论。持续公网吞吐和生产 SLA 不属于已验证保证。 + +- [x] D1a 用户整体确认产品与设计方向，授权进入实施。 +- [x] D1b 在verification.md增量索引冻结可执行验收与协议兼容边界；逐项区分受控运行、公网试验和未完成验收，不代表全部场景通过。 +- [x] D2 定义配置、分页/抽屉、策略快照、出处和等待反馈合同，同步 OpenAPI 与 MD；完成静态解析、引用及旧完成接口兼容性对照，交叉审查的暂停合同问题已修复。此项不代表 API 已实现或运行验收通过。 +- [x] D3 Java 持久状态、去重归因、公平续排、统计查询和权限实现。 +- [x] D4 Go 执行官方查询、README、作者一层扩展、分片与恢复实现；策略目录由 Java 统一维护。 +- [x] D5 多凭证限流及采集服务离线/多实例恢复完善。 +- [x] D6 探索配置、列表抽屉和运行反馈实现；后续 U 节将其并入统一技能同步入口。 +- [x] D7 执行设计矩阵、对照召回、限定规模性能及停机恢复验证，完成预算初值的小范围校验。 +- [x] D8 已完成公开GitHub小范围代码搜索对照与固定抽样Tree核查；存在新增仓库也存在同blob复制，首期保留备选。未宣称完整包有效性或独立额度恢复通过。 +- [x] D9 更新部署/操作说明，记录可回退条件和证据，交付人工预览。 + + +## U. 技能同步入口整合（实现及验收） + +方案见[统一管理入口](unified-admin.md)。审核合同保持；Q3/Q6/Q7确认新增平台@github采集安全暂停/恢复，TEAM不受该开关控制，技术合同已冻结于 unified-admin-contract.md。 + +- [x] U1 抽取可复用页面内容，统一三页签、配置抽屉与URL状态，保留sourceId/runId及旧探索路由兼容。 +- [x] U2 整合来源配置、统一仓库列表、运行视图和设置，侧栏仅保留技能同步入口。 +- [x] U3 收敛查询与轮询，保留分页、筛选、详情按需加载、权限及错误状态。 +- [x] U4 补组件/导航回归、桌面窄屏及浅深色浏览器验证，更新文案和使用指南。 +- [x] U5 Java/Go相关质量门、Web质量门、OpenSpec校验与最新预览复查；按实际结果回填证据。 + +- [x] U6 实施平台@github采集安全暂停/恢复合同；验证持久状态、在途检查点、已接收包继续处理、TEAM不被暂停及原停用来源不被恢复（Q3/Q6/Q7已确认，技术合同已冻结于 unified-admin-contract.md）。 + +- [x] U7 补齐统一仓库服务端分页/筛选/去重及多出处展示，使用指定仓库产品名称（Q4/Q5已确认）。 diff --git a/openspec/changes/add-github-skill-sync/unified-admin-contract.md b/openspec/changes/add-github-skill-sync/unified-admin-contract.md new file mode 100644 index 000000000..1622f26e8 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/unified-admin-contract.md @@ -0,0 +1,64 @@ +# 统一技能同步管理：实施合同 + +状态：2026-10-13，产品已确认；本页将研究结果冻结为实施合同，代码与生成类型须按此对齐。平台暂停协议及统一查询合同已冻结，实施验收结果另行记录。 + +## 统一仓库读接口 + +根路径 `/api/v1/admin/github-sync`，SUPER_ADMIN会话+既有CSRF规则，namespace由服务器解析为@github，不接受客户端扩大到TEAM。GET响应no-store，外层复用ApiResponse。 + +- GET `/repositories`：page>=0，size=1..100（默认20），q使用现有长度限制/LIKE转义；可选originType、state、sourceStatus和asOf。非法枚举400。 +- GET `/repositories/{repositoryId}`：不存在或不在平台范围404。 +- GET `/repositories/{repositoryId}/origins`：有界分页，合并并去重source/candidate的出处。 +- GET `/repositories/{repositoryId}/paths`：复用逐路径安全投影，资格改为统一仓库存在性；保持NO_CHANGE审核关联约束。 + +Page为items/total/page/size/asOf。Summary为repositoryId、可空sourceId/candidateId、owner、name、firstSeenAt、state、可空sourceStatus/candidateStatus、去重有界originTypes。Detail包含repository以及可空latestRunId、lastCheckedAt、nextCheckAt、lastErrorCode。Origin包含originType/originKey/firstSeenAt/可空lastSeenAt、entryId/channel/queryId/runId/seedRepositoryId。不存在的观测不伪填。 + +state用于登记阶段：有来源REGISTERED；无来源为PENDING/VALIDATING/NO_VALID_SKILL/VALIDATION_FAILED/UNREACHABLE；异常BOUND但无来源使用BOUND_WITHOUT_SOURCE，不虚称有效。sourceStatus独立表达已登记来源的ACTIVE/PAUSED/UNREACHABLE。等待/审核不是同一枚举，不混合成一个误导性成功状态。 + +基集source LEFT JOIN candidate，以namespace+repository匹配，再UNION ALL无source的candidate。筛选出处使用EXISTS，过滤后分页，排序firstSeenAt DESC、repositoryId DESC；只关联本页详情，不把历史路径联入COUNT。firstSeenAt取首次发现/创建最早值，名称以source当前值优先。asOf只抑制后续新增导致的页漂移，不是跨请求历史快照；单请求total/items使用一致事务快照。 + +完整出处按type/key去重，首次取min，lastSeen仅真实观测；删除规则后仍保留出处。被清理run不返回失效链接。仅按查询计划证据增加必要索引，不增加无依据汇总表。 + +## 指定仓库添加 + +重复的同namespace/类型/目标请求幂等返回既有入口，不恢复已停用状态，不反复生成新任务；同稳定仓库ID归并依旧在来源解析阶段完成。未解析ID的配置不混入统一仓库total，按entry UUID在配置中展示真实解析任务状态，提供最近run/errorCode/retryAt（未知为空）。关联键使用任务snapshot真实ruleId，不按名字匹配；禁止返回原始snapshot或外部异常正文。 + +## 运行列表增量 + +扩展既有GET `/runs`的可选kind/status筛选，保留page/size及原调用兼容。kind维持真实DISCOVERY/REPOSITORY_SYNC两类主运行；后者页面标「仓库检查与采集」，HEAD_ONLY/FULL具体阶段在任务详情展示，不强行拆成三个虚构运行类型。status沿原枚举。所有筛选在分页前，COUNT与items口径一致，继续排除搜索预览及已清理详情。 + +增安全target摘要及targetCount：单目标显示明确仓库/账号/查询标识，多目标显示数量并引导分页任务；没有目标保持空，不把第一个任务冒充全部。时间沿用createdAt并准确标「创建时间」，不伪装尚未采集的实际开始时间。 + +## 现有类型与权限兼容 + +保留旧sources、deep-discovery及run详情API；新接口通过Springdoc生成前端schema，不手工编辑生成文件。准备清单复用已有真实配置/状态API，不额外造汇总DTO；各请求失败/未知分别体现，不能默认为就绪。仓库ID安全整数范围沿既有策略。 + +## 平台采集控制合同（冻结） + +### 管理接口与持久状态 + +新增GET/PUT `/api/v1/admin/github-sync/platform-control`，SUPER_ADMIN、CSRF、审计及no-store沿已有方式。PUT为 `{expectedVersion, paused}`，乐观冲突返回409；同一版本重复相同意图允许幂等返回，不能多次翻转版本。响应为namespaceId、version、desiredPaused、state、pendingStopCount、expiredUnconfirmedCount、updatedAt（无数据时不得假造）。state为RUNNING/PAUSING/PAUSED；RUNNING仅表示控制未暂停，不代表Go在线，UI仍独立显示worker状态。 + +V119新增固定引用真实平台namespace的控制行：namespace_id、paused=false、单调version、updated_at/updated_by。任务增加platform_pause_version/generation/token_hash/stop_ack_at，捕获当前暂停epoch的在途租约，摘要不输出管理端。暂停锁控制行与捕获task；claim及admission在同事务先持平台控制共享锁，之后沿既有官方配置/预算/准入/task锁顺序，不反向死锁。暂停生效后不能越过门创建新平台租约或准入；TEAM仍按原条件领取。scheduler可保留有去重约束的已排任务，必须保证不因暂停重复排队，领取门是最终保证。 + +当前版本未确认任务存在时为PAUSING，包括过期/离线而未确认的任务。不能将租约超时或Go心跳丢失当停止证明；响应分开给出pending和expired数量。无在途或全部明确停止才PAUSED。恢复不依赖ack，切换paused=false并递增version，原进度及独立启停状态保留。保留策略不能清理当前暂停尚未确认记录而伪造PAUSED。 + +### Worker协议 + +能力 `platform-pause/v1` 仅作为平台任务领取要求，TEAM不强制新增能力；升级期间旧worker不能取得新的平台工作。旧在途任务不能假称可安全停止，显示待确认。claim附可空platformControlVersion识别受控任务。 + +新增POST `/internal/github-sync/v1/tasks/{taskId}/platform-control` 与 `/platform-pause`。control提交既有lease proof，返回action CONTINUE/PAUSE及platformVersion。pause提交observedVersion和相应kind的已确认discoveryCheckpoint/RepositoryProgress，所有进度校验复用现有安全规则。 + +仅执行线程在停止外部工作、完成必要结算及本地清理后发送停止确认。Java验证当前paused/version、捕获generation/token，原子保存进度/ack并将任务恢复为QUEUED但受平台门阻止领取，不记FAILED。旧epoch返回CONTROL_CHANGED，worker重新读取控制；旧generation不得改新租约。重复相同确认幂等。过期worker凭本epoch捕获proof只可确认停止，不能提交未持久进度或新包，不能影响新generation。 + +正常完成/失败回报只有协议明确保证外部执行结束且匹配捕获proof时才能确认停止；未知错误、过期回报不自动充当ack。控制网络故障时如实PAUSING，不能承诺即时停止。 + +### 安全点与已接收边界 + +受控平台任务最多每5秒目标频率观察控制，实际停止时间受请求超时和网络条件约束；TEAM不增加该轮询。观察协程仅传递停止意图，不带旧checkpoint旁路ack。执行线程在发现每请求/已接收批次、metadata/HEAD、Tree每请求、Git每网络步骤、准入前及上传接收后检查；与既有配额钩子组合，不覆盖。正在执行的外部获取有界结束或取消，不能等完整大仓库结束。 + +Java `acceptReport` 提交摄入、准入及upload ACCEPTED事务是已接收边界，单纯PUT对象不是。暂停前已准入且已完整打包的成果可按原租约完成上传/接收后ack；不完整本地包清理并释放准入，只保留已确认进度。发现已获取页可经现有幂等batch提交再ack；拒绝时停在最后确认checkpoint。已接收摄入的后续校验/扫描/待审不加控制门。 + +### 验证与部署兼容 + +真实PG竞态证明暂停/claim/admission线性化；覆盖legacy/official/preview发现、HEAD/FULL、TEAM、过期及失联、快速暂停恢复旧ack、确认响应丢失、重启与独立disabled。API新字段在Springdoc生成类型同步。V119追加迁移，不回写既有迁移；Java/Go部署要配套，新Java已启用能力门时旧Go平台采集会等待升级，TEAM规则不变。回退不能简单只换旧Go，需考虑能力和进行中暂停；本地发布前在隔离环境验证，不让预览进入自动采集。 diff --git a/openspec/changes/add-github-skill-sync/unified-admin.md b/openspec/changes/add-github-skill-sync/unified-admin.md new file mode 100644 index 000000000..b8e51143a --- /dev/null +++ b/openspec/changes/add-github-skill-sync/unified-admin.md @@ -0,0 +1,103 @@ +# 技能同步统一管理入口 + +本文记录已确认的统一管理产品决策。正式任务见 [tasks.md](tasks.md) 的 U 节，领域术语见 [glossary.md](glossary.md)，验证入口见 [verification.md](verification.md)。 + +## 1. 目标与范围 + +平台管理员只面对一个「技能同步」入口，以少量配置使系统持续发现、采集及更新公开GitHub技能，并能查看结果、定位异常和进入独立同步审核队列。自动探索属于采集方式，不再作为另一套并列产品。 + +本轮包含前端页面整合、统一仓库服务端查询、平台采集安全暂停/恢复，以及必要的Java/Go协议和文档调整。不改变审核发布规则、namespace归属、TEAM权限、公开仓库限制和现有来源调度策略。没有确认任何新增托管平台、自动发布、一次性探索按钮或全站召回承诺。 + +## 2. 页面框架 + +固定标题「技能同步」，说明当前来源为GitHub；顶部提供「采集配置」「系统设置」入口。日常只有三个页签： + +| 页签 | 默认内容 | 详情与操作 | +|---|---|---| +| 概览 | 平台采集状态、Go在线状态、已登记仓库/待验证线索/未解决同步项、需要处理的问题、最近活动 | 暂停/恢复平台采集；问题直达相关配置；活动直达运行详情；未解决项入口直达既有同步审核队列，并说明数量还包含阻塞等未解决状态 | +| 仓库 | 全部发现方式、全部处理状态的统一分页列表 | 按发现方式及处理状态筛选；仓库详情抽屉展示出处、检查/采集结果、技能状态和对应运行/审核链接 | +| 运行记录 | 真实运行按时间排序的统一列表 | 显示工作类型、目标、状态、结果摘要和创建时间；支持类型/状态筛选；查询、分页、任务及错误细节按需展开 | + +概览以运行情况及需要处理的问题为主，不堆积空卡片和默认展开的渠道图表。正常时明确显示运行正常；额度冷却等可自动恢复状态说明等待原因及可靠的预计时间，不一律标为需要人工修复。Go离线、平台手动暂停、容量暂停和GitHub额度等待分别展示，不相互冒充。 + +运行记录中的发现仓库、检查更新、采集技能应从真实run/task语义映射，不能虚构全站探索轮次，也不能把一个包含多个阶段的真实运行拆成多条假记录。多任务运行的主类型及详情展示按[接口合同](unified-admin-contract.md)执行。 + +## 3. 采集配置和首次准备 + +采集配置以抽屉打开，突出两个常用入口： +- 自动探索：状态、简短成果及启停，渠道进度折叠查看；配置表单保留预算档位、排除项，系统维护查询和补漏策略。 +- 指定仓库：已添加目标及「添加仓库」。只要求GitHub仓库URL或owner/repo，目标固定@github；复用已有周期、忽略和容量策略，不重复填写。 + +指定账号、自定义搜索规则放入「其他采集方式」按需展开，保留已有能力，不拆成普通/高级两套配置。系统设置抽屉维护凭证、水位及调度等公共配置；同一配置只有一处维护入口。复杂内容在抽屉中分区，不堆入单一大弹窗。 + +首次进入显示基于真实状态的准备清单：采集服务、容量配置、可选凭证、是否有启用的采集方式。缺项直达具体设置，已配置项无需重复填写，完成后清单收起；无凭证可以匿名运行，需说明额度限制，不能直接算错误。不强制向导，不擅自启用探索，不以自动填入未经确认的水位代替配置。 + +添加仓库保存后由系统自动安排检查，不保证瞬间执行。平台暂停时可以保存，并显示等待平台采集恢复。重复添加关联同一仓库并展示既有状态，不能重复创建或擅自恢复暂停状态。详情只提供实际支持的单仓库设置，不把未实现的逐仓库周期配置说成现有能力。 + +## 4. 统一仓库模型与查询 + +「指定仓库」取代产品文案中的「精选仓库」，表示管理员主动选择，不表示质量认证；既有CURATED_REPOSITORY协议值不因文案更名盲目调整。 + +自动探索搜索命中先作为待验证线索，管理员指定的仓库可先登记正式来源再扫描。两者进入采集后共用包获取、校验、扫描、审核及后续更新流程。发现方式与处理状态分别表达，仓库登记不等于存在有效Skill或已经发布。 + +同namespace内按稳定repository ID关联候选与正式来源，一仓库一行并保留全部命中出处；不同namespace仍独立。列表默认每页20，服务端完成筛选、排序、去重、total和分页，不在浏览器拼接两套分页结果。现有官方探索查询限定官方出处，不能直接作为统一列表使用；须覆盖指定仓库、指定账号、自定义规则及官方探索的适用数据。尚未解析repository ID的指定入口状态应能在配置中看见，不应伪造统一仓库身份。 + +统一查询的字段、稳定排序及关联边界见接口合同；SQL与HTTP性能须在交付验收中测量，不能仅以分页字段存在推断性能。 + +## 5. 平台采集安全暂停 + +按钮固定为「暂停平台采集／恢复平台采集」，仅超级管理员操作，只影响平台@github，TEAM不受此开关控制。TEAM仍受原有共享额度和容量规则约束，不能据此承诺资源完全隔离。 + +- 暂停后停止领取新的平台发现、检查和下载任务。 +- 在途采集到安全检查点停止并持久保存进度，不等待整个大仓库完成。 +- 已上传并被Java接收的包继续校验、扫描及进入待审；人工审核与已发布技能使用不受影响。 +- 展示暂停中与已暂停，不能保存控制字段后立即伪称所有worker停止。 +- 恢复从原进度继续，不启用原本独立停用的入口或来源。 + +决策理由：管理员需要一个控制平台采集负载的入口，但不应以停止平台服务来影响TEAM，也不应丢弃已接收成果。前端批量关闭入口会破坏原启停意图；Go离线状态、自动探索开关和容量暂停都不具备此语义，因此需要独立的持久控制。 + +平台控制API及Java/Go协议已实现，暂停时效、租约/离线判定、安全检查点、已接收边界与代际一致性以[接口合同](unified-admin-contract.md)为准。不得仅因数据库状态更新就宣称在途任务已停止。本轮未加入独立仓库暂停按钮。 + +## 6. 主要使用流程 + +1. 首次配置：打开概览→查看准备清单→补必要配置→启用自动探索或添加指定仓库→系统自动安排工作。 +2. 日常观察：概览确认运行/等待原因→仓库看结果→按需进入运行或审核，不必逐一阅读任务技术日志。 +3. 指定目标：采集配置→添加仓库→系统显示已保存/等待/检查状态→在统一列表查看同一仓库。 +4. 排查异常：问题入口→相关仓库或运行详情→根据真实原因定位配置；自动重试的工作显示恢复条件，不要求反复点击重试。 +5. 暂停恢复：暂停平台采集→暂停中→到安全检查点后已暂停→恢复平台采集；TEAM继续原流程，已接收包继续处理。 + +## 7. 路由、展示及性能约束 + +保留/dashboard/github-sync主路由，侧栏只有技能同步；sourceId/runId旧深链继续可用。旧/dashboard/github-deep-discovery兼容进入主入口并定位自动探索配置/状态。页签、筛选和必要分页状态支持刷新与前进后退；抽屉跳转详情后返回不丢列表位置。 + +复用现有色彩token、小尺寸按钮、Radix抽屉/页签和分页组件，不增加无依据的生产依赖。不直接嵌套带整页标题/轮询的旧页面；抽取内容组件，统一路由负责组织。使用已有生成类型和TanStack Query；详情按需读取，隐藏页签或浏览器隐藏时停轮询，同参数查询共享缓存，禁止全量加载仓库/文件。 + +保留SUPER_ADMIN前后台鉴权及TEAM现有入口，清楚区分空态、无配置、加载失败、离线、限流及容量暂停。更新中英文等现有语言，不仅修改中文菜单。 + +接口与状态设计见[实施合同](unified-admin-contract.md)。 + +## 8. 实施与验收顺序 + +1. 根据已确认产品规则补齐平台暂停与统一仓库查询的接口、持久状态及回退设计；更新现有OpenAPI/MD及spec，不维护另一份正式需求。 +2. 实施并验证服务端合同，再接页面抽取、三页签、配置抽屉及旧路由兼容；保留可独立检查的阶段。 +3. 验证服务端去重/分页/权限/性能、平台暂停与TEAM隔离、在途安全暂停、重启/租约过期/恢复及已接收包继续处理。 +4. 验证1440px/390px和浅深色，首次准备、添加/重复仓库、统一运行列表、错误引导、返回筛选、隐藏页签无多余轮询。区分真实只读与受控接口证据。 +5. Web定向/全量测试、typecheck、lint、build，变更服务端的相关及必要扩大回归；OpenSpec严格校验和diff检查。最终构建与集成验收应绑定交付提交。 + +## 9. 已确认决策索引 + +| 问题 | 用户选择 | 结论 | +|---|---|---| +| Q1 | A | 概览、仓库、运行记录三个日常页签 | +| Q2 | A | 真实状态准备清单，不强制向导 | +| Q3 | A | 增加统一暂停能力，作用范围由Q6限定 | +| Q4 | A | 指定仓库／添加仓库命名 | +| Q5 | A | 默认全部仓库统一列表，后端分页去重 | +| Q6 | 不包括TEAM | 仅暂停平台@github | +| Q7 | A | 在途安全暂停，已接收包继续，恢复原进度 | +| Q8 | A | 突出自动探索/指定仓库，其他方式按需展开 | +| Q9 | A | 添加仓库仅填地址，沿用策略，重复添加不重建 | +| Q10 | A | 真实运行统一列表，按类型和状态筛选 | +| Q11 | A | 概览以运行情况、简短统计及待处理问题为主 | + +原五页签、仅前端整合、全局暂停等早期草案已被以上决定取代。产品决策和技术协议已实施；实际验收范围与未覆盖边界见[验证说明](verification.md)，不将产品确认或局部验证表述为全部场景通过。 diff --git a/openspec/changes/add-github-skill-sync/verification.md b/openspec/changes/add-github-skill-sync/verification.md new file mode 100644 index 000000000..b5dbc7776 --- /dev/null +++ b/openspec/changes/add-github-skill-sync/verification.md @@ -0,0 +1,40 @@ +# GitHub 技能同步验证说明 + +正式预期来自三个 [Spec](specs/) 以及 [统一管理合同](unified-admin-contract.md)。本文件说明可复现的验证入口；运行日志、截图、临时账号和部署方凭证不属于公共文档。历史测试结果不能替代当前提交与当前 main 整合后的验收。 + +## 本地检查 + +从仓库根目录执行已有工具链： + +```bash +make test-backend +make typecheck-web +make lint-web +make test-frontend +make build-frontend +(cd github-sync-go && go test -count=1 ./... && go vet ./...) +openspec validate add-github-skill-sync --strict +bash scripts/tests/validate-release-config-test.sh +bash charts/skillhub/tests/configuration-contracts.sh +``` + +后端定向测试必须包含 `-am`，不能使用本机旧模块产物替代依赖构建。Go 普通测试不访问 GitHub；显式公网用例和 Go–Java HTTP 互操作入口见 [Go README](../../../github-sync-go/README.md)。数据库、MinIO、并发及重启用例需要 Docker；条件跳过必须独立报告，不能计为通过。 + +## 集成验收范围 + +| 范围 | 必须观察的行为 | +| --- | --- | +| 初始化与迁移 | main V1–V75 原 checksum 保持；新 V76–V119 唯一且连续；空库与 main 旧库可升级；保留标识冲突明确拒绝、不接管原数据 | +| 发现与来源 | 指定仓库、指定账号、自定义规则、官方探索；候选按仓库 ID 去重、多个出处保留；TEAM 权限与可见性隔离 | +| 执行与恢复 | 分页和检查点原子、超限与不完整结果保留缺口、树枚举失败隔离、固定 commit、租约接管和旧租约拒绝 | +| 容量与额度 | 包/文件/临时卷上限、水位回落后的当前内容重核、多个 Token 不累加额度、配置池不可用时等待而不回退匿名 | +| 暂停与配置 | 平台暂停不影响 TEAM、在途任务安全确认、已接收包继续处理、探索停用不停止已登记来源、版本冲突和 CSRF 拒绝 | +| 包与审核 | 本地和 S3 模式实际上传、摘要复核、真实扫描、独立同步队列、批准/拒绝、更新待审不替换已发布版本、普通审核行为保持 | +| 浏览器与性能 | 三页签及抽屉、按需正文/差异和缓存、深浅主题与窄屏、大列表服务端分页、响应大小/延迟/数据库执行计划 | +| 安全与升级 | API/日志/指标不泄漏凭证或正文；Java/Go 配套升级能力门；最终镜像与验收提交一致 | + +最终交付应使用隔离的 release Compose、MinIO 与 Scanner，记录实际提交、配置生效、迁移历史、业务断言及清理结果。配置和启动命令见 [管理员手册](../../../docs/github-sync-operations.md)。不要把编译、mock 响应、历史样本或条件跳过当作部署与公网采集通过。 + +## 验证边界 + +ARM64 整镜像、持续公网召回、长期容量、所有旧二进制混跑及反向数据库降级需要各自独立验收；本地功能用例不能给出这些保证。搜索受 GitHub 索引、结果上限和额度限制，不承诺全站穷尽或生产 SLA。完整任务完成状态保留在 [tasks.md](tasks.md)，延期项保持未完成。 diff --git a/scripts/tests/validate-release-config-test.sh b/scripts/tests/validate-release-config-test.sh index c241e3088..4d1370ccc 100755 --- a/scripts/tests/validate-release-config-test.sh +++ b/scripts/tests/validate-release-config-test.sh @@ -71,6 +71,22 @@ write_env "$valid_env" "release-download-secret-32-bytes-minimum" printf '%s\n' "SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true" >>"$valid_env" "$SCRIPT" "$valid_env" >/dev/null +# The S3 client supports the default credential chain when both static keys are absent. +iam_env="$tmp/iam.env" +write_env "$iam_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' 'SKILLHUB_STORAGE_S3_ACCESS_KEY=' 'SKILLHUB_STORAGE_S3_SECRET_KEY=' >>"$iam_env" +"$SCRIPT" "$iam_env" >/dev/null + +partial_s3_key_env="$tmp/partial-s3-key.env" +write_env "$partial_s3_key_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' 'SKILLHUB_STORAGE_S3_SECRET_KEY=' >>"$partial_s3_key_env" +expect_fail "$partial_s3_key_env" "SKILLHUB_STORAGE_S3_SECRET_KEY is required" + +partial_s3_secret_env="$tmp/partial-s3-secret.env" +write_env "$partial_s3_secret_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' 'SKILLHUB_STORAGE_S3_ACCESS_KEY=' >>"$partial_s3_secret_env" +expect_fail "$partial_s3_secret_env" "SKILLHUB_STORAGE_S3_ACCESS_KEY is required" + compose_default_redirect="$tmp/compose-default-redirect.txt" SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET=release-download-secret-32-bytes-minimum \ SKILLHUB_PUBLIC_BASE_URL=https://skillhub.example.com \ diff --git a/scripts/validate-release-config.sh b/scripts/validate-release-config.sh index 356d241da..d2f770356 100755 --- a/scripts/validate-release-config.sh +++ b/scripts/validate-release-config.sh @@ -338,8 +338,12 @@ case "$storage_provider" in s3) require_non_empty SKILLHUB_STORAGE_S3_ENDPOINT require_non_empty SKILLHUB_STORAGE_S3_BUCKET - require_non_empty SKILLHUB_STORAGE_S3_ACCESS_KEY - require_non_empty SKILLHUB_STORAGE_S3_SECRET_KEY + if [ -n "${SKILLHUB_STORAGE_S3_ACCESS_KEY:-}" ] && [ -z "${SKILLHUB_STORAGE_S3_SECRET_KEY:-}" ]; then + error "SKILLHUB_STORAGE_S3_SECRET_KEY is required when SKILLHUB_STORAGE_S3_ACCESS_KEY is set" + fi + if [ -z "${SKILLHUB_STORAGE_S3_ACCESS_KEY:-}" ] && [ -n "${SKILLHUB_STORAGE_S3_SECRET_KEY:-}" ]; then + error "SKILLHUB_STORAGE_S3_ACCESS_KEY is required when SKILLHUB_STORAGE_S3_SECRET_KEY is set" + fi require_non_empty SKILLHUB_STORAGE_S3_REGION validate_url SKILLHUB_STORAGE_S3_ENDPOINT validate_url SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT diff --git a/server/Dockerfile b/server/Dockerfile index 801ced808..eb9026fb0 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -11,13 +11,18 @@ COPY skillhub-search/pom.xml skillhub-search/ COPY skillhub-infra/pom.xml skillhub-infra/ COPY skillhub-storage/pom.xml skillhub-storage/ COPY skillhub-notification/pom.xml skillhub-notification/ +COPY skillhub-github-sync/pom.xml skillhub-github-sync/ COPY skillhub-app/pom.xml skillhub-app/ COPY mvnw . COPY .mvn .mvn -RUN ./mvnw dependency:go-offline -B +RUN --mount=type=secret,id=maven_settings,target=/root/.m2/settings.xml,required=false \ + ./mvnw dependency:go-offline -B COPY . . -RUN ./mvnw package -DskipTests -B +# The shared build context permits target/ for Dockerfile.dev. Recompile +# from source here instead of reusing host build outputs copied above. +RUN --mount=type=secret,id=maven_settings,target=/root/.m2/settings.xml,required=false \ + ./mvnw clean package -DskipTests -B # ---- Runtime Stage ---- # The Noble variant publishes a linux/riscv64 image; the Alpine JRE currently diff --git a/server/pom.xml b/server/pom.xml index 6c59c4643..b177e79b4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -33,6 +33,7 @@ skillhub-storage skillhub-infra skillhub-notification + skillhub-github-sync @@ -67,6 +68,11 @@ skillhub-notification ${project.version} + + com.iflytek.skillhub + skillhub-github-sync + ${project.version} + diff --git a/server/skillhub-app/pom.xml b/server/skillhub-app/pom.xml index 856939bd1..454bb8aa8 100644 --- a/server/skillhub-app/pom.xml +++ b/server/skillhub-app/pom.xml @@ -94,6 +94,10 @@ com.iflytek.skillhub skillhub-notification + + com.iflytek.skillhub + skillhub-github-sync + org.springframework.boot spring-boot-starter-data-jpa diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/AsyncConfig.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/AsyncConfig.java index 10a6f5a5f..4164a2bb2 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/AsyncConfig.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/AsyncConfig.java @@ -4,7 +4,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.support.ContextPropagatingTaskDecorator; import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.util.concurrent.Executor; @@ -16,7 +15,6 @@ import java.util.concurrent.ThreadPoolExecutor; */ @Configuration @EnableAsync -@EnableScheduling public class AsyncConfig { @Bean(name = "skillhubEventExecutor") diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/GithubSyncInternalSecurityConfig.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/GithubSyncInternalSecurityConfig.java new file mode 100644 index 000000000..00b2ff0fe --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/GithubSyncInternalSecurityConfig.java @@ -0,0 +1,102 @@ +package com.iflytek.skillhub.config; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.annotation.Order; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; + +/** A dedicated, stateless machine-identity boundary for the Go acquisition worker. */ +@Configuration +public class GithubSyncInternalSecurityConfig { + @Bean + @Order(-1) + public SecurityFilterChain githubSyncInternalFilterChain( + HttpSecurity http, + @Value("${skillhub.github-sync.service-tokens:}") String configuredTokens) throws Exception { + http.securityMatcher("/internal/github-sync/v1/**") + .csrf(csrf -> csrf.disable()) + .requestCache(cache -> cache.disable()) + .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) + .authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .exceptionHandling(exceptions -> exceptions.authenticationEntryPoint((request, response, error) -> + reject(response))) + .addFilterBefore(new ServiceTokenFilter(configuredTokens), UsernamePasswordAuthenticationFilter.class); + return http.build(); + } + + private static void reject(HttpServletResponse response) throws IOException { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.getWriter().write("{\"code\":\"AUTH_INVALID\",\"message\":\"Invalid service identity\"}"); + } + + private static final class ServiceTokenFilter extends OncePerRequestFilter { + private final List tokenHashes; + + private ServiceTokenFilter(String configuredTokens) { + tokenHashes = Arrays.stream(configuredTokens.split(",", -1)) + .map(String::trim) + .filter(token -> !token.isEmpty()) + .map(token -> { + if (token.length() < 32) { + throw new IllegalArgumentException("GitHub sync service tokens must have at least 32 characters"); + } + return hash(token); + }) + .toList(); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, + FilterChain chain) throws ServletException, IOException { + String authorization = request.getHeader("Authorization"); + if (authorization == null || !authorization.startsWith("Bearer ") + || authorization.length() <= "Bearer ".length()) { + reject(response); + return; + } + byte[] suppliedHash = hash(authorization.substring("Bearer ".length())); + boolean matched = false; + for (byte[] expectedHash : tokenHashes) { + matched |= MessageDigest.isEqual(expectedHash, suppliedHash); + } + if (!matched) { + reject(response); + return; + } + var authentication = new UsernamePasswordAuthenticationToken( + "github-sync-worker", null, + List.of(new SimpleGrantedAuthority("ROLE_GITHUB_SYNC_WORKER"))); + SecurityContextHolder.getContext().setAuthentication(authentication); + chain.doFilter(request, response); + } + + private static byte[] hash(String token) { + try { + return MessageDigest.getInstance("SHA-256").digest(token.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SchedulingConfig.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SchedulingConfig.java new file mode 100644 index 000000000..b5074b862 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SchedulingConfig.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +@Configuration +@ConditionalOnProperty(prefix = "skillhub.scheduling", name = "enabled", + havingValue = "true", matchIfMissing = true) +@EnableScheduling +public class SchedulingConfig { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SkillPublishProperties.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SkillPublishProperties.java index b6440ae5e..f316ea375 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SkillPublishProperties.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/config/SkillPublishProperties.java @@ -11,7 +11,7 @@ import java.util.Set; @ConfigurationProperties(prefix = "skillhub.publish") public class SkillPublishProperties { - private int maxFileCount = 500; + private int maxFileCount = 100; private long maxSingleFileSize = 10 * 1024 * 1024; // 10MB private long maxPackageSize = 100 * 1024 * 1024; private Set allowedFileExtensions = new LinkedHashSet<>(SkillPackagePolicy.ALLOWED_EXTENSIONS); diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialControlController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialControlController.java new file mode 100644 index 000000000..df06814d6 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialControlController.java @@ -0,0 +1,78 @@ +package com.iflytek.skillhub.controller; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import java.time.Instant; +import com.iflytek.skillhub.githubsync.GithubOfficialLifecycleRepository; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import io.swagger.v3.oas.annotations.Hidden; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; +import java.util.UUID; + +@RestController +@Hidden +@RequestMapping("/internal/github-sync/v1/tasks") +public class GithubOfficialControlController { + private final GithubOfficialLifecycleRepository official; + + public GithubOfficialControlController(GithubOfficialLifecycleRepository official) { + this.official = official; + } + + @PostMapping("/{taskId}/official-control") + public ResponseEntity control(@PathVariable UUID taskId, @RequestBody ControlRequest request) { + requireLease(request.lease()); + var control = official.control(taskId, request.lease().generation(), request.lease().token()); + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body( + new ControlResponse(control.action(), control.configurationVersion(), control.reason(), control.retryAt())); + } + + @PostMapping("/{taskId}/official-pause") + public ResponseEntity pause(@PathVariable UUID taskId, @RequestBody PauseRequest request) { + requireLease(request.lease()); + if ("DISCOVERY".equals(request.kind()) && request.repositoryProgress() == null) { + official.pauseDiscovery(taskId, request.lease().generation(), request.lease().token(), request.discoveryCheckpoint()); + } else if ("REPOSITORY_SYNC".equals(request.kind()) && request.discoveryCheckpoint() == null + && request.repositoryProgress() != null) { + official.pauseRepository(taskId, request.lease().generation(), request.lease().token(), request.repositoryProgress()); + } else { + throw new GithubOfficialProtocolException(422, "RESULT_CONFLICT"); + } + return ResponseEntity.ok(Map.of("status", "PAUSED")); + } + + @PostMapping("/{taskId}/official-checkpoint-recovery") + public ResponseEntity recover(@PathVariable UUID taskId, @RequestBody RecoveryRequest request) { + requireLease(request.lease()); + UUID replacement = official.recover(taskId, request.lease().generation(), request.lease().token(), + request.reason(), request.rejectedCheckpoint()); + return ResponseEntity.ok(Map.of("status", "REPLAY_SCHEDULED", "replacementTaskId", replacement)); + } + + private void requireLease(GithubSyncInternalController.LeaseProof lease) { + if (lease == null) { + throw new IllegalArgumentException("lease is required"); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public record ControlResponse(String action, long configurationVersion, String reason, Instant retryAt) {} + + public record ControlRequest(GithubSyncInternalController.LeaseProof lease) {} + public record PauseRequest(GithubSyncInternalController.LeaseProof lease, String kind, + String discoveryCheckpoint, GithubOfficialLifecycleRepository.RepositoryProgress repositoryProgress) { + @JsonAnySetter + public void rejectUnknown(String name, Object value) { + throw new IllegalArgumentException("Unknown official pause field: " + name); + } + } + public record RecoveryRequest(GithubSyncInternalController.LeaseProof lease, String reason, String rejectedCheckpoint) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialProtocolAdvice.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialProtocolAdvice.java new file mode 100644 index 000000000..6b4ea02db --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubOfficialProtocolAdvice.java @@ -0,0 +1,17 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import org.springframework.core.annotation.Order; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@Order(0) +@RestControllerAdvice(assignableTypes = {GithubSyncInternalController.class, GithubOfficialControlController.class, GithubPlatformControlController.class}) +public class GithubOfficialProtocolAdvice { + @ExceptionHandler(GithubOfficialProtocolException.class) + public ResponseEntity officialFailure(GithubOfficialProtocolException exception) { + return ResponseEntity.status(exception.status()).body( + new GithubSyncInternalController.ApiError(exception.code(), exception.code())); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubPlatformControlController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubPlatformControlController.java new file mode 100644 index 000000000..e38b73523 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubPlatformControlController.java @@ -0,0 +1,54 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.GithubOfficialLifecycleRepository; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import com.iflytek.skillhub.githubsync.GithubPlatformControlRepository; +import io.swagger.v3.oas.annotations.Hidden; +import java.util.Map; +import java.util.UUID; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@Hidden +@RequestMapping("/internal/github-sync/v1/tasks") +public class GithubPlatformControlController { + private final GithubPlatformControlRepository controls; + + public GithubPlatformControlController(GithubPlatformControlRepository controls) { + this.controls = controls; + } + + @PostMapping("/{taskId}/platform-control") + public ResponseEntity control(@PathVariable UUID taskId, + @RequestBody ControlRequest body) { + requireLease(body.lease()); + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(controls.control(taskId, body.lease().generation(), body.lease().token())); + } + + @PostMapping("/{taskId}/platform-pause") + public ResponseEntity> pause(@PathVariable UUID taskId, @RequestBody PauseRequest body) { + requireLease(body.lease()); + controls.pause(taskId, body.lease().generation(), body.lease().token(), body.observedVersion(), + body.kind(), body.discoveryCheckpoint(), body.repositoryProgress()); + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(Map.of("status", "PAUSED")); + } + + private void requireLease(GithubSyncInternalController.LeaseProof proof) { + if (proof == null) { + throw new GithubOfficialProtocolException(422, "LEASE_REQUIRED"); + } + } + + public record ControlRequest(GithubSyncInternalController.LeaseProof lease) {} + + public record PauseRequest(GithubSyncInternalController.LeaseProof lease, long observedVersion, String kind, + String discoveryCheckpoint, + GithubOfficialLifecycleRepository.RepositoryProgress repositoryProgress) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalController.java new file mode 100644 index 000000000..eafc1b1ac --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalController.java @@ -0,0 +1,430 @@ +package com.iflytek.skillhub.controller; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubOfficialLifecycleRepository; +import org.springframework.beans.factory.annotation.Autowired; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCandidate; +import com.iflytek.skillhub.githubsync.GithubSyncDeferralReason; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.githubsync.GithubPackageAdmissionResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryPathResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryMetadataResult; +import com.iflytek.skillhub.githubsync.GithubSyncFailureReason; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskLease; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import io.swagger.v3.oas.annotations.Hidden; +import org.springframework.http.ResponseEntity; +import org.springframework.http.CacheControl; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +@RestController +@Hidden +@RequestMapping("/internal/github-sync/v1/tasks") +public class GithubSyncInternalController { + private final GithubSyncLeaseService leases; + private final GithubCredentialLeaseAppService credentialLeases; + private final ObjectMapper objectMapper; + private final SkillPublishProperties publishProperties; + private final GithubOfficialLifecycleRepository official; + + public GithubSyncInternalController(GithubSyncLeaseService leases, + GithubCredentialLeaseAppService credentialLeases, + ObjectMapper objectMapper, + SkillPublishProperties publishProperties) { + this(leases, credentialLeases, objectMapper, publishProperties, null); + } + + @Autowired + public GithubSyncInternalController(GithubSyncLeaseService leases, + GithubCredentialLeaseAppService credentialLeases, + ObjectMapper objectMapper, + SkillPublishProperties publishProperties, + GithubOfficialLifecycleRepository official) { + this.official = official; + this.leases = leases; + this.credentialLeases = credentialLeases; + this.objectMapper = objectMapper; + this.publishProperties = publishProperties; + } + + @PostMapping("/claim") + public ResponseEntity claim(@RequestBody ClaimRequest request) { + if (request.protocolMajor() == null) { + throw new IllegalArgumentException("protocolMajor is required"); + } + if (request.protocolMajor() != 1) { + return ResponseEntity.status(426) + .body(new ApiError("PROTOCOL_UNSUPPORTED", "Unsupported protocol major")); + } + var claimed = request.capabilities() == null || request.capabilities().isEmpty() + ? credentialLeases.claim(request.kind(), request.workerId()) + : credentialLeases.claim(request.kind(), request.workerId(), request.capabilities()); + return claimed + .>map(task -> ResponseEntity.ok() + .cacheControl(CacheControl.noStore()).body(toResponse(task))) + .orElseGet(() -> ResponseEntity.noContent().build()); + } + + @PostMapping("/{taskId}/renew") + public ResponseEntity renew(@PathVariable UUID taskId, @RequestBody RenewRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + return leases.renew(taskId, request.lease().generation(), request.lease().token()) + .>map(expiresAt -> ResponseEntity.ok(new RenewResponse(expiresAt))) + .orElseGet(() -> ResponseEntity.status(409) + .body(new ApiError("LEASE_STALE", "Lease is no longer current"))); + } + + @PostMapping("/{taskId}/repository-metadata") + public ResponseEntity repositoryMetadata(@PathVariable UUID taskId, + @RequestBody RepositoryMetadataRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + GithubRepositoryMetadataResult result = leases.updateRepositoryMetadata(taskId, + request.lease().generation(), request.lease().token(), request.repositoryId(), + request.owner(), request.name(), request.defaultBranch(), request.ownerId()); + return switch (result) { + case UPDATED -> ResponseEntity.noContent().build(); + case LEASE_STALE -> staleLease(); + case REPOSITORY_MISMATCH -> ResponseEntity.status(409) + .body(new ApiError("REPOSITORY_MISMATCH", "GitHub repository ID differs from task source")); + }; + } + + @PostMapping("/{taskId}/credential-status") + public ResponseEntity credentialStatus(@PathVariable UUID taskId, + @RequestBody CredentialStatusRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + return credentialLeases.reportStatus(taskId, request.lease().generation(), + request.lease().token(), request.credentialId(), request.status(), request.retryAt(), + request.githubQuota() == null ? null : new GithubCredentialLeaseAppService.Quota( + request.githubQuota().resource(), request.githubQuota().remaining(), + request.githubQuota().resetAt()), request.officialQuotaObservation()) + ? ResponseEntity.noContent().build() : staleLease(); + } + + @PostMapping("/{taskId}/official-quota") + public ResponseEntity officialQuota(@PathVariable UUID taskId, @RequestBody OfficialQuotaRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + return credentialLeases.reportOfficialQuota(taskId, request.lease().generation(), request.lease().token(), request.observation()) + ? ResponseEntity.noContent().build() : staleLease(); + } + + public record OfficialQuotaRequest(LeaseProof lease, + com.iflytek.skillhub.githubsync.GithubOfficialQuotaRepository.Observation observation) {} + + @PostMapping("/{taskId}/package-admissions") + public ResponseEntity admitPackage(@PathVariable UUID taskId, @RequestBody PackageAdmissionRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + GithubPackageAdmissionResult result = leases.admitPackage(taskId, request.lease().generation(), + request.lease().token(), request.commitSha(), request.skillPath()); + return switch (result.status()) { + case ADMITTED -> ResponseEntity.ok(new PackageAdmissionResponse(result.admissionId())); + case LEASE_STALE -> staleLease(); + case COMMIT_MISMATCH -> ResponseEntity.status(409) + .body(new ApiError("COMMIT_MISMATCH", "Task is pinned to a different commit")); + case BACKPRESSURE_ACTIVE -> ResponseEntity.status(409) + .body(new ApiError("BACKPRESSURE_ACTIVE", "Package acquisition is paused")); + case PATH_IGNORED -> ResponseEntity.status(409) + .body(new ApiError("PATH_IGNORED", "Skill path is ignored by source policy")); + }; + } + + @PostMapping("/{taskId}/discovered-repositories") + public ResponseEntity reportDiscoveryBatch(@PathVariable UUID taskId, + @RequestBody DiscoveryBatchRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + GithubDiscoveryBatch batch = new GithubDiscoveryBatch(request.previousCheckpoint(), + request.checkpoint(), request.repositories(), + request.deferredCount() == null ? 0 : request.deferredCount(), request.deferralReason()); + if (request.officialBatch() != null) { + official.report(taskId, request.lease().generation(), request.lease().token(), batch, request.officialBatch()); + return ResponseEntity.accepted().build(); + } + if (batch.repositories().stream().anyMatch(candidate -> candidate.officialProvenance() != null)) { + return ResponseEntity.status(409).body(new ApiError("RESULT_CONFLICT", "Official provenance requires an official batch")); + } + return switch (leases.reportDiscoveryBatch(taskId, request.lease().generation(), + request.lease().token(), batch)) { + case ACCEPTED -> ResponseEntity.accepted().build(); + case LEASE_STALE -> ResponseEntity.status(409) + .body(new ApiError("LEASE_STALE", "Lease is no longer current")); + case CHECKPOINT_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("CHECKPOINT_CONFLICT", "Discovery checkpoint has advanced")); + case RESULT_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("RESULT_CONFLICT", "Discovery batch conflicts with a prior report")); + case BUDGET_EXCEEDED -> ResponseEntity.unprocessableEntity() + .body(new ApiError("CANDIDATE_BUDGET_EXCEEDED", "Discovery candidate budget exceeded")); + }; + } + + @PostMapping("/{taskId}/preview-results") + public ResponseEntity completePreview(@PathVariable UUID taskId, + @RequestBody PreviewResultRequest request) { + if (request.lease() == null || request.selectors() == null + || request.selectors().isEmpty() || request.selectors().size() > 20) { + throw new IllegalArgumentException("preview lease and 1 to 20 selector results are required"); + } + for (PreviewSelectorResult selector : request.selectors()) { + if (selector == null || selector.kind() == null + || !List.of("KEYWORD", "TOPIC").contains(selector.kind()) + || selector.value() == null || selector.value().isBlank() + || selector.priority() == null + || !List.of("CORE", "EXTENSION").contains(selector.priority()) + || selector.totalCount() < 0 || selector.samples() == null + || selector.samples().size() > 5) { + throw new IllegalArgumentException("invalid preview selector result"); + } + for (GithubDiscoveryCandidate sample : selector.samples()) { + if (sample == null || sample.repositoryId() <= 0 + || sample.owner() == null || sample.name() == null) { + throw new IllegalArgumentException("invalid preview repository sample"); + } + } + } + String result; + try { + result = objectMapper.writeValueAsString(java.util.Map.of("selectors", request.selectors())); + } catch (JsonProcessingException exception) { + throw new IllegalArgumentException("invalid preview result", exception); + } + return leases.completePreview(taskId, request.lease().generation(), request.lease().token(), result) + ? ResponseEntity.ok(new CompleteResponse("ACQUISITION_COMPLETE")) : staleLease(); + } + + @PostMapping("/{taskId}/complete") + public ResponseEntity complete(@PathVariable UUID taskId, @RequestBody CompleteRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + if ("HEAD_ONLY".equals(request.coverage())) { + if (request.pathResults() != null || request.incompleteSubtrees() != null + || request.pathCheckpoint() != null || request.discoveryCheckpoint() != null + || request.retryAt() != null || request.deferralReason() != null + || request.githubQuota() != null) { + return unsupportedCoverage(); + } + if (leases.completeHeadOnly(taskId, request.lease().generation(), + request.lease().token(), request.observedCommitSha())) { + return ResponseEntity.ok(new CompleteResponse("ACQUISITION_COMPLETE")); + } + return staleLease(); + } + if ("COMPLETE".equals(request.coverage()) || "INCOMPLETE".equals(request.coverage())) { + if (request.githubQuota() != null) { + return unsupportedCoverage(); + } + if (request.discoveryCheckpoint() != null) { + if (request.observedCommitSha() != null || request.pathResults() != null + || request.incompleteSubtrees() != null || request.pathCheckpoint() != null) { + return unsupportedCoverage(); + } + if (official != null && official.isOfficialDiscovery(taskId)) { + return ResponseEntity.ok(new CompleteResponse(official.complete(taskId, + request.lease().generation(), request.lease().token(), request.discoveryCheckpoint(), + request.coverage(), request.retryAt()))); + } + return switch (leases.completeDiscovery(taskId, request.lease().generation(), + request.lease().token(), request.discoveryCheckpoint(), + "COMPLETE".equals(request.coverage()), request.retryAt(), request.deferralReason())) { + case ACQUISITION_COMPLETE -> ResponseEntity.ok(new CompleteResponse("ACQUISITION_COMPLETE")); + case RETRY_WAIT -> ResponseEntity.ok(new CompleteResponse("RETRY_WAIT")); + case LEASE_STALE -> staleLease(); + case CHECKPOINT_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("CHECKPOINT_CONFLICT", "Discovery checkpoint has advanced")); + }; + } + return switch (leases.completeRepository(taskId, request.lease().generation(), + request.lease().token(), request.observedCommitSha(), + "COMPLETE".equals(request.coverage()), request.pathCheckpoint(), + request.incompleteSubtrees(), request.pathResults(), request.retryAt(), + request.deferralReason())) { + case ACQUISITION_COMPLETE -> ResponseEntity.ok(new CompleteResponse("ACQUISITION_COMPLETE")); + case PARTIAL -> ResponseEntity.ok(new CompleteResponse("PARTIAL")); + case RETRY_WAIT -> ResponseEntity.ok(new CompleteResponse("RETRY_WAIT")); + case LEASE_STALE -> staleLease(); + case COMMIT_MISMATCH -> ResponseEntity.status(409) + .body(new ApiError("COMMIT_MISMATCH", "Task is pinned to a different commit")); + case CHECKPOINT_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("CHECKPOINT_CONFLICT", "Path checkpoint has advanced")); + case RESULT_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("RESULT_CONFLICT", "Path results conflict with persisted acquisition")); + case COVERAGE_CONFLICT -> ResponseEntity.status(409) + .body(new ApiError("COVERAGE_CONFLICT", "Repository acquisition remains incomplete")); + }; + } + return unsupportedCoverage(); + } + + private ResponseEntity staleLease() { + return ResponseEntity.status(409) + .body(new ApiError("LEASE_STALE", "Lease is no longer current")); + } + + private ResponseEntity unsupportedCoverage() { + return ResponseEntity.unprocessableEntity() + .body(new ApiError("COVERAGE_UNSUPPORTED", "This completion coverage is not implemented")); + } + + @PostMapping("/{taskId}/fail") + public ResponseEntity fail(@PathVariable UUID taskId, @RequestBody FailRequest request) { + if (request.lease() == null || request.reason() == null) { + throw new IllegalArgumentException("lease and reason are required"); + } + return leases.fail(taskId, request.lease().generation(), request.lease().token(), + request.reason(), request.detail(), request.githubQuota() == null ? null + : new GithubSyncLeaseService.Quota( + request.githubQuota().resource(), + request.githubQuota().remaining(), + request.githubQuota().resetAt()), request.retryAt()) + .>map(retryAt -> ResponseEntity.ok(new FailResponse("RETRY_WAIT", retryAt))) + .orElseGet(() -> ResponseEntity.status(409) + .body(new ApiError("LEASE_STALE", "Lease is no longer current"))); + } + + private ClaimedTaskResponse toResponse(GithubCredentialLeaseAppService.Claimed claimed) { + GithubSyncLeaseService.LeasedTask leased = claimed.lease(); + GithubSyncTaskLease task = leased.task(); + JsonNode snapshot; + try { + snapshot = objectMapper.readTree(task.snapshotJson()); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted GitHub sync task snapshot", exception); + } + boolean official = snapshot.has("officialDiscovery"); + if (official) { + snapshot = snapshot.get("officialDiscovery"); + } + if (task.kind() == GithubSyncTaskKind.DISCOVERY && snapshot instanceof ObjectNode discovery) { + ObjectNode discoveryCopy = discovery.deepCopy(); + if (task.discoveryCheckpoint() == null) { + discoveryCopy.remove("checkpoint"); + } else { + discoveryCopy.put("checkpoint", task.discoveryCheckpoint()); + } + snapshot = discoveryCopy; + } + if (task.kind() == GithubSyncTaskKind.REPOSITORY_SYNC && task.fixedCommitSha() != null + && snapshot instanceof ObjectNode repository) { + ObjectNode repositoryCopy = repository.deepCopy(); + repositoryCopy.put("fixedCommitSha", task.fixedCommitSha()); + if (task.pathCheckpoint() != null) { + repositoryCopy.put("pathCheckpoint", task.pathCheckpoint()); + } + try { + repositoryCopy.set("incompleteSubtrees", objectMapper.readTree(task.incompleteSubtreesJson())); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted incomplete GitHub subtrees", exception); + } + var retryPaths = repositoryCopy.putArray("retryPaths"); + task.retryPaths().forEach(retryPaths::add); + var acceptedPaths = repositoryCopy.putArray("acceptedPaths"); + task.acceptedPaths().forEach(acceptedPaths::add); + snapshot = repositoryCopy; + } + return new ClaimedTaskResponse( + task.taskId(), task.runId(), task.kind().name(), + task.mode() == null ? null : task.mode().name(), + task.namespaceId(), task.sourceId(), + new LeaseProof(task.leaseGeneration(), leased.token()), task.leaseExpiresAt(), + task.kind() == GithubSyncTaskKind.DISCOVERY && !official ? snapshot : null, + task.kind() == GithubSyncTaskKind.REPOSITORY_SYNC ? snapshot : null, + claimed.credential() == null ? null : new CredentialResponse( + claimed.credential().credentialId(), claimed.credential().token()), + publishProperties.getMaxPackageSize(), publishProperties.getMaxSingleFileSize(), + publishProperties.getMaxFileCount(), publishProperties.getMaxPackageSize(), + official ? snapshot : null, + task.requiredCapabilities().isEmpty() ? null : task.requiredCapabilities(), task.platformControlVersion()); + } + + public record ClaimRequest(String workerId, Integer protocolMajor, GithubSyncTaskKind kind, + List capabilities) { + public ClaimRequest(String workerId, Integer protocolMajor, GithubSyncTaskKind kind) { + this(workerId, protocolMajor, kind, null); + } + } + public record LeaseProof(long generation, String token) {} + public record RepositoryMetadataRequest(LeaseProof lease, long repositoryId, + String owner, String name, String defaultBranch, Long ownerId) { + public RepositoryMetadataRequest(LeaseProof lease, long repositoryId, String owner, String name, String defaultBranch) { + this(lease, repositoryId, owner, name, defaultBranch, null); + } + } + public record RenewRequest(LeaseProof lease) {} + public record RenewResponse(Instant leaseExpiresAt) {} + public record CredentialStatusRequest(LeaseProof lease, UUID credentialId, + String status, Instant retryAt, CredentialQuotaRequest githubQuota, + com.iflytek.skillhub.githubsync.GithubOfficialQuotaRepository.Observation officialQuotaObservation) { + public CredentialStatusRequest(LeaseProof lease, UUID credentialId, String status, Instant retryAt, CredentialQuotaRequest githubQuota) { + this(lease, credentialId, status, retryAt, githubQuota, null); + } + } + public record CredentialQuotaRequest(String resource, int remaining, Instant resetAt) {} + public record PackageAdmissionRequest(LeaseProof lease, String commitSha, String skillPath) {} + public record PackageAdmissionResponse(UUID admissionId) {} + public record DiscoveryBatchRequest(LeaseProof lease, List repositories, + String previousCheckpoint, String checkpoint, Integer deferredCount, + GithubSyncDeferralReason deferralReason, GithubOfficialDiscoveryBatch officialBatch) { + public DiscoveryBatchRequest(LeaseProof lease, List repositories, + String previousCheckpoint, String checkpoint, Integer deferredCount, GithubSyncDeferralReason deferralReason) { + this(lease, repositories, previousCheckpoint, checkpoint, deferredCount, deferralReason, null); + } + } + public record PreviewSelectorResult(String kind, String value, String priority, int totalCount, + boolean incompleteResults, boolean limitReached, + List samples) {} + public record PreviewResultRequest(LeaseProof lease, List selectors) {} + public record CompleteRequest(LeaseProof lease, String coverage, String observedCommitSha, + List incompleteSubtrees, String pathCheckpoint, + List pathResults, + String discoveryCheckpoint, GithubSyncDeferralReason deferralReason, + JsonNode githubQuota, Instant retryAt) {} + public record CompleteResponse(String status) {} + public record FailRequest(LeaseProof lease, GithubSyncFailureReason reason, Instant retryAt, + CredentialQuotaRequest githubQuota, String detail) {} + public record FailResponse(String status, Instant retryAt) {} + public record ApiError(String code, String message) {} + public record CredentialResponse(UUID credentialId, String token) {} + @JsonInclude(JsonInclude.Include.NON_NULL) + public record ClaimedTaskResponse( + UUID taskId, UUID runId, String kind, String mode, long namespaceId, Long sourceId, + LeaseProof lease, Instant leaseExpiresAt, JsonNode discovery, JsonNode repository, + CredentialResponse credential, + long maxPackageBytes, long maxSingleFileBytes, int maxFilesPerPackage, + long maxUploadBytes, JsonNode officialDiscovery, List requiredCapabilities, Long platformControlVersion) { + public ClaimedTaskResponse(UUID taskId, UUID runId, String kind, String mode, long namespaceId, + Long sourceId, LeaseProof lease, Instant leaseExpiresAt, JsonNode discovery, + JsonNode repository, CredentialResponse credential, long maxPackageBytes, + long maxSingleFileBytes, int maxFilesPerPackage, long maxUploadBytes) { + this(taskId, runId, kind, mode, namespaceId, sourceId, lease, leaseExpiresAt, discovery, + repository, credential, maxPackageBytes, maxSingleFileBytes, maxFilesPerPackage, + maxUploadBytes, null, null, null); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalErrorHandler.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalErrorHandler.java new file mode 100644 index 000000000..c05340e64 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncInternalErrorHandler.java @@ -0,0 +1,20 @@ +package com.iflytek.skillhub.controller; + +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; + +@RestControllerAdvice(assignableTypes = {GithubSyncInternalController.class, GithubSyncUploadController.class, GithubPlatformControlController.class}) +@Order(Ordered.HIGHEST_PRECEDENCE) +public class GithubSyncInternalErrorHandler { + @ExceptionHandler({IllegalArgumentException.class, HttpMessageNotReadableException.class, + MethodArgumentTypeMismatchException.class}) + public ResponseEntity badRequest(Exception ignored) { + return ResponseEntity.badRequest() + .body(new GithubSyncInternalController.ApiError("INVALID_REQUEST", "Invalid request")); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncUploadController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncUploadController.java new file mode 100644 index 000000000..2b4c67db1 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncUploadController.java @@ -0,0 +1,103 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.GithubPackageReportResult; +import com.iflytek.skillhub.service.GithubSyncUploadAppService; +import io.swagger.v3.oas.annotations.Hidden; +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.util.UUID; + +@RestController +@Hidden +@RequestMapping("/internal/github-sync/v1") +public class GithubSyncUploadController { + private final GithubSyncUploadAppService uploads; + + public GithubSyncUploadController(GithubSyncUploadAppService uploads) { + this.uploads = uploads; + } + + @PostMapping("/tasks/{taskId}/upload-authorizations") + public ResponseEntity authorize(@PathVariable UUID taskId, + @RequestBody UploadAuthorizationRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + try { + return uploads.authorize(taskId, request.lease().generation(), request.lease().token(), + request.admissionId(), request.commitSha(), request.skillPath(), + request.sizeBytes(), request.sha256()) + .>map(authorization -> ResponseEntity.ok() + .cacheControl(CacheControl.noStore()).body(authorization)) + .orElseGet(() -> ResponseEntity.status(409) + .body(new GithubSyncInternalController.ApiError("ADMISSION_STALE", + "Package admission is no longer current"))); + } catch (GithubSyncUploadAppService.UploadTooLargeException exception) { + return ResponseEntity.status(413) + .body(new GithubSyncInternalController.ApiError("PACKAGE_TOO_LARGE", + "Package exceeds the active upload size limit")); + } + } + + @PutMapping(value = "/uploads/{uploadId}/content", consumes = "application/octet-stream") + public ResponseEntity uploadLocal(@PathVariable UUID uploadId, + @RequestHeader(value = "X-Upload-Ticket", required = false) String ticket, + HttpServletRequest request) throws IOException { + return switch (uploads.uploadLocal(uploadId, ticket, request.getInputStream())) { + case UPLOADED -> ResponseEntity.noContent().build(); + case TICKET_INVALID -> ResponseEntity.status(401) + .body(new GithubSyncInternalController.ApiError("UPLOAD_TICKET_INVALID", + "Upload ticket is invalid or expired")); + case TOO_LARGE -> ResponseEntity.status(413) + .body(new GithubSyncInternalController.ApiError("PACKAGE_TOO_LARGE", + "Package exceeds the authorized size")); + case CONTENT_MISMATCH -> ResponseEntity.unprocessableEntity() + .body(new GithubSyncInternalController.ApiError("UPLOAD_MISMATCH", + "Uploaded package size or digest does not match")); + case LEASE_STALE -> ResponseEntity.status(409) + .body(new GithubSyncInternalController.ApiError("LEASE_STALE", + "Lease is no longer current")); + }; + } + + @PostMapping("/tasks/{taskId}/packages/report") + public ResponseEntity report(@PathVariable UUID taskId, @RequestBody PackageReportRequest request) { + if (request.lease() == null) { + throw new IllegalArgumentException("lease is required"); + } + GithubPackageReportResult result = uploads.report(taskId, request.lease().generation(), + request.lease().token(), request.admissionId(), request.commitSha(), + request.skillPath(), request.uploadId(), request.sizeBytes(), request.sha256()); + return switch (result.status()) { + case RECEIVED -> ResponseEntity.accepted().body(new PackageReportResponse(result.ingestionId(), "RECEIVED")); + case LEASE_STALE -> ResponseEntity.status(409) + .body(new GithubSyncInternalController.ApiError("LEASE_STALE", "Lease is no longer current")); + case RESULT_CONFLICT -> ResponseEntity.status(409) + .body(new GithubSyncInternalController.ApiError("RESULT_CONFLICT", "Package report conflicts with persisted state")); + case UPLOAD_NOT_READY, OBJECT_MISSING -> ResponseEntity.status(409) + .body(new GithubSyncInternalController.ApiError("UPLOAD_NOT_READY", "Staged package is not ready")); + case OBJECT_MISMATCH -> ResponseEntity.unprocessableEntity() + .body(new GithubSyncInternalController.ApiError("UPLOAD_MISMATCH", "Staged package size differs")); + case TOO_LARGE -> ResponseEntity.status(413) + .body(new GithubSyncInternalController.ApiError("PACKAGE_TOO_LARGE", "Staged package exceeds the active size limit")); + case READY_TO_CHECK_OBJECT -> throw new IllegalStateException("Package report was not settled"); + }; + } + + public record UploadAuthorizationRequest(GithubSyncInternalController.LeaseProof lease, + UUID admissionId, String commitSha, String skillPath, long sizeBytes, String sha256) {} + public record PackageReportRequest(GithubSyncInternalController.LeaseProof lease, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256) {} + public record PackageReportResponse(UUID ingestionId, String status) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncWorkerHeartbeatController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncWorkerHeartbeatController.java new file mode 100644 index 000000000..0090cca75 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/GithubSyncWorkerHeartbeatController.java @@ -0,0 +1,35 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import com.iflytek.skillhub.dto.GithubSyncWorkerHeartbeatRequest; +import io.swagger.v3.oas.annotations.Hidden; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@Hidden +@RequestMapping("/internal/github-sync/v1") +public class GithubSyncWorkerHeartbeatController { + private final GithubSyncWorkerPresenceRepository presence; + + public GithubSyncWorkerHeartbeatController(GithubSyncWorkerPresenceRepository presence) { + this.presence = presence; + } + + @PostMapping("/heartbeat") + public ResponseEntity heartbeat(@RequestBody(required = false) GithubSyncWorkerHeartbeatRequest request) { + if (request == null || request.workerId() == null) { + presence.heartbeat(); + } else { + presence.heartbeat(request.workerId(), request.protocolMajor(), request.capabilities()); + } + return ResponseEntity.noContent().build(); + } + + public ResponseEntity heartbeat() { + return heartbeat(null); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubCredentialController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubCredentialController.java new file mode 100644 index 000000000..adcc6b99b --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubCredentialController.java @@ -0,0 +1,80 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubCredentialEnabledRequest; +import com.iflytek.skillhub.dto.GithubCredentialResponse; +import com.iflytek.skillhub.dto.GithubCredentialUpsertRequest; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubCredentialAdminAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.UUID; + +@RestController +@RequestMapping("/api/v1/admin/github-sync/credentials") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name = "GitHub Sync Admin") +public class AdminGithubCredentialController extends BaseApiController { + private final GithubCredentialAdminAppService credentials; + + public AdminGithubCredentialController(ApiResponseFactory responseFactory, + GithubCredentialAdminAppService credentials) { + super(responseFactory); + this.credentials = credentials; + } + + @GetMapping + @Operation(summary = "查看系统级公开 GitHub 凭证的脱敏状态") + public ResponseEntity>> list() { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.read", credentials.list())); + } + + @PostMapping + @Operation(summary = "加密保存系统级公开 GitHub 凭证") + public ResponseEntity> create( + @Valid @RequestBody GithubCredentialUpsertRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.created", credentials.create( + body, principal.userId(), AuditRequestContext.from(request)))); + } + + @PutMapping("/{id}") + @Operation(summary = "替换系统级公开 GitHub 凭证并恢复健康状态") + public ResponseEntity> replace( + @PathVariable UUID id, @Valid @RequestBody GithubCredentialUpsertRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.updated", credentials.replace( + id, body, principal.userId(), AuditRequestContext.from(request)))); + } + + @PutMapping("/{id}/enabled") + @Operation(summary = "启用或停用系统级公开 GitHub 凭证") + public ResponseEntity> setEnabled( + @PathVariable UUID id, @Valid @RequestBody GithubCredentialEnabledRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.updated", credentials.setEnabled( + id, body.enabled(), principal.userId(), AuditRequestContext.from(request)))); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryController.java new file mode 100644 index 000000000..bdad522cf --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryController.java @@ -0,0 +1,54 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryConfigurationRequest; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryConfigurationResponse; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubDeepDiscoveryConfigurationAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/admin/github-sync/deep-discovery/configuration") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name = "GitHub Sync Admin") +public class AdminGithubDeepDiscoveryController extends BaseApiController { + private final GithubDeepDiscoveryConfigurationAppService configurations; + + public AdminGithubDeepDiscoveryController(ApiResponseFactory responseFactory, + GithubDeepDiscoveryConfigurationAppService configurations) { + super(responseFactory); + this.configurations = configurations; + } + + @GetMapping + @Operation(summary = "读取官方 GitHub 深度探索配置", description = "配置启用与采集服务在线及执行状态独立") + public ResponseEntity> get() { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.read", configurations.get())); + } + + @PutMapping + @Operation(summary = "按配置版本更新官方 GitHub 深度探索设置") + public ResponseEntity> update( + @Valid @RequestBody GithubDeepDiscoveryConfigurationRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()) + .body(ok("response.success.updated", configurations.update( + body, principal.userId(), AuditRequestContext.from(request)))); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryReadController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryReadController.java new file mode 100644 index 000000000..7f4e0b547 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDeepDiscoveryReadController.java @@ -0,0 +1,59 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.service.GithubDeepDiscoveryReadAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.time.Instant; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/api/v1/admin/github-sync/deep-discovery") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name="GitHub Sync Admin") +public class AdminGithubDeepDiscoveryReadController extends BaseApiController { + private final GithubDeepDiscoveryReadAppService reads; + public AdminGithubDeepDiscoveryReadController(ApiResponseFactory factory,GithubDeepDiscoveryReadAppService reads) { + super(factory);this.reads=reads; + } + @GetMapping("/summary") @Operation(summary="查看官方探索近期成果与当前积压") + public ResponseEntity> summary( + @RequestParam @DateTimeFormat(iso=DateTimeFormat.ISO.DATE_TIME) Instant from, + @RequestParam @DateTimeFormat(iso=DateTimeFormat.ISO.DATE_TIME) Instant until) { return response(reads.summary(from,until)); } + @GetMapping("/progress") @Operation(summary="查看官方探索渠道进度与等待原因") + public ResponseEntity> progress() { return response(reads.progress()); } + @GetMapping("/repositories") @Operation(summary="分页查询官方探索命中的仓库") + public ResponseEntity>> repositories( + @RequestParam(defaultValue="0") int page,@RequestParam(defaultValue="20") int size, + @RequestParam(required=false) @DateTimeFormat(iso=DateTimeFormat.ISO.DATE_TIME) Instant asOf, + @RequestParam(required=false) String q,@RequestParam(required=false) RepositoryStatus status, + @RequestParam(required=false) Channel channel,@RequestParam(required=false) Boolean paused) { + return response(reads.repositories(page,size,asOf,q,status,channel,paused)); + } + @GetMapping("/repositories/{repositoryId}") @Operation(summary="按需读取仓库检查摘要") + public ResponseEntity> repository(@PathVariable long repositoryId) { + return response(reads.repository(repositoryId)); + } + @GetMapping("/repositories/{repositoryId}/provenance") @Operation(summary="分页查询仓库命中出处") + public ResponseEntity>> provenance(@PathVariable long repositoryId, + @RequestParam(defaultValue="0") int page,@RequestParam(defaultValue="20") int size, + @RequestParam(required=false) @DateTimeFormat(iso=DateTimeFormat.ISO.DATE_TIME) Instant asOf) { + return response(reads.provenance(repositoryId,page,size,asOf)); + } + @GetMapping("/repositories/{repositoryId}/paths") @Operation(summary="分页查询仓库技能路径最新结果") + public ResponseEntity>> paths(@PathVariable long repositoryId, + @RequestParam(defaultValue="0") int page,@RequestParam(defaultValue="20") int size, + @RequestParam(required=false) @DateTimeFormat(iso=DateTimeFormat.ISO.DATE_TIME) Instant asOf) { + return response(reads.paths(repositoryId,page,size,asOf)); + } + private ResponseEntity> response(T value) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(ok("response.success.read",value)); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDiscoveryController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDiscoveryController.java new file mode 100644 index 000000000..1f25eff07 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubDiscoveryController.java @@ -0,0 +1,119 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubCuratedOwnerRequest; +import com.iflytek.skillhub.dto.GithubCuratedRepositoryRequest; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryEnabledRequest; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryResponse; +import com.iflytek.skillhub.dto.GithubSearchRuleRequest; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubDiscoveryAdminAppService; +import com.iflytek.skillhub.service.GithubSearchPreviewAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.UUID; + +@RestController +@RequestMapping("/api/v1/admin/github-sync/discovery-entries") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name = "GitHub Sync Admin") +public class AdminGithubDiscoveryController extends BaseApiController { + private final GithubDiscoveryAdminAppService discovery; + private final GithubSearchPreviewAppService previews; + + public AdminGithubDiscoveryController(ApiResponseFactory responseFactory, + GithubDiscoveryAdminAppService discovery, + GithubSearchPreviewAppService previews) { + super(responseFactory); + this.discovery = discovery; + this.previews = previews; + } + + @GetMapping + @Operation(summary = "查看 GitHub 发现规则与精选入口") + public ApiResponse> list() { + return ok("response.success.read", discovery.list()); + } + + @PostMapping("/search-rules") + @Operation(summary = "创建初始停用的仓库搜索规则") + public ApiResponse createSearchRule( + @Valid @RequestBody GithubSearchRuleRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.created", discovery.createSearchRule( + body, principal.userId(), AuditRequestContext.from(request))); + } + + @PostMapping("/search-rules/preview") + @Operation(summary = "预览搜索规则的仓库命中情况，不登记来源") + public ApiResponse previewSearchRule( + @Valid @RequestBody GithubSearchRuleRequest body, + @AuthenticationPrincipal PlatformPrincipal principal) { + return ok("response.success.created", previews.queue(body, principal.userId())); + } + + @GetMapping("/search-rules/previews/{id}") + @Operation(summary = "查看搜索规则预览结果") + public ApiResponse searchRulePreview(@PathVariable UUID id) { + return ok("response.success.read", previews.get(id)); + } + + @PutMapping("/search-rules/{id}") + @Operation(summary = "编辑仓库搜索规则并生成新版本") + public ApiResponse updateSearchRule( + @PathVariable UUID id, + @Valid @RequestBody GithubSearchRuleRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.updated", discovery.updateSearchRule( + id, body, principal.userId(), AuditRequestContext.from(request))); + } + + @PostMapping("/curated-owners") + @Operation(summary = "登记精选 GitHub 用户或组织") + public ApiResponse createCuratedOwner( + @Valid @RequestBody GithubCuratedOwnerRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.created", discovery.createCuratedOwner( + body, principal.userId(), AuditRequestContext.from(request))); + } + + @PostMapping("/curated-repositories") + @Operation(summary = "登记精选 GitHub 公开仓库") + public ApiResponse createCuratedRepository( + @Valid @RequestBody GithubCuratedRepositoryRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.created", discovery.createCuratedRepository( + body, principal.userId(), AuditRequestContext.from(request))); + } + + @PutMapping("/{id}/enabled") + @Operation(summary = "启用或停用 GitHub 发现入口") + public ApiResponse setEnabled( + @PathVariable UUID id, + @Valid @RequestBody GithubDiscoveryEntryEnabledRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.updated", discovery.setEnabled( + id, body.enabled(), principal.userId(), AuditRequestContext.from(request))); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubPlatformControlController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubPlatformControlController.java new file mode 100644 index 000000000..bc807d5c3 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubPlatformControlController.java @@ -0,0 +1,64 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.githubsync.GithubPlatformControlRepository; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubPlatformControlAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.time.Instant; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Tag(name = "GitHub Sync Admin") +@RestController +@RequestMapping("/api/v1/admin/github-sync/platform-control") +@PreAuthorize("hasRole('SUPER_ADMIN')") +public class AdminGithubPlatformControlController extends BaseApiController { + private final GithubPlatformControlAppService controls; + public AdminGithubPlatformControlController(ApiResponseFactory factory, GithubPlatformControlAppService controls) { + super(factory); + this.controls = controls; + } + + @Operation(summary = "读取平台技能采集控制状态") + @GetMapping + public ResponseEntity> get() { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(ok("response.success.read", response(controls.get()))); + } + + @Operation(summary = "暂停或恢复平台技能采集") + @PutMapping + public ResponseEntity> update(@Valid @RequestBody UpdateRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(ok("response.success.updated", + response(controls.update(body.expectedVersion(), body.paused(), principal.userId(), AuditRequestContext.from(request))))); + } + + @Schema(name = "GithubPlatformControlUpdateRequest") + public record UpdateRequest(@NotNull @Min(1) Long expectedVersion, @NotNull Boolean paused) {} + + @Schema(name = "GithubPlatformControlState") + public record PlatformControlState(long namespaceId, long version, boolean desiredPaused, String state, + long pendingStopCount, long expiredUnconfirmedCount, Instant updatedAt) {} + + private PlatformControlState response(GithubPlatformControlRepository.State state) { + return new PlatformControlState(state.namespaceId(), state.version(), state.desiredPaused(), state.state(), + state.pendingStopCount(), state.expiredUnconfirmedCount(), state.updatedAt()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncController.java new file mode 100644 index 000000000..95a3f818e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncController.java @@ -0,0 +1,225 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubSyncAdmissionPolicyResponse; +import com.iflytek.skillhub.dto.GithubSyncAdmissionPolicyUpdateRequest; +import com.iflytek.skillhub.dto.GithubSyncRunSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncRunDetailResponse; +import com.iflytek.skillhub.dto.GithubSyncRunTaskResponse; +import com.iflytek.skillhub.dto.GithubSyncRunPathResponse; +import com.iflytek.skillhub.dto.GithubSyncWorkerStatusResponse; +import com.iflytek.skillhub.dto.GithubSyncSourceSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSkillPathResponse; +import com.iflytek.skillhub.dto.GithubSyncManualBindingRequest; +import com.iflytek.skillhub.dto.GithubSyncManualBindingResponse; +import com.iflytek.skillhub.dto.GithubSourceIgnoreRequest; +import com.iflytek.skillhub.dto.GithubSourceIgnoreResponse; +import com.iflytek.skillhub.dto.GithubSourceRescanResponse; +import com.iflytek.skillhub.dto.GithubSyncRetrySkillRequest; +import com.iflytek.skillhub.dto.GithubSyncSchedulePolicyRequest; +import com.iflytek.skillhub.dto.GithubSyncSchedulePolicyResponse; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.RunKind; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.RunStatus; +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicy; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubSyncAdmissionPolicyAppService; +import com.iflytek.skillhub.service.GithubSourceIgnoreAppService; +import com.iflytek.skillhub.service.GithubSyncManualBindingAppService; +import com.iflytek.skillhub.service.GithubSourceRescanAppService; +import com.iflytek.skillhub.service.GithubSyncSchedulePolicyAppService; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.UUID; + +@RestController +@RequestMapping("/api/v1/admin/github-sync") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name = "GitHub Sync Admin") +public class AdminGithubSyncController extends BaseApiController { + private final GithubSyncAdmissionPolicyAppService admissionPolicies; + private final GithubSyncManagementQueryRepository management; + private final GithubSyncWorkerPresenceRepository workerPresence; + private final GithubSourceIgnoreAppService ignores; + private final GithubSyncManualBindingAppService manualBindings; + private final GithubSourceRescanAppService rescans; + private final GithubSyncSchedulePolicyAppService schedulePolicies; + + public AdminGithubSyncController(ApiResponseFactory responseFactory, + GithubSyncAdmissionPolicyAppService admissionPolicies, + GithubSyncManagementQueryRepository management, + GithubSyncWorkerPresenceRepository workerPresence, + GithubSourceIgnoreAppService ignores, + GithubSyncManualBindingAppService manualBindings, + GithubSourceRescanAppService rescans, + GithubSyncSchedulePolicyAppService schedulePolicies) { + super(responseFactory); + this.admissionPolicies = admissionPolicies; + this.management = management; + this.workerPresence = workerPresence; + this.ignores = ignores; + this.manualBindings = manualBindings; + this.rescans = rescans; + this.schedulePolicies = schedulePolicies; + } + + @GetMapping("/schedule-policy") + @Operation(summary = "读取 GitHub 同步三类任务目标周期") + public ApiResponse schedulePolicy() { + return ok("response.success.read", schedulePolicies.get()); + } + + @GetMapping("/worker-status") + @Operation(summary = "查看 GitHub 采集服务最后在线时间") + public ApiResponse workerStatus() { + return ok("response.success.read", workerPresence.status()); + } + + @PutMapping("/schedule-policy") + @Operation(summary = "按类别调整 GitHub 同步任务目标周期") + public ApiResponse updateSchedulePolicy( + @Valid @RequestBody GithubSyncSchedulePolicyRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ok("response.success.updated", schedulePolicies.update(body, principal.userId(), + AuditRequestContext.from(request))); + } + + @PostMapping("/sources/{sourceId}/rescan") + @Operation(summary = "按现有任务队列安排平台 GitHub 来源重新检查") + public ApiResponse rescan(@PathVariable long sourceId, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ok("response.success.updated", rescans.platform(sourceId, principal.userId(), + AuditRequestContext.from(request))); + } + + @PostMapping("/sources/{sourceId}/retry-rejected-skill") + @Operation(summary = "重新采集并送审一个已拒绝的 GitHub Skill") + public ApiResponse retryRejectedSkill(@PathVariable long sourceId, + @Valid @RequestBody GithubSyncRetrySkillRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ok("response.success.updated", rescans.retryPlatformSkill(sourceId, body.skillPath(), + principal.userId(), AuditRequestContext.from(request))); + } + + @PostMapping("/conflicts/{ingestionId}/bind-skill") + @Operation(summary = "确认平台 GitHub 来源与既有 Skill ID 的绑定") + public ApiResponse bindExistingSkill( + @PathVariable UUID ingestionId, @Valid @RequestBody GithubSyncManualBindingRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ok("response.success.updated", manualBindings.bindPlatform( + ingestionId, body.skillId(), body.reason(), principal.userId(), + AuditRequestContext.from(request))); + } + + @GetMapping("/sources") + @Operation(summary = "分页查看平台 GitHub 来源及摄入进度") + public ApiResponse> sources( + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", management.sources(management.platformNamespaceId(), page, size)); + } + + @GetMapping("/runs") + @Operation(summary = "分页查看平台 GitHub 发现与仓库采集运行") + public ApiResponse> runs( + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size, + @RequestParam(required = false) RunKind kind, + @RequestParam(required = false) RunStatus status) { + return ok("response.success.read", management.runs(management.platformNamespaceId(), page, size, kind, status)); + } + + @GetMapping("/runs/{runId}") + @Operation(summary = "查看平台 GitHub 同步单次运行详情") + public ApiResponse runDetail(@PathVariable UUID runId) { + return ok("response.success.read", management.runDetail(management.platformNamespaceId(), runId)); + } + + @GetMapping("/runs/{runId}/tasks") + @Operation(summary = "分页查看平台 GitHub 同步单次运行的任务") + public ApiResponse> runTasks( + @PathVariable UUID runId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", management.runTasks( + management.platformNamespaceId(), runId, page, size)); + } + + @GetMapping("/runs/{runId}/paths") + @Operation(summary = "分页查看平台 GitHub 同步单次运行的逐 Skill 路径结果") + public ApiResponse> runPaths( + @PathVariable UUID runId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", management.runPaths( + management.platformNamespaceId(), runId, page, size)); + } + + @GetMapping("/sources/{sourceId}/skills") + @Operation(summary = "分页查看平台 GitHub 来源的逐 Skill 路径状态") + public ApiResponse> skillPaths( + @PathVariable long sourceId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", management.skillPaths( + management.platformNamespaceId(), sourceId, page, size)); + } + + @GetMapping("/sources/{sourceId}/ignore-patterns") + @Operation(summary = "查看平台 GitHub 来源的候选路径忽略规则") + public ApiResponse ignorePatterns(@PathVariable long sourceId) { + return ok("response.success.read", ignores.platform(sourceId)); + } + + @PutMapping("/sources/{sourceId}/ignore-patterns") + @Operation(summary = "设置平台 GitHub 来源的候选路径忽略规则") + public ApiResponse updateIgnorePatterns( + @PathVariable long sourceId, @Valid @RequestBody GithubSourceIgnoreRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, HttpServletRequest request) { + return ok("response.success.updated", ignores.updatePlatform(sourceId, body.ignorePatterns(), + principal.userId(), AuditRequestContext.from(request))); + } + + @GetMapping("/admission-policy") + @Operation(summary = "读取 GitHub 同步准入水位与当前积压") + public ApiResponse getAdmissionPolicy() { + return ok("response.success.read", admissionPolicyResponse(admissionPolicies.get())); + } + + @PutMapping("/admission-policy") + @Operation(summary = "更新 GitHub 同步准入水位与候选预算") + public ApiResponse updateAdmissionPolicy( + @Valid @RequestBody GithubSyncAdmissionPolicyUpdateRequest body, + @AuthenticationPrincipal PlatformPrincipal principal, + HttpServletRequest request) { + return ok("response.success.updated", admissionPolicyResponse(admissionPolicies.update( + body.globalHigh(), body.globalLow(), body.namespaceHigh(), body.namespaceLow(), + body.candidatePendingLimit(), body.candidatePerLeaseLimit(), + principal.userId(), AuditRequestContext.from(request)))); + } + + private GithubSyncAdmissionPolicyResponse admissionPolicyResponse(GithubSyncAdmissionPolicy policy) { + var snapshot = management.admissionSnapshot(); + return GithubSyncAdmissionPolicyResponse.from(policy, snapshot.globalUnresolvedCount(), + snapshot.platformUnresolvedCount(), snapshot.platformPendingCandidateCount(), + snapshot.platformPaused()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncRepositoryController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncRepositoryController.java new file mode 100644 index 000000000..06f1e8f26 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/admin/AdminGithubSyncRepositoryController.java @@ -0,0 +1,75 @@ +package com.iflytek.skillhub.controller.admin; + +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.PathResult; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Detail; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Origin; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.OriginType; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Page; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.SourceStatus; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.State; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Summary; +import com.iflytek.skillhub.repository.GithubSyncRepositoryQueryRepository; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.time.Instant; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/admin/github-sync/repositories") +@PreAuthorize("hasRole('SUPER_ADMIN')") +@Tag(name = "GitHub Sync Admin") +public class AdminGithubSyncRepositoryController extends BaseApiController { + private final GithubSyncRepositoryQueryRepository queries; + + public AdminGithubSyncRepositoryController(ApiResponseFactory factory, GithubSyncRepositoryQueryRepository queries) { + super(factory); + this.queries = queries; + } + + @GetMapping + @Operation(summary = "分页查看平台全部发现方式的仓库") + public ResponseEntity>> repositories( + @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "20") int size, + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant asOf, + @RequestParam(required = false) String q, @RequestParam(required = false) OriginType originType, + @RequestParam(required = false) State state, @RequestParam(required = false) SourceStatus sourceStatus) { + return response(queries.repositories(page, size, asOf, q, originType, state, sourceStatus)); + } + + @GetMapping("/{repositoryId}") + @Operation(summary = "查看统一仓库登记和最近检查摘要") + public ResponseEntity> repository(@PathVariable long repositoryId) { + return response(queries.detail(repositoryId)); + } + + @GetMapping("/{repositoryId}/origins") + @Operation(summary = "分页查看仓库全部持久出处") + public ResponseEntity>> origins(@PathVariable long repositoryId, + @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "20") int size, + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant asOf) { + return response(queries.origins(repositoryId, page, size, asOf)); + } + + @GetMapping("/{repositoryId}/paths") + @Operation(summary = "分页查看统一仓库技能路径及审核链接") + public ResponseEntity>> paths(@PathVariable long repositoryId, + @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "20") int size, + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant asOf) { + return response(queries.paths(repositoryId, page, size, asOf)); + } + + private ResponseEntity> response(T value) { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(ok("response.success.read", value)); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubSyncReviewSourceController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubSyncReviewSourceController.java new file mode 100644 index 000000000..1b08416b9 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubSyncReviewSourceController.java @@ -0,0 +1,81 @@ +package com.iflytek.skillhub.controller.portal; + +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubSyncReviewSourceResponse; +import com.iflytek.skillhub.dto.GithubSyncReviewQueueItemResponse; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.service.GithubSyncReviewSourceAppService; +import com.iflytek.skillhub.service.GithubSyncReviewQueueAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; + +import java.util.Map; + +@RestController +@RequestMapping({"/api/v1/reviews/sync", "/api/web/reviews/sync"}) +public class GithubSyncReviewSourceController extends BaseApiController { + private final GithubSyncReviewSourceAppService sources; + private final GithubSyncReviewQueueAppService queue; + + public GithubSyncReviewSourceController(ApiResponseFactory responseFactory, + GithubSyncReviewSourceAppService sources, + GithubSyncReviewQueueAppService queue) { + super(responseFactory); + this.sources = sources; + this.queue = queue; + } + + @GetMapping + @Operation(summary = "按审核权限查询 GitHub 同步审核队列") + public ApiResponse> list( + @RequestParam String status, + @RequestParam(required = false) Long namespaceId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size, + @RequestParam(defaultValue = "DESC") String sortDirection, + @RequestAttribute("userId") String userId, + @RequestAttribute(value = "userNsRoles", required = false) + Map userNsRoles) { + return ok("response.success.read", queue.list(status, namespaceId, page, size, + sortDirection, userId, userNsRoles)); + } + + @GetMapping("/{id}/source") + @Operation(summary = "查看同步审核的 GitHub 来源和许可证证据") + public ApiResponse getSource( + @PathVariable Long id, + @RequestAttribute("userId") String userId, + @RequestAttribute(value = "userNsRoles", required = false) + Map userNsRoles) { + return ok("response.success.read", sources.get(id, userId, userNsRoles)); + } + + @GetMapping("/{id}/base-file") + @Operation(summary = "预览同步审核基线版本的变更文件") + public ResponseEntity getBaseFile( + @PathVariable Long id, + @RequestParam String path, + @Parameter(description = "是否按 256 KiB 上限读取预览；缺省 false 保留原读取行为") + @RequestParam(defaultValue = "false") boolean preview, + @RequestAttribute("userId") String userId, + @RequestAttribute(value = "userNsRoles", required = false) + Map userNsRoles) { + if (path == null || path.isBlank() || path.contains("..") || path.startsWith("/")) { + return ResponseEntity.badRequest().build(); + } + return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(new InputStreamResource(sources.getBaseFile(id, path, userId, userNsRoles, preview))); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubTeamSourceController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubTeamSourceController.java new file mode 100644 index 000000000..2e944dd5e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/GithubTeamSourceController.java @@ -0,0 +1,173 @@ +package com.iflytek.skillhub.controller.portal; + +import com.iflytek.skillhub.controller.BaseApiController; +import com.iflytek.skillhub.dto.ApiResponse; +import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryEnabledRequest; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryResponse; +import com.iflytek.skillhub.dto.GithubTeamRepositoryRequest; +import com.iflytek.skillhub.dto.GithubTeamSourceVisibilityRequest; +import com.iflytek.skillhub.dto.GithubSyncRunSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSourceSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSkillPathResponse; +import com.iflytek.skillhub.dto.GithubSyncManualBindingRequest; +import com.iflytek.skillhub.dto.GithubSyncManualBindingResponse; +import com.iflytek.skillhub.dto.GithubSourceIgnoreRequest; +import com.iflytek.skillhub.dto.GithubSourceIgnoreResponse; +import com.iflytek.skillhub.dto.GithubSourceRescanResponse; +import com.iflytek.skillhub.dto.GithubSyncRetrySkillRequest; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.service.GithubTeamSourceAppService; +import com.iflytek.skillhub.service.GithubSourceIgnoreAppService; +import com.iflytek.skillhub.service.GithubSyncManualBindingAppService; +import com.iflytek.skillhub.service.GithubSourceRescanAppService; +import io.swagger.v3.oas.annotations.Operation; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.UUID; + +@RestController +@RequestMapping({"/api/v1/namespaces/{namespaceSlug}/github-sources", + "/api/web/namespaces/{namespaceSlug}/github-sources"}) +public class GithubTeamSourceController extends BaseApiController { + private final GithubTeamSourceAppService sources; + private final GithubSourceIgnoreAppService ignores; + private final GithubSyncManualBindingAppService manualBindings; + private final GithubSourceRescanAppService rescans; + + public GithubTeamSourceController(ApiResponseFactory responseFactory, GithubTeamSourceAppService sources, + GithubSourceIgnoreAppService ignores, + GithubSyncManualBindingAppService manualBindings, + GithubSourceRescanAppService rescans) { + super(responseFactory); + this.sources = sources; + this.ignores = ignores; + this.manualBindings = manualBindings; + this.rescans = rescans; + } + + @PostMapping("/resolved/{sourceId}/rescan") + @Operation(summary = "按现有任务队列安排团队 GitHub 来源重新检查") + public ApiResponse rescan( + @PathVariable String namespaceSlug, @PathVariable long sourceId, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", rescans.team(namespaceSlug, sourceId, userId, + AuditRequestContext.from(request))); + } + + @PostMapping("/resolved/{sourceId}/retry-rejected-skill") + @Operation(summary = "重新采集并送审一个已拒绝的团队 GitHub Skill") + public ApiResponse retryRejectedSkill( + @PathVariable String namespaceSlug, @PathVariable long sourceId, + @Valid @RequestBody GithubSyncRetrySkillRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", rescans.retryTeamSkill(namespaceSlug, sourceId, + body.skillPath(), userId, AuditRequestContext.from(request))); + } + + @PostMapping("/conflicts/{ingestionId}/bind-skill") + @Operation(summary = "确认团队 GitHub 来源与既有 Skill ID 的绑定") + public ApiResponse bindExistingSkill( + @PathVariable String namespaceSlug, @PathVariable UUID ingestionId, + @Valid @RequestBody GithubSyncManualBindingRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", manualBindings.bindTeam( + namespaceSlug, ingestionId, body.skillId(), body.reason(), userId, + AuditRequestContext.from(request))); + } + + @GetMapping + @Operation(summary = "查看团队 GitHub 公开仓库来源配置") + public ApiResponse> list( + @PathVariable String namespaceSlug, @RequestAttribute("userId") String userId) { + return ok("response.success.read", sources.list(namespaceSlug, userId)); + } + + @GetMapping("/resolved") + @Operation(summary = "分页查看团队已解析 GitHub 来源与摄入进度") + public ApiResponse> resolved( + @PathVariable String namespaceSlug, @RequestAttribute("userId") String userId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", sources.resolvedSources(namespaceSlug, userId, page, size)); + } + + @GetMapping("/runs") + @Operation(summary = "分页查看团队 GitHub 来源的发现与采集运行") + public ApiResponse> runs( + @PathVariable String namespaceSlug, @RequestAttribute("userId") String userId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", sources.runs(namespaceSlug, userId, page, size)); + } + + @GetMapping("/resolved/{sourceId}/skills") + @Operation(summary = "分页查看团队 GitHub 来源的逐 Skill 路径状态") + public ApiResponse> skillPaths( + @PathVariable String namespaceSlug, @PathVariable long sourceId, + @RequestAttribute("userId") String userId, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "20") int size) { + return ok("response.success.read", sources.skillPaths( + namespaceSlug, userId, sourceId, page, size)); + } + + @GetMapping("/resolved/{sourceId}/ignore-patterns") + @Operation(summary = "查看团队 GitHub 来源的候选路径忽略规则") + public ApiResponse ignorePatterns( + @PathVariable String namespaceSlug, @PathVariable long sourceId, + @RequestAttribute("userId") String userId) { + return ok("response.success.read", ignores.team(namespaceSlug, sourceId, userId)); + } + + @PutMapping("/resolved/{sourceId}/ignore-patterns") + @Operation(summary = "设置团队 GitHub 来源的候选路径忽略规则") + public ApiResponse updateIgnorePatterns( + @PathVariable String namespaceSlug, @PathVariable long sourceId, + @Valid @RequestBody GithubSourceIgnoreRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", ignores.updateTeam(namespaceSlug, sourceId, + body.ignorePatterns(), userId, AuditRequestContext.from(request))); + } + + @PostMapping + @Operation(summary = "为团队登记 GitHub 公开仓库") + public ApiResponse create( + @PathVariable String namespaceSlug, @Valid @RequestBody GithubTeamRepositoryRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.created", sources.create(namespaceSlug, body, userId, + AuditRequestContext.from(request))); + } + + @PutMapping("/{entryId}/enabled") + @Operation(summary = "暂停或恢复团队 GitHub 来源同步") + public ApiResponse setEnabled( + @PathVariable String namespaceSlug, @PathVariable UUID entryId, + @Valid @RequestBody GithubDiscoveryEntryEnabledRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", sources.setEnabled(namespaceSlug, entryId, body.enabled(), + userId, AuditRequestContext.from(request))); + } + + @PutMapping("/{entryId}/visibility") + @Operation(summary = "设置团队来源后续导入内容的可见范围") + public ApiResponse setVisibility( + @PathVariable String namespaceSlug, @PathVariable UUID entryId, + @Valid @RequestBody GithubTeamSourceVisibilityRequest body, + @RequestAttribute("userId") String userId, HttpServletRequest request) { + return ok("response.success.updated", sources.setVisibility(namespaceSlug, entryId, body.visibility(), + userId, AuditRequestContext.from(request))); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/ReviewController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/ReviewController.java index b4115e0bc..13d449d07 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/ReviewController.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/ReviewController.java @@ -13,6 +13,8 @@ import com.iflytek.skillhub.dto.ReviewTaskRequest; import com.iflytek.skillhub.dto.ReviewTaskResponse; import com.iflytek.skillhub.service.AuditRequestContext; import com.iflytek.skillhub.service.GovernanceWorkflowAppService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import jakarta.servlet.http.HttpServletRequest; import java.util.List; import java.util.Map; @@ -183,11 +185,15 @@ public class ReviewController extends BaseApiController { @GetMapping("/{id}/skill-detail") public ApiResponse getReviewSkillDetail(@PathVariable Long id, + @Parameter(description = "是否包含 README.md/SKILL.md 正文；缺省 true，false 仅返回元数据") + @RequestParam(defaultValue = "true") boolean includeDocumentation, @RequestAttribute("userId") String userId, @RequestAttribute(value = "userNsRoles", required = false) Map userNsRoles) { return ok( "response.success.read", - governanceWorkflowAppService.getReviewSkillDetail(id, userId, userNsRoles) + includeDocumentation + ? governanceWorkflowAppService.getReviewSkillDetail(id, userId, userNsRoles) + : governanceWorkflowAppService.getReviewSkillDetail(id, userId, userNsRoles, false) ); } @@ -200,6 +206,8 @@ public class ReviewController extends BaseApiController { public ResponseEntity getReviewFile( @PathVariable Long id, @RequestParam("path") String path, + @Parameter(description = "是否按 256 KiB 上限读取预览；缺省 false 保留完整文件读取") + @RequestParam(defaultValue = "false") boolean preview, @RequestAttribute("userId") String userId, @RequestAttribute(value = "userNsRoles", required = false) Map userNsRoles) { @@ -209,7 +217,7 @@ public class ReviewController extends BaseApiController { } java.io.InputStream content = governanceWorkflowAppService.getReviewFileContent( - id, path, userId, userNsRoles + id, path, userId, userNsRoles, preview ); return ResponseEntity.ok() diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java index 7b125a007..4bc7f2695 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java @@ -22,6 +22,7 @@ import com.iflytek.skillhub.dto.SkillVersionCompareLineResponse; import com.iflytek.skillhub.dto.SkillVersionCompareResponse; import com.iflytek.skillhub.dto.SkillVersionDetailResponse; import com.iflytek.skillhub.dto.SkillVersionResponse; +import com.iflytek.skillhub.repository.GithubSkillSourceQueryRepository; import com.iflytek.skillhub.metrics.SkillHubMetrics; import com.iflytek.skillhub.ratelimit.RateLimit; import com.iflytek.skillhub.service.SkillLabelAppService; @@ -65,6 +66,7 @@ public class SkillController extends BaseApiController { private final ComplianceSnapshotProjectionService complianceSnapshotProjectionService; private final SkillSuiteAppService skillSuiteAppService; private final SkillHubMetrics metrics; + private final GithubSkillSourceQueryRepository githubSkillSourceQueryRepository; public SkillController( SkillQueryService skillQueryService, @@ -73,6 +75,7 @@ public class SkillController extends BaseApiController { ComplianceSnapshotProjectionService complianceSnapshotProjectionService, SkillSuiteAppService skillSuiteAppService, SkillHubMetrics metrics, + GithubSkillSourceQueryRepository githubSkillSourceQueryRepository, ApiResponseFactory responseFactory) { super(responseFactory); this.skillQueryService = skillQueryService; @@ -81,6 +84,7 @@ public class SkillController extends BaseApiController { this.complianceSnapshotProjectionService = complianceSnapshotProjectionService; this.skillSuiteAppService = skillSuiteAppService; this.metrics = metrics; + this.githubSkillSourceQueryRepository = githubSkillSourceQueryRepository; } /** @@ -138,7 +142,10 @@ public class SkillController extends BaseApiController { && !detail.ownerId().equals(userId) ? null : SkillAvatarUrls.current(detail.activeAvatarId(), detail.activeAvatarAudience(), SkillVisibility.valueOf(detail.visibility()), namespace, detail.slug()), - detail.ownerReadOnly() + detail.ownerReadOnly(), + detail.publishedVersion() == null ? null + : githubSkillSourceQueryRepository.findPublishedVersion(detail.publishedVersion().id()) + .orElse(null) ); return ok("response.success.read", response); @@ -188,6 +195,7 @@ public class SkillController extends BaseApiController { PageResponse response = PageResponse.from(versions.map(v -> new SkillVersionResponse( v.getId(), v.getVersion(), + v.getGithubSyncRevision(), v.getStatus().name(), v.getChangelog(), v.getFileCount(), @@ -516,7 +524,7 @@ public class SkillController extends BaseApiController { return null; } return new SkillLifecycleVersionResponse(projection.id(), projection.version(), projection.status(), - projection.displayName(), projection.summary()); + projection.displayName(), projection.summary(), projection.syncRevision()); } private SkillVersionCompareResponse toCompareResponse(SkillQueryService.SkillVersionCompareDTO compare) { diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialEnabledRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialEnabledRequest.java new file mode 100644 index 000000000..1a950805a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialEnabledRequest.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotNull; + +public record GithubCredentialEnabledRequest(@NotNull Boolean enabled) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialResponse.java new file mode 100644 index 000000000..cf0870153 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialResponse.java @@ -0,0 +1,23 @@ +package com.iflytek.skillhub.dto; + +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.githubsync.GithubCredentialQuota; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record GithubCredentialResponse(UUID id, String label, String fingerprint, + String status, String keyVersion, boolean keyAvailable, Instant rateLimitedUntil, + Instant lastUsedAt, Instant createdAt, Instant updatedAt, + List quotas) { + public static GithubCredentialResponse from(GithubSystemCredential credential, + GithubCredentialCipher cipher, + List quotas) { + return new GithubCredentialResponse(credential.id(), credential.label(), + credential.fingerprint(), credential.status(), credential.keyVersion(), + cipher.supports(credential.keyVersion()), credential.rateLimitedUntil(), + credential.lastUsedAt(), credential.createdAt(), credential.updatedAt(), quotas); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialUpsertRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialUpsertRequest.java new file mode 100644 index 000000000..1a004064b --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCredentialUpsertRequest.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; + +public record GithubCredentialUpsertRequest(@NotBlank String label, @NotBlank String token) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedOwnerRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedOwnerRequest.java new file mode 100644 index 000000000..084d99ff6 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedOwnerRequest.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; + +public record GithubCuratedOwnerRequest(@NotBlank String ownerLogin, Integer targetHours) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedRepositoryRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedRepositoryRequest.java new file mode 100644 index 000000000..61f1f42e4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubCuratedRepositoryRequest.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; + +public record GithubCuratedRepositoryRequest(@NotBlank String ownerLogin, + @NotBlank String repositoryName) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationRequest.java new file mode 100644 index 000000000..ef9651c4a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationRequest.java @@ -0,0 +1,33 @@ +package com.iflytek.skillhub.dto; + +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import java.util.List; + +public record GithubDeepDiscoveryConfigurationRequest( + @NotNull @Min(1) Long expectedVersion, + @NotNull Boolean enabled, + @NotNull GithubDeepDiscoveryConfiguration.BudgetProfile budgetProfile, + @NotNull @Size(max = 1000) List<@NotNull @Valid ExclusionRequest> exclusions) { + + @JsonAnySetter + public void rejectUnknownProperty(String name, Object value) { + throw new IllegalArgumentException("Unknown GitHub discovery configuration field"); + } + + public record ExclusionRequest( + @NotNull GithubDeepDiscoveryConfiguration.ExclusionKind kind, + @NotBlank @Size(max = 140) String value) { + + @JsonAnySetter + public void rejectUnknownProperty(String name, Object value) { + throw new IllegalArgumentException("Unknown GitHub discovery exclusion field"); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationResponse.java new file mode 100644 index 000000000..ecebaa186 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryConfigurationResponse.java @@ -0,0 +1,16 @@ +package com.iflytek.skillhub.dto; + +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import io.swagger.v3.oas.annotations.media.Schema; + +import java.time.Instant; +import java.util.List; + +public record GithubDeepDiscoveryConfigurationResponse( + long version, + boolean enabled, + GithubDeepDiscoveryConfiguration.BudgetProfile budgetProfile, + List exclusions, + @Schema(nullable = true, description = "尚未装载官方策略时为空；配置启用不表示正在执行") String strategyVersion, + Instant updatedAt, + @Schema(nullable = true) String updatedBy) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryReadResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryReadResponse.java new file mode 100644 index 000000000..a4a6ba1db --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDeepDiscoveryReadResponse.java @@ -0,0 +1,51 @@ +package com.iflytek.skillhub.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +/** Bounded read projections; no source contents, credentials, or raw error messages. */ +public final class GithubDeepDiscoveryReadResponse { + private GithubDeepDiscoveryReadResponse() {} + @Schema(name = "GithubDeepDiscoveryChannel") + public enum Channel { REPOSITORY_METADATA, README, OWNER_EXPANSION } + @Schema(name = "GithubDeepDiscoveryRepositoryStatus") + public enum RepositoryStatus { PENDING, VALIDATING, VALID, NO_VALID_SKILL, VALIDATION_FAILED, UNREACHABLE } + public record DiscoveryPage(List items, long total, int page, int size, Instant asOf) {} + @Schema(name = "GithubDeepDiscoveryWait") + public record Wait(String reason, List affectedWork, List continuingWork, + Instant retryAt, String recoveryCondition, String action) {} + @Schema(name = "GithubDeepDiscoveryRecent") + public record Recent(Instant from, Instant until, Long newCandidateRepositories, Long firstValidRepositories, + Long newSkills, Long updatedVersions, boolean complete, Instant availableSince) {} + @Schema(name = "GithubDeepDiscoveryCurrent") + public record Current(long pendingRepositoryChecks, long pendingReviewVersions, Instant observedAt) {} + @Schema(name = "GithubDeepDiscoverySummary") + public record Summary(long configurationVersion, String strategyVersion, String state, Instant observedAt, + GithubOfficialWorkerAvailabilityResponse worker, Recent recent, Current current, + List waits, List recentRunIds) {} + @Schema(name = "GithubDeepDiscoveryQuota") + public record Quota(String resource, String subjectKey, String state, Long remaining, Instant observedAt, + Instant validUntil, Instant retryAt) {} + @Schema(name = "GithubDeepDiscoveryChannelProgress") + public record ChannelProgress(Channel channel, String state, long completedShards, long pendingShards, + long gapShards, Instant lastProgressAt, List waits) {} + @Schema(name = "GithubDeepDiscoveryProgress") + public record Progress(String strategyVersion, Instant observedAt, List channels, + List quotas, boolean quotaObservationsTruncated) {} + @Schema(name = "GithubDeepDiscoveryRepositorySummary") + public record RepositorySummary(long repositoryId, String owner, String name, Instant firstDiscoveredAt, + Instant lastDiscoveredAt, RepositoryStatus status, List channels, + Long sourceId, boolean paused, String waitReason, long candidateId) {} + @Schema(name = "GithubDeepDiscoveryRepositoryDetail") + public record RepositoryDetail(RepositorySummary repository, Instant lastCheckedAt, Instant nextEligibleAt, + String notImportedReason, List waits, UUID latestRunId, + Instant historyAvailableSince) {} + @Schema(name = "GithubDeepDiscoveryProvenance") + public record Provenance(UUID id, Channel channel, String strategyVersion, UUID queryId, long queryVersion, + UUID shardId, UUID runId, Instant firstSeenAt, Instant lastSeenAt, Long seedRepositoryId) {} + @Schema(name = "GithubDeepDiscoveryPathResult") + public record PathResult(String skillPath, String outcome, String reasonCode, Long skillId, Long versionId, + Long reviewId, UUID runId, Instant observedAt) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryEnabledRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryEnabledRequest.java new file mode 100644 index 000000000..63cc91a1f --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryEnabledRequest.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotNull; + +public record GithubDiscoveryEntryEnabledRequest(@NotNull Boolean enabled) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryResponse.java new file mode 100644 index 000000000..b7ed27cb1 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubDiscoveryEntryResponse.java @@ -0,0 +1,29 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record GithubDiscoveryEntryResponse( + UUID id, + String entryType, + String title, + String ownerLogin, + String repositoryName, + Long repositoryId, + String sourceVisibility, + List selectors, + long ruleVersion, + boolean enabled, + int targetHours, + Instant nextDueAt, + Instant updatedAt, + Execution latestExecution) { + public record Execution(UUID taskId, UUID runId, String status, String errorCode, Instant retryAt) {} + public GithubDiscoveryEntryResponse(UUID id, String entryType, String title, String ownerLogin, + String repositoryName, Long repositoryId, String sourceVisibility, List selectors, + long ruleVersion, boolean enabled, int targetHours, Instant nextDueAt, Instant updatedAt) { + this(id, entryType, title, ownerLogin, repositoryName, repositoryId, sourceVisibility, selectors, + ruleVersion, enabled, targetHours, nextDueAt, updatedAt, null); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubOfficialWorkerAvailabilityResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubOfficialWorkerAvailabilityResponse.java new file mode 100644 index 000000000..c28be08d0 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubOfficialWorkerAvailabilityResponse.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; + +/** Contact and protocol compatibility, not proof of successful task execution. */ +public record GithubOfficialWorkerAvailabilityResponse( + String state, Instant lastSeenAt, long compatibleInstanceCount, Instant observedAt) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchRuleRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchRuleRequest.java new file mode 100644 index 000000000..62185c83c --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchRuleRequest.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; + +import java.util.List; + +public record GithubSearchRuleRequest( + @NotBlank String title, + @NotEmpty List selectors, + Integer targetHours) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchSelectorRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchSelectorRequest.java new file mode 100644 index 000000000..256953c90 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSearchSelectorRequest.java @@ -0,0 +1,4 @@ +package com.iflytek.skillhub.dto; + +public record GithubSearchSelectorRequest(String kind, String value, String priority) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSkillSourceResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSkillSourceResponse.java new file mode 100644 index 000000000..844777452 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSkillSourceResponse.java @@ -0,0 +1,16 @@ +package com.iflytek.skillhub.dto; + +import java.util.List; + +/** Public provenance for the currently published GitHub-synced version. */ +public record GithubSkillSourceResponse( + String ownerLogin, + String repositoryName, + String skillPath, + String commitSha, + int syncRevision, + String licenseStatus, + List licenseDocuments +) { + public record LicenseDocument(String scope, String path, boolean bundled) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreRequest.java new file mode 100644 index 000000000..40f1db1c4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreRequest.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +public record GithubSourceIgnoreRequest(@NotNull List ignorePatterns) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreResponse.java new file mode 100644 index 000000000..79c4fdad0 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceIgnoreResponse.java @@ -0,0 +1,5 @@ +package com.iflytek.skillhub.dto; + +import java.util.List; + +public record GithubSourceIgnoreResponse(long sourceId, List ignorePatterns) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceRescanResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceRescanResponse.java new file mode 100644 index 000000000..ac1d97605 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSourceRescanResponse.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSourceRescanResponse(long sourceId, String status, UUID taskId, + String taskStatus, Instant nextCheckAt) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyResponse.java new file mode 100644 index 000000000..8024e8c99 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyResponse.java @@ -0,0 +1,32 @@ +package com.iflytek.skillhub.dto; + +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicy; + +import java.time.Instant; + +public record GithubSyncAdmissionPolicyResponse( + Integer globalHigh, + Integer globalLow, + Integer namespaceHigh, + Integer namespaceLow, + int candidatePendingLimit, + int candidatePerLeaseLimit, + boolean globalPaused, + int globalUnresolvedCount, + int platformUnresolvedCount, + int platformPendingCandidateCount, + boolean platformPaused, + Instant updatedAt) { + + public static GithubSyncAdmissionPolicyResponse from(GithubSyncAdmissionPolicy policy, + int globalUnresolvedCount, + int platformUnresolvedCount, + int platformPendingCandidateCount, + boolean platformPaused) { + return new GithubSyncAdmissionPolicyResponse(policy.globalHigh(), policy.globalLow(), + policy.namespaceHigh(), policy.namespaceLow(), policy.candidatePendingLimit(), + policy.candidatePerLeaseLimit(), policy.globalPaused(), + globalUnresolvedCount, platformUnresolvedCount, + platformPendingCandidateCount, platformPaused, policy.updatedAt()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyUpdateRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyUpdateRequest.java new file mode 100644 index 000000000..edf4c4a79 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncAdmissionPolicyUpdateRequest.java @@ -0,0 +1,13 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +public record GithubSyncAdmissionPolicyUpdateRequest( + @NotNull @Min(1) Integer globalHigh, + @NotNull @Min(0) Integer globalLow, + @NotNull @Min(1) Integer namespaceHigh, + @NotNull @Min(0) Integer namespaceLow, + @NotNull @Min(1) Integer candidatePendingLimit, + @NotNull @Min(1) Integer candidatePerLeaseLimit) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingRequest.java new file mode 100644 index 000000000..aa9894657 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingRequest.java @@ -0,0 +1,11 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Positive; +import jakarta.validation.constraints.Size; + +public record GithubSyncManualBindingRequest( + @NotNull @Positive Long skillId, + @NotBlank @Size(min = 10, max = 1000) String reason +) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingResponse.java new file mode 100644 index 000000000..b43faf89a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncManualBindingResponse.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.dto; + +import java.util.UUID; + +public record GithubSyncManualBindingResponse( + UUID ingestionId, + long sourceId, + long skillId, + String skillSlug, + String skillPath, + String status +) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRepositoryResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRepositoryResponse.java new file mode 100644 index 000000000..d6501c4c0 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRepositoryResponse.java @@ -0,0 +1,34 @@ +package com.iflytek.skillhub.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +/** Platform repository projections; registration is not a quality or publication verdict. */ +public final class GithubSyncRepositoryResponse { + private GithubSyncRepositoryResponse() {} + @Schema(name = "GithubSyncRepositoryState") + public enum State { REGISTERED, PENDING, VALIDATING, NO_VALID_SKILL, VALIDATION_FAILED, UNREACHABLE, BOUND_WITHOUT_SOURCE } + @Schema(name = "GithubSyncRepositoryOriginType") + public enum OriginType { CURATED_REPOSITORY, CURATED_OWNER, SEARCH_RULE, OFFICIAL_DISCOVERY, TEAM_CONFIG } + @Schema(name = "GithubSyncRepositorySourceStatus") + public enum SourceStatus { ACTIVE, PAUSED, UNREACHABLE } + public record Page(List items, long total, int page, int size, Instant asOf) {} + @Schema(name = "GithubSyncRepositorySummary") + public record Summary(long repositoryId, Long sourceId, Long candidateId, String owner, String name, + Instant firstSeenAt, State state, SourceStatus sourceStatus, String candidateStatus, + List originTypes) {} + @Schema(name = "GithubSyncRepositoryDetail") + public record Detail(Summary repository, UUID latestRunId, Instant lastCheckedAt, + Instant nextCheckAt, String lastErrorCode) {} + @Schema(name = "GithubSyncRepositoryOrigin") + public record Origin(OriginType originType, String originKey, Instant firstSeenAt, Instant lastSeenAt, + UUID entryId, String channel, UUID queryId, UUID runId, Long seedRepositoryId) {} + @Schema(name = "GithubSyncRunKind") + public enum RunKind { DISCOVERY, REPOSITORY_SYNC } + @Schema(name = "GithubSyncRunStatus") + public enum RunStatus { QUEUED, LEASED, RETRY_WAIT, PARTIAL, ACQUISITION_COMPLETE, FAILED } + @Schema(name = "GithubSyncRunTarget") + public record Target(String type, String value) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRetrySkillRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRetrySkillRequest.java new file mode 100644 index 000000000..a547bca21 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRetrySkillRequest.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Size; + +public record GithubSyncRetrySkillRequest( + @NotBlank @Size(max = 1024) String skillPath) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewQueueItemResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewQueueItemResponse.java new file mode 100644 index 000000000..2833f7590 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewQueueItemResponse.java @@ -0,0 +1,14 @@ +package com.iflytek.skillhub.dto; + +import java.util.UUID; + +public record GithubSyncReviewQueueItemResponse( + ReviewTaskResponse review, + UUID ingestionId, + UUID runId, + long repositoryId, + String ownerLogin, + String repositoryName, + String skillPath, + String commitSha +) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewSourceResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewSourceResponse.java new file mode 100644 index 000000000..b7fb7f14b --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncReviewSourceResponse.java @@ -0,0 +1,38 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record GithubSyncReviewSourceResponse( + UUID ingestionId, + UUID runId, + Long sourceId, + long repositoryId, + String ownerLogin, + String repositoryName, + String commitSha, + String skillPath, + String contentSha256, + int syncRevision, + String ingestionStatus, + Instant receivedAt, + LicenseEvidence evidence, + Long baseVersionId, + List fileChanges +) { + public GithubSyncReviewSourceResponse withFileChanges(List changes) { + return new GithubSyncReviewSourceResponse(ingestionId, runId, sourceId, repositoryId, + ownerLogin, repositoryName, commitSha, skillPath, contentSha256, + syncRevision, ingestionStatus, receivedAt, evidence, baseVersionId, changes); + } + + public record FileChange(String path, String changeType, Long oldSize, Long newSize, + String oldSha256, String newSha256) {} + + public record LicenseEvidence(String status, List documents) {} + + public record EvidenceDocument(String scope, String path, String gitSha, + String sha256, long size, boolean bundled, + String text) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunDetailResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunDetailResponse.java new file mode 100644 index 000000000..81f11d1dd --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunDetailResponse.java @@ -0,0 +1,21 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncRunDetailResponse( + UUID id, + String kind, + String status, + Instant createdAt, + Instant finishedAt, + long taskCount, + long pathCount, + Instant nextRetryAt, + long receivedPackages, + long processingPackages, + long pendingReviewPackages, + long blockedPackages, + long failedPackages, + String lastErrorCode) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunPathResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunPathResponse.java new file mode 100644 index 000000000..3fa155cd6 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunPathResponse.java @@ -0,0 +1,19 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncRunPathResponse( + UUID taskId, + Long sourceId, + String repository, + String skillPath, + String commitSha, + String outcome, + String reason, + String ingestionStatus, + String ingestionErrorCode, + String reviewStatus, + Instant retryAt, + Instant updatedAt) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunSummaryResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunSummaryResponse.java new file mode 100644 index 000000000..773d488db --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunSummaryResponse.java @@ -0,0 +1,28 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncRunSummaryResponse( + UUID id, + String kind, + String status, + Instant createdAt, + Instant finishedAt, + Instant nextRetryAt, + long taskCount, + long receivedPackages, + long processingPackages, + long pendingReviewPackages, + long blockedPackages, + long failedPackages, + String lastErrorCode, + GithubSyncRepositoryResponse.Target target, + long targetCount) { + public GithubSyncRunSummaryResponse(UUID id, String kind, String status, Instant createdAt, Instant finishedAt, + Instant nextRetryAt, long taskCount, long receivedPackages, long processingPackages, + long pendingReviewPackages, long blockedPackages, long failedPackages, String lastErrorCode) { + this(id, kind, status, createdAt, finishedAt, nextRetryAt, taskCount, receivedPackages, processingPackages, + pendingReviewPackages, blockedPackages, failedPackages, lastErrorCode, null, 0); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunTaskResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunTaskResponse.java new file mode 100644 index 000000000..0120df70e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncRunTaskResponse.java @@ -0,0 +1,18 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncRunTaskResponse( + UUID id, + Long sourceId, + String repository, + String kind, + String mode, + String status, + String fixedCommitSha, + String lastErrorCode, + Instant retryAt, + Instant createdAt, + Instant completedAt) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyRequest.java new file mode 100644 index 000000000..14f373764 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyRequest.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; + +public record GithubSyncSchedulePolicyRequest( + @Min(1) @Max(720) int repositoryHours, + @Min(1) @Max(720) int curatedOwnerHours, + @Min(1) @Max(720) int searchRuleHours) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyResponse.java new file mode 100644 index 000000000..d9085b484 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSchedulePolicyResponse.java @@ -0,0 +1,10 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; + +public record GithubSyncSchedulePolicyResponse( + Integer repositoryHours, + int curatedOwnerHours, + int searchRuleHours, + String effectiveRepositoryInterval, + Instant updatedAt) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSkillPathResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSkillPathResponse.java new file mode 100644 index 000000000..ff9a97ea1 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSkillPathResponse.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncSkillPathResponse( + String skillPath, + String commitSha, + String acquisitionOutcome, + String acquisitionReason, + Instant acquisitionRetryAt, + Instant acquisitionUpdatedAt, + UUID ingestionId, + String ingestionStatus, + String ingestionErrorCode, + String targetSkillSlug, + Long skillId, + String upstreamPresence, + String presenceCommitSha, + Instant missingSince, + Long reviewId, + String reviewStatus, + String latestBindingIngestionStatus, + boolean retryRequested, + boolean retryInProgress, + boolean ignoredByCurrentPolicy) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSourceSummaryResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSourceSummaryResponse.java new file mode 100644 index 000000000..2f4cad17b --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncSourceSummaryResponse.java @@ -0,0 +1,30 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSyncSourceSummaryResponse( + long id, + long repositoryId, + String ownerLogin, + String repositoryName, + String defaultBranch, + String visibility, + String status, + String lastObservedCommit, + String lastFullyProcessedCommit, + Instant nextCheckAt, + String lastErrorCode, + UUID latestTaskId, + String latestTaskStatus, + String latestTaskMode, + String latestTaskErrorCode, + String latestTaskErrorDetail, + Instant latestTaskRetryAt, + long receivedPackages, + long processingPackages, + long pendingReviewPackages, + long blockedPackages, + long failedPackages, + long trackedSkills) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerHeartbeatRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerHeartbeatRequest.java new file mode 100644 index 000000000..76c3ad5f6 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerHeartbeatRequest.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.dto; + +import com.fasterxml.jackson.annotation.JsonAnySetter; + +import java.util.List; + +public record GithubSyncWorkerHeartbeatRequest(String workerId, Integer protocolMajor, List capabilities) { + public GithubSyncWorkerHeartbeatRequest { + boolean legacy = workerId == null && protocolMajor == null && capabilities == null; + if (!legacy && (invalid(workerId) || !Integer.valueOf(1).equals(protocolMajor) + || capabilities == null || capabilities.size() > 32 + || capabilities.stream().anyMatch(GithubSyncWorkerHeartbeatRequest::invalid))) { + throw new IllegalArgumentException("Invalid GitHub worker heartbeat"); + } + capabilities = capabilities == null ? null : List.copyOf(capabilities); + } + + @JsonAnySetter + public void rejectUnknown(String name, Object value) { + throw new IllegalArgumentException("Unsupported GitHub worker heartbeat field"); + } + + private static boolean invalid(String value) { + return value == null || value.isBlank() || value.length() > 128 + || value.chars().anyMatch(Character::isISOControl); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerStatusResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerStatusResponse.java new file mode 100644 index 000000000..c7013dee9 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubSyncWorkerStatusResponse.java @@ -0,0 +1,5 @@ +package com.iflytek.skillhub.dto; + +import java.time.Instant; + +public record GithubSyncWorkerStatusResponse(Instant lastSeenAt, boolean online) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamRepositoryRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamRepositoryRequest.java new file mode 100644 index 000000000..65e3e8d30 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamRepositoryRequest.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; + +public record GithubTeamRepositoryRequest(@NotBlank String ownerLogin, + @NotBlank String repositoryName, + String visibility) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamSourceVisibilityRequest.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamSourceVisibilityRequest.java new file mode 100644 index 000000000..cca4ebf3c --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/GithubTeamSourceVisibilityRequest.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.dto; + +import jakarta.validation.constraints.NotBlank; + +public record GithubTeamSourceVisibilityRequest(@NotBlank String visibility) { +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ReviewTaskResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ReviewTaskResponse.java index ac064064e..fe2b0288e 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ReviewTaskResponse.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/ReviewTaskResponse.java @@ -23,15 +23,24 @@ public record ReviewTaskResponse( Long avatarCandidateId, boolean avatarRemoval, String previousVisibility, - String requestedVisibility + String requestedVisibility, + String originType ) { + public ReviewTaskResponse(Long id, Long skillVersionId, String namespace, String skillSlug, String version, String status, String submittedBy, String submittedByName, String reviewedBy, String reviewedByName, String reviewComment, Instant submittedAt, Instant reviewedAt, String subjectType, Long subjectId, Long subjectVersionId, String subjectSlug, String originType) { + this(id, skillVersionId, namespace, skillSlug, version, status, submittedBy, submittedByName, reviewedBy, reviewedByName, reviewComment, submittedAt, reviewedAt, subjectType, subjectId, subjectVersionId, subjectSlug, null, false, null, null, originType); + } + + public ReviewTaskResponse(Long id, Long skillVersionId, String namespace, String skillSlug, String version, String status, String submittedBy, String submittedByName, String reviewedBy, String reviewedByName, String reviewComment, Instant submittedAt, Instant reviewedAt, String subjectType, Long subjectId, Long subjectVersionId, String subjectSlug, Long avatarCandidateId, boolean avatarRemoval, String previousVisibility, String requestedVisibility) { + this(id, skillVersionId, namespace, skillSlug, version, status, submittedBy, submittedByName, reviewedBy, reviewedByName, reviewComment, submittedAt, reviewedAt, subjectType, subjectId, subjectVersionId, subjectSlug, avatarCandidateId, avatarRemoval, previousVisibility, requestedVisibility, "USER"); + } + public ReviewTaskResponse(Long id, Long skillVersionId, String namespace, String skillSlug, String version, String status, String submittedBy, String submittedByName, String reviewedBy, String reviewedByName, String reviewComment, Instant submittedAt, Instant reviewedAt, String subjectType, Long subjectId, Long subjectVersionId, String subjectSlug, Long avatarCandidateId, boolean avatarRemoval) { this(id, skillVersionId, namespace, skillSlug, version, status, submittedBy, submittedByName, reviewedBy, reviewedByName, reviewComment, submittedAt, reviewedAt, subjectType, subjectId, - subjectVersionId, subjectSlug, avatarCandidateId, avatarRemoval, null, null); + subjectVersionId, subjectSlug, avatarCandidateId, avatarRemoval, null, null, "USER"); } public ReviewTaskResponse(Long id, Long skillVersionId, String namespace, String skillSlug, String version, String status, String submittedBy, String submittedByName, String reviewedBy, String reviewedByName, diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillDetailResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillDetailResponse.java index e807b8335..fcd94f5a8 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillDetailResponse.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillDetailResponse.java @@ -32,7 +32,8 @@ public record SkillDetailResponse( List entryForSuites, PageResponse memberOfSuites, String avatarUrl, - boolean ownerReadOnly + boolean ownerReadOnly, + GithubSkillSourceResponse githubSource ) { public SkillDetailResponse(Long id, String slug, @@ -62,6 +63,6 @@ public record SkillDetailResponse( List entryForSuites, PageResponse memberOfSuites, String avatarUrl) { - this(id, slug, displayName, ownerId, ownerDisplayName, summary, visibility, status, downloadCount, starCount, subscriptionCount, ratingAvg, ratingCount, hidden, namespace, labels, canManageLifecycle, canSubmitPromotion, canInteract, canReport, headlineVersion, publishedVersion, ownerPreviewVersion, ownerPreviewReviewComment, resolutionMode, entryForSuites, memberOfSuites, avatarUrl, false); + this(id, slug, displayName, ownerId, ownerDisplayName, summary, visibility, status, downloadCount, starCount, subscriptionCount, ratingAvg, ratingCount, hidden, namespace, labels, canManageLifecycle, canSubmitPromotion, canInteract, canReport, headlineVersion, publishedVersion, ownerPreviewVersion, ownerPreviewReviewComment, resolutionMode, entryForSuites, memberOfSuites, avatarUrl, false, null); } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillLifecycleVersionResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillLifecycleVersionResponse.java index 5d526080f..f41ea0f5f 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillLifecycleVersionResponse.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillLifecycleVersionResponse.java @@ -1,13 +1,16 @@ package com.iflytek.skillhub.dto; +import com.fasterxml.jackson.annotation.JsonInclude; + public record SkillLifecycleVersionResponse( Long id, String version, String status, String displayName, - String summary + String summary, + @JsonInclude(JsonInclude.Include.NON_NULL) Integer syncRevision ) { public SkillLifecycleVersionResponse(Long id, String version, String status) { - this(id, version, status, null, null); + this(id, version, status, null, null, null); } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillVersionResponse.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillVersionResponse.java index e5a3c4c53..5187e44f7 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillVersionResponse.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/dto/SkillVersionResponse.java @@ -6,6 +6,7 @@ import java.time.Instant; public record SkillVersionResponse( Long id, String version, + Integer syncRevision, String status, @Schema(nullable = true) String changelog, int fileCount, diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java index 856fe1d14..e12c1cabc 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java @@ -1,6 +1,7 @@ package com.iflytek.skillhub.exception; import com.iflytek.skillhub.auth.exception.AuthFlowException; +import com.iflytek.skillhub.auth.policy.RouteSecurityPolicyRegistry; import com.iflytek.skillhub.dto.ApiResponse; import com.iflytek.skillhub.dto.ApiResponseFactory; import com.iflytek.skillhub.domain.shared.exception.LocalizedDomainException; @@ -184,15 +185,20 @@ public class GlobalExceptionHandler { @ExceptionHandler(StorageAccessException.class) public ResponseEntity> handleStorageAccess(StorageAccessException ex, HttpServletRequest request) { metrics.incrementStorageAccessFailure(ex.getOperation()); + if (isGithubSyncSensitiveRequest(request)) { + logger.warn("GitHub sync storage unavailable [requestId={}, operation={}]", + requestIdAccessor.current(), ex.getOperation()); + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body( + apiResponseFactory.error(503, "error.storage.unavailable")); + } logger.warn( - "Object storage unavailable [requestId={}, method={}, path={}, authentication={}, operation={}, key={}]", + "Object storage unavailable [requestId={}, method={}, path={}, authentication={}, operation={}, type={}]", requestIdAccessor.current(), request.getMethod(), sensitiveLogSanitizer.sanitizeRequestTarget(request), resolveAuthenticationState(request), ex.getOperation(), - ex.getKey(), - ex + ex.getClass().getSimpleName() ); return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body( apiResponseFactory.error(503, "error.storage.unavailable")); @@ -207,19 +213,30 @@ public class GlobalExceptionHandler { @ExceptionHandler(Exception.class) public ResponseEntity> handleGlobalException(Exception ex, HttpServletRequest request) { + if (isGithubSyncSensitiveRequest(request)) { + logger.error("Unhandled GitHub sync API exception [requestId={}, type={}]", + requestIdAccessor.current(), ex.getClass().getSimpleName()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body( + apiResponseFactory.error(500, "error.internal")); + } logger.error( - "Unhandled API exception [requestId={}, method={}, path={}, authentication={}]", + "Unhandled API exception [requestId={}, method={}, path={}, authentication={}, type={}]", requestIdAccessor.current(), request.getMethod(), sensitiveLogSanitizer.sanitizeRequestTarget(request), resolveAuthenticationState(request), - ex + ex.getClass().getSimpleName() ); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body( apiResponseFactory.error(500, "error.internal")); } private void logHandledException(HttpStatus status, String messageCode, HttpServletRequest request) { + if (isGithubSyncSensitiveRequest(request)) { + logger.info("GitHub sync API request failed [requestId={}, status={}, code={}]", + requestIdAccessor.current(), status.value(), messageCode); + return; + } logger.info( "API request failed [requestId={}, status={}, method={}, path={}, authentication={}, code={}]", requestIdAccessor.current(), @@ -246,4 +263,10 @@ public class GlobalExceptionHandler { } return "anonymous"; } + + private boolean isGithubSyncSensitiveRequest(HttpServletRequest request) { + String path = RouteSecurityPolicyRegistry.requestPath(request); + return path != null && (path.startsWith("/internal/github-sync/") + || path.startsWith("/api/v1/admin/github-sync/credentials")); + } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/filter/RequestLoggingFilter.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/filter/RequestLoggingFilter.java index 00e08dbaf..42e5797a5 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/filter/RequestLoggingFilter.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/filter/RequestLoggingFilter.java @@ -1,5 +1,6 @@ package com.iflytek.skillhub.filter; +import com.iflytek.skillhub.auth.policy.RouteSecurityPolicyRegistry; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -14,11 +15,10 @@ import org.springframework.web.util.ContentCachingRequestWrapper; import org.springframework.web.util.ContentCachingResponseWrapper; import java.io.IOException; -import java.util.Locale; import java.util.Set; /** - * Logs inbound HTTP requests with only core parameters to keep log files compact. + * Logs request method, path, status and timing without user-controlled query or header values. */ @Component @Order(Ordered.HIGHEST_PRECEDENCE + 1) @@ -26,13 +26,14 @@ public class RequestLoggingFilter extends OncePerRequestFilter { private static final Logger log = LoggerFactory.getLogger(RequestLoggingFilter.class); private static final Set SKIP_PREFIXES = Set.of( - "/actuator", "/favicon.ico", "/assets/" + "/actuator", "/favicon.ico", "/assets/", "/internal/github-sync/", + "/api/v1/admin/github-sync/credentials" ); @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { - String uri = request.getRequestURI(); + String uri = RouteSecurityPolicyRegistry.requestPath(request); if (shouldSkip(uri)) { filterChain.doFilter(request, response); return; @@ -54,47 +55,14 @@ public class RequestLoggingFilter extends OncePerRequestFilter { private void logRequest(ContentCachingRequestWrapper request, ContentCachingResponseWrapper response, long duration) { String requestUri = request.getRequestURI(); - String queryString = request.getQueryString(); - String fullUrl = queryString != null ? requestUri + "?" + sanitizeQueryString(queryString) : requestUri; - - String contentType = request.getContentType(); - String userAgent = request.getHeader("User-Agent"); - StringBuilder sb = new StringBuilder(); - sb.append(request.getMethod()).append(" ").append(fullUrl); + sb.append(request.getMethod()).append(" ").append(requestUri); sb.append(" | ").append(response.getStatus()); sb.append(" | ").append(duration).append("ms"); sb.append(" | ").append(request.getRemoteAddr()); - if (contentType != null) { - sb.append(" | Content-Type: ").append(contentType); - } - if (userAgent != null) { - sb.append(" | UA: ").append(truncate(userAgent, 80)); - } - log.info(sb.toString()); } - private String sanitizeQueryString(String queryString) { - return java.util.Arrays.stream(queryString.split("&", -1)) - .map(parameter -> { - int separator = parameter.indexOf('='); - if (separator < 0) { - return parameter; - } - String name = parameter.substring(0, separator).toLowerCase(Locale.ROOT); - return isSensitiveQueryParameter(name) - ? parameter.substring(0, separator) + "=[REDACTED]" - : parameter; - }) - .collect(java.util.stream.Collectors.joining("&")); - } - - private boolean isSensitiveQueryParameter(String name) { - return Set.of("code", "state", "error", "error_description", "error_uri", "access_token", - "refresh_token", "id_token", "client_secret").contains(name); - } - private boolean shouldSkip(String uri) { for (String prefix : SKIP_PREFIXES) { if (uri.startsWith(prefix)) { @@ -104,10 +72,4 @@ public class RequestLoggingFilter extends OncePerRequestFilter { return false; } - private String truncate(String value, int maxLength) { - if (value == null || value.length() <= maxLength) { - return value; - } - return value.substring(0, maxLength) + "...[truncated]"; - } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/listener/GithubSyncReviewResolutionListener.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/listener/GithubSyncReviewResolutionListener.java new file mode 100644 index 000000000..9b19494b9 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/listener/GithubSyncReviewResolutionListener.java @@ -0,0 +1,76 @@ +package com.iflytek.skillhub.listener; + +import com.iflytek.skillhub.domain.event.ReviewApprovedEvent; +import com.iflytek.skillhub.domain.event.ReviewRejectedEvent; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +import java.util.UUID; + +@Component +public class GithubSyncReviewResolutionListener { + private final JdbcTemplate jdbc; + + public GithubSyncReviewResolutionListener(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @TransactionalEventListener(phase = TransactionPhase.BEFORE_COMMIT) + public void approved(ReviewApprovedEvent event) { + settle(event.reviewId(), "APPROVED"); + } + + @TransactionalEventListener(phase = TransactionPhase.BEFORE_COMMIT) + public void rejected(ReviewRejectedEvent event) { + settle(event.reviewId(), "REJECTED"); + } + + private void settle(Long reviewId, String status) { + UUID ingestionId = jdbc.query(""" + SELECT github_sync_ingestion_id FROM review_task WHERE id = ? + """, result -> result.next() ? result.getObject(1, UUID.class) : null, reviewId); + if (ingestionId == null) { + return; + } + if ("APPROVED".equals(status)) { + Boolean needsResolution = jdbc.query(""" + SELECT namespace.type = 'CATALOG' + AND COALESCE(ingestion.license_evidence_json ->> 'status', 'UNKNOWN') <> 'RECOGNIZED' + AND NULLIF(BTRIM(review.review_comment), '') IS NULL AS required + FROM review_task review + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = review.github_sync_ingestion_id + JOIN namespace ON namespace.id = ingestion.namespace_id + WHERE review.id = ? + """, result -> result.next() ? result.getBoolean("required") : false, reviewId); + if (Boolean.TRUE.equals(needsResolution)) { + throw new DomainBadRequestException("review.sync.licenseResolutionRequired"); + } + } + UUID latestId = jdbc.query(""" + SELECT binding.latest_ingestion_id + FROM github_source_skill_binding binding + JOIN github_sync_package_ingestion ingestion + ON ingestion.namespace_id = binding.namespace_id + AND ingestion.repository_id = binding.repository_id + AND ingestion.skill_path = binding.skill_path + WHERE ingestion.id = ? + FOR UPDATE OF binding + """, result -> result.next() ? result.getObject(1, UUID.class) : null, ingestionId); + if (!ingestionId.equals(latestId)) { + throw new IllegalStateException("GitHub sync review is no longer the latest target"); + } + int updated = jdbc.update(""" + UPDATE github_sync_package_ingestion ingestion + SET status = ?, updated_at = CURRENT_TIMESTAMP + WHERE ingestion.id = ? + AND ingestion.status = 'READY_FOR_REVIEW' + """, status, ingestionId); + if (updated != 1) { + throw new IllegalStateException("GitHub sync review is no longer the latest target"); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetrics.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetrics.java new file mode 100644 index 000000000..f25842c62 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetrics.java @@ -0,0 +1,145 @@ +package com.iflytek.skillhub.metrics; + +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.storage.ObjectStorageService; +import com.iflytek.skillhub.storage.StorageProperties; +import com.iflytek.skillhub.storage.StoragePrefixUsage; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; + +/** Samples upload reservations and provider-specific staging capacity without trusting client sizes. */ +@Component +public class GithubSyncStagingCapacityMetrics { + private static final Logger log = LoggerFactory.getLogger(GithubSyncStagingCapacityMetrics.class); + private static final String STAGING_PREFIX = "github-sync/staging/"; + private static final int S3_MAX_OBJECTS_PER_SAMPLE = 10_000; + private final JdbcTemplate jdbc; + private final StorageProperties storage; + private final ObjectStorageService objects; + private final SkillPublishProperties publishPolicy; + private final long s3CapacityLimitBytes; + private final AtomicLong reservedBytes = new AtomicLong(); + private final AtomicLong localAvailableBytes = new AtomicLong(-1); + private final AtomicLong localCapacityWarning = new AtomicLong(); + private final AtomicLong s3UsageBytes = new AtomicLong(-1); + private final AtomicLong s3ObjectCount = new AtomicLong(-1); + private final AtomicLong s3ScanComplete = new AtomicLong(); + private final AtomicLong s3CapacityWarning = new AtomicLong(); + + public GithubSyncStagingCapacityMetrics(JdbcTemplate jdbc, StorageProperties storage, + ObjectStorageService objects, SkillPublishProperties publishPolicy, MeterRegistry registry, + @Value("${skillhub.github-sync.staging-capacity-limit-bytes:0}") long s3CapacityLimitBytes) { + if (s3CapacityLimitBytes < 0) { + throw new IllegalArgumentException("GitHub sync staging capacity limit cannot be negative"); + } + this.jdbc = jdbc; + this.storage = storage; + this.objects = objects; + this.publishPolicy = publishPolicy; + this.s3CapacityLimitBytes = s3CapacityLimitBytes; + Gauge.builder("skillhub.github.sync.staging.reserved.bytes", reservedBytes, AtomicLong::get) + .description("Declared bytes for staged uploads not yet cleaned; actual S3 object size may differ") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.local.available.bytes", localAvailableBytes, + AtomicLong::get) + .description("Usable bytes on the local staging filesystem, or -1 for S3") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.local.capacity.warning", localCapacityWarning, + AtomicLong::get) + .description("1 when local staging cannot fit one maximum-size package") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.s3.used.bytes", s3UsageBytes, AtomicLong::get) + .description("Actual bytes in the GitHub sync S3 staging prefix, or -1 if unavailable") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.s3.objects", s3ObjectCount, AtomicLong::get) + .description("Actual S3 staging object count, or -1 if unavailable") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.s3.scan.complete", s3ScanComplete, AtomicLong::get) + .description("1 when the bounded S3 staging prefix scan covered every object") + .register(registry); + Gauge.builder("skillhub.github.sync.staging.s3.capacity.warning", s3CapacityWarning, + AtomicLong::get) + .description("1 when S3 staging exceeds its configured byte budget or cannot be measured") + .register(registry); + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.upload-cleanup-interval-ms:300000}") + public void sample() { + Long reserved = jdbc.queryForObject(""" + SELECT COALESCE(sum(upload.size_bytes), 0)::bigint + FROM github_sync_staged_upload upload + LEFT JOIN github_sync_package_ingestion ingestion ON ingestion.upload_id = upload.id + WHERE (ingestion.id IS NULL AND upload.status <> 'CLEANED') + OR (ingestion.id IS NOT NULL AND ingestion.object_cleaned_at IS NULL) + """, Long.class); + reservedBytes.set(reserved == null ? 0 : reserved); + if (!"local".equals(storage.getProvider())) { + localAvailableBytes.set(-1); + localCapacityWarning.set(0); + return; + } + try { + Path path = Path.of(storage.getLocal().getBasePath()).toAbsolutePath(); + while (!Files.exists(path)) { + path = path.getParent(); + if (path == null) { + throw new IOException("local staging filesystem does not exist"); + } + } + long available = Files.getFileStore(path).getUsableSpace(); + localAvailableBytes.set(available); + boolean low = available < publishPolicy.getMaxPackageSize(); + localCapacityWarning.set(low ? 1 : 0); + if (low) { + log.warn("GitHub sync local staging capacity is below one maximum-size package: " + + "availableBytes={}, packageLimitBytes={}", + available, publishPolicy.getMaxPackageSize()); + } + } catch (IOException exception) { + localAvailableBytes.set(-1); + localCapacityWarning.set(1); + log.warn("Could not read GitHub sync local staging capacity: {}", + exception.getClass().getSimpleName()); + } + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.staging-capacity-sample-interval-ms:3600000}") + public void sampleS3() { + if (!"s3".equals(storage.getProvider())) { + return; + } + try { + StoragePrefixUsage usage = objects.listPrefixUsage(STAGING_PREFIX, S3_MAX_OBJECTS_PER_SAMPLE) + .orElseThrow(() -> new IllegalStateException("S3 prefix usage is unavailable")); + s3ObjectCount.set(usage.objectCount()); + s3ScanComplete.set(usage.complete() ? 1 : 0); + s3UsageBytes.set(usage.complete() ? usage.bytes() : -1); + boolean warning = !usage.complete() + || (s3CapacityLimitBytes == 0 && usage.objectCount() > 0) + || (s3CapacityLimitBytes > 0 && usage.bytes() >= s3CapacityLimitBytes); + s3CapacityWarning.set(warning ? 1 : 0); + if (warning) { + log.warn("GitHub sync S3 staging capacity needs attention: objects={}, " + + "scanComplete={}, usedBytes={}, configuredLimitBytes={}", + usage.objectCount(), usage.complete(), usage.bytes(), s3CapacityLimitBytes); + } + } catch (RuntimeException exception) { + s3UsageBytes.set(-1); + s3ObjectCount.set(-1); + s3ScanComplete.set(0); + s3CapacityWarning.set(1); + log.warn("Could not measure GitHub sync S3 staging capacity"); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDeepDiscoveryQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDeepDiscoveryQueryRepository.java new file mode 100644 index 000000000..5cd31c0b8 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDeepDiscoveryQueryRepository.java @@ -0,0 +1,459 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +/** Presentation joins use JDBC to page candidates before bounded provenance/path aggregation. */ +@Repository +@Transactional(readOnly = true) +public class GithubDeepDiscoveryQueryRepository { + private final NamedParameterJdbcTemplate jdbc; + private final com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate capacity; + public GithubDeepDiscoveryQueryRepository(JdbcTemplate jdbc, + com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate capacity) { + this.jdbc = new NamedParameterJdbcTemplate(jdbc); + this.capacity = capacity; + } + public record BudgetWait(boolean blocked, Instant retryAt) {} + + public BudgetWait discoveryBudgetWait() { + return jdbc.getJdbcTemplate().queryForObject(""" + SELECT budget.next_lease_eligible_at > clock_timestamp() OR EXISTS ( + SELECT 1 FROM github_sync_task WHERE kind='DISCOVERY' AND official_query_id IS NOT NULL + AND status='LEASED' AND lease_expires_at > clock_timestamp()) AS blocked, + CASE WHEN budget.next_lease_eligible_at > clock_timestamp() + THEN budget.next_lease_eligible_at END AS retry_at + FROM github_official_discovery_budget budget WHERE id + """, (rs, row) -> new BudgetWait(rs.getBoolean("blocked"), + rs.getTimestamp("retry_at") == null ? null : rs.getTimestamp("retry_at").toInstant())); + } + + private static final String STATUS = """ + CASE c.status WHEN 'BOUND' THEN 'VALID' WHEN 'REJECTED' THEN + CASE WHEN c.last_error_code='NO_VALID_SKILL' THEN 'NO_VALID_SKILL' ELSE 'VALIDATION_FAILED' END + ELSE c.status END + """; + private static final String BASE = """ + FROM github_discovery_candidate c JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id + WHERE c.namespace_id=:ns AND e.has_official + """; + private static final String PROJECTION = """ + SELECT c.*, e.source_id, NOT e.acquisition_allowed AS paused, + CASE WHEN e.source_status='PAUSED' THEN 'SOURCE_PAUSED' + WHEN NOT e.acquisition_allowed AND e.excluded THEN 'EXCLUDED' + WHEN NOT e.acquisition_allowed THEN 'DISABLED' ELSE NULL END AS wait_reason, + """ + STATUS + " AS display_status "; + private static final String CHANNELS = """ + SELECT s.*, o.channels, o.last_discovered_at FROM selected s + LEFT JOIN LATERAL (SELECT array_agg(DISTINCT q.definition_json->>'channel') AS channels, + max(o.last_seen_at) AS last_discovered_at + FROM github_discovery_candidate_origin o JOIN github_deep_discovery_query q ON q.id::text=o.origin_key + WHERE o.candidate_id=s.id AND o.origin_type='OFFICIAL_DISCOVERY') o ON TRUE + """; + + public DiscoveryPage repositories(long namespace, int page, int size, Instant asOf, + String search, RepositoryStatus status, Channel channel, Boolean paused) { + var params = parameters(namespace).addValue("asOf", Timestamp.from(asOf)); + String filters = " AND c.first_seen_at<=:asOf"; + if (search != null && !search.isBlank()) { + filters += " AND strpos(lower(c.owner_login || '/' || c.repository_name), lower(:search))>0"; + params.addValue("search", search.trim()); + } + if (status != null) { filters += " AND (" + STATUS + ")=:status"; params.addValue("status", status.name()); } + if (paused != null) { filters += " AND NOT e.acquisition_allowed=:paused"; params.addValue("paused", paused); } + if (channel != null) { + filters += " " + """ + AND EXISTS (SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_deep_discovery_query q ON q.id::text=origin.origin_key + WHERE origin.candidate_id=c.id AND origin.origin_type='OFFICIAL_DISCOVERY' + AND q.definition_json->>'channel'=:channel) + """; + params.addValue("channel", channel.name()); + } + String where = BASE + filters; + long total = count("SELECT count(*) " + where, params); + params.addValue("limit", size).addValue("offset", (long)page*size); + String selected; + if (paused == null) { + // Eligibility affects projected pause/source fields, not candidate selection here. + // Keep it correlated after pagination instead of expanding it for every candidate. + selected = "WITH candidate_page AS MATERIALIZED (SELECT c.* FROM github_discovery_candidate c " + + "WHERE c.namespace_id=:ns AND EXISTS (SELECT 1 FROM github_discovery_candidate_origin origin " + + "WHERE origin.candidate_id=c.id AND origin.origin_type='OFFICIAL_DISCOVERY')" + filters + + " ORDER BY c.first_seen_at DESC,c.id DESC LIMIT :limit OFFSET :offset), selected AS (" + + PROJECTION + " FROM candidate_page c JOIN LATERAL (" + + "SELECT * FROM github_candidate_discovery_eligibility eligibility " + + "WHERE eligibility.candidate_id=c.id AND eligibility.namespace_id=c.namespace_id " + + "AND eligibility.repository_id=c.repository_id OFFSET 0) e ON TRUE) "; + } else { + selected = "WITH selected AS (" + PROJECTION + where + + " ORDER BY c.first_seen_at DESC,c.id DESC LIMIT :limit OFFSET :offset) "; + } + var rows = jdbc.query(selected + CHANNELS + " ORDER BY s.first_seen_at DESC,s.id DESC", + params, (rs,n)->repository(rs)); + return new DiscoveryPage<>(rows,total,page,size,asOf); + } + + public RepositorySummary repository(long namespace, long repositoryId) { + var rows = jdbc.query("WITH selected AS (" + PROJECTION + BASE + + " AND c.repository_id=:repo AND e.namespace_id=:ns AND e.repository_id=:repo) " + CHANNELS, + parameters(namespace).addValue("repo",repositoryId), (rs,n)->repository(rs)); + if (rows.isEmpty()) throw new DomainNotFoundException("error.githubSync.source.notFound"); + return rows.getFirst(); + } + private RepositorySummary repository(ResultSet rs) throws SQLException { + var values=rs.getArray("channels"); + List channels=values==null?List.of():Arrays.stream((String[])values.getArray()) + .filter(java.util.Objects::nonNull).map(Channel::valueOf).sorted().toList(); + return new RepositorySummary(rs.getLong("repository_id"),rs.getString("owner_login"), + rs.getString("repository_name"),instant(rs,"first_seen_at"),instant(rs,"last_discovered_at"), + RepositoryStatus.valueOf(rs.getString("display_status")),channels,rs.getObject("source_id",Long.class), + rs.getBoolean("paused"),rs.getString("wait_reason"),rs.getLong("id")); + } + + public RepositoryDetail detail(long namespace,long repositoryId) { + var repository=repository(namespace,repositoryId); + var params=parameters(namespace).addValue("repo",Long.toString(repositoryId)); + record Task(Instant completed, Instant eligible, UUID run) {} + var tasks=jdbc.query(""" + SELECT t.completed_at, CASE WHEN t.status IN ('QUEUED','RETRY_WAIT') THEN GREATEST(t.due_at,t.retry_at) END eligible, + CASE WHEN r.details_purged_at IS NULL THEN r.id END run_id + FROM github_sync_task t JOIN github_sync_run r ON r.id=t.run_id + WHERE t.namespace_id=:ns AND t.kind='REPOSITORY_SYNC' AND t.snapshot_json->>'repositoryId'=:repo + ORDER BY t.created_at DESC,t.id DESC LIMIT 1 + """,params,(rs,n)->new Task(instant(rs,"completed_at"),instant(rs,"eligible"),rs.getObject("run_id",UUID.class))); + var task=tasks.isEmpty()?null:tasks.getFirst(); + Timestamp completed=jdbc.queryForObject(""" + SELECT max(completed_at) FROM github_sync_task WHERE namespace_id=:ns AND kind='REPOSITORY_SYNC' + AND snapshot_json->>'repositoryId'=:repo + """,params,Timestamp.class); + Instant checked=completed==null?null:completed.toInstant(); + Instant next=jdbc.query(""" + SELECT CASE WHEN e.acquisition_allowed THEN COALESCE(s.next_check_at,c.next_validation_at) END eligible + FROM github_discovery_candidate c JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id + LEFT JOIN github_source_binding s ON s.id=e.source_id WHERE c.id=:candidate + """,new MapSqlParameterSource("candidate",repository.candidateId()),rs->rs.next()?instant(rs,"eligible"):null); + if (task!=null && task.eligible()!=null && !repository.paused()) next=task.eligible(); + String reason=switch(repository.status()) { + case PENDING,VALIDATING -> "NOT_CHECKED"; + case NO_VALID_SKILL -> "NO_VALID_SKILL"; + case VALIDATION_FAILED -> "VALIDATION_FAILED"; + case UNREACHABLE -> "SOURCE_UNREADABLE"; + case VALID -> null; + }; + if ("EXCLUDED".equals(repository.waitReason()) || "DISABLED".equals(repository.waitReason())) reason=repository.waitReason(); + var ingestion=jdbc.query(""" + SELECT last_error_code FROM github_sync_package_ingestion WHERE namespace_id=:ns AND repository_id=:repository + AND status IN ('BLOCKED','READY_FOR_REVIEW') ORDER BY updated_at DESC,id DESC LIMIT 1 + """,params.addValue("repository",repositoryId),(rs,n)->rs.getString("last_error_code")); + if (!ingestion.isEmpty() && ingestion.getFirst()!=null) reason= + ingestion.getFirst().contains("CONFLICT") || ingestion.getFirst().startsWith("PATH_MOVE_") + ?"CONFLICT":"VALIDATION_FAILED"; + if (count(""" + SELECT count(*) FROM review_task r JOIN github_sync_package_ingestion i ON i.id=r.github_sync_ingestion_id + WHERE i.namespace_id=:ns AND i.repository_id=:repository AND r.status='PENDING' + """,params)>0) reason="REVIEW_PENDING"; + return new RepositoryDetail(repository,checked,next,reason,List.of(),task==null?null:task.run(),null); + } + + public DiscoveryPage provenance(long namespace,long repositoryId,int page,int size,Instant asOf) { + var repository=repository(namespace,repositoryId); + var params=parameters(namespace).addValue("candidate",repository.candidateId()).addValue("asOf",Timestamp.from(asOf)) + .addValue("limit",size).addValue("offset",(long)page*size); + String where=" FROM github_discovery_candidate_origin o WHERE o.candidate_id=:candidate AND o.origin_type='OFFICIAL_DISCOVERY' AND o.first_seen_at<=:asOf"; + long total=count("SELECT count(*)"+where,params); + var rows=jdbc.query("WITH selected AS (SELECT o.*"+where+" ORDER BY o.last_seen_at DESC,o.id DESC LIMIT :limit OFFSET :offset) " + """ + SELECT o.*,q.id query_id,q.query_version,q.definition_json->>'channel' channel, + CASE WHEN r.details_purged_at IS NULL THEN r.id END run_id + FROM selected o JOIN github_deep_discovery_query q ON q.id::text=o.origin_key + LEFT JOIN github_sync_run r ON r.id=o.recent_run_id + ORDER BY o.last_seen_at DESC,o.id DESC + """,params,(rs,n)->new Provenance(rs.getObject("id",UUID.class),Channel.valueOf(rs.getString("channel")), + rs.getString("latest_strategy_version"),rs.getObject("query_id",UUID.class),rs.getLong("query_version"), + rs.getObject("recent_shard_id",UUID.class),rs.getObject("run_id",UUID.class),instant(rs,"first_seen_at"), + instant(rs,"last_seen_at"),rs.getObject("recent_seed_repository_id",Long.class))); + return new DiscoveryPage<>(rows,total,page,size,asOf); + } + + public DiscoveryPage paths(long namespace,long repositoryId,int page,int size,Instant asOf) { + repository(namespace,repositoryId); + return pathsForRepository(namespace,repositoryId,page,size,asOf); + } + + /** Shared safe path projection; caller must check repository visibility before invoking. */ + DiscoveryPage pathsForRepository(long namespace,long repositoryId,int page,int size,Instant asOf) { + var params=parameters(namespace).addValue("repo",Long.toString(repositoryId)).addValue("asOf",Timestamp.from(asOf)) + .addValue("limit",size).addValue("offset",(long)page*size); + String latest=""" + WITH latest AS (SELECT DISTINCT ON (p.skill_path) p.*,t.run_id + FROM github_sync_task t JOIN github_sync_path_result p ON p.task_id=t.id + WHERE t.namespace_id=:ns AND t.kind='REPOSITORY_SYNC' AND t.snapshot_json->>'repositoryId'=:repo + AND p.created_at<=:asOf + ORDER BY p.skill_path,t.created_at DESC,t.id DESC,p.updated_at DESC,p.id DESC) + """; + long total=count(latest+" SELECT count(*) FROM latest",params); + var rows=jdbc.query(latest+""" + ,selected AS (SELECT * FROM latest ORDER BY skill_path LIMIT :limit OFFSET :offset) + SELECT p.*,COALESCE(i.published_skill_id,reused.published_skill_id) published_skill_id, + COALESCE(i.published_version_id,reused.published_version_id) published_version_id,review.id review_id, + CASE WHEN r.details_purged_at IS NULL THEN r.id END visible_run_id + FROM selected p LEFT JOIN github_sync_package_ingestion i ON i.id=p.ingestion_id + LEFT JOIN LATERAL ( + SELECT published.id,published.published_skill_id,published.published_version_id + FROM github_source_skill_binding binding + JOIN github_sync_package_ingestion published ON published.id=binding.latest_ingestion_id + AND published.namespace_id=binding.namespace_id AND published.repository_id=binding.repository_id + AND published.skill_path=binding.skill_path AND published.content_sha256=binding.latest_content_sha256 + AND published.published_skill_id=binding.skill_id AND published.created_at<=i.created_at + JOIN skill_version version ON version.id=published.published_version_id + AND version.skill_id=binding.skill_id AND version.github_sync_ingestion_id=published.id + WHERE i.status='NO_CHANGE' AND binding.namespace_id=i.namespace_id + AND binding.repository_id=i.repository_id AND binding.skill_path=i.skill_path + AND binding.latest_content_sha256=i.content_sha256 + ) reused ON TRUE + LEFT JOIN LATERAL (SELECT id FROM review_task WHERE github_sync_ingestion_id=COALESCE(reused.id,i.id) + ORDER BY submitted_at DESC,id DESC LIMIT 1) review ON TRUE + LEFT JOIN github_sync_run r ON r.id=p.run_id ORDER BY p.skill_path + """,params,(rs,n)->new PathResult(rs.getString("skill_path"),rs.getString("outcome"), + reasonCode(rs.getString("reason")),rs.getObject("published_skill_id",Long.class), + rs.getObject("published_version_id",Long.class),rs.getObject("review_id",Long.class), + rs.getObject("visible_run_id",UUID.class),instant(rs,"updated_at"))); + return new DiscoveryPage<>(rows,total,page,size,asOf); + } + + public Recent recent(long namespace,Instant from,Instant until) { + Instant since=jdbc.queryForObject("SELECT available_since FROM github_discovery_metrics_epoch WHERE id",new MapSqlParameterSource(),Timestamp.class).toInstant(); + if(from.isBefore(since)) return new Recent(from,until,null,null,null,null,false,since); + var metrics=jdbc.query(""" + SELECT event_type,count(*) value FROM github_discovery_outcome_event + WHERE namespace_id=:ns AND official_at_event AND occurred_at>=:from AND occurred_at<:until GROUP BY event_type + """,parameters(namespace).addValue("from",Timestamp.from(from)).addValue("until",Timestamp.from(until)), + rs->{var map=new java.util.HashMap();while(rs.next())map.put(rs.getString("event_type"),rs.getLong("value"));return map;}); + boolean uncertain=count(""" + SELECT count(*) FROM github_discovery_outcome_event + WHERE namespace_id=:ns AND official_at_event AND event_type='FIRST_VALID' AND NOT first_occurrence_known + AND occurred_at>=:from AND occurred_at<:until + """,parameters(namespace).addValue("from",Timestamp.from(from)).addValue("until",Timestamp.from(until)))>0; + return new Recent(from,until,metrics.getOrDefault("CANDIDATE_CREATED",0L),uncertain?null:metrics.getOrDefault("FIRST_VALID",0L), + metrics.getOrDefault("NEW_SKILL",0L),metrics.getOrDefault("UPDATED_VERSION",0L),!uncertain,since); + } + public Current current(long namespace,Instant observed) { + var params=parameters(namespace); + long pending=count("SELECT count(*) "+BASE+" AND c.status IN ('PENDING','VALIDATING','REJECTED')",params); + long reviews=count(""" + SELECT count(DISTINCT r.subject_version_id) FROM review_task r + JOIN github_sync_package_ingestion i ON i.id=r.github_sync_ingestion_id + JOIN github_discovery_candidate c ON c.namespace_id=i.namespace_id AND c.repository_id=i.repository_id + WHERE i.namespace_id=:ns AND r.subject_type='SKILL_VERSION' AND r.status='PENDING' + AND EXISTS(SELECT 1 FROM github_discovery_candidate_origin o WHERE o.candidate_id=c.id AND o.origin_type='OFFICIAL_DISCOVERY') + """,params); + return new Current(pending,reviews,observed); + } + public List recentRuns(long namespace) { + return jdbc.query(""" + SELECT r.id FROM github_sync_run r WHERE r.namespace_id=:ns AND r.details_purged_at IS NULL + AND EXISTS(SELECT 1 FROM github_sync_task t WHERE t.run_id=r.id AND (t.official_query_id IS NOT NULL OR + EXISTS(SELECT 1 FROM github_discovery_candidate c JOIN github_discovery_candidate_origin o ON o.candidate_id=c.id + WHERE c.namespace_id=t.namespace_id AND c.repository_id::text=t.snapshot_json->>'repositoryId' AND o.origin_type='OFFICIAL_DISCOVERY'))) + ORDER BY r.created_at DESC,r.id DESC LIMIT 5 + """,parameters(namespace),(rs,n)->rs.getObject("id",UUID.class)); + } + public record ChannelState(Channel channel,long completed,long pending,long gaps,Instant progress,boolean running,Instant retry) {} + public List channelStates(long namespace) { + return jdbc.query(""" + SELECT q.definition_json->>'channel' channel, + COALESCE(sum(CASE WHEN q.query_source='OWNER' THEN CASE WHEN q.active_task_id IS NULL AND q.last_full_recheck_at IS NOT NULL AND q.gap_reason IS NULL AND q.next_due_at>CURRENT_TIMESTAMP THEN 1 ELSE 0 END ELSE s.completed END),0) completed, + COALESCE(sum(CASE WHEN q.query_source='OWNER' THEN CASE WHEN (q.active_task_id IS NOT NULL OR q.next_due_at<=CURRENT_TIMESTAMP) AND q.gap_reason IS NULL THEN 1 ELSE 0 END ELSE CASE WHEN s.total=0 AND q.next_due_at<=CURRENT_TIMESTAMP THEN 1 ELSE s.pending END END),0) pending, + COALESCE(sum(CASE WHEN q.query_source='OWNER' THEN CASE WHEN q.gap_reason IS NOT NULL THEN 1 ELSE 0 END ELSE s.gaps END),0) gaps, + max(q.last_progress_at) progress, + bool_or(t.status='LEASED' AND t.lease_expires_at>CURRENT_TIMESTAMP AND t.last_progress_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds') running, + min(GREATEST(t.due_at,t.retry_at)) FILTER(WHERE t.status IN ('QUEUED','RETRY_WAIT')) retry + FROM github_deep_discovery_query q LEFT JOIN github_sync_task t ON t.id=q.active_task_id + LEFT JOIN LATERAL (SELECT count(*) total,count(*) FILTER(WHERE state='COMPLETE') completed, + count(*) FILTER(WHERE state IN ('PENDING','ACTIVE')) pending,count(*) FILTER(WHERE state='GAP') gaps + FROM github_official_discovery_shard WHERE query_id=q.id) s ON TRUE + WHERE q.namespace_id=:ns AND (q.retired_at IS NULL OR q.active_task_id IS NOT NULL + OR EXISTS(SELECT 1 FROM github_official_discovery_shard active_root + WHERE active_root.query_id=q.id AND active_root.parent_id IS NULL AND active_root.state<>'COMPLETE')) GROUP BY q.definition_json->>'channel' + """,parameters(namespace),(rs,n)->new ChannelState(Channel.valueOf(rs.getString("channel")),rs.getLong("completed"), + rs.getLong("pending"),rs.getLong("gaps"),instant(rs,"progress"),rs.getBoolean("running"),instant(rs,"retry"))); + } + /** Each row is one last observation, never a sum of token allowances. */ + public List quotas() { + return jdbc.query(""" + SELECT o.resource,o.observed_at,o.valid_until, + CASE WHEN o.valid_until<=CURRENT_TIMESTAMP THEN 'UNKNOWN' + WHEN o.state='PRIMARY_EXHAUSTED' THEN 'EXHAUSTED' + WHEN o.state='SECONDARY_COOLDOWN' THEN 'COOLING_DOWN' ELSE o.state END display_state, + CASE WHEN o.valid_until>CURRENT_TIMESTAMP AND o.state IN ('OBSERVED','PRIMARY_EXHAUSTED') THEN o.remaining::bigint END remaining, + CASE WHEN GREATEST(CASE WHEN o.valid_until>CURRENT_TIMESTAMP AND o.state='PRIMARY_EXHAUSTED' THEN o.reset_at END,c.primary_until,s.cooldown_until)>CURRENT_TIMESTAMP + THEN GREATEST(CASE WHEN o.valid_until>CURRENT_TIMESTAMP AND o.state='PRIMARY_EXHAUSTED' THEN o.reset_at END,c.primary_until,s.cooldown_until) END retry_at + FROM github_official_quota_observation o + LEFT JOIN github_official_resource_cooldown c ON c.resource=o.resource AND o.credential_id IS NULL + LEFT JOIN github_official_secondary_cooldown s ON s.id + ORDER BY o.observed_at DESC,o.observation_key,o.resource LIMIT 101 + """,new MapSqlParameterSource(),(rs,n)->new Quota(rs.getString("resource"),null,rs.getString("display_state"), + rs.getObject("remaining",Long.class),instant(rs,"observed_at"),instant(rs,"valid_until"),instant(rs,"retry_at"))); + } + public record QuotaWait(String reason,String resource,Instant retryAt) {} + public List quotaWaits() { + return jdbc.query(""" + SELECT 'QUOTA_COOLDOWN' reason,NULL::text resource,cooldown_until retry_at + FROM github_official_secondary_cooldown WHERE cooldown_until>CURRENT_TIMESTAMP + UNION ALL + SELECT 'PRIMARY_RATE_LIMIT',resource,primary_until FROM github_official_resource_cooldown + WHERE primary_until>CURRENT_TIMESTAMP + AND NOT EXISTS(SELECT 1 FROM github_system_credential) + """,new MapSqlParameterSource(),(rs,n)->new QuotaWait(rs.getString("reason"),rs.getString("resource"),instant(rs,"retry_at"))); + } + public boolean admissionBlocked(long namespace) { + return !capacity.allowsFullAcquisition(namespace, Instant.now()); + } + /** Same eligibility as credential selection, without locking rows or reading secrets. */ + public boolean credentialPoolUnavailable(String resource) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS(SELECT 1 FROM github_system_credential) AND NOT EXISTS( + SELECT 1 FROM github_system_credential c WHERE c.status='ACTIVE' + AND (c.rate_limited_until IS NULL OR c.rate_limited_until<=CURRENT_TIMESTAMP) + AND NOT EXISTS(SELECT 1 FROM github_system_credential_quota q + WHERE q.credential_id=c.id AND q.resource=:resource AND q.remaining=0 AND q.reset_at>CURRENT_TIMESTAMP)) + """,new MapSqlParameterSource("resource",resource),Boolean.class)); + } + public boolean candidateCapacityBlocked(long namespace) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT (SELECT count(*) FROM github_discovery_candidate WHERE namespace_id=:ns + AND status IN ('PENDING','VALIDATING')) >= candidate_pending_limit + FROM github_sync_admission_policy WHERE id + """,parameters(namespace),Boolean.class)); + } + public record WaitEvidence(String reason,String work,Instant retryAt) {} + public List discoveryWaits(long namespace,Channel channel) { + var params=parameters(namespace).addValue("channel",channel==null?null:channel.name()); + return jdbc.query(""" + WITH active_queries AS ( + SELECT q.* FROM github_deep_discovery_query q WHERE q.namespace_id=:ns + AND (CAST(:channel AS text) IS NULL OR q.definition_json->>'channel'=:channel) + AND (q.retired_at IS NULL OR q.active_task_id IS NOT NULL OR EXISTS( + SELECT 1 FROM github_official_discovery_shard s WHERE s.query_id=q.id AND s.parent_id IS NULL AND s.state<>'COMPLETE')) + ), evidence AS ( + SELECT q.wait_reason reason,GREATEST(t.due_at,t.retry_at) retry_at FROM active_queries q + LEFT JOIN github_sync_task t ON t.id=q.active_task_id WHERE q.wait_reason IS NOT NULL AND q.wait_reason NOT IN ('DISABLED','EXCLUDED','SOURCE_PAUSED') + UNION ALL SELECT t.last_error_code,GREATEST(t.due_at,t.retry_at) FROM active_queries q + JOIN github_sync_task t ON t.id=q.active_task_id WHERE t.status IN ('QUEUED','RETRY_WAIT') AND t.last_error_code IS NOT NULL AND t.last_error_code NOT IN ('DISABLED','EXCLUDED','SOURCE_PAUSED') + UNION ALL SELECT t.official_pause_reason,GREATEST(t.due_at,t.retry_at) FROM active_queries q + JOIN github_sync_task t ON t.id=q.active_task_id WHERE t.status IN ('QUEUED','RETRY_WAIT') AND t.official_pause_reason IS NOT NULL AND t.official_pause_reason NOT IN ('DISABLED','EXCLUDED','SOURCE_PAUSED') + UNION ALL SELECT 'COVERAGE_GAP',s.retry_at FROM active_queries q + JOIN github_official_discovery_shard s ON s.query_id=q.id WHERE s.state='GAP' + UNION ALL SELECT 'COVERAGE_GAP',GREATEST(q.next_due_at,t.retry_at) FROM active_queries q + LEFT JOIN github_sync_task t ON t.id=q.active_task_id WHERE q.query_source='OWNER' AND q.gap_reason IS NOT NULL + UNION ALL SELECT 'EXCLUDED',NULL FROM active_queries q + JOIN github_official_owner_eligibility e ON e.query_id=q.id WHERE e.excluded + ) SELECT reason,CASE WHEN bool_or(retry_at IS NULL) THEN NULL ELSE max(retry_at) END retry_at + FROM evidence WHERE reason IN ('DISABLED','EXCLUDED','SOURCE_PAUSED','BUDGET','PRIMARY_RATE_LIMIT','SECONDARY_COOLDOWN','QUOTA_COOLDOWN','CREDENTIAL_UNAVAILABLE','CANDIDATE_CAPACITY','REVIEW_BACKPRESSURE','COVERAGE_GAP','GITHUB_RATE_LIMITED','BACKPRESSURE_ACTIVE','CANDIDATE_BUDGET_EXCEEDED') GROUP BY reason ORDER BY reason LIMIT 32 + """,params,(rs,n)->new WaitEvidence(rs.getString("reason"),"NEW_DISCOVERY",instant(rs,"retry_at"))); + } + public List repositoryWaits(long namespace,Long repositoryId) { + var params=parameters(namespace).addValue("repo",repositoryId==null?null:repositoryId.toString()).addValue("numericRepo",repositoryId); + return jdbc.query(""" + WITH relevant AS ( + SELECT t.*,CASE WHEN e.source_id IS NULL THEN 'CANDIDATE_VALIDATION' ELSE 'SOURCE_UPDATE' END work + FROM github_sync_task t JOIN github_discovery_candidate c ON c.namespace_id=t.namespace_id + AND c.repository_id::text=t.snapshot_json->>'repositoryId' + JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id + WHERE t.namespace_id=:ns AND t.kind='REPOSITORY_SYNC' AND e.has_official + AND (CAST(:repo AS text) IS NULL OR t.snapshot_json->>'repositoryId'=:repo) + AND t.status IN ('QUEUED','LEASED','RETRY_WAIT') + ), evidence AS ( + SELECT official_pause_reason reason,work,GREATEST(due_at,retry_at) retry_at FROM relevant + WHERE status IN ('QUEUED','RETRY_WAIT') AND official_pause_reason IS NOT NULL AND official_pause_reason NOT IN ('DISABLED','EXCLUDED','SOURCE_PAUSED') + UNION ALL SELECT last_error_code,work,GREATEST(due_at,retry_at) FROM relevant + WHERE status='RETRY_WAIT' AND last_error_code IS NOT NULL AND last_error_code NOT IN ('DISABLED','EXCLUDED','SOURCE_PAUSED') + UNION ALL SELECT CASE WHEN e.source_status='PAUSED' THEN 'SOURCE_PAUSED' + WHEN e.excluded THEN 'EXCLUDED' ELSE 'DISABLED' END, + CASE WHEN e.source_id IS NULL THEN 'CANDIDATE_VALIDATION' ELSE 'SOURCE_UPDATE' END,NULL + FROM github_candidate_discovery_eligibility e WHERE e.namespace_id=:ns AND e.has_official + AND NOT e.acquisition_allowed AND (CAST(:numericRepo AS bigint) IS NULL OR e.repository_id=:numericRepo) + ) SELECT reason,work,CASE WHEN bool_or(retry_at IS NULL) THEN NULL ELSE max(retry_at) END retry_at + FROM evidence WHERE reason IN ('DISABLED','EXCLUDED','SOURCE_PAUSED','BUDGET','PRIMARY_RATE_LIMIT','SECONDARY_COOLDOWN','QUOTA_COOLDOWN','CREDENTIAL_UNAVAILABLE','CANDIDATE_CAPACITY','REVIEW_BACKPRESSURE','COVERAGE_GAP','GITHUB_RATE_LIMITED','BACKPRESSURE_ACTIVE','CANDIDATE_BUDGET_EXCEEDED') GROUP BY reason,work ORDER BY reason,work LIMIT 32 + """,params,(rs,n)->new WaitEvidence(rs.getString("reason"),rs.getString("work"),instant(rs,"retry_at"))); + } + public boolean compatibleWorker(Channel channel) { + String capabilities=channel==Channel.OWNER_EXPANSION + ? "ARRAY['official-discovery/v1','official-checkpoint/v1','official-pause/v1','official-owner/v1','platform-pause/v1']::text[]" + : "ARRAY['official-discovery/v1','official-checkpoint/v1','official-pause/v1','platform-pause/v1']::text[]"; + return count("SELECT count(*) FROM github_sync_worker_instance WHERE protocol_major=1 AND last_seen_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds' AND capabilities @> "+capabilities,new MapSqlParameterSource())>0; + } + public boolean repositoryNeedsOfficialWorker(long namespace,long repositoryId) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT official_only FROM github_candidate_discovery_eligibility WHERE namespace_id=:ns AND repository_id=:repo + """,parameters(namespace).addValue("repo",repositoryId),Boolean.class)); + } + public boolean hasOfficialOnlyCandidates(long namespace) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_candidate_discovery_eligibility + WHERE namespace_id=:ns AND has_official AND official_only) + """,parameters(namespace),Boolean.class)); + } + public boolean pauseCapableWorker() { + return count(""" + SELECT count(*) FROM github_sync_worker_instance WHERE protocol_major=1 + AND last_seen_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds' + AND capabilities @> ARRAY['official-pause/v1','platform-pause/v1']::text[] + """,new MapSqlParameterSource())>0; + } + public List progressingWork(long namespace,Long repositoryId,Channel channel) { + if (channel != null && repositoryId == null) { + // Channel progress only concerns official discovery. Avoid expanding candidate + // eligibility for every repository when no repository work can match this filter. + return jdbc.query(""" + SELECT DISTINCT 'NEW_DISCOVERY' AS work FROM github_sync_task t + JOIN github_deep_discovery_query q ON q.id=t.official_query_id + JOIN github_deep_discovery_config config ON config.namespace_id=t.namespace_id + WHERE t.namespace_id=:ns AND t.kind='DISCOVERY' AND t.status='LEASED' + AND t.lease_expires_at>CURRENT_TIMESTAMP + AND t.last_progress_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds' + AND q.definition_json->>'channel'=:channel AND config.enabled + AND NOT EXISTS (SELECT 1 FROM github_official_owner_eligibility oe + WHERE oe.query_id=q.id AND oe.excluded) + """,parameters(namespace).addValue("channel",channel.name()),(rs,n)->rs.getString("work")); + } + return jdbc.query(""" + WITH tasks AS ( + SELECT t.kind,t.mode,COALESCE(e.source_id,t.source_id) source_id FROM github_sync_task t + LEFT JOIN github_discovery_candidate c ON c.namespace_id=t.namespace_id + AND c.repository_id::text=t.snapshot_json->>'repositoryId' + AND (CAST(:numericRepo AS bigint) IS NULL OR c.repository_id=:numericRepo) + LEFT JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id AND e.namespace_id=:ns + AND (CAST(:numericRepo AS bigint) IS NULL OR e.repository_id=:numericRepo) + LEFT JOIN github_deep_discovery_query q ON q.id=t.official_query_id + JOIN github_deep_discovery_config config ON config.namespace_id=t.namespace_id + WHERE t.namespace_id=:ns AND t.status='LEASED' AND t.lease_expires_at>CURRENT_TIMESTAMP + AND t.last_progress_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds' + AND (CAST(:repo AS text) IS NULL OR t.snapshot_json->>'repositoryId'=:repo) + AND (CAST(:channel AS text) IS NULL OR q.definition_json->>'channel'=:channel) + AND ((t.official_query_id IS NOT NULL AND config.enabled AND NOT EXISTS( + SELECT 1 FROM github_official_owner_eligibility oe WHERE oe.query_id=q.id AND oe.excluded)) + OR (e.has_official AND e.acquisition_allowed)) + ) SELECT DISTINCT CASE WHEN kind='DISCOVERY' THEN 'NEW_DISCOVERY' + WHEN source_id IS NULL THEN 'CANDIDATE_VALIDATION' ELSE 'SOURCE_UPDATE' END work FROM tasks + UNION SELECT 'PACKAGE_ACQUISITION' FROM tasks WHERE kind='REPOSITORY_SYNC' AND mode='FULL' + """,parameters(namespace).addValue("repo",repositoryId==null?null:repositoryId.toString()) + .addValue("channel",channel==null?null:channel.name()).addValue("numericRepo",repositoryId),(rs,n)->rs.getString("work")); + } + private long count(String sql,MapSqlParameterSource params) { return jdbc.queryForObject(sql,params,Long.class); } + private static MapSqlParameterSource parameters(long namespace) { return new MapSqlParameterSource("ns",namespace); } + private static Instant instant(ResultSet rs,String column) throws SQLException { var value=rs.getTimestamp(column);return value==null?null:value.toInstant(); } + private static String reasonCode(String value) { return value!=null && value.matches("[A-Z][A-Z0-9_]{0,99}")?value:null; } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDiscoveryEntryStatusQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDiscoveryEntryStatusQueryRepository.java new file mode 100644 index 000000000..c956cea56 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubDiscoveryEntryStatusQueryRepository.java @@ -0,0 +1,65 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.dto.GithubDiscoveryEntryResponse.Execution; +import java.sql.Timestamp; +import java.util.UUID; +import java.util.Map; +import java.util.List; +import java.util.HashMap; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +/** Reads only durable execution identity and safe codes; never returns task snapshots or error bodies. */ +@Repository +public class GithubDiscoveryEntryStatusQueryRepository { + private final JdbcTemplate jdbc; + public GithubDiscoveryEntryStatusQueryRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public Map latestForEntries(long namespace, List entries) { + if (entries.isEmpty()) { + return Map.of(); + } + var named = new NamedParameterJdbcTemplate(jdbc); + var values = new MapSqlParameterSource("ns", namespace) + .addValue("entries", entries.stream().map(UUID::toString).toList()); + return named.query(""" + SELECT DISTINCT ON (t.snapshot_json->>'ruleId') t.snapshot_json->>'ruleId' entry_id, + t.id,t.status,t.last_error_code,t.retry_at, + CASE WHEN r.details_purged_at IS NULL THEN r.id END run_id + FROM github_sync_task t JOIN github_sync_run r ON r.id=t.run_id AND r.namespace_id=t.namespace_id + WHERE t.namespace_id=:ns AND t.snapshot_json->>'ruleId' IN (:entries) + AND t.snapshot_json->>'entryType' IN ('CURATED_REPOSITORY','CURATED_OWNER','SEARCH_RULE') + ORDER BY t.snapshot_json->>'ruleId',t.created_at DESC,t.id DESC + """, values, rs -> { + Map result = new HashMap<>(); + while (rs.next()) { + Timestamp retry = rs.getTimestamp("retry_at"); + result.put(UUID.fromString(rs.getString("entry_id")), new Execution(rs.getObject("id", UUID.class), + rs.getObject("run_id", UUID.class),rs.getString("status"),rs.getString("last_error_code"), + retry == null ? null : retry.toInstant())); + } + return result; + }); + } + public Execution latest(long namespace, UUID entryId) { + return jdbc.query(""" + SELECT t.id,t.status,t.last_error_code,t.retry_at, + CASE WHEN r.details_purged_at IS NULL THEN r.id END run_id + FROM github_sync_task t JOIN github_sync_run r ON r.id=t.run_id AND r.namespace_id=t.namespace_id + WHERE t.namespace_id=? AND t.snapshot_json->>'ruleId'=? + AND t.snapshot_json->>'entryType' IN ('CURATED_REPOSITORY','CURATED_OWNER','SEARCH_RULE') + ORDER BY t.created_at DESC,t.id DESC LIMIT 1 + """, rs -> { + if (!rs.next()) { + return null; + } + Timestamp retry = rs.getTimestamp("retry_at"); + return new Execution(rs.getObject("id", UUID.class), rs.getObject("run_id", UUID.class), + rs.getString("status"), rs.getString("last_error_code"), retry == null ? null : retry.toInstant()); + }, namespace, entryId.toString()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSkillSourceQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSkillSourceQueryRepository.java new file mode 100644 index 000000000..6b0b55ad4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSkillSourceQueryRepository.java @@ -0,0 +1,70 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.dto.GithubSkillSourceResponse; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@Repository +public class GithubSkillSourceQueryRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper objectMapper; + + public GithubSkillSourceQueryRepository(JdbcTemplate jdbc, ObjectMapper objectMapper) { + this.jdbc = jdbc; + this.objectMapper = objectMapper; + } + + /** Call only after SkillQueryService has authorized the viewer and selected the published version. */ + public Optional findPublishedVersion(long versionId) { + return jdbc.query(""" + SELECT COALESCE(source.owner_login, task.snapshot_json ->> 'owner') AS owner_login, + COALESCE(source.repository_name, task.snapshot_json ->> 'name') AS repository_name, + ingestion.skill_path, ingestion.commit_sha, version.github_sync_revision, + ingestion.license_evidence_json::text AS evidence_json + FROM skill_version version + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = version.github_sync_ingestion_id + AND ingestion.published_version_id = version.id + JOIN github_sync_task task ON task.id = ingestion.task_id + LEFT JOIN github_source_binding source + ON source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id + WHERE version.id = ? AND version.status = 'PUBLISHED' + """, result -> { + if (!result.next()) { + return Optional.empty(); + } + JsonNode evidence = evidence(result.getString("evidence_json")); + return Optional.of(new GithubSkillSourceResponse( + result.getString("owner_login"), result.getString("repository_name"), + result.getString("skill_path"), result.getString("commit_sha"), + result.getInt("github_sync_revision"), + evidence.path("status").asText("UNKNOWN"), licenseDocuments(evidence))); + }, versionId); + } + + private JsonNode evidence(String json) { + try { + return objectMapper.readTree(json); + } catch (IOException exception) { + throw new IllegalStateException("Invalid stored GitHub license evidence", exception); + } + } + + private List licenseDocuments(JsonNode evidence) { + List documents = new ArrayList<>(); + for (JsonNode document : evidence.path("documents")) { + documents.add(new GithubSkillSourceResponse.LicenseDocument( + document.path("scope").asText(), document.path("path").asText(), + document.path("bundled").asBoolean())); + } + return List.copyOf(documents); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncManagementQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncManagementQueryRepository.java new file mode 100644 index 000000000..185218e7e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncManagementQueryRepository.java @@ -0,0 +1,460 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.dto.GithubSyncRunSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncRunDetailResponse; +import com.iflytek.skillhub.dto.GithubSyncRunTaskResponse; +import com.iflytek.skillhub.dto.GithubSyncRunPathResponse; +import com.iflytek.skillhub.dto.GithubSyncSourceSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSkillPathResponse; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.RunKind; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.RunStatus; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Target; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.githubsync.GithubSkillPathPolicy; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +/** Pages source and run rows before joining their current acquisition and ingestion summaries. */ +@Repository +public class GithubSyncManagementQueryRepository { + private final JdbcTemplate jdbc; + private final GithubSourceIgnoreRepository ignores; + private final JdbcGithubSyncCapacityGate capacityGate; + + public GithubSyncManagementQueryRepository(JdbcTemplate jdbc, GithubSourceIgnoreRepository ignores, + JdbcGithubSyncCapacityGate capacityGate) { + this.jdbc = jdbc; + this.ignores = ignores; + this.capacityGate = capacityGate; + } + + public long platformNamespaceId() { + return Objects.requireNonNull(jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class), "@github catalog namespace is missing"); + } + + @Transactional(readOnly = true) + public AdmissionSnapshot admissionSnapshot() { + long platformNamespaceId = platformNamespaceId(); + Instant now = Instant.now(); + Integer pendingCandidates = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND status IN ('PENDING', 'VALIDATING') + """, Integer.class, platformNamespaceId); + Boolean platformPaused = jdbc.query(""" + SELECT paused FROM github_sync_namespace_admission_state WHERE namespace_id = ? + """, result -> result.next() && result.getBoolean("paused"), platformNamespaceId); + return new AdmissionSnapshot(capacityGate.activeCount(null, now), + capacityGate.activeCount(platformNamespaceId, now), + pendingCandidates == null ? 0 : pendingCandidates, Boolean.TRUE.equals(platformPaused)); + } + + public record AdmissionSnapshot(int globalUnresolvedCount, int platformUnresolvedCount, + int platformPendingCandidateCount, boolean platformPaused) {} + + @Transactional(readOnly = true) + public PageResponse sources(long namespaceId, int page, int size) { + validatePage(page, size); + List items = jdbc.query(""" + WITH selected AS ( + SELECT source.* FROM github_source_binding source + WHERE source.namespace_id = ? + ORDER BY source.id DESC LIMIT ? OFFSET ? + ) + SELECT source.id, source.repository_id, source.owner_login, source.repository_name, + source.default_branch, source.visibility, source.status, + source.last_observed_commit, source.last_fully_processed_commit, + source.next_check_at, source.last_error_code, + task.id AS task_id, task.status AS task_status, task.mode AS task_mode, + task.last_error_code AS task_error_code, + task.last_error_detail AS task_error_detail, task.retry_at AS task_retry_at, + COALESCE(packages.received, 0) AS received_packages, + COALESCE(packages.processing, 0) AS processing_packages, + COALESCE(packages.pending_review, 0) AS pending_review_packages, + COALESCE(packages.blocked, 0) AS blocked_packages, + COALESCE(packages.failed, 0) AS failed_packages, + COALESCE(tracked.skills, 0) AS tracked_skills + FROM selected source + LEFT JOIN LATERAL ( + SELECT task.id, task.status, task.mode, task.last_error_code, + task.last_error_detail, task.retry_at + FROM github_sync_task task WHERE task.source_id = source.id + ORDER BY task.created_at DESC, task.id DESC LIMIT 1 + ) task ON TRUE + LEFT JOIN LATERAL ( + SELECT count(*) FILTER (WHERE latest.status = 'RECEIVED') AS received, + count(*) FILTER (WHERE latest.status IN + ('VALIDATING', 'RETRY_WAIT', 'VALIDATED')) AS processing, + count(*) FILTER (WHERE latest.status = 'READY_FOR_REVIEW') AS pending_review, + count(*) FILTER (WHERE latest.status = 'BLOCKED') AS blocked, + count(*) FILTER (WHERE latest.status = 'FAILED') AS failed + FROM ( + SELECT DISTINCT ON (ingestion.skill_path) + ingestion.skill_path, ingestion.status + FROM github_sync_package_ingestion ingestion + WHERE ingestion.namespace_id = source.namespace_id + AND ingestion.repository_id = source.repository_id + ORDER BY ingestion.skill_path, ingestion.created_at DESC, ingestion.id DESC + ) latest + ) packages ON TRUE + LEFT JOIN LATERAL ( + SELECT count(*) AS skills FROM github_source_skill_binding binding + WHERE binding.source_id = source.id + ) tracked ON TRUE + ORDER BY source.id DESC + """, (result, row) -> mapSource(result), namespaceId, size, (long) page * size); + Long total = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE namespace_id = ? + """, Long.class, namespaceId); + return new PageResponse<>(items, total == null ? 0 : total, page, size); + } + + @Transactional(readOnly = true) + public PageResponse runs(long namespaceId, int page, int size) { + return runs(namespaceId, page, size, null, null); + } + + @Transactional(readOnly = true) + public PageResponse runs(long namespaceId, int page, int size, RunKind kind, RunStatus status) { + validatePage(page, size); + List items = jdbc.query(""" + WITH selected AS ( + SELECT run.* FROM github_sync_run run WHERE run.namespace_id = ? + AND (CAST(? AS text) IS NULL OR run.kind=?) + AND (CAST(? AS text) IS NULL OR run.status=?) + AND run.details_purged_at IS NULL + AND NOT EXISTS (SELECT 1 FROM github_sync_task preview + WHERE preview.run_id = run.id + AND preview.snapshot_json ->> 'entryType' = 'SEARCH_PREVIEW') + ORDER BY run.created_at DESC, run.id DESC LIMIT ? OFFSET ? + ) + SELECT run.id, run.kind, run.status, run.created_at, run.finished_at, + (SELECT count(*) FROM github_sync_task task WHERE task.run_id = run.id) + AS task_count, + (SELECT min(task.retry_at) FROM github_sync_task task + WHERE task.run_id = run.id AND task.status = 'RETRY_WAIT') + AS next_retry_at, + COALESCE(packages.received, 0) AS received_packages, + COALESCE(packages.processing, 0) AS processing_packages, + COALESCE(packages.pending_review, 0) AS pending_review_packages, + COALESCE(packages.blocked, 0) AS blocked_packages, + COALESCE(packages.failed, 0) AS failed_packages, + latest_error.error_code AS last_error_code, + targets.target_count, targets.target_type, targets.target_value + FROM selected run + LEFT JOIN LATERAL ( + SELECT count(*) target_count, + CASE WHEN count(*)=1 THEN min(type) END target_type, + CASE WHEN count(*)=1 THEN min(value) END target_value + FROM ( + SELECT DISTINCT ON (type,COALESCE(repository_key,lower(value))) type,value + FROM ( + SELECT task.created_at,task.id, + COALESCE(source.repository_id::text,task.snapshot_json->>'repositoryId') repository_key, + CASE WHEN source.id IS NOT NULL OR task.kind='REPOSITORY_SYNC' + OR task.snapshot_json->>'entryType'='CURATED_REPOSITORY' THEN 'REPOSITORY' + WHEN task.snapshot_json->>'entryType'='CURATED_OWNER' THEN 'OWNER' + WHEN task.official_query_id IS NOT NULL OR task.snapshot_json->>'ruleId' IS NOT NULL THEN 'QUERY' + END type, + CASE WHEN source.id IS NOT NULL THEN source.owner_login||'/'||source.repository_name + WHEN task.kind='REPOSITORY_SYNC' THEN (task.snapshot_json->>'owner')||'/'||(task.snapshot_json->>'name') + WHEN task.snapshot_json->>'entryType'='CURATED_REPOSITORY' THEN + (task.snapshot_json->>'owner')||'/'||(task.snapshot_json->>'repositoryName') + WHEN task.snapshot_json->>'entryType'='CURATED_OWNER' THEN task.snapshot_json->>'owner' + WHEN task.official_query_id IS NOT NULL THEN task.official_query_id::text + ELSE task.snapshot_json->>'ruleId' END value + FROM github_sync_task task LEFT JOIN github_source_binding source ON source.id=task.source_id + AND source.namespace_id=task.namespace_id + WHERE task.run_id=run.id AND task.namespace_id=run.namespace_id + ) described WHERE type IS NOT NULL AND value IS NOT NULL + ORDER BY type,COALESCE(repository_key,lower(value)),created_at DESC,id DESC + ) distinct_targets + ) targets ON TRUE + LEFT JOIN LATERAL ( + SELECT count(*) FILTER (WHERE ingestion.status = 'RECEIVED') AS received, + count(*) FILTER (WHERE ingestion.status IN + ('VALIDATING', 'RETRY_WAIT', 'VALIDATED')) AS processing, + count(*) FILTER (WHERE ingestion.status = 'READY_FOR_REVIEW') AS pending_review, + count(*) FILTER (WHERE ingestion.status = 'BLOCKED') AS blocked, + count(*) FILTER (WHERE ingestion.status = 'FAILED') AS failed + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + WHERE task.run_id = run.id + ) packages ON TRUE + LEFT JOIN LATERAL ( + SELECT errors.error_code FROM ( + SELECT task.last_error_code AS error_code, task.updated_at + FROM github_sync_task task + WHERE task.run_id = run.id + AND task.status IN ('RETRY_WAIT', 'PARTIAL', 'FAILED') + AND task.last_error_code IS NOT NULL + UNION ALL + SELECT ingestion.last_error_code AS error_code, ingestion.updated_at + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + WHERE task.run_id = run.id + AND ingestion.status IN ('RETRY_WAIT', 'BLOCKED', 'FAILED') + AND ingestion.last_error_code IS NOT NULL + ) errors + ORDER BY errors.updated_at DESC LIMIT 1 + ) latest_error ON TRUE + ORDER BY run.created_at DESC, run.id DESC + """, (result, row) -> mapRun(result), namespaceId, kind == null ? null : kind.name(), kind == null ? null : kind.name(), + status == null ? null : status.name(), status == null ? null : status.name(), size, (long) page * size); + Long total = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_run run WHERE run.namespace_id = ? + AND (CAST(? AS text) IS NULL OR run.kind=?) + AND (CAST(? AS text) IS NULL OR run.status=?) + AND run.details_purged_at IS NULL + AND NOT EXISTS (SELECT 1 FROM github_sync_task preview + WHERE preview.run_id = run.id + AND preview.snapshot_json ->> 'entryType' = 'SEARCH_PREVIEW') + """, Long.class, namespaceId, kind == null ? null : kind.name(), kind == null ? null : kind.name(), + status == null ? null : status.name(), status == null ? null : status.name()); + return new PageResponse<>(items, total == null ? 0 : total, page, size); + } + + @Transactional(readOnly = true) + public GithubSyncRunDetailResponse runDetail(long namespaceId, UUID runId) { + return jdbc.query(""" + SELECT run.id, run.kind, run.status, run.created_at, run.finished_at, + (SELECT count(*) FROM github_sync_task task WHERE task.run_id = run.id) AS task_count, + (SELECT count(*) FROM github_sync_task task + JOIN github_sync_path_result path ON path.task_id = task.id + WHERE task.run_id = run.id) AS path_count, + (SELECT min(task.retry_at) FROM github_sync_task task + WHERE task.run_id = run.id AND task.status = 'RETRY_WAIT') AS next_retry_at, + COALESCE(packages.received, 0) AS received_packages, + COALESCE(packages.processing, 0) AS processing_packages, + COALESCE(packages.pending_review, 0) AS pending_review_packages, + COALESCE(packages.blocked, 0) AS blocked_packages, + COALESCE(packages.failed, 0) AS failed_packages, + (SELECT task.last_error_code FROM github_sync_task task + WHERE task.run_id = run.id AND task.last_error_code IS NOT NULL + ORDER BY task.updated_at DESC, task.id DESC LIMIT 1) AS last_error_code + FROM github_sync_run run + LEFT JOIN LATERAL ( + SELECT count(*) FILTER (WHERE ingestion.status = 'RECEIVED') AS received, + count(*) FILTER (WHERE ingestion.status IN + ('VALIDATING', 'RETRY_WAIT', 'VALIDATED')) AS processing, + count(*) FILTER (WHERE ingestion.status = 'READY_FOR_REVIEW') AS pending_review, + count(*) FILTER (WHERE ingestion.status = 'BLOCKED') AS blocked, + count(*) FILTER (WHERE ingestion.status = 'FAILED') AS failed + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + WHERE task.run_id = run.id + ) packages ON TRUE + WHERE run.id = ? AND run.namespace_id = ? AND run.details_purged_at IS NULL + AND NOT EXISTS (SELECT 1 FROM github_sync_task preview + WHERE preview.run_id = run.id + AND preview.snapshot_json ->> 'entryType' = 'SEARCH_PREVIEW') + """, result -> { + if (!result.next()) { + throw new DomainNotFoundException("error.githubSync.run.notFound"); + } + return new GithubSyncRunDetailResponse(result.getObject("id", UUID.class), + result.getString("kind"), result.getString("status"), + instant(result, "created_at"), instant(result, "finished_at"), + result.getLong("task_count"), result.getLong("path_count"), + instant(result, "next_retry_at"), result.getLong("received_packages"), + result.getLong("processing_packages"), result.getLong("pending_review_packages"), + result.getLong("blocked_packages"), result.getLong("failed_packages"), + result.getString("last_error_code")); + }, runId, namespaceId); + } + + @Transactional(readOnly = true) + public PageResponse runTasks( + long namespaceId, UUID runId, int page, int size) { + validatePage(page, size); + GithubSyncRunDetailResponse run = runDetail(namespaceId, runId); + List items = jdbc.query(""" + SELECT task.id, task.source_id, source.owner_login, source.repository_name, + task.kind, task.mode, task.status, task.fixed_commit_sha, + task.last_error_code, task.retry_at, task.created_at, task.completed_at + FROM github_sync_task task + LEFT JOIN github_source_binding source ON source.id = task.source_id + WHERE task.run_id = ? AND task.namespace_id = ? + ORDER BY task.created_at, task.id LIMIT ? OFFSET ? + """, (result, row) -> new GithubSyncRunTaskResponse( + result.getObject("id", UUID.class), result.getObject("source_id", Long.class), + repository(result), result.getString("kind"), result.getString("mode"), + result.getString("status"), result.getString("fixed_commit_sha"), + result.getString("last_error_code"), instant(result, "retry_at"), + instant(result, "created_at"), instant(result, "completed_at")), + runId, namespaceId, size, (long) page * size); + return new PageResponse<>(items, run.taskCount(), page, size); + } + + @Transactional(readOnly = true) + public PageResponse runPaths( + long namespaceId, UUID runId, int page, int size) { + validatePage(page, size); + GithubSyncRunDetailResponse run = runDetail(namespaceId, runId); + List items = jdbc.query(""" + SELECT task.id AS task_id, task.source_id, source.owner_login, source.repository_name, + path.skill_path, path.commit_sha, path.outcome, path.reason, + path.retry_at, path.updated_at, ingestion.status AS ingestion_status, + ingestion.last_error_code AS ingestion_error_code, + review.status AS review_status + FROM github_sync_task task + JOIN github_sync_path_result path ON path.task_id = task.id + LEFT JOIN github_source_binding source ON source.id = task.source_id + LEFT JOIN github_sync_package_ingestion ingestion ON ingestion.id = path.ingestion_id + LEFT JOIN LATERAL ( + SELECT item.status FROM review_task item + WHERE item.github_sync_ingestion_id = ingestion.id + ORDER BY item.id DESC LIMIT 1 + ) review ON TRUE + WHERE task.run_id = ? AND task.namespace_id = ? + ORDER BY path.updated_at DESC, path.id DESC LIMIT ? OFFSET ? + """, (result, row) -> new GithubSyncRunPathResponse( + result.getObject("task_id", UUID.class), result.getObject("source_id", Long.class), + repository(result), result.getString("skill_path"), result.getString("commit_sha"), + result.getString("outcome"), result.getString("reason"), + result.getString("ingestion_status"), result.getString("ingestion_error_code"), + result.getString("review_status"), + instant(result, "retry_at"), instant(result, "updated_at")), + runId, namespaceId, size, (long) page * size); + return new PageResponse<>(items, run.pathCount(), page, size); + } + + private String repository(ResultSet result) throws SQLException { + String owner = result.getString("owner_login"); + String name = result.getString("repository_name"); + return owner == null || name == null ? null : owner + "/" + name; + } + + @Transactional(readOnly = true) + public PageResponse skillPaths( + long namespaceId, long sourceId, int page, int size) { + validatePage(page, size); + List ignorePatterns = ignores.find(namespaceId, sourceId) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.source.notFound")); + List items = jdbc.query(""" + WITH latest AS ( + SELECT DISTINCT ON (path.skill_path) + path.skill_path, path.commit_sha, path.outcome, path.reason, + path.retry_at, path.updated_at, path.ingestion_id + FROM github_sync_task task + JOIN github_sync_path_result path ON path.task_id = task.id + WHERE task.namespace_id = ? AND task.source_id = ? + ORDER BY path.skill_path, task.created_at DESC, + path.updated_at DESC, path.id DESC + ), selected AS ( + SELECT * FROM latest ORDER BY skill_path LIMIT ? OFFSET ? + ) + SELECT path.skill_path, path.commit_sha, path.outcome, path.reason, + path.retry_at, path.updated_at, path.ingestion_id, + ingestion.status AS ingestion_status, + ingestion.last_error_code AS ingestion_error_code, + ingestion.target_skill_slug, + binding.skill_id, binding.upstream_presence, + binding.presence_commit_sha, binding.missing_since, + latest_ingestion.status AS latest_binding_ingestion_status, + binding.retry_requested_at IS NOT NULL AS retry_requested, + (binding.retry_requested_at IS NOT NULL AND + (binding.retry_scheduled_at IS NULL OR EXISTS ( + SELECT 1 FROM github_sync_task retry_task + WHERE retry_task.source_id = binding.source_id + AND retry_task.mode = 'FULL' + AND retry_task.status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') + AND retry_task.updated_at >= binding.retry_scheduled_at + ) OR EXISTS ( + SELECT 1 FROM github_sync_package_ingestion pending + WHERE pending.namespace_id = binding.namespace_id + AND pending.repository_id = binding.repository_id + AND pending.skill_path = binding.skill_path + AND pending.created_at >= binding.retry_requested_at + AND pending.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', + 'VALIDATED', 'READY_FOR_REVIEW') + ))) AS retry_in_progress, + review.id AS review_id, review.status AS review_status + FROM selected path + LEFT JOIN github_sync_package_ingestion ingestion ON ingestion.id = path.ingestion_id + LEFT JOIN github_source_skill_binding binding + ON binding.source_id = ? AND binding.skill_path = path.skill_path + LEFT JOIN github_sync_package_ingestion latest_ingestion + ON latest_ingestion.id = binding.latest_ingestion_id + LEFT JOIN review_task review + ON review.github_sync_ingestion_id = ingestion.id + ORDER BY path.skill_path + """, (result, row) -> new GithubSyncSkillPathResponse( + result.getString("skill_path"), result.getString("commit_sha"), + result.getString("outcome"), result.getString("reason"), + instant(result, "retry_at"), instant(result, "updated_at"), + result.getObject("ingestion_id", UUID.class), result.getString("ingestion_status"), + result.getString("ingestion_error_code"), result.getString("target_skill_slug"), + result.getObject("skill_id", Long.class), + result.getString("upstream_presence"), result.getString("presence_commit_sha"), + instant(result, "missing_since"), + result.getObject("review_id", Long.class), result.getString("review_status"), + result.getString("latest_binding_ingestion_status"), + result.getBoolean("retry_requested"), + result.getBoolean("retry_in_progress"), + GithubSkillPathPolicy.ignored(result.getString("skill_path"), ignorePatterns)), + namespaceId, sourceId, size, (long) page * size, sourceId); + Long total = jdbc.queryForObject(""" + SELECT count(DISTINCT path.skill_path) + FROM github_sync_task task + JOIN github_sync_path_result path ON path.task_id = task.id + WHERE task.namespace_id = ? AND task.source_id = ? + """, Long.class, namespaceId, sourceId); + return new PageResponse<>(items, total == null ? 0 : total, page, size); + } + + private void validatePage(int page, int size) { + if (page < 0 || size < 1 || size > 100) { + throw new DomainBadRequestException("error.badRequest"); + } + } + + private GithubSyncSourceSummaryResponse mapSource(ResultSet result) throws SQLException { + return new GithubSyncSourceSummaryResponse( + result.getLong("id"), result.getLong("repository_id"), + result.getString("owner_login"), result.getString("repository_name"), + result.getString("default_branch"), result.getString("visibility"), + result.getString("status"), result.getString("last_observed_commit"), + result.getString("last_fully_processed_commit"), instant(result, "next_check_at"), + result.getString("last_error_code"), result.getObject("task_id", UUID.class), + result.getString("task_status"), result.getString("task_mode"), + result.getString("task_error_code"), result.getString("task_error_detail"), + instant(result, "task_retry_at"), + result.getLong("received_packages"), result.getLong("processing_packages"), + result.getLong("pending_review_packages"), result.getLong("blocked_packages"), + result.getLong("failed_packages"), + result.getLong("tracked_skills")); + } + + private GithubSyncRunSummaryResponse mapRun(ResultSet result) throws SQLException { + return new GithubSyncRunSummaryResponse(result.getObject("id", UUID.class), + result.getString("kind"), result.getString("status"), + instant(result, "created_at"), instant(result, "finished_at"), + instant(result, "next_retry_at"), + result.getLong("task_count"), result.getLong("received_packages"), + result.getLong("processing_packages"), result.getLong("pending_review_packages"), + result.getLong("blocked_packages"), result.getLong("failed_packages"), + result.getString("last_error_code"), + result.getString("target_type") == null ? null : new Target(result.getString("target_type"), result.getString("target_value")), + result.getLong("target_count")); + } + + private Instant instant(ResultSet result, String column) throws SQLException { + Timestamp value = result.getTimestamp(column); + return value == null ? null : value.toInstant(); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncRepositoryQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncRepositoryQueryRepository.java new file mode 100644 index 000000000..13804384a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncRepositoryQueryRepository.java @@ -0,0 +1,209 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.PathResult; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Detail; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Origin; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.OriginType; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Page; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.SourceStatus; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.State; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.Summary; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Arrays; +import java.util.UUID; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; + +/** Unifies source/candidate identities before pagination; provenance never multiplies repository rows. */ +@Repository +@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ) +public class GithubSyncRepositoryQueryRepository { + private final NamedParameterJdbcTemplate jdbc; + private final GithubSyncManagementQueryRepository management; + private final GithubDeepDiscoveryQueryRepository paths; + public GithubSyncRepositoryQueryRepository(JdbcTemplate jdbc, GithubSyncManagementQueryRepository management, + GithubDeepDiscoveryQueryRepository paths) { + this.jdbc = new NamedParameterJdbcTemplate(jdbc); + this.management = management; + this.paths = paths; + } + private static final String BASE = """ + WITH repositories AS ( + SELECT s.repository_id,s.id source_id,c.id candidate_id,s.owner_login,s.repository_name, + LEAST(s.created_at,c.first_seen_at) first_seen_at,'REGISTERED' state, + s.status source_status,c.status candidate_status,s.next_check_at,s.last_error_code + FROM github_source_binding s LEFT JOIN github_discovery_candidate c + ON c.namespace_id=s.namespace_id AND c.repository_id=s.repository_id + WHERE s.namespace_id=:ns + UNION ALL + SELECT c.repository_id,NULL::bigint,c.id,c.owner_login,c.repository_name,c.first_seen_at, + CASE c.status WHEN 'BOUND' THEN 'BOUND_WITHOUT_SOURCE' + WHEN 'REJECTED' THEN CASE WHEN c.last_error_code='NO_VALID_SKILL' THEN 'NO_VALID_SKILL' + ELSE 'VALIDATION_FAILED' END ELSE c.status END, + NULL::varchar,c.status,NULL::timestamptz,c.last_error_code + FROM github_discovery_candidate c WHERE c.namespace_id=:ns + AND NOT EXISTS (SELECT 1 FROM github_source_binding s + WHERE s.namespace_id=c.namespace_id AND s.repository_id=c.repository_id) + ) + """; + private static final String ORIGIN_TYPES = """ + SELECT r.*,ARRAY(SELECT origin_type FROM ( + SELECT origin_type FROM github_source_origin WHERE source_id=r.source_id + UNION SELECT origin_type FROM github_discovery_candidate_origin WHERE candidate_id=r.candidate_id + ) origins ORDER BY origin_type) origin_types FROM selected r + """; + + public Page repositories(int page, int size, Instant asOf, String q, + OriginType originType, State state, SourceStatus sourceStatus) { + Instant bound = validate(page, size, asOf); + if (q != null && q.length() > 200) { + invalid(); + } + var parameters = parameters().addValue("asOf", Timestamp.from(bound)) + .addValue("limit", size).addValue("offset", (long) page * size); + String filter = " WHERE r.first_seen_at<=:asOf"; + if (q != null && !q.isBlank()) { + filter += " AND (r.owner_login||'/'||r.repository_name) ILIKE :q ESCAPE '!'"; + parameters.addValue("q", "%" + q.trim().replace("!", "!!").replace("%", "!%").replace("_", "!_") + "%"); + } + if (state != null) { + filter += " AND r.state=:state"; + parameters.addValue("state", state.name()); + } + if (sourceStatus != null) { + filter += " AND r.source_status=:sourceStatus"; + parameters.addValue("sourceStatus", sourceStatus.name()); + } + if (originType != null) { + filter += """ + AND (EXISTS (SELECT 1 FROM github_source_origin o WHERE o.source_id=r.source_id AND o.origin_type=:origin) + OR EXISTS (SELECT 1 FROM github_discovery_candidate_origin o WHERE o.candidate_id=r.candidate_id AND o.origin_type=:origin)) + """; + parameters.addValue("origin", originType.name()); + } + Long total = jdbc.queryForObject(BASE + " SELECT count(*) FROM repositories r" + filter, parameters, Long.class); + var items = jdbc.query(BASE + ",selected AS MATERIALIZED (SELECT r.* FROM repositories r" + filter + + " ORDER BY first_seen_at DESC,repository_id DESC LIMIT :limit OFFSET :offset) " + + ORIGIN_TYPES + " ORDER BY first_seen_at DESC,repository_id DESC", parameters, (rs, n) -> summary(rs)); + return new Page<>(items, total == null ? 0 : total, page, size, bound); + } + + private Summary repository(long repositoryId) { + if (repositoryId < 1) { + invalid(); + } + var rows = jdbc.query(BASE + ",selected AS (SELECT * FROM repositories WHERE repository_id=:repo) " + + ORIGIN_TYPES, parameters().addValue("repo", repositoryId), (rs, n) -> summary(rs)); + if (rows.isEmpty()) { + throw new DomainNotFoundException("error.githubSync.source.notFound"); + } + return rows.getFirst(); + } + + public Detail detail(long repositoryId) { + Summary repository = repository(repositoryId); + var parameters = parameters().addValue("repo", repositoryId).addValue("repoText", Long.toString(repositoryId)) + .addValue("source", repository.sourceId()); + var runIds = jdbc.query(""" + SELECT t.run_id FROM github_sync_task t JOIN github_sync_run r ON r.id=t.run_id + WHERE t.namespace_id=:ns AND r.namespace_id=:ns AND r.details_purged_at IS NULL + AND (t.source_id=:source OR t.snapshot_json->>'repositoryId'=:repoText) + AND COALESCE(t.snapshot_json->>'entryType','')<>'SEARCH_PREVIEW' + ORDER BY t.created_at DESC,t.id DESC LIMIT 1 + """, parameters, (rs, n) -> rs.getObject("run_id", UUID.class)); + Instant checked = jdbc.queryForObject(""" + SELECT max(completed_at) FROM github_sync_task WHERE namespace_id=:ns AND kind='REPOSITORY_SYNC' + AND status='ACQUISITION_COMPLETE' AND (source_id=:source OR snapshot_json->>'repositoryId'=:repoText) + """, parameters, (rs, n) -> instant(rs, 1)); + return jdbc.query(BASE + " SELECT * FROM repositories WHERE repository_id=:repo", parameters, + rs -> { + rs.next(); + return new Detail(repository, runIds.isEmpty() ? null : runIds.getFirst(), + checked, instant(rs, "next_check_at"), rs.getString("last_error_code")); + }); + } + + public Page origins(long repositoryId, int page, int size, Instant asOf) { + Instant bound = validate(page, size, asOf); + Summary repository = repository(repositoryId); + var parameters = parameters().addValue("source", repository.sourceId()).addValue("candidate", repository.candidateId()) + .addValue("asOf", Timestamp.from(bound)).addValue("limit", size).addValue("offset", (long) page * size); + String base = """ + WITH combined AS ( + SELECT origin_type,origin_key,created_at first_seen_at,NULL::timestamptz last_seen_at + FROM github_source_origin WHERE source_id=:source AND created_at<=:asOf + UNION ALL + SELECT origin_type,origin_key,first_seen_at,last_seen_at FROM github_discovery_candidate_origin + WHERE candidate_id=:candidate AND first_seen_at<=:asOf + ), origins AS (SELECT origin_type,origin_key,min(first_seen_at) first_seen_at,max(last_seen_at) last_seen_at + FROM combined GROUP BY origin_type,origin_key) + """; + Long total = jdbc.queryForObject(base + " SELECT count(*) FROM origins", parameters, Long.class); + var items = jdbc.query(base + """ + ,selected AS (SELECT * FROM origins ORDER BY first_seen_at DESC,origin_type,origin_key LIMIT :limit OFFSET :offset) + SELECT o.*,entry.id entry_id,q.id query_id,q.definition_json->>'channel' channel, + CASE WHEN r.details_purged_at IS NULL THEN r.id END run_id,co.recent_seed_repository_id + FROM selected o + LEFT JOIN LATERAL (SELECT e.id FROM github_discovery_entry e WHERE e.namespace_id=:ns + AND e.entry_type=o.origin_type AND ((o.origin_type='CURATED_OWNER' AND lower(e.owner_login)=lower(o.origin_key)) + OR (o.origin_type<>'CURATED_OWNER' AND e.id::text=o.origin_key)) LIMIT 1) entry ON TRUE + LEFT JOIN github_deep_discovery_query q ON o.origin_type='OFFICIAL_DISCOVERY' + AND q.namespace_id=:ns AND q.id::text=o.origin_key + LEFT JOIN github_discovery_candidate_origin co ON co.candidate_id=:candidate + AND co.origin_type=o.origin_type AND co.origin_key=o.origin_key + LEFT JOIN github_sync_run r ON r.id=co.recent_run_id AND r.namespace_id=:ns + ORDER BY o.first_seen_at DESC,o.origin_type,o.origin_key + """, parameters, (rs, n) -> new Origin(OriginType.valueOf(rs.getString("origin_type")), rs.getString("origin_key"), + instant(rs, "first_seen_at"), instant(rs, "last_seen_at"), rs.getObject("entry_id", UUID.class), + rs.getString("channel"), rs.getObject("query_id", UUID.class), rs.getObject("run_id", UUID.class), + rs.getObject("recent_seed_repository_id", Long.class))); + return new Page<>(items, total == null ? 0 : total, page, size, bound); + } + + public Page paths(long repositoryId, int page, int size, Instant asOf) { + Instant bound = validate(page, size, asOf); + repository(repositoryId); + var result = paths.pathsForRepository(management.platformNamespaceId(), repositoryId, page, size, bound); + return new Page<>(result.items(), result.total(), page, size, bound); + } + private MapSqlParameterSource parameters() { + return new MapSqlParameterSource("ns", management.platformNamespaceId()); + } + + private Summary summary(ResultSet rs) throws SQLException { + String sourceStatus = rs.getString("source_status"); + String[] origins = (String[]) rs.getArray("origin_types").getArray(); + return new Summary(rs.getLong("repository_id"), rs.getObject("source_id", Long.class), rs.getObject("candidate_id", Long.class), + rs.getString("owner_login"), rs.getString("repository_name"), instant(rs, "first_seen_at"), + State.valueOf(rs.getString("state")), sourceStatus == null ? null : SourceStatus.valueOf(sourceStatus), + rs.getString("candidate_status"), Arrays.stream(origins).map(OriginType::valueOf).toList()); + } + private static Instant validate(int page, int size, Instant asOf) { + Instant now = Instant.now(); + if (page < 0 || size < 1 || size > 100 || asOf != null && asOf.isAfter(now)) { + invalid(); + } + return asOf == null ? now : asOf; + } + private static void invalid() { + throw new DomainBadRequestException("error.badRequest"); + } + + private static Instant instant(ResultSet rs, String column) throws SQLException { + Timestamp value = rs.getTimestamp(column); + return value == null ? null : value.toInstant(); + } + private static Instant instant(ResultSet rs, int column) throws SQLException { + Timestamp value = rs.getTimestamp(column); + return value == null ? null : value.toInstant(); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncReviewSourceQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncReviewSourceQueryRepository.java new file mode 100644 index 000000000..612ccdcf8 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncReviewSourceQueryRepository.java @@ -0,0 +1,149 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.dto.GithubSyncReviewSourceResponse; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class GithubSyncReviewSourceQueryRepository { + private final JdbcTemplate jdbc; + private final NamedParameterJdbcTemplate namedJdbc; + private final ObjectMapper objectMapper; + + public GithubSyncReviewSourceQueryRepository(JdbcTemplate jdbc, ObjectMapper objectMapper) { + this.jdbc = jdbc; + this.namedJdbc = new NamedParameterJdbcTemplate(jdbc); + this.objectMapper = objectMapper; + } + + public record QueueSource(UUID ingestionId, UUID runId, long repositoryId, + String ownerLogin, String repositoryName, + String skillPath, String commitSha) {} + + public Map findQueueSources(List reviewIds) { + if (reviewIds.isEmpty()) { + return Map.of(); + } + return namedJdbc.query(""" + SELECT review.id AS review_id, ingestion.id AS ingestion_id, task.run_id, + ingestion.repository_id, + COALESCE(task.snapshot_json ->> 'owner', source.owner_login) AS owner_login, + COALESCE(task.snapshot_json ->> 'name', source.repository_name) AS repository_name, + ingestion.skill_path, ingestion.commit_sha + FROM review_task review + JOIN github_sync_package_ingestion ingestion ON ingestion.id = review.github_sync_ingestion_id + JOIN github_sync_task task ON task.id = ingestion.task_id + LEFT JOIN github_source_binding source + ON source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id + WHERE review.id IN (:reviewIds) AND review.origin_type = 'GITHUB_SYNC' + """, Map.of("reviewIds", reviewIds), result -> { + Map sources = new HashMap<>(); + while (result.next()) { + sources.put(result.getLong("review_id"), new QueueSource( + result.getObject("ingestion_id", UUID.class), + result.getObject("run_id", UUID.class), + result.getLong("repository_id"), + result.getString("owner_login"), result.getString("repository_name"), + result.getString("skill_path"), result.getString("commit_sha"))); + } + return sources; + }); + } + + public Optional findByReviewId(Long reviewId) { + Optional row = jdbc.query(""" + SELECT ingestion.id, task.run_id, source.id AS source_id, ingestion.repository_id, + COALESCE(task.snapshot_json ->> 'owner', source.owner_login) AS owner_login, + COALESCE(task.snapshot_json ->> 'name', source.repository_name) AS repository_name, + ingestion.commit_sha, + ingestion.skill_path, ingestion.content_sha256, ingestion.status, + ingestion.created_at, ingestion.license_evidence_json::text AS evidence_json, + review.skill_version_id, reviewed_version.github_sync_revision, + (SELECT previous.id FROM skill_version previous + WHERE previous.skill_id = reviewed_version.skill_id + AND previous.id <> review.skill_version_id + AND previous.published_at <= review.submitted_at + ORDER BY previous.published_at DESC, previous.id DESC + LIMIT 1) AS base_version_id + FROM review_task review + JOIN skill_version reviewed_version ON reviewed_version.id = review.skill_version_id + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = review.github_sync_ingestion_id + JOIN github_sync_task task ON task.id = ingestion.task_id + LEFT JOIN github_source_binding source + ON source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id + WHERE review.id = ? AND review.origin_type = 'GITHUB_SYNC' + """, result -> result.next() ? Optional.of(new SourceRow(new GithubSyncReviewSourceResponse( + result.getObject("id", UUID.class), result.getObject("run_id", UUID.class), + result.getObject("source_id", Long.class), + result.getLong("repository_id"), result.getString("owner_login"), + result.getString("repository_name"), result.getString("commit_sha"), + result.getString("skill_path"), result.getString("content_sha256"), + result.getInt("github_sync_revision"), + result.getString("status"), result.getTimestamp("created_at").toInstant(), + evidence(result.getString("evidence_json")), + result.getObject("base_version_id", Long.class), + List.of()), result.getLong("skill_version_id"))) : Optional.empty(), reviewId); + return row.map(value -> value.response().withFileChanges( + fileChanges(value.versionId(), value.response().baseVersionId()))); + } + + private record SourceRow(GithubSyncReviewSourceResponse response, long versionId) {} + + private List fileChanges(long versionId, Long baseVersionId) { + return jdbc.query(""" + SELECT COALESCE(current_file.file_path, previous_file.file_path) AS path, + CASE WHEN previous_file.id IS NULL THEN 'ADDED' + WHEN current_file.id IS NULL THEN 'REMOVED' + ELSE 'MODIFIED' END AS change_type, + previous_file.file_size AS old_size, current_file.file_size AS new_size, + previous_file.sha256 AS old_sha256, current_file.sha256 AS new_sha256 + FROM (SELECT * FROM skill_file WHERE version_id = ?) current_file + FULL OUTER JOIN (SELECT * FROM skill_file WHERE version_id = ?) previous_file + ON previous_file.file_path = current_file.file_path + WHERE current_file.id IS NULL OR previous_file.id IS NULL + OR current_file.sha256 IS DISTINCT FROM previous_file.sha256 + ORDER BY path + """, (result, rowNum) -> new GithubSyncReviewSourceResponse.FileChange( + result.getString("path"), result.getString("change_type"), + result.getObject("old_size", Long.class), result.getObject("new_size", Long.class), + result.getString("old_sha256"), result.getString("new_sha256")), + versionId, baseVersionId); + } + + private GithubSyncReviewSourceResponse.LicenseEvidence evidence(String json) { + try { + JsonNode root = objectMapper.readTree(json); + List documents = new ArrayList<>(); + for (JsonNode document : root.path("documents")) { + documents.add(new GithubSyncReviewSourceResponse.EvidenceDocument( + document.path("scope").asText(), document.path("path").asText(), + optional(document, "gitSha"), optional(document, "sha256"), + document.path("size").asLong(), document.path("bundled").asBoolean(), + optional(document, "text"))); + } + return new GithubSyncReviewSourceResponse.LicenseEvidence( + root.path("status").asText("UNKNOWN"), List.copyOf(documents)); + } catch (IOException exception) { + throw new IllegalStateException("Invalid stored GitHub license evidence", exception); + } + } + + private String optional(JsonNode node, String field) { + JsonNode value = node.path(field); + return value.isTextual() ? value.asText() : null; + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncWorkerPresenceRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncWorkerPresenceRepository.java new file mode 100644 index 000000000..98985922e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/GithubSyncWorkerPresenceRepository.java @@ -0,0 +1,99 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.dto.GithubSyncWorkerStatusResponse; +import com.iflytek.skillhub.dto.GithubOfficialWorkerAvailabilityResponse; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryCatalog; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; + +/** Operational status shared by the worker heartbeat and the admin read model. */ +@Repository +public class GithubSyncWorkerPresenceRepository { + private final JdbcTemplate jdbc; + + public GithubSyncWorkerPresenceRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public void heartbeat() { + jdbc.update(""" + INSERT INTO github_sync_worker_presence (id, last_seen_at) + VALUES (1, CURRENT_TIMESTAMP) + ON CONFLICT (id) DO UPDATE SET last_seen_at = + GREATEST(github_sync_worker_presence.last_seen_at, CURRENT_TIMESTAMP) + """); + } + + @Transactional + public void heartbeat(String workerId, int protocolMajor, List capabilities) { + heartbeat(); + jdbc.update(connection -> { + var statement = connection.prepareStatement(""" + INSERT INTO github_sync_worker_instance (worker_id, protocol_major, capabilities, last_seen_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (worker_id) DO UPDATE SET protocol_major = EXCLUDED.protocol_major, + capabilities = EXCLUDED.capabilities, + last_seen_at = GREATEST(github_sync_worker_instance.last_seen_at, EXCLUDED.last_seen_at) + WHERE github_sync_worker_instance.last_seen_at <= EXCLUDED.last_seen_at + """); + statement.setString(1, workerId); + statement.setInt(2, protocolMajor); + statement.setArray(3, connection.createArrayOf("text", capabilities.toArray())); + return statement; + }); + } + + public GithubOfficialWorkerAvailabilityResponse officialStatus() { + var required = new java.util.ArrayList<>(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES); + required.add(com.iflytek.skillhub.githubsync.GithubPlatformControlRepository.CAPABILITY); + return jdbc.query(connection -> { + var statement = connection.prepareStatement(""" + SELECT CURRENT_TIMESTAMP AS observed_at, + GREATEST((SELECT max(last_seen_at) FROM github_sync_worker_instance), + (SELECT last_seen_at FROM github_sync_worker_presence WHERE id = 1)) AS last_seen_at, + (SELECT count(*) FROM github_sync_worker_instance + WHERE protocol_major = 1 AND capabilities @> ?::text[] + AND last_seen_at >= CURRENT_TIMESTAMP - INTERVAL '90 seconds') AS compatible_count + """); + statement.setArray(1, connection.createArrayOf("text", required.toArray())); + return statement; + }, result -> { + result.next(); + Instant observed = result.getTimestamp("observed_at").toInstant(); + Timestamp last = result.getTimestamp("last_seen_at"); + Instant lastSeen = last == null ? null : last.toInstant(); + long compatible = result.getLong("compatible_count"); + String state = compatible > 0 ? "AVAILABLE" : lastSeen == null ? "NEVER_SEEN" + : !lastSeen.isBefore(observed.minusSeconds(90)) ? "INCOMPATIBLE" : "UNREACHABLE"; + return new GithubOfficialWorkerAvailabilityResponse(state, lastSeen, compatible, observed); + }); + } + + @Transactional + public int deleteExpiredInstances(Instant before, int limit) { + if (before == null || limit < 1 || limit > 500) { + throw new IllegalArgumentException("Invalid worker presence retention batch"); + } + return jdbc.update(""" + DELETE FROM github_sync_worker_instance WHERE worker_id IN ( + SELECT worker_id FROM github_sync_worker_instance WHERE last_seen_at < ? + ORDER BY last_seen_at, worker_id LIMIT ? FOR UPDATE SKIP LOCKED) + """, Timestamp.from(before), limit); + } + + public GithubSyncWorkerStatusResponse status() { + return jdbc.query(""" + SELECT last_seen_at, + last_seen_at >= CURRENT_TIMESTAMP - INTERVAL '90 seconds' AS online + FROM github_sync_worker_presence WHERE id = 1 + """, result -> result.next() + ? new GithubSyncWorkerStatusResponse( + result.getTimestamp("last_seen_at").toInstant(), result.getBoolean("online")) + : new GithubSyncWorkerStatusResponse(null, false)); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaGovernanceQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaGovernanceQueryRepository.java index 7a9ed2536..4b417e2b9 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaGovernanceQueryRepository.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaGovernanceQueryRepository.java @@ -223,7 +223,7 @@ public class JpaGovernanceQueryRepository implements GovernanceQueryRepository { task.getReviewedBy(), reviewedBy != null ? reviewedBy.getDisplayName() : null, task.getReviewComment(), task.getSubmittedAt(), task.getReviewedAt(), task.getSubjectType().name(), task.getSubjectId(), - task.getSubjectVersionId(), suite.getSlug()); + task.getSubjectVersionId(), suite.getSlug(), task.getOriginType().name()); } Long skillId = task.getSkillId() != null ? task.getSkillId() @@ -253,7 +253,7 @@ public class JpaGovernanceQueryRepository implements GovernanceQueryRepository { task.getSubjectVersionId() != null ? task.getSubjectVersionId() : task.getSkillVersionId(), skill.getSlug(), task.getAvatarCandidateId(), task.isAvatarRemoval(), task.getPreviousVisibility() == null ? null : task.getPreviousVisibility().name(), - task.getRequestedVisibility() == null ? null : task.getRequestedVisibility().name() + task.getRequestedVisibility() == null ? null : task.getRequestedVisibility().name(), task.getOriginType().name() ); } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaMySkillQueryRepository.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaMySkillQueryRepository.java index 3d1b4b65e..ae9fcaae5 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaMySkillQueryRepository.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/repository/JpaMySkillQueryRepository.java @@ -257,7 +257,7 @@ public class JpaMySkillQueryRepository implements MySkillQueryRepository { if (namespace == null) { return false; } - if (namespace.getType() == NamespaceType.GLOBAL) { + if (namespace.getType() != NamespaceType.TEAM) { return false; } if (namespace.getStatus() != NamespaceStatus.ACTIVE || skill.getStatus() != SkillStatus.ACTIVE) { @@ -274,6 +274,6 @@ public class JpaMySkillQueryRepository implements MySkillQueryRepository { return null; } return new SkillLifecycleVersionResponse(projection.id(), projection.version(), projection.status(), - projection.displayName(), projection.summary()); + projection.displayName(), projection.summary(), projection.syncRevision()); } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/security/SensitiveLogSanitizer.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/security/SensitiveLogSanitizer.java index ce38aea7d..f51b9b123 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/security/SensitiveLogSanitizer.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/security/SensitiveLogSanitizer.java @@ -1,48 +1,15 @@ package com.iflytek.skillhub.security; import jakarta.servlet.http.HttpServletRequest; -import java.util.Arrays; -import java.util.Locale; -import java.util.Set; -import java.util.stream.Collectors; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; /** - * Applies lightweight redaction rules before sensitive strings are written to logs. + * Logs the request path without user-controlled query parameter names or values. */ @Component public class SensitiveLogSanitizer { - private static final Set SENSITIVE_KEYS = Set.of( - "password", "passwd", "pwd", "token", "authorization", "cookie", - "secret", "api_key", "apikey", "access_key", "refresh_token", "code"); - public String sanitizeRequestTarget(HttpServletRequest request) { - String uri = request.getRequestURI(); - String query = request.getQueryString(); - if (!StringUtils.hasText(query)) { - return uri; - } - return uri + "?" + sanitizeQuery(query); - } - - String sanitizeQuery(String query) { - return Arrays.stream(query.split("&")) - .map(this::sanitizeQueryPart) - .collect(Collectors.joining("&")); - } - - private String sanitizeQueryPart(String queryPart) { - int idx = queryPart.indexOf('='); - if (idx < 0) { - return queryPart; - } - String key = queryPart.substring(0, idx); - String normalizedKey = key.trim().toLowerCase(Locale.ROOT); - if (SENSITIVE_KEYS.contains(normalizedKey)) { - return key + "=[REDACTED]"; - } - return queryPart; + return request.getRequestURI(); } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialAdminAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialAdminAppService.java new file mode 100644 index 000000000..fce050de4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialAdminAppService.java @@ -0,0 +1,152 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubCredentialResponse; +import com.iflytek.skillhub.dto.GithubCredentialUpsertRequest; +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; + +@Service +public class GithubCredentialAdminAppService { + private final GithubSystemCredentialRepository credentials; + private final GithubCredentialCipher cipher; + private final AuditLogService auditLogService; + private final RequestIdAccessor requestIdAccessor; + + public GithubCredentialAdminAppService(GithubSystemCredentialRepository credentials, + GithubCredentialCipher cipher, AuditLogService auditLogService, + RequestIdAccessor requestIdAccessor) { + this.credentials = credentials; + this.cipher = cipher; + this.auditLogService = auditLogService; + this.requestIdAccessor = requestIdAccessor; + } + + public List list() { + return credentials.list().stream().map(this::toResponse).toList(); + } + + @Transactional + public GithubCredentialResponse create(GithubCredentialUpsertRequest body, String actorId, + AuditRequestContext request) { + requireKey(); + String label = label(body.label()); + String token = token(body.token()); + UUID id = UUID.randomUUID(); + var encrypted = cipher.encrypt(id, token); + String fingerprint = fingerprint(token); + GithubSystemCredential credential = new GithubSystemCredential(id, label, fingerprint, + encrypted.keyVersion(), encrypted.nonce(), encrypted.ciphertext(), "ACTIVE", + null, null, Instant.now(), Instant.now()); + try { + credentials.create(credential); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.credential.duplicate"); + } + audit("CREATE_GITHUB_CREDENTIAL", id, fingerprint, actorId, request); + return toResponse(credentials.find(id).orElseThrow()); + } + + @Transactional + public GithubCredentialResponse replace(UUID id, GithubCredentialUpsertRequest body, + String actorId, AuditRequestContext request) { + requireKey(); + require(id); + String label = label(body.label()); + String token = token(body.token()); + var encrypted = cipher.encrypt(id, token); + String fingerprint = fingerprint(token); + try { + credentials.replace(id, label, fingerprint, encrypted.keyVersion(), + encrypted.nonce(), encrypted.ciphertext()); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.credential.duplicate"); + } + audit("REPLACE_GITHUB_CREDENTIAL", id, fingerprint, actorId, request); + return toResponse(require(id)); + } + + @Transactional + public GithubCredentialResponse setEnabled(UUID id, boolean enabled, + String actorId, AuditRequestContext request) { + GithubSystemCredential previous = require(id); + if (enabled && "INVALID".equals(previous.status())) { + throw new DomainBadRequestException("error.githubSync.credential.replaceRequired"); + } + if (enabled && !cipher.supports(previous.keyVersion())) { + throw new DomainBadRequestException("error.githubSync.credential.keyUnavailable"); + } + String nextStatus = enabled ? "ACTIVE" : "DISABLED"; + if (!nextStatus.equals(previous.status())) { + if (!credentials.setEnabled(id, enabled)) { + throw new DomainBadRequestException("error.githubSync.credential.replaceRequired"); + } + audit(enabled ? "ENABLE_GITHUB_CREDENTIAL" : "DISABLE_GITHUB_CREDENTIAL", + id, previous.fingerprint(), actorId, request); + } + return toResponse(require(id)); + } + + private GithubCredentialResponse toResponse(GithubSystemCredential credential) { + return GithubCredentialResponse.from(credential, cipher, credentials.quotas(credential.id())); + } + + private GithubSystemCredential require(UUID id) { + return credentials.find(id) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.credential.notFound")); + } + + private void requireKey() { + if (!cipher.configured()) { + throw new DomainBadRequestException("error.githubSync.credential.keyUnavailable"); + } + } + + private String label(String value) { + if (value == null || value.isBlank() || value.trim().length() > 100) { + throw new DomainBadRequestException("error.githubSync.credential.label"); + } + return value.trim(); + } + + private String token(String value) { + if (value == null || value.length() < 20 || value.length() > 512 + || value.chars().anyMatch(character -> character < 33 || character > 126)) { + throw new DomainBadRequestException("error.githubSync.credential.token"); + } + return value; + } + + private String fingerprint(String token) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest).substring(0, 16); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private void audit(String action, UUID id, String fingerprint, String actorId, + AuditRequestContext request) { + auditLogService.record(actorId, action, "GITHUB_CREDENTIAL", null, + requestIdAccessor.current(), request.clientIp(), request.userAgent(), + AuditDetail.builder().put("credentialId", id).put("fingerprint", fingerprint).build()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppService.java new file mode 100644 index 000000000..f03b5128d --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppService.java @@ -0,0 +1,220 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import com.iflytek.skillhub.githubsync.GithubOfficialQuotaRepository; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryBudgetRepository; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Service +public class GithubCredentialLeaseAppService { + private final GithubSyncLeaseService leases; + private final GithubSystemCredentialRepository credentials; + private final GithubCredentialCipher cipher; + private final ObjectMapper mapper; + private final GithubOfficialQuotaRepository officialQuota; + private final GithubOfficialDiscoveryBudgetRepository officialBudget; + + public GithubCredentialLeaseAppService(GithubSyncLeaseService leases, + GithubSystemCredentialRepository credentials, GithubCredentialCipher cipher, + ObjectMapper mapper) { + this(leases, credentials, cipher, mapper, null); + } + + public GithubCredentialLeaseAppService(GithubSyncLeaseService leases, + GithubSystemCredentialRepository credentials, GithubCredentialCipher cipher, + ObjectMapper mapper, GithubOfficialQuotaRepository officialQuota) { + this(leases, credentials, cipher, mapper, officialQuota, null); + } + + @org.springframework.beans.factory.annotation.Autowired + public GithubCredentialLeaseAppService(GithubSyncLeaseService leases, + GithubSystemCredentialRepository credentials, GithubCredentialCipher cipher, + ObjectMapper mapper, GithubOfficialQuotaRepository officialQuota, + GithubOfficialDiscoveryBudgetRepository officialBudget) { + this.officialBudget = officialBudget; + this.leases = leases; + this.credentials = credentials; + this.cipher = cipher; + this.mapper = mapper; + this.officialQuota = officialQuota; + } + + @Transactional + public Optional claim(GithubSyncTaskKind kind, String workerId) { + return claim(kind, workerId, List.of()); + } + + @Transactional + public Optional claim(GithubSyncTaskKind kind, String workerId, List capabilities) { + var task = capabilities.isEmpty() ? leases.claim(kind, workerId) : leases.claim(kind, workerId, capabilities); + return task.flatMap(leased -> { + Instant now = Instant.now(); + String resource = quotaResource(leased.task().kind(), leased.task().snapshotJson()); + Instant cooldown = officialQuota == null ? null : officialQuota.cooldown(resource, now); + if (cooldown != null) { + officialQuota.deferClaim(leased.task().taskId(), leased.task().leaseGeneration(), cooldown, now); + return Optional.empty(); + } + var available = credentials.availableForClaim(now, resource); + if (available.isEmpty()) { + if (officialQuota != null && credentials.hasConfiguredCredentials()) { + officialQuota.deferClaim(leased.task().taskId(), leased.task().leaseGeneration(), now.plusSeconds(60), now, "CREDENTIAL_UNAVAILABLE"); + return Optional.empty(); + } + if (officialQuota != null) officialQuota.clearClaimWait(leased.task().taskId(), leased.task().leaseGeneration()); + reserve(leased); + return Optional.of(new Claimed(leased, null)); + } + var credential = available.get(); + String token = cipher.decrypt(credential.id(), credential.keyVersion(), + credential.nonce(), credential.ciphertext()); + if (!credentials.assignToLease(leased.task().taskId(), + leased.task().leaseGeneration(), credential.id(), + credential.fingerprint(), Instant.now())) { + throw new IllegalStateException("Could not bind GitHub credential to current lease"); + } + if (officialQuota != null) officialQuota.clearClaimWait(leased.task().taskId(), leased.task().leaseGeneration()); + reserve(leased); + return Optional.of(new Claimed(leased, new LeasedCredential(credential.id(), token))); + }); + } + + private void reserve(GithubSyncLeaseService.LeasedTask leased) { + if (officialBudget != null && leased.task().kind() == GithubSyncTaskKind.DISCOVERY) { + officialBudget.reserveSuccessfulClaim(leased.task().taskId(), leased.task().leaseGeneration()); + } + } + + private String quotaResource(GithubSyncTaskKind kind, String snapshotJson) { + if (kind == GithubSyncTaskKind.REPOSITORY_SYNC) { + return "CORE"; + } + try { + var snapshot = mapper.readTree(snapshotJson); + if (snapshot.has("officialDiscovery")) { + return switch (snapshot.path("officialDiscovery").path("channel").asText()) { + case "REPOSITORY_METADATA", "README" -> "SEARCH"; + case "OWNER_EXPANSION" -> "CORE"; + default -> throw new IllegalStateException("Unsupported official discovery channel"); + }; + } + String entryType = snapshot.path("entryType").asText(); + return switch (entryType) { + case "SEARCH_RULE", "SEARCH_PREVIEW" -> "SEARCH"; + case "CURATED_OWNER", "CURATED_REPOSITORY" -> "CORE"; + default -> throw new IllegalStateException("Unsupported GitHub discovery entry type"); + }; + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid GitHub discovery task snapshot", exception); + } + } + + @Transactional + public boolean reportStatus(UUID taskId, long generation, String leaseToken, + UUID credentialId, String status, Instant retryAt, Quota quota) { + return reportStatus(taskId, generation, leaseToken, credentialId, status, retryAt, quota, null); + } + + @Transactional + public boolean reportStatus(UUID taskId, long generation, String leaseToken, + UUID credentialId, String status, Instant retryAt, Quota quota, + GithubOfficialQuotaRepository.Observation observation) { + if (taskId == null || credentialId == null || generation < 1 || leaseToken == null + || !leaseToken.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid GitHub credential lease proof"); + } + Instant now = Instant.now(); + String tokenHash = hash(leaseToken); + if (quota != null && (quota.resource() == null + || !java.util.Set.of("CORE", "SEARCH", "GRAPHQL", "OTHER").contains(quota.resource()) + || quota.remaining() < 0 || quota.resetAt() == null)) { + throw new IllegalArgumentException("Invalid GitHub quota observation"); + } + if (observation != null) { + if (officialQuota == null || status == null + || "OBSERVED".equals(status) && (quota == null || retryAt != null || !"OBSERVED".equals(observation.state())) + || "AUTH_INVALID".equals(status) && (retryAt != null || !"AUTH_INVALID".equals(observation.state())) + || "RATE_LIMITED".equals(status) && (retryAt == null || !retryAt.isAfter(now) + || !List.of("PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN").contains(observation.state())) + || !List.of("OBSERVED", "AUTH_INVALID", "RATE_LIMITED").contains(status)) { + throw new IllegalArgumentException("Credential status and official quota observation disagree"); + } + if (quota != null && (!quota.resource().equals(observation.resource()) + || observation.remaining() != null && quota.remaining() != observation.remaining() + || observation.resetAt() != null && !quota.resetAt().equals(observation.resetAt()))) { + throw new IllegalArgumentException("Credential quota values disagree"); + } + if ("SECONDARY_COOLDOWN".equals(observation.state()) && observation.retryAfterSeconds() == null) { + observation = new GithubOfficialQuotaRepository.Observation(observation.resource(), observation.observedAt(), + observation.state(), observation.remaining(), observation.resetAt(), + Math.toIntExact(java.time.Duration.between(now, retryAt).getSeconds() + 1)); + } + return officialQuota.record(taskId, generation, tokenHash, credentialId, observation, now); + } + boolean accepted = switch (status) { + case "OBSERVED" -> { + if (retryAt != null || quota == null) { + throw new IllegalArgumentException("Quota observation requires quota without retry time"); + } + yield credentials.recordQuotaForLease(taskId, generation, tokenHash, credentialId, + quota.resource(), quota.remaining(), quota.resetAt(), now); + } + case "AUTH_INVALID" -> { + if (retryAt != null) { + throw new IllegalArgumentException("Invalid credential cannot have a retry time"); + } + yield credentials.markInvalid(taskId, generation, tokenHash, credentialId, now); + } + case "RATE_LIMITED" -> { + if (retryAt == null || !retryAt.isAfter(now)) { + throw new IllegalArgumentException("Rate limit requires a future retry time"); + } + yield credentials.markRateLimited(taskId, generation, tokenHash, + credentialId, retryAt, now); + } + default -> throw new IllegalArgumentException("Unsupported GitHub credential status"); + }; + if (accepted && quota != null && !"OBSERVED".equals(status)) { + credentials.recordQuota(credentialId, quota.resource(), quota.remaining(), + quota.resetAt(), now); + } + return accepted; + } + + @Transactional + public boolean reportOfficialQuota(UUID taskId, long generation, String token, + GithubOfficialQuotaRepository.Observation observation) { + if (taskId == null || generation < 1 || token == null || !token.matches("[0-9a-f]{64}") || officialQuota == null) { + throw new IllegalArgumentException("Invalid official quota lease proof"); + } + return officialQuota.record(taskId, generation, hash(token), null, observation, Instant.now()); + } + + private String hash(String token) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + public record LeasedCredential(UUID credentialId, String token) {} + public record Claimed(GithubSyncLeaseService.LeasedTask lease, LeasedCredential credential) {} + public record Quota(String resource, int remaining, Instant resetAt) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryConfigurationAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryConfigurationAppService.java new file mode 100644 index 000000000..b1aa8c298 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryConfigurationAppService.java @@ -0,0 +1,81 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainConflictException; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryConfigurationRequest; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryConfigurationResponse; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfigurationRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.LinkedHashSet; +import java.util.List; + +@Service +public class GithubDeepDiscoveryConfigurationAppService { + private final GithubDeepDiscoveryConfigurationRepository configurations; + private final AuditLogService audits; + private final RequestIdAccessor requestIds; + + public GithubDeepDiscoveryConfigurationAppService(GithubDeepDiscoveryConfigurationRepository configurations, + AuditLogService audits, RequestIdAccessor requestIds) { + this.configurations = configurations; + this.audits = audits; + this.requestIds = requestIds; + } + + public GithubDeepDiscoveryConfigurationResponse get() { + return response(configurations.get()); + } + + @Transactional + public GithubDeepDiscoveryConfigurationResponse update(GithubDeepDiscoveryConfigurationRequest request, + String actorId, AuditRequestContext context) { + if (request.expectedVersion() == null || request.expectedVersion() < 1 + || request.enabled() == null || request.budgetProfile() == null + || request.exclusions() == null || request.exclusions().size() > 1000) { + throw new DomainBadRequestException("error.githubSync.deepDiscovery.configurationInvalid"); + } + var exclusions = new LinkedHashSet(); + for (var exclusion : request.exclusions()) { + if (exclusion == null) { + throw new DomainBadRequestException("error.githubSync.deepDiscovery.exclusionInvalid"); + } + try { + exclusions.add(new GithubDeepDiscoveryConfiguration.Exclusion(exclusion.kind(), exclusion.value())); + } catch (IllegalArgumentException exception) { + throw new DomainBadRequestException("error.githubSync.deepDiscovery.exclusionInvalid"); + } + } + GithubDeepDiscoveryConfiguration previous = configurations.get(); + if (previous.version() != request.expectedVersion()) { + throw new DomainConflictException("error.githubSync.deepDiscovery.configurationConflict"); + } + GithubDeepDiscoveryConfiguration updated = configurations.update(request.expectedVersion(), + request.enabled(), request.budgetProfile(), List.copyOf(exclusions), actorId) + .orElseThrow(() -> new DomainConflictException("error.githubSync.deepDiscovery.configurationConflict")); + audits.record(actorId, "UPDATE_GITHUB_DEEP_DISCOVERY_CONFIGURATION", "GITHUB_DEEP_DISCOVERY_CONFIG", + null, requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder() + .put("namespaceId", updated.namespaceId()) + .put("previousVersion", previous.version()) + .put("version", updated.version()) + .put("previousEnabled", previous.enabled()) + .put("enabled", updated.enabled()) + .put("previousBudgetProfile", previous.budgetProfile()) + .put("budgetProfile", updated.budgetProfile()) + .put("previousExclusions", previous.exclusions()) + .put("exclusions", updated.exclusions()).build()); + return response(updated); + } + + private GithubDeepDiscoveryConfigurationResponse response(GithubDeepDiscoveryConfiguration configuration) { + return new GithubDeepDiscoveryConfigurationResponse(configuration.version(), configuration.enabled(), + configuration.budgetProfile(), configuration.exclusions(), configuration.strategyVersion(), + configuration.updatedAt(), configuration.updatedBy()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryReadAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryReadAppService.java new file mode 100644 index 000000000..e3e3a81a7 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDeepDiscoveryReadAppService.java @@ -0,0 +1,226 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.dto.GithubOfficialWorkerAvailabilityResponse; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfigurationRepository; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@Transactional(readOnly = true) +public class GithubDeepDiscoveryReadAppService { + private final GithubDeepDiscoveryQueryRepository queries; + private final GithubDeepDiscoveryConfigurationRepository configurations; + private final GithubSyncWorkerPresenceRepository workers; + public GithubDeepDiscoveryReadAppService(GithubDeepDiscoveryQueryRepository queries, + GithubDeepDiscoveryConfigurationRepository configurations, GithubSyncWorkerPresenceRepository workers) { + this.queries=queries;this.configurations=configurations;this.workers=workers; + } + public Summary summary(Instant from,Instant until) { + Instant now=Instant.now(); + if(from==null || until==null || !from.isBefore(until) || until.isAfter(now) + || Duration.between(from,until).compareTo(Duration.ofDays(90))>0) invalid(); + var config=configurations.get();var worker=workers.officialStatus(); + var channels=queries.channelStates(config.namespaceId()); + var waits=waits(config,null,null); + boolean running=channels.stream().anyMatch(c->c.running() && queries.compatibleWorker(c.channel())); + String state=!config.enabled()?"DISABLED":running?"RUNNING" + :!waits.isEmpty() || channels.stream().anyMatch(c->c.pending()>0 || c.gaps()>0)?"WAITING":"IDLE"; + return new Summary(config.version(),config.strategyVersion(),state,now,worker, + queries.recent(config.namespaceId(),from,until),queries.current(config.namespaceId(),now), + waits,queries.recentRuns(config.namespaceId())); + } + public Progress progress() { + var config=configurations.get();var worker=workers.officialStatus(); + var states=queries.channelStates(config.namespaceId()); + var channels=new ArrayList(); + for(Channel channel:Channel.values()) { + var row=states.stream().filter(c->c.channel()==channel).findFirst().orElse(null); + var waits=waits(config,channel,null); + String state=!config.enabled()?"DISABLED":row!=null && row.running() && queries.compatibleWorker(channel)?"RUNNING" + :!waits.isEmpty() || row!=null && (row.pending()>0 || row.gaps()>0)?"WAITING":"IDLE"; + channels.add(new ChannelProgress(channel,state,row==null?0:row.completed(),row==null?0:row.pending(), + row==null?0:row.gaps(),row==null?null:row.progress(),waits)); + } + var quotas=queries.quotas(); + return new Progress(config.strategyVersion(),Instant.now(),channels, + List.copyOf(quotas.subList(0,Math.min(100,quotas.size()))),quotas.size()>100); + } + /** Merge bounded reasons from current controls and durable task evidence, never raw errors. */ + private List waits(GithubDeepDiscoveryConfiguration config, Channel channel, RepositorySummary repository) { + long namespace=config.namespaceId(); + Long repositoryId=repository==null?null:repository.repositoryId(); + boolean detail=repository!=null; + boolean source=detail && repository.sourceId()!=null; + boolean contactsOnline=workers.status().online(); + var evidence=new ArrayList(); + List repositoryWork=source?List.of("SOURCE_UPDATE","PACKAGE_ACQUISITION") + :List.of("CANDIDATE_VALIDATION","PACKAGE_ACQUISITION"); + if(!contactsOnline) { + add(evidence,"WORKER_UNAVAILABLE",detail?repositoryWork:List.of("NEW_DISCOVERY","CANDIDATE_VALIDATION","PACKAGE_ACQUISITION","SOURCE_UPDATE"),null, + "等待采集服务恢复联系；到达重试时间不表示任务已执行。","CHECK_WORKER"); + } else if(detail) { + if(queries.repositoryNeedsOfficialWorker(namespace,repositoryId) && !queries.pauseCapableWorker()) + add(evidence,"CAPABILITY_UNAVAILABLE",repositoryWork,null, + "等待支持官方候选暂停协议的实例恢复联系。","CHECK_WORKER"); + } else { + boolean compatible=channel==null + ? queries.compatibleWorker(Channel.REPOSITORY_METADATA) && queries.compatibleWorker(Channel.OWNER_EXPANSION) + : queries.compatibleWorker(channel); + if(!compatible) add(evidence,"CAPABILITY_UNAVAILABLE",List.of("NEW_DISCOVERY"),null, + "至少一个探索渠道没有兼容实例；不代表其他渠道或普通来源实例已离线。","CHECK_WORKER"); + if(channel==null && queries.hasOfficialOnlyCandidates(namespace) && !queries.pauseCapableWorker()) + add(evidence,"CAPABILITY_UNAVAILABLE",List.of("CANDIDATE_VALIDATION","PACKAGE_ACQUISITION"),null, + "等待支持官方候选暂停协议的实例恢复联系。","CHECK_WORKER"); + } + if (detail && queries.credentialPoolUnavailable("CORE")) { + add(evidence,"CREDENTIAL_UNAVAILABLE",repositoryWork,null, + "当前没有可用于仓库读取的已配置凭证；检查凭证启用状态、有效性和资源额度，恢复后自动领取。","MANAGE_CREDENTIALS"); + } else if (!detail) { + boolean searchNeeded=channel==null || channel!=Channel.OWNER_EXPANSION; + boolean coreNeeded=channel==null || channel==Channel.OWNER_EXPANSION; + var affected=new java.util.LinkedHashSet(); + if(searchNeeded && queries.credentialPoolUnavailable("SEARCH")) affected.add("NEW_DISCOVERY"); + if(coreNeeded && queries.credentialPoolUnavailable("CORE")) { + affected.add("NEW_DISCOVERY"); + if(channel==null) affected.addAll(List.of("CANDIDATE_VALIDATION","PACKAGE_ACQUISITION","SOURCE_UPDATE")); + } + if(!affected.isEmpty()) add(evidence,"CREDENTIAL_UNAVAILABLE",List.copyOf(affected),null, + "对应资源没有可用的已配置凭证；检查凭证启用状态、有效性和额度，恢复后自动领取。此状态不等于确定限流。","MANAGE_CREDENTIALS"); + } + if(!detail) { + var budgetWait=queries.discoveryBudgetWait(); + if(config.enabled() && budgetWait!=null && budgetWait.blocked()) add(evidence,"BUDGET",List.of("NEW_DISCOVERY"),budgetWait.retryAt(), + "新官方租约须同时等待预算时间到达和在途官方租约完成或过期；到时不保证立即执行。已有租约继续，普通来源不受此预算影响。","EDIT_CONFIGURATION"); + if(!config.enabled()) add(evidence,"DISABLED",List.of("NEW_DISCOVERY"),null, + "管理员启用探索后保留进度续跑；来源仍使用各自开关。","EDIT_CONFIGURATION"); + if(queries.candidateCapacityBlocked(namespace)) add(evidence,"CANDIDATE_CAPACITY",List.of("NEW_DISCOVERY"),null, + "候选处理释放容量后恢复接收新仓库；已存在候选仍可处理。","MANAGE_ADMISSION"); + for(var row:queries.discoveryWaits(namespace,channel)) addTaskEvidence(evidence,row); + } + if(detail || channel==null) { + for(var row:queries.repositoryWaits(namespace,repositoryId)) addTaskEvidence(evidence,row); + } + if(queries.admissionBlocked(namespace)) { + var affected=detail && !source?List.of("CANDIDATE_VALIDATION","PACKAGE_ACQUISITION"):List.of("PACKAGE_ACQUISITION"); + add(evidence,"REVIEW_BACKPRESSURE",affected,null, + "水位配置有效且未解决项低于恢复线后允许新包；已准入包可结算，来源轻量检查不因此视为停止。","MANAGE_ADMISSION"); + } + for(var quota:queries.quotaWaits()) { + var affected=new java.util.LinkedHashSet(); + boolean allResources=quota.resource()==null; + if(detail) { + if(allResources || "CORE".equals(quota.resource())) affected.addAll(repositoryWork); + } else { + if(allResources || channel==null && ("SEARCH".equals(quota.resource()) || "CORE".equals(quota.resource())) + || channel==Channel.OWNER_EXPANSION && "CORE".equals(quota.resource()) + || channel!=null && channel!=Channel.OWNER_EXPANSION && "SEARCH".equals(quota.resource())) + affected.add("NEW_DISCOVERY"); + if(channel==null && (allResources || "CORE".equals(quota.resource()))) + affected.addAll(List.of("CANDIDATE_VALIDATION","PACKAGE_ACQUISITION","SOURCE_UPDATE")); + } + if(!affected.isEmpty()) add(evidence,quota.reason(),List.copyOf(affected),quota.retryAt(), + "共享保护期间暂停对应资源的新任务领取和新包准入；已准入包可结算。旧协议在途任务未必逐请求停止，保护原因不等于确证次级限流。","MANAGE_CREDENTIALS"); + } + var observedContinuing=contactsOnline?queries.progressingWork(namespace,repositoryId,channel):List.of(); + return mergeWaits(evidence,observedContinuing); + } + private static void addTaskEvidence(List target,GithubDeepDiscoveryQueryRepository.WaitEvidence evidence) { + String raw=evidence.reason(); + String reason=waitReason(raw); + if(reason==null) return; + // Pool state is re-evaluated above, so an old task label cannot outlive credential recovery. + if("CREDENTIAL_UNAVAILABLE".equals(reason)) return; + if(List.of("PRIMARY_RATE_LIMIT","SECONDARY_COOLDOWN","QUOTA_COOLDOWN").contains(reason) + && evidence.retryAt()!=null && !evidence.retryAt().isAfter(Instant.now())) return; + // Current eligibility supplies these controls; stale task pause labels must not override a resumed source. + String recovery="满足持久任务的恢复条件后自动领取；时间仅是最早可领取，不保证执行。"; + String action=switch(reason) { + case "CREDENTIAL_UNAVAILABLE","PRIMARY_RATE_LIMIT","SECONDARY_COOLDOWN","QUOTA_COOLDOWN" -> "MANAGE_CREDENTIALS"; + case "REVIEW_BACKPRESSURE","CANDIDATE_CAPACITY" -> "MANAGE_ADMISSION"; + case "EXCLUDED","DISABLED" -> "EDIT_CONFIGURATION"; + default -> null; + }; + if("SOURCE_PAUSED".equals(raw)) { + recovery="由来源管理员恢复该仓库来源；探索开关和其他命中入口不会覆盖来源自身暂停。"; + action=null; + } else if("EXCLUDED".equals(raw)) { + recovery="移除当前命中的排除项后按保留进度继续；已登记来源与独立入口依据自身资格运行。"; + } else if("DISABLED".equals(raw)) { + recovery="官方独占候选随探索启停；重开后按保留进度续跑。"; + } + List affected=List.of(evidence.work()); + if("CANDIDATE_VALIDATION".equals(evidence.work()) || "SOURCE_PAUSED".equals(raw)) + affected=List.of(evidence.work(),"PACKAGE_ACQUISITION"); + add(target,reason,affected,evidence.retryAt(),recovery,action); + } + private static void add(List target,String reason,List affected,Instant retry,String recovery,String action) { + target.add(new Wait(reason,affected,List.of(),retry,recovery,action)); + } + private static List mergeWaits(List evidence,List observedContinuing) { + var reasons=new java.util.LinkedHashMap>(); + evidence.forEach(wait->reasons.computeIfAbsent(wait.reason(),ignored->new ArrayList<>()).add(wait)); + var blocked=evidence.stream().filter(wait->!"BUDGET".equals(wait.reason())).flatMap(wait->wait.affectedWork().stream()).collect(java.util.stream.Collectors.toSet()); + var continuing=observedContinuing.stream().filter(work->!blocked.contains(work)).distinct().sorted().toList(); + var result=new ArrayList(); + for(var entry:reasons.entrySet()) { + var values=entry.getValue(); + var affected=values.stream().flatMap(wait->wait.affectedWork().stream()).distinct().sorted().toList(); + Instant retry=values.stream().anyMatch(wait->wait.retryAt()==null)?null: + values.stream().map(Wait::retryAt).max(Instant::compareTo).orElse(null); + String recovery=values.stream().map(Wait::recoveryCondition).distinct().collect(java.util.stream.Collectors.joining(" ")); + if(recovery.length()>500) recovery=recovery.substring(0,500); + String action=values.stream().map(Wait::action).filter(java.util.Objects::nonNull).findFirst().orElse(null); + result.add(new Wait(entry.getKey(),affected,continuing,retry,recovery,action)); + } + if(result.size()>32) throw new IllegalStateException("Discovery wait reason catalog exceeds response bound"); + return List.copyOf(result); + } + private static String waitReason(String reason) { + if(reason==null) return null; + return switch(reason) { + case "DISABLED","EXCLUDED","BUDGET","PRIMARY_RATE_LIMIT","SECONDARY_COOLDOWN","QUOTA_COOLDOWN", + "CREDENTIAL_UNAVAILABLE","CANDIDATE_CAPACITY","REVIEW_BACKPRESSURE","COVERAGE_GAP" -> reason; + case "SOURCE_PAUSED" -> "DISABLED"; + case "GITHUB_RATE_LIMITED" -> "PRIMARY_RATE_LIMIT"; + case "BACKPRESSURE_ACTIVE" -> "REVIEW_BACKPRESSURE"; + case "CANDIDATE_BUDGET_EXCEEDED" -> "BUDGET"; + default -> null; + }; + } + public DiscoveryPage repositories(int page,int size,Instant asOf,String q,RepositoryStatus status,Channel channel,Boolean paused) { + Instant bound=page(page,size,asOf);if(q!=null && q.length()>200) invalid(); + return queries.repositories(configurations.get().namespaceId(),page,size,bound,q,status,channel,paused); + } + public RepositoryDetail repository(long id) { + identity(id); + var config=configurations.get(); + var detail=queries.detail(config.namespaceId(),id); + var waits=waits(config,null,detail.repository()); + String reason=detail.notImportedReason(); + if("NOT_CHECKED".equals(reason) && waits.stream().anyMatch(w-> + "REVIEW_BACKPRESSURE".equals(w.reason()))) reason="WAITING_CAPACITY"; + return new RepositoryDetail(detail.repository(),detail.lastCheckedAt(),detail.nextEligibleAt(),reason, + waits,detail.latestRunId(),detail.historyAvailableSince()); + } + public DiscoveryPage provenance(long id,int page,int size,Instant asOf) { + identity(id);Instant bound=page(page,size,asOf);return queries.provenance(configurations.get().namespaceId(),id,page,size,bound); + } + public DiscoveryPage paths(long id,int page,int size,Instant asOf) { + identity(id);Instant bound=page(page,size,asOf);return queries.paths(configurations.get().namespaceId(),id,page,size,bound); + } + private static Instant page(int page,int size,Instant asOf) { + Instant now=Instant.now();if(page<0 || page>1000 || size<1 || size>100 || (long)page*size>100000 || asOf!=null && asOf.isAfter(now)) invalid(); + return asOf==null?now:asOf; + } + private static void identity(long id) { if(id<1) invalid(); } + private static void invalid() { throw new DomainBadRequestException("error.githubSync.deepDiscovery.queryInvalid"); } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDiscoveryAdminAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDiscoveryAdminAppService.java new file mode 100644 index 000000000..49fdb9ad9 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubDiscoveryAdminAppService.java @@ -0,0 +1,249 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubCuratedOwnerRequest; +import com.iflytek.skillhub.dto.GithubCuratedRepositoryRequest; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryResponse; +import com.iflytek.skillhub.dto.GithubSearchRuleRequest; +import com.iflytek.skillhub.dto.GithubSearchSelectorRequest; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntry; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubDiscoveryEntryStatusQueryRepository; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +@Service +public class GithubDiscoveryAdminAppService { + private static final Pattern OWNER = Pattern.compile("^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$"); + private static final Pattern TOPIC = Pattern.compile("^[A-Za-z0-9][A-Za-z0-9-]*$"); + private static final Pattern REPOSITORY = Pattern.compile("^[A-Za-z0-9_.-]{1,100}$"); + + private final GithubDiscoveryEntryRepository entries; + private final GithubSyncSchedulePolicyRepository schedulePolicies; + private final ObjectMapper objectMapper; + private final AuditLogService auditLogService; + private final RequestIdAccessor requestIdAccessor; + private final GithubDiscoveryEntryStatusQueryRepository entryStatus; + + public GithubDiscoveryAdminAppService(GithubDiscoveryEntryRepository entries, + GithubSyncSchedulePolicyRepository schedulePolicies, + ObjectMapper objectMapper, + AuditLogService auditLogService, + RequestIdAccessor requestIdAccessor, + GithubDiscoveryEntryStatusQueryRepository entryStatus) { + this.entries = entries; + this.schedulePolicies = schedulePolicies; + this.objectMapper = objectMapper; + this.auditLogService = auditLogService; + this.requestIdAccessor = requestIdAccessor; + this.entryStatus = entryStatus; + } + + public List list() { + var configured = entries.listPlatformEntries(); + if (configured.isEmpty()) return List.of(); + var executions = entryStatus.latestForEntries(configured.getFirst().namespaceId(), + configured.stream().map(GithubDiscoveryEntry::id).toList()); + return configured.stream().map(entry -> toResponse(entry, executions.get(entry.id()))).toList(); + } + + @Transactional + public GithubDiscoveryEntryResponse createSearchRule(GithubSearchRuleRequest request, + String actorId, AuditRequestContext context) { + String title = normalizeTitle(request.title()); + String criteriaJson = serializeSelectors(request.selectors()); + int interval = intervalSeconds(request.targetHours(), schedulePolicies.getForUpdate().searchRuleHours()); + GithubDiscoveryEntry created; + try { + created = entries.createSearchRule(title, criteriaJson, interval); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.duplicate"); + } + recordAudit("CREATE_GITHUB_DISCOVERY_ENTRY", created, actorId, context); + return toResponse(created); + } + + @Transactional + public GithubDiscoveryEntryResponse updateSearchRule(UUID id, GithubSearchRuleRequest request, + String actorId, AuditRequestContext context) { + GithubDiscoveryEntry existing = requireEntry(id); + if (!"SEARCH_RULE".equals(existing.entryType())) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.type"); + } + String title = normalizeTitle(request.title()); + String criteriaJson = serializeSelectors(request.selectors()); + int interval = intervalSeconds(request.targetHours(), schedulePolicies.getForUpdate().searchRuleHours()); + GithubDiscoveryEntry updated; + try { + updated = entries.updateSearchRule(id, title, criteriaJson, interval, existing.enabled()); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.duplicate"); + } + recordAudit("UPDATE_GITHUB_DISCOVERY_ENTRY", updated, actorId, context); + return toResponse(updated); + } + + @Transactional + public GithubDiscoveryEntryResponse createCuratedOwner(GithubCuratedOwnerRequest request, + String actorId, AuditRequestContext context) { + String ownerLogin = request.ownerLogin().trim(); + if (!OWNER.matcher(ownerLogin).matches()) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.owner"); + } + int interval = intervalSeconds(request.targetHours(), schedulePolicies.getForUpdate().curatedOwnerHours()); + GithubDiscoveryEntry created; + try { + created = entries.createCuratedOwner(ownerLogin, interval); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.duplicate"); + } + recordAudit("CREATE_GITHUB_DISCOVERY_ENTRY", created, actorId, context); + return toResponse(created); + } + + @Transactional + public GithubDiscoveryEntryResponse createCuratedRepository(GithubCuratedRepositoryRequest request, + String actorId, AuditRequestContext context) { + String ownerLogin = request.ownerLogin().trim(); + String repositoryName = request.repositoryName().trim(); + if (!OWNER.matcher(ownerLogin).matches() || !REPOSITORY.matcher(repositoryName).matches() + || repositoryName.equals(".") || repositoryName.equals("..")) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.repository"); + } + GithubDiscoveryEntry created; + try { + created = entries.createCuratedRepository(ownerLogin, repositoryName); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.duplicate"); + } + recordAudit("CREATE_GITHUB_DISCOVERY_ENTRY", created, actorId, context); + return toResponse(created); + } + + @Transactional + public GithubDiscoveryEntryResponse setEnabled(UUID id, boolean enabled, + String actorId, AuditRequestContext context) { + GithubDiscoveryEntry existing = requireEntry(id); + GithubDiscoveryEntry updated = existing.enabled() == enabled + ? existing : entries.setEnabled(id, enabled); + if (existing.enabled() != enabled) { + recordAudit("SET_GITHUB_DISCOVERY_ENTRY_ENABLED", updated, actorId, context); + } + return toResponse(updated); + } + + private GithubDiscoveryEntry requireEntry(UUID id) { + return entries.findPlatformEntry(id) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.discoveryEntry.notFound")); + } + + String serializeSelectors(List requested) { + if (requested == null || requested.isEmpty() || requested.size() > 20) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + List normalized = new ArrayList<>(); + Set seen = new HashSet<>(); + for (GithubSearchSelectorRequest selector : requested) { + if (selector == null || selector.kind() == null || selector.value() == null + || selector.priority() == null) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + String value = selector.value().trim(); + if (value.isEmpty() || value.codePointCount(0, value.length()) > 100 + || !Set.of("CORE", "EXTENSION").contains(selector.priority())) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + if ("KEYWORD".equals(selector.kind())) { + boolean valid = value.codePoints().allMatch(codePoint -> + Character.isLetterOrDigit(codePoint) || codePoint == ' ' + || codePoint == '-' || codePoint == '_'); + if (!valid) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + } else if (!"TOPIC".equals(selector.kind()) || !TOPIC.matcher(value).matches()) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + String coordinate = selector.kind() + ":" + value.toLowerCase(Locale.ROOT); + if (!seen.add(coordinate)) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.selectors"); + } + normalized.add(new GithubSearchSelectorRequest(selector.kind(), value, selector.priority())); + } + try { + return objectMapper.writeValueAsString(Map.of("selectors", normalized)); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Could not serialize GitHub search criteria", exception); + } + } + + private String normalizeTitle(String title) { + if (title == null || title.isBlank() || title.trim().codePointCount(0, title.trim().length()) > 128) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.title"); + } + return title.trim(); + } + + private int intervalSeconds(Integer targetHours, int fallbackHours) { + int hours = targetHours == null ? fallbackHours : targetHours; + if (hours < 1 || hours > 720) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.interval"); + } + return hours * 3600; + } + + private GithubDiscoveryEntryResponse toResponse(GithubDiscoveryEntry entry) { + return toResponse(entry, entryStatus.latest(entry.namespaceId(), entry.id())); + } + + private GithubDiscoveryEntryResponse toResponse(GithubDiscoveryEntry entry, GithubDiscoveryEntryResponse.Execution execution) { + List selectors = List.of(); + if (entry.criteriaJson() != null) { + try { + JsonNode stored = objectMapper.readTree(entry.criteriaJson()).path("selectors"); + if (!stored.isArray()) { + throw new IllegalStateException("GitHub search criteria is malformed"); + } + List parsed = new ArrayList<>(); + for (JsonNode selector : stored) { + parsed.add(new GithubSearchSelectorRequest(selector.path("kind").asText(), + selector.path("value").asText(), selector.path("priority").asText())); + } + selectors = List.copyOf(parsed); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Could not parse GitHub search criteria", exception); + } + } + return new GithubDiscoveryEntryResponse(entry.id(), entry.entryType(), entry.title(), + entry.ownerLogin(), entry.repositoryName(), entry.repositoryId(), + entry.sourceVisibility(), selectors, + entry.ruleVersion(), entry.enabled(), + entry.targetIntervalSeconds() / 3600, entry.nextDueAt(), entry.updatedAt(), + execution); + } + + private void recordAudit(String action, GithubDiscoveryEntry entry, + String actorId, AuditRequestContext context) { + auditLogService.record(actorId, action, "GITHUB_DISCOVERY_ENTRY", null, + requestIdAccessor.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("entryId", entry.id()).put("entryType", entry.entryType()) + .put("ruleVersion", entry.ruleVersion()).put("enabled", entry.enabled()).build()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubPlatformControlAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubPlatformControlAppService.java new file mode 100644 index 000000000..d168c2ae4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubPlatformControlAppService.java @@ -0,0 +1,46 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainConflictException; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import com.iflytek.skillhub.githubsync.GithubPlatformControlRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +public class GithubPlatformControlAppService { + private final GithubPlatformControlRepository controls; + private final AuditLogService audits; + private final RequestIdAccessor requestIds; + + public GithubPlatformControlAppService(GithubPlatformControlRepository controls, AuditLogService audits, + RequestIdAccessor requestIds) { + this.controls = controls; + this.audits = audits; + this.requestIds = requestIds; + } + + public GithubPlatformControlRepository.State get() { + return controls.get(); + } + + @Transactional + public GithubPlatformControlRepository.State update(long expectedVersion, boolean paused, String actor, + AuditRequestContext context) { + GithubPlatformControlRepository.State state; + try { + state = controls.update(expectedVersion, paused, actor); + } catch (GithubOfficialProtocolException exception) { + if ("CONTROL_CHANGED".equals(exception.code())) { + throw new DomainConflictException("error.request.conflict"); + } + throw exception; + } + audits.record(actor, "UPDATE_GITHUB_PLATFORM_CONTROL", "GITHUB_PLATFORM_CONTROL", null, requestIds.current(), + context.clientIp(), context.userAgent(), AuditDetail.builder().put("namespaceId", state.namespaceId()) + .put("version", state.version()).put("paused", state.desiredPaused()).build()); + return state; + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSearchPreviewAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSearchPreviewAppService.java new file mode 100644 index 000000000..6f6fb12d1 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSearchPreviewAppService.java @@ -0,0 +1,49 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubSearchRuleRequest; +import com.iflytek.skillhub.githubsync.GithubSearchPreviewRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.UUID; + +@Service +public class GithubSearchPreviewAppService { + private final GithubSearchPreviewRepository previews; + private final GithubDiscoveryAdminAppService discovery; + private final ObjectMapper mapper; + + public GithubSearchPreviewAppService(GithubSearchPreviewRepository previews, + GithubDiscoveryAdminAppService discovery, ObjectMapper mapper) { + this.previews = previews; + this.discovery = discovery; + this.mapper = mapper; + } + + @Transactional + public Preview queue(GithubSearchRuleRequest request, String actorId) { + String criteria = discovery.serializeSelectors(request.selectors()); + return get(previews.queue(criteria, actorId, Instant.now())); + } + + public Preview get(UUID id) { + GithubSearchPreviewRepository.Preview persisted = previews.find(id) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.discoveryEntry.notFound")); + try { + JsonNode snapshot = mapper.readTree(persisted.snapshotJson()); + return new Preview(persisted.id(), persisted.status(), persisted.errorCode(), persisted.retryAt(), + snapshot.path("criteria"), snapshot.path("searchWindow").path("until").asText(), + snapshot.get("previewResult"), persisted.createdAt(), persisted.updatedAt()); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted GitHub search preview", exception); + } + } + + public record Preview(UUID id, String status, String errorCode, Instant retryAt, JsonNode criteria, + String searchedUntil, JsonNode result, Instant createdAt, Instant updatedAt) {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceIgnoreAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceIgnoreAppService.java new file mode 100644 index 000000000..6bd5356a0 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceIgnoreAppService.java @@ -0,0 +1,102 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository; +import com.iflytek.skillhub.domain.namespace.NamespaceRepository; +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.domain.namespace.NamespaceStatus; +import com.iflytek.skillhub.domain.namespace.NamespaceType; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubSourceIgnoreResponse; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import com.iflytek.skillhub.githubsync.GithubSkillPathPolicy; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Service +public class GithubSourceIgnoreAppService { + private final GithubSourceIgnoreRepository sources; + private final GithubSyncManagementQueryRepository management; + private final NamespaceRepository namespaces; + private final NamespaceMemberRepository members; + private final AuditLogService auditLogs; + private final RequestIdAccessor requestIds; + + public GithubSourceIgnoreAppService(GithubSourceIgnoreRepository sources, + GithubSyncManagementQueryRepository management, NamespaceRepository namespaces, + NamespaceMemberRepository members, AuditLogService auditLogs, + RequestIdAccessor requestIds) { + this.sources = sources; + this.management = management; + this.namespaces = namespaces; + this.members = members; + this.auditLogs = auditLogs; + this.requestIds = requestIds; + } + + public GithubSourceIgnoreResponse platform(long sourceId) { + return get(management.platformNamespaceId(), sourceId); + } + + @Transactional + public GithubSourceIgnoreResponse updatePlatform(long sourceId, List patterns, + String actorId, AuditRequestContext context) { + return update(management.platformNamespaceId(), sourceId, patterns, actorId, context); + } + + public GithubSourceIgnoreResponse team(String slug, long sourceId, String userId) { + return get(teamNamespaceId(slug, userId, false), sourceId); + } + + @Transactional + public GithubSourceIgnoreResponse updateTeam(String slug, long sourceId, List patterns, + String userId, AuditRequestContext context) { + return update(teamNamespaceId(slug, userId, true), sourceId, patterns, userId, context); + } + + private GithubSourceIgnoreResponse get(long namespaceId, long sourceId) { + return new GithubSourceIgnoreResponse(sourceId, sources.find(namespaceId, sourceId) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.source.notFound"))); + } + + private GithubSourceIgnoreResponse update(long namespaceId, long sourceId, List patterns, + String actorId, AuditRequestContext context) { + List normalized; + try { + normalized = GithubSkillPathPolicy.normalizePatterns(patterns); + } catch (IllegalArgumentException exception) { + throw new DomainBadRequestException("error.githubSync.source.ignorePatterns"); + } + List saved = sources.replace(namespaceId, sourceId, normalized) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.source.notFound")); + auditLogs.record(actorId, "UPDATE_GITHUB_SOURCE_IGNORE_PATTERNS", "GITHUB_SOURCE", sourceId, + requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("namespaceId", namespaceId) + .put("patternCount", saved.size()).build()); + return new GithubSourceIgnoreResponse(sourceId, saved); + } + + private long teamNamespaceId(String slug, String userId, boolean write) { + var namespace = namespaces.findBySlug(slug) + .orElseThrow(() -> new DomainNotFoundException("error.namespace.slug.notFound", slug)); + if (namespace.getType() != NamespaceType.TEAM) { + throw new DomainNotFoundException("error.namespace.slug.notFound", slug); + } + NamespaceRole role = members.findByNamespaceIdAndUserId(namespace.getId(), userId) + .map(member -> member.getRole()).orElse(null); + if (role != NamespaceRole.OWNER && role != NamespaceRole.ADMIN) { + throw new DomainForbiddenException("error.githubSync.teamSource.forbidden"); + } + if (write && namespace.getStatus() != NamespaceStatus.ACTIVE) { + throw new DomainBadRequestException("error.namespace.readonly", slug); + } + return namespace.getId(); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceRescanAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceRescanAppService.java new file mode 100644 index 000000000..0a34c85c3 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSourceRescanAppService.java @@ -0,0 +1,109 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubSourceRescanResponse; +import com.iflytek.skillhub.githubsync.GithubSourceTaskRepository; +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; + +@Service +public class GithubSourceRescanAppService { + private final GithubSourceTaskRepository tasks; + private final GithubSyncPublicationRepository publications; + private final GithubSourceIgnoreRepository ignores; + private final GithubTeamSourceAppService teamSources; + private final GithubSyncManagementQueryRepository management; + private final AuditLogService audits; + private final RequestIdAccessor requestIds; + + public GithubSourceRescanAppService(GithubSourceTaskRepository tasks, + GithubSyncPublicationRepository publications, GithubSourceIgnoreRepository ignores, + GithubTeamSourceAppService teamSources, GithubSyncManagementQueryRepository management, + AuditLogService audits, RequestIdAccessor requestIds) { + this.tasks = tasks; + this.publications = publications; + this.ignores = ignores; + this.teamSources = teamSources; + this.management = management; + this.audits = audits; + this.requestIds = requestIds; + } + + @Transactional + public GithubSourceRescanResponse platform(long sourceId, String actorId, + AuditRequestContext context) { + return request(management.platformNamespaceId(), sourceId, actorId, context); + } + + @Transactional + public GithubSourceRescanResponse team(String slug, long sourceId, String actorId, + AuditRequestContext context) { + long namespaceId = teamSources.requireTeamAdmin(slug, actorId, true).getId(); + return request(namespaceId, sourceId, actorId, context); + } + + @Transactional + public GithubSourceRescanResponse retryPlatformSkill(long sourceId, String skillPath, + String actorId, AuditRequestContext context) { + return retrySkill(management.platformNamespaceId(), sourceId, skillPath, actorId, context); + } + + @Transactional + public GithubSourceRescanResponse retryTeamSkill(String slug, long sourceId, String skillPath, + String actorId, AuditRequestContext context) { + long namespaceId = teamSources.requireTeamAdmin(slug, actorId, true).getId(); + return retrySkill(namespaceId, sourceId, skillPath, actorId, context); + } + + private GithubSourceRescanResponse retrySkill(long namespaceId, long sourceId, + String skillPath, String actorId, AuditRequestContext context) { + if (skillPath == null || skillPath.isBlank() || skillPath.length() > 1024) { + throw new DomainBadRequestException("error.badRequest"); + } + Instant now = Instant.now(); + var result = tasks.requestRescan(namespaceId, sourceId, now) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.source.notFound")); + if ("PAUSED".equals(result.status())) { + throw new DomainBadRequestException("error.githubSync.source.paused"); + } + boolean ignored; + try { + ignored = ignores.ignoredForSource(sourceId, skillPath); + } catch (IllegalArgumentException exception) { + throw new DomainBadRequestException("error.badRequest"); + } + if (ignored || !publications.requestRejectedRetry(namespaceId, sourceId, skillPath, now)) { + throw new DomainBadRequestException("error.badRequest"); + } + audits.record(actorId, "RETRY_REJECTED_GITHUB_SKILL", "GITHUB_SOURCE", sourceId, + requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("namespaceId", namespaceId).put("skillPath", skillPath) + .put("taskId", result.taskId()).build()); + return new GithubSourceRescanResponse(sourceId, result.status(), result.taskId(), + result.taskStatus(), result.nextCheckAt()); + } + + private GithubSourceRescanResponse request(long namespaceId, long sourceId, + String actorId, AuditRequestContext context) { + var result = tasks.requestRescan(namespaceId, sourceId, Instant.now()) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.source.notFound")); + if ("PAUSED".equals(result.status())) { + throw new DomainBadRequestException("error.githubSync.source.paused"); + } + audits.record(actorId, "REQUEST_GITHUB_SOURCE_RESCAN", "GITHUB_SOURCE", sourceId, + requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("namespaceId", namespaceId) + .put("status", result.status()).put("taskId", result.taskId()).build()); + return new GithubSourceRescanResponse(sourceId, result.status(), result.taskId(), + result.taskStatus(), result.nextCheckAt()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncAdmissionPolicyAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncAdmissionPolicyAppService.java new file mode 100644 index 000000000..1016d84d6 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncAdmissionPolicyAppService.java @@ -0,0 +1,64 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicy; +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicyRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +public class GithubSyncAdmissionPolicyAppService { + private final GithubSyncAdmissionPolicyRepository policies; + private final AuditLogService auditLogService; + private final RequestIdAccessor requestIdAccessor; + + public GithubSyncAdmissionPolicyAppService(GithubSyncAdmissionPolicyRepository policies, + AuditLogService auditLogService, + RequestIdAccessor requestIdAccessor) { + this.policies = policies; + this.auditLogService = auditLogService; + this.requestIdAccessor = requestIdAccessor; + } + + public GithubSyncAdmissionPolicy get() { + return policies.get(); + } + + @Transactional + public GithubSyncAdmissionPolicy update(int globalHigh, int globalLow, + int namespaceHigh, int namespaceLow, + int candidatePendingLimit, int candidatePerLeaseLimit, + String actorId, AuditRequestContext request) { + if (globalHigh <= 0 || globalLow < 0 || globalLow >= globalHigh + || namespaceHigh <= 0 || namespaceLow < 0 || namespaceLow >= namespaceHigh + || candidatePendingLimit < 1 || candidatePendingLimit > 100000 + || candidatePerLeaseLimit < 1 || candidatePerLeaseLimit > 10000 + || candidatePerLeaseLimit > candidatePendingLimit) { + throw new DomainBadRequestException("error.githubSync.admissionPolicy.invalid"); + } + GithubSyncAdmissionPolicy previous = policies.getForUpdate(); + GithubSyncAdmissionPolicy updated = policies.update( + globalHigh, globalLow, namespaceHigh, namespaceLow, + candidatePendingLimit, candidatePerLeaseLimit); + auditLogService.record(actorId, "UPDATE_GITHUB_SYNC_ADMISSION_POLICY", "GITHUB_SYNC_POLICY", + null, requestIdAccessor.current(), request.clientIp(), request.userAgent(), + AuditDetail.builder() + .put("previousGlobalHigh", previous.globalHigh()) + .put("previousGlobalLow", previous.globalLow()) + .put("previousNamespaceHigh", previous.namespaceHigh()) + .put("previousNamespaceLow", previous.namespaceLow()) + .put("previousCandidatePendingLimit", previous.candidatePendingLimit()) + .put("previousCandidatePerLeaseLimit", previous.candidatePerLeaseLimit()) + .put("globalHigh", updated.globalHigh()) + .put("globalLow", updated.globalLow()) + .put("namespaceHigh", updated.namespaceHigh()) + .put("namespaceLow", updated.namespaceLow()) + .put("candidatePendingLimit", updated.candidatePendingLimit()) + .put("candidatePerLeaseLimit", updated.candidatePerLeaseLimit()) + .build()); + return updated; + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncArchiveValidator.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncArchiveValidator.java new file mode 100644 index 000000000..d398f299f --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncArchiveValidator.java @@ -0,0 +1,470 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.domain.skill.validation.PackageEntry; +import com.iflytek.skillhub.domain.skill.validation.SkillPackagePolicy; +import com.iflytek.skillhub.domain.skill.validation.SkillPackageValidator; +import com.iflytek.skillhub.domain.skill.validation.ValidationResult; +import com.iflytek.skillhub.githubsync.GithubSkillPathPolicy; +import com.iflytek.skillhub.service.archive.ZipArchiveSafety; +import com.iflytek.skillhub.storage.ObjectStorageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +/** Revalidates one untrusted staged ZIP within the active publish limits. */ +@Service +public class GithubSyncArchiveValidator { + private static final Logger log = LoggerFactory.getLogger(GithubSyncArchiveValidator.class); + private static final int BUFFER_SIZE = 8192; + private static final String EVIDENCE_MANIFEST = "__skillhub_sync_evidence__.json"; + private static final String EVIDENCE_PREFIX = "__skillhub_sync_evidence__/"; + private static final int MAX_MANIFEST_BYTES = 16 * 1024; + private static final int MAX_EVIDENCE_FILES = 16; + private static final int MAX_EVIDENCE_FILE_BYTES = 64 * 1024; + private static final int MAX_EVIDENCE_TOTAL_BYTES = 256 * 1024; + private static final Pattern EVIDENCE_NAME = Pattern.compile( + "(?i)^(license|licence|copying|notice|copyright)(?:[-_.][a-z0-9][a-z0-9._-]*)?$"); + private static final Pattern LICENSE_NAME = Pattern.compile( + "(?i)^(license|licence|copying)(?:[-_.][a-z0-9][a-z0-9._-]*)?$"); + + private final ObjectStorageService storage; + private final SkillPublishProperties policy; + private final SkillPackageValidator packageValidator; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public GithubSyncArchiveValidator(ObjectStorageService storage, SkillPublishProperties policy, + SkillPackageValidator packageValidator) { + this.storage = storage; + this.policy = policy; + this.packageValidator = packageValidator; + } + + public ArchiveValidation validate(String objectKey, long expectedSize, String expectedSha256, + String skillPath) + throws IOException { + return withValidatedEntries(objectKey, expectedSize, expectedSha256, skillPath, + null, (result, entries) -> result); + } + + /** Keeps the bounded extracted files available only while the caller processes the package. */ + public T withValidatedEntries(String objectKey, long expectedSize, String expectedSha256, + String skillPath, String expectedContentSha256, + BiFunction, T> consumer) + throws IOException { + long maxPackageSize = policy.getMaxPackageSize(); + if (expectedSize <= 0 || expectedSize > maxPackageSize || maxPackageSize <= 0 + || policy.getMaxSingleFileSize() <= 0 || policy.getMaxFileCount() <= 0) { + throw invalid("PACKAGE_LIMIT"); + } + if (skillPath == null || skillPath.isBlank() || ignoredOrInvalid(skillPath)) { + throw invalid("SKILL_PATH_IGNORED"); + } + Path tempDir = Files.createTempDirectory("skillhub-github-ingestion-"); + List files = new ArrayList<>(); + Path archivePath = tempDir.resolve("package.zip"); + try { + long requiredFree = Math.addExact(expectedSize, maxPackageSize); + if (Files.getFileStore(tempDir).getUsableSpace() < requiredFree) { + throw new InsufficientLocalCapacityException(); + } + copyAndVerifyArchive(objectKey, archivePath, expectedSize, expectedSha256, maxPackageSize); + try { + long maxEntries = Math.addExact(Math.multiplyExact((long) policy.getMaxFileCount(), 2L), 100L); + ZipArchiveSafety.rejectSymbolicLinks(archivePath, (int) Math.min(maxEntries, Integer.MAX_VALUE), + Math.min(maxEntries * 4096L, maxPackageSize)); + } catch (IllegalArgumentException exception) { + throw invalid("INVALID_ARCHIVE"); + } + try { + return inspectZip(archivePath, tempDir, files, maxPackageSize, + expectedContentSha256, consumer); + } catch (ZipException exception) { + throw invalid("INVALID_ARCHIVE"); + } + } finally { + for (Path file : files) { + clean(file); + } + clean(archivePath); + clean(tempDir); + } + } + + private boolean ignoredOrInvalid(String skillPath) { + try { + return GithubSkillPathPolicy.ignored(skillPath, List.of()); + } catch (IllegalArgumentException exception) { + return true; + } + } + + private void copyAndVerifyArchive(String objectKey, Path archivePath, long expectedSize, + String expectedSha256, long maximum) throws IOException { + MessageDigest digest = newDigest(); + long copied = 0; + byte[] buffer = new byte[BUFFER_SIZE]; + try (InputStream input = storage.getObject(objectKey); + OutputStream output = Files.newOutputStream(archivePath)) { + int count; + while ((count = input.read(buffer)) != -1) { + copied += count; + if (copied > maximum || copied > expectedSize) { + throw invalid("ARCHIVE_SIZE_MISMATCH"); + } + digest.update(buffer, 0, count); + output.write(buffer, 0, count); + } + } + if (copied != expectedSize || !HexFormat.of().formatHex(digest.digest()).equals(expectedSha256)) { + throw invalid("ARCHIVE_DIGEST_MISMATCH"); + } + } + + private T inspectZip(Path archivePath, Path tempDir, List files, + long maxExpandedSize, String expectedContentSha256, + BiFunction, T> consumer) + throws IOException { + List packageEntries = new ArrayList<>(); + List digests = new ArrayList<>(); + Set paths = new HashSet<>(); + Map evidenceContents = new LinkedHashMap<>(); + byte[] evidenceManifest = null; + long expanded = 0; + long evidenceExpanded = 0; + int allEntries = 0; + try (ZipFile zip = new ZipFile(archivePath.toFile())) { + var entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + allEntries++; + if (allEntries > Math.multiplyExact(policy.getMaxFileCount(), 2) + 100) { + throw invalid("PACKAGE_FILE_LIMIT"); + } + String rawPath = entry.getName(); + if (rawPath == null || rawPath.length() > 1024 || rawPath.indexOf('\\') >= 0) { + throw invalid("PACKAGE_PATH_INVALID"); + } + String path; + try { + path = SkillPackagePolicy.normalizeEntryPath( + entry.isDirectory() ? rawPath.replaceFirst("/+$", "") : rawPath); + } catch (IllegalArgumentException exception) { + throw invalid("PACKAGE_PATH_INVALID"); + } + if (entry.isDirectory()) { + continue; + } + if (!paths.add(path)) { + throw invalid("PACKAGE_STRUCTURE_INVALID"); + } + if (path.equals(EVIDENCE_MANIFEST)) { + evidenceManifest = boundedEntry(zip, entry, MAX_MANIFEST_BYTES); + continue; + } + if (path.startsWith(EVIDENCE_PREFIX)) { + String evidenceName = path.substring(EVIDENCE_PREFIX.length()); + if (!EVIDENCE_NAME.matcher(evidenceName).matches() + || evidenceContents.size() >= MAX_EVIDENCE_FILES) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + byte[] content = boundedEntry(zip, entry, MAX_EVIDENCE_FILE_BYTES); + evidenceExpanded += content.length; + if (evidenceExpanded > MAX_EVIDENCE_TOTAL_BYTES) { + throw invalid("LICENSE_EVIDENCE_LIMIT"); + } + evidenceContents.put(evidenceName, content); + continue; + } + if (path.endsWith("/SKILL.md") || + packageEntries.size() >= policy.getMaxFileCount()) { + throw invalid("PACKAGE_STRUCTURE_INVALID"); + } + Path localPath = tempDir.resolve("entry-" + packageEntries.size()); + files.add(localPath); + MessageDigest fileDigest = newDigest(); + long fileSize = 0; + byte[] buffer = new byte[BUFFER_SIZE]; + try (InputStream input = zip.getInputStream(entry); + OutputStream output = Files.newOutputStream(localPath)) { + int count; + while ((count = input.read(buffer)) != -1) { + fileSize += count; + expanded += count; + if (fileSize > policy.getMaxSingleFileSize() || expanded > maxExpandedSize) { + throw invalid("PACKAGE_SIZE_LIMIT"); + } + fileDigest.update(buffer, 0, count); + output.write(buffer, 0, count); + } + } + long finalFileSize = fileSize; + packageEntries.add(PackageEntry.streaming(path, fileSize, "application/octet-stream", + () -> Files.newInputStream(localPath))); + digests.add(new EntryDigest(path, finalFileSize, + HexFormat.of().formatHex(fileDigest.digest()))); + } + } + ValidationResult validation = packageValidator.validate(packageEntries); + if (!validation.passed()) { + throw invalid("PACKAGE_POLICY_INVALID"); + } + List rootEvidence = parseRootEvidence(evidenceManifest, evidenceContents); + String evidenceJson = evidenceSnapshot(rootEvidence, packageEntries, digests); + boolean evidenceWarning = rootEvidence.stream().anyMatch(evidence -> !evidence.bundled()) + || !hasLicenseEvidence(rootEvidence, packageEntries); + ArchiveValidation result = new ArchiveValidation( + contentFingerprint(digests, rootEvidence), + validation.warnings().size() + (evidenceWarning ? 1 : 0), evidenceJson); + if (expectedContentSha256 != null + && !expectedContentSha256.equals(result.contentSha256())) { + throw invalid("PACKAGE_CONTENT_CHANGED"); + } + return consumer.apply(result, List.copyOf(packageEntries)); + } + + private byte[] boundedEntry(ZipFile zip, ZipEntry entry, int limit) throws IOException { + if (entry.getSize() > limit) { + throw invalid("LICENSE_EVIDENCE_LIMIT"); + } + try (InputStream input = zip.getInputStream(entry)) { + byte[] content = input.readNBytes(limit + 1); + if (content.length > limit) { + throw invalid("LICENSE_EVIDENCE_LIMIT"); + } + return content; + } + } + + private List parseRootEvidence(byte[] manifest, + Map contents) { + if (manifest == null) { + if (!contents.isEmpty()) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + return List.of(); + } + JsonNode root; + try { + root = objectMapper.readTree(manifest); + } catch (IOException exception) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + JsonNode files = root == null ? null : root.path("rootFiles"); + if (root == null || !root.path("formatVersion").canConvertToInt() + || root.path("formatVersion").asInt() != 1 || files == null + || !files.isArray() || files.size() > MAX_EVIDENCE_FILES) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + List result = new ArrayList<>(); + Set seen = new HashSet<>(); + long bundledBytes = 0; + for (JsonNode file : files) { + JsonNode pathNode = file.path("path"); + JsonNode shaNode = file.path("gitSha"); + JsonNode sizeNode = file.path("size"); + JsonNode bundledNode = file.path("bundled"); + if (!pathNode.isTextual() || !shaNode.isTextual() || !sizeNode.isIntegralNumber() + || !bundledNode.isBoolean() || !sizeNode.canConvertToLong()) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + String path = pathNode.asText(); + String gitSha = shaNode.asText(); + long size = sizeNode.asLong(); + boolean bundled = bundledNode.asBoolean(); + if (!EVIDENCE_NAME.matcher(path).matches() || !seen.add(path) + || !gitSha.matches("[0-9a-f]{40}") || size < 0) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + boolean shouldBundle = size <= MAX_EVIDENCE_FILE_BYTES + && size <= MAX_EVIDENCE_TOTAL_BYTES - bundledBytes; + if (bundled != shouldBundle) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + byte[] content = contents.get(path); + if (bundled) { + if (content == null || content.length != size + || !gitSha.equals(gitBlobSha(content))) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + bundledBytes += size; + } else if (content != null) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + result.add(new RootEvidence(path, gitSha, size, bundled, content)); + } + Set bundledPaths = new HashSet<>(); + result.stream().filter(RootEvidence::bundled).map(RootEvidence::path) + .forEach(bundledPaths::add); + if (!bundledPaths.equals(contents.keySet())) { + throw invalid("LICENSE_EVIDENCE_INVALID"); + } + return List.copyOf(result); + } + + private String evidenceSnapshot(List rootEvidence, + List entries, + List digests) throws IOException { + List> documents = new ArrayList<>(); + for (RootEvidence evidence : rootEvidence) { + Map document = new LinkedHashMap<>(); + document.put("scope", "REPOSITORY_ROOT"); + document.put("path", evidence.path()); + document.put("gitSha", evidence.gitSha()); + document.put("size", evidence.size()); + document.put("bundled", evidence.bundled()); + if (evidence.content() != null) { + document.put("sha256", sha256(evidence.content())); + String preview = utf8Preview(evidence.content()); + if (preview != null) { + document.put("text", preview); + } + } + documents.add(document); + } + for (PackageEntry entry : entries) { + if (!EVIDENCE_NAME.matcher(entry.path()).matches()) { + continue; + } + Map document = new LinkedHashMap<>(); + document.put("scope", "SKILL_ROOT"); + document.put("path", entry.path()); + document.put("size", entry.size()); + digests.stream().filter(digest -> digest.path().equals(entry.path())) + .findFirst().ifPresent(digest -> document.put("sha256", digest.sha256())); + if (entry.size() <= MAX_EVIDENCE_FILE_BYTES) { + String preview = utf8Preview(entry.content()); + if (preview != null) { + document.put("text", preview); + } + } + documents.add(document); + } + String status = !hasLicenseEvidence(rootEvidence, entries) ? "MISSING" + : rootEvidence.stream().anyMatch(evidence -> !evidence.bundled()) + ? "PRESENT_UNPREVIEWABLE" : "PRESENT_UNCLASSIFIED"; + return objectMapper.writeValueAsString(Map.of( + "formatVersion", 1, "status", status, "documents", documents)); + } + + private boolean hasLicenseEvidence(List rootEvidence, + List entries) { + return rootEvidence.stream().anyMatch(evidence -> LICENSE_NAME.matcher(evidence.path()).matches()) + || entries.stream().anyMatch(entry -> LICENSE_NAME.matcher(entry.path()).matches()); + } + + private String utf8Preview(byte[] content) { + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(content)).toString(); + } catch (CharacterCodingException exception) { + return null; + } + } + + private String gitBlobSha(byte[] content) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + digest.update(("blob " + content.length + "\0").getBytes(StandardCharsets.US_ASCII)); + digest.update(content); + return HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("Git object hash is unavailable", exception); + } + } + + private String sha256(byte[] content) { + return HexFormat.of().formatHex(newDigest().digest(content)); + } + + private String contentFingerprint(List entries, List rootEvidence) { + MessageDigest digest = newDigest(); + entries.stream().sorted(Comparator.comparing(EntryDigest::path)).forEach(entry -> { + byte[] path = entry.path().getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(4).putInt(path.length).array()); + digest.update(path); + digest.update(ByteBuffer.allocate(8).putLong(entry.size()).array()); + digest.update(HexFormat.of().parseHex(entry.sha256())); + }); + if (!rootEvidence.isEmpty()) { + digest.update((byte) 0xff); + rootEvidence.stream().sorted(Comparator.comparing(RootEvidence::path)).forEach(evidence -> { + byte[] path = evidence.path().getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(4).putInt(path.length).array()); + digest.update(path); + digest.update(ByteBuffer.allocate(8).putLong(evidence.size()).array()); + digest.update(HexFormat.of().parseHex(evidence.gitSha())); + }); + } + return HexFormat.of().formatHex(digest.digest()); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private void clean(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + log.warn("Could not remove GitHub ingestion temporary file: {}", + exception.getClass().getSimpleName()); + } + } + + private static ValidationFailure invalid(String code) { + return new ValidationFailure(code); + } + + public record ArchiveValidation(String contentSha256, int warningCount, + String evidenceJson) {} + private record EntryDigest(String path, long size, String sha256) {} + private record RootEvidence(String path, String gitSha, long size, + boolean bundled, byte[] content) {} + + public static class ValidationFailure extends RuntimeException { + private final String code; + + private ValidationFailure(String code) { + super(code); + this.code = code; + } + + public String code() { return code; } + } + + public static class InsufficientLocalCapacityException extends IOException {} +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncManualBindingAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncManualBindingAppService.java new file mode 100644 index 000000000..81adf0b96 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncManualBindingAppService.java @@ -0,0 +1,75 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.dto.GithubSyncManualBindingResponse; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncManualBindingRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.UUID; + +@Service +public class GithubSyncManualBindingAppService { + private final JdbcGithubSyncManualBindingRepository bindings; + private final GithubTeamSourceAppService teamSources; + private final GithubSyncManagementQueryRepository management; + private final AuditLogService audits; + private final RequestIdAccessor requestIds; + private final GithubSourceIgnoreRepository ignores; + + public GithubSyncManualBindingAppService(JdbcGithubSyncManualBindingRepository bindings, + GithubTeamSourceAppService teamSources, GithubSyncManagementQueryRepository management, + AuditLogService audits, RequestIdAccessor requestIds, + GithubSourceIgnoreRepository ignores) { + this.bindings = bindings; + this.teamSources = teamSources; + this.management = management; + this.audits = audits; + this.requestIds = requestIds; + this.ignores = ignores; + } + + @Transactional + public GithubSyncManualBindingResponse bindPlatform(UUID ingestionId, long skillId, + String reason, String actorId, AuditRequestContext context) { + return bind(management.platformNamespaceId(), ingestionId, skillId, reason, actorId, context); + } + + @Transactional + public GithubSyncManualBindingResponse bindTeam(String namespaceSlug, UUID ingestionId, + long skillId, String reason, String actorId, AuditRequestContext context) { + long namespaceId = teamSources.requireTeamAdmin(namespaceSlug, actorId, true).getId(); + return bind(namespaceId, ingestionId, skillId, reason, actorId, context); + } + + private GithubSyncManualBindingResponse bind(long namespaceId, UUID ingestionId, long skillId, + String reason, String actorId, AuditRequestContext context) { + if (reason == null || reason.trim().length() < 10 || reason.length() > 1000) { + throw new DomainBadRequestException("error.githubSync.binding.reason"); + } + var result = bindings.bind(namespaceId, ingestionId, skillId); + if (result.previousPath() != null + && (ignores.ignoredForSource(result.sourceId(), result.previousPath()) + || ignores.ignoredForSource(result.sourceId(), result.skillPath()))) { + throw new DomainBadRequestException("error.githubSync.binding.notResolvable"); + } + audits.record(actorId, result.previousPath() == null + ? "BIND_GITHUB_SOURCE_TO_SKILL" : "CONFIRM_GITHUB_SKILL_PATH_MOVE", + "SKILL", skillId, + requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("namespaceId", result.namespaceId()) + .put("repositoryId", result.repositoryId()) + .put("sourceId", result.sourceId()) + .put("ingestionId", result.ingestionId()) + .put("skillPath", result.skillPath()) + .put("previousPath", result.previousPath()) + .put("reason", reason.trim()).build()); + return new GithubSyncManualBindingResponse(result.ingestionId(), result.sourceId(), + result.skillId(), result.skillSlug(), result.skillPath(), "QUEUED_FOR_RETRY"); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncPublicationAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncPublicationAppService.java new file mode 100644 index 000000000..e088af883 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncPublicationAppService.java @@ -0,0 +1,130 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.skill.SkillVisibility; +import com.iflytek.skillhub.domain.skill.service.SkillPublishService; +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.time.Instant; + +@Service +public class GithubSyncPublicationAppService { + private final GithubSyncPublicationRepository publications; + private final GithubSyncArchiveValidator archives; + private final SkillPublishService skills; + private final GithubSourceIgnoreRepository ignores; + private final AuditLogService audits; + + public GithubSyncPublicationAppService(GithubSyncPublicationRepository publications, + GithubSyncArchiveValidator archives, + SkillPublishService skills, + GithubSourceIgnoreRepository ignores, + AuditLogService audits) { + this.publications = publications; + this.archives = archives; + this.skills = skills; + this.ignores = ignores; + this.audits = audits; + } + + /** One database transaction covers the Skill version, review and path binding. */ + @Transactional(rollbackFor = Exception.class) + public void process(GithubSyncPublicationRepository.ClaimedPublication claim) throws IOException { + if (!publications.lockAndVerify(claim, Instant.now())) { + return; + } + var currentTarget = publications.currentTarget(claim); + if (currentTarget.newerAcquisition()) { + if (!publications.superseded(claim, Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + var targetClaim = claim.withExpectedSkillId(currentTarget.skillId()); + if (claim.sourceId() == null) { + if (!publications.blocked(claim, "SOURCE_NOT_BOUND", Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if ("PAUSED".equals(claim.sourceStatus())) { + if (!publications.retry(claim, "SOURCE_PAUSED", Instant.now().plusSeconds(3600), + Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if (ignores.ignoredForSource(claim.sourceId(), claim.skillPath())) { + if (!publications.ignored(claim, Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + var pathMove = publications.inspectPathMove(targetClaim); + if (pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.RETRY) { + if (!publications.retry(claim, "PATH_MOVE_SCAN_INCOMPLETE", + Instant.now().plusSeconds(900), Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if (pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.INCOMPLETE) { + if (!publications.blocked(claim, "PATH_MOVE_SCAN_INCOMPLETE", Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if (pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.STALE) { + if (!publications.blocked(claim, "PATH_MOVE_STALE_SNAPSHOT", Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if (pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.AMBIGUOUS + || pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.CANDIDATE + && ignores.ignoredForSource(claim.sourceId(), pathMove.previousPath())) { + if (!publications.blocked(claim, "PATH_MOVE_AMBIGUOUS", Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + if (pathMove.outcome() == GithubSyncPublicationRepository.PathMoveOutcome.CANDIDATE) { + if (!publications.movePath(targetClaim, pathMove, Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + audits.record("github-sync-publisher", "AUTO_MOVE_GITHUB_SKILL_PATH", "SKILL", + pathMove.skillId(), null, null, null, + AuditDetail.builder().put("namespaceId", claim.namespaceId()) + .put("repositoryId", claim.repositoryId()) + .put("sourceId", claim.sourceId()) + .put("ingestionId", claim.id()) + .put("previousPath", pathMove.previousPath()) + .put("skillPath", claim.skillPath()).build()); + return; + } + if (claim.contentSha256().equals(currentTarget.contentSha256()) + && !currentTarget.retryRequested()) { + if (!publications.noChange(claim, Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + return; + } + + SkillPublishService.PublishResult result = archives.withValidatedEntries( + claim.objectKey(), claim.sizeBytes(), claim.archiveSha256(), claim.skillPath(), + claim.contentSha256(), (validation, entries) -> skills.publishGithubSyncFromEntries( + claim.namespaceSlug(), entries, SkillVisibility.valueOf(claim.visibility()), + claim.id(), validation.contentSha256(), targetClaim.expectedSkillId())); + if (!publications.readyForReview(claim, result.skillId(), result.version().getId(), + Instant.now())) { + throw new IllegalStateException("GitHub publication lease changed"); + } + publications.recordOutcome(claim, result.skillId(), result.version().getId(), + result.newSkill(), Instant.now()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppService.java new file mode 100644 index 000000000..17f330555 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppService.java @@ -0,0 +1,42 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.dto.GithubSyncReviewQueueItemResponse; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.dto.ReviewTaskResponse; +import com.iflytek.skillhub.repository.GithubSyncReviewSourceQueryRepository; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +@Service +public class GithubSyncReviewQueueAppService { + private final ReviewPortalAppService reviews; + private final GithubSyncReviewSourceQueryRepository sources; + + public GithubSyncReviewQueueAppService(ReviewPortalAppService reviews, + GithubSyncReviewSourceQueryRepository sources) { + this.reviews = reviews; + this.sources = sources; + } + + public PageResponse list(String status, Long namespaceId, + int page, int size, String sortDirection, String userId, + Map namespaceRoles) { + PageResponse visible = reviews.listSyncReviews( + status, namespaceId, page, size, sortDirection, userId, namespaceRoles); + Map metadata = sources.findQueueSources( + visible.items().stream().map(ReviewTaskResponse::id).toList()); + List items = visible.items().stream().map(review -> { + GithubSyncReviewSourceQueryRepository.QueueSource source = metadata.get(review.id()); + if (source == null) { + throw new IllegalStateException("GitHub sync review has no ingestion source"); + } + return new GithubSyncReviewQueueItemResponse(review, source.ingestionId(), source.runId(), + source.repositoryId(), source.ownerLogin(), source.repositoryName(), + source.skillPath(), source.commitSha()); + }).toList(); + return new PageResponse<>(items, visible.total(), visible.page(), visible.size()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewSourceAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewSourceAppService.java new file mode 100644 index 000000000..80a4aa71a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncReviewSourceAppService.java @@ -0,0 +1,79 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.auth.rbac.RbacService; +import com.iflytek.skillhub.domain.namespace.Namespace; +import com.iflytek.skillhub.domain.namespace.NamespaceRepository; +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.domain.review.ReviewOriginType; +import com.iflytek.skillhub.domain.review.ReviewService; +import com.iflytek.skillhub.domain.review.ReviewTask; +import com.iflytek.skillhub.domain.review.ReviewTaskRepository; +import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubSyncReviewSourceResponse; +import com.iflytek.skillhub.domain.skill.service.SkillQueryService; +import com.iflytek.skillhub.repository.GithubSyncReviewSourceQueryRepository; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.io.InputStream; + +@Service +public class GithubSyncReviewSourceAppService { + private final ReviewTaskRepository reviews; + private final NamespaceRepository namespaces; + private final ReviewService reviewService; + private final RbacService rbac; + private final GithubSyncReviewSourceQueryRepository sources; + private final SkillQueryService skillQueries; + + public GithubSyncReviewSourceAppService(ReviewTaskRepository reviews, + NamespaceRepository namespaces, + ReviewService reviewService, + RbacService rbac, + GithubSyncReviewSourceQueryRepository sources, + SkillQueryService skillQueries) { + this.reviews = reviews; + this.namespaces = namespaces; + this.reviewService = reviewService; + this.rbac = rbac; + this.sources = sources; + this.skillQueries = skillQueries; + } + + public GithubSyncReviewSourceResponse get(Long reviewId, String userId, + Map namespaceRoles) { + ReviewTask review = reviews.findById(reviewId) + .orElseThrow(() -> new DomainNotFoundException("review_task.not_found", reviewId)); + if (review.getOriginType() != ReviewOriginType.GITHUB_SYNC) { + throw new DomainNotFoundException("review_task.not_found", reviewId); + } + Namespace namespace = namespaces.findById(review.getNamespaceId()) + .orElseThrow(() -> new DomainNotFoundException("namespace.not_found", review.getNamespaceId())); + if (!reviewService.canViewReview(review, userId, namespace.getType(), + namespaceRoles == null ? Map.of() : namespaceRoles, rbac.getUserRoleCodes(userId))) { + throw new DomainForbiddenException("review.no_permission"); + } + return sources.findByReviewId(reviewId) + .orElseThrow(() -> new DomainNotFoundException("review_task.not_found", reviewId)); + } + + public InputStream getBaseFile(Long reviewId, String path, String userId, + Map namespaceRoles) { + return getBaseFile(reviewId, path, userId, namespaceRoles, false); + } + + public InputStream getBaseFile(Long reviewId, String path, String userId, + Map namespaceRoles, boolean preview) { + GithubSyncReviewSourceResponse source = get(reviewId, userId, namespaceRoles); + boolean changedBaseFile = source.baseVersionId() != null && source.fileChanges().stream() + .anyMatch(file -> file.path().equals(path) && file.oldSha256() != null + && file.oldSize() != null && file.oldSize() <= 1024 * 1024); + if (!changedBaseFile) { + throw new DomainNotFoundException("review_task.not_found", reviewId); + } + return preview + ? skillQueries.getFilePreviewContentByVersionId(source.baseVersionId(), path) + : skillQueries.getFileContentByVersionId(source.baseVersionId(), path); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncSchedulePolicyAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncSchedulePolicyAppService.java new file mode 100644 index 000000000..eaf68b6f8 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncSchedulePolicyAppService.java @@ -0,0 +1,67 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.dto.GithubSyncSchedulePolicyRequest; +import com.iflytek.skillhub.dto.GithubSyncSchedulePolicyResponse; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicy; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Duration; + +@Service +public class GithubSyncSchedulePolicyAppService { + private final GithubSyncSchedulePolicyRepository policies; + private final AuditLogService audits; + private final RequestIdAccessor requestIds; + private final Duration legacyRepositoryInterval; + + public GithubSyncSchedulePolicyAppService(GithubSyncSchedulePolicyRepository policies, + AuditLogService audits, RequestIdAccessor requestIds, + @Value("${skillhub.github-sync.repository-check-interval:PT24H}") Duration legacyRepositoryInterval) { + this.policies = policies; + this.audits = audits; + this.requestIds = requestIds; + this.legacyRepositoryInterval = legacyRepositoryInterval; + } + + public GithubSyncSchedulePolicyResponse get() { + return response(policies.get()); + } + + @Transactional + public GithubSyncSchedulePolicyResponse update(GithubSyncSchedulePolicyRequest request, + String actorId, AuditRequestContext context) { + if (request.repositoryHours() < 1 || request.repositoryHours() > 720 + || request.curatedOwnerHours() < 1 || request.curatedOwnerHours() > 720 + || request.searchRuleHours() < 1 || request.searchRuleHours() > 720) { + throw new DomainBadRequestException("error.githubSync.schedulePolicy.invalid"); + } + GithubSyncSchedulePolicy previous = policies.getForUpdate(); + GithubSyncSchedulePolicy updated = policies.update(request.repositoryHours(), + request.curatedOwnerHours(), request.searchRuleHours()); + audits.record(actorId, "UPDATE_GITHUB_SYNC_SCHEDULE_POLICY", "GITHUB_SYNC_POLICY", null, + requestIds.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder() + .put("previousRepositoryHours", previous.repositoryHours()) + .put("previousCuratedOwnerHours", previous.curatedOwnerHours()) + .put("previousSearchRuleHours", previous.searchRuleHours()) + .put("repositoryHours", updated.repositoryHours()) + .put("curatedOwnerHours", updated.curatedOwnerHours()) + .put("searchRuleHours", updated.searchRuleHours()).build()); + return response(updated); + } + + private GithubSyncSchedulePolicyResponse response(GithubSyncSchedulePolicy policy) { + Duration effective = policy.repositoryHours() == null + ? legacyRepositoryInterval : Duration.ofHours(policy.repositoryHours()); + return new GithubSyncSchedulePolicyResponse(policy.repositoryHours(), + policy.curatedOwnerHours(), policy.searchRuleHours(), effective.toString(), + policy.updatedAt()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncUploadAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncUploadAppService.java new file mode 100644 index 000000000..e399afd7d --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubSyncUploadAppService.java @@ -0,0 +1,278 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.githubsync.GithubSyncUploadRepository; +import com.iflytek.skillhub.githubsync.GithubPackageReportResult; +import com.iflytek.skillhub.storage.ObjectStorageService; +import com.iflytek.skillhub.storage.PresignedUpload; +import com.iflytek.skillhub.storage.StorageAccessException; +import com.iflytek.skillhub.storage.StorageProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Duration; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Supplier; + +@Service +public class GithubSyncUploadAppService { + private static final Logger log = LoggerFactory.getLogger(GithubSyncUploadAppService.class); + private final GithubSyncUploadRepository uploads; + private final ObjectStorageService storage; + private final StorageProperties storageProperties; + private final SkillPublishProperties publishProperties; + private final Duration ticketTtl; + private final SecureRandom random = new SecureRandom(); + + public GithubSyncUploadAppService(GithubSyncUploadRepository uploads, ObjectStorageService storage, + StorageProperties storageProperties, SkillPublishProperties publishProperties, + @Value("${skillhub.github-sync.upload-ticket-ttl:PT5M}") Duration ticketTtl) { + if (ticketTtl.isNegative() || ticketTtl.isZero() || ticketTtl.compareTo(Duration.ofHours(1)) > 0) { + throw new IllegalArgumentException("GitHub sync upload ticket TTL must be positive and at most one hour"); + } + this.uploads = uploads; + this.storage = storage; + this.storageProperties = storageProperties; + this.publishProperties = publishProperties; + this.ticketTtl = ticketTtl; + } + + public Optional authorize(UUID taskId, long generation, String leaseToken, + UUID admissionId, String commitSha, String skillPath, long sizeBytes, String sha256) { + if (taskId == null || admissionId == null || generation < 1 || !isHex(leaseToken, 64) + || !isHex(commitSha, 40) || skillPath == null || skillPath.isBlank() + || skillPath.length() > 1024 || !isHex(sha256, 64)) { + throw new IllegalArgumentException("Invalid GitHub sync upload authorization request"); + } + long maxBytes = publishProperties.getMaxPackageSize(); + if (maxBytes <= 0) { + throw new IllegalStateException("Skill package size policy must be positive"); + } + if (sizeBytes <= 0 || sizeBytes > maxBytes) { + throw new UploadTooLargeException(); + } + String provider = storageProperties.getProvider(); + if (!"local".equals(provider) && !"s3".equals(provider)) { + throw new IllegalStateException("Unsupported GitHub sync staging storage provider"); + } + UUID uploadId = UUID.randomUUID(); + String objectKey = "github-sync/staging/" + uploadId + ".zip"; + String mode = "s3".equals(provider) ? "S3_PRESIGNED_PUT" : "LOCAL_STREAM"; + String ticket = null; + if ("local".equals(provider)) { + byte[] ticketBytes = new byte[32]; + random.nextBytes(ticketBytes); + ticket = HexFormat.of().formatHex(ticketBytes); + } + Instant now = Instant.now(); + Instant expiresAt = now.plus(ticketTtl); + if (!uploads.issue(taskId, generation, sha256(leaseToken), admissionId, commitSha, skillPath, + uploadId, objectKey, mode, ticket == null ? null : sha256(ticket), + sizeBytes, sha256, expiresAt, now)) { + return Optional.empty(); + } + if ("local".equals(provider)) { + return Optional.of(new UploadAuthorization(uploadId, "PUT", mode, + "/internal/github-sync/v1/uploads/" + uploadId + "/content", + ticket, expiresAt, maxBytes, Map.of())); + } + try { + PresignedUpload target = safeStorageCall(uploadId, "presignPut", + () -> storage.presignPut(objectKey, ticketTtl)) + .orElseThrow(() -> new IllegalStateException("S3 storage cannot sign uploads")); + return Optional.of(new UploadAuthorization(uploadId, "PUT", mode, + target.url(), null, expiresAt, maxBytes, target.headers())); + } catch (RuntimeException exception) { + uploads.failUpload(uploadId, Instant.now()); + throw exception; + } + } + + public LocalUploadResult uploadLocal(UUID uploadId, String ticket, InputStream body) throws IOException { + if (uploadId == null || !isHex(ticket, 64) || body == null) { + return LocalUploadResult.TICKET_INVALID; + } + Optional current = + uploads.beginLocalUpload(uploadId, sha256(ticket), Instant.now()); + if (current.isEmpty()) { + return LocalUploadResult.TICKET_INVALID; + } + GithubSyncUploadRepository.LocalUpload upload = current.get(); + BoundedDigestInputStream bounded = new BoundedDigestInputStream(body, upload.sizeBytes()); + boolean uploaded = false; + boolean storageCompleted = false; + try { + storage.putObject(upload.objectKey(), bounded, upload.sizeBytes(), "application/zip"); + storageCompleted = true; + if (bounded.bytesRead() != upload.sizeBytes() + || !bounded.digestHex().equals(upload.sha256())) { + return LocalUploadResult.CONTENT_MISMATCH; + } + if (!uploads.finishLocalUpload(uploadId, Instant.now())) { + return LocalUploadResult.LEASE_STALE; + } + uploaded = true; + return LocalUploadResult.UPLOADED; + } catch (RuntimeException exception) { + if (bounded.exceeded()) { + return LocalUploadResult.TOO_LARGE; + } + if (!storageCompleted) { + throw new StorageAccessException("putObject", uploadId.toString(), null); + } + throw new IllegalStateException("GitHub sync local upload failed"); + } finally { + // A failed upload is never accepted as a package, including digest mismatch or lease expiry. + // Keep FAILED in the upload ledger when best-effort object deletion fails. + if (!uploaded && uploads.failUpload(uploadId, Instant.now())) { + try { + storage.deleteObject(upload.objectKey()); + } catch (RuntimeException exception) { + log.warn("Could not clean up failed GitHub sync staged upload {}: {}", + uploadId, exception.getClass().getSimpleName()); + } + } + } + } + + public GithubPackageReportResult report(UUID taskId, long generation, String leaseToken, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256) { + if (taskId == null || admissionId == null || uploadId == null || generation < 1 + || !isHex(leaseToken, 64) || !isHex(commitSha, 40) + || skillPath == null || skillPath.isBlank() || skillPath.length() > 1024 + || !isHex(sha256, 64) || sizeBytes <= 0) { + throw new IllegalArgumentException("Invalid GitHub sync package report"); + } + String tokenHash = sha256(leaseToken); + GithubPackageReportResult checked = uploads.lookupReport(taskId, generation, tokenHash, + admissionId, commitSha, skillPath, uploadId, sizeBytes, sha256, Instant.now()); + if (checked.status() != GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT) { + return checked; + } + long maxBytes = publishProperties.getMaxPackageSize(); + if (maxBytes <= 0) { + throw new IllegalStateException("Skill package size policy must be positive"); + } + if (sizeBytes > maxBytes) { + discardOversizedUpload(uploadId, checked.objectKey()); + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.TOO_LARGE); + } + if (!safeStorageCall(uploadId, "exists", () -> storage.exists(checked.objectKey()))) { + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.OBJECT_MISSING); + } + long actualSize = safeStorageCall(uploadId, "getMetadata", + () -> storage.getMetadata(checked.objectKey())).size(); + if (actualSize > maxBytes) { + discardOversizedUpload(uploadId, checked.objectKey()); + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.TOO_LARGE); + } + if (actualSize != sizeBytes) { + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.OBJECT_MISMATCH); + } + return uploads.acceptReport(taskId, generation, tokenHash, admissionId, + commitSha, skillPath, uploadId, sizeBytes, sha256, Instant.now()); + } + + private void discardOversizedUpload(UUID uploadId, String objectKey) { + if (!uploads.failUpload(uploadId, Instant.now())) { + return; + } + try { + storage.deleteObject(objectKey); + } catch (RuntimeException exception) { + // FAILED uploads remain eligible for the existing compensation cleanup task. + log.warn("Could not immediately clean up oversized GitHub sync staged upload {}", uploadId); + } + } + + private T safeStorageCall(UUID uploadId, String operation, Supplier action) { + try { + return action.get(); + } catch (RuntimeException exception) { + throw new StorageAccessException(operation, uploadId.toString(), null); + } + } + + private static boolean isHex(String value, int length) { + return value != null && value.length() == length && value.chars().allMatch(character -> + (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')); + } + + private static String sha256(String value) { + return HexFormat.of().formatHex(newDigest().digest(value.getBytes(StandardCharsets.UTF_8))); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public record UploadAuthorization(UUID uploadId, String method, String mode, String uploadUrl, + String uploadTicket, Instant expiresAt, long maxBytes, + Map headers) {} + + public enum LocalUploadResult { UPLOADED, TICKET_INVALID, TOO_LARGE, CONTENT_MISMATCH, LEASE_STALE } + + public static class UploadTooLargeException extends RuntimeException {} + + private static final class BoundedDigestInputStream extends FilterInputStream { + private final long maxBytes; + private final MessageDigest digest = newDigest(); + private long bytesRead; + private boolean exceeded; + + private BoundedDigestInputStream(InputStream input, long maxBytes) { + super(input); + this.maxBytes = maxBytes; + } + + @Override + public int read() throws IOException { + int value = in.read(); + if (value < 0) return -1; + if (bytesRead >= maxBytes) { + exceeded = true; + throw new IOException("GitHub sync upload exceeds authorized size"); + } + digest.update((byte) value); + bytesRead++; + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int count = in.read(buffer, offset, (int) Math.min(length, maxBytes - bytesRead + 1)); + if (count < 0) return -1; + if (bytesRead + count > maxBytes) { + exceeded = true; + throw new IOException("GitHub sync upload exceeds authorized size"); + } + digest.update(buffer, offset, count); + bytesRead += count; + return count; + } + + private long bytesRead() { return bytesRead; } + private boolean exceeded() { return exceeded; } + private String digestHex() { return HexFormat.of().formatHex(digest.digest()); } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubTeamSourceAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubTeamSourceAppService.java new file mode 100644 index 000000000..e101654d4 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GithubTeamSourceAppService.java @@ -0,0 +1,179 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.domain.audit.AuditDetail; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.namespace.Namespace; +import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository; +import com.iflytek.skillhub.domain.namespace.NamespaceRepository; +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.domain.namespace.NamespaceStatus; +import com.iflytek.skillhub.domain.namespace.NamespaceType; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubDiscoveryEntryResponse; +import com.iflytek.skillhub.dto.GithubTeamRepositoryRequest; +import com.iflytek.skillhub.dto.GithubSyncRunSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSourceSummaryResponse; +import com.iflytek.skillhub.dto.GithubSyncSkillPathResponse; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntry; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +@Service +public class GithubTeamSourceAppService { + private static final Pattern OWNER = Pattern.compile("^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$"); + private static final Pattern REPOSITORY = Pattern.compile("^[A-Za-z0-9_.-]{1,100}$"); + private static final Set VISIBILITIES = Set.of("PUBLIC", "NAMESPACE_ONLY"); + + private final NamespaceRepository namespaces; + private final NamespaceMemberRepository members; + private final GithubDiscoveryEntryRepository entries; + private final GithubSyncManagementQueryRepository management; + private final AuditLogService auditLogService; + private final RequestIdAccessor requestIdAccessor; + + public GithubTeamSourceAppService(NamespaceRepository namespaces, + NamespaceMemberRepository members, + GithubDiscoveryEntryRepository entries, + GithubSyncManagementQueryRepository management, + AuditLogService auditLogService, + RequestIdAccessor requestIdAccessor) { + this.namespaces = namespaces; + this.members = members; + this.entries = entries; + this.management = management; + this.auditLogService = auditLogService; + this.requestIdAccessor = requestIdAccessor; + } + + public List list(String namespaceSlug, String userId) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, false); + return entries.listTeamRepositories(namespace.getId()).stream().map(this::response).toList(); + } + + public PageResponse resolvedSources( + String namespaceSlug, String userId, int page, int size) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, false); + return management.sources(namespace.getId(), page, size); + } + + public PageResponse runs( + String namespaceSlug, String userId, int page, int size) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, false); + return management.runs(namespace.getId(), page, size); + } + + public PageResponse skillPaths( + String namespaceSlug, String userId, long sourceId, int page, int size) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, false); + return management.skillPaths(namespace.getId(), sourceId, page, size); + } + + @Transactional + public GithubDiscoveryEntryResponse create(String namespaceSlug, GithubTeamRepositoryRequest request, + String userId, AuditRequestContext context) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, true); + if (request.ownerLogin() == null || request.repositoryName() == null) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.repository"); + } + String owner = request.ownerLogin().trim(); + String repository = request.repositoryName().trim(); + if (!OWNER.matcher(owner).matches() || !REPOSITORY.matcher(repository).matches() + || repository.equals(".") || repository.equals("..")) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.repository"); + } + String visibility = visibility(request.visibility() == null ? "NAMESPACE_ONLY" : request.visibility()); + GithubDiscoveryEntry entry; + try { + entry = entries.createTeamRepository(namespace.getId(), owner, repository, visibility); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.discoveryEntry.duplicate"); + } + audit("CREATE_GITHUB_TEAM_SOURCE", entry, userId, context); + return response(entry); + } + + @Transactional + public GithubDiscoveryEntryResponse setEnabled(String namespaceSlug, UUID entryId, boolean enabled, + String userId, AuditRequestContext context) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, true); + GithubDiscoveryEntry previous = requireEntry(namespace.getId(), entryId); + if (previous.enabled() == enabled) { + return response(previous); + } + GithubDiscoveryEntry updated = entries.setTeamRepositoryEnabled(namespace.getId(), entryId, enabled); + audit("SET_GITHUB_TEAM_SOURCE_ENABLED", updated, userId, context); + return response(updated); + } + + @Transactional + public GithubDiscoveryEntryResponse setVisibility(String namespaceSlug, UUID entryId, + String requestedVisibility, String userId, + AuditRequestContext context) { + Namespace namespace = requireTeamAdmin(namespaceSlug, userId, true); + GithubDiscoveryEntry previous = requireEntry(namespace.getId(), entryId); + String visibility = visibility(requestedVisibility); + if (visibility.equals(previous.sourceVisibility())) { + return response(previous); + } + GithubDiscoveryEntry updated = entries.setTeamRepositoryVisibility(namespace.getId(), entryId, visibility); + audit("SET_GITHUB_TEAM_SOURCE_VISIBILITY", updated, userId, context); + return response(updated); + } + + Namespace requireTeamAdmin(String slug, String userId, boolean write) { + Namespace namespace = namespaces.findBySlug(slug) + .orElseThrow(() -> new DomainNotFoundException("error.namespace.slug.notFound", slug)); + if (namespace.getType() != NamespaceType.TEAM) { + throw new DomainNotFoundException("error.namespace.slug.notFound", slug); + } + NamespaceRole role = members.findByNamespaceIdAndUserId(namespace.getId(), userId) + .map(member -> member.getRole()).orElse(null); + if (role != NamespaceRole.OWNER && role != NamespaceRole.ADMIN) { + throw new DomainForbiddenException("error.githubSync.teamSource.forbidden"); + } + if (write && namespace.getStatus() != NamespaceStatus.ACTIVE) { + throw new DomainBadRequestException("error.namespace.readonly", slug); + } + return namespace; + } + + private GithubDiscoveryEntry requireEntry(long namespaceId, UUID entryId) { + return entries.findTeamRepository(namespaceId, entryId) + .orElseThrow(() -> new DomainNotFoundException("error.githubSync.discoveryEntry.notFound")); + } + + private String visibility(String value) { + if (value == null || !VISIBILITIES.contains(value)) { + throw new DomainBadRequestException("error.githubSync.teamSource.visibility"); + } + return value; + } + + private GithubDiscoveryEntryResponse response(GithubDiscoveryEntry entry) { + return new GithubDiscoveryEntryResponse(entry.id(), entry.entryType(), entry.title(), + entry.ownerLogin(), entry.repositoryName(), entry.repositoryId(), + entry.sourceVisibility(), List.of(), entry.ruleVersion(), entry.enabled(), + entry.targetIntervalSeconds() / 3600, entry.nextDueAt(), entry.updatedAt()); + } + + private void audit(String action, GithubDiscoveryEntry entry, + String actorId, AuditRequestContext context) { + auditLogService.record(actorId, action, "GITHUB_DISCOVERY_ENTRY", null, + requestIdAccessor.current(), context.clientIp(), context.userAgent(), + AuditDetail.builder().put("entryId", entry.id()).put("namespaceId", entry.namespaceId()) + .put("repositoryId", entry.repositoryId()).put("visibility", entry.sourceVisibility()) + .put("enabled", entry.enabled()).build()); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GovernanceWorkflowAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GovernanceWorkflowAppService.java index f4a0d6aee..709b1cb71 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GovernanceWorkflowAppService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/GovernanceWorkflowAppService.java @@ -86,6 +86,17 @@ public class GovernanceWorkflowAppService { return reviewPortalAppService.listReviews(status, namespaceId, page, size, sortDirection, userId, userNsRoles); } + public PageResponse listSyncReviews(String status, + Long namespaceId, + int page, + int size, + String sortDirection, + String userId, + Map userNsRoles) { + return reviewPortalAppService.listSyncReviews( + status, namespaceId, page, size, sortDirection, userId, userNsRoles); + } + public PageResponse listPendingReviews(Long namespaceId, int page, int size, @@ -135,6 +146,21 @@ public class GovernanceWorkflowAppService { ); } + public ReviewSkillDetailResponse getReviewSkillDetail(Long reviewTaskId, + String userId, + Map userNsRoles, + boolean includeDocumentation) { + if (includeDocumentation) { + return getReviewSkillDetail(reviewTaskId, userId, userNsRoles); + } + return reviewSkillDetailAppService.getReviewSkillDetail( + reviewTaskId, + userId, + userNsRoles != null ? userNsRoles : Map.of(), + includeDocumentation + ); + } + public SkillDownloadService.DownloadResult downloadReviewPackage(Long reviewTaskId, String userId, Map userNsRoles) { @@ -153,11 +179,20 @@ public class GovernanceWorkflowAppService { String filePath, String userId, Map userNsRoles) { + return getReviewFileContent(reviewTaskId, filePath, userId, userNsRoles, false); + } + + public InputStream getReviewFileContent(Long reviewTaskId, + String filePath, + String userId, + Map userNsRoles, + boolean preview) { return reviewSkillDetailAppService.getReviewFileContent( reviewTaskId, filePath, userId, - userNsRoles != null ? userNsRoles : Map.of() + userNsRoles != null ? userNsRoles : Map.of(), + preview ); } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/NamespacePortalQueryAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/NamespacePortalQueryAppService.java index 43c506fbe..4627910bc 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/NamespacePortalQueryAppService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/NamespacePortalQueryAppService.java @@ -146,7 +146,7 @@ public class NamespacePortalQueryAppService { @Transactional(readOnly = true) public PageResponse listMembers(String slug, Pageable pageable, String userId, Set platformRoles) { Namespace namespace = namespaceService.getNamespaceBySlug(slug); - if (namespace.getType() == NamespaceType.GLOBAL) { + if (namespace.getType() != NamespaceType.TEAM) { Set roles = platformRoles != null ? platformRoles : Set.of(); if (!roles.contains("SUPER_ADMIN") && !roles.contains("USER_ADMIN")) { throw new DomainForbiddenException("error.namespace.global.members.platformAdmin.required"); diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewPortalAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewPortalAppService.java index f2432107e..cbe59f02e 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewPortalAppService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewPortalAppService.java @@ -8,6 +8,7 @@ import com.iflytek.skillhub.domain.namespace.NamespaceRepository; import com.iflytek.skillhub.domain.namespace.NamespaceRole; import com.iflytek.skillhub.domain.review.ReviewService; import com.iflytek.skillhub.domain.review.ReviewTask; +import com.iflytek.skillhub.domain.review.ReviewOriginType; import com.iflytek.skillhub.domain.review.ReviewTaskRepository; import com.iflytek.skillhub.domain.review.ReviewTaskStatus; import com.iflytek.skillhub.domain.review.ReviewSubjectType; @@ -156,6 +157,29 @@ public class ReviewPortalAppService { String sortDirection, String userId, Map userNsRoles) { + return listReviewsByOrigin(status, namespaceId, page, size, sortDirection, + userId, userNsRoles, ReviewOriginType.USER); + } + + public PageResponse listSyncReviews(String status, + Long namespaceId, + int page, + int size, + String sortDirection, + String userId, + Map userNsRoles) { + return listReviewsByOrigin(status, namespaceId, page, size, sortDirection, + userId, userNsRoles, ReviewOriginType.GITHUB_SYNC); + } + + private PageResponse listReviewsByOrigin(String status, + Long namespaceId, + int page, + int size, + String sortDirection, + String userId, + Map userNsRoles, + ReviewOriginType originType) { ReviewTaskStatus reviewStatus = ReviewTaskStatus.valueOf(status.toUpperCase()); Map namespaceRoles = normalizeRoles(userNsRoles); Set platformRoles = platformRoles(userId); @@ -174,12 +198,17 @@ public class ReviewPortalAppService { platformRoles)) { throw new DomainForbiddenException("review.no_permission"); } - tasks = reviewTaskRepository.findByNamespaceIdAndStatus(namespaceId, reviewStatus, pageable); + tasks = originType == ReviewOriginType.USER + ? reviewTaskRepository.findByNamespaceIdAndStatus(namespaceId, reviewStatus, pageable) + : reviewTaskRepository.findByNamespaceIdAndOriginTypeAndStatus( + namespaceId, originType, reviewStatus, pageable); } else { if (!hasPlatformReviewRole(platformRoles)) { throw new DomainForbiddenException("review.no_permission"); } - tasks = reviewTaskRepository.findByStatus(reviewStatus, pageable); + tasks = originType == ReviewOriginType.USER + ? reviewTaskRepository.findByStatus(reviewStatus, pageable) + : reviewTaskRepository.findByOriginTypeAndStatus(originType, reviewStatus, pageable); } List visibleItems = tasks.getContent().stream() diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewSkillDetailAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewSkillDetailAppService.java index 0a6001501..f7f49e81f 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewSkillDetailAppService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/ReviewSkillDetailAppService.java @@ -61,7 +61,6 @@ public class ReviewSkillDetailAppService { this.skills = skills; this.candidates = candidates; } - public ReviewSkillDetailAppService(ReviewTaskRepository reviewTaskRepository, NamespaceRepository namespaceRepository, ReviewService reviewService, @@ -102,8 +101,30 @@ public class ReviewSkillDetailAppService { } SkillQueryService.ReviewSkillSnapshotDTO snapshot = skillQueryService.getReviewSkillSnapshot(reviewVersionId(context.task())); + return toResponse(reviewId, context, snapshot); + } + public ReviewSkillDetailResponse getReviewSkillDetail(Long reviewId, + String userId, + Map userNsRoles, + boolean includeDocumentation) { + if (includeDocumentation) { + return getReviewSkillDetail(reviewId, userId, userNsRoles); + } + ReviewAccessContext context = loadAuthorizedContext(reviewId, userId, userNsRoles); + if (context.task().getSubjectType() == ReviewSubjectType.SKILL_METADATA && context.task().getRequestedVisibility() == null) { + return metadataDetail(context); + } + SkillQueryService.ReviewSkillSnapshotDTO snapshot = + skillQueryService.getReviewSkillSnapshot(reviewVersionId(context.task()), false); + + return toResponse(reviewId, context, snapshot); + } + + private ReviewSkillDetailResponse toResponse(Long reviewId, ReviewAccessContext context, + SkillQueryService.ReviewSkillSnapshotDTO snapshot) { SkillLifecycleVersionResponse activeVersion = toLifecycleVersion(snapshot.activeVersion()); + SkillDetailResponse skill = new SkillDetailResponse( snapshot.skill().getId(), snapshot.skill().getSlug(), @@ -132,13 +153,14 @@ public class ReviewSkillDetailAppService { "REVIEW_TASK", List.of(), new PageResponse<>(List.of(), 0, 0, 20), - SkillAvatarUrls.current(snapshot.skill(), context.namespace().getSlug()) + SkillAvatarUrls.current(snapshot.skill(), context.namespace().getSlug()), false, null ); List versions = snapshot.versions().stream() .map(version -> new SkillVersionResponse( version.getId(), version.getVersion(), + version.getGithubSyncRevision(), version.getStatus().name(), version.getChangelog(), version.getFileCount(), @@ -174,7 +196,8 @@ public class ReviewSkillDetailAppService { throw new DomainBadRequestException("review.metadata.no_package"); } SkillQueryService.ReviewSkillSnapshotDTO snapshot = - skillQueryService.getReviewSkillSnapshot(reviewVersionId(context.task())); + skillQueryService.getReviewSkillSnapshot(reviewVersionId(context.task()), false); + return skillDownloadService.downloadReviewVersion(snapshot.skill(), snapshot.activeVersion()); } @@ -187,14 +210,21 @@ public class ReviewSkillDetailAppService { String filePath, String userId, Map userNsRoles) { + return getReviewFileContent(reviewId, filePath, userId, userNsRoles, false); + } + + public InputStream getReviewFileContent(Long reviewId, + String filePath, + String userId, + Map userNsRoles, + boolean preview) { ReviewAccessContext context = loadAuthorizedContext(reviewId, userId, userNsRoles); if (reviewVersionId(context.task()) == null) { throw new DomainBadRequestException("review.metadata.no_package"); } - return skillQueryService.getFileContentByVersionId( - reviewVersionId(context.task()), - filePath - ); + return preview + ? skillQueryService.getFilePreviewContentByVersionId(reviewVersionId(context.task()), filePath) + : skillQueryService.getFileContentByVersionId(reviewVersionId(context.task()), filePath); } private ReviewSkillDetailResponse.AvatarChange avatarChange(ReviewAccessContext context) { @@ -280,7 +310,7 @@ public class ReviewSkillDetailAppService { } } return new SkillLifecycleVersionResponse(version.getId(), version.getVersion(), version.getStatus().name(), - metadata == null ? null : metadata.displayName(), metadata == null ? null : metadata.description()); + metadata == null ? null : metadata.displayName(), metadata == null ? null : metadata.description(), version.getGithubSyncRevision()); } private record ReviewAccessContext(ReviewTask task, Namespace namespace) {} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/SkillSearchAppService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/SkillSearchAppService.java index 4935b66c9..e12ae9fa0 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/SkillSearchAppService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/SkillSearchAppService.java @@ -292,7 +292,7 @@ public class SkillSearchAppService { projection.version(), projection.status(), projection.displayName(), - projection.summary() + projection.summary(), projection.syncRevision() ); } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/archive/ZipArchiveSafety.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/archive/ZipArchiveSafety.java new file mode 100644 index 000000000..b9bd0ffec --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/archive/ZipArchiveSafety.java @@ -0,0 +1,101 @@ +package com.iflytek.skillhub.service.archive; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** Checks ZIP central-directory metadata that java.util.zip does not expose. */ +public final class ZipArchiveSafety { + private static final int CENTRAL_HEADER_SIZE = 46; + private static final int EOCD_MIN_SIZE = 22; + private static final int EOCD_MAX_SEARCH = 65_557; + private static final int UNIX_FILE_TYPE_MASK = 0170000; + private static final int UNIX_SYMBOLIC_LINK = 0120000; + + private ZipArchiveSafety() {} + + public static void rejectSymbolicLinks(Path archivePath) throws IOException { + rejectSymbolicLinks(archivePath, Integer.MAX_VALUE, Long.MAX_VALUE); + } + + public static void rejectSymbolicLinks(Path archivePath, int maxEntries, long maxCentralBytes) + throws IOException { + try (FileChannel channel = FileChannel.open(archivePath, StandardOpenOption.READ)) { + long fileSize = channel.size(); + int tailLength = (int) Math.min(fileSize, EOCD_MAX_SEARCH); + ByteBuffer tail = ByteBuffer.allocate(tailLength).order(ByteOrder.LITTLE_ENDIAN); + readFully(channel, tail, fileSize - tailLength); + int eocd = findEndOfCentralDirectory(tail); + if (eocd < 0 || eocd + EOCD_MIN_SIZE > tailLength) { + throw invalid("Invalid ZIP end-of-central-directory record"); + } + int entryCount = Short.toUnsignedInt(tail.getShort(eocd + 10)); + long centralSize = Integer.toUnsignedLong(tail.getInt(eocd + 12)); + long centralOffset = Integer.toUnsignedLong(tail.getInt(eocd + 16)); + if (entryCount == 0xffff || centralSize == 0xffffffffL || centralOffset == 0xffffffffL) { + throw invalid("ZIP64 archives are not supported"); + } + if (centralOffset + centralSize > fileSize) { + throw invalid("Invalid ZIP central-directory bounds"); + } + if (entryCount > maxEntries || centralSize > maxCentralBytes) { + throw invalid("ZIP central directory exceeds package limits"); + } + + long cursor = centralOffset; + for (int index = 0; index < entryCount; index++) { + ByteBuffer header = ByteBuffer.allocate(CENTRAL_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + readFully(channel, header, cursor); + if (header.getInt(0) != 0x02014b50) { + throw invalid("Invalid ZIP central-directory entry"); + } + int creatorSystem = Byte.toUnsignedInt(header.get(5)); + int nameLength = Short.toUnsignedInt(header.getShort(28)); + int extraLength = Short.toUnsignedInt(header.getShort(30)); + int commentLength = Short.toUnsignedInt(header.getShort(32)); + long externalAttributes = Integer.toUnsignedLong(header.getInt(38)); + int unixMode = (int) (externalAttributes >>> 16); + if (creatorSystem == 3 && (unixMode & UNIX_FILE_TYPE_MASK) == UNIX_SYMBOLIC_LINK) { + throw invalid("ZIP archive must not contain symbolic links"); + } + cursor += CENTRAL_HEADER_SIZE + nameLength + extraLength + commentLength; + if (cursor > centralOffset + centralSize) { + throw invalid("Invalid ZIP central-directory entry bounds"); + } + } + } + } + + private static void readFully(FileChannel channel, ByteBuffer buffer, long position) throws IOException { + while (buffer.hasRemaining()) { + int read = channel.read(buffer, position + buffer.position()); + if (read < 0) { + throw invalid("Unexpected end of ZIP archive"); + } + } + } + + private static int findEndOfCentralDirectory(ByteBuffer tail) { + byte[] bytes = tail.array(); + int signature = 0x06054b50; + for (int index = bytes.length - EOCD_MIN_SIZE; index >= 0; index--) { + if ((bytes[index] & 0xff) == (signature & 0xff) + && (bytes[index + 1] & 0xff) == ((signature >>> 8) & 0xff) + && (bytes[index + 2] & 0xff) == ((signature >>> 16) & 0xff) + && (bytes[index + 3] & 0xff) == ((signature >>> 24) & 0xff)) { + int commentLength = Short.toUnsignedInt(tail.getShort(index + 20)); + if (index + EOCD_MIN_SIZE + commentLength == bytes.length) { + return index; + } + } + } + return -1; + } + + private static IllegalArgumentException invalid(String detail) { + return new IllegalArgumentException(detail); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/bundle/SkillSuiteBundleArchiveService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/bundle/SkillSuiteBundleArchiveService.java index d0d7a9be1..fa4bde201 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/bundle/SkillSuiteBundleArchiveService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/bundle/SkillSuiteBundleArchiveService.java @@ -2,6 +2,7 @@ package com.iflytek.skillhub.service.bundle; import com.iflytek.skillhub.config.SkillPublishProperties; import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.service.archive.ZipArchiveSafety; import com.iflytek.skillhub.storage.ObjectStorageService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,12 +13,8 @@ import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -37,11 +34,6 @@ public class SkillSuiteBundleArchiveService { private static final Logger log = LoggerFactory.getLogger(SkillSuiteBundleArchiveService.class); private static final int BUFFER_SIZE = 8192; private static final int MAX_MEMBERS = 100; - private static final int ZIP_CENTRAL_HEADER_SIZE = 46; - private static final int ZIP_EOCD_MIN_SIZE = 22; - private static final int ZIP_EOCD_MAX_SEARCH = 65_557; - private static final int UNIX_FILE_TYPE_MASK = 0170000; - private static final int UNIX_SYMBOLIC_LINK = 0120000; private final SkillSuiteBundlePackageAnalyzer analyzer; private final ObjectStorageService objectStorageService; @@ -81,7 +73,11 @@ public class SkillSuiteBundleArchiveService { try { String archiveSha256 = copyUpload(upload, archivePath); - rejectSymbolicLinks(archivePath); + try { + ZipArchiveSafety.rejectSymbolicLinks(archivePath); + } catch (IllegalArgumentException exception) { + throw invalid(exception.getMessage()); + } try (InputStream archiveInput = Files.newInputStream(archivePath)) { objectStorageService.putObject( archiveObjectKey, archiveInput, Files.size(archivePath), "application/zip"); @@ -220,77 +216,6 @@ public class SkillSuiteBundleArchiveService { return copied; } - /** java.util.zip does not expose Unix modes, so inspect central-directory attributes directly. */ - private void rejectSymbolicLinks(Path archivePath) throws IOException { - try (FileChannel channel = FileChannel.open(archivePath, StandardOpenOption.READ)) { - long fileSize = channel.size(); - int tailLength = (int) Math.min(fileSize, ZIP_EOCD_MAX_SEARCH); - ByteBuffer tail = ByteBuffer.allocate(tailLength).order(ByteOrder.LITTLE_ENDIAN); - readFully(channel, tail, fileSize - tailLength); - int eocd = findEndOfCentralDirectory(tail); - if (eocd < 0 || eocd + ZIP_EOCD_MIN_SIZE > tailLength) { - throw invalid("Invalid ZIP end-of-central-directory record"); - } - int entryCount = Short.toUnsignedInt(tail.getShort(eocd + 10)); - long centralSize = Integer.toUnsignedLong(tail.getInt(eocd + 12)); - long centralOffset = Integer.toUnsignedLong(tail.getInt(eocd + 16)); - if (entryCount == 0xffff || centralSize == 0xffffffffL || centralOffset == 0xffffffffL) { - throw invalid("ZIP64 Bundle archives are not supported"); - } - if (centralOffset + centralSize > fileSize) { - throw invalid("Invalid ZIP central-directory bounds"); - } - - long cursor = centralOffset; - for (int index = 0; index < entryCount; index++) { - ByteBuffer header = ByteBuffer.allocate(ZIP_CENTRAL_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); - readFully(channel, header, cursor); - if (header.getInt(0) != 0x02014b50) { - throw invalid("Invalid ZIP central-directory entry"); - } - int creatorSystem = Byte.toUnsignedInt(header.get(5)); - int nameLength = Short.toUnsignedInt(header.getShort(28)); - int extraLength = Short.toUnsignedInt(header.getShort(30)); - int commentLength = Short.toUnsignedInt(header.getShort(32)); - long externalAttributes = Integer.toUnsignedLong(header.getInt(38)); - int unixMode = (int) (externalAttributes >>> 16); - if (creatorSystem == 3 && (unixMode & UNIX_FILE_TYPE_MASK) == UNIX_SYMBOLIC_LINK) { - throw invalid("Bundle archive must not contain symbolic links"); - } - cursor += ZIP_CENTRAL_HEADER_SIZE + nameLength + extraLength + commentLength; - if (cursor > centralOffset + centralSize) { - throw invalid("Invalid ZIP central-directory entry bounds"); - } - } - } - } - - private void readFully(FileChannel channel, ByteBuffer buffer, long position) throws IOException { - while (buffer.hasRemaining()) { - int read = channel.read(buffer, position + buffer.position()); - if (read < 0) { - throw invalid("Unexpected end of ZIP archive"); - } - } - } - - private int findEndOfCentralDirectory(ByteBuffer tail) { - byte[] bytes = tail.array(); - int signature = 0x06054b50; - for (int index = bytes.length - ZIP_EOCD_MIN_SIZE; index >= 0; index--) { - if ((bytes[index] & 0xff) == (signature & 0xff) - && (bytes[index + 1] & 0xff) == ((signature >>> 8) & 0xff) - && (bytes[index + 2] & 0xff) == ((signature >>> 16) & 0xff) - && (bytes[index + 3] & 0xff) == ((signature >>> 24) & 0xff)) { - int commentLength = Short.toUnsignedInt(tail.getShort(index + 20)); - if (index + ZIP_EOCD_MIN_SIZE + commentLength == bytes.length) { - return index; - } - } - } - return -1; - } - private String contentType(String path) { String lower = path.toLowerCase(); if (lower.endsWith(".py")) return "text/x-python"; diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/AbstractStreamConsumer.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/AbstractStreamConsumer.java index a91fdb0fb..6efc15953 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/AbstractStreamConsumer.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/AbstractStreamConsumer.java @@ -236,7 +236,7 @@ public abstract class AbstractStreamConsumer { markCompleted(payload); acknowledge(messageId); } catch (Exception e) { - messageObservationSupport.recordCurrentError(e); + messageObservationSupport.recordCurrentError(observationError(e)); if (shouldDeferFailure(payload, e)) { markDeferred(payload, e); } else { @@ -303,6 +303,10 @@ public abstract class AbstractStreamConsumer { return false; } + protected Exception observationError(Exception error) { + return error; + } + protected void markDeferred(T payload, Exception error) { log.warn("Deferring {} task without acknowledging its stream entry: {}", taskDisplayName(), payloadIdentifier(payload), error); diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/ScanTaskConsumer.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/ScanTaskConsumer.java index 490ce90b0..dabbfe964 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/ScanTaskConsumer.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/stream/ScanTaskConsumer.java @@ -115,6 +115,11 @@ public class ScanTaskConsumer extends AbstractStreamConsumer 0 || reconciled > 0) { + log.info("GitHub candidate scheduling: queued={}, reconciled={}", queued, reconciled); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubDiscoverySchedulerTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubDiscoverySchedulerTask.java new file mode 100644 index 000000000..206dae92a --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubDiscoverySchedulerTask.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Instant; + +@Component +public class GithubDiscoverySchedulerTask { + private static final Logger log = LoggerFactory.getLogger(GithubDiscoverySchedulerTask.class); + private final GithubDiscoveryEntryRepository entries; + + public GithubDiscoverySchedulerTask(GithubDiscoveryEntryRepository entries) { + this.entries = entries; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.discovery-scheduler-interval-ms:60000}") + public void enqueueDue() { + int queued = entries.enqueueDue(Instant.now(), 10); + if (queued > 0) { + log.info("Queued {} GitHub discovery tasks", queued); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubOfficialDiscoverySchedulerTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubOfficialDiscoverySchedulerTask.java new file mode 100644 index 000000000..034b6f1fd --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubOfficialDiscoverySchedulerTask.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Instant; + +@Component +public class GithubOfficialDiscoverySchedulerTask { + private static final Logger log = LoggerFactory.getLogger(GithubOfficialDiscoverySchedulerTask.class); + private final GithubOfficialDiscoveryRepository queries; + + public GithubOfficialDiscoverySchedulerTask(GithubOfficialDiscoveryRepository queries) { + this.queries = queries; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.discovery-scheduler-interval-ms:60000}") + public void enqueueDue() { + int queued = queries.enqueueDue(Instant.now(), 10); + if (queued > 0) { + log.info("Queued {} official GitHub discovery tasks", queued); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSourceSchedulerTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSourceSchedulerTask.java new file mode 100644 index 000000000..93b0d909e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSourceSchedulerTask.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubSourceTaskRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Instant; + +@Component +public class GithubSourceSchedulerTask { + private static final Logger log = LoggerFactory.getLogger(GithubSourceSchedulerTask.class); + private final GithubSourceTaskRepository sources; + + public GithubSourceSchedulerTask(GithubSourceTaskRepository sources) { + this.sources = sources; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.source-scheduler-interval-ms:60000}") + public void enqueueDue() { + int queued = sources.enqueueDue(Instant.now(), 10); + if (queued > 0) { + log.info("Queued {} GitHub source sync tasks", queued); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncHistoryCleanupTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncHistoryCleanupTask.java new file mode 100644 index 000000000..0bce33d4e --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncHistoryCleanupTask.java @@ -0,0 +1,51 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubSyncHistoryRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; + +@Component +public class GithubSyncHistoryCleanupTask { + private static final Logger log = LoggerFactory.getLogger(GithubSyncHistoryCleanupTask.class); + private static final int DISCOVERY_BATCH_SIZE = 500; + private static final int REPOSITORY_BATCH_SIZE = 25; + private final GithubSyncHistoryRepository history; + private final Duration retention; + + public GithubSyncHistoryCleanupTask(GithubSyncHistoryRepository history, + @Value("${skillhub.github-sync.completed-detail-retention:P90D}") Duration retention) { + if (retention.compareTo(Duration.ofDays(1)) < 0) { + throw new IllegalArgumentException("GitHub sync history retention must be at least one day"); + } + this.history = history; + this.retention = retention; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.history-cleanup-interval-ms:3600000}") + public void cleanup() { + Instant cutoff = Instant.now().minus(retention); + int tasks = 0; + int runs = 0; + for (int batch = 0; batch < 10; batch++) { + var discovery = history.deleteExpiredDiscoveryHistory(cutoff, DISCOVERY_BATCH_SIZE); + var repository = history.purgeExpiredRepositoryDetails(cutoff, Instant.now(), + REPOSITORY_BATCH_SIZE); + tasks += discovery.tasks() + repository.tasks(); + runs += discovery.runs() + repository.runs(); + if (discovery.tasks() < DISCOVERY_BATCH_SIZE && discovery.runs() < DISCOVERY_BATCH_SIZE + && repository.tasks() < REPOSITORY_BATCH_SIZE + && repository.runs() < REPOSITORY_BATCH_SIZE) { + break; + } + } + if (tasks > 0 || runs > 0) { + log.info("Cleaned up expired GitHub sync history [tasks={}, runs={}]", tasks, runs); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncIngestionTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncIngestionTask.java new file mode 100644 index 000000000..cb660814d --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncIngestionTask.java @@ -0,0 +1,114 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubSyncIngestionRepository; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import com.iflytek.skillhub.service.GithubSyncArchiveValidator; +import com.iflytek.skillhub.storage.ObjectStorageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +@Component +public class GithubSyncIngestionTask { + private static final Logger log = LoggerFactory.getLogger(GithubSyncIngestionTask.class); + private final GithubSyncIngestionRepository ingestions; + private final GithubSyncArchiveValidator validator; + private final ObjectStorageService storage; + private final GithubSourceIgnoreRepository ignores; + private final Duration validationLease; + + public GithubSyncIngestionTask(GithubSyncIngestionRepository ingestions, + GithubSyncArchiveValidator validator, ObjectStorageService storage, + GithubSourceIgnoreRepository ignores, + @Value("${skillhub.github-sync.ingestion-validation-lease:PT30M}") Duration validationLease) { + if (validationLease.isNegative() || validationLease.isZero() + || validationLease.compareTo(Duration.ofHours(4)) > 0) { + throw new IllegalArgumentException("GitHub sync ingestion validation lease is invalid"); + } + this.ingestions = ingestions; + this.validator = validator; + this.storage = storage; + this.ignores = ignores; + this.validationLease = validationLease; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.ingestion-validation-interval-ms:5000}") + public void validatePending() { + for (int processed = 0; processed < 8; processed++) { + Instant now = Instant.now(); + Optional claim = + ingestions.claim(UUID.randomUUID(), now, now.plus(validationLease)); + if (claim.isEmpty()) { + return; + } + validateOne(claim.get()); + } + } + + private void validateOne(GithubSyncIngestionRepository.ClaimedIngestion ingestion) { + try { + if (ingestion.sourceId() != null + && ignores.ignoredForSource(ingestion.sourceId(), ingestion.skillPath())) { + fail(ingestion, "SKILL_PATH_IGNORED", null, "IGNORED"); + return; + } + if (!storage.exists(ingestion.objectKey())) { + if (ingestion.attempt() >= 3) { + fail(ingestion, "OBJECT_MISSING", null, "RETRYABLE_FAILURE"); + } else { + fail(ingestion, "OBJECT_MISSING", nextRetry(ingestion.attempt()), null); + } + return; + } + long actualSize = storage.getMetadata(ingestion.objectKey()).size(); + if (actualSize != ingestion.sizeBytes()) { + fail(ingestion, "ARCHIVE_SIZE_MISMATCH", null, "RETRYABLE_FAILURE"); + return; + } + GithubSyncArchiveValidator.ArchiveValidation result = validator.validate( + ingestion.objectKey(), ingestion.sizeBytes(), ingestion.sha256(), ingestion.skillPath()); + if (!ingestions.validated(ingestion.id(), ingestion.validationToken(), + result.contentSha256(), result.warningCount(), result.evidenceJson(), + Instant.now())) { + log.debug("GitHub sync ingestion validation lease became stale: {}", ingestion.id()); + } + } catch (GithubSyncArchiveValidator.ValidationFailure exception) { + String pathOutcome = switch (exception.code()) { + case "ARCHIVE_SIZE_MISMATCH", "ARCHIVE_DIGEST_MISMATCH" -> "RETRYABLE_FAILURE"; + default -> "TERMINAL_FAILURE"; + }; + fail(ingestion, exception.code(), null, pathOutcome); + } catch (GithubSyncArchiveValidator.InsufficientLocalCapacityException exception) { + fail(ingestion, "LOCAL_CAPACITY", Instant.now().plus(Duration.ofHours(1)), null); + } catch (IOException exception) { + fail(ingestion, "INGESTION_IO", nextRetry(ingestion.attempt()), null); + } catch (RuntimeException exception) { + log.warn("GitHub sync ingestion {} failed with {}", ingestion.id(), + exception.getClass().getSimpleName()); + fail(ingestion, "INGESTION_INTERNAL", nextRetry(ingestion.attempt()), null); + } + } + + private void fail(GithubSyncIngestionRepository.ClaimedIngestion ingestion, + String code, Instant retryAt, String pathOutcome) { + if (!ingestions.failed(ingestion.id(), ingestion.validationToken(), + code, retryAt, pathOutcome, Instant.now())) { + log.debug("GitHub sync ingestion failure arrived after lease changed: {}", ingestion.id()); + } + } + + private Instant nextRetry(int attempt) { + Duration delay = attempt <= 1 ? Duration.ofMinutes(15) + : attempt == 2 ? Duration.ofHours(1) + : attempt == 3 ? Duration.ofHours(6) : Duration.ofDays(1); + return Instant.now().plus(delay); + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncPublicationTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncPublicationTask.java new file mode 100644 index 000000000..cd8a1650f --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncPublicationTask.java @@ -0,0 +1,92 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import com.iflytek.skillhub.service.GithubSyncArchiveValidator; +import com.iflytek.skillhub.service.GithubSyncPublicationAppService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +@Component +public class GithubSyncPublicationTask { + private static final Logger log = LoggerFactory.getLogger(GithubSyncPublicationTask.class); + + private final GithubSyncPublicationRepository publications; + private final GithubSyncPublicationAppService service; + private final Duration lease; + + public GithubSyncPublicationTask(GithubSyncPublicationRepository publications, + GithubSyncPublicationAppService service, + @Value("${skillhub.github-sync.publication-lease:PT4H}") Duration lease) { + if (lease.isNegative() || lease.isZero() || lease.compareTo(Duration.ofHours(24)) > 0) { + throw new IllegalArgumentException("GitHub publication lease is invalid"); + } + this.publications = publications; + this.service = service; + this.lease = lease; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.publication-interval-ms:5000}") + public void publishValidated() { + for (int processed = 0; processed < 4; processed++) { + Instant now = Instant.now(); + Optional claim = + publications.claim(UUID.randomUUID(), now, now.plus(lease)); + if (claim.isEmpty()) { + return; + } + publishOne(claim.get()); + } + } + + private void publishOne(GithubSyncPublicationRepository.ClaimedPublication claim) { + try { + service.process(claim); + } catch (GithubSyncArchiveValidator.ValidationFailure exception) { + settleBlocked(claim, exception.code()); + } catch (DomainBadRequestException exception) { + if ("error.skill.publish.githubSync.slugConflict".equals(exception.messageCode()) + && exception.messageArgs().length > 0 + && exception.messageArgs()[0] instanceof String slug) { + if (!publications.blockedSlugConflict(claim, slug, Instant.now())) { + log.debug("GitHub publication lease became stale: {}", claim.id()); + } + } else { + settleBlocked(claim, "PUBLISH_POLICY_BLOCKED"); + } + log.info("GitHub publication {} blocked by {}", claim.id(), exception.messageCode()); + } catch (GithubSyncArchiveValidator.InsufficientLocalCapacityException exception) { + settleRetry(claim, "LOCAL_CAPACITY"); + } catch (IOException exception) { + settleRetry(claim, "PUBLICATION_IO"); + } catch (RuntimeException exception) { + log.warn("GitHub publication {} failed with {}", claim.id(), + exception.getClass().getSimpleName()); + settleRetry(claim, "PUBLICATION_INTERNAL"); + } + } + + private void settleBlocked(GithubSyncPublicationRepository.ClaimedPublication claim, String code) { + if (!publications.blocked(claim, code, Instant.now())) { + log.debug("GitHub publication lease became stale: {}", claim.id()); + } + } + + private void settleRetry(GithubSyncPublicationRepository.ClaimedPublication claim, String code) { + Duration delay = claim.attempt() == 1 ? Duration.ofMinutes(15) + : claim.attempt() == 2 ? Duration.ofHours(1) + : claim.attempt() == 3 ? Duration.ofHours(6) : Duration.ofDays(1); + if (!publications.retry(claim, code, Instant.now().plus(delay), Instant.now())) { + log.debug("GitHub publication lease became stale: {}", claim.id()); + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncStagedUploadCleanupTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncStagedUploadCleanupTask.java new file mode 100644 index 000000000..1552eaaa0 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncStagedUploadCleanupTask.java @@ -0,0 +1,58 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.githubsync.GithubSyncUploadRepository; +import com.iflytek.skillhub.githubsync.GithubSyncIngestionRepository; +import com.iflytek.skillhub.storage.ObjectStorageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; + +@Component +public class GithubSyncStagedUploadCleanupTask { + private static final Logger log = LoggerFactory.getLogger(GithubSyncStagedUploadCleanupTask.class); + private final GithubSyncUploadRepository uploads; + private final GithubSyncIngestionRepository ingestions; + private final ObjectStorageService storage; + private final Duration signedUrlGrace; + + public GithubSyncStagedUploadCleanupTask(GithubSyncUploadRepository uploads, + GithubSyncIngestionRepository ingestions, ObjectStorageService storage, + @Value("${skillhub.github-sync.signed-upload-cleanup-grace:PT1H}") Duration signedUrlGrace) { + if (signedUrlGrace.isNegative() || signedUrlGrace.compareTo(Duration.ofHours(24)) > 0) { + throw new IllegalArgumentException("GitHub sync signed upload cleanup grace is invalid"); + } + this.uploads = uploads; + this.ingestions = ingestions; + this.storage = storage; + this.signedUrlGrace = signedUrlGrace; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.upload-cleanup-interval-ms:300000}") + public void cleanup() { + for (GithubSyncUploadRepository.CleanupUpload upload : + uploads.claimCleanup(Instant.now(), signedUrlGrace, 100)) { + try { + storage.deleteObject(upload.objectKey()); + uploads.markCleaned(upload.uploadId(), Instant.now()); + } catch (RuntimeException exception) { + log.warn("Could not clean up GitHub sync staged upload {}: {}", + upload.uploadId(), exception.getClass().getSimpleName()); + } + } + for (GithubSyncIngestionRepository.CleanupIngestion ingestion : + ingestions.claimCleanup(Instant.now(), signedUrlGrace, 100)) { + try { + storage.deleteObject(ingestion.objectKey()); + ingestions.markCleaned(ingestion.id(), Instant.now()); + } catch (RuntimeException exception) { + log.warn("Could not clean up terminal GitHub sync ingestion {}: {}", + ingestion.id(), exception.getClass().getSimpleName()); + } + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncWorkerPresenceCleanupTask.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncWorkerPresenceCleanupTask.java new file mode 100644 index 000000000..d15da3802 --- /dev/null +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/GithubSyncWorkerPresenceCleanupTask.java @@ -0,0 +1,36 @@ +package com.iflytek.skillhub.task; + +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; + +/** Discard expired process diagnostics without touching durable task progress or legacy contact. */ +@Component +public class GithubSyncWorkerPresenceCleanupTask { + private static final int BATCH_SIZE = 500; + private final GithubSyncWorkerPresenceRepository presence; + private final Duration retention; + + public GithubSyncWorkerPresenceCleanupTask(GithubSyncWorkerPresenceRepository presence, + @Value("${skillhub.github-sync.completed-detail-retention:P90D}") Duration retention) { + if (retention.compareTo(Duration.ofDays(1)) < 0) { + throw new IllegalArgumentException("GitHub sync presence retention must be at least one day"); + } + this.presence = presence; + this.retention = retention; + } + + @Scheduled(fixedDelayString = "${skillhub.github-sync.history-cleanup-interval-ms:3600000}") + public void cleanup() { + Instant before = Instant.now().minus(retention); + for (int batch = 0; batch < 10; batch++) { + if (presence.deleteExpiredInstances(before, BATCH_SIZE) < BATCH_SIZE) { + break; + } + } + } +} diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcher.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcher.java index 242a32d20..0ae74a735 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcher.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcher.java @@ -73,8 +73,10 @@ public class ScanTaskOutboxDispatcher { private void handlePublishFailure(ScanTaskOutbox outbox, Exception error) { Instant now = Instant.now(clock); int nextAttempt = outbox.getRetryCount() + 1; + String errorType = error.getClass().getSimpleName(); + String failureCode = "SCAN_TASK_PUBLISH_FAILED:" + errorType; if (nextAttempt >= maxAttempts) { - outbox.markFailed(now, error.toString()); + outbox.markFailed(now, failureCode); repository.save(outbox); versionRepository.findById(outbox.getVersionId()) .filter(version -> version.getStatus() == SkillVersionStatus.SCANNING) @@ -82,15 +84,15 @@ public class ScanTaskOutboxDispatcher { version.setStatus(SkillVersionStatus.SCAN_FAILED); versionRepository.save(version); }); - log.error("Scan task publish failed permanently: taskId={}, versionId={}, attempts={}", - outbox.getTaskId(), outbox.getVersionId(), outbox.getRetryCount(), error); + log.error("Scan task publish failed permanently: taskId={}, versionId={}, attempts={}, errorType={}", + outbox.getTaskId(), outbox.getVersionId(), outbox.getRetryCount(), errorType); return; } Duration delay = retryDelay(nextAttempt); - outbox.markRetry(now, delay, error.toString()); + outbox.markRetry(now, delay, failureCode); repository.save(outbox); - log.warn("Failed to publish scan task; will retry taskId={}, retryCount={}, nextDelay={}", - outbox.getTaskId(), outbox.getRetryCount(), delay, error); + log.warn("Failed to publish scan task; will retry taskId={}, retryCount={}, nextDelay={}, errorType={}", + outbox.getTaskId(), outbox.getRetryCount(), delay, errorType); } @Scheduled(cron = "0 20 2 * * ?") diff --git a/server/skillhub-app/src/main/resources/application.yml b/server/skillhub-app/src/main/resources/application.yml index fdbb39a46..3b2f58f82 100644 --- a/server/skillhub-app/src/main/resources/application.yml +++ b/server/skillhub-app/src/main/resources/application.yml @@ -142,6 +142,25 @@ skillhub: run-timeout-ms: ${SKILLHUB_AUTHORING_RUN_TIMEOUT_MS:900000} # Active runs older than this are swept to TIMED_OUT (crash recovery) stale-run-minutes: ${SKILLHUB_AUTHORING_STALE_RUN_MINUTES:30} + github-sync: + discovery-scheduler-interval-ms: ${SKILLHUB_GITHUB_SYNC_DISCOVERY_SCHEDULER_INTERVAL_MS:60000} + candidate-scheduler-interval-ms: ${SKILLHUB_GITHUB_SYNC_CANDIDATE_SCHEDULER_INTERVAL_MS:60000} + source-scheduler-interval-ms: ${SKILLHUB_GITHUB_SYNC_SOURCE_SCHEDULER_INTERVAL_MS:60000} + upload-ticket-ttl: ${SKILLHUB_GITHUB_SYNC_UPLOAD_TICKET_TTL:PT5M} + signed-upload-cleanup-grace: ${SKILLHUB_GITHUB_SYNC_SIGNED_UPLOAD_CLEANUP_GRACE:PT1H} + upload-cleanup-interval-ms: ${SKILLHUB_GITHUB_SYNC_UPLOAD_CLEANUP_INTERVAL_MS:300000} + staging-capacity-limit-bytes: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES:0} + staging-capacity-sample-interval-ms: ${SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_SAMPLE_INTERVAL_MS:3600000} + ingestion-validation-lease: ${SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_LEASE:PT30M} + ingestion-validation-interval-ms: ${SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_INTERVAL_MS:5000} + publication-lease: ${SKILLHUB_GITHUB_SYNC_PUBLICATION_LEASE:PT4H} + publication-interval-ms: ${SKILLHUB_GITHUB_SYNC_PUBLICATION_INTERVAL_MS:5000} + service-tokens: ${SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS:} + credential-keys: ${SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS:} + active-credential-key-version: ${SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION:} + lease-duration: ${SKILLHUB_GITHUB_SYNC_LEASE_DURATION:PT5M} + repository-check-interval: ${SKILLHUB_GITHUB_SYNC_REPOSITORY_CHECK_INTERVAL:PT24H} + suite: # Fail closed for deployment modes whose upgrade topology is unknown. Single-instance # distributions explicitly enable this after ruling out mixed application versions. diff --git a/server/skillhub-app/src/main/resources/db/migration/V100__github_candidate_origins.sql b/server/skillhub-app/src/main/resources/db/migration/V100__github_candidate_origins.sql new file mode 100644 index 000000000..4094f6bd5 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V100__github_candidate_origins.sql @@ -0,0 +1,28 @@ +-- Candidate provenance must outlive the discovery task detail retention window. +CREATE TABLE github_discovery_candidate_origin ( + candidate_id BIGINT NOT NULL REFERENCES github_discovery_candidate(id) ON DELETE CASCADE, + origin_type VARCHAR(32) NOT NULL CHECK (origin_type IN ('SEARCH_RULE', 'CURATED_OWNER')), + origin_key VARCHAR(256) NOT NULL, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (candidate_id, origin_type, origin_key) +); + +INSERT INTO github_discovery_candidate_origin + (candidate_id, origin_type, origin_key, first_seen_at, last_seen_at) +SELECT hit.candidate_id, + task.snapshot_json ->> 'entryType', + CASE WHEN task.snapshot_json ->> 'entryType' = 'CURATED_OWNER' + THEN task.snapshot_json ->> 'owner' + ELSE task.snapshot_json ->> 'ruleId' END, + MIN(hit.created_at), MAX(hit.created_at) +FROM github_discovery_task_hit hit +JOIN github_sync_task task ON task.id = hit.task_id +WHERE task.snapshot_json ->> 'entryType' IN ('SEARCH_RULE', 'CURATED_OWNER') + AND CASE WHEN task.snapshot_json ->> 'entryType' = 'CURATED_OWNER' + THEN task.snapshot_json ->> 'owner' + ELSE task.snapshot_json ->> 'ruleId' END IS NOT NULL +GROUP BY hit.candidate_id, task.snapshot_json ->> 'entryType', + CASE WHEN task.snapshot_json ->> 'entryType' = 'CURATED_OWNER' + THEN task.snapshot_json ->> 'owner' + ELSE task.snapshot_json ->> 'ruleId' END; diff --git a/server/skillhub-app/src/main/resources/db/migration/V101__github_sync_history_retention_index.sql b/server/skillhub-app/src/main/resources/db/migration/V101__github_sync_history_retention_index.sql new file mode 100644 index 000000000..575cbed6f --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V101__github_sync_history_retention_index.sql @@ -0,0 +1,7 @@ +CREATE INDEX idx_github_sync_discovery_history_cleanup + ON github_sync_task (completed_at, id) + WHERE kind = 'DISCOVERY' AND status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED'); + +CREATE INDEX idx_github_sync_discovery_run_cleanup + ON github_sync_run (finished_at, id) + WHERE kind = 'DISCOVERY' AND status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V102__github_repository_task_detail_retention.sql b/server/skillhub-app/src/main/resources/db/migration/V102__github_repository_task_detail_retention.sql new file mode 100644 index 000000000..c7194fcb5 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V102__github_repository_task_detail_retention.sql @@ -0,0 +1,19 @@ +-- Repository task and run IDs remain as audit references after their operational +-- details expire. Latest per-path source state is retained separately. +ALTER TABLE github_sync_task ADD COLUMN details_purged_at TIMESTAMPTZ; +ALTER TABLE github_sync_run ADD COLUMN details_purged_at TIMESTAMPTZ; + +CREATE INDEX idx_github_sync_repository_detail_cleanup + ON github_sync_task (completed_at, id) + WHERE kind = 'REPOSITORY_SYNC' + AND status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND details_purged_at IS NULL; + +CREATE INDEX idx_github_sync_run_detail_cleanup + ON github_sync_run (finished_at, id) + WHERE kind = 'REPOSITORY_SYNC' + AND status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND details_purged_at IS NULL; + +CREATE INDEX idx_github_sync_path_skill_task + ON github_sync_path_result (skill_path, task_id, updated_at DESC, id DESC); diff --git a/server/skillhub-app/src/main/resources/db/migration/V103__github_sync_rejected_skill_retry.sql b/server/skillhub-app/src/main/resources/db/migration/V103__github_sync_rejected_skill_retry.sql new file mode 100644 index 000000000..2586f6d38 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V103__github_sync_rejected_skill_retry.sql @@ -0,0 +1,9 @@ +ALTER TABLE github_source_skill_binding + ADD COLUMN retry_requested_at TIMESTAMPTZ, + ADD COLUMN retry_scheduled_at TIMESTAMPTZ, + ADD CONSTRAINT ck_github_skill_retry_schedule CHECK + (retry_scheduled_at IS NULL OR retry_requested_at IS NOT NULL); + +CREATE INDEX idx_github_skill_binding_retry_requested + ON github_source_skill_binding (source_id) + WHERE retry_requested_at IS NOT NULL AND retry_scheduled_at IS NULL; diff --git a/server/skillhub-app/src/main/resources/db/migration/V104__github_sync_worker_presence.sql b/server/skillhub-app/src/main/resources/db/migration/V104__github_sync_worker_presence.sql new file mode 100644 index 000000000..7a8439ae4 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V104__github_sync_worker_presence.sql @@ -0,0 +1,4 @@ +CREATE TABLE github_sync_worker_presence ( + id SMALLINT PRIMARY KEY CHECK (id = 1), + last_seen_at TIMESTAMPTZ NOT NULL +); diff --git a/server/skillhub-app/src/main/resources/db/migration/V105__github_sync_latest_source_ingestion.sql b/server/skillhub-app/src/main/resources/db/migration/V105__github_sync_latest_source_ingestion.sql new file mode 100644 index 000000000..bbbe2951c --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V105__github_sync_latest_source_ingestion.sql @@ -0,0 +1,4 @@ +CREATE INDEX idx_github_sync_ingestion_latest_source_path + ON github_sync_package_ingestion + (namespace_id, repository_id, skill_path, created_at DESC, id DESC) + INCLUDE (status); diff --git a/server/skillhub-app/src/main/resources/db/migration/V106__github_source_partial_scan_and_rescan.sql b/server/skillhub-app/src/main/resources/db/migration/V106__github_source_partial_scan_and_rescan.sql new file mode 100644 index 000000000..72a7ab952 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V106__github_source_partial_scan_and_rescan.sql @@ -0,0 +1,5 @@ +-- Distinguish a completely enumerated revision with terminal Skill failures +-- from a revision whose repository scan still needs to resume. +ALTER TABLE github_source_binding + ADD COLUMN last_partial_commit VARCHAR(40), + ADD COLUMN manual_rescan_requested_at TIMESTAMPTZ; diff --git a/server/skillhub-app/src/main/resources/db/migration/V107__github_sync_content_revision.sql b/server/skillhub-app/src/main/resources/db/migration/V107__github_sync_content_revision.sql new file mode 100644 index 000000000..6292b743c --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V107__github_sync_content_revision.sql @@ -0,0 +1,34 @@ +-- Content revisions are local to a Skill. A retry of identical content keeps its +-- revision; returning to an earlier digest after a different digest gets a new one. +ALTER TABLE skill_version ADD COLUMN github_sync_revision INTEGER; +ALTER TABLE github_source_skill_binding + ADD COLUMN latest_revision INTEGER NOT NULL DEFAULT 0 CHECK (latest_revision >= 0); + +WITH ordered AS ( + SELECT version.id, version.skill_id, ingestion.content_sha256, + LAG(ingestion.content_sha256) OVER ( + PARTITION BY version.skill_id ORDER BY version.id + ) AS previous_content_sha256 + FROM skill_version version + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = version.github_sync_ingestion_id +), numbered AS ( + SELECT id, SUM(CASE WHEN content_sha256 IS DISTINCT FROM previous_content_sha256 + THEN 1 ELSE 0 END) OVER ( + PARTITION BY skill_id ORDER BY id + ) AS revision + FROM ordered +) +UPDATE skill_version version +SET github_sync_revision = numbered.revision +FROM numbered WHERE version.id = numbered.id; + +UPDATE github_source_skill_binding binding +SET latest_revision = COALESCE(( + SELECT MAX(version.github_sync_revision) + FROM skill_version version WHERE version.skill_id = binding.skill_id +), 0); + +ALTER TABLE skill_version ADD CONSTRAINT ck_skill_version_github_sync_revision + CHECK (github_sync_revision IS NULL + OR (github_sync_ingestion_id IS NOT NULL AND github_sync_revision > 0)); diff --git a/server/skillhub-app/src/main/resources/db/migration/V108__github_deep_discovery_configuration.sql b/server/skillhub-app/src/main/resources/db/migration/V108__github_deep_discovery_configuration.sql new file mode 100644 index 000000000..08bece30a --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V108__github_deep_discovery_configuration.sql @@ -0,0 +1,17 @@ +CREATE TABLE github_deep_discovery_config ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id = TRUE), + namespace_id BIGINT NOT NULL UNIQUE REFERENCES namespace(id), + enabled BOOLEAN NOT NULL DEFAULT FALSE, + budget_profile VARCHAR(16) NOT NULL DEFAULT 'BALANCED' + CHECK (budget_profile IN ('ECONOMY', 'BALANCED', 'ACCELERATED')), + config_version BIGINT NOT NULL DEFAULT 1 CHECK (config_version > 0), + exclusions_json JSONB NOT NULL DEFAULT '[]'::jsonb + CHECK (jsonb_typeof(exclusions_json) = 'array' AND jsonb_array_length(exclusions_json) <= 1000), + strategy_version VARCHAR(128), + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(128) +); + +-- No strategy is installed by this configuration-only migration. +INSERT INTO github_deep_discovery_config (namespace_id) +VALUES ((SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG')); diff --git a/server/skillhub-app/src/main/resources/db/migration/V109__github_official_discovery_queries.sql b/server/skillhub-app/src/main/resources/db/migration/V109__github_official_discovery_queries.sql new file mode 100644 index 000000000..655d97cf8 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V109__github_official_discovery_queries.sql @@ -0,0 +1,44 @@ +-- Monotonic activation prevents mixed Java versions from rolling the catalog backward. +ALTER TABLE github_deep_discovery_config + ADD COLUMN strategy_generation BIGINT NOT NULL DEFAULT 0 CHECK (strategy_generation >= 0); + +CREATE TABLE github_deep_discovery_query ( + id UUID PRIMARY KEY, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + query_key VARCHAR(128) NOT NULL, + query_version BIGINT NOT NULL CHECK (query_version > 0), + strategy_version VARCHAR(128) NOT NULL, + definition_json JSONB NOT NULL CHECK (jsonb_typeof(definition_json) = 'object'), + covered_until TIMESTAMPTZ, + checkpoint VARCHAR(4096), + next_due_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + active_task_id UUID REFERENCES github_sync_task(id), + last_progress_at TIMESTAMPTZ, + wait_reason VARCHAR(64), + retired_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_deep_discovery_query UNIQUE (namespace_id, query_key, query_version), + CONSTRAINT uq_github_deep_discovery_query_namespace UNIQUE (id, namespace_id) +); + +CREATE INDEX idx_github_deep_discovery_query_due + ON github_deep_discovery_query (next_due_at, id) + WHERE retired_at IS NULL AND active_task_id IS NULL; + +ALTER TABLE github_sync_task + ADD COLUMN required_capabilities TEXT[] NOT NULL DEFAULT '{}'::text[], + ADD COLUMN official_query_id UUID, + ADD COLUMN last_progress_at TIMESTAMPTZ, + ADD CONSTRAINT ck_github_sync_task_capability_count + CHECK (cardinality(required_capabilities) <= 32 + AND array_position(required_capabilities, NULL) IS NULL), + ADD CONSTRAINT ck_github_sync_task_official_capabilities + CHECK (official_query_id IS NULL OR (kind = 'DISCOVERY' AND required_capabilities @> + ARRAY['official-discovery/v1', 'official-checkpoint/v1', 'official-pause/v1']::text[])), + ADD CONSTRAINT fk_github_sync_task_official_query + FOREIGN KEY (official_query_id, namespace_id) + REFERENCES github_deep_discovery_query(id, namespace_id); + +CREATE UNIQUE INDEX uq_github_official_query_open_task + ON github_sync_task (official_query_id) + WHERE official_query_id IS NOT NULL AND status IN ('QUEUED', 'LEASED', 'RETRY_WAIT'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V110__github_official_discovery_results.sql b/server/skillhub-app/src/main/resources/db/migration/V110__github_official_discovery_results.sql new file mode 100644 index 000000000..068f2c347 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V110__github_official_discovery_results.sql @@ -0,0 +1,30 @@ +ALTER TABLE github_discovery_candidate_origin + DROP CONSTRAINT github_discovery_candidate_origin_origin_type_check, + ADD CONSTRAINT github_discovery_candidate_origin_origin_type_check + CHECK (origin_type IN ('SEARCH_RULE', 'CURATED_OWNER', 'OFFICIAL_DISCOVERY')); +ALTER TABLE github_source_origin + DROP CONSTRAINT ck_github_source_origin_type, + ADD CONSTRAINT ck_github_source_origin_type + CHECK (origin_type IN ('SEARCH_RULE', 'CURATED_OWNER', 'CURATED_REPOSITORY', 'TEAM_CONFIG', 'OFFICIAL_DISCOVERY')); +ALTER TABLE github_sync_task ADD COLUMN official_pause_reason VARCHAR(64); +ALTER TABLE github_deep_discovery_query ADD COLUMN last_full_recheck_at TIMESTAMPTZ, + ADD COLUMN gap_reason VARCHAR(64); +ALTER TABLE github_discovery_batch ADD COLUMN official_metadata JSONB; + +-- Current independent entry state, rather than a historical origin alone, grants eligibility. +CREATE VIEW github_candidate_discovery_eligibility AS +SELECT candidate.id AS candidate_id, candidate.namespace_id, candidate.repository_id, + source.id AS source_id, source.status AS source_status, + EXISTS (SELECT 1 FROM github_discovery_candidate_origin origin + WHERE origin.candidate_id = candidate.id AND origin.origin_type = 'OFFICIAL_DISCOVERY') AS has_official, + EXISTS ( + SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_discovery_entry entry ON entry.namespace_id = candidate.namespace_id AND entry.enabled + AND ((origin.origin_type = 'SEARCH_RULE' AND entry.entry_type = 'SEARCH_RULE' + AND origin.origin_key = entry.id::text) + OR (origin.origin_type = 'CURATED_OWNER' AND entry.entry_type = 'CURATED_OWNER' + AND lower(origin.origin_key) = lower(entry.owner_login))) + WHERE origin.candidate_id = candidate.id) AS has_independent +FROM github_discovery_candidate candidate +LEFT JOIN github_source_binding source + ON source.namespace_id = candidate.namespace_id AND source.repository_id = candidate.repository_id; diff --git a/server/skillhub-app/src/main/resources/db/migration/V111__github_official_candidate_controls.sql b/server/skillhub-app/src/main/resources/db/migration/V111__github_official_candidate_controls.sql new file mode 100644 index 000000000..965e9542a --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V111__github_official_candidate_controls.sql @@ -0,0 +1,34 @@ +-- Live eligibility: registered source policy wins, then current independent entries, then official controls. +CREATE OR REPLACE VIEW github_candidate_discovery_eligibility AS +WITH origin_state AS ( +SELECT candidate.id AS candidate_id, candidate.namespace_id, candidate.repository_id, + source.id AS source_id, source.status AS source_status, + EXISTS (SELECT 1 FROM github_discovery_candidate_origin origin + WHERE origin.candidate_id = candidate.id AND origin.origin_type = 'OFFICIAL_DISCOVERY') AS has_official, + EXISTS ( + SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_discovery_entry entry ON entry.namespace_id = candidate.namespace_id AND entry.enabled + AND ((origin.origin_type = 'SEARCH_RULE' AND entry.entry_type = 'SEARCH_RULE' + AND origin.origin_key = entry.id::text) + OR (origin.origin_type = 'CURATED_OWNER' AND entry.entry_type = 'CURATED_OWNER' + AND lower(origin.origin_key) = lower(entry.owner_login))) + WHERE origin.candidate_id = candidate.id) AS has_independent +FROM github_discovery_candidate candidate +LEFT JOIN github_source_binding source + ON source.namespace_id = candidate.namespace_id AND source.repository_id = candidate.repository_id +), controls AS ( + SELECT origin_state.*, COALESCE(config.enabled, FALSE) AS configuration_enabled, + EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(config.exclusions_json, '[]'::jsonb)) exclusion + JOIN github_discovery_candidate candidate ON candidate.id = origin_state.candidate_id + WHERE (exclusion ->> 'kind' = 'OWNER' AND exclusion ->> 'value' = lower(candidate.owner_login)) + OR (exclusion ->> 'kind' = 'REPOSITORY' + AND exclusion ->> 'value' = lower(candidate.owner_login || '/' || candidate.repository_name))) AS excluded + FROM origin_state LEFT JOIN github_deep_discovery_config config ON config.namespace_id = origin_state.namespace_id +) +SELECT candidate_id, namespace_id, repository_id, source_id, source_status, has_official, has_independent, + has_official AND NOT has_independent AND source_id IS NULL AS official_only, + configuration_enabled, excluded, + CASE WHEN source_id IS NOT NULL THEN source_status <> 'PAUSED' + WHEN NOT has_official OR has_independent THEN TRUE + ELSE configuration_enabled AND NOT excluded END AS acquisition_allowed +FROM controls; diff --git a/server/skillhub-app/src/main/resources/db/migration/V112__github_official_discovery_shards.sql b/server/skillhub-app/src/main/resources/db/migration/V112__github_official_discovery_shards.sql new file mode 100644 index 000000000..a917bdf9d --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V112__github_official_discovery_shards.sql @@ -0,0 +1,28 @@ +-- One sequential tree per query. Completed trees are removed when their next root is scheduled; +-- long-lived provenance and confirmed coverage remain on candidate origins and query progress. +CREATE TABLE github_official_discovery_shard ( + id UUID PRIMARY KEY, + query_id UUID NOT NULL REFERENCES github_deep_discovery_query(id), + root_id UUID NOT NULL REFERENCES github_official_discovery_shard(id) ON DELETE CASCADE, + parent_id UUID REFERENCES github_official_discovery_shard(id) ON DELETE CASCADE, + depth INTEGER NOT NULL CHECK (depth BETWEEN 0 AND 32), + range_since TIMESTAMPTZ NOT NULL, + range_until TIMESTAMPTZ NOT NULL, + snapshot_json JSONB NOT NULL CHECK (jsonb_typeof(snapshot_json) = 'object'), + state VARCHAR(16) NOT NULL CHECK (state IN ('PENDING', 'ACTIVE', 'SPLIT', 'COMPLETE', 'GAP')), + task_id UUID UNIQUE REFERENCES github_sync_task(id) ON DELETE SET NULL, + retry_at TIMESTAMPTZ, + gap_reason VARCHAR(64), + gap_checkpoint VARCHAR(4096), + last_lease_generation BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + CHECK (range_since < range_until), + CHECK (date_trunc('second', range_since) = range_since AND date_trunc('second', range_until) = range_until), + CHECK ((parent_id IS NULL AND root_id = id AND depth = 0) OR (parent_id IS NOT NULL AND root_id <> id AND depth > 0)) +); +CREATE UNIQUE INDEX uq_github_official_shard_open_root ON github_official_discovery_shard (query_id) + WHERE parent_id IS NULL AND state <> 'COMPLETE'; +CREATE INDEX idx_github_official_shard_pending ON github_official_discovery_shard (query_id, range_since, depth) + WHERE state = 'PENDING'; +CREATE INDEX idx_github_official_shard_root ON github_official_discovery_shard (root_id); diff --git a/server/skillhub-app/src/main/resources/db/migration/V113__github_official_owner_expansion.sql b/server/skillhub-app/src/main/resources/db/migration/V113__github_official_owner_expansion.sql new file mode 100644 index 000000000..f3227c8a7 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V113__github_official_owner_expansion.sql @@ -0,0 +1,33 @@ +-- Stable GitHub account identity is distinct from SkillHub's own owner identifiers. +ALTER TABLE github_discovery_candidate ADD COLUMN github_owner_id BIGINT CHECK (github_owner_id > 0); +ALTER TABLE github_source_binding ADD COLUMN github_owner_id BIGINT CHECK (github_owner_id > 0); + +-- Validation evidence survives ingestion/task history cleanup and later independent discovery. +CREATE TABLE github_official_validated_seed ( + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + github_owner_id BIGINT NOT NULL CHECK (github_owner_id > 0), + owner_login VARCHAR(39) NOT NULL, + ingestion_id UUID NOT NULL, + validated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (namespace_id, repository_id, github_owner_id) +); +ALTER TABLE github_deep_discovery_query + ADD COLUMN query_source VARCHAR(16) NOT NULL DEFAULT 'CATALOG' CHECK (query_source IN ('CATALOG', 'OWNER')), + ADD COLUMN github_owner_id BIGINT CHECK (github_owner_id > 0), + ADD COLUMN owner_login VARCHAR(39), + ADD COLUMN seed_repository_id BIGINT, + ADD CONSTRAINT ck_github_official_owner_query CHECK ( + query_source = 'CATALOG' OR (github_owner_id IS NOT NULL AND owner_login IS NOT NULL AND seed_repository_id IS NOT NULL AND seed_repository_id > 0)); +CREATE UNIQUE INDEX uq_github_official_owner_query ON github_deep_discovery_query(namespace_id, github_owner_id) + WHERE query_source = 'OWNER'; + +-- Refreshable account coordinates never change the stable account identity of an owner query. +CREATE VIEW github_official_owner_eligibility AS +SELECT query.id AS query_id, + EXISTS (SELECT 1 FROM jsonb_array_elements(config.exclusions_json) exclusion + WHERE exclusion ->> 'kind' = 'OWNER' + AND lower(exclusion ->> 'value') = lower(query.owner_login)) AS excluded +FROM github_deep_discovery_query query +JOIN github_deep_discovery_config config ON config.namespace_id = query.namespace_id +WHERE query.query_source = 'OWNER'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V114__github_sync_worker_instances.sql b/server/skillhub-app/src/main/resources/db/migration/V114__github_sync_worker_instances.sql new file mode 100644 index 000000000..c01e4c3fb --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V114__github_sync_worker_instances.sql @@ -0,0 +1,8 @@ +-- Legacy presence remains a contact signal only; capabilities belong to a process instance. +CREATE TABLE github_sync_worker_instance ( + worker_id VARCHAR(128) PRIMARY KEY, + protocol_major INTEGER NOT NULL CHECK (protocol_major = 1), + capabilities TEXT[] NOT NULL CHECK (cardinality(capabilities) <= 32), + last_seen_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX idx_github_sync_worker_instance_seen ON github_sync_worker_instance (last_seen_at, worker_id); diff --git a/server/skillhub-app/src/main/resources/db/migration/V115__github_discovery_outcomes.sql b/server/skillhub-app/src/main/resources/db/migration/V115__github_discovery_outcomes.sql new file mode 100644 index 000000000..bacc70ff8 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V115__github_discovery_outcomes.sql @@ -0,0 +1,64 @@ +-- Long-lived business facts deliberately have no foreign key to disposable task history. +CREATE TABLE github_discovery_metrics_epoch ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + available_since TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); +INSERT INTO github_discovery_metrics_epoch (id) VALUES (TRUE); +CREATE TABLE github_discovery_outcome_event ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + namespace_id BIGINT NOT NULL, + repository_id BIGINT NOT NULL, + event_type VARCHAR(32) NOT NULL CHECK (event_type IN + ('CANDIDATE_CREATED','FIRST_VALID','NEW_SKILL','UPDATED_VERSION')), + subject_key BIGINT NOT NULL, + occurred_at TIMESTAMPTZ, + official_at_event BOOLEAN NOT NULL, + first_occurrence_known BOOLEAN NOT NULL DEFAULT TRUE, + query_ids UUID[] NOT NULL DEFAULT '{}', + UNIQUE (namespace_id, event_type, subject_key) +); +-- Historical markers prevent a later official hit from becoming a new discovery. +-- NULL timestamps/false attribution explicitly carry no historical metric claim. +INSERT INTO github_discovery_outcome_event + (namespace_id, repository_id, event_type, subject_key, official_at_event) +SELECT namespace_id, repository_id, 'CANDIDATE_CREATED', repository_id, FALSE +FROM github_discovery_candidate; +-- Existing validated packages and published bindings are factual prior validation evidence. +INSERT INTO github_discovery_outcome_event + (namespace_id, repository_id, event_type, subject_key, official_at_event) +SELECT namespace_id, repository_id, 'FIRST_VALID', repository_id, FALSE FROM ( + SELECT namespace_id, repository_id FROM github_sync_package_ingestion WHERE validated_at IS NOT NULL + UNION SELECT namespace_id, repository_id FROM github_source_skill_binding + UNION SELECT namespace_id, repository_id FROM github_official_validated_seed +) prior_validation ON CONFLICT DO NOTHING; +-- Missing old evidence cannot prove whether the next validation is the first-ever one. +CREATE TABLE github_discovery_legacy_unknown_validation ( + namespace_id BIGINT NOT NULL, + repository_id BIGINT NOT NULL, + PRIMARY KEY (namespace_id, repository_id) +); +INSERT INTO github_discovery_legacy_unknown_validation +SELECT c.namespace_id,c.repository_id FROM ( + SELECT namespace_id,repository_id FROM github_discovery_candidate + UNION SELECT namespace_id,repository_id FROM github_source_binding +) c +WHERE NOT EXISTS(SELECT 1 FROM github_discovery_outcome_event e + WHERE e.namespace_id=c.namespace_id AND e.repository_id=c.repository_id AND e.event_type='FIRST_VALID'); +CREATE INDEX idx_discovery_outcome_recent ON github_discovery_outcome_event + (namespace_id, occurred_at, event_type) WHERE official_at_event; +ALTER TABLE github_discovery_candidate_origin + ADD COLUMN id UUID NOT NULL DEFAULT gen_random_uuid(), + ADD COLUMN recent_shard_id UUID, + ADD COLUMN recent_run_id UUID, + ADD COLUMN recent_seed_repository_id BIGINT, + ADD COLUMN latest_strategy_version VARCHAR(128); +CREATE UNIQUE INDEX idx_discovery_origin_id ON github_discovery_candidate_origin(id); +CREATE INDEX idx_discovery_origin_recent ON github_discovery_candidate_origin + (candidate_id, last_seen_at DESC, id DESC) WHERE origin_type='OFFICIAL_DISCOVERY'; +CREATE INDEX idx_discovery_origin_query ON github_discovery_candidate_origin + (origin_key, candidate_id) WHERE origin_type='OFFICIAL_DISCOVERY'; +CREATE INDEX idx_discovery_candidate_all_recent ON github_discovery_candidate + (namespace_id, first_seen_at DESC, id DESC); +CREATE INDEX idx_github_task_repository_recent ON github_sync_task + (namespace_id, (snapshot_json->>'repositoryId'), created_at DESC, id DESC) + WHERE kind='REPOSITORY_SYNC'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V116__github_official_quota_control.sql b/server/skillhub-app/src/main/resources/db/migration/V116__github_official_quota_control.sql new file mode 100644 index 000000000..8be72a546 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V116__github_official_quota_control.sql @@ -0,0 +1,23 @@ +-- Observations are not a reservation ledger. Unknown account identities share conservative gates. +CREATE TABLE github_official_quota_observation ( + observation_key VARCHAR(64) NOT NULL, + resource VARCHAR(16) NOT NULL CHECK (resource IN ('CORE','SEARCH','GRAPHQL','OTHER')), + credential_id UUID REFERENCES github_system_credential(id) ON DELETE CASCADE, + github_subject_id BIGINT, + state VARCHAR(32) NOT NULL CHECK (state IN ('OBSERVED','UNKNOWN','PRIMARY_EXHAUSTED','SECONDARY_COOLDOWN','AUTH_INVALID')), + remaining INTEGER CHECK (remaining >= 0), + reset_at TIMESTAMPTZ, + observed_at TIMESTAMPTZ NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + valid_until TIMESTAMPTZ NOT NULL, + PRIMARY KEY (observation_key, resource) +); +CREATE TABLE github_official_resource_cooldown ( + resource VARCHAR(16) PRIMARY KEY CHECK (resource IN ('CORE','SEARCH','GRAPHQL','OTHER')), + primary_until TIMESTAMPTZ NOT NULL +); +CREATE TABLE github_official_secondary_cooldown ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + cooldown_until TIMESTAMPTZ +); +INSERT INTO github_official_secondary_cooldown (id) VALUES (TRUE); diff --git a/server/skillhub-app/src/main/resources/db/migration/V117__github_official_discovery_budget.sql b/server/skillhub-app/src/main/resources/db/migration/V117__github_official_discovery_budget.sql new file mode 100644 index 000000000..4ac1c8369 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V117__github_official_discovery_budget.sql @@ -0,0 +1,18 @@ +-- One deployment-wide pacing gate. No accumulated idle credit and no refund after a grant. +CREATE TABLE github_official_discovery_budget ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + next_lease_eligible_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +INSERT INTO github_official_discovery_budget(id) VALUES(TRUE); +CREATE TABLE github_official_discovery_reservation ( + task_id UUID NOT NULL REFERENCES github_sync_task(id) ON DELETE CASCADE, + lease_generation BIGINT NOT NULL, + budget_profile VARCHAR(16) NOT NULL, + reserved_requests INTEGER NOT NULL, + granted_at TIMESTAMPTZ NOT NULL, + next_lease_eligible_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY(task_id, lease_generation) +); +ALTER TABLE github_deep_discovery_query ADD COLUMN last_successful_claim_at TIMESTAMPTZ; +CREATE INDEX idx_github_official_active_lease ON github_sync_task(lease_expires_at) + WHERE kind='DISCOVERY' AND official_query_id IS NOT NULL AND status='LEASED'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V118__github_candidate_eligibility_projection.sql b/server/skillhub-app/src/main/resources/db/migration/V118__github_candidate_eligibility_projection.sql new file mode 100644 index 000000000..35f99dd67 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V118__github_candidate_eligibility_projection.sql @@ -0,0 +1,35 @@ +-- Reuse the candidate already read by origin_state when evaluating exclusions. +-- Preserve the public view columns and all live source/entry/configuration policy. +CREATE OR REPLACE VIEW github_candidate_discovery_eligibility AS +WITH origin_state AS ( +SELECT candidate.id AS candidate_id, candidate.namespace_id, candidate.repository_id, + candidate.owner_login, candidate.repository_name, + source.id AS source_id, source.status AS source_status, + EXISTS (SELECT 1 FROM github_discovery_candidate_origin origin + WHERE origin.candidate_id = candidate.id AND origin.origin_type = 'OFFICIAL_DISCOVERY') AS has_official, + EXISTS ( + SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_discovery_entry entry ON entry.namespace_id = candidate.namespace_id AND entry.enabled + AND ((origin.origin_type = 'SEARCH_RULE' AND entry.entry_type = 'SEARCH_RULE' + AND origin.origin_key = entry.id::text) + OR (origin.origin_type = 'CURATED_OWNER' AND entry.entry_type = 'CURATED_OWNER' + AND lower(origin.origin_key) = lower(entry.owner_login))) + WHERE origin.candidate_id = candidate.id) AS has_independent +FROM github_discovery_candidate candidate +LEFT JOIN github_source_binding source + ON source.namespace_id = candidate.namespace_id AND source.repository_id = candidate.repository_id +), controls AS ( + SELECT origin_state.*, COALESCE(config.enabled, FALSE) AS configuration_enabled, + EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(config.exclusions_json, '[]'::jsonb)) exclusion + WHERE (exclusion ->> 'kind' = 'OWNER' AND exclusion ->> 'value' = lower(origin_state.owner_login)) + OR (exclusion ->> 'kind' = 'REPOSITORY' + AND exclusion ->> 'value' = lower(origin_state.owner_login || '/' || origin_state.repository_name))) AS excluded + FROM origin_state LEFT JOIN github_deep_discovery_config config ON config.namespace_id = origin_state.namespace_id +) +SELECT candidate_id, namespace_id, repository_id, source_id, source_status, has_official, has_independent, + has_official AND NOT has_independent AND source_id IS NULL AS official_only, + configuration_enabled, excluded, + CASE WHEN source_id IS NOT NULL THEN source_status <> 'PAUSED' + WHEN NOT has_official OR has_independent THEN TRUE + ELSE configuration_enabled AND NOT excluded END AS acquisition_allowed +FROM controls; diff --git a/server/skillhub-app/src/main/resources/db/migration/V119__github_platform_collection_control.sql b/server/skillhub-app/src/main/resources/db/migration/V119__github_platform_collection_control.sql new file mode 100644 index 000000000..81e81fad6 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V119__github_platform_collection_control.sql @@ -0,0 +1,15 @@ +CREATE TABLE github_platform_collection_control ( + namespace_id BIGINT PRIMARY KEY REFERENCES namespace(id), + paused BOOLEAN NOT NULL DEFAULT FALSE, + version BIGINT NOT NULL DEFAULT 1, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(128) +); +INSERT INTO github_platform_collection_control(namespace_id) +SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'; + +ALTER TABLE github_sync_task + ADD COLUMN platform_pause_version BIGINT, + ADD COLUMN platform_pause_generation BIGINT, + ADD COLUMN platform_pause_token_hash CHAR(64), + ADD COLUMN platform_stop_ack_at TIMESTAMPTZ; diff --git a/server/skillhub-app/src/main/resources/db/migration/V76__github_catalog_namespace.sql b/server/skillhub-app/src/main/resources/db/migration/V76__github_catalog_namespace.sql new file mode 100644 index 000000000..e23a680f3 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V76__github_catalog_namespace.sql @@ -0,0 +1,22 @@ +-- The catalog slug and actor are reserved. A collision must be handled explicitly by an +-- administrator; this migration must never adopt or overwrite an existing user's data. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM namespace WHERE slug = 'github') THEN + RAISE EXCEPTION 'Cannot create @github catalog: namespace slug github already exists'; + END IF; + IF EXISTS (SELECT 1 FROM user_account WHERE id = 'github-sync-publisher') THEN + RAISE EXCEPTION 'Cannot create GitHub catalog actor: user id github-sync-publisher already exists'; + END IF; +END $$; + +INSERT INTO user_account (id, display_name, status, system_account) +VALUES ('github-sync-publisher', 'GitHub Skill Sync', 'ACTIVE', TRUE); + +INSERT INTO namespace (slug, display_name, type, description, status, created_by) +VALUES ('github', 'GitHub Skills', 'CATALOG', 'Public GitHub skill catalog', 'ACTIVE', 'github-sync-publisher'); + +INSERT INTO namespace_member (namespace_id, user_id, role) +SELECT id, 'github-sync-publisher', 'OWNER' +FROM namespace +WHERE slug = 'github'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V77__github_source_bindings.sql b/server/skillhub-app/src/main/resources/db/migration/V77__github_source_bindings.sql new file mode 100644 index 000000000..933a194d4 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V77__github_source_bindings.sql @@ -0,0 +1,36 @@ +CREATE TABLE github_source_binding ( + id BIGSERIAL PRIMARY KEY, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + owner_login VARCHAR(128) NOT NULL, + repository_name VARCHAR(256) NOT NULL, + default_branch VARCHAR(256), + visibility VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'ACTIVE', + last_observed_commit VARCHAR(40), + last_fully_processed_commit VARCHAR(40), + next_check_at TIMESTAMPTZ, + unreachable_attempts INTEGER NOT NULL DEFAULT 0 CHECK (unreachable_attempts >= 0), + last_error_code VARCHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_source_namespace_repository UNIQUE (namespace_id, repository_id), + CONSTRAINT uq_github_source_id_namespace UNIQUE (id, namespace_id), + CONSTRAINT ck_github_source_status CHECK (status IN ('ACTIVE', 'PAUSED', 'UNREACHABLE')), + CONSTRAINT ck_github_source_visibility CHECK (visibility IN ('PUBLIC', 'NAMESPACE_ONLY')) +); + +CREATE INDEX idx_github_source_due + ON github_source_binding (next_check_at, id) + WHERE status IN ('ACTIVE', 'UNREACHABLE'); + +CREATE TABLE github_source_origin ( + id BIGSERIAL PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES github_source_binding(id), + origin_type VARCHAR(32) NOT NULL, + origin_key VARCHAR(256) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_source_origin UNIQUE (source_id, origin_type, origin_key), + CONSTRAINT ck_github_source_origin_type CHECK + (origin_type IN ('SEARCH_RULE', 'CURATED_REPOSITORY', 'CURATED_OWNER', 'TEAM_CONFIG')) +); diff --git a/server/skillhub-app/src/main/resources/db/migration/V78__github_sync_tasks.sql b/server/skillhub-app/src/main/resources/db/migration/V78__github_sync_tasks.sql new file mode 100644 index 000000000..12bbfba8f --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V78__github_sync_tasks.sql @@ -0,0 +1,77 @@ +CREATE TABLE github_sync_run ( + id UUID PRIMARY KEY, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + kind VARCHAR(24) NOT NULL CHECK (kind IN ('DISCOVERY', 'REPOSITORY_SYNC')), + status VARCHAR(24) NOT NULL DEFAULT 'QUEUED', + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMPTZ, + CONSTRAINT uq_github_sync_run_id_namespace UNIQUE (id, namespace_id), + CONSTRAINT ck_github_sync_run_status CHECK + (status IN ('QUEUED', 'LEASED', 'RETRY_WAIT', 'PARTIAL', 'ACQUISITION_COMPLETE', 'FAILED')) +); + +CREATE TABLE github_sync_task ( + id UUID PRIMARY KEY, + run_id UUID NOT NULL, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + source_id BIGINT, + kind VARCHAR(24) NOT NULL CHECK (kind IN ('DISCOVERY', 'REPOSITORY_SYNC')), + mode VARCHAR(16) CHECK (mode IN ('FULL', 'HEAD_ONLY')), + status VARCHAR(24) NOT NULL DEFAULT 'QUEUED', + dedupe_key VARCHAR(256) NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + snapshot_json JSONB NOT NULL, + due_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + retry_at TIMESTAMPTZ, + lease_generation BIGINT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_token_hash CHAR(64), + lease_expires_at TIMESTAMPTZ, + leased_by VARCHAR(128), + fixed_commit_sha VARCHAR(40), + path_checkpoint VARCHAR(4096), + incomplete_subtrees JSONB NOT NULL DEFAULT '[]'::jsonb, + discovery_checkpoint VARCHAR(4096), + last_error_code VARCHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + CONSTRAINT ck_github_sync_task_status CHECK + (status IN ('QUEUED', 'LEASED', 'RETRY_WAIT', 'PARTIAL', 'ACQUISITION_COMPLETE', 'FAILED')), + CONSTRAINT ck_github_sync_task_snapshot CHECK (jsonb_typeof(snapshot_json) = 'object'), + CONSTRAINT ck_github_sync_task_incomplete_subtrees CHECK (jsonb_typeof(incomplete_subtrees) = 'array'), + CONSTRAINT ck_github_sync_task_dedupe_key CHECK (length(dedupe_key) > 0), + CONSTRAINT ck_github_sync_task_shape CHECK + ((kind = 'DISCOVERY' AND mode IS NULL AND source_id IS NULL) + OR (kind = 'REPOSITORY_SYNC' AND mode IS NOT NULL)), + CONSTRAINT fk_github_sync_task_run_namespace FOREIGN KEY (run_id, namespace_id) + REFERENCES github_sync_run(id, namespace_id), + CONSTRAINT fk_github_sync_task_source_namespace FOREIGN KEY (source_id, namespace_id) + REFERENCES github_source_binding(id, namespace_id), + CONSTRAINT ck_github_sync_task_lease CHECK + ((status = 'LEASED' AND lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND leased_by IS NOT NULL) + OR (status <> 'LEASED' AND lease_token_hash IS NULL AND lease_expires_at IS NULL AND leased_by IS NULL)) +); + +CREATE UNIQUE INDEX uq_github_sync_task_open_work + ON github_sync_task (namespace_id, dedupe_key) + WHERE status IN ('QUEUED', 'LEASED', 'RETRY_WAIT'); + +CREATE INDEX idx_github_sync_task_claim + ON github_sync_task (priority DESC, due_at, id) + WHERE status IN ('QUEUED', 'LEASED', 'RETRY_WAIT'); + +CREATE TABLE github_sync_path_result ( + id BIGSERIAL PRIMARY KEY, + task_id UUID NOT NULL REFERENCES github_sync_task(id), + commit_sha VARCHAR(40) NOT NULL, + skill_path VARCHAR(1024) NOT NULL, + outcome VARCHAR(32) NOT NULL CHECK + (outcome IN ('ACCEPTED', 'UNCHANGED', 'IGNORED', 'RETRYABLE_FAILURE', 'TERMINAL_FAILURE')), + ingestion_id UUID, + sha256 CHAR(64), + reason VARCHAR(500), + retry_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_sync_path_result UNIQUE (task_id, commit_sha, skill_path) +); diff --git a/server/skillhub-app/src/main/resources/db/migration/V79__github_discovery_candidates.sql b/server/skillhub-app/src/main/resources/db/migration/V79__github_discovery_candidates.sql new file mode 100644 index 000000000..bfb033408 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V79__github_discovery_candidates.sql @@ -0,0 +1,38 @@ +CREATE TABLE github_discovery_candidate ( + id BIGSERIAL PRIMARY KEY, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + owner_login VARCHAR(128) NOT NULL, + repository_name VARCHAR(256) NOT NULL, + default_branch VARCHAR(256), + status VARCHAR(20) NOT NULL DEFAULT 'PENDING', + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_discovery_candidate UNIQUE (namespace_id, repository_id), + CONSTRAINT ck_github_discovery_candidate_status CHECK + (status IN ('PENDING', 'VALIDATING', 'BOUND', 'REJECTED', 'UNREACHABLE')) +); + +CREATE INDEX idx_github_discovery_candidate_pending + ON github_discovery_candidate (namespace_id, first_seen_at, id) + WHERE status = 'PENDING'; + +CREATE TABLE github_discovery_task_hit ( + task_id UUID NOT NULL REFERENCES github_sync_task(id) ON DELETE CASCADE, + repository_id BIGINT NOT NULL, + candidate_id BIGINT NOT NULL REFERENCES github_discovery_candidate(id), + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (task_id, repository_id) +); + +CREATE TABLE github_discovery_batch ( + task_id UUID NOT NULL REFERENCES github_sync_task(id) ON DELETE CASCADE, + checkpoint VARCHAR(4096) NOT NULL, + request_sha256 CHAR(64) NOT NULL, + deferred_count INTEGER NOT NULL DEFAULT 0 CHECK (deferred_count >= 0), + deferral_reason VARCHAR(32), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (task_id, checkpoint), + CONSTRAINT ck_github_discovery_batch_checkpoint CHECK (length(checkpoint) > 0) +); diff --git a/server/skillhub-app/src/main/resources/db/migration/V80__github_package_admissions.sql b/server/skillhub-app/src/main/resources/db/migration/V80__github_package_admissions.sql new file mode 100644 index 000000000..edaaa827c --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V80__github_package_admissions.sql @@ -0,0 +1,46 @@ +-- Watermarks have no guessed defaults: the admin configuration flow must set both pairs +-- before Java may grant package acquisition. +CREATE TABLE github_sync_admission_policy ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + global_high INTEGER, + global_low INTEGER, + namespace_high INTEGER, + namespace_low INTEGER, + global_paused BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_github_sync_global_watermarks CHECK + ((global_high IS NULL AND global_low IS NULL) + OR (global_high IS NOT NULL AND global_low IS NOT NULL + AND global_high > 0 AND global_low >= 0 AND global_low < global_high)), + CONSTRAINT ck_github_sync_namespace_watermarks CHECK + ((namespace_high IS NULL AND namespace_low IS NULL) + OR (namespace_high IS NOT NULL AND namespace_low IS NOT NULL + AND namespace_high > 0 AND namespace_low >= 0 AND namespace_low < namespace_high)) +); + +INSERT INTO github_sync_admission_policy (id) VALUES (TRUE); + +CREATE TABLE github_sync_namespace_admission_state ( + namespace_id BIGINT PRIMARY KEY REFERENCES namespace(id), + paused BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE github_sync_package_admission ( + id UUID PRIMARY KEY, + task_id UUID NOT NULL REFERENCES github_sync_task(id), + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + commit_sha CHAR(40) NOT NULL, + skill_path VARCHAR(1024) NOT NULL, + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE' CHECK (status IN ('ACTIVE', 'RELEASED', 'ACCEPTED')), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_github_package_admission_attempt + UNIQUE (task_id, commit_sha, skill_path, lease_generation) +); + +CREATE INDEX idx_github_package_admission_active_coordinate + ON github_sync_package_admission (namespace_id, repository_id, skill_path) + WHERE status = 'ACTIVE'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V81__github_sync_staged_uploads.sql b/server/skillhub-app/src/main/resources/db/migration/V81__github_sync_staged_uploads.sql new file mode 100644 index 000000000..f566df33e --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V81__github_sync_staged_uploads.sql @@ -0,0 +1,22 @@ +CREATE TABLE github_sync_staged_upload ( + id UUID PRIMARY KEY, + admission_id UUID NOT NULL REFERENCES github_sync_package_admission(id), + object_key VARCHAR(256) NOT NULL UNIQUE, + mode VARCHAR(24) NOT NULL CHECK (mode IN ('LOCAL_STREAM', 'S3_PRESIGNED_PUT')), + ticket_hash CHAR(64), + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + sha256 CHAR(64) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'ISSUED' + CHECK (status IN ('ISSUED', 'UPLOADING', 'UPLOADED', 'FAILED', + 'ACCEPTED', 'DELETING', 'CLEANED')), + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_github_sync_upload_ticket CHECK + ((mode = 'LOCAL_STREAM' AND ticket_hash IS NOT NULL) + OR (mode = 'S3_PRESIGNED_PUT' AND ticket_hash IS NULL)) +); + +CREATE INDEX idx_github_sync_staged_upload_cleanup + ON github_sync_staged_upload (status, expires_at) + WHERE status IN ('ISSUED', 'UPLOADING', 'UPLOADED', 'FAILED', 'DELETING'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V82__github_sync_package_ingestion.sql b/server/skillhub-app/src/main/resources/db/migration/V82__github_sync_package_ingestion.sql new file mode 100644 index 000000000..86327f25c --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V82__github_sync_package_ingestion.sql @@ -0,0 +1,49 @@ +CREATE TABLE github_sync_package_ingestion ( + id UUID PRIMARY KEY, + task_id UUID NOT NULL REFERENCES github_sync_task(id), + admission_id UUID NOT NULL REFERENCES github_sync_package_admission(id), + upload_id UUID NOT NULL UNIQUE REFERENCES github_sync_staged_upload(id), + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + commit_sha CHAR(40) NOT NULL, + skill_path VARCHAR(1024) NOT NULL, + object_key VARCHAR(256) NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + sha256 CHAR(64) NOT NULL, + status VARCHAR(24) NOT NULL DEFAULT 'RECEIVED' CHECK + (status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', 'READY_FOR_REVIEW', + 'VALIDATED', 'NO_CHANGE', 'BLOCKED', 'FAILED', + 'APPROVED', 'REJECTED', 'SUPERSEDED')), + validation_token UUID, + validation_expires_at TIMESTAMPTZ, + validation_attempts INTEGER NOT NULL DEFAULT 0 CHECK (validation_attempts >= 0), + validated_at TIMESTAMPTZ, + content_sha256 CHAR(64), + license_evidence_json JSONB NOT NULL DEFAULT '{}'::jsonb, + validation_warning_count INTEGER NOT NULL DEFAULT 0 CHECK (validation_warning_count >= 0), + object_cleanup_claimed_at TIMESTAMPTZ, + object_cleaned_at TIMESTAMPTZ, + last_error_code VARCHAR(64), + retry_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_github_sync_license_evidence_json CHECK (jsonb_typeof(license_evidence_json) = 'object'), + CONSTRAINT ck_github_sync_ingestion_validation_lease CHECK + ((status = 'VALIDATING' AND validation_token IS NOT NULL AND validation_expires_at IS NOT NULL) + OR (status <> 'VALIDATING' AND validation_token IS NULL AND validation_expires_at IS NULL)), + CONSTRAINT uq_github_sync_ingestion_path UNIQUE (task_id, commit_sha, skill_path) +); + +CREATE INDEX idx_github_sync_ingestion_unresolved_coordinate + ON github_sync_package_ingestion (namespace_id, repository_id, skill_path) + WHERE status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', 'VALIDATED', 'READY_FOR_REVIEW', 'BLOCKED'); + +CREATE INDEX idx_github_sync_ingestion_validation_claim + ON github_sync_package_ingestion (created_at, id) + WHERE status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT'); + +CREATE INDEX idx_github_sync_ingestion_object_cleanup + ON github_sync_package_ingestion (updated_at, id) + WHERE status IN ('FAILED', 'NO_CHANGE', 'READY_FOR_REVIEW', + 'APPROVED', 'REJECTED', 'SUPERSEDED') + AND object_cleaned_at IS NULL; diff --git a/server/skillhub-app/src/main/resources/db/migration/V83__github_sync_review_origin.sql b/server/skillhub-app/src/main/resources/db/migration/V83__github_sync_review_origin.sql new file mode 100644 index 000000000..446ebe628 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V83__github_sync_review_origin.sql @@ -0,0 +1,18 @@ +ALTER TABLE review_task + ADD COLUMN origin_type VARCHAR(24) NOT NULL DEFAULT 'USER', + ADD COLUMN github_sync_ingestion_id UUID REFERENCES github_sync_package_ingestion(id); + +ALTER TABLE review_task + ADD CONSTRAINT ck_review_task_origin_type + CHECK (origin_type IN ('USER', 'GITHUB_SYNC')), + ADD CONSTRAINT ck_review_task_github_sync_origin + CHECK ((origin_type = 'USER' AND github_sync_ingestion_id IS NULL) + OR (origin_type = 'GITHUB_SYNC' AND github_sync_ingestion_id IS NOT NULL + AND subject_type = 'SKILL_VERSION')); + +CREATE UNIQUE INDEX uq_review_task_github_sync_ingestion + ON review_task (github_sync_ingestion_id) + WHERE github_sync_ingestion_id IS NOT NULL; + +CREATE INDEX idx_review_task_origin_status + ON review_task (origin_type, status, submitted_at DESC, id DESC); diff --git a/server/skillhub-app/src/main/resources/db/migration/V84__github_discovery_entries.sql b/server/skillhub-app/src/main/resources/db/migration/V84__github_discovery_entries.sql new file mode 100644 index 000000000..e17ffd71a --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V84__github_discovery_entries.sql @@ -0,0 +1,44 @@ +CREATE TABLE github_discovery_entry ( + id UUID PRIMARY KEY, + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + entry_type VARCHAR(24) NOT NULL, + title VARCHAR(128) NOT NULL, + owner_login VARCHAR(128), + criteria_json JSONB, + rule_version BIGINT NOT NULL DEFAULT 1 CHECK (rule_version > 0), + enabled BOOLEAN NOT NULL DEFAULT FALSE, + target_interval_seconds INTEGER NOT NULL CHECK + (target_interval_seconds BETWEEN 3600 AND 2592000), + next_due_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_github_discovery_entry_type CHECK + (entry_type IN ('SEARCH_RULE', 'CURATED_OWNER')), + CONSTRAINT ck_github_discovery_entry_shape CHECK + ((entry_type = 'SEARCH_RULE' AND owner_login IS NULL + AND jsonb_typeof(criteria_json) = 'object') + OR (entry_type = 'CURATED_OWNER' AND owner_login IS NOT NULL + AND criteria_json IS NULL)) +); + +CREATE UNIQUE INDEX uq_github_discovery_rule_title + ON github_discovery_entry (namespace_id, lower(title)) + WHERE entry_type = 'SEARCH_RULE'; + +CREATE UNIQUE INDEX uq_github_discovery_curated_owner + ON github_discovery_entry (namespace_id, lower(owner_login)) + WHERE entry_type = 'CURATED_OWNER'; + +CREATE INDEX idx_github_discovery_entry_due + ON github_discovery_entry (next_due_at, id) + WHERE enabled = TRUE; + +-- Initial search terms are only an editable starting point. Upgrade must not trigger discovery. +INSERT INTO github_discovery_entry + (id, namespace_id, entry_type, title, criteria_json, enabled, target_interval_seconds) +SELECT '82c4e916-42f1-4a7b-9c30-116d5faac201'::uuid, id, + 'SEARCH_RULE', '公开 Agent Skill 仓库', + '{"selectors":[{"kind":"KEYWORD","value":"agent skills","priority":"CORE"}, + {"kind":"TOPIC","value":"agent-skills","priority":"EXTENSION"}]}'::jsonb, + FALSE, 259200 +FROM namespace WHERE slug = 'github' AND type = 'CATALOG'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V85__github_discovery_budget.sql b/server/skillhub-app/src/main/resources/db/migration/V85__github_discovery_budget.sql new file mode 100644 index 000000000..a675b8eb9 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V85__github_discovery_budget.sql @@ -0,0 +1,5 @@ +ALTER TABLE github_sync_admission_policy + ADD COLUMN candidate_pending_limit INTEGER NOT NULL DEFAULT 1000 + CHECK (candidate_pending_limit BETWEEN 1 AND 100000), + ADD COLUMN candidate_per_lease_limit INTEGER NOT NULL DEFAULT 100 + CHECK (candidate_per_lease_limit BETWEEN 1 AND 10000); diff --git a/server/skillhub-app/src/main/resources/db/migration/V86__github_sync_run_status.sql b/server/skillhub-app/src/main/resources/db/migration/V86__github_sync_run_status.sql new file mode 100644 index 000000000..6473c146d --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V86__github_sync_run_status.sql @@ -0,0 +1,32 @@ +-- A run may contain more than one repository task. Keep its visible status in sync +-- with the persisted task states, including retries and partial completion. +CREATE FUNCTION refresh_github_sync_run_status() RETURNS trigger AS $$ +BEGIN + -- Serialize concurrent task updates for the same run before reading siblings. + PERFORM 1 FROM github_sync_run WHERE id = NEW.run_id FOR NO KEY UPDATE; + + UPDATE github_sync_run AS run + SET status = summary.status, + finished_at = CASE WHEN summary.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + THEN COALESCE(run.finished_at, CURRENT_TIMESTAMP) ELSE NULL END + FROM ( + SELECT CASE + WHEN bool_or(status = 'LEASED') THEN 'LEASED' + WHEN bool_or(status = 'QUEUED') THEN 'QUEUED' + WHEN bool_or(status = 'RETRY_WAIT') THEN 'RETRY_WAIT' + WHEN bool_or(status = 'PARTIAL') + OR (bool_or(status = 'FAILED') AND bool_or(status = 'ACQUISITION_COMPLETE')) + THEN 'PARTIAL' + WHEN bool_and(status = 'FAILED') THEN 'FAILED' + ELSE 'ACQUISITION_COMPLETE' + END AS status + FROM github_sync_task WHERE run_id = NEW.run_id + ) AS summary + WHERE run.id = NEW.run_id; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER github_sync_run_status_after_task + AFTER INSERT OR UPDATE OF status ON github_sync_task + FOR EACH ROW EXECUTE FUNCTION refresh_github_sync_run_status(); diff --git a/server/skillhub-app/src/main/resources/db/migration/V87__github_candidate_validation_tasks.sql b/server/skillhub-app/src/main/resources/db/migration/V87__github_candidate_validation_tasks.sql new file mode 100644 index 000000000..23e7984b3 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V87__github_candidate_validation_tasks.sql @@ -0,0 +1,16 @@ +ALTER TABLE github_discovery_candidate + ADD COLUMN validation_task_id UUID REFERENCES github_sync_task(id), + ADD COLUMN next_validation_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + ADD COLUMN unreachable_attempts INTEGER NOT NULL DEFAULT 0 + CHECK (unreachable_attempts >= 0), + ADD COLUMN last_error_code VARCHAR(64), + ADD CONSTRAINT ck_github_candidate_validation_task + CHECK (status <> 'VALIDATING' OR validation_task_id IS NOT NULL); + +CREATE UNIQUE INDEX uq_github_candidate_validation_task + ON github_discovery_candidate (validation_task_id) + WHERE validation_task_id IS NOT NULL; + +CREATE INDEX idx_github_candidate_validation_due + ON github_discovery_candidate (next_validation_at, id) + WHERE status IN ('PENDING', 'REJECTED'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V88__github_source_scheduler_index.sql b/server/skillhub-app/src/main/resources/db/migration/V88__github_source_scheduler_index.sql new file mode 100644 index 000000000..3fbd66526 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V88__github_source_scheduler_index.sql @@ -0,0 +1,3 @@ +CREATE INDEX idx_github_sync_task_source_recent + ON github_sync_task (source_id, completed_at DESC, id DESC) + WHERE source_id IS NOT NULL AND status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V89__github_sync_publication.sql b/server/skillhub-app/src/main/resources/db/migration/V89__github_sync_publication.sql new file mode 100644 index 000000000..e811d72cd --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V89__github_sync_publication.sql @@ -0,0 +1,46 @@ +-- Each GitHub Skill path has one namespace-local Skill identity. Publication work is leased +-- separately from archive validation so a restarted Java instance can resume it. +ALTER TABLE github_source_binding + ADD CONSTRAINT uq_github_source_full_coordinate UNIQUE (id, namespace_id, repository_id); + +ALTER TABLE skill_version + ADD COLUMN github_sync_ingestion_id UUID UNIQUE REFERENCES github_sync_package_ingestion(id); + +ALTER TABLE skill + ADD COLUMN github_sync_managed BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE TABLE github_source_skill_binding ( + id BIGSERIAL PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES github_source_binding(id), + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + repository_id BIGINT NOT NULL CHECK (repository_id > 0), + skill_path VARCHAR(1024) NOT NULL, + skill_id BIGINT NOT NULL REFERENCES skill(id), + latest_content_sha256 CHAR(64) NOT NULL, + latest_ingestion_id UUID NOT NULL REFERENCES github_sync_package_ingestion(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT fk_github_skill_binding_source_coordinate FOREIGN KEY + (source_id, namespace_id, repository_id) + REFERENCES github_source_binding(id, namespace_id, repository_id), + CONSTRAINT uq_github_skill_binding_path UNIQUE (namespace_id, repository_id, skill_path), + CONSTRAINT uq_github_skill_binding_skill UNIQUE (namespace_id, skill_id) +); + +CREATE INDEX idx_github_skill_binding_content + ON github_source_skill_binding (source_id, latest_content_sha256); + +ALTER TABLE github_sync_package_ingestion + ADD COLUMN publication_token UUID, + ADD COLUMN publication_expires_at TIMESTAMPTZ, + ADD COLUMN publication_retry_at TIMESTAMPTZ, + ADD COLUMN publication_attempts INTEGER NOT NULL DEFAULT 0 CHECK (publication_attempts >= 0), + ADD COLUMN published_skill_id BIGINT REFERENCES skill(id), + ADD COLUMN published_version_id BIGINT REFERENCES skill_version(id), + ADD CONSTRAINT ck_github_sync_publication_lease CHECK + ((publication_token IS NULL AND publication_expires_at IS NULL) + OR (publication_token IS NOT NULL AND publication_expires_at IS NOT NULL)); + +CREATE INDEX idx_github_sync_publication_claim + ON github_sync_package_ingestion (publication_retry_at, validated_at, id) + WHERE status = 'VALIDATED'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V90__github_curated_repository_entries.sql b/server/skillhub-app/src/main/resources/db/migration/V90__github_curated_repository_entries.sql new file mode 100644 index 000000000..f323746d7 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V90__github_curated_repository_entries.sql @@ -0,0 +1,27 @@ +ALTER TABLE github_discovery_entry + ADD COLUMN repository_name VARCHAR(100), + ADD COLUMN repository_id BIGINT CHECK (repository_id > 0); + +ALTER TABLE github_discovery_entry + DROP CONSTRAINT ck_github_discovery_entry_type, + ADD CONSTRAINT ck_github_discovery_entry_type CHECK + (entry_type IN ('SEARCH_RULE', 'CURATED_OWNER', 'CURATED_REPOSITORY')), + DROP CONSTRAINT ck_github_discovery_entry_shape, + ADD CONSTRAINT ck_github_discovery_entry_shape CHECK + ((entry_type = 'SEARCH_RULE' AND owner_login IS NULL AND repository_name IS NULL + AND jsonb_typeof(criteria_json) = 'object') + OR (entry_type = 'CURATED_OWNER' AND owner_login IS NOT NULL + AND repository_name IS NULL AND criteria_json IS NULL) + OR (entry_type = 'CURATED_REPOSITORY' AND owner_login IS NOT NULL + AND repository_name IS NOT NULL AND criteria_json IS NULL)); + +CREATE UNIQUE INDEX uq_github_discovery_curated_repository + ON github_discovery_entry (namespace_id, lower(owner_login), lower(repository_name)) + WHERE entry_type = 'CURATED_REPOSITORY'; + +ALTER TABLE github_discovery_batch + ADD COLUMN curated_repository_id BIGINT CHECK (curated_repository_id > 0); + +CREATE UNIQUE INDEX uq_github_discovery_batch_curated_repository + ON github_discovery_batch (task_id) + WHERE curated_repository_id IS NOT NULL; diff --git a/server/skillhub-app/src/main/resources/db/migration/V91__github_team_repository_entries.sql b/server/skillhub-app/src/main/resources/db/migration/V91__github_team_repository_entries.sql new file mode 100644 index 000000000..b024a760b --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V91__github_team_repository_entries.sql @@ -0,0 +1,18 @@ +ALTER TABLE github_discovery_entry + ADD COLUMN source_visibility VARCHAR(20) + CHECK (source_visibility IN ('PUBLIC', 'NAMESPACE_ONLY')); + +UPDATE github_discovery_entry SET source_visibility = 'PUBLIC' +WHERE entry_type = 'CURATED_REPOSITORY'; + +ALTER TABLE github_discovery_entry + DROP CONSTRAINT ck_github_discovery_entry_shape, + ADD CONSTRAINT ck_github_discovery_entry_shape CHECK + ((entry_type = 'SEARCH_RULE' AND owner_login IS NULL AND repository_name IS NULL + AND source_visibility IS NULL AND jsonb_typeof(criteria_json) = 'object') + OR (entry_type = 'CURATED_OWNER' AND owner_login IS NOT NULL + AND repository_name IS NULL AND source_visibility IS NULL + AND criteria_json IS NULL) + OR (entry_type = 'CURATED_REPOSITORY' AND owner_login IS NOT NULL + AND repository_name IS NOT NULL AND source_visibility IS NOT NULL + AND criteria_json IS NULL)); diff --git a/server/skillhub-app/src/main/resources/db/migration/V92__github_sync_management_read_indexes.sql b/server/skillhub-app/src/main/resources/db/migration/V92__github_sync_management_read_indexes.sql new file mode 100644 index 000000000..5636c3170 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V92__github_sync_management_read_indexes.sql @@ -0,0 +1,9 @@ +CREATE INDEX idx_github_source_namespace_recent + ON github_source_binding (namespace_id, id DESC); + +CREATE INDEX idx_github_sync_task_source_created + ON github_sync_task (source_id, created_at DESC, id DESC) + WHERE source_id IS NOT NULL; + +CREATE INDEX idx_github_sync_task_run + ON github_sync_task (run_id); diff --git a/server/skillhub-app/src/main/resources/db/migration/V93__github_system_credentials.sql b/server/skillhub-app/src/main/resources/db/migration/V93__github_system_credentials.sql new file mode 100644 index 000000000..c86dc0ecd --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V93__github_system_credentials.sql @@ -0,0 +1,32 @@ +CREATE TABLE github_system_credential ( + id UUID PRIMARY KEY, + label VARCHAR(100) NOT NULL, + fingerprint CHAR(16) NOT NULL UNIQUE, + key_version VARCHAR(40) NOT NULL, + nonce BYTEA NOT NULL CHECK (octet_length(nonce) = 12), + ciphertext BYTEA NOT NULL CHECK (octet_length(ciphertext) >= 17), + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE' + CHECK (status IN ('ACTIVE', 'INVALID', 'DISABLED')), + rate_limited_until TIMESTAMPTZ, + last_used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_github_credential_label CHECK (length(btrim(label)) > 0) +); + +ALTER TABLE github_sync_task + ADD COLUMN credential_id UUID REFERENCES github_system_credential(id), + ADD COLUMN credential_fingerprint CHAR(16); + +CREATE INDEX idx_github_system_credential_available + ON github_system_credential (last_used_at ASC NULLS FIRST, id) + WHERE status = 'ACTIVE'; + +CREATE TABLE github_system_credential_quota ( + credential_id UUID NOT NULL REFERENCES github_system_credential(id) ON DELETE CASCADE, + resource VARCHAR(16) NOT NULL CHECK (resource IN ('CORE', 'SEARCH', 'GRAPHQL', 'OTHER')), + remaining INTEGER NOT NULL CHECK (remaining >= 0), + reset_at TIMESTAMPTZ NOT NULL, + observed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (credential_id, resource) +); diff --git a/server/skillhub-app/src/main/resources/db/migration/V94__github_source_ignore_patterns.sql b/server/skillhub-app/src/main/resources/db/migration/V94__github_source_ignore_patterns.sql new file mode 100644 index 000000000..6c11e75eb --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V94__github_source_ignore_patterns.sql @@ -0,0 +1,4 @@ +ALTER TABLE github_source_binding + ADD COLUMN ignore_patterns JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD CONSTRAINT ck_github_source_ignore_patterns_array + CHECK (jsonb_typeof(ignore_patterns) = 'array'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V95__github_sync_failure_detail.sql b/server/skillhub-app/src/main/resources/db/migration/V95__github_sync_failure_detail.sql new file mode 100644 index 000000000..8d9395335 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V95__github_sync_failure_detail.sql @@ -0,0 +1,6 @@ +ALTER TABLE github_sync_task + ADD COLUMN last_error_detail VARCHAR(100); + +ALTER TABLE github_sync_task + ADD CONSTRAINT ck_github_sync_task_error_detail + CHECK (last_error_detail IS NULL OR last_error_detail ~ '^[A-Z0-9_:]{1,100}$'); diff --git a/server/skillhub-app/src/main/resources/db/migration/V96__github_sync_manual_skill_binding.sql b/server/skillhub-app/src/main/resources/db/migration/V96__github_sync_manual_skill_binding.sql new file mode 100644 index 000000000..e45f56207 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V96__github_sync_manual_skill_binding.sql @@ -0,0 +1,8 @@ +-- An administrator may reserve an existing Skill identity before the blocked package is retried. +-- These pointers become non-null when publication creates the first synced version. +ALTER TABLE github_source_skill_binding + ALTER COLUMN latest_content_sha256 DROP NOT NULL, + ALTER COLUMN latest_ingestion_id DROP NOT NULL; + +ALTER TABLE github_sync_package_ingestion + ADD COLUMN target_skill_slug VARCHAR(128); diff --git a/server/skillhub-app/src/main/resources/db/migration/V97__github_skill_upstream_presence.sql b/server/skillhub-app/src/main/resources/db/migration/V97__github_skill_upstream_presence.sql new file mode 100644 index 000000000..9c90c4c66 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V97__github_skill_upstream_presence.sql @@ -0,0 +1,13 @@ +-- Track what a complete repository scan observed without changing Skill publication state. +ALTER TABLE github_source_skill_binding + ADD COLUMN upstream_presence VARCHAR(16) NOT NULL DEFAULT 'UNKNOWN', + ADD COLUMN presence_commit_sha VARCHAR(40), + ADD COLUMN missing_since TIMESTAMPTZ, + ADD CONSTRAINT ck_github_skill_upstream_presence + CHECK (upstream_presence IN ('UNKNOWN', 'PRESENT', 'MISSING')), + ADD CONSTRAINT ck_github_skill_missing_since + CHECK ((upstream_presence = 'MISSING') = (missing_since IS NOT NULL)); + +CREATE INDEX idx_github_skill_missing_source + ON github_source_skill_binding (source_id, missing_since) + WHERE upstream_presence = 'MISSING'; diff --git a/server/skillhub-app/src/main/resources/db/migration/V98__github_sync_schedule_policy.sql b/server/skillhub-app/src/main/resources/db/migration/V98__github_sync_schedule_policy.sql new file mode 100644 index 000000000..f03270a01 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V98__github_sync_schedule_policy.sql @@ -0,0 +1,10 @@ +CREATE TABLE github_sync_schedule_policy ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + repository_hours INTEGER CHECK (repository_hours BETWEEN 1 AND 720), + curated_owner_hours INTEGER NOT NULL DEFAULT 24 CHECK (curated_owner_hours BETWEEN 1 AND 720), + search_rule_hours INTEGER NOT NULL DEFAULT 72 CHECK (search_rule_hours BETWEEN 1 AND 720), + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- NULL keeps the existing deployment-level repository interval until an admin saves this policy. +INSERT INTO github_sync_schedule_policy (id) VALUES (TRUE); diff --git a/server/skillhub-app/src/main/resources/db/migration/V99__github_sync_claim_fairness.sql b/server/skillhub-app/src/main/resources/db/migration/V99__github_sync_claim_fairness.sql new file mode 100644 index 000000000..74903349e --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V99__github_sync_claim_fairness.sql @@ -0,0 +1,8 @@ +-- Persist the last successful lease per namespace so a large source queue in one +-- namespace cannot indefinitely hide another namespace's due source tasks. +CREATE TABLE github_sync_claim_cursor ( + kind VARCHAR(24) NOT NULL CHECK (kind IN ('DISCOVERY', 'REPOSITORY_SYNC')), + namespace_id BIGINT NOT NULL REFERENCES namespace(id), + last_claimed_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (kind, namespace_id) +); diff --git a/server/skillhub-app/src/main/resources/messages.properties b/server/skillhub-app/src/main/resources/messages.properties index 575e95b91..56bf53a85 100644 --- a/server/skillhub-app/src/main/resources/messages.properties +++ b/server/skillhub-app/src/main/resources/messages.properties @@ -62,6 +62,35 @@ error.auth.sessionBootstrap.providerUnsupported=Unsupported session bootstrap pr error.auth.sessionBootstrap.notAuthenticated=No authenticated external session found error.badRequest=Invalid request error.notFound=Resource not found +error.githubSync.admissionPolicy.invalid=GitHub sync watermarks must satisfy 0 <= low < high; candidate limits must be positive and the per-lease limit must not exceed the pending limit +error.githubSync.discoveryEntry.duplicate=GitHub discovery entry already exists +error.githubSync.discoveryEntry.notFound=GitHub discovery entry not found +error.githubSync.discoveryEntry.type=This discovery entry is not a search rule +error.githubSync.discoveryEntry.owner=Invalid GitHub user or organization name +error.githubSync.discoveryEntry.repository=Invalid public GitHub repository coordinate +error.githubSync.teamSource.forbidden=Only namespace owners and admins can manage GitHub sources +error.githubSync.teamSource.visibility=Source visibility must be PUBLIC or NAMESPACE_ONLY +error.githubSync.source.notFound=GitHub source not found +error.githubSync.run.notFound=GitHub sync run not found +error.githubSync.source.paused=GitHub source is paused; resume it before requesting a rescan +error.githubSync.schedulePolicy.invalid=Sync target intervals must be between 1 and 720 hours +error.githubSync.binding.conflictNotFound=GitHub sync conflict not found in this namespace +error.githubSync.binding.notResolvable=This GitHub sync item is not an unresolved Skill slug conflict +error.githubSync.binding.invalidTarget=Choose an active Skill with the same slug in this namespace +error.githubSync.binding.alreadyBound=The Skill or GitHub path already has an authoritative source binding +error.githubSync.binding.pendingReview=Resolve the Skill's pending user review before binding a GitHub source +error.githubSync.binding.reason=Provide a binding reason of 10 to 1000 characters +error.githubSync.source.ignorePatterns=Invalid GitHub source ignore paths +error.githubSync.discoveryEntry.selectors=Use 1 to 20 valid keyword or topic selectors, without GitHub query syntax +error.githubSync.discoveryEntry.title=Discovery rule title must contain 1 to 128 characters +error.githubSync.discoveryEntry.interval=Discovery target interval must be between 1 and 720 hours +error.githubSync.credential.duplicate=This GitHub credential is already configured +error.githubSync.credential.notFound=GitHub credential not found +error.githubSync.credential.keyUnavailable=GitHub credential encryption key is unavailable +error.githubSync.credential.label=Credential label must contain 1 to 100 characters +error.githubSync.credential.token=Invalid GitHub token format +error.githubSync.credential.replaceRequired=Replace the invalid GitHub credential before enabling it + error.suite.workspace.invalidFilter=Invalid suite status filter error.methodNotAllowed=HTTP method is not supported error.unsupportedMediaType=Unsupported media type @@ -158,6 +187,7 @@ error.skill.report.status.invalid=Unsupported skill report status: {0} error.skill.version.latest.unavailable=No published version available for skill: {0} error.skill.version.latest.notFound=Latest published version not found error.skill.file.notFound=File not found: {0} +error.skill.file.previewTooLarge=File is too large for inline preview: {0} error.skill.tag.latest.reserved=Tag name ''latest'' is reserved error.skill.tag.latest.delete=Tag name ''latest'' is reserved and cannot be deleted error.skill.tag.notFound=Tag not found: {0} @@ -179,6 +209,12 @@ error.admin.user.status.unsupported=Only ACTIVE or DISABLED status can be manage error.skill.publish.nameConflict=A published skill with name ''{0}'' already exists in this namespace error.skill.publish.nameConflict.private=A private skill with name ''{0}'' has already been published in this namespace error.skill.publish.concurrentConflict=A concurrent publish for skill ''{0}'' is already being processed; please retry. +error.skill.publish.githubSync.namespaceInvalid=GitHub sync requires a catalog or team namespace +error.skill.publish.githubSync.slugConflict=GitHub sync skill ''{0}'' conflicts with an existing skill; a namespace administrator must resolve it +error.skill.publish.githubSync.pendingUserReview=Skill ''{0}'' has a pending user review and cannot be replaced by sync +error.skill.publish.githubSync.versionExists=The GitHub sync version for skill ''{0}'' already exists +review.submit.githubSyncManaged=Sync version {0} must be managed through GitHub sync review +review.sync.licenseResolutionRequired=The GitHub skill license is not confirmed; add the resolution basis to the review comment before approval error.skill.approve.nameConflict=Cannot approve: a published skill with name ''{0}'' already exists in this namespace error.skill.version.submit.notUploaded=Version ''{0}'' is not in UPLOADED status and cannot be submitted for review error.skill.version.confirm.notUploaded=Version ''{0}'' is not in UPLOADED status and cannot be confirmed @@ -350,3 +386,9 @@ error.skill.visibility.scanIncomplete=The current security scan has not passed. error.skill.visibility.avatarPending=Finish or withdraw the pending avatar review before changing visibility. error.skill.owner.readOnly=You are no longer a member of this namespace. You can only view and download this skill. + +error.githubSync.deepDiscovery.configurationInvalid=Provide a positive configuration version, enabled state, budget profile and at most 1000 exclusions +error.githubSync.deepDiscovery.exclusionInvalid=Exclusions must be valid GitHub owner names or owner/repository coordinates; URLs and wildcards are not supported +error.githubSync.deepDiscovery.configurationConflict=Official discovery configuration has changed. Reload the latest configuration before saving + +error.githubSync.deepDiscovery.queryInvalid=Invalid deep discovery query parameters. diff --git a/server/skillhub-app/src/main/resources/messages_ru.properties b/server/skillhub-app/src/main/resources/messages_ru.properties index 3ee5537ff..3b7efbb1b 100644 --- a/server/skillhub-app/src/main/resources/messages_ru.properties +++ b/server/skillhub-app/src/main/resources/messages_ru.properties @@ -60,6 +60,35 @@ error.auth.sessionBootstrap.providerUnsupported=Неподдерживаемый error.auth.sessionBootstrap.notAuthenticated=Внешняя аутентифицированная сессия не найдена error.badRequest=Некорректный запрос error.notFound=Ресурс не найден +error.githubSync.admissionPolicy.invalid=Пороги синхронизации GitHub должны удовлетворять условию 0 <= нижний < верхний; лимиты кандидатов должны быть положительными, лимит на аренду не выше общего +error.githubSync.discoveryEntry.duplicate=Источник обнаружения GitHub уже существует +error.githubSync.discoveryEntry.notFound=Источник обнаружения GitHub не найден +error.githubSync.discoveryEntry.type=Этот источник обнаружения не является правилом поиска +error.githubSync.discoveryEntry.owner=Недопустимое имя пользователя или организации GitHub +error.githubSync.discoveryEntry.repository=Недопустимые координаты публичного репозитория GitHub +error.githubSync.teamSource.forbidden=Управлять источниками GitHub могут только владелец или администратор пространства имен +error.githubSync.teamSource.visibility=Видимость источника должна быть PUBLIC или NAMESPACE_ONLY +error.githubSync.source.notFound=Источник GitHub не найден +error.githubSync.run.notFound=Запуск синхронизации GitHub не найден +error.githubSync.source.paused=Источник GitHub приостановлен; возобновите его перед повторной проверкой +error.githubSync.schedulePolicy.invalid=Периоды синхронизации должны составлять от 1 до 720 часов +error.githubSync.binding.conflictNotFound=Конфликт синхронизации GitHub в этом namespace не найден +error.githubSync.binding.notResolvable=Этот элемент не является нерешённым конфликтом имени скилла +error.githubSync.binding.invalidTarget=Выберите активный скилл с тем же именем в этом namespace +error.githubSync.binding.alreadyBound=У скилла или пути GitHub уже есть основной источник +error.githubSync.binding.pendingReview=Сначала обработайте ожидающую проверку пользовательскую версию скилла +error.githubSync.binding.reason=Укажите основание привязки длиной от 10 до 1000 символов +error.githubSync.source.ignorePatterns=Некорректные пути исключения источника GitHub +error.githubSync.discoveryEntry.selectors=Укажите от 1 до 20 допустимых ключевых слов или тем без синтаксиса запросов GitHub +error.githubSync.discoveryEntry.title=Название правила должно содержать от 1 до 128 символов +error.githubSync.discoveryEntry.interval=Период обнаружения должен составлять от 1 до 720 часов +error.githubSync.credential.duplicate=Эти учётные данные GitHub уже добавлены +error.githubSync.credential.notFound=Учётные данные GitHub не найдены +error.githubSync.credential.keyUnavailable=Ключ шифрования учётных данных GitHub недоступен +error.githubSync.credential.label=Название должно содержать от 1 до 100 символов +error.githubSync.credential.token=Неверный формат токена GitHub +error.githubSync.credential.replaceRequired=Замените недействительный токен GitHub перед включением + error.suite.workspace.invalidFilter=Некорректный фильтр статуса набора error.methodNotAllowed=HTTP-метод не поддерживается error.unsupportedMediaType=Неподдерживаемый тип медиа @@ -155,6 +184,7 @@ error.skill.report.status.invalid=Неподдерживаемый статус error.skill.version.latest.unavailable=Нет опубликованной версии для скилла: {0} error.skill.version.latest.notFound=Последняя опубликованная версия не найдена error.skill.file.notFound=Файл не найден: {0} +error.skill.file.previewTooLarge=Файл слишком большой для встроенного просмотра: {0} error.skill.tag.latest.reserved=Имя тега ''latest'' зарезервировано error.skill.tag.latest.delete=Имя тега ''latest'' зарезервировано и не может быть удалено error.skill.tag.notFound=Тег не найден: {0} @@ -175,6 +205,12 @@ error.admin.user.status.invalid=Неверный статус пользоват error.admin.user.status.unsupported=Здесь можно управлять только статусами ACTIVE или DISABLED error.skill.publish.nameConflict=Опубликованный скилл с именем ''{0}'' уже существует в этом пространстве имён error.skill.publish.nameConflict.private=Приватный скилл с именем ''{0}'' уже опубликован в этом пространстве имён +error.skill.publish.githubSync.namespaceInvalid=Синхронизация GitHub доступна только для каталога или пространства команды +error.skill.publish.githubSync.slugConflict=Скилл GitHub ''{0}'' конфликтует с существующим скиллом; требуется решение администратора пространства +error.skill.publish.githubSync.pendingUserReview=У скилла ''{0}'' есть ожидающее ревью пользователя; синхронизация не может его заменить +error.skill.publish.githubSync.versionExists=Версия скилла GitHub ''{0}'' уже существует +review.submit.githubSyncManaged=Версией синхронизации {0} можно управлять только через ревью GitHub +review.sync.licenseResolutionRequired=Лицензия синхронизированного скилла не подтверждена; укажите основание решения в комментарии перед одобрением error.skill.approve.nameConflict=Нельзя одобрить: опубликованный скилл с именем ''{0}'' уже существует в этом пространстве имён error.skill.version.submit.notUploaded=Версия ''{0}'' не в статусе UPLOADED и не может быть отправлена на ревью error.skill.version.confirm.notUploaded=Версия ''{0}'' не в статусе UPLOADED и не может быть подтверждена @@ -293,3 +329,9 @@ error.skill.visibility.scanIncomplete=Текущая проверка безоп error.skill.visibility.avatarPending=Завершите или отзовите проверку аватара перед изменением видимости. error.skill.owner.readOnly=You are no longer a member of this namespace. You can only view and download this skill. + +error.githubSync.deepDiscovery.configurationInvalid=Укажите положительную версию конфигурации, состояние включения, профиль бюджета и не более 1000 исключений +error.githubSync.deepDiscovery.exclusionInvalid=Исключения должны содержать допустимое имя владельца GitHub или координаты владелец/репозиторий; URL и шаблоны не поддерживаются +error.githubSync.deepDiscovery.configurationConflict=Конфигурация официального поиска изменена. Загрузите актуальную конфигурацию перед сохранением + +error.githubSync.deepDiscovery.queryInvalid=Недопустимые параметры запроса глубокого обнаружения. diff --git a/server/skillhub-app/src/main/resources/messages_zh.properties b/server/skillhub-app/src/main/resources/messages_zh.properties index 7df810b7a..6b2787c2e 100644 --- a/server/skillhub-app/src/main/resources/messages_zh.properties +++ b/server/skillhub-app/src/main/resources/messages_zh.properties @@ -62,6 +62,35 @@ error.auth.sessionBootstrap.providerUnsupported=不支持的会话引导提供 error.auth.sessionBootstrap.notAuthenticated=未检测到已认证的外部会话 error.badRequest=请求参数不合法 error.notFound=未找到资源 +error.githubSync.admissionPolicy.invalid=GitHub 同步积压水位须满足 0 ≤ 低水位 < 高水位；候选上限须为正数，单次租约上限不得超过待验证总上限 +error.githubSync.discoveryEntry.duplicate=该 GitHub 发现入口已存在 +error.githubSync.discoveryEntry.notFound=未找到 GitHub 发现入口 +error.githubSync.discoveryEntry.type=该发现入口不是仓库搜索规则 +error.githubSync.discoveryEntry.owner=GitHub 用户或组织名称格式不正确 +error.githubSync.discoveryEntry.repository=GitHub 公开仓库坐标格式不正确 +error.githubSync.teamSource.forbidden=只有 namespace 所有者或管理员可以管理 GitHub 来源 +error.githubSync.teamSource.visibility=来源可见范围只能设为 PUBLIC 或 NAMESPACE_ONLY +error.githubSync.source.notFound=未找到 GitHub 来源 +error.githubSync.run.notFound=未找到 GitHub 同步运行记录 +error.githubSync.source.paused=GitHub 来源已暂停，请先恢复后再安排重扫 +error.githubSync.schedulePolicy.invalid=同步目标周期必须为 1 到 720 小时 +error.githubSync.binding.conflictNotFound=当前 namespace 中未找到该 GitHub 同步冲突 +error.githubSync.binding.notResolvable=该同步项不是待处理的 Skill 同名冲突 +error.githubSync.binding.invalidTarget=请选择当前 namespace 中同名且未归档的 Skill +error.githubSync.binding.alreadyBound=该 Skill 或 GitHub 路径已有权威来源绑定 +error.githubSync.binding.pendingReview=请先处理该 Skill 的普通用户待审版本，再绑定 GitHub 来源 +error.githubSync.binding.reason=请填写 10 至 1000 字的绑定依据 +error.githubSync.source.ignorePatterns=GitHub 来源忽略路径不符合要求 +error.githubSync.discoveryEntry.selectors=请配置 1 至 20 个有效的关键词或 topic 条件，不要输入原始 GitHub 查询语法 +error.githubSync.discoveryEntry.title=发现规则名称长度须为 1 至 128 个字符 +error.githubSync.discoveryEntry.interval=发现目标周期须为 1 至 720 小时 +error.githubSync.credential.duplicate=该 GitHub 凭证已配置 +error.githubSync.credential.notFound=未找到 GitHub 凭证 +error.githubSync.credential.keyUnavailable=GitHub 凭证加密密钥不可用 +error.githubSync.credential.label=凭证名称长度须为 1 至 100 个字符 +error.githubSync.credential.token=GitHub Token 格式无效 +error.githubSync.credential.replaceRequired=请先替换已失效的 GitHub 凭证，再启用 + error.suite.workspace.invalidFilter=套件状态筛选不合法 error.methodNotAllowed=不支持的请求方法 error.unsupportedMediaType=不支持的请求内容类型 @@ -158,6 +187,7 @@ error.skill.report.status.invalid=不支持的技能举报状态：{0} error.skill.version.latest.unavailable=技能没有可下载的已发布版本：{0} error.skill.version.latest.notFound=未找到最新已发布版本 error.skill.file.notFound=未找到文件：{0} +error.skill.file.previewTooLarge=文件过大，无法在线预览：{0} error.skill.tag.latest.reserved=标签 latest 为系统保留标签 error.skill.tag.latest.delete=标签 latest 为系统保留标签，不能删除 error.skill.tag.notFound=未找到标签：{0} @@ -179,6 +209,12 @@ error.admin.user.status.unsupported=这里只允许管理 ACTIVE 或 DISABLED error.skill.publish.nameConflict=该命名空间下已存在名为"{0}"的已发布技能，无法提交 error.skill.publish.nameConflict.private=该命名空间下已存在名为"{0}"的已发布私有技能，无法提交 error.skill.publish.concurrentConflict=技能"{0}"存在并发的发布请求正在处理，请稍后重试。 +error.skill.publish.githubSync.namespaceInvalid=GitHub 同步仅支持目录或团队命名空间 +error.skill.publish.githubSync.slugConflict=GitHub 同步技能"{0}"与已有技能冲突，请由命名空间管理员处理 +error.skill.publish.githubSync.pendingUserReview=技能"{0}"仍有普通用户的待审版本，无法同步覆盖 +error.skill.publish.githubSync.versionExists=GitHub 同步技能"{0}"的版本已存在 +review.submit.githubSyncManaged=同步版本 {0} 只能通过 GitHub 同步审核处理 +review.sync.licenseResolutionRequired=该 GitHub 同步技能的许可证尚未确认；请在审核意见中写明处理依据后再批准 error.skill.approve.nameConflict=无法通过审核：该命名空间下已存在名为"{0}"的已发布技能 error.skill.version.submit.notUploaded=版本"{0}"不在 UPLOADED 状态，无法提交审核 error.skill.version.confirm.notUploaded=版本"{0}"不在 UPLOADED 状态，无法确认发布 @@ -352,3 +388,9 @@ error.skill.visibility.scanIncomplete=本次安全扫描尚未通过，无法批 error.skill.visibility.avatarPending=请先完成或撤回待审核头像，再修改可见范围。 error.skill.owner.readOnly=已不在该空间，只能查看和下载技能。 + +error.githubSync.deepDiscovery.configurationInvalid=请提供有效的配置版本、启停状态、预算档位和最多1000条排除项 +error.githubSync.deepDiscovery.exclusionInvalid=排除项须为有效的GitHub作者名称或作者/仓库坐标，不支持URL或通配符 +error.githubSync.deepDiscovery.configurationConflict=官方探索配置已被修改，请重新读取最新配置后再保存 + +error.githubSync.deepDiscovery.queryInvalid=深度探索查询参数无效。 diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/config/AsyncConfigTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/config/AsyncConfigTest.java index a5ac41d6a..a3bcb5c8b 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/config/AsyncConfigTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/config/AsyncConfigTest.java @@ -3,14 +3,16 @@ package com.iflytek.skillhub.config; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; class AsyncConfigTest { @Test - void asyncConfig_enablesAsyncAndScheduling() { + void asyncExecutionAndConditionalSchedulingHaveSeparateConfigurations() { assertThat(AsyncConfig.class).hasAnnotation(EnableAsync.class); - assertThat(AsyncConfig.class).hasAnnotation(EnableScheduling.class); + assertThat(SchedulingConfig.class).hasAnnotation(EnableScheduling.class); + assertThat(SchedulingConfig.class).hasAnnotation(ConditionalOnProperty.class); } } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/AdminGithubDiscoveryControllerSecurityTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/AdminGithubDiscoveryControllerSecurityTest.java new file mode 100644 index 000000000..9680c51db --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/AdminGithubDiscoveryControllerSecurityTest.java @@ -0,0 +1,110 @@ +package com.iflytek.skillhub.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.service.GithubDiscoveryAdminAppService; +import com.iflytek.skillhub.service.GithubSearchPreviewAppService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.authentication; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class AdminGithubDiscoveryControllerSecurityTest { + private static final String ENTRIES = "/api/v1/admin/github-sync/discovery-entries"; + private static final String RULE = """ + {"title":"Public skills","selectors":[ + {"kind":"KEYWORD","value":"agent skills","priority":"CORE"}]} + """; + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private MockMvc mockMvc; + @Autowired private ObjectMapper mapper; + @MockBean private GithubDiscoveryAdminAppService discovery; + @MockBean private GithubSearchPreviewAppService previews; + + @Test + void anonymousAndOrdinaryUserCannotReadRulesOrQueuePreview() throws Exception { + mockMvc.perform(get(ENTRIES)).andExpect(status().isUnauthorized()); + mockMvc.perform(get(ENTRIES).with(authentication(auth("ordinary", "USER")))) + .andExpect(status().isForbidden()); + mockMvc.perform(post(ENTRIES + "/search-rules/preview") + .with(authentication(auth("ordinary", "USER"))).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(RULE)) + .andExpect(status().isForbidden()); + verifyNoInteractions(discovery, previews); + } + + @Test + void superAdminCanReadRulesAndQueuePreviewThroughSecurityChain() throws Exception { + when(discovery.list()).thenReturn(List.of()); + var preview = new GithubSearchPreviewAppService.Preview( + UUID.randomUUID(), "QUEUED", null, null, mapper.createObjectNode(), + Instant.now().toString(), null, Instant.now(), Instant.now()); + when(previews.queue(any(), eq("admin"))).thenReturn(preview); + + mockMvc.perform(get(ENTRIES).with(authentication(auth("admin", "SUPER_ADMIN")))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data").isArray()); + mockMvc.perform(post(ENTRIES + "/search-rules/preview") + .with(authentication(auth("admin", "SUPER_ADMIN"))).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(RULE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.id").value(preview.id().toString())); + verify(previews).queue(any(), eq("admin")); + } + + private UsernamePasswordAuthenticationToken auth(String userId, String role) { + var principal = new PlatformPrincipal(userId, userId, null, null, null, Set.of(role)); + return new UsernamePasswordAuthenticationToken(principal, null, + List.of(new SimpleGrantedAuthority("ROLE_" + role))); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubContentRevisionConcurrencyIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubContentRevisionConcurrencyIntegrationTest.java new file mode 100644 index 000000000..a17853235 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubContentRevisionConcurrencyIntegrationTest.java @@ -0,0 +1,209 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.HexFormat; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = {"skillhub.builtin-skills.enabled=false", + "skillhub.security.scanner.enabled=true", "skillhub.security.scanner.mode=upload"}) +class GithubContentRevisionConcurrencyIntegrationTest { + @org.junit.jupiter.api.io.TempDir static java.nio.file.Path storageDirectory; + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("skillhub.storage.local.base-path", () -> storageDirectory.toString()); + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + @org.springframework.boot.test.mock.mockito.MockBean + org.springframework.data.redis.core.StringRedisTemplate redisTemplate; + @org.springframework.boot.test.mock.mockito.MockBean + org.redisson.api.RedissonClient redissonClient; + @org.springframework.boot.test.mock.mockito.MockBean + com.iflytek.skillhub.stream.ScanTaskConsumer scanTaskConsumer; + @Autowired JdbcTemplate jdbc; + @Autowired GithubSyncIngestionRepository ingestions; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired GithubSyncPublicationRepository publications; + @Autowired com.iflytek.skillhub.service.GithubSyncPublicationAppService publicationService; + @Autowired com.iflytek.skillhub.service.GithubSyncArchiveValidator archiveValidator; + @Autowired com.iflytek.skillhub.storage.ObjectStorageService storage; + @Autowired org.springframework.transaction.PlatformTransactionManager transactionManager; + @Autowired com.iflytek.skillhub.repository.GithubSyncReviewSourceQueryRepository reviewSources; + @Autowired com.iflytek.skillhub.repository.GithubSkillSourceQueryRepository marketSources; + long namespace; + UUID catalogQuery; + static final String TOKEN = "e".repeat(64), DIGEST = "b".repeat(64); + + @BeforeEach void setup() { + namespace = jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id", Long.class); + scheduler.enqueueDue(Instant.now(), 1); // Disabled configuration reconciles the real Java catalog only. + catalogQuery = jdbc.queryForObject("SELECT id FROM github_deep_discovery_query ORDER BY query_key LIMIT 1", UUID.class); + jdbc.update("UPDATE github_deep_discovery_query SET next_due_at=now()+INTERVAL '2 days'"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + } + + @Test + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.NOT_SUPPORTED) + void rollbackAndIndependentConcurrentTransactionsDoNotConsumeOrDuplicateRevisions() throws Exception { + var transaction = new org.springframework.transaction.support.TransactionTemplate(transactionManager); + Fixture first = prepareArchive(1801, "A"); + var claim = publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(600)).orElseThrow(); + assertThatThrownBy(() -> transaction.executeWithoutResult(status -> { + try { publicationService.process(claim); } catch (java.io.IOException exception) { throw new RuntimeException(exception); } + assertRevision(first, 1); + throw new IllegalStateException("rollback after revision assignment"); + })).isInstanceOf(IllegalStateException.class).hasMessage("rollback after revision assignment"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_source_skill_binding WHERE repository_id=1801", Integer.class)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM skill_version WHERE github_sync_ingestion_id=?", Integer.class, first.ingestion())).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM review_task WHERE github_sync_ingestion_id=?", Integer.class, first.ingestion())).isZero(); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id=?", String.class, first.ingestion())).isEqualTo("VALIDATED"); + publicationService.process(claim); + assertRevision(first, 1); + Fixture second = prepareArchive(1801, "B"); + var secondClaim = publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(600)).orElseThrow(); + Fixture third = prepareArchive(1801, "A"); + var thirdClaim = publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(600)).orElseThrow(); + var locked = new java.util.concurrent.CountDownLatch(1); + var entered = new java.util.concurrent.CountDownLatch(1); + var release = new java.util.concurrent.CountDownLatch(1); + var olderPid = new java.util.concurrent.atomic.AtomicInteger(); + var newerPid = new java.util.concurrent.atomic.AtomicInteger(); + try (var executor = java.util.concurrent.Executors.newFixedThreadPool(2)) { + var older = executor.submit(() -> transaction.executeWithoutResult(status -> { + olderPid.set(jdbc.queryForObject("SELECT pg_backend_pid()", Integer.class)); + assertThat(publications.lockAndVerify(secondClaim, Instant.now())).isTrue(); + locked.countDown(); + try { + assertThat(release.await(30, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + publicationService.process(secondClaim); + } catch (Exception exception) { throw new RuntimeException(exception); } + })); + java.util.concurrent.Future newer; + try { + assertThat(locked.await(10, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + newer = executor.submit(() -> transaction.executeWithoutResult(status -> { + newerPid.set(jdbc.queryForObject("SELECT pg_backend_pid()", Integer.class)); + entered.countDown(); + try { publicationService.process(thirdClaim); } + catch (java.io.IOException exception) { throw new RuntimeException(exception); } + })); + assertThat(entered.await(10, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + assertThat(newerPid.get()).isNotEqualTo(olderPid.get()); + // A third connection observes actual PostgreSQL lock contention; thread timing is not proof. + try (var observer = java.sql.DriverManager.getConnection( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + var statement = observer.prepareStatement("SELECT ? = ANY(pg_blocking_pids(?))")) { + statement.setInt(1, olderPid.get()); + statement.setInt(2, newerPid.get()); + statement.setQueryTimeout(2); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10); + boolean blockedByOlder = false; + while (System.nanoTime() < deadline) { + try (var result = statement.executeQuery()) { + result.next(); + blockedByOlder = result.getBoolean(1); + } + if (blockedByOlder) break; + Thread.sleep(10); // Poll pacing only; the database predicate determines success. + } + assertThat(blockedByOlder).as("newer database session is blocked by the older transaction").isTrue(); + } + } finally { release.countDown(); } + older.get(30, java.util.concurrent.TimeUnit.SECONDS); + newer.get(30, java.util.concurrent.TimeUnit.SECONDS); + } + assertRevision(second, 2); + assertRevision(third, 3); + assertThat(jdbc.queryForObject("SELECT latest_revision FROM github_source_skill_binding WHERE repository_id=1801", Integer.class)).isEqualTo(3); + assertThat(jdbc.queryForObject("SELECT count(*) FROM skill_version WHERE skill_id=(SELECT published_skill_id FROM github_sync_package_ingestion WHERE id=?)", Integer.class, third.ingestion())).isEqualTo(3); + } + + private void assertRevision(Fixture fixture, int expected) { + long version = jdbc.queryForObject("SELECT published_version_id FROM github_sync_package_ingestion WHERE id=?", Long.class, fixture.ingestion()); + assertThat(jdbc.queryForObject("SELECT github_sync_revision FROM skill_version WHERE id=?", Integer.class, version)).isEqualTo(expected); + String technical = jdbc.queryForObject("SELECT version FROM skill_version WHERE id=?", String.class, version); + assertThat(technical).startsWith("0.0.0-g"); + long review = jdbc.queryForObject("SELECT id FROM review_task WHERE github_sync_ingestion_id=?", Long.class, fixture.ingestion()); + assertThat(reviewSources.findByReviewId(review).orElseThrow().syncRevision()).isEqualTo(expected); + String status = jdbc.queryForObject("SELECT status FROM skill_version WHERE id=?", String.class, version); + // A database fixture exercises the published-only projection, not approval workflow acceptance. + jdbc.update("UPDATE skill_version SET status='PUBLISHED' WHERE id=?", version); + assertThat(marketSources.findPublishedVersion(version).orElseThrow().syncRevision()).isEqualTo(expected); + jdbc.update("UPDATE skill_version SET status=? WHERE id=?", status, version); + assertThat(jdbc.queryForObject("SELECT version FROM skill_version WHERE id=?", String.class, version)).isEqualTo(technical); + } + + private Fixture prepareArchive(long repository, String content) throws Exception { + Fixture fixture = fixture(repository, 161, true); + jdbc.update("INSERT INTO github_sync_path_result(task_id,commit_sha,skill_path,outcome,ingestion_id,sha256) SELECT task_id,commit_sha,skill_path,'ACCEPTED',id,sha256 FROM github_sync_package_ingestion WHERE id=?",fixture.ingestion()); + var bytes = new java.io.ByteArrayOutputStream(); + try (var zip = new java.util.zip.ZipOutputStream(bytes)) { + zip.putNextEntry(new java.util.zip.ZipEntry("SKILL.md")); + zip.write(("---\nname: actual-owner-outcome-" + repository + "\ndescription: Exercise durable GitHub publication outcomes.\nlicense: MIT\n---\n# Actual owner outcome\n\n" + content + "\n").getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + byte[] archive = bytes.toByteArray(); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(archive)); + String key = jdbc.queryForObject("SELECT object_key FROM github_sync_package_ingestion WHERE id=?", String.class, fixture.ingestion()); + storage.putObject(key, new java.io.ByteArrayInputStream(archive), archive.length, "application/zip"); + var validation = archiveValidator.validate(key, archive.length, digest, "skills/demo"); + jdbc.update("UPDATE github_sync_package_ingestion SET size_bytes=?,sha256=? WHERE id=?", archive.length, digest, fixture.ingestion()); + assertThat(ingestions.validated(fixture.ingestion(), fixture.validationToken(), validation.contentSha256(), 0, "{}", Instant.now())).isTrue(); + return fixture; + } + + private void origin(long candidate, UUID query) { + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?) ON CONFLICT DO NOTHING", candidate, query.toString()); + } + private Fixture fixture(long repo, long owner, boolean official) throws Exception { + UUID task = UUID.randomUUID(), run = UUID.randomUUID(), admission = UUID.randomUUID(), upload = UUID.randomUUID(), ingestion = UUID.randomUUID(), validation = UUID.randomUUID(); + String commit = task.toString().replace("-", "") + "00000000"; + long candidate = jdbc.queryForObject("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name,default_branch,github_owner_id) VALUES (?,?,'seed-owner','skills','main',?) ON CONFLICT (namespace_id,repository_id) DO UPDATE SET last_seen_at=now() RETURNING id", Long.class, namespace, repo, owner); + if (official) origin(candidate, catalogQuery); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'REPOSITORY_SYNC')", run, namespace); + String hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(TOKEN.getBytes(StandardCharsets.UTF_8))); + jdbc.update(""" + INSERT INTO github_sync_task(id,run_id,namespace_id,kind,mode,status,dedupe_key,snapshot_json,lease_generation,lease_token_hash,lease_expires_at,leased_by) + VALUES (?,?,?,'REPOSITORY_SYNC','FULL','LEASED',?,jsonb_build_object('repositoryId',?::bigint,'owner','seed-owner','name','skills','ownerId',?::bigint),1,?,now()+INTERVAL '1 hour','owner-test') + """, task, run, namespace, task.toString(), repo, owner, hash); + jdbc.update("UPDATE github_sync_task SET created_at=clock_timestamp() WHERE id=?", task); + jdbc.update("UPDATE github_discovery_candidate SET status='VALIDATING',validation_task_id=? WHERE id=?", task, candidate); + jdbc.update("INSERT INTO github_sync_package_admission(id,task_id,namespace_id,repository_id,commit_sha,skill_path,lease_generation) VALUES (?,?,?,?,?,'skills/demo',1)", admission, task, namespace, repo, commit); + jdbc.update("INSERT INTO github_sync_staged_upload(id,admission_id,object_key,mode,ticket_hash,size_bytes,sha256,status,expires_at) VALUES (?,?,?,'LOCAL_STREAM',?,1,?,'ACCEPTED',now()+INTERVAL '1 hour')", upload, admission, "owner-test/"+upload, DIGEST, DIGEST); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion(id,task_id,admission_id,upload_id,namespace_id,repository_id,commit_sha,skill_path,object_key,size_bytes,sha256,status,validation_token,validation_expires_at) + VALUES (?,?,?,?,?,?,?,'skills/demo',?,1,?,'VALIDATING',?,?) + """, ingestion, task, admission, upload, namespace, repo, commit, "owner-test/"+upload, DIGEST, validation, Timestamp.from(Instant.now().plusSeconds(3600))); + return new Fixture(task, ingestion, validation, candidate); + } + private record Fixture(UUID task, UUID ingestion, UUID validationToken, long candidate) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubDeepDiscoveryReadModelIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubDeepDiscoveryReadModelIntegrationTest.java new file mode 100644 index 000000000..d5b6bba10 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubDeepDiscoveryReadModelIntegrationTest.java @@ -0,0 +1,319 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import com.iflytek.skillhub.service.GithubDeepDiscoveryReadAppService; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubDeepDiscoveryReadModelIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired JdbcTemplate jdbc; + @Autowired GithubDeepDiscoveryQueryRepository queries; + @Autowired GithubDeepDiscoveryReadAppService reads; + @Autowired GithubSyncWorkerPresenceRepository presence; + private long namespace; + + @BeforeEach void setup() { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update("UPDATE github_sync_admission_policy SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000 WHERE id"); + namespace=jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id",Long.class); + } + + @Test void exclusionProjectionMatchesOriginalViewAfterRenamesAndPolicyChanges() throws Exception { + String migration = new org.springframework.core.io.ClassPathResource( + "db/migration/V111__github_official_candidate_controls.sql") + .getContentAsString(java.nio.charset.StandardCharsets.UTF_8); + String original = migration.substring(migration.indexOf("WITH origin_state")); + original = original.substring(0, original.lastIndexOf(';')); + long other = otherNamespace(); + for (long repo : List.of(700L, 701L, 702L, 703L, 704L)) candidate(namespace, repo, repo != 704); + candidate(other, 700, true); // Missing configuration produces the same nullable source fields. + UUID entry = independent(namespace, 701); + source(namespace, 702, "ACTIVE"); + source(namespace, 703, "PAUSED"); + for (int step = 0; step < 7; step++) { + var expected = jdbc.queryForList("SELECT * FROM (" + original + ") original ORDER BY candidate_id"); + var actual = jdbc.queryForList("SELECT * FROM github_candidate_discovery_eligibility ORDER BY candidate_id"); + assertThat(actual).as("complete eligibility rows at step %s", step).isEqualTo(expected); + assertThat(actual.getFirst().keySet()).containsExactly("candidate_id", "namespace_id", "repository_id", + "source_id", "source_status", "has_official", "has_independent", "official_only", + "configuration_enabled", "excluded", "acquisition_allowed"); + switch (step) { + case 0 -> jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"renamed\"},{\"kind\":\"REPOSITORY\",\"value\":\"fixture/updated\"},{}]' WHERE id"); + case 1 -> jdbc.update("UPDATE github_discovery_candidate SET owner_login='ReNamed' WHERE repository_id IN (700,702,703)"); + case 2 -> jdbc.update("UPDATE github_discovery_candidate SET repository_name='Updated' WHERE namespace_id=? AND repository_id=701", namespace); + case 3 -> jdbc.update("UPDATE github_discovery_entry SET enabled=FALSE WHERE id=?", entry); + case 4 -> jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE,exclusions_json='[]' WHERE id"); + case 5 -> jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE,exclusions_json='[{\"kind\":\"OWNER\",\"value\":null},{\"kind\":null,\"value\":\"renamed\"}]' WHERE id"); + default -> { } + } + } + java.util.function.Function> shape = sql -> jdbc.query(sql, rs -> { + var metadata = rs.getMetaData(); + var columns = new java.util.ArrayList(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + columns.add(metadata.getColumnName(i) + ":" + metadata.getColumnType(i)); + } + return columns; + }); + assertThat(shape.apply("SELECT * FROM github_candidate_discovery_eligibility WHERE FALSE")) + .isEqualTo(shape.apply("SELECT * FROM (" + original + ") original WHERE FALSE")); + } + + @Test void targetedRepositoryMatchesListEligibilityAcrossNamespacesAndAllOrigins() { + long other=otherNamespace(); + for(long repo:List.of(100L,101L,102L,103L)) candidate(namespace,repo,true); + candidate(other,100,true); + independent(namespace,101); + source(namespace,102,"ACTIVE");source(namespace,103,"PAUSED"); + jdbc.update("UPDATE github_discovery_candidate SET status='BOUND' WHERE namespace_id=? AND repository_id IN (102,103)",namespace); + candidate(namespace,104,false); + for(int step=0;step<3;step++) { + var all=queries.repositories(namespace,0,20,Instant.now(),null,null,null,null).items(); + assertThat(all).hasSize(4); + for(var expected:all) assertThat(queries.repository(namespace,expected.repositoryId())).isEqualTo(expected); + assertThat(queries.repository(namespace,101).paused()).isFalse(); + assertThat(queries.repository(namespace,102).paused()).isFalse(); + assertThat(queries.repository(namespace,103).paused()).isTrue(); + if(step>0) assertThat(queries.repository(namespace,100).paused()).isTrue(); + assertThat(queries.repository(other,100).candidateId()).isNotEqualTo(queries.repository(namespace,100).candidateId()); + assertThatThrownBy(() -> queries.repository(other,101)).isInstanceOf(DomainNotFoundException.class); + assertThatThrownBy(() -> queries.repository(namespace,104)).isInstanceOf(DomainNotFoundException.class); + if(step==0) jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + else jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE,exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"fixture\"}]' WHERE id"); + } + } + + @Test void existenceShortCircuitKeepsNamespaceAndIndependentAndBoundSemantics() { + long other=otherNamespace();candidate(other,200,true); + assertOfficialOnly(false);candidate(namespace,200,false);assertOfficialOnly(false); + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) SELECT id,'OFFICIAL_DISCOVERY',? FROM github_discovery_candidate WHERE namespace_id=?",UUID.randomUUID().toString(),namespace); + assertOfficialOnly(true);UUID entry=independent(namespace,200);assertOfficialOnly(false); + jdbc.update("UPDATE github_discovery_entry SET enabled=FALSE WHERE id=?",entry);assertOfficialOnly(true); + source(namespace,200,"PAUSED");assertOfficialOnly(false); + assertThat(queries.hasOfficialOnlyCandidates(other)).isTrue(); + } + + @Test void channelFastPathEqualsOriginalAcrossTaskLeaseControlAndRepositoryWork() { + long other=otherNamespace(); + UUID search=officialTask(Channel.REPOSITORY_METADATA);officialTask(Channel.README);UUID owner=officialTask(Channel.OWNER_EXPANSION); + jdbc.update("UPDATE github_sync_task SET snapshot_json='{\"repositoryId\":399}' WHERE id=?",search); + candidate(namespace,300,true);long source=source(namespace,300,"ACTIVE");repositoryTask(300,source); + for(int stage=0;stage<7;stage++) { + assertEquivalentProgress(namespace);assertEquivalentProgress(other); + switch(stage) { + case 0 -> jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"fixture-owner\"}]' WHERE id"); + case 1 -> jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + case 2 -> { jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE,exclusions_json='[]' WHERE id");jdbc.update("UPDATE github_sync_task SET lease_expires_at=CURRENT_TIMESTAMP-INTERVAL '1 second' WHERE id=?",owner); } + case 3 -> jdbc.update("UPDATE github_sync_task SET last_progress_at=CURRENT_TIMESTAMP-INTERVAL '91 seconds'"); + case 4 -> jdbc.update("UPDATE github_sync_task SET last_progress_at=CURRENT_TIMESTAMP,lease_expires_at=CURRENT_TIMESTAMP+INTERVAL '1 hour'"); + case 5 -> jdbc.update("UPDATE github_sync_task SET status='RETRY_WAIT',lease_expires_at=NULL,lease_token_hash=NULL,leased_by=NULL,official_pause_reason='BUDGET'"); + default -> { } + } + } + } + + @Test void workerCapabilitiesBudgetAndPauseFeedbackStayAuthoritative() { + officialTask(Channel.REPOSITORY_METADATA);officialTask(Channel.OWNER_EXPANSION); + assertThat(reads.progress().channels()).allSatisfy(c -> assertThat(c.waits()).anyMatch(w -> w.reason().equals("WORKER_UNAVAILABLE"))); + presence.heartbeat("search",1,platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)); + assertThat(channel(Channel.REPOSITORY_METADATA).state()).isEqualTo("RUNNING"); + assertThat(channel(Channel.OWNER_EXPANSION).waits()).anyMatch(w -> w.reason().equals("CAPABILITY_UNAVAILABLE")); + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=CURRENT_TIMESTAMP+INTERVAL '1 hour'"); + assertThat(channel(Channel.REPOSITORY_METADATA).waits()).anySatisfy(w -> { + assertThat(w.reason()).isEqualTo("BUDGET");assertThat(w.continuingWork()).contains("NEW_DISCOVERY"); + }); + presence.heartbeat("owner",1,platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES)); + assertThat(channel(Channel.OWNER_EXPANSION).state()).isEqualTo("RUNNING"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + assertThat(reads.progress().channels()).allSatisfy(c -> {assertThat(c.state()).isEqualTo("DISABLED");assertThat(c.waits()).allSatisfy(w -> assertThat(w.continuingWork()).doesNotContain("NEW_DISCOVERY"));}); + } + + @Test void paginatedCandidateProjectionMatchesOriginalForFiltersAndStableOrdering() { + long other=otherNamespace(); + UUID metadataTask=officialTask(Channel.REPOSITORY_METADATA),readmeTask=officialTask(Channel.README); + UUID metadata=jdbc.queryForObject("SELECT official_query_id FROM github_sync_task WHERE id=?",UUID.class,metadataTask); + UUID readme=jdbc.queryForObject("SELECT official_query_id FROM github_sync_task WHERE id=?",UUID.class,readmeTask); + String[] states={"PENDING","VALIDATING","BOUND","REJECTED","REJECTED","UNREACHABLE"}; + for(int i=0;i<42;i++) { + long repo=1000+i,id=candidate(namespace,repo,true); + UUID validationTask=null; + if (states[i%states.length].equals("VALIDATING")) { + validationTask=UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,kind,mode,dedupe_key,snapshot_json) VALUES (?,?,?,'REPOSITORY_SYNC','FULL',?,'{}')", + validationTask,run("REPOSITORY_SYNC"),namespace,validationTask.toString()); + } + jdbc.update("UPDATE github_discovery_candidate SET status=?,repository_name=?,last_error_code=?,validation_task_id=? WHERE id=?", + states[i%states.length],"skills-"+i,i%states.length==3?"NO_VALID_SKILL":null,validationTask,id); + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?)",id,metadata.toString()); + if(i%3==0) jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?)",id,readme.toString()); + if(i%states.length==2) source(namespace,repo,i%12==2?"PAUSED":"ACTIVE"); + if(i%5==0) independent(namespace,repo); + } + candidate(other,1000,true);candidate(namespace,2000,false); + long future=candidate(namespace,2001,true); + jdbc.update("UPDATE github_discovery_candidate SET first_seen_at=CURRENT_TIMESTAMP+INTERVAL '1 day' WHERE id=?",future); + jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"fixture\"}]' WHERE id"); + Instant asOf=Instant.now(); + var filters=new java.util.ArrayList();filters.add(new ListFilter(null,null,null)); + for(var status:RepositoryStatus.values()) filters.add(new ListFilter(null,status,null)); + for(var channel:Channel.values()) filters.add(new ListFilter(null,null,channel)); + for(String search:List.of(" FIXTURE/skills-1 ","skills","missing"," ")) filters.add(new ListFilter(search,null,null)); + filters.add(new ListFilter("skills-1",RepositoryStatus.VALIDATION_FAILED,Channel.README)); + filters.add(new ListFilter("fixture",RepositoryStatus.PENDING,Channel.REPOSITORY_METADATA)); + for(long ns:List.of(namespace,other)) for(Boolean paused:java.util.Arrays.asList(null,true,false)) + for(int page:List.of(0,5,100)) for(var filter:filters) { + var expected=originalPage(ns,page,7,asOf,filter,paused); + assertThat(queries.repositories(ns,page,7,asOf,filter.search(),filter.status(),filter.channel(),paused)) + .as("ns=%s page=%s paused=%s filters=%s",ns,page,paused,filter).isEqualTo(expected); + } + var first=queries.repositories(namespace,0,7,asOf,null,null,null,null); + var next=queries.repositories(namespace,1,7,asOf,null,null,null,null); + assertThat(first.total()).isEqualTo(42);assertThat(next.total()).isEqualTo(42); + assertThat(first.items()).extracting(RepositorySummary::candidateId).isSortedAccordingTo(java.util.Comparator.reverseOrder()); + assertThat(first.items()).extracting(RepositorySummary::candidateId) + .doesNotContainAnyElementsOf(next.items().stream().map(RepositorySummary::candidateId).toList()); + assertThat(queries.repositories(namespace,0,7,asOf,null,null,null,null)).isEqualTo(first); + } + + private record ListFilter(String search,RepositoryStatus status,Channel channel) {} + private DiscoveryPage originalPage(long ns,int page,int size,Instant asOf,ListFilter filter,Boolean paused) { + var params=new org.springframework.jdbc.core.namedparam.MapSqlParameterSource("ns",ns) + .addValue("asOf",java.sql.Timestamp.from(asOf)).addValue("limit",size).addValue("offset",page*size) + .addValue("search",filter.search()==null || filter.search().isBlank()?null:filter.search().trim()) + .addValue("status",filter.status()==null?null:filter.status().name()) + .addValue("channel",filter.channel()==null?null:filter.channel().name()).addValue("paused",paused); + var template=new org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate(jdbc); + long total=template.queryForObject("SELECT count(*) "+ORIGINAL_LIST_FROM,params,Long.class); + var rows=template.query("WITH selected AS ("+ORIGINAL_LIST_PROJECTION+ORIGINAL_LIST_FROM + +" ORDER BY c.first_seen_at DESC,c.id DESC LIMIT :limit OFFSET :offset) "+ORIGINAL_LIST_CHANNELS + +" ORDER BY s.first_seen_at DESC,s.id DESC",params,(rs,row) -> { + var values=rs.getArray("channels"); + var channels=values==null?List.of():java.util.Arrays.stream((String[])values.getArray()) + .filter(java.util.Objects::nonNull).map(Channel::valueOf).sorted().toList(); + return new RepositorySummary(rs.getLong("repository_id"),rs.getString("owner_login"),rs.getString("repository_name"), + rs.getTimestamp("first_seen_at").toInstant(),rs.getTimestamp("last_discovered_at")==null?null:rs.getTimestamp("last_discovered_at").toInstant(), + RepositoryStatus.valueOf(rs.getString("display_status")),channels,rs.getObject("source_id",Long.class), + rs.getBoolean("paused"),rs.getString("wait_reason"),rs.getLong("id")); + }); + return new DiscoveryPage<>(rows,total,page,size,asOf); + } + private static final String ORIGINAL_LIST_STATUS="CASE c.status WHEN 'BOUND' THEN 'VALID' WHEN 'REJECTED' THEN CASE WHEN c.last_error_code='NO_VALID_SKILL' THEN 'NO_VALID_SKILL' ELSE 'VALIDATION_FAILED' END ELSE c.status END"; + private static final String ORIGINAL_LIST_FROM=""" + FROM github_discovery_candidate c JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id + WHERE c.namespace_id=:ns AND e.has_official AND c.first_seen_at<=:asOf + AND (CAST(:search AS text) IS NULL OR strpos(lower(c.owner_login || '/' || c.repository_name),lower(:search))>0) + AND (CAST(:paused AS boolean) IS NULL OR NOT e.acquisition_allowed=:paused) + AND (CAST(:channel AS text) IS NULL OR EXISTS(SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_deep_discovery_query q ON q.id::text=origin.origin_key + WHERE origin.candidate_id=c.id AND origin.origin_type='OFFICIAL_DISCOVERY' AND q.definition_json->>'channel'=:channel)) + """+" AND (CAST(:status AS text) IS NULL OR ("+ORIGINAL_LIST_STATUS+")=:status)"; + private static final String ORIGINAL_LIST_PROJECTION=""" + SELECT c.*,e.source_id,NOT e.acquisition_allowed AS paused, + CASE WHEN e.source_status='PAUSED' THEN 'SOURCE_PAUSED' + WHEN NOT e.acquisition_allowed AND e.excluded THEN 'EXCLUDED' + WHEN NOT e.acquisition_allowed THEN 'DISABLED' ELSE NULL END AS wait_reason, + """+ORIGINAL_LIST_STATUS+" AS display_status "; + private static final String ORIGINAL_LIST_CHANNELS=""" + SELECT s.*,o.channels,o.last_discovered_at FROM selected s + LEFT JOIN LATERAL (SELECT array_agg(DISTINCT q.definition_json->>'channel') AS channels,max(o.last_seen_at) AS last_discovered_at + FROM github_discovery_candidate_origin o JOIN github_deep_discovery_query q ON q.id::text=o.origin_key + WHERE o.candidate_id=s.id AND o.origin_type='OFFICIAL_DISCOVERY') o ON TRUE + """; + + private ChannelProgress channel(Channel channel) { return reads.progress().channels().stream().filter(c -> c.channel()==channel).findFirst().orElseThrow(); } + private void assertOfficialOnly(boolean expected) { + assertThat(queries.hasOfficialOnlyCandidates(namespace)).isEqualTo(expected); + assertThat(queries.hasOfficialOnlyCandidates(namespace)).isEqualTo(jdbc.queryForObject("SELECT count(*)>0 FROM github_candidate_discovery_eligibility WHERE namespace_id=? AND has_official AND official_only",Boolean.class,namespace)); + } + private void assertEquivalentProgress(long ns) { + var channels=new java.util.ArrayList();channels.add(null);channels.addAll(List.of(Channel.values())); + for(Channel channel:channels) for(Long repo:java.util.Arrays.asList(null,300L,399L)) { + var params=new org.springframework.jdbc.core.namedparam.MapSqlParameterSource("ns",ns) + .addValue("repo",repo==null?null:repo.toString()).addValue("channel",channel==null?null:channel.name()); + var original=new org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate(jdbc) + .query(ORIGINAL_PROGRESS,params,(rs,row) -> rs.getString("work")); + assertThat(queries.progressingWork(ns,repo,channel)).as("ns=%s repo=%s channel=%s",ns,repo,channel).containsExactlyInAnyOrderElementsOf(original); + } + } + private long otherNamespace() { return jdbc.queryForObject("INSERT INTO namespace(slug,display_name,type) VALUES ('read-model-other','Other','TEAM') RETURNING id",Long.class); } + private long candidate(long ns,long repo,boolean official) { + long id=jdbc.queryForObject("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) VALUES (?,?,'fixture','skills') RETURNING id",Long.class,ns,repo); + if(official) jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?)",id,UUID.randomUUID().toString()); + return id; + } + private UUID independent(long ns,long repo) { + UUID id=UUID.randomUUID();jdbc.update("INSERT INTO github_discovery_entry(id,namespace_id,entry_type,title,criteria_json,enabled,target_interval_seconds) VALUES (?, ?,'SEARCH_RULE',?,'{}',TRUE,3600)",id,ns,"fixture-"+repo); + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) SELECT id,'SEARCH_RULE',? FROM github_discovery_candidate WHERE namespace_id=? AND repository_id=?",id.toString(),ns,repo);return id; + } + private long source(long ns,long repo,String status) { return jdbc.queryForObject("INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility,status) VALUES (?,?,'fixture','skills','PUBLIC',?) RETURNING id",Long.class,ns,repo,status); } + private UUID officialTask(Channel channel) { + UUID query=UUID.randomUUID(),task=UUID.randomUUID(),run=run("DISCOVERY"); + jdbc.update("INSERT INTO github_deep_discovery_query(id,namespace_id,query_key,query_version,strategy_version,definition_json) VALUES (?,?,?,1,'fixture',jsonb_build_object('channel',?::text))",query,namespace,query.toString(),channel.name()); + if(channel==Channel.OWNER_EXPANSION) jdbc.update("UPDATE github_deep_discovery_query SET query_source='OWNER',github_owner_id=900,owner_login='fixture-owner',seed_repository_id=300 WHERE id=?",query); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,kind,dedupe_key,snapshot_json,official_query_id,required_capabilities,status,lease_token_hash,lease_expires_at,leased_by,last_progress_at) VALUES (?,?,?,'DISCOVERY',?,'{}',?,ARRAY['official-discovery/v1','official-checkpoint/v1','official-pause/v1'],'LEASED',?,CURRENT_TIMESTAMP+INTERVAL '1 hour','fixture',CURRENT_TIMESTAMP)",task,run,namespace,task.toString(),query,"a".repeat(64)); + jdbc.update("UPDATE github_deep_discovery_query SET active_task_id=? WHERE id=?",task,query);return task; + } + private void repositoryTask(long repo,long source) { + UUID task=UUID.randomUUID(),run=run("REPOSITORY_SYNC"); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,source_id,kind,mode,dedupe_key,snapshot_json,status,lease_token_hash,lease_expires_at,leased_by,last_progress_at) VALUES (?,?,?,?,'REPOSITORY_SYNC','FULL',?,jsonb_build_object('repositoryId',?::bigint),'LEASED',?,CURRENT_TIMESTAMP+INTERVAL '1 hour','fixture',CURRENT_TIMESTAMP)",task,run,namespace,source,task.toString(),repo,"b".repeat(64)); + } + private UUID run(String kind) { UUID id=UUID.randomUUID();jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,?)",id,namespace,kind);return id; } + // Frozen pre-optimization SQL is an equivalence oracle, not a new production path. + private static final String ORIGINAL_PROGRESS = """ + WITH tasks AS ( + SELECT t.kind,t.mode,COALESCE(e.source_id,t.source_id) source_id FROM github_sync_task t + LEFT JOIN github_discovery_candidate c ON c.namespace_id=t.namespace_id + AND c.repository_id::text=t.snapshot_json->>'repositoryId' + LEFT JOIN github_candidate_discovery_eligibility e ON e.candidate_id=c.id + LEFT JOIN github_deep_discovery_query q ON q.id=t.official_query_id + JOIN github_deep_discovery_config config ON config.namespace_id=t.namespace_id + WHERE t.namespace_id=:ns AND t.status='LEASED' AND t.lease_expires_at>CURRENT_TIMESTAMP + AND t.last_progress_at>CURRENT_TIMESTAMP-INTERVAL '90 seconds' + AND (CAST(:repo AS text) IS NULL OR t.snapshot_json->>'repositoryId'=:repo) + AND (CAST(:channel AS text) IS NULL OR q.definition_json->>'channel'=:channel) + AND ((t.official_query_id IS NOT NULL AND config.enabled AND NOT EXISTS( + SELECT 1 FROM github_official_owner_eligibility oe WHERE oe.query_id=q.id AND oe.excluded)) + OR (e.has_official AND e.acquisition_allowed)) + ) SELECT DISTINCT CASE WHEN kind='DISCOVERY' THEN 'NEW_DISCOVERY' + WHEN source_id IS NULL THEN 'CANDIDATE_VALIDATION' ELSE 'SOURCE_UPDATE' END work FROM tasks + UNION SELECT 'PACKAGE_ACQUISITION' FROM tasks WHERE kind='REPOSITORY_SYNC' AND mode='FULL' + """; +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialBudgetIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialBudgetIntegrationTest.java new file mode 100644 index 000000000..cbb773bcd --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialBudgetIntegrationTest.java @@ -0,0 +1,210 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.infra.jpa.JdbcGithubOfficialDiscoveryBudgetRepository; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles("test") +@Testcontainers +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubOfficialBudgetIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired JdbcTemplate jdbc; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired GithubSyncTaskRepository tasks; + @Autowired GithubOfficialDiscoveryBudgetRepository budget; + @Autowired GithubOfficialQuotaRepository quota; + @Autowired GithubCredentialLeaseAppService claims; + @Autowired GithubDeepDiscoveryQueryRepository queries; + @Autowired PlatformTransactionManager transactions; + + @BeforeEach void setup() { + jdbc.execute("TRUNCATE github_sync_task,github_deep_discovery_query,github_system_credential,github_discovery_candidate CASCADE"); + jdbc.execute("TRUNCATE github_official_discovery_reservation"); + jdbc.execute("DELETE FROM github_official_resource_cooldown"); + jdbc.execute("DELETE FROM github_official_secondary_cooldown"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE,budget_profile='BALANCED' WHERE id"); + advanceBudget(); + jdbc.update("UPDATE github_sync_admission_policy SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000,candidate_pending_limit=10000 WHERE id"); + } + + @Test void profilesChargeSnapshotRequestsWithoutIdleCredit() { + for (var entry: java.util.Map.of("ECONOMY",600,"BALANCED",200,"ACCELERATED",100).entrySet()) { + jdbc.update("UPDATE github_deep_discovery_config SET budget_profile=? WHERE id",entry.getKey()); + expireAll();advanceBudget();enqueue(1); + var lease=claim(); + assertThat(jdbc.queryForObject("SELECT reserved_requests FROM github_official_discovery_reservation WHERE task_id=? AND lease_generation=?",Integer.class,lease.task().taskId(),lease.task().leaseGeneration())).isEqualTo(10); + assertThat(jdbc.queryForObject("SELECT extract(epoch from next_lease_eligible_at-granted_at)::integer FROM github_official_discovery_reservation WHERE task_id=? AND lease_generation=?",Integer.class,lease.task().taskId(),lease.task().leaseGeneration())).isEqualTo(entry.getValue()); + assertThat(queries.discoveryBudgetWait().retryAt()).isAfter(Instant.now()); + } + } + + @Test void profilePauseAndRepositoryRecreationCannotResetReservation() { + enqueue(1);var lease=claim();Instant next=next(); + new TransactionTemplate(transactions).executeWithoutResult(status -> new JdbcGithubOfficialDiscoveryBudgetRepository(jdbc).reserveSuccessfulClaim(lease.task().taskId(),lease.task().leaseGeneration())); + assertThat(next()).isEqualTo(next); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE,budget_profile='ACCELERATED' WHERE id"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + expireAll(); + assertThat(claimOptional()).isEmpty();assertThat(next()).isEqualTo(next); + advanceBudget();var second=claim(); + assertThat(second.task().leaseGeneration()).isGreaterThan(lease.task().leaseGeneration()); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isEqualTo(2); + assertThat(jdbc.queryForObject("SELECT extract(epoch from next_lease_eligible_at-granted_at)::integer FROM github_official_discovery_reservation WHERE task_id=? AND lease_generation=?",Integer.class,second.task().taskId(),second.task().leaseGeneration())).isEqualTo(100); + } + + @Test void concurrentWorkersShareOnePermitAndActiveLeaseSurvivesPacingExpiry() throws Exception { + enqueue(3);var start=new CountDownLatch(1); + try(var pool=Executors.newFixedThreadPool(2)) { + var first=pool.submit(()->{start.await();return claimOptional();}); + var second=pool.submit(()->{start.await();return claimOptional();});start.countDown(); + assertThat(List.of(first.get(20,TimeUnit.SECONDS),second.get(20,TimeUnit.SECONDS)).stream().filter(java.util.Optional::isPresent).count()).isEqualTo(1); + } + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isEqualTo(1); + advanceBudget();assertThat(claimOptional()).isEmpty(); + } + + @Test void unavailableCredentialsAndQuotaDoNotSpendBudget() { + enqueue(1);Instant previous=next(); + jdbc.update("INSERT INTO github_system_credential(id,label,fingerprint,key_version,nonce,ciphertext,status) VALUES (?,'fixture','budgetfixture','v1',?,?,'INVALID')",UUID.randomUUID(),new byte[12],new byte[17]); + assertThat(claimOptional()).isEmpty();assertThat(next()).isEqualTo(previous); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isZero(); + jdbc.update("DELETE FROM github_system_credential"); + jdbc.update("UPDATE github_sync_task SET due_at=clock_timestamp()-INTERVAL '1 second'"); + jdbc.update("INSERT INTO github_official_resource_cooldown(resource,primary_until) VALUES ('SEARCH',clock_timestamp()+INTERVAL '1 hour')"); + assertThat(claimOptional()).isEmpty();assertThat(next()).isEqualTo(previous); + jdbc.update("DELETE FROM github_official_resource_cooldown"); + jdbc.update("UPDATE github_sync_task SET due_at=clock_timestamp()-INTERVAL '1 second'"); + assertThat(claimOptional()).isPresent(); + } + + @Test void blockedOfficialBudgetDoesNotStarveLegacyDiscovery() { + enqueue(1);claim(); + UUID task=queueLegacy(); + assertThat(claim().task().taskId()).isEqualTo(task); + UUID nextLegacy=queueLegacy(); + expireAll();advanceBudget(); + var official=claim(); + assertThat(official.task().taskId()).isNotEqualTo(nextLegacy); + assertThat(claim().task().taskId()).isEqualTo(nextLegacy); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isEqualTo(2); + } + + @Test void pendingCapacityFilledAfterEnqueuePreventsDebitButKeepsLegacyRunnable() { + enqueue(1);Instant before=next(); + jdbc.update("UPDATE github_sync_admission_policy SET candidate_pending_limit=1 WHERE id"); + jdbc.update("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) SELECT namespace_id,999,'fixture','repo' FROM github_deep_discovery_config WHERE id"); + assertThat(claimOptional()).isEmpty();assertThat(next()).isEqualTo(before); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isZero(); + UUID legacy=queueLegacy();assertThat(claim().task().taskId()).isEqualTo(legacy); + jdbc.update("UPDATE github_discovery_candidate SET status='REJECTED'"); + assertThat(claimOptional()).isPresent(); + } + + @Test void actualSnapshotRequestAllowanceDeterminesTheCharge() { + enqueue(1); + jdbc.update("UPDATE github_sync_task SET snapshot_json=jsonb_set(snapshot_json,'{officialDiscovery,budget,maxRequests}','3') WHERE official_query_id IS NOT NULL"); + claim(); + assertThat(jdbc.queryForObject("SELECT reserved_requests FROM github_official_discovery_reservation",Integer.class)).isEqualTo(3); + assertThat(jdbc.queryForObject("SELECT extract(epoch from next_lease_eligible_at-granted_at)::integer FROM github_official_discovery_reservation",Integer.class)).isEqualTo(60); + } + + @Test void oldestSuccessfullyClaimedQueryWinsOverPriorityAndRenewals() { + enqueue(3);var first=claim(); + jdbc.update("UPDATE github_sync_task SET priority=100000 WHERE id=?",first.task().taskId()); + expireAll();advanceBudget();var second=claim(); + assertThat(second.task().taskId()).isNotEqualTo(first.task().taskId()); + expireAll();advanceBudget();var third=claim(); + assertThat(third.task().taskId()).isNotIn(first.task().taskId(),second.task().taskId()); + } + + @Test void skewedCallerClockCannotCreateExpiredOfficialLeaseOrRenewal() { + enqueue(1); + Instant skew=Instant.now().minusSeconds(3600); + var lease=new TransactionTemplate(transactions).execute(status -> { + var result=tasks.claim(GithubSyncTaskKind.DISCOVERY,"skewed","a".repeat(64),skew,skew.plusSeconds(300),platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow(); + budget.reserveSuccessfulClaim(result.taskId(),result.leaseGeneration()); + return result; + }); + assertThat(lease.leaseExpiresAt()).isAfter(Instant.now().plusSeconds(290)); + Instant future=Instant.now().plusSeconds(3600); + var renewal=new TransactionTemplate(transactions).execute(status -> tasks.renew(lease.taskId(),lease.leaseGeneration(),"a".repeat(64),future,future.plusSeconds(300))); + assertThat(renewal.orElseThrow()).isBetween(Instant.now().plusSeconds(290),Instant.now().plusSeconds(310)); + Boolean accepted=new TransactionTemplate(transactions).execute(status -> quota.record(lease.taskId(),lease.leaseGeneration(),"a".repeat(64),null, + new GithubOfficialQuotaRepository.Observation("SEARCH",future,"OBSERVED",100,future.plusSeconds(600),null),future)); + assertThat(accepted).isTrue(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isEqualTo(1); + } + + @Test void expiredBeforeCredentialFinalizationRollsBackWithoutCharge() { + enqueue(1);Instant before=next(); + assertThatThrownBy(() -> new TransactionTemplate(transactions).executeWithoutResult(status -> { + var lease=tasks.claim(GithubSyncTaskKind.DISCOVERY,"expired","b".repeat(64),Instant.now(),Instant.now().plusSeconds(300),platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow(); + expireAll();budget.reserveSuccessfulClaim(lease.taskId(),lease.leaseGeneration()); + })).isInstanceOf(org.springframework.dao.InvalidDataAccessApiUsageException.class) + .hasRootCauseInstanceOf(IllegalStateException.class); + assertThat(next()).isEqualTo(before); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isZero(); + assertThat(claimOptional()).isPresent(); + } + + @Test void retainedTaskCleanupDoesNotResetGlobalPacing() { + enqueue(1);var lease=claim();Instant before=next(); + jdbc.update("UPDATE github_deep_discovery_query SET active_task_id=NULL WHERE active_task_id=?",lease.task().taskId()); + jdbc.update("DELETE FROM github_official_discovery_shard WHERE task_id=?",lease.task().taskId()); + jdbc.update("DELETE FROM github_sync_task WHERE id=?",lease.task().taskId()); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_reservation",Integer.class)).isZero(); + assertThat(next()).isEqualTo(before); + } + + private UUID queueLegacy() { + long ns=jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id",Long.class); + UUID run=UUID.randomUUID(),task=UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'DISCOVERY')",run,ns); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,kind,dedupe_key,snapshot_json,priority) VALUES (?,?,?,'DISCOVERY',?,?::jsonb,100000)",task,run,ns,task.toString(),"{\"entryType\":\"SEARCH_RULE\",\"ruleId\":\""+UUID.randomUUID()+"\"}"); + return task; + } + + private void enqueue(int count) { assertThat(scheduler.enqueueDue(Instant.now(),count)).isEqualTo(count); } + private void advanceBudget() { jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 day'"); } + private void expireAll() { jdbc.update("UPDATE github_sync_task SET lease_expires_at=clock_timestamp()-INTERVAL '1 second' WHERE status='LEASED' AND official_query_id IS NOT NULL"); } + private Instant next() { return jdbc.queryForObject("SELECT next_lease_eligible_at FROM github_official_discovery_budget",java.sql.Timestamp.class).toInstant(); } + private java.util.Optional claimOptional() { return claims.claim(GithubSyncTaskKind.DISCOVERY,"budget-worker",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)); } + private GithubSyncLeaseService.LeasedTask claim() { return claimOptional().orElseThrow().lease(); } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDeepDiscoveryIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDeepDiscoveryIntegrationTest.java new file mode 100644 index 000000000..4c9937548 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDeepDiscoveryIntegrationTest.java @@ -0,0 +1,294 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubOfficialDeepDiscoveryIntegrationTest { + private static final String BASE = "/api/v1/admin/github-sync/deep-discovery"; + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired MockMvc mvc; + @Autowired ObjectMapper mapper; + @Autowired JdbcTemplate jdbc; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired GithubSyncLeaseService leases; + @Autowired GithubOfficialLifecycleRepository lifecycle; + @Autowired GithubOfficialQuotaRepository quota; + @Autowired GithubCredentialLeaseAppService credentialClaims; + + @BeforeEach void setup() { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update("UPDATE github_sync_admission_policy SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000 WHERE id"); + } + + @Test void adminSecurityCsrfCasAndBoundedReadQueries() throws Exception { + mvc.perform(get(BASE+"/configuration")).andExpect(status().isUnauthorized()); + mvc.perform(get(BASE+"/configuration").with(authentication(auth("USER")))).andExpect(status().isForbidden()); + String body="{\"expectedVersion\":1,\"enabled\":true,\"budgetProfile\":\"BALANCED\",\"exclusions\":[]}"; + mvc.perform(put(BASE+"/configuration").with(authentication(auth("USER"))).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(body)).andExpect(status().isForbidden()); + mvc.perform(put(BASE+"/configuration").with(authentication(auth("SUPER_ADMIN"))) + .contentType(MediaType.APPLICATION_JSON).content(body)).andExpect(status().isUnauthorized()); + mvc.perform(put(BASE+"/configuration").with(authentication(auth("SUPER_ADMIN"))).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(body)).andExpect(status().isOk()).andExpect(jsonPath("$.data.version").value(2)); + mvc.perform(put(BASE+"/configuration").with(authentication(auth("SUPER_ADMIN"))).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(body)).andExpect(status().isConflict()); + var lease=claim(); report(lease,null,checkpoint(lease,2,"IN_PROGRESS"),41,"allowed"); + for(String path:List.of("/repositories","/repositories/41","/repositories/41/provenance","/repositories/41/paths","/progress")) + mvc.perform(get(BASE+path).with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control","no-store")); + for(int mask=0;mask<16;mask++) { + var request=get(BASE+"/repositories").with(authentication(auth("SUPER_ADMIN"))); + if((mask&1)!=0) request.param("q","allowed"); + if((mask&2)!=0) request.param("status","PENDING"); + if((mask&4)!=0) request.param("channel",snapshot(lease).path("channel").asText()); + if((mask&8)!=0) request.param("paused","false"); + mvc.perform(request).andExpect(status().isOk()).andExpect(jsonPath("$.data.total").value(1)); + } + mvc.perform(get(BASE+"/repositories").param("channel",snapshot(lease).path("channel").asText()) + .param("paused","true").with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isOk()).andExpect(jsonPath("$.data.total").value(0)); + mvc.perform(get(BASE+"/summary").param("from",jdbc.queryForObject("SELECT available_since FROM github_discovery_metrics_epoch WHERE id",java.sql.Timestamp.class).toInstant().minusSeconds(1).toString()) + .param("until",Instant.now().toString()).with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isOk()).andExpect(jsonPath("$.data.recent.complete").value(false)); + mvc.perform(get(BASE+"/summary").param("from",jdbc.queryForObject("SELECT available_since FROM github_discovery_metrics_epoch WHERE id",java.sql.Timestamp.class).toInstant().toString()) + .param("until",Instant.now().toString()).with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isOk()).andExpect(jsonPath("$.data.recent.complete").value(true)) + .andExpect(jsonPath("$.data.recent.newCandidateRepositories").value(1)) + .andExpect(jsonPath("$.data.current.pendingRepositoryChecks").value(1)); + mvc.perform(get(BASE+"/repositories").param("page","1001").with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isBadRequest()); + mvc.perform(get(BASE+"/repositories").param("size","101").with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isBadRequest()); + mvc.perform(get(BASE+"/repositories").param("asOf",Instant.now().plusSeconds(600).toString()).with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isBadRequest()); + mvc.perform(get(BASE+"/repositories/999999").with(authentication(auth("SUPER_ADMIN")))) + .andExpect(status().isNotFound()); + } + + @Test void capabilityIsolationReplayConflictAndExclusionArePersisted() throws Exception { + assertThat(scheduler.enqueueDue(Instant.now(),1)).isEqualTo(1); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY,"legacy", platform(java.util.List.of()))).isEmpty(); + var lease=leases.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow(); + String checkpoint=checkpoint(lease,2,"IN_PROGRESS"); + report(lease,null,checkpoint,42,"allowed"); + report(lease,null,checkpoint,42,"allowed"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate WHERE repository_id=42",Integer.class)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event WHERE event_type='CANDIDATE_CREATED' AND repository_id=42",Integer.class)).isEqualTo(1); + assertThatThrownBy(()->report(lease,null,checkpoint,43,"allowed")).isInstanceOf(GithubOfficialProtocolException.class); + jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"excluded\"}]'::jsonb WHERE id"); + String next=checkpoint(lease,3,"IN_PROGRESS"); + report(lease,checkpoint,next,44,"excluded"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate WHERE repository_id=44",Integer.class)).isZero(); + assertThat(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?",String.class,lease.task().taskId())).isEqualTo(next); + assertThat(jdbc.queryForObject("SELECT (official_metadata->>'excludedCount')::int FROM github_discovery_batch WHERE task_id=? AND checkpoint=?",Integer.class,lease.task().taskId(),next)).isEqualTo(1); + } + + @Test void pauseRejectsOldLeaseAndResumesSavedCheckpoint() throws Exception { + var lease=claim();String cp=checkpoint(lease,2,"IN_PROGRESS");report(lease,null,cp,45,"allowed"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + lifecycle.pauseDiscovery(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(),cp); + assertThatThrownBy(()->lifecycle.control(lease.task().taskId(),lease.task().leaseGeneration(),lease.token())) + .isInstanceOf(GithubOfficialProtocolException.class); + assertThatThrownBy(()->report(lease,cp,checkpoint(lease,3,"IN_PROGRESS"),46,"allowed")) + .isInstanceOf(GithubOfficialProtocolException.class); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES))).isEmpty(); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 second'"); + var resumed=leases.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow(); + assertThat(resumed.task().taskId()).isEqualTo(lease.task().taskId()); + assertThat(resumed.task().leaseGeneration()).isGreaterThan(lease.task().leaseGeneration()); + assertThat(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?",String.class,resumed.task().taskId())).isEqualTo(cp); + } + + @Test void primarySearchIsResourceScopedAndSecondaryIsSharedWithLeaseProof() throws Exception { + var lease=claim();Instant now=Instant.now();String hash=hash(lease.token()); + var primary=new GithubOfficialQuotaRepository.Observation("SEARCH",now,"PRIMARY_EXHAUSTED",0,now.plusSeconds(120),null); + assertThat(quota.record(lease.task().taskId(),lease.task().leaseGeneration(),hash,UUID.randomUUID(),primary,now)).isFalse(); + assertThat(quota.record(lease.task().taskId(),lease.task().leaseGeneration(),hash,null,primary,now)).isTrue(); + assertThat(quota.cooldown("SEARCH",now)).isAfter(now); + assertThat(quota.cooldown("CORE",now)).isNull(); + assertThat(jdbc.queryForObject("SELECT observation_key FROM github_official_quota_observation",String.class)).isEqualTo("anonymous"); + queueCore(88001); + assertThat(credentialClaims.claim(GithubSyncTaskKind.REPOSITORY_SYNC,"core-worker", platform(java.util.List.of()))).isPresent(); + assertThat(quota.record(lease.task().taskId(),lease.task().leaseGeneration(),hash,null, + new GithubOfficialQuotaRepository.Observation("SEARCH",now.plusSeconds(1),"SECONDARY_COOLDOWN",null,null,60),now.plusSeconds(1))).isTrue(); + assertThat(quota.cooldown("CORE",now)).isAfter(now); + UUID blocked=queueCore(88002); + assertThat(credentialClaims.claim(GithubSyncTaskKind.REPOSITORY_SYNC,"core-worker", platform(java.util.List.of()))).isEmpty(); + assertThat(jdbc.queryForObject("SELECT official_pause_reason FROM github_sync_task WHERE id=?",String.class,blocked)).isEqualTo("QUOTA_COOLDOWN"); + assertThat(quota.record(lease.task().taskId(),lease.task().leaseGeneration(),"a".repeat(64),null,primary,now)).isFalse(); + } + + @Test void configuredInvalidTokenDefersInsteadOfFallingBackToAnonymous() { + jdbc.update(""" + INSERT INTO github_system_credential(id,label,fingerprint,key_version,nonce,ciphertext,status) + VALUES (?,'fixture','0123456789abcdef','fixture',?,?,'INVALID') + """,UUID.randomUUID(),new byte[12],new byte[17]); + assertThat(scheduler.enqueueDue(Instant.now(),1)).isEqualTo(1); + assertThat(credentialClaims.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES))).isEmpty(); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE official_query_id IS NOT NULL",String.class)).isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject("SELECT official_pause_reason FROM github_sync_task WHERE official_query_id IS NOT NULL",String.class)).isEqualTo("CREDENTIAL_UNAVAILABLE"); + jdbc.update("DELETE FROM github_system_credential"); + jdbc.update("UPDATE github_sync_task SET due_at=CURRENT_TIMESTAMP-INTERVAL '1 second',retry_at=CURRENT_TIMESTAMP-INTERVAL '1 second'"); + assertThat(credentialClaims.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES))).isPresent(); + assertThat(jdbc.queryForObject("SELECT official_pause_reason FROM github_sync_task WHERE official_query_id IS NOT NULL",String.class)).isNull(); + } + + @Test void configuredDisabledTokenDefersProductionClaimWithoutAnonymousFallback() { + jdbc.update(""" + INSERT INTO github_system_credential(id,label,fingerprint,key_version,nonce,ciphertext,status) + VALUES (?,'disabled fixture','fedcba9876543210','fixture',?,?,'DISABLED') + """, UUID.randomUUID(), new byte[12], new byte[17]); + assertThat(scheduler.enqueueDue(Instant.now(), 1)).isEqualTo(1); + assertThat(credentialClaims.claim(GithubSyncTaskKind.DISCOVERY, "official", + platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES))).isEmpty(); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE official_query_id IS NOT NULL", String.class)) + .isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject("SELECT official_pause_reason FROM github_sync_task WHERE official_query_id IS NOT NULL", String.class)) + .isEqualTo("CREDENTIAL_UNAVAILABLE"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE official_query_id IS NOT NULL AND credential_id IS NOT NULL", Integer.class)) + .isZero(); + } + + @Test void excludedCandidatesStillConsumeBoundedOriginalResponseBudget() throws Exception { + jdbc.update("UPDATE github_sync_admission_policy SET candidate_per_lease_limit=1 WHERE id"); + jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"excluded\"}]'::jsonb WHERE id"); + var lease=claim();String cp=checkpoint(lease,2,"IN_PROGRESS"); + report(lease,null,cp,71,"excluded"); + report(lease,null,cp,71,"excluded"); + assertThatThrownBy(()->report(lease,cp,checkpoint(lease,3,"IN_PROGRESS"),72,"excluded")) + .isInstanceOf(GithubOfficialProtocolException.class); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate",Integer.class)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_batch",Integer.class)).isEqualTo(1); + } + + @Test void gapDoesNotBlockSiblingAndOnlyAllLeavesAdvanceCoverage() throws Exception { + var root=claim();var snapshot=snapshot(root);UUID query=UUID.fromString(snapshot.path("queryId").asText()); + jdbc.update("UPDATE github_deep_discovery_query SET next_due_at=CURRENT_TIMESTAMP+INTERVAL '2 days' WHERE id<>?",query); + Instant since=Instant.parse(snapshot.path("range").path("since").asText()); + Instant until=Instant.parse(snapshot.path("range").path("until").asText()); + Instant middle=since.plusSeconds(java.time.Duration.between(since,until).getSeconds()/2); + String split=checkpoint(root,1,"SPLIT"); + var children=List.of(new GithubOfficialDiscoveryBatch.Range(since,middle,null),new GithubOfficialDiscoveryBatch.Range(middle,until,null)); + reportEmpty(root,null,split,"SPLIT",children,null); + reportEmpty(root,null,split,"SPLIT",children,null); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard",Integer.class)).isEqualTo(3); + lifecycle.complete(root.task().taskId(),root.task().leaseGeneration(),root.token(),split,"COMPLETE",null); + var first=claim();String gap=checkpoint(first,1,"GAP"); + reportEmpty(first,null,gap,"GAP",List.of(),"UNSPLITTABLE"); + assertThat(lifecycle.complete(first.task().taskId(),first.task().leaseGeneration(),first.token(),gap,"INCOMPLETE",Instant.now().plusSeconds(600))).isEqualTo("PARTIAL"); + var sibling=claim();assertThat(sibling.task().taskId()).isNotEqualTo(first.task().taskId()); + String complete=checkpoint(sibling,2,"COMPLETE");reportEmpty(sibling,null,complete,"COMPLETE",List.of(),null); + lifecycle.complete(sibling.task().taskId(),sibling.task().leaseGeneration(),sibling.token(),complete,"COMPLETE",null); + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query)).isNull(); + jdbc.update("UPDATE github_official_discovery_shard SET retry_at=CURRENT_TIMESTAMP-INTERVAL '1 second' WHERE state='GAP'"); + jdbc.update("UPDATE github_deep_discovery_query SET next_due_at=CURRENT_TIMESTAMP-INTERVAL '1 second' WHERE id=?",query); + var retry=claim();assertThat(retry.task().taskId()).isEqualTo(first.task().taskId()); + assertThat(retry.task().leaseGeneration()).isGreaterThan(first.task().leaseGeneration()); + String done=checkpoint(retry,2,"COMPLETE");reportEmpty(retry,retry.task().discoveryCheckpoint(),done,"COMPLETE",List.of(),null); + lifecycle.complete(retry.task().taskId(),retry.task().leaseGeneration(),retry.token(),done,"COMPLETE",null); + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query).toInstant()).isEqualTo(until); + } + + private void reportEmpty(GithubSyncLeaseService.LeasedTask lease,String previous,String cp,String coverage, + List children,String gap) { + lifecycle.report(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(), + new GithubDiscoveryBatch(previous,cp,List.of(),0,null), + new GithubOfficialDiscoveryBatch(lease.task().taskId(),coverage,1,children,gap)); + } + + private UUID queueCore(long repositoryId) { + long namespace=jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id",Long.class); + Long source=jdbc.queryForObject(""" + INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility) + VALUES (?,?,'fixture','skills','PUBLIC') RETURNING id + """,Long.class,namespace,repositoryId); + UUID run=UUID.randomUUID(),task=UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'REPOSITORY_SYNC')",run,namespace); + jdbc.update(""" + INSERT INTO github_sync_task(id,run_id,namespace_id,source_id,kind,mode,dedupe_key,snapshot_json) + VALUES (?,?,?,?,'REPOSITORY_SYNC','HEAD_ONLY',?,jsonb_build_object('repositoryId',?::bigint,'owner','fixture','name','skills')) + """,task,run,namespace,source,task.toString(),repositoryId); + return task; + } + + private GithubSyncLeaseService.LeasedTask claim() { + assertThat(scheduler.enqueueDue(Instant.now(),1)).isEqualTo(1); + // Advance only pacing time in this checkpoint-focused fixture; budget tests verify real delays. + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 second'"); + return credentialClaims.claim(GithubSyncTaskKind.DISCOVERY,"official",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow().lease(); + } + private JsonNode snapshot(GithubSyncLeaseService.LeasedTask lease) throws Exception { + return mapper.readTree(lease.task().snapshotJson()).path("officialDiscovery"); + } + private String checkpoint(GithubSyncLeaseService.LeasedTask lease,int page,String coverage) throws Exception { + var result=mapper.createObjectNode();JsonNode snapshot=snapshot(lease); + result.put("formatVersion",1).put("nextPage",page).put("coverage",coverage); + for(String field:List.of("strategyVersion","queryId","queryVersion","shardId")) result.set(field,snapshot.get(field)); + return result.toString(); + } + private void report(GithubSyncLeaseService.LeasedTask lease,String previous,String checkpoint,long repo,String owner) throws Exception { + JsonNode s=snapshot(lease); + var provenance=new GithubOfficialDiscoveryBatch.Provenance(s.path("channel").asText(),UUID.fromString(s.path("queryId").asText()), + s.path("queryVersion").asLong(),s.path("strategyVersion").asText(),lease.task().taskId(),null); + lifecycle.report(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(), + new GithubDiscoveryBatch(previous,checkpoint,List.of(new GithubDiscoveryCandidate(repo,owner,"skills","main",List.of(provenance))),0,null), + new GithubOfficialDiscoveryBatch(lease.task().taskId(),"IN_PROGRESS",1,List.of(),null)); + } + private String hash(String token) throws Exception { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(token.getBytes(StandardCharsets.UTF_8))); } + private UsernamePasswordAuthenticationToken auth(String role) { + var p=new PlatformPrincipal("github-sync-publisher","fixture",null,null,null,Set.of(role)); + return new UsernamePasswordAuthenticationToken(p,null,List.of(new SimpleGrantedAuthority("ROLE_"+role))); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDiscoveryProcessInteropTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDiscoveryProcessInteropTest.java new file mode 100644 index 000000000..413bfa99f --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialDiscoveryProcessInteropTest.java @@ -0,0 +1,190 @@ +package com.iflytek.skillhub.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Real Go OS processes and Java HTTP/PostgreSQL; GitHub responses are controlled fixtures. */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +@Testcontainers +@Import(GithubOfficialDiscoveryProcessInteropTest.RedisFixtureConfig.class) +@TestPropertySource(properties = { + "skillhub.github-sync.service-tokens=fixture-official-process-service-token-32-chars", + "skillhub.builtin-skills.enabled=false" +}) +class GithubOfficialDiscoveryProcessInteropTest { + private static final String SERVICE_TOKEN = "fixture-official-process-service-token-32-chars"; + + @Container + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @Container + static final GenericContainer REDIS = new GenericContainer<>("redis:7-alpine").withExposedPorts(6379); + + @TestConfiguration + static class RedisFixtureConfig { + @Bean + RedisProperties redisProperties() { + RedisProperties properties = new RedisProperties(); + properties.setHost(REDIS.getHost()); + properties.setPort(REDIS.getMappedPort(6379)); + return properties; + } + } + + @DynamicPropertySource + static void properties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.data.redis.host", REDIS::getHost); + registry.add("spring.data.redis.port", () -> REDIS.getMappedPort(6379)); + } + + @Autowired JdbcTemplate jdbc; + @Autowired ObjectMapper mapper; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @LocalServerPort int port; + @TempDir Path temporary; + + @Test + void killedGoProcessIsReplacedWithoutReplayingAcknowledgedPage() throws Exception { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000 WHERE id + """); + assertThat(scheduler.enqueueDue(Instant.now(), 1)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE official_query_id IS NOT NULL AND status='QUEUED' + """, UUID.class); + Path executable = temporary.resolve("official-worker.test"); + Path buildLog = temporary.resolve("build.log"); + Process build = new ProcessBuilder("go", "test", "-c", "-o", executable.toString(), "./internal/worker") + .directory(findGoModule().toFile()).redirectErrorStream(true).redirectOutput(buildLog.toFile()).start(); + awaitExit(build, buildLog, 120); + + Path barrier = temporary.resolve("page-two.ready"); + Path firstLog = temporary.resolve("first.log"); + Process first = process(executable, taskId, "before-kill", barrier, firstLog).start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + while (!Files.exists(barrier) && first.isAlive() && System.nanoTime() < deadline) { + Thread.sleep(25); + } + assertThat(Files.exists(barrier)).as("first process: %s", Files.readString(firstLog)).isTrue(); + String saved = jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?", + String.class, taskId); + assertThat(mapper.readTree(saved).path("nextPage").asInt()).isEqualTo(2); + assertThat(mapper.readTree(saved).path("coverage").asText()).isEqualTo("IN_PROGRESS"); + long firstGeneration = jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id=?", + Long.class, taskId); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate WHERE repository_id=987654901", + Integer.class)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate WHERE repository_id=987654902", + Integer.class)).isZero(); + + first.destroyForcibly(); + assertThat(first.waitFor(10, TimeUnit.SECONDS)).isTrue(); + assertThat(first.exitValue()).isNotZero(); + jdbc.update(""" + UPDATE github_sync_task SET lease_expires_at=CURRENT_TIMESTAMP-INTERVAL '1 second' WHERE id=? + """, taskId); + // This crash-recovery fixture advances pacing instead of waiting 200 seconds. + // The independent budget integration suite verifies the persistent delay itself. + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 second'"); + assertThat(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?", + String.class, taskId)).isEqualTo(saved); + + Path resumedLog = temporary.resolve("resumed.log"); + Process resumed = process(executable, taskId, "resume", barrier, resumedLog).start(); + awaitExit(resumed, resumedLog, 60); + assertThat(jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id=?", + Long.class, taskId)).isEqualTo(firstGeneration + 1); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id=?", + String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + String completed = jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?", + String.class, taskId); + assertThat(mapper.readTree(completed).path("nextPage").asInt()).isEqualTo(2); + assertThat(mapper.readTree(completed).path("coverage").asText()).isEqualTo("COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate WHERE repository_id IN (987654901,987654902) + """, Integer.class)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_outcome_event + WHERE event_type='CANDIDATE_CREATED' AND repository_id IN (987654901,987654902) + """, Integer.class)).isEqualTo(2); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_batch WHERE task_id=?", + Integer.class, taskId)).isEqualTo(3); + } finally { + if (first.isAlive()) { + first.destroyForcibly(); + first.waitFor(10, TimeUnit.SECONDS); + } + } + } + + private ProcessBuilder process(Path executable, UUID taskId, String phase, Path barrier, Path log) { + ProcessBuilder builder = new ProcessBuilder(executable.toString(), + "-test.run=^TestInteropOfficialDiscoveryProcess$", "-test.v", "-test.timeout=90s") + .directory(findGoModule().toFile()).redirectErrorStream(true).redirectOutput(log.toFile()); + builder.environment().keySet().removeIf(key -> key.startsWith("SKILLHUB_") + && (key.contains("LIVE") || key.contains("INTEROP"))); + builder.environment().put("SKILLHUB_INTEROP_OFFICIAL_PROCESS", phase); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", "http://127.0.0.1:" + port + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + builder.environment().put("SKILLHUB_INTEROP_BARRIER", barrier.toString()); + return builder; + } + + private static void awaitExit(Process process, Path log, int timeoutSeconds) throws Exception { + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + throw new AssertionError("Go process timed out: " + Files.readString(log)); + } + assertThat(process.exitValue()).as("Go process: %s", Files.readString(log)).isZero(); + } + + private static Path findGoModule() { + Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath(); + while (current != null) { + Path candidate = current.resolve("github-sync-go"); + if (Files.isRegularFile(candidate.resolve("go.mod"))) { + return candidate; + } + current = current.getParent(); + } + throw new IllegalStateException("Cannot locate github-sync-go module"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialOwnerOutcomeIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialOwnerOutcomeIntegrationTest.java new file mode 100644 index 000000000..79712ee00 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialOwnerOutcomeIntegrationTest.java @@ -0,0 +1,324 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.infra.jpa.JdbcGithubOfficialOwnerRepository; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = {"skillhub.builtin-skills.enabled=false", + "skillhub.security.scanner.enabled=true", "skillhub.security.scanner.mode=upload"}) +class GithubOfficialOwnerOutcomeIntegrationTest { + @org.junit.jupiter.api.io.TempDir static java.nio.file.Path storageDirectory; + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("skillhub.storage.local.base-path", () -> storageDirectory.toString()); + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + @org.springframework.boot.test.mock.mockito.MockBean + org.springframework.data.redis.core.StringRedisTemplate redisTemplate; + @org.springframework.boot.test.mock.mockito.MockBean + org.redisson.api.RedissonClient redissonClient; + @org.springframework.boot.test.mock.mockito.MockBean + com.iflytek.skillhub.stream.ScanTaskConsumer scanTaskConsumer; + @Autowired JdbcTemplate jdbc; + @Autowired ObjectMapper mapper; + @Autowired com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository readModels; + @Autowired GithubSyncIngestionRepository ingestions; + @Autowired GithubSyncLeaseService leases; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired JdbcGithubOfficialOwnerRepository owners; + @Autowired GithubOfficialLifecycleRepository lifecycle; + @Autowired GithubDiscoveryOutcomeRepository outcomes; + @Autowired GithubSyncPublicationRepository publications; + @Autowired com.iflytek.skillhub.service.GithubSyncPublicationAppService publicationService; + @Autowired com.iflytek.skillhub.service.GithubSyncArchiveValidator archiveValidator; + @Autowired com.iflytek.skillhub.storage.ObjectStorageService storage; + @Autowired com.iflytek.skillhub.repository.GithubSyncReviewSourceQueryRepository reviewSources; + @Autowired com.iflytek.skillhub.repository.GithubSkillSourceQueryRepository marketSources; + long namespace; + UUID catalogQuery; + static final String TOKEN = "e".repeat(64), DIGEST = "b".repeat(64); + + @BeforeEach void setup() { + namespace = jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id", Long.class); + scheduler.enqueueDue(Instant.now(), 1); // Disabled configuration reconciles the real Java catalog only. + catalogQuery = jdbc.queryForObject("SELECT id FROM github_deep_discovery_query ORDER BY query_key LIMIT 1", UUID.class); + jdbc.update("UPDATE github_deep_discovery_query SET next_due_at=now()+INTERVAL '2 days'"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + } + + @Test void onlyValidatedIndependentSeedsExpandOnceAndLaterIndependentHitCanQualify() throws Exception { + Fixture first = fixture(1101, 101, true); + owners.reconcile(namespace, Instant.now()); + assertThat(ownerCount()).isZero(); // Uploaded/VALIDATING is not a valid package. + assertThat(ingestions.validated(first.ingestion(), UUID.randomUUID(), DIGEST, 0, "{}", Instant.now())).isFalse(); + assertThat(ownerCount()).isZero(); + validate(first); + owners.reconcile(namespace, Instant.now()); + assertThat(ownerCount()).isEqualTo(1); + UUID ownerQuery = jdbc.queryForObject("SELECT id FROM github_deep_discovery_query WHERE query_source='OWNER'", UUID.class); + Fixture expanded = fixture(1102, 202, false); + origin(expanded.candidate(), ownerQuery); + validate(expanded); + Fixture nonOfficial = fixture(1103, 303, false); + validate(nonOfficial); + Fixture onlyBound = fixture(1104, 404, true); + jdbc.update("UPDATE github_discovery_candidate SET status='BOUND' WHERE id=?", onlyBound.candidate()); + owners.reconcile(namespace, Instant.now()); + assertThat(ownerCount()).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_source_binding WHERE repository_id=1103", Integer.class)).isEqualTo(1); + origin(expanded.candidate(), catalogQuery); + owners.reconcile(namespace, Instant.now()); + assertThat(ownerCount()).isEqualTo(2); + Fixture sameOwner = fixture(1105, 101, true); + validate(sameOwner); + owners.reconcile(namespace, Instant.now()); + assertThat(ownerCount()).isEqualTo(2); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event WHERE repository_id=1101 AND event_type='FIRST_VALID'", Integer.class)).isEqualTo(1); + assertThat(ingestions.validated(first.ingestion(), first.validationToken(), DIGEST, 0, "{}", Instant.now())).isFalse(); + } + + @Test void transferredRepositoryCannotRetargetAnOwnerAndLegacyMetadataClearsUnknownIdentity() throws Exception { + Fixture seed = fixture(1201, 111, true); + validate(seed); + owners.reconcile(namespace, Instant.now()); + assertThat(leases.updateRepositoryMetadata(seed.task(), 1, TOKEN, 1201, "new-owner", "skills", "main", 222L)) + .isEqualTo(GithubRepositoryMetadataResult.UPDATED); + assertThat(jdbc.queryForObject("SELECT github_owner_id FROM github_source_binding WHERE repository_id=1201", Long.class)).isEqualTo(222L); + owners.reconcile(namespace, Instant.now()); + assertThat(jdbc.queryForList("SELECT github_owner_id FROM github_deep_discovery_query WHERE query_source='OWNER'", Long.class)).containsExactly(111L); + assertThat(leases.updateRepositoryMetadata(seed.task(), 1, TOKEN, 1201, "unknown-owner", "skills", "main")) + .isEqualTo(GithubRepositoryMetadataResult.UPDATED); + assertThat(jdbc.queryForObject("SELECT github_owner_id FROM github_source_binding WHERE repository_id=1201", Long.class)).isNull(); + assertThat(jdbc.queryForObject("SELECT github_owner_id FROM github_discovery_candidate WHERE repository_id=1201", Long.class)).isNull(); + assertThat(jdbc.queryForObject("SELECT jsonb_exists(snapshot_json, 'ownerId') FROM github_sync_task WHERE id=?", Boolean.class, seed.task())).isFalse(); + assertThat(jdbc.queryForList("SELECT github_owner_id FROM github_deep_discovery_query WHERE query_source='OWNER'", Long.class)).containsExactly(111L); + } + + @Test void canonicalOwnerReportAppliesExclusionWithoutChangingFrozenIdentity() throws Exception { + Fixture seed = fixture(1301, 131, true); + validate(seed); + assertThat(scheduler.enqueueDue(Instant.now(), 1)).isEqualTo(1); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY, "search-only",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES))).isEmpty(); + var lease = leases.claim(GithubSyncTaskKind.DISCOVERY, "owner-worker",platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES)).orElseThrow(); + var snapshot = mapper.readTree(lease.task().snapshotJson()).path("officialDiscovery"); + assertThat(snapshot.path("ownerId").asLong()).isEqualTo(131); + assertThat(snapshot.path("mode").asText()).isEqualTo("OWNER_RECHECK"); + jdbc.update("UPDATE github_deep_discovery_config SET exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"renamed-owner\"}]'::jsonb WHERE id"); + var cp = mapper.createObjectNode().put("formatVersion", 1).put("nextPage", 12345).put("coverage", "IN_PROGRESS"); + for (String field : List.of("queryId", "queryVersion", "strategyVersion", "shardId")) cp.set(field, snapshot.get(field)); + var batch = new GithubDiscoveryBatch(null, cp.toString(), List.of(), 0, null); + var metadata = new GithubOfficialDiscoveryBatch(lease.task().taskId(), "IN_PROGRESS", 1, List.of(), null, "renamed-owner"); + lifecycle.report(lease.task().taskId(), lease.task().leaseGeneration(), lease.token(), batch, metadata); + lifecycle.report(lease.task().taskId(), lease.task().leaseGeneration(), lease.token(), batch, metadata); + assertThat(lifecycle.control(lease.task().taskId(), lease.task().leaseGeneration(), lease.token()).reason()).isEqualTo("EXCLUDED"); + assertThat(jdbc.queryForObject("SELECT snapshot_json->'officialDiscovery'->>'ownerId' FROM github_sync_task WHERE id=?", String.class, lease.task().taskId())).isEqualTo("131"); + assertThat(jdbc.queryForObject("SELECT owner_login FROM github_deep_discovery_query WHERE query_source='OWNER'", String.class)).isEqualTo("renamed-owner"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard", Integer.class)).isZero(); + lifecycle.pauseDiscovery(lease.task().taskId(), lease.task().leaseGeneration(), lease.token(), cp.toString()); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY, "owner-worker",platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES))).isEmpty(); + } + + @Test void firstValidationAttributionIsNotRewrittenByLaterOfficialOrigins() throws Exception { + Fixture fixture = fixture(1401, 141, false); + validate(fixture); + origin(fixture.candidate(), catalogQuery); + outcomes.firstValid(namespace, 1401, Instant.now()); + assertThat(jdbc.queryForObject("SELECT official_at_event FROM github_discovery_outcome_event WHERE repository_id=1401 AND event_type='FIRST_VALID'", Boolean.class)).isFalse(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event WHERE repository_id=1401 AND event_type='FIRST_VALID'", Integer.class)).isEqualTo(1); + } + + @Test void realSkillVersionEventsAreSeparateAndNoChangeDoesNotCreateAnOutcome() throws Exception { + Fixture fixture = fixture(1501, 151, true); + validate(fixture); + var claim = publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(600)).orElseThrow(); + long skill = jdbc.queryForObject("INSERT INTO skill(namespace_id,slug,owner_id) VALUES (?,'owner-outcome','github-sync-publisher') RETURNING id", Long.class, namespace); + long first = jdbc.queryForObject("INSERT INTO skill_version(skill_id,version,created_by) VALUES (?,'1.0.0','github-sync-publisher') RETURNING id", Long.class, skill); + long second = jdbc.queryForObject("INSERT INTO skill_version(skill_id,version,created_by) VALUES (?,'1.0.1','github-sync-publisher') RETURNING id", Long.class, skill); + publications.recordOutcome(claim, skill, first, true, Instant.now()); + publications.recordOutcome(claim, skill, first, true, Instant.now()); + publications.recordOutcome(claim, skill, second, false, Instant.now()); + publications.recordOutcome(claim, skill, second, false, Instant.now()); + assertThat(jdbc.queryForList("SELECT event_type FROM github_discovery_outcome_event WHERE repository_id=1501 AND event_type IN ('NEW_SKILL','UPDATED_VERSION') ORDER BY event_type", String.class)) + .containsExactly("NEW_SKILL", "UPDATED_VERSION"); + assertThat(jdbc.queryForObject("SELECT subject_key FROM github_discovery_outcome_event WHERE event_type='NEW_SKILL'", Long.class)).isEqualTo(skill); + assertThat(jdbc.queryForObject("SELECT subject_key FROM github_discovery_outcome_event WHERE event_type='UPDATED_VERSION'", Long.class)).isEqualTo(second); + int count = jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event", Integer.class); + assertThat(publications.noChange(claim, Instant.now())).isTrue(); + assertThat(publications.noChange(claim, Instant.now())).isFalse(); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id=?", String.class, fixture.ingestion())).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event", Integer.class)).isEqualTo(count); + } + + @Test void actualPublicationCreatesOneSkillThenOneVersionAndIdenticalArchiveIsNoChange() throws Exception { + Fixture first = publishArchive(1601, "First content."); + long skill = jdbc.queryForObject("SELECT published_skill_id FROM github_sync_package_ingestion WHERE id=?", Long.class, first.ingestion()); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id=?", String.class, first.ingestion())).isEqualTo("READY_FOR_REVIEW"); + Fixture updated = publishArchive(1601, "Updated content."); + assertThat(jdbc.queryForObject("SELECT published_skill_id FROM github_sync_package_ingestion WHERE id=?", Long.class, updated.ingestion())).isEqualTo(skill); + assertThat(jdbc.queryForObject("SELECT count(*) FROM skill_version WHERE skill_id=?", Integer.class, skill)).isEqualTo(2); + Fixture identical = publishArchive(1601, "Updated content."); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id=?", String.class, identical.ingestion())).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM skill_version WHERE skill_id=?", Integer.class, skill)).isEqualTo(2); + assertThat(jdbc.queryForList("SELECT event_type FROM github_discovery_outcome_event WHERE repository_id=1601 AND event_type IN ('NEW_SKILL','UPDATED_VERSION') ORDER BY event_type", String.class)) + .containsExactly("NEW_SKILL", "UPDATED_VERSION"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM review_task WHERE skill_id=?", Integer.class, skill)).isEqualTo(2); + long version=jdbc.queryForObject("SELECT published_version_id FROM github_sync_package_ingestion WHERE id=?",Long.class,updated.ingestion()); + long review=jdbc.queryForObject("SELECT id FROM review_task WHERE github_sync_ingestion_id=?",Long.class,updated.ingestion()); + var path=readModels.paths(namespace,1601,0,20,Instant.now()).items().getFirst(); + assertThat(path.skillId()).isEqualTo(skill);assertThat(path.versionId()).isEqualTo(version);assertThat(path.reviewId()).isEqualTo(review); + assertThat(path.outcome()).isEqualTo("ACCEPTED"); + assertThat(path.runId()).isEqualTo(jdbc.queryForObject("SELECT run_id FROM github_sync_task WHERE id=?",UUID.class,identical.task())); + + String content=jdbc.queryForObject("SELECT content_sha256 FROM github_sync_package_ingestion WHERE id=?",String.class,identical.ingestion()); + jdbc.update("UPDATE github_sync_package_ingestion SET content_sha256=? WHERE id=?","c".repeat(64),identical.ingestion()); + assertNoPathPublication(1601); + jdbc.update("UPDATE github_sync_package_ingestion SET content_sha256=?,status='BLOCKED' WHERE id=?",content,identical.ingestion()); + assertNoPathPublication(1601); // An unresolved new attempt must not inherit an older review. + jdbc.update("UPDATE github_sync_package_ingestion SET status='NO_CHANGE' WHERE id=?",identical.ingestion()); + Fixture unrelated=fixture(1602,161,true); + jdbc.update("UPDATE github_sync_package_ingestion SET content_sha256=?,published_skill_id=?,published_version_id=? WHERE id=?",content,skill,version,unrelated.ingestion()); + jdbc.update("UPDATE github_source_skill_binding SET latest_ingestion_id=? WHERE namespace_id=? AND repository_id=1601",unrelated.ingestion(),namespace); + assertNoPathPublication(1601); // Even matching hashes and IDs cannot cross repository coordinates. + jdbc.update("UPDATE github_source_skill_binding SET latest_ingestion_id=? WHERE namespace_id=? AND repository_id=1601",updated.ingestion(),namespace); + long firstVersion=jdbc.queryForObject("SELECT published_version_id FROM github_sync_package_ingestion WHERE id=?",Long.class,first.ingestion()); + jdbc.update("UPDATE github_sync_package_ingestion SET published_version_id=? WHERE id=?",firstVersion,updated.ingestion()); + assertNoPathPublication(1601); // Version must belong to the resolved publication ingestion. + jdbc.update("UPDATE github_sync_package_ingestion SET published_version_id=? WHERE id=?",version,updated.ingestion()); + assertThat(readModels.paths(namespace,1601,0,20,Instant.now()).items().getFirst().reviewId()).isEqualTo(review); + } + + @Test void revisionsKeepExplicitRetryAndIncreaseForChangeAndContentReturn() throws Exception { + Fixture first = publishArchive(1701, "A"); + assertRevision(first, 1); + jdbc.update("UPDATE github_sync_package_ingestion SET status='REJECTED' WHERE id=?", first.ingestion()); + long source = jdbc.queryForObject("SELECT source_id FROM github_source_skill_binding WHERE repository_id=1701", Long.class); + assertThat(publications.requestRejectedRetry(namespace, source, "skills/demo", Instant.now())).isTrue(); + Fixture retry = publishArchive(1701, "A"); + assertRevision(retry, 1); + Fixture changed = publishArchive(1701, "B"); + assertRevision(changed, 2); + Fixture returned = publishArchive(1701, "A"); + assertRevision(returned, 3); + Fixture noChange = publishArchive(1701, "A"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id=?", String.class, noChange.ingestion())).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForList("SELECT github_sync_revision FROM skill_version WHERE skill_id=(SELECT published_skill_id FROM github_sync_package_ingestion WHERE id=?) ORDER BY id", Integer.class, returned.ingestion())) + .containsExactly(1, 1, 2, 3); + assertThat(jdbc.queryForObject("SELECT latest_revision FROM github_source_skill_binding WHERE repository_id=1701", Integer.class)).isEqualTo(3); + } + + private void assertRevision(Fixture fixture, int expected) { + long version = jdbc.queryForObject("SELECT published_version_id FROM github_sync_package_ingestion WHERE id=?", Long.class, fixture.ingestion()); + assertThat(jdbc.queryForObject("SELECT github_sync_revision FROM skill_version WHERE id=?", Integer.class, version)).isEqualTo(expected); + String technical = jdbc.queryForObject("SELECT version FROM skill_version WHERE id=?", String.class, version); + assertThat(technical).startsWith("0.0.0-g"); + long review = jdbc.queryForObject("SELECT id FROM review_task WHERE github_sync_ingestion_id=?", Long.class, fixture.ingestion()); + assertThat(reviewSources.findByReviewId(review).orElseThrow().syncRevision()).isEqualTo(expected); + String status = jdbc.queryForObject("SELECT status FROM skill_version WHERE id=?", String.class, version); + // A database fixture exercises the published-only projection, not approval workflow acceptance. + jdbc.update("UPDATE skill_version SET status='PUBLISHED' WHERE id=?", version); + assertThat(marketSources.findPublishedVersion(version).orElseThrow().syncRevision()).isEqualTo(expected); + jdbc.update("UPDATE skill_version SET status=? WHERE id=?", status, version); + assertThat(jdbc.queryForObject("SELECT version FROM skill_version WHERE id=?", String.class, version)).isEqualTo(technical); + } + + private void assertNoPathPublication(long repository) { + var path=readModels.paths(namespace,repository,0,20,Instant.now()).items().getFirst(); + assertThat(path.skillId()).isNull();assertThat(path.versionId()).isNull();assertThat(path.reviewId()).isNull(); + } + + private Fixture publishArchive(long repository, String content) throws Exception { + Fixture fixture = prepareArchive(repository, content); + var claim = publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(600)).orElseThrow(); + assertThat(claim.id()).isEqualTo(fixture.ingestion()); + publicationService.process(claim); + publicationService.process(claim); + return fixture; + } + + private Fixture prepareArchive(long repository, String content) throws Exception { + Fixture fixture = fixture(repository, 161, true); + jdbc.update("INSERT INTO github_sync_path_result(task_id,commit_sha,skill_path,outcome,ingestion_id,sha256) SELECT task_id,commit_sha,skill_path,'ACCEPTED',id,sha256 FROM github_sync_package_ingestion WHERE id=?",fixture.ingestion()); + var bytes = new java.io.ByteArrayOutputStream(); + try (var zip = new java.util.zip.ZipOutputStream(bytes)) { + zip.putNextEntry(new java.util.zip.ZipEntry("SKILL.md")); + zip.write(("---\nname: actual-owner-outcome-" + repository + "\ndescription: Exercise durable GitHub publication outcomes.\nlicense: MIT\n---\n# Actual owner outcome\n\n" + content + "\n").getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + byte[] archive = bytes.toByteArray(); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(archive)); + String key = jdbc.queryForObject("SELECT object_key FROM github_sync_package_ingestion WHERE id=?", String.class, fixture.ingestion()); + storage.putObject(key, new java.io.ByteArrayInputStream(archive), archive.length, "application/zip"); + var validation = archiveValidator.validate(key, archive.length, digest, "skills/demo"); + jdbc.update("UPDATE github_sync_package_ingestion SET size_bytes=?,sha256=? WHERE id=?", archive.length, digest, fixture.ingestion()); + assertThat(ingestions.validated(fixture.ingestion(), fixture.validationToken(), validation.contentSha256(), 0, "{}", Instant.now())).isTrue(); + return fixture; + } + + private int ownerCount() { return jdbc.queryForObject("SELECT count(*) FROM github_deep_discovery_query WHERE query_source='OWNER'", Integer.class); } + private void origin(long candidate, UUID query) { + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?) ON CONFLICT DO NOTHING", candidate, query.toString()); + } + private void validate(Fixture fixture) { + assertThat(ingestions.validated(fixture.ingestion(), fixture.validationToken(), DIGEST, 0, "{}", Instant.now())).isTrue(); + } + private Fixture fixture(long repo, long owner, boolean official) throws Exception { + UUID task = UUID.randomUUID(), run = UUID.randomUUID(), admission = UUID.randomUUID(), upload = UUID.randomUUID(), ingestion = UUID.randomUUID(), validation = UUID.randomUUID(); + String commit = task.toString().replace("-", "") + "00000000"; + long candidate = jdbc.queryForObject("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name,default_branch,github_owner_id) VALUES (?,?,'seed-owner','skills','main',?) ON CONFLICT (namespace_id,repository_id) DO UPDATE SET last_seen_at=now() RETURNING id", Long.class, namespace, repo, owner); + if (official) origin(candidate, catalogQuery); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'REPOSITORY_SYNC')", run, namespace); + String hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(TOKEN.getBytes(StandardCharsets.UTF_8))); + jdbc.update(""" + INSERT INTO github_sync_task(id,run_id,namespace_id,kind,mode,status,dedupe_key,snapshot_json,lease_generation,lease_token_hash,lease_expires_at,leased_by) + VALUES (?,?,?,'REPOSITORY_SYNC','FULL','LEASED',?,jsonb_build_object('repositoryId',?::bigint,'owner','seed-owner','name','skills','ownerId',?::bigint),1,?,now()+INTERVAL '1 hour','owner-test') + """, task, run, namespace, task.toString(), repo, owner, hash); + jdbc.update("UPDATE github_sync_task SET created_at=clock_timestamp() WHERE id=?", task); + jdbc.update("UPDATE github_discovery_candidate SET status='VALIDATING',validation_task_id=? WHERE id=?", task, candidate); + jdbc.update("INSERT INTO github_sync_package_admission(id,task_id,namespace_id,repository_id,commit_sha,skill_path,lease_generation) VALUES (?,?,?,?,?,'skills/demo',1)", admission, task, namespace, repo, commit); + jdbc.update("INSERT INTO github_sync_staged_upload(id,admission_id,object_key,mode,ticket_hash,size_bytes,sha256,status,expires_at) VALUES (?,?,?,'LOCAL_STREAM',?,1,?,'ACCEPTED',now()+INTERVAL '1 hour')", upload, admission, "owner-test/"+upload, DIGEST, DIGEST); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion(id,task_id,admission_id,upload_id,namespace_id,repository_id,commit_sha,skill_path,object_key,size_bytes,sha256,status,validation_token,validation_expires_at) + VALUES (?,?,?,?,?,?,?,'skills/demo',?,1,?,'VALIDATING',?,?) + """, ingestion, task, admission, upload, namespace, repo, commit, "owner-test/"+upload, DIGEST, validation, Timestamp.from(Instant.now().plusSeconds(3600))); + // This class wraps several logical worker attempts in one test transaction. + // Preserve their real-world creation order instead of PostgreSQL now()'s fixed transaction time. + jdbc.update("UPDATE github_sync_package_ingestion SET created_at=clock_timestamp() WHERE id=?", ingestion); + return new Fixture(task, ingestion, validation, candidate); + } + private record Fixture(UUID task, UUID ingestion, UUID validationToken, long candidate) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialPresenceQuotaIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialPresenceQuotaIntegrationTest.java new file mode 100644 index 000000000..0fa443a92 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialPresenceQuotaIntegrationTest.java @@ -0,0 +1,301 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import com.iflytek.skillhub.service.GithubDeepDiscoveryReadAppService; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.infra.jpa.JdbcGithubOfficialQuotaRepository; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubOfficialPresenceQuotaIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + private static final String TEST_KEY = java.util.Base64.getEncoder().encodeToString(new java.security.SecureRandom().generateSeed(32)); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("skillhub.github-sync.credential-keys", () -> "test=" + TEST_KEY); + registry.add("skillhub.github-sync.active-credential-key-version", () -> "test"); + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired JdbcTemplate jdbc; + @Autowired org.springframework.transaction.PlatformTransactionManager transactionManager; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired GithubCredentialLeaseAppService claims; + @Autowired GithubCredentialCipher cipher; + @Autowired GithubSystemCredentialRepository credentials; + @Autowired GithubSyncWorkerPresenceRepository presence; + @Autowired GithubDeepDiscoveryReadAppService reads; + @Autowired GithubDeepDiscoveryQueryRepository queries; + @Autowired GithubOfficialQuotaRepository quotas; + + @BeforeEach void setup() { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update("UPDATE github_sync_admission_policy SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000 WHERE id"); + } + + @Test void onlineWorkerWithoutPlatformCheckpointsIsNotCompatible() { + presence.heartbeat("old-official", 1, GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES); + assertThat(presence.status().online()).isTrue(); + assertThat(presence.officialStatus().state()).isEqualTo("INCOMPATIBLE"); + assertThat(queries.compatibleWorker(Channel.REPOSITORY_METADATA)).isFalse(); + assertThat(queries.compatibleWorker(Channel.OWNER_EXPANSION)).isFalse(); + assertThat(queries.pauseCapableWorker()).isFalse(); + presence.heartbeat("old-official", 1, platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES)); + assertThat(presence.officialStatus().state()).isEqualTo("AVAILABLE"); + assertThat(queries.compatibleWorker(Channel.REPOSITORY_METADATA)).isTrue(); + assertThat(queries.compatibleWorker(Channel.OWNER_EXPANSION)).isTrue(); + assertThat(queries.pauseCapableWorker()).isTrue(); + } + + @Test void presenceIsChannelSpecificAndReconnectAndCleanupPreserveOnlineWorkers() { + presence.heartbeat("search-only",1,platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)); + presence.heartbeat("owner-capable",1,platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES)); + assertThat(queries.compatibleWorker(Channel.REPOSITORY_METADATA)).isTrue(); + assertThat(queries.compatibleWorker(Channel.OWNER_EXPANSION)).isTrue(); + jdbc.update("UPDATE github_sync_worker_instance SET last_seen_at=CURRENT_TIMESTAMP-INTERVAL '2 minutes' WHERE worker_id='owner-capable'"); + assertThat(channel(Channel.REPOSITORY_METADATA).waits()).noneMatch(w -> w.reason().equals("CAPABILITY_UNAVAILABLE")); + assertThat(channel(Channel.OWNER_EXPANSION).waits()).anyMatch(w -> w.reason().equals("CAPABILITY_UNAVAILABLE")); + jdbc.update("UPDATE github_sync_worker_instance SET last_seen_at=CURRENT_TIMESTAMP-INTERVAL '2 minutes'"); + jdbc.update("UPDATE github_sync_worker_presence SET last_seen_at=CURRENT_TIMESTAMP-INTERVAL '2 minutes'"); + assertThat(reads.progress().channels()).allSatisfy(c -> { + assertThat(c.state()).isEqualTo("WAITING"); + assertThat(c.waits()).anySatisfy(w -> { + assertThat(w.reason()).isEqualTo("WORKER_UNAVAILABLE"); + assertThat(w.retryAt()).isNull(); + assertThat(w.recoveryCondition()).contains("不表示任务已执行"); + }); + }); + presence.heartbeat("owner-capable",1,platform(GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES)); + assertThat(reads.progress().channels()).allSatisfy(c -> assertThat(c.waits()) + .noneMatch(w -> List.of("WORKER_UNAVAILABLE","CAPABILITY_UNAVAILABLE").contains(w.reason()))); + jdbc.update("UPDATE github_sync_worker_instance SET last_seen_at=CURRENT_TIMESTAMP-INTERVAL '100 days' WHERE worker_id='search-only'"); + assertThat(presence.deleteExpiredInstances(Instant.now().minusSeconds(90L*86400),500)).isEqualTo(1); + assertThat(jdbc.queryForList("SELECT worker_id FROM github_sync_worker_instance",String.class)).containsExactly("owner-capable"); + assertThat(new GithubSyncWorkerPresenceRepository(jdbc).officialStatus().compatibleInstanceCount()).isEqualTo(1); + } + + @Test void twoEncryptedTokensRotateAfterResourceExhaustionAndNeverFallBackToAnonymous() { + UUID first=credential("first"),second=credential("second"); + var leased=claim(); + assertThat(leased.credential().credentialId()).isIn(first,second); + UUID used=leased.credential().credentialId(); + UUID other=used.equals(first)?second:first; + observe(leased,"PRIMARY_EXHAUSTED",0,null); + assertThat(quotas.cooldown("CORE",Instant.now())).isNull(); + assertThat(quotas.cooldownForLease(leased.lease().task().taskId(),Instant.now())).isAfter(Instant.now()); + assertThat(credentials.availableForClaim(Instant.now(),"CORE")).isPresent(); + assertThat(credentials.availableForClaim(Instant.now(),"SEARCH")).get().extracting(GithubSystemCredential::id).isEqualTo(other); + releaseForNextClaim(); + var next=claim(); + assertThat(next.credential()).isNotNull(); + assertThat(next.credential().credentialId()).isEqualTo(other); + observe(next,"PRIMARY_EXHAUSTED",0,null); + releaseForNextClaim(); + assertThat(claimOptional()).isEmpty(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE status='LEASED' AND credential_id IS NULL",Long.class)).isZero(); + assertThat(queries.credentialPoolUnavailable("SEARCH")).isTrue(); + assertThat(queries.credentialPoolUnavailable("CORE")).isFalse(); + } + + @Test void secondaryCooldownSurvivesRepositoryRecreationAndBlocksTheOtherToken() { + credential("first");credential("second"); + var leased=claim(); + observe(leased,"SECONDARY_COOLDOWN",null,120); + var recreated=new JdbcGithubOfficialQuotaRepository(jdbc); + assertThat(recreated.cooldown("SEARCH",Instant.now())).isAfter(Instant.now()); + assertThat(recreated.cooldown("CORE",Instant.now())).isAfter(Instant.now()); + assertThat(credentials.availableForClaim(Instant.now(),"SEARCH")).isPresent(); + releaseForNextClaim(); + assertThat(claimOptional()).isEmpty(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE official_pause_reason='QUOTA_COOLDOWN'",Long.class)).isPositive(); + jdbc.update("UPDATE github_official_secondary_cooldown SET cooldown_until=CURRENT_TIMESTAMP-INTERVAL '1 second'"); + releaseForNextClaim(); + assertThat(claimOptional()).isPresent(); + } + + @Test void expiredQuotaObservationsAreUnknownAndNotAnAggregatedRealTimeBalance() { + credential("first");credential("second"); + var one=claim();observe(one,"OBSERVED",12,null); + releaseForNextClaim(); + var two=claim();observe(two,"OBSERVED",21,null); + assertThat(two.credential().credentialId()).isNotEqualTo(one.credential().credentialId()); + assertThat(reads.progress().quotas()).hasSize(2).extracting(Quota::remaining).containsExactlyInAnyOrder(12L,21L); + jdbc.update("UPDATE github_official_quota_observation SET observed_at=CURRENT_TIMESTAMP-INTERVAL '10 minutes',valid_until=CURRENT_TIMESTAMP-INTERVAL '5 minutes'"); + assertThat(reads.progress().quotas()).hasSize(2).allSatisfy(q -> { + assertThat(q.state()).isEqualTo("UNKNOWN"); + assertThat(q.remaining()).isNull(); + assertThat(q.subjectKey()).isNull(); + assertThat(q.retryAt()).isNull(); + }); + } + + @Test void anonymousPrimaryCooldownDoesNotBlockConfiguredCredentials() { + var anonymous=claim();assertThat(anonymous.credential()).isNull(); + observe(anonymous,"PRIMARY_EXHAUSTED",0,null); + assertThat(quotas.cooldown("SEARCH",Instant.now())).isAfter(Instant.now()); + credential("new-authenticated"); + assertThat(quotas.cooldown("SEARCH",Instant.now())).isNull(); + assertThat(quotas.cooldownForLease(anonymous.lease().task().taskId(),Instant.now())).isAfter(Instant.now()); + releaseForNextClaim(); + var authenticated=claim();assertThat(authenticated.credential()).isNotNull(); + assertThat(quotas.cooldownForLease(authenticated.lease().task().taskId(),Instant.now())).isNull(); + } + + @Test void credentialReplacementRejectsOldLeaseObservationsAndAuthInvalidDisablesCurrentCredential() { + UUID id=credential("replaceable");var before=claim(); + var replacement=cipher.encrypt(id,"synthetic-replacement-"+UUID.randomUUID()); + credentials.replace(id,"replacement","f".repeat(16),replacement.keyVersion(),replacement.nonce(),replacement.ciphertext()); + Instant now=Instant.now(); + assertThat(claims.reportOfficialQuota(before.lease().task().taskId(),before.lease().task().leaseGeneration(),before.lease().token(), + new GithubOfficialQuotaRepository.Observation("SEARCH",now,"AUTH_INVALID",null,null,null))).isFalse(); + assertThat(credentials.find(id).orElseThrow().status()).isEqualTo("ACTIVE"); + releaseForNextClaim();var current=claim();observe(current,"AUTH_INVALID",null,null); + assertThat(credentials.find(id).orElseThrow().status()).isEqualTo("INVALID"); + releaseForNextClaim();assertThat(claimOptional()).isEmpty(); + } + + @Test void replacingExhaustedCredentialClearsOnlyItsOldObservationsAndQuota() { + UUID id=credential("exhausted");var old=claim();observe(old,"PRIMARY_EXHAUSTED",0,null); + assertThat(queries.credentialPoolUnavailable("SEARCH")).isTrue(); + var same=credentials.find(id).orElseThrow(); + credentials.replace(id,"renamed",same.fingerprint(),same.keyVersion(),same.nonce(),same.ciphertext()); + assertThat(queries.credentialPoolUnavailable("SEARCH")).isTrue(); + replaceCredential(id); + assertThat(credentials.quotas(id)).isEmpty();assertThat(reads.progress().quotas()).isEmpty(); + assertThat(queries.credentialPoolUnavailable("SEARCH")).isFalse(); + releaseForNextClaim();assertThat(claim().credential().credentialId()).isEqualTo(id); + } + + @Test + @Transactional(propagation=org.springframework.transaction.annotation.Propagation.NOT_SUPPORTED) + void replacementWhileReportWaitsOnCredentialCannotPoisonNewToken() throws Exception { + var tx=new org.springframework.transaction.support.TransactionTemplate(transactionManager); + try (var executor=java.util.concurrent.Executors.newSingleThreadExecutor()) { + for (String mode:List.of("AUTH_INVALID","PRIMARY_EXHAUSTED","LEGACY_QUOTA","LEGACY_TASK_QUOTA","LEGACY_INVALID","LEGACY_LIMITED")) { + UUID id=tx.execute(status -> credential("concurrent-"+mode)); + var old=tx.execute(status -> claim()); + assertThat(old.credential().credentialId()).isEqualTo(id); + var started=new java.util.concurrent.CountDownLatch(1); + var reporterPid=new java.util.concurrent.atomic.AtomicInteger(); + var report=new java.util.concurrent.atomic.AtomicReference>(); + tx.executeWithoutResult(status -> { + jdbc.queryForObject("SELECT id FROM github_system_credential WHERE id=? FOR UPDATE",UUID.class,id); + report.set(executor.submit(() -> tx.execute(reportTx -> { + reporterPid.set(jdbc.queryForObject("SELECT pg_backend_pid()",Integer.class));started.countDown(); + var lease=old.lease();Instant now=Instant.now(); + String hash=org.apache.commons.codec.digest.DigestUtils.sha256Hex(lease.token()); + return switch(mode) { + case "LEGACY_QUOTA" -> credentials.recordQuotaForLease(lease.task().taskId(),lease.task().leaseGeneration(),hash,id,"SEARCH",0,now.plusSeconds(300),now); + case "LEGACY_TASK_QUOTA" -> credentials.recordQuotaForTaskLease(lease.task().taskId(),lease.task().leaseGeneration(),hash,"SEARCH",0,now.plusSeconds(300),now); + case "LEGACY_INVALID" -> credentials.markInvalid(lease.task().taskId(),lease.task().leaseGeneration(),hash,id,now); + case "LEGACY_LIMITED" -> credentials.markRateLimited(lease.task().taskId(),lease.task().leaseGeneration(),hash,id,now.plusSeconds(300),now); + default -> claims.reportOfficialQuota(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(), + new GithubOfficialQuotaRepository.Observation("SEARCH",now,mode,mode.equals("PRIMARY_EXHAUSTED")?0:null, + mode.equals("PRIMARY_EXHAUSTED")?now.plusSeconds(300):null,null)); + }; + }))); + try { + assertThat(started.await(10,java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + long deadline=System.nanoTime()+java.time.Duration.ofSeconds(10).toNanos();boolean blocked=false; + while (System.nanoTime() { + releaseForNextClaim();jdbc.update("UPDATE github_system_credential SET status='DISABLED' WHERE id=?",id); + }); + } + } finally { + // Only this class's Testcontainers database; preserve no fixtures across methods. + tx.executeWithoutResult(status -> { + jdbc.execute("TRUNCATE github_deep_discovery_query,github_sync_run,github_system_credential CASCADE"); + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()"); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + }); + } + } + + private void replaceCredential(UUID id) { + var encrypted=cipher.encrypt(id,"synthetic-replacement-"+UUID.randomUUID()); + credentials.replace(id,"replacement",UUID.randomUUID().toString().replace("-","").substring(0,16), + encrypted.keyVersion(),encrypted.nonce(),encrypted.ciphertext()); + } + + private ChannelProgress channel(Channel channel) { + return reads.progress().channels().stream().filter(c -> c.channel()==channel).findFirst().orElseThrow(); + } + private UUID credential(String label) { + UUID id=UUID.randomUUID();String token="synthetic-local-only-"+UUID.randomUUID(); + var encrypted=cipher.encrypt(id,token); + credentials.create(new GithubSystemCredential(id,label,id.toString().replace("-","").substring(0,16),encrypted.keyVersion(), + encrypted.nonce(),encrypted.ciphertext(),"ACTIVE",null,null,Instant.now(),Instant.now())); + assertThat(cipher.decrypt(id,encrypted.keyVersion(),encrypted.nonce(),encrypted.ciphertext())).isEqualTo(token); + return id; + } + private GithubCredentialLeaseAppService.Claimed claim() { + return claimOptional().orElseThrow(); + } + private java.util.Optional claimOptional() { + scheduler.enqueueDue(Instant.now(),1); + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 second'"); + return claims.claim(GithubSyncTaskKind.DISCOVERY,"quota-fixture",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)); + } + private void releaseForNextClaim() { + // Advance scheduling/lease clocks only; quota state and token choice stay production-controlled. + jdbc.update("UPDATE github_sync_task SET lease_expires_at=clock_timestamp()-INTERVAL '1 second' WHERE status='LEASED'"); + jdbc.update("UPDATE github_sync_task SET due_at=clock_timestamp()-INTERVAL '1 second',retry_at=clock_timestamp()-INTERVAL '1 second' WHERE status='RETRY_WAIT'"); + } + private void observe(GithubCredentialLeaseAppService.Claimed claimed,String state,Integer remaining,Integer retryAfter) { + var lease=claimed.lease();Instant now=Instant.now(); + assertThat(claims.reportOfficialQuota(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(), + new GithubOfficialQuotaRepository.Observation("SEARCH",now,state,remaining, + remaining==null?null:now.plusSeconds(300),retryAfter))).isTrue(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialRecoveryBoundaryIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialRecoveryBoundaryIntegrationTest.java new file mode 100644 index 000000000..1a5e8e4b4 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubOfficialRecoveryBoundaryIntegrationTest.java @@ -0,0 +1,262 @@ +package com.iflytek.skillhub.controller; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubOfficialRecoveryBoundaryIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired com.iflytek.skillhub.infra.jpa.JdbcGithubOfficialShardRepository shards; + @Autowired GithubCandidateTaskRepository candidates; + @Autowired ObjectMapper mapper; + @Autowired JdbcTemplate jdbc; + @Autowired GithubOfficialDiscoveryRepository scheduler; + @Autowired GithubSyncLeaseService leases; + @Autowired GithubOfficialLifecycleRepository lifecycle; + @Autowired GithubCredentialLeaseAppService credentialClaims; + + @BeforeEach void setup() { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + jdbc.update("UPDATE github_sync_admission_policy SET global_high=10000,global_low=5000,namespace_high=10000,namespace_low=5000 WHERE id"); + } + + @Test void catalogUpgradePreservesAdminPolicyCompatibleCheckpointAndHistoricalStart() throws Exception { + var lease=claim();UUID query=query(lease);String cp=checkpoint(lease,"IN_PROGRESS"); + report(lease,null,cp,"IN_PROGRESS",List.of()); + // Represent an earlier compatible strategy without changing its query semantics. + cp=((com.fasterxml.jackson.databind.node.ObjectNode)mapper.readTree(cp)).put("strategyVersion","previous").toString(); + jdbc.update("UPDATE github_sync_task SET snapshot_json=jsonb_set(snapshot_json,'{officialDiscovery,strategyVersion}','\"previous\"'),discovery_checkpoint=? WHERE id=?",cp,lease.task().taskId()); + jdbc.update("UPDATE github_deep_discovery_query SET strategy_version='previous',checkpoint=?,covered_until='2020-01-01T00:00:00Z' WHERE id=?",cp,query); + String original=jdbc.queryForObject("SELECT snapshot_json::text FROM github_sync_task WHERE id=?",String.class,lease.task().taskId()); + UUID missing=jdbc.queryForObject("SELECT id FROM github_deep_discovery_query WHERE id<>? ORDER BY query_key LIMIT 1",UUID.class,query); + String key=jdbc.queryForObject("SELECT query_key FROM github_deep_discovery_query WHERE id=?",String.class,missing); + jdbc.update("DELETE FROM github_deep_discovery_query WHERE id=?",missing); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE,budget_profile='ECONOMY',config_version=41,exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"excluded\"}]',strategy_generation=?,strategy_version='previous' WHERE id",GithubOfficialDiscoveryCatalog.GENERATION-1); + assertThat(scheduler.enqueueDue(Instant.now(),10)).isZero(); + assertThat(jdbc.queryForMap("SELECT enabled,budget_profile,config_version,exclusions_json::text FROM github_deep_discovery_config WHERE id")) + .containsEntry("enabled",false).containsEntry("budget_profile","ECONOMY").containsEntry("config_version",41L); + assertThat(jdbc.queryForObject("SELECT exclusions_json->0->>'value' FROM github_deep_discovery_config WHERE id",String.class)).isEqualTo("excluded"); + assertThat(jdbc.queryForObject("SELECT strategy_generation FROM github_deep_discovery_config WHERE id",Long.class)).isEqualTo(GithubOfficialDiscoveryCatalog.GENERATION); + assertThat(jdbc.queryForObject("SELECT checkpoint FROM github_deep_discovery_query WHERE id=?",String.class,query)).isEqualTo(cp); + assertThat(jdbc.queryForObject("SELECT snapshot_json::text FROM github_sync_task WHERE id=?",String.class,lease.task().taskId())).isEqualTo(original); + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query).toInstant()).isEqualTo(Instant.parse("2020-01-01T00:00:00Z")); + UUID added=jdbc.queryForObject("SELECT id FROM github_deep_discovery_query WHERE query_key=?",UUID.class,key); + assertThat(added).isNotEqualTo(missing); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id"); + onlyDue(added);assertThat(scheduler.enqueueDue(Instant.now(),1)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT snapshot_json#>>'{officialDiscovery,range,since}' FROM github_sync_task WHERE official_query_id=?",String.class,added)).isEqualTo(Instant.EPOCH.toString()); + jdbc.update("UPDATE github_deep_discovery_config SET strategy_generation=?,strategy_version='future-release' WHERE id",GithubOfficialDiscoveryCatalog.GENERATION+1); + int count=jdbc.queryForObject("SELECT count(*) FROM github_deep_discovery_query",Integer.class); + assertThat(scheduler.enqueueDue(Instant.now().plusSeconds(86400),100)).isZero(); + assertThat(jdbc.queryForObject("SELECT strategy_version FROM github_deep_discovery_config WHERE id",String.class)).isEqualTo("future-release"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_deep_discovery_query",Integer.class)).isEqualTo(count); + } + + @Test void retiredQueryFinishesExistingRootWithoutStartingAnother() throws Exception { + var root=claim();UUID query=query(root);var official=snapshot(root); + jdbc.update("UPDATE github_deep_discovery_query SET query_key='retired-fixture' WHERE id=?",query); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=FALSE WHERE id"); + scheduler.enqueueDue(Instant.now(),1); + assertThat(jdbc.queryForObject("SELECT retired_at FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query)).isNotNull(); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE WHERE id");onlyDue(query); + String cp=checkpoint(root,"SPLIT");report(root,null,cp,"SPLIT",halves(official)); + lifecycle.complete(root.task().taskId(),root.task().leaseGeneration(),root.token(),cp,"COMPLETE",null); + for(int i=0;i<2;i++) { + var child=claim();assertThat(query(child)).isEqualTo(query); + String done=checkpoint(child,"COMPLETE");report(child,null,done,"COMPLETE",List.of()); + lifecycle.complete(child.task().taskId(),child.task().leaseGeneration(),child.token(),done,"COMPLETE",null); + } + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query).toInstant()).isEqualTo(Instant.parse(official.path("range").path("until").asText())); + onlyDue(query); + assertThat(scheduler.enqueueDue(Instant.now(),1)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE query_id=? AND parent_id IS NULL",Integer.class,query)).isEqualTo(1); + } + + @Test void threeOriginsRemainIsolatedAcrossDisableExclusionAndResume() { + long ns=namespace();UUID independent=UUID.randomUUID(); + jdbc.update("INSERT INTO github_discovery_entry(id,namespace_id,entry_type,title,criteria_json,enabled,target_interval_seconds) VALUES (?,?,'SEARCH_RULE','independent fixture','{}',TRUE,3600)",independent,ns); + for(long repo:List.of(101L,102L,103L)) { + long candidate=jdbc.queryForObject("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) VALUES (?,?,'fixture','skills') RETURNING id",Long.class,ns,repo); + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'OFFICIAL_DISCOVERY',?)",candidate,UUID.randomUUID().toString()); + if(repo==102) jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,'SEARCH_RULE',?)",candidate,independent.toString()); + } + long source=jdbc.queryForObject("INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility) VALUES (?,103,'fixture','skills','PUBLIC') RETURNING id",Long.class,ns); + assertThat(candidates.enqueueDue(Instant.now().plusSeconds(1),10)).isEqualTo(2); + UUID run=UUID.randomUUID(),sourceTask=UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'REPOSITORY_SYNC')",run,ns); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,source_id,kind,mode,dedupe_key,snapshot_json) VALUES (?,?,?,?,'REPOSITORY_SYNC','HEAD_ONLY',?,'{\"repositoryId\":103}')",sourceTask,run,ns,source,sourceTask.toString()); + jdbc.update("UPDATE github_sync_task SET fixed_commit_sha=?,path_checkpoint='saved-path' WHERE kind='REPOSITORY_SYNC' AND source_id IS NULL","a".repeat(40)); + for(String phase:List.of("disabled","resumed","excluded","unexcluded")) { + jdbc.update("UPDATE github_deep_discovery_config SET enabled=?,exclusions_json=?::jsonb WHERE id",!phase.equals("disabled"),phase.equals("excluded")?"[{\"kind\":\"OWNER\",\"value\":\"fixture\"}]":"[]"); + resetRepositoryTasks(); + Set expected=("disabled".equals(phase)||"excluded".equals(phase))?Set.of(102L,103L):Set.of(101L,102L,103L); + assertThat(claimRepositories()).containsExactlyInAnyOrderElementsOf(expected); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE path_checkpoint='saved-path'",Integer.class)).isEqualTo(2); + } + jdbc.update("UPDATE github_source_binding SET status='PAUSED' WHERE id=?",source);resetRepositoryTasks(); + assertThat(claimRepositories()).containsExactlyInAnyOrder(101L,102L); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_source_binding WHERE namespace_id=?",Integer.class,ns)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE kind='REPOSITORY_SYNC'",Integer.class)).isEqualTo(3); + } + + @Test void depthThirtyTwoAndCumulativeChildren126AreEnforced() throws Exception { + var lease=claim();UUID query=query(lease);var official=snapshot(lease); + UUID root=jdbc.queryForObject("SELECT id FROM github_official_discovery_shard WHERE task_id=?",UUID.class,lease.task().taskId()); + jdbc.update("UPDATE github_official_discovery_shard SET task_id=NULL,state='SPLIT' WHERE id=?",root); + UUID parent=root;JsonNode remaining=official; + for(int depth=1;depth<=31;depth++) { + Instant since=Instant.parse(remaining.path("range").path("since").asText()); + Instant until=Instant.parse(remaining.path("range").path("until").asText()); + insertShard(query,root,parent,depth,withRange(remaining,since,since.plusSeconds(1)),"COMPLETE",null); + remaining=withRange(remaining,since.plusSeconds(1),until); + parent=insertShard(query,root,parent,depth,remaining,depth==31?"ACTIVE":"SPLIT",depth==31?lease.task().taskId():null); + } + shards.split(lease.task().taskId(),query,remaining,halves(remaining)); + assertThat(jdbc.queryForObject("SELECT max(depth) FROM github_official_discovery_shard WHERE root_id=?",Integer.class,root)).isEqualTo(32); + UUID deepest=jdbc.queryForObject("SELECT id FROM github_official_discovery_shard WHERE depth=32 LIMIT 1",UUID.class); + jdbc.update("UPDATE github_official_discovery_shard SET task_id=NULL WHERE task_id=?",lease.task().taskId()); + jdbc.update("UPDATE github_official_discovery_shard SET task_id=?,state='ACTIVE' WHERE id=?",lease.task().taskId(),deepest); + var deepestSnapshot=mapper.readTree(jdbc.queryForObject("SELECT snapshot_json::text FROM github_official_discovery_shard WHERE id=?",String.class,deepest)); + assertThatThrownBy(()->shards.split(lease.task().taskId(),query,deepestSnapshot,halves(deepestSnapshot))).isInstanceOf(GithubOfficialProtocolException.class); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE root_id=?",Integer.class,root)).isEqualTo(65); + } + + @Test void cumulativeChildrenBudgetRejects127thAnd128thNodes() throws Exception { + var lease=claim();UUID query=query(lease);var official=snapshot(lease); + UUID root=jdbc.queryForObject("SELECT id FROM github_official_discovery_shard WHERE task_id=?",UUID.class,lease.task().taskId()); + jdbc.update("UPDATE github_official_discovery_shard SET state='SPLIT',task_id=NULL WHERE id=?",root); + record Node(UUID id,int depth,JsonNode snapshot) {} + var queue=new java.util.ArrayDeque();queue.add(new Node(root,0,official)); + for(int index=0;index<62;index++) { + var parent=queue.removeFirst(); + jdbc.update("UPDATE github_official_discovery_shard SET state='SPLIT' WHERE id=?",parent.id()); + for(var range:halves(parent.snapshot())) { + JsonNode childSnapshot=withRange(parent.snapshot(),range.since(),range.until()); + UUID child=insertShard(query,root,parent.id(),parent.depth()+1,childSnapshot,"COMPLETE",null); + queue.add(new Node(child,parent.depth()+1,childSnapshot)); + } + } + var selected=queue.removeFirst();UUID leaf=selected.id(); + jdbc.update("UPDATE github_official_discovery_shard SET state='ACTIVE',task_id=? WHERE id=?",lease.task().taskId(),leaf); + shards.split(lease.task().taskId(),query,selected.snapshot(),halves(selected.snapshot())); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE root_id=? AND parent_id IS NOT NULL",Integer.class,root)).isEqualTo(126); + UUID child=jdbc.queryForObject("SELECT id FROM github_official_discovery_shard WHERE parent_id=? LIMIT 1",UUID.class,leaf); + jdbc.update("UPDATE github_official_discovery_shard SET task_id=NULL WHERE task_id=?",lease.task().taskId()); + jdbc.update("UPDATE github_official_discovery_shard SET task_id=?,state='ACTIVE' WHERE id=?",lease.task().taskId(),child); + var childSnapshot=mapper.readTree(jdbc.queryForObject("SELECT snapshot_json::text FROM github_official_discovery_shard WHERE id=?",String.class,child)); + assertThatThrownBy(()->shards.split(lease.task().taskId(),query,childSnapshot,halves(childSnapshot))).isInstanceOf(GithubOfficialProtocolException.class); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE root_id=? AND parent_id IS NOT NULL",Integer.class,root)).isEqualTo(126); + } + + @Test void deletingDiagnosticTaskPreservesGapBackoffCoverageAndCheckpoint() throws Exception { + var rootLease=claim();UUID query=query(rootLease);onlyDue(query); + var original=snapshot(rootLease);String split=checkpoint(rootLease,"SPLIT"); + report(rootLease,null,split,"SPLIT",halves(original)); + lifecycle.complete(rootLease.task().taskId(),rootLease.task().leaseGeneration(),rootLease.token(),split,"COMPLETE",null); + var gapLease=claim();String cp=checkpoint(gapLease,"GAP");report(gapLease,null,cp,"GAP",List.of()); + Instant retry=Instant.now().plusSeconds(600); + lifecycle.complete(gapLease.task().taskId(),gapLease.task().leaseGeneration(),gapLease.token(),cp,"INCOMPLETE",retry); + var sibling=claim();String done=checkpoint(sibling,"COMPLETE");report(sibling,null,done,"COMPLETE",List.of()); + lifecycle.complete(sibling.task().taskId(),sibling.task().leaseGeneration(),sibling.token(),done,"COMPLETE",null); + UUID gap=jdbc.queryForObject("SELECT id FROM github_official_discovery_shard WHERE task_id=?",UUID.class,gapLease.task().taskId()); + jdbc.update("DELETE FROM github_sync_task WHERE id=?",gapLease.task().taskId()); + assertThat(jdbc.queryForObject("SELECT task_id FROM github_official_discovery_shard WHERE id=?",UUID.class,gap)).isNull(); + assertThat(jdbc.queryForObject("SELECT gap_checkpoint FROM github_official_discovery_shard WHERE id=?",String.class,gap)).isEqualTo(cp); + assertThat(jdbc.queryForObject("SELECT retry_at FROM github_official_discovery_shard WHERE id=?",java.sql.Timestamp.class,gap).toInstant()).isCloseTo(retry,org.assertj.core.api.Assertions.within(1,java.time.temporal.ChronoUnit.MICROS)); + assertThat(scheduler.enqueueDue(retry.minusSeconds(1),1)).isZero(); + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query)).isNull(); + assertThat(scheduler.enqueueDue(retry.plusSeconds(1),1)).isEqualTo(1); + UUID replacement=jdbc.queryForObject("SELECT task_id FROM github_official_discovery_shard WHERE id=?",UUID.class,gap); + assertThat(replacement).isNotEqualTo(gapLease.task().taskId()); + JsonNode saved=mapper.readTree(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?",String.class,replacement)); + assertThat(saved.path("shardId").asText()).isEqualTo(replacement.toString()); + assertThat(saved.path("nextPage").asInt()).isEqualTo(2); + assertThat(jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id=?",Long.class,replacement)).isEqualTo(gapLease.task().leaseGeneration()); + assertThat(jdbc.queryForObject("SELECT (snapshot_json#>>'{officialDiscovery,budget,maxChildShards}')::integer FROM github_sync_task WHERE id=?",Integer.class,replacement)).isEqualTo(original.path("budget").path("maxChildShards").asInt()-2); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE query_id=?",Integer.class,query)).isEqualTo(3); + assertThat(jdbc.queryForObject("SELECT covered_until FROM github_deep_discovery_query WHERE id=?",java.sql.Timestamp.class,query)).isNull(); + } + + private GithubSyncLeaseService.LeasedTask claim() { + assertThat(scheduler.enqueueDue(Instant.now(),1)).isEqualTo(1); + jdbc.update("UPDATE github_official_discovery_budget SET next_lease_eligible_at=clock_timestamp()-INTERVAL '1 second'"); + return credentialClaims.claim(GithubSyncTaskKind.DISCOVERY,"recovery",platform(GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES)).orElseThrow().lease(); + } + private UUID query(GithubSyncLeaseService.LeasedTask lease) throws Exception { return UUID.fromString(snapshot(lease).path("queryId").asText()); } + private JsonNode snapshot(GithubSyncLeaseService.LeasedTask lease) throws Exception { return mapper.readTree(lease.task().snapshotJson()).path("officialDiscovery"); } + private String checkpoint(GithubSyncLeaseService.LeasedTask lease,String coverage) throws Exception { + var cp=mapper.createObjectNode().put("formatVersion",1).put("nextPage",2).put("coverage",coverage); + for(String field:List.of("strategyVersion","queryId","queryVersion","shardId")) cp.set(field,snapshot(lease).get(field)); + return cp.toString(); + } + private List halves(JsonNode snapshot) { + Instant since=Instant.parse(snapshot.path("range").path("since").asText()),until=Instant.parse(snapshot.path("range").path("until").asText()); + Instant middle=since.plusSeconds(java.time.Duration.between(since,until).getSeconds()/2); + return List.of(new GithubOfficialDiscoveryBatch.Range(since,middle,null),new GithubOfficialDiscoveryBatch.Range(middle,until,null)); + } + private void report(GithubSyncLeaseService.LeasedTask lease,String previous,String cp,String state,List children) { + lifecycle.report(lease.task().taskId(),lease.task().leaseGeneration(),lease.token(),new GithubDiscoveryBatch(previous,cp,List.of(),0,null), + new GithubOfficialDiscoveryBatch(lease.task().taskId(),state,1,children,"GAP".equals(state)?"UNSPLITTABLE":null)); + } + private void onlyDue(UUID query) { jdbc.update("UPDATE github_deep_discovery_query SET next_due_at=CASE WHEN id=? THEN CURRENT_TIMESTAMP-INTERVAL '1 second' ELSE CURRENT_TIMESTAMP+INTERVAL '2 days' END",query); } + private long namespace() { return jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id",Long.class); } + private void resetRepositoryTasks() { jdbc.update("UPDATE github_sync_task SET status='QUEUED',lease_token_hash=NULL,lease_expires_at=NULL,leased_by=NULL,due_at=CURRENT_TIMESTAMP-INTERVAL '1 second' WHERE kind='REPOSITORY_SYNC'"); } + private Set claimRepositories() { + Set result=new java.util.HashSet<>(); + for(int i=0;i<4;i++) { + var lease=leases.claim(GithubSyncTaskKind.REPOSITORY_SYNC,"origin-worker",platform(List.of("official-pause/v1"))); + if(lease.isEmpty()) return result; + try { result.add(mapper.readTree(lease.get().task().snapshotJson()).path("repositoryId").asLong()); } + catch(Exception error) { throw new AssertionError(error); } + } + throw new AssertionError("Unexpected duplicate repository lease"); + } + private JsonNode withRange(JsonNode snapshot,Instant since,Instant until) { + var result=(com.fasterxml.jackson.databind.node.ObjectNode)snapshot.deepCopy(); + result.putObject("range").put("since",since.toString()).put("until",until.toString()); + return result; + } + private UUID insertShard(UUID query,UUID root,UUID parent,int depth,JsonNode snapshot,String state,UUID task) { + UUID id=UUID.randomUUID(); + jdbc.update("INSERT INTO github_official_discovery_shard(id,query_id,root_id,parent_id,depth,range_since,range_until,snapshot_json,state,task_id) VALUES (?,?,?,?,?,?::timestamptz,?::timestamptz,?::jsonb,?,?)", + id,query,root,parent,depth,snapshot.path("range").path("since").asText(),snapshot.path("range").path("until").asText(),snapshot.toString(),state,task); + return id; + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubPlatformControlIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubPlatformControlIntegrationTest.java new file mode 100644 index 000000000..f197332c0 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubPlatformControlIntegrationTest.java @@ -0,0 +1,269 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.*; +import com.iflytek.skillhub.infra.jpa.JdbcGithubOfficialDiscoveryBudgetRepository; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.transaction.PlatformTransactionManager; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = {"skillhub.builtin-skills.enabled=false", + "skillhub.github-sync.service-tokens=platform-control-test-only-service-token-32"}) +class GithubPlatformControlIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url",POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username",POSTGRES::getUsername); + registry.add("spring.datasource.password",POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name",()->"org.postgresql.Driver"); + registry.add("spring.jpa.database-platform",()->"org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled",()->true); + registry.add("spring.jpa.hibernate.ddl-auto",()->"validate"); + } + @Autowired JdbcTemplate jdbc; + @Autowired org.springframework.test.web.servlet.MockMvc mvc; + @org.springframework.boot.test.web.server.LocalServerPort int port; + @Autowired GithubPlatformControlRepository control; + @Autowired GithubSyncLeaseService leases; + @Autowired GithubSyncHistoryRepository history; + @Autowired PlatformTransactionManager transactions; + long ns; + @BeforeEach void setup() { + jdbc.execute("TRUNCATE github_sync_task CASCADE"); + jdbc.update("UPDATE github_platform_collection_control SET paused=false,version=1"); + ns=control.get().namespaceId(); + } + @Test void capabilityGatePauseAcknowledgeAndResumePreserveCheckpoint() { + UUID id=queue(ns); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY,"old-worker")).isEmpty(); + var lease=claim(); + assertThat(lease.task().platformControlVersion()).isEqualTo(1); + jdbc.update("UPDATE github_sync_task SET discovery_checkpoint='confirmed-page' WHERE id=?",id); + var paused=control.update(1,true,"tester"); + assertThat(paused.state()).isEqualTo("PAUSING");assertThat(paused.pendingStopCount()).isEqualTo(1); + assertThat(control.control(id,lease.task().leaseGeneration(),lease.token()).action()).isEqualTo("PAUSE"); + assertThat(leases.claim(GithubSyncTaskKind.DISCOVERY,"worker",List.of(GithubPlatformControlRepository.CAPABILITY))).isEmpty(); + assertThatThrownBy(()->control.pause(id,lease.task().leaseGeneration(),lease.token(),2,"DISCOVERY","unconfirmed",null)).isInstanceOf(GithubOfficialProtocolException.class); + control.pause(id,lease.task().leaseGeneration(),lease.token(),2,"DISCOVERY","confirmed-page",null); + control.pause(id,lease.task().leaseGeneration(),lease.token(),2,"DISCOVERY","confirmed-page",null); + assertThat(control.get().state()).isEqualTo("PAUSED"); + assertThat(control.update(2,false,"tester").state()).isEqualTo("RUNNING"); + var resumed=claim();assertThat(resumed.task().discoveryCheckpoint()).isEqualTo("confirmed-page"); + assertThat(resumed.task().leaseGeneration()).isEqualTo(2); + assertThat(resumed.task().requiredCapabilities()).containsExactly(GithubPlatformControlRepository.CAPABILITY); + } + @Test void expiredWorkRequiresExplicitAckAndCannotContinueAfterResume() { + UUID id=queue(ns);var lease=claim();control.update(1,true,"tester"); + jdbc.update("UPDATE github_sync_task SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=?",id); + assertThat(control.get().state()).isEqualTo("PAUSING");assertThat(control.get().expiredUnconfirmedCount()).isEqualTo(1); + assertThat(control.control(id,lease.task().leaseGeneration(),lease.token()).action()).isEqualTo("PAUSE"); + control.pause(id,lease.task().leaseGeneration(),lease.token(),2,"DISCOVERY","must-not-persist",null); + assertThat(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id=?",String.class,id)).isNull(); + assertThat(control.get().state()).isEqualTo("PAUSED");control.update(2,false,"tester"); + assertThatThrownBy(()->control.control(id,lease.task().leaseGeneration(),lease.token())).isInstanceOf(GithubOfficialProtocolException.class); + } + @Test void rapidResumeRejectsOldEpochAndNormalTerminalProofCanAcknowledge() { + UUID id=queue(ns);var lease=claim();control.update(1,true,"tester");control.update(2,false,"tester");control.update(3,true,"tester"); + assertThatThrownBy(()->control.pause(id,lease.task().leaseGeneration(),lease.token(),2,"DISCOVERY",null,null)).isInstanceOf(GithubOfficialProtocolException.class); + jdbc.update("UPDATE github_sync_task SET status='ACQUISITION_COMPLETE',lease_token_hash=NULL,lease_expires_at=NULL,leased_by=NULL WHERE id=?",id); + assertThat(control.get().state()).isEqualTo("PAUSING"); + control.pause(id,lease.task().leaseGeneration(),lease.token(),4,"DISCOVERY",null,null); + assertThat(control.get().state()).isEqualTo("PAUSED"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id=?",String.class,id)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(control.update(3,true,"tester").version()).isEqualTo(4); + assertThatThrownBy(()->control.update(1,false,"tester")).isInstanceOf(GithubOfficialProtocolException.class); + } + @Test void teamClaimsRemainIndependentWithoutCapability() { + long team=jdbc.queryForObject("INSERT INTO namespace(slug,display_name,type) VALUES ('platform-team','Platform Team','TEAM') RETURNING id",Long.class); + UUID id=queue(team);control.update(1,true,"tester"); + var lease=leases.claim(GithubSyncTaskKind.DISCOVERY,"legacy-team").orElseThrow(); + assertThat(lease.task().taskId()).isEqualTo(id);assertThat(lease.task().platformControlVersion()).isNull(); + assertThat(control.get().state()).isEqualTo("PAUSED"); + } + @Test void pauseAndClaimSerializeAtTheDatabaseGate() throws Exception { + queue(ns); + var locked=new CountDownLatch(1);var release=new CountDownLatch(1);var entered=new CountDownLatch(1); + var olderPid=new java.util.concurrent.atomic.AtomicInteger();var newerPid=new java.util.concurrent.atomic.AtomicInteger(); + var tx=new TransactionTemplate(transactions); + try(var pool=Executors.newFixedThreadPool(2)) { + var pause=pool.submit(()->tx.execute(status->{ + olderPid.set(jdbc.queryForObject("SELECT pg_backend_pid()",Integer.class)); + control.update(1,true,"tester");locked.countDown(); + try {if(!release.await(20,TimeUnit.SECONDS)) throw new IllegalStateException("release timeout");} + catch(InterruptedException e){Thread.currentThread().interrupt();throw new RuntimeException(e);} + return null; + })); + java.util.concurrent.Future> claim=null; + try { + assertThat(locked.await(10,TimeUnit.SECONDS)).isTrue(); + claim=pool.submit(()->tx.execute(status->{ + newerPid.set(jdbc.queryForObject("SELECT pg_backend_pid()",Integer.class));entered.countDown(); + return leases.claim(GithubSyncTaskKind.DISCOVERY,"blocked-worker",List.of(GithubPlatformControlRepository.CAPABILITY)); + })); + assertThat(entered.await(10,TimeUnit.SECONDS)).isTrue(); + try(var connection=java.sql.DriverManager.getConnection(POSTGRES.getJdbcUrl(),POSTGRES.getUsername(),POSTGRES.getPassword()); + var statement=connection.prepareStatement("SELECT ? = ANY(pg_blocking_pids(?))")) { + statement.setInt(1,olderPid.get());statement.setInt(2,newerPid.get());statement.setQueryTimeout(2); + boolean blocked=false;long deadline=System.nanoTime()+TimeUnit.SECONDS.toNanos(10); + while(System.nanoTime()tx.execute(status->{ + olderPid.set(jdbc.queryForObject("SELECT pg_backend_pid()",Integer.class)); + control.update(1,true,"tester");locked.countDown(); + try {if(!release.await(20,TimeUnit.SECONDS)) throw new IllegalStateException("release timeout");} + catch(InterruptedException e){Thread.currentThread().interrupt();throw new RuntimeException(e);} + return null; + })); + java.util.concurrent.Future claim=null; + try { + assertThat(locked.await(10,TimeUnit.SECONDS)).isTrue(); + claim=pool.submit(()->tx.execute(status->{ + newerPid.set(jdbc.queryForObject("SELECT pg_backend_pid()",Integer.class));entered.countDown(); + return leases.admitPackage(id,lease.task().leaseGeneration(),lease.token(),"a".repeat(40),"new-path"); + })); + assertThat(entered.await(10,TimeUnit.SECONDS)).isTrue(); + try(var connection=java.sql.DriverManager.getConnection(POSTGRES.getJdbcUrl(),POSTGRES.getUsername(),POSTGRES.getPassword()); + var statement=connection.prepareStatement("SELECT ? = ANY(pg_blocking_pids(?))")) { + statement.setInt(1,olderPid.get());statement.setInt(2,newerPid.get());statement.setQueryTimeout(2); + boolean blocked=false;long deadline=System.nanoTime()+TimeUnit.SECONDS.toNanos(10); + while(System.nanoTime() POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void properties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + registry.add("skillhub.github-sync.credential-keys", + () -> "v1=" + Base64.getEncoder().encodeToString(CREDENTIAL_KEY)); + registry.add("skillhub.github-sync.active-credential-key-version", () -> "v1"); + } + + @Autowired private MockMvc mockMvc; + @Autowired private ObjectMapper mapper; + @Autowired private JdbcTemplate jdbc; + @Autowired private GithubDiscoveryEntryRepository discoveryEntries; + @Autowired private MeterRegistry meterRegistry; + @LocalServerPort private int port; + + @Test + void credentialRemainsInInternalLeaseAndOutOfOrdinaryOutputs() throws Exception { + String token = "ghp_" + UUID.randomUUID().toString().replace("-", ""); + String contentMarker = "SKILL_BODY_" + UUID.randomUUID().toString().replace("-", ""); + Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + ListAppender logs = new ListAppender<>(); + logs.start(); + root.addAppender(logs); + try { + var admin = authentication(new UsernamePasswordAuthenticationToken( + new PlatformPrincipal("github-sync-publisher", "privacy-admin", null, + null, null, Set.of("SUPER_ADMIN")), null, + List.of(new SimpleGrantedAuthority("ROLE_SUPER_ADMIN")))); + String created = mockMvc.perform(post(CREDENTIALS).with(admin).with(csrf()) + .contentType(MediaType.APPLICATION_JSON) + .content(mapper.writeValueAsBytes(Map.of("label", "Privacy fixture", "token", token)))) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + String listed = mockMvc.perform(get(CREDENTIALS).with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + assertThat(created).doesNotContain(token, "ciphertext", "nonce"); + assertThat(listed).doesNotContain(token, "ciphertext", "nonce"); + + HttpClient client = HttpClient.newHttpClient(); + String baseUrl = "http://127.0.0.1:" + port; + var ordinary = client.send(HttpRequest.newBuilder(URI.create(baseUrl + + "/api/v1/admin/github-sync/discovery-entries?unexpected=" + contentMarker)) + .header("User-Agent", contentMarker) + .header("Authorization", "Bearer " + token).GET().build(), + HttpResponse.BodyHandlers.ofString()); + assertThat(ordinary.statusCode()).isEqualTo(401); + assertThat(ordinary.body()).doesNotContain(token, contentMarker); + String malformed = "{\"token\":\"" + token + "\",\"body\":\"" + contentMarker + "\""; + String error = mockMvc.perform(post(CREDENTIALS).with(admin).with(csrf()) + .contentType(MediaType.APPLICATION_JSON).content(malformed)) + .andExpect(status().isBadRequest()).andReturn().getResponse().getContentAsString(); + assertThat(error).doesNotContain(token, contentMarker); + var internalError = client.send(HttpRequest.newBuilder(URI.create(baseUrl + CLAIM)) + .header("Authorization", "Bearer " + SERVICE_TOKEN) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(malformed)).build(), + HttpResponse.BodyHandlers.ofString()); + assertThat(internalError.statusCode()).isEqualTo(400); + assertThat(internalError.body()).doesNotContain(token, contentMarker); + + var entry = discoveryEntries.createCuratedRepository("privacy-owner", "privacy-repo"); + assertThat(discoveryEntries.enqueueDue(Instant.now(), 10)).isEqualTo(1); + var claim = client.send(HttpRequest.newBuilder(URI.create(baseUrl + CLAIM)) + .header("Authorization", "Bearer " + SERVICE_TOKEN) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + "{\"workerId\":\"privacy-worker\",\"protocolMajor\":1,\"capabilities\":[\"platform-pause/v1\"],\"kind\":\"DISCOVERY\"}")) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(claim.statusCode()).isEqualTo(200); + String lease = claim.body(); + assertThat(lease).contains(token, entry.id().toString()); + UUID taskId = UUID.fromString(mapper.readTree(lease).path("taskId").asText()); + String task = jdbc.queryForObject(""" + SELECT row_to_json(task)::text FROM ( + SELECT snapshot_json, credential_id, credential_fingerprint, + last_error_code, discovery_checkpoint + FROM github_sync_task WHERE id = ? + ) task + """, String.class, taskId); + assertThat(task).doesNotContain(token, contentMarker); + String audit = jdbc.queryForObject(""" + SELECT detail_json::text FROM audit_log + WHERE action = 'CREATE_GITHUB_CREDENTIAL' + ORDER BY id DESC LIMIT 1 + """, String.class); + assertThat(audit).doesNotContain(token, contentMarker); + + String logText = logs.list.stream().map(event -> event.getFormattedMessage() + + (event.getThrowableProxy() == null ? "" + : ThrowableProxyUtil.asString(event.getThrowableProxy())) + + event.getMDCPropertyMap()).collect(Collectors.joining("\n")); + assertThat(logText).doesNotContain(token, contentMarker, SERVICE_TOKEN); + String metricIds = meterRegistry.getMeters().stream().map(meter -> meter.getId().toString()) + .collect(Collectors.joining("\n")); + assertThat(metricIds).doesNotContain(token, contentMarker, SERVICE_TOKEN); + } finally { + root.detachAppender(logs); + logs.stop(); + } + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncHttpIngestionInteropTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncHttpIngestionInteropTest.java new file mode 100644 index 000000000..32dc48bfd --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncHttpIngestionInteropTest.java @@ -0,0 +1,3098 @@ +package com.iflytek.skillhub.controller; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import ch.qos.logback.core.read.ListAppender; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.domain.review.ReviewOriginType; +import com.iflytek.skillhub.domain.review.ReviewService; +import com.iflytek.skillhub.domain.review.ReviewTaskRepository; +import com.iflytek.skillhub.domain.review.ReviewTaskStatus; +import com.iflytek.skillhub.dto.GithubSearchRuleRequest; +import com.iflytek.skillhub.dto.GithubSearchSelectorRequest; +import com.iflytek.skillhub.dto.GithubCuratedOwnerRequest; +import com.iflytek.skillhub.service.AuditRequestContext; +import com.iflytek.skillhub.githubsync.GithubCandidateTaskRepository; +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import com.iflytek.skillhub.githubsync.GithubSourceTaskRepository; +import com.iflytek.skillhub.githubsync.GithubPackageAdmissionResult; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskRepository; +import com.iflytek.skillhub.githubsync.GithubSyncIngestionRepository; +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncIngestionRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPublicationRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.metrics.GithubSyncStagingCapacityMetrics; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import com.iflytek.skillhub.service.GithubDiscoveryAdminAppService; +import com.iflytek.skillhub.service.GithubSearchPreviewAppService; +import com.iflytek.skillhub.storage.ObjectStorageService; +import com.iflytek.skillhub.storage.StoragePrefixUsage; +import com.iflytek.skillhub.task.GithubSyncIngestionTask; +import com.iflytek.skillhub.task.GithubSyncPublicationTask; +import com.iflytek.skillhub.task.ScanTaskOutboxDispatcher; +import com.sun.net.httpserver.HttpServer; +import io.micrometer.core.instrument.MeterRegistry; +import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.data.domain.PageRequest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.util.FileSystemUtils; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.ServerSocket; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.authentication; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** Opt-in wire and PostgreSQL check using the ZIP produced by the real Go acquisition test. */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE", matches = ".+") +@Import(GithubSyncHttpIngestionInteropTest.RedisFixtureConfig.class) +@TestPropertySource(properties = { + "skillhub.github-sync.service-tokens=fixture-service-token-for-http-interop-32-chars", + "skillhub.builtin-skills.enabled=false", + "skillhub.security.scanner.enabled=true", + "skillhub.publish.max-package-size=65536", + "skillhub.github-sync.staging-capacity-limit-bytes=1024", + "skillhub.github-sync.ingestion-validation-interval-ms=3600000", + "skillhub.github-sync.publication-interval-ms=3600000", + "skillhub.security.outbox.dispatch-interval-ms=3600000" +}) +class GithubSyncHttpIngestionInteropTest { + private static final String COMMIT = "49f948faa9258a0c61caceaf225e179651397431"; + private static final String SKILL_PATH = "skills/.system/skill-creator"; + private static final String SERVICE_TOKEN = "fixture-service-token-for-http-interop-32-chars"; + private static final byte[] CREDENTIAL_KEY = new byte[32]; + static { + new SecureRandom().nextBytes(CREDENTIAL_KEY); + } + private static final boolean S3_MODE = "s3".equalsIgnoreCase( + System.getenv("SKILLHUB_GITHUB_SYNC_INTEROP_STORAGE_MODE")); + private static final String MINIO_ACCESS_KEY = "interop" + UUID.randomUUID().toString().replace("-", ""); + private static final String MINIO_SECRET_KEY = UUID.randomUUID().toString().replace("-", ""); + private static final String S3_BUCKET = "skillhub-interop-" + UUID.randomUUID().toString().replace("-", ""); + private static final Path STORAGE_ROOT = Path.of(System.getProperty("java.io.tmpdir"), + "skillhub-github-http-interop-" + UUID.randomUUID()); + private static final GenericContainer MINIO = new GenericContainer<>("quay.io/minio/minio:latest") + .withExposedPorts(9000) + .withEnv("MINIO_ROOT_USER", MINIO_ACCESS_KEY) + .withEnv("MINIO_ROOT_PASSWORD", MINIO_SECRET_KEY) + .withCommand("server", "/data") + .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000)); + private static String s3Endpoint; + + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @Container + private static final GenericContainer REDIS = new GenericContainer<>("redis:7-alpine") + .withExposedPorts(6379); + private static HttpServer scanner; + private static String scannerBaseUrl; + private static final AtomicReference scannerFailureMarker = new AtomicReference<>(); + + @TestConfiguration + static class RedisFixtureConfig { + @Bean + RedisProperties redisProperties() { + RedisProperties properties = new RedisProperties(); + properties.setHost(REDIS.getHost()); + properties.setPort(REDIS.getMappedPort(6379)); + return properties; + } + } + + /** Claims from another JVM, then exits without settling the validation lease. */ + public static final class IngestionClaimProcess { + public static void main(String[] args) { + DriverManagerDataSource source = new DriverManagerDataSource( + System.getenv("SKILLHUB_RESTART_JDBC_URL"), + System.getenv("SKILLHUB_RESTART_JDBC_USER"), + System.getenv("SKILLHUB_RESTART_JDBC_PASSWORD")); + JdbcTemplate jdbc = new JdbcTemplate(source); + var repository = new JdbcGithubSyncIngestionRepository(jdbc, + new JdbcGithubSyncSourceCheckpointRepository(jdbc), + new JdbcGithubSyncSchedulePolicyRepository(jdbc), Duration.ofHours(24)); + var claim = repository.claim(UUID.randomUUID(), Instant.now(), + Instant.now().plusSeconds(2)).orElseThrow(); + if (!claim.id().equals(UUID.fromString(args[0]))) { + throw new IllegalStateException("Claimed a different ingestion"); + } + System.out.println("CLAIMED"); + } + } + + /** Claims publication work from a separate JVM, then exits before committing a Skill. */ + public static final class PublicationClaimProcess { + public static void main(String[] args) { + DriverManagerDataSource source = new DriverManagerDataSource( + System.getenv("SKILLHUB_RESTART_JDBC_URL"), + System.getenv("SKILLHUB_RESTART_JDBC_USER"), + System.getenv("SKILLHUB_RESTART_JDBC_PASSWORD")); + var repository = new JdbcGithubSyncPublicationRepository(new JdbcTemplate(source)); + var claim = repository.claim(UUID.randomUUID(), Instant.now(), + Instant.now().plusSeconds(2)).orElseThrow(); + if (!claim.id().equals(UUID.fromString(args[0]))) { + throw new IllegalStateException("Claimed a different publication"); + } + System.out.println("CLAIMED"); + } + } + + @DynamicPropertySource + static void properties(DynamicPropertyRegistry registry) { + if ("1".equals(System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES_INTEROP")) + || System.getenv("SKILLHUB_GITHUB_SYNC_INTEROP_IMAGE") != null) { + registry.add("skillhub.publish.max-package-size", () -> 104857600); + } + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("skillhub.github-sync.credential-keys", + () -> "v1=" + Base64.getEncoder().encodeToString(CREDENTIAL_KEY)); + registry.add("skillhub.github-sync.active-credential-key-version", () -> "v1"); + registry.add("skillhub.storage.local.base-path", STORAGE_ROOT::toString); + registry.add("spring.data.redis.host", REDIS::getHost); + registry.add("spring.data.redis.port", () -> REDIS.getMappedPort(6379)); + scannerBaseUrl = System.getenv("SKILLHUB_GITHUB_SYNC_INTEROP_SCANNER_URL"); + boolean externalScanner = scannerBaseUrl != null && !scannerBaseUrl.isBlank(); + if (!externalScanner) { + try { + scanner = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + } catch (IOException exception) { + throw new IllegalStateException("Cannot start scanner fixture", exception); + } + scanner.createContext("/scan", exchange -> { + exchange.getRequestBody().readAllBytes(); + String failureMarker = scannerFailureMarker.get(); + byte[] response = (failureMarker == null ? """ + {"scan_id":"interop-safe","skill_name":"skill-creator","is_safe":true, + "max_severity":"LOW","findings_count":0,"findings":[], + "scan_duration_seconds":0.01} + """ : "{\"detail\":\"" + failureMarker + "\"}") + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(failureMarker == null ? 200 : 400, response.length); + try (var output = exchange.getResponseBody()) { + output.write(response); + } + }); + scanner.start(); + scannerBaseUrl = "http://127.0.0.1:" + scanner.getAddress().getPort(); + } + registry.add("skillhub.security.scanner.base-url", () -> scannerBaseUrl); + registry.add("skillhub.security.scanner.mode", () -> externalScanner ? "upload" : "local"); + if (S3_MODE) { + MINIO.start(); + s3Endpoint = "http://" + MINIO.getHost() + ":" + MINIO.getMappedPort(9000); + registry.add("skillhub.storage.provider", () -> "s3"); + registry.add("skillhub.storage.s3.endpoint", () -> s3Endpoint); + registry.add("skillhub.storage.s3.bucket", () -> S3_BUCKET); + registry.add("skillhub.storage.s3.access-key", () -> MINIO_ACCESS_KEY); + registry.add("skillhub.storage.s3.secret-key", () -> MINIO_SECRET_KEY); + registry.add("skillhub.storage.s3.region", () -> "us-east-1"); + registry.add("skillhub.storage.s3.auto-create-bucket", () -> true); + registry.add("skillhub.storage.s3.force-path-style", () -> true); + } + } + + @AfterAll + static void cleanStorage() throws IOException { + if (scanner != null) { + scanner.stop(0); + } + FileSystemUtils.deleteRecursively(STORAGE_ROOT); + if (MINIO.isRunning()) { + MINIO.stop(); + } + } + + @Autowired private JdbcTemplate jdbc; + @Autowired private MockMvc mockMvc; + @Autowired private ObjectMapper objectMapper; + @Autowired private GithubSyncIngestionTask ingestionTask; + @Autowired private GithubSyncPublicationTask publicationTask; + @Autowired private ObjectStorageService storage; + @Autowired private GithubSyncStagingCapacityMetrics capacityMetrics; + @Autowired private GithubSearchPreviewAppService searchPreviews; + @Autowired private GithubDiscoveryAdminAppService discoveryAdmin; + @Autowired private GithubDiscoveryEntryRepository discoveryEntries; + @Autowired private GithubSyncWorkerPresenceRepository workerPresence; + @Autowired private GithubSyncManagementQueryRepository managementQueries; + @Autowired private GithubCredentialCipher credentialCipher; + @Autowired private GithubSystemCredentialRepository systemCredentials; + @Autowired private MeterRegistry meterRegistry; + @Autowired private ReviewTaskRepository reviews; + @Autowired private ReviewService reviewService; + @Autowired private ScanTaskOutboxDispatcher scanTaskOutboxDispatcher; + @Autowired private GithubSyncPublicationRepository publications; + @Autowired private GithubSyncIngestionRepository ingestions; + @Autowired private GithubSourceTaskRepository sourceTasks; + @Autowired private GithubSyncTaskRepository syncTasks; + @Autowired private GithubCandidateTaskRepository candidateTasks; + @LocalServerPort private int port; + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_OPENAPI_OUTPUT", matches = ".+") + void exportedOpenApiIncludesLiveAdmissionCounts() throws Exception { + HttpResponse response = HttpClient.newHttpClient().send( + HttpRequest.newBuilder(URI.create(baseUrl() + "/v3/api-docs")).GET().build(), + HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).isEqualTo(200); + assertThat(response.body()).contains("globalUnresolvedCount", "platformUnresolvedCount", + "platformPendingCandidateCount", "platformPaused", "failedPackages"); + Files.writeString(Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_OPENAPI_OUTPUT")), response.body()); + } + + @Test + void admissionPolicyHttpIncludesCurrentBacklogAndCandidateCount() throws Exception { + long namespaceId = managementQueries.platformNamespaceId(); + var before = managementQueries.admissionSnapshot(); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, snapshot_json, + lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', 987654334), 1, ?, + CURRENT_TIMESTAMP + INTERVAL '5 minutes', 'overview-test') + """, taskId, runId, namespaceId, "repository:987654334", "a".repeat(64)); + UUID admissionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, + lease_generation) + VALUES (?, ?, ?, 987654334, ?, 'skills/overview-test', 1) + """, admissionId, taskId, namespaceId, "b".repeat(40)); + long candidateId = jdbc.queryForObject(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, status, + next_validation_at) + VALUES (?, 987654335, 'fixture', 'overview-candidate', 'PENDING', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + try { + var admin = authentication(new UsernamePasswordAuthenticationToken( + new PlatformPrincipal("github-sync-publisher", "overview-admin", null, + null, null, Set.of("SUPER_ADMIN")), null, + List.of(new SimpleGrantedAuthority("ROLE_SUPER_ADMIN")))); + String response = mockMvc.perform(get("/api/v1/admin/github-sync/admission-policy").with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + var data = objectMapper.readTree(response).path("data"); + assertThat(data.path("globalUnresolvedCount").asInt()) + .isEqualTo(before.globalUnresolvedCount() + 1); + assertThat(data.path("platformUnresolvedCount").asInt()) + .isEqualTo(before.platformUnresolvedCount() + 1); + assertThat(data.path("platformPendingCandidateCount").asInt()) + .isEqualTo(before.platformPendingCandidateCount() + 1); + } finally { + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE id = ?", + admissionId); + jdbc.update(""" + UPDATE github_sync_task SET status = 'FAILED', lease_expires_at = NULL, + lease_token_hash = NULL, leased_by = NULL WHERE id = ? + """, taskId); + jdbc.update("UPDATE github_discovery_candidate SET status = 'REJECTED' WHERE id = ?", + candidateId); + } + } + + @Test + void goHeartbeatPersistsOnlineAndOfflineWorkerState() throws Exception { + jdbc.update("DELETE FROM github_sync_worker_presence"); + assertThat(workerPresence.status().lastSeenAt()).isNull(); + assertThat(workerPresence.status().online()).isFalse(); + + assertThat(runWorkerHeartbeat()).contains("PASS"); + assertThat(workerPresence.status().lastSeenAt()).isNotNull(); + assertThat(workerPresence.status().online()).isTrue(); + + jdbc.update(""" + UPDATE github_sync_worker_presence + SET last_seen_at = CURRENT_TIMESTAMP - INTERVAL '91 seconds' + WHERE id = 1 + """); + assertThat(workerPresence.status().online()).isFalse(); + assertThat(runWorkerHeartbeat()).contains("PASS"); + assertThat(workerPresence.status().online()).isTrue(); + } + + private String runWorkerHeartbeat() throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropWorkerHeartbeat$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_HEARTBEAT", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go worker heartbeat timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go worker heartbeat: %s", output).isZero(); + return output; + } + + @Test + void acceptedPackageContinuesAfterCompleteJavaApplicationRestart() throws Exception { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES ('interop-restart-team', 'Interop Restart Team', 'TEAM') RETURNING id + """, Long.class); + long repositoryId = 10023; + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, repositoryId); + UUID ingestionId; + try (JavaApplicationProcess first = startJavaApplication(3_600_000)) { + // The first fixed-delay tick runs at startup; receive the package after that tick. + Thread.sleep(1_000); + ingestionId = UUID.fromString(runGoClient(taskId, first.port())); + assertThat(ingestionStatus(ingestionId)).isEqualTo("RECEIVED"); + } + assertThat(ingestionStatus(ingestionId)).isEqualTo("RECEIVED"); + + try (JavaApplicationProcess restarted = startJavaApplication(500)) { + Instant deadline = Instant.now().plusSeconds(90); + while (!"READY_FOR_REVIEW".equals(ingestionStatus(ingestionId)) + && Instant.now().isBefore(deadline)) { + assertThat(restarted.process().isAlive()).isTrue(); + Thread.sleep(250); + } + assertThat(ingestionStatus(ingestionId)).isEqualTo("READY_FOR_REVIEW"); + } + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE github_sync_ingestion_id = ? AND origin_type = 'GITHUB_SYNC' + AND status = 'PENDING' + """, Integer.class, ingestionId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill_version + WHERE id = (SELECT published_version_id FROM github_sync_package_ingestion + WHERE id = ?) + """, Integer.class, ingestionId)).isEqualTo(1); + } + + @Test + void goLeaseIsReclaimedAfterJavaStopsDuringRepositoryTask() throws Exception { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES ('interop-outage-team', 'Interop Outage Team', 'TEAM') RETURNING id + """, Long.class); + long repositoryId = 10026; + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, repositoryId); + Path archive = Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE")); + Path markerDirectory = Files.createTempDirectory("skillhub-java-outage-"); + Path ready = markerDirectory.resolve("ready"); + Path outage = markerDirectory.resolve("outage"); + Path goOutput = markerDirectory.resolve("go.log"); + Process goProcess = null; + try { + int javaPort; + UUID ingestionId; + try (JavaApplicationProcess first = startJavaApplication(3_600_000)) { + javaPort = first.port(); + Thread.sleep(1_000); + ProcessBuilder builder = goClientBuilder(taskId, null, COMMIT, archive, + "wait-for-java-outage", null, "http://127.0.0.1:" + javaPort); + builder.environment().put("SKILLHUB_INTEROP_READY_MARKER", ready.toString()); + builder.environment().put("SKILLHUB_INTEROP_OUTAGE_MARKER", outage.toString()); + builder.redirectOutput(goOutput.toFile()); + goProcess = builder.start(); + Instant deadline = Instant.now().plusSeconds(60); + while (!Files.exists(ready) && Instant.now().isBefore(deadline)) { + if (!goProcess.isAlive()) { + throw new IllegalStateException("Go client stopped before receipt: " + + childFailureSummary(goOutput)); + } + Thread.sleep(100); + } + assertThat(Files.exists(ready)).isTrue(); + ingestionId = UUID.fromString(Files.readString(ready).trim()); + assertThat(ingestionStatus(ingestionId)).isEqualTo("RECEIVED"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("LEASED"); + } + + assertThat(goProcess.isAlive()).isTrue(); + Files.createFile(outage); + assertThat(goProcess.waitFor(15, TimeUnit.SECONDS)).isTrue(); + assertThat(goProcess.exitValue()) + .as("Go client after Java outage: %s", childFailureSummary(goOutput)).isZero(); + assertThat(Files.readString(goOutput).trim()).isEqualTo(ingestionId.toString()); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("LEASED"); + + // Advance the persisted lease clock rather than waiting for the production TTL. + jdbc.update(""" + UPDATE github_sync_task + SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + try (JavaApplicationProcess restarted = startJavaApplication(3_600_000, javaPort)) { + assertThat(runGoClient(taskId, null, COMMIT, archive, "resume-only", + ingestionId.toString(), "http://127.0.0.1:" + restarted.port())) + .isEqualTo(ingestionId.toString()); + } + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id = ?", + Long.class, taskId)).isEqualTo(2); + assertSingleReceivedPackage(taskId, ingestionId.toString()); + } finally { + if (goProcess != null && goProcess.isAlive()) { + goProcess.descendants().forEach(ProcessHandle::destroyForcibly); + goProcess.destroyForcibly(); + goProcess.waitFor(5, TimeUnit.SECONDS); + } + Files.deleteIfExists(ready); + Files.deleteIfExists(outage); + Files.deleteIfExists(goOutput); + Files.deleteIfExists(markerDirectory); + } + } + + private String ingestionStatus(UUID ingestionId) { + return jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, ingestionId); + } + + private JavaApplicationProcess startJavaApplication(int ingestionIntervalMs) throws Exception { + return startJavaApplication(ingestionIntervalMs, 0); + } + + private JavaApplicationProcess startJavaApplication(int ingestionIntervalMs, + int requestedPort) throws Exception { + int childPort = requestedPort; + if (childPort == 0) { + try (ServerSocket socket = new ServerSocket(0)) { + childPort = socket.getLocalPort(); + } + } + Path log = Files.createTempFile("skillhub-java-restart-", ".log"); + String applicationClasspath = Arrays.stream( + System.getProperty("java.class.path").split(File.pathSeparator)) + .filter(entry -> !Path.of(entry).getFileName().toString().equals("test-classes")) + .collect(Collectors.joining(File.pathSeparator)); + ProcessBuilder builder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Dspring.devtools.restart.enabled=false", "-cp", applicationClasspath, + "com.iflytek.skillhub.SkillhubApplication"); + builder.redirectErrorStream(true).redirectOutput(log.toFile()); + Map environment = builder.environment(); + environment.remove("SPRING_PROFILES_ACTIVE"); + environment.put("SERVER_PORT", Integer.toString(childPort)); + environment.put("SPRING_DATASOURCE_URL", POSTGRES.getJdbcUrl()); + environment.put("SPRING_DATASOURCE_USERNAME", POSTGRES.getUsername()); + environment.put("SPRING_DATASOURCE_PASSWORD", POSTGRES.getPassword()); + environment.put("SPRING_DATA_REDIS_HOST", REDIS.getHost()); + environment.put("SPRING_DATA_REDIS_PORT", REDIS.getMappedPort(6379).toString()); + environment.put("SKILLHUB_GITHUB_SYNC_SERVICE_TOKENS", SERVICE_TOKEN); + environment.put("SKILLHUB_GITHUB_SYNC_CREDENTIAL_KEYS", + "v1=" + Base64.getEncoder().encodeToString(CREDENTIAL_KEY)); + environment.put("SKILLHUB_GITHUB_SYNC_ACTIVE_CREDENTIAL_KEY_VERSION", "v1"); + environment.put("SKILLHUB_GITHUB_SYNC_INGESTION_VALIDATION_INTERVAL_MS", + Integer.toString(ingestionIntervalMs)); + environment.put("SKILLHUB_GITHUB_SYNC_PUBLICATION_INTERVAL_MS", + Integer.toString(ingestionIntervalMs)); + environment.put("SKILLHUB_GITHUB_SYNC_DISCOVERY_SCHEDULER_INTERVAL_MS", "3600000"); + environment.put("SKILLHUB_GITHUB_SYNC_CANDIDATE_SCHEDULER_INTERVAL_MS", "3600000"); + environment.put("SKILLHUB_GITHUB_SYNC_SOURCE_SCHEDULER_INTERVAL_MS", "3600000"); + environment.put("SKILLHUB_GITHUB_SYNC_UPLOAD_CLEANUP_INTERVAL_MS", "3600000"); + environment.put("SKILLHUB_BUILTIN_SKILLS_ENABLED", "false"); + environment.put("SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET", + "fixture-download-cookie-secret-for-java-restart"); + environment.put("SKILLHUB_SECURITY_SCANNER_URL", scannerBaseUrl); + environment.put("STORAGE_BASE_PATH", STORAGE_ROOT.toString()); + if (S3_MODE) { + environment.put("SKILLHUB_STORAGE_PROVIDER", "s3"); + environment.put("SKILLHUB_STORAGE_S3_ENDPOINT", s3Endpoint); + environment.put("SKILLHUB_STORAGE_S3_BUCKET", S3_BUCKET); + environment.put("SKILLHUB_STORAGE_S3_ACCESS_KEY", MINIO_ACCESS_KEY); + environment.put("SKILLHUB_STORAGE_S3_SECRET_KEY", MINIO_SECRET_KEY); + environment.put("SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET", "true"); + } else { + environment.put("SKILLHUB_STORAGE_PROVIDER", "local"); + } + Process process = builder.start(); + JavaApplicationProcess application = new JavaApplicationProcess(process, log, childPort); + try { + // Full application restarts compete with the parent test JVM and local preview services. + Instant deadline = Instant.now().plusSeconds(120); + while (Instant.now().isBefore(deadline)) { + if (!process.isAlive()) { + throw new IllegalStateException("Java application exited before becoming healthy: " + + process.exitValue() + ": " + childFailureSummary(log)); + } + try { + HttpURLConnection connection = (HttpURLConnection) URI.create( + "http://127.0.0.1:" + childPort + "/actuator/health") + .toURL().openConnection(); + connection.setConnectTimeout(500); + connection.setReadTimeout(500); + if (connection.getResponseCode() == 200) { + connection.disconnect(); + return application; + } + connection.disconnect(); + } catch (IOException ignored) { + // The application has not bound its HTTP listener yet. + } + Thread.sleep(250); + } + throw new IllegalStateException("Java application did not become healthy: " + + childFailureSummary(log)); + } catch (Exception exception) { + application.close(); + throw exception; + } + } + + private String childFailureSummary(Path log) throws IOException { + String output = Files.readString(log); + output = output.replace(POSTGRES.getPassword(), "[redacted]") + .replace(SERVICE_TOKEN, "[redacted]") + .replace(MINIO_ACCESS_KEY, "[redacted]") + .replace(MINIO_SECRET_KEY, "[redacted]") + .replace(Base64.getEncoder().encodeToString(CREDENTIAL_KEY), "[redacted]"); + return output.substring(Math.max(0, output.length() - 4000)); + } + + private record JavaApplicationProcess(Process process, Path log, int port) + implements AutoCloseable { + @Override + public void close() throws Exception { + process.destroy(); + if (!process.waitFor(15, TimeUnit.SECONDS)) { + process.destroyForcibly(); + assertThat(process.waitFor(5, TimeUnit.SECONDS)).isTrue(); + } + Files.deleteIfExists(log); + } + } + + @Test + void invalidGoArchiveDoesNotExposeSkillBodyInFailureOutputs() throws Exception { + String bodyMarker = "SKILL_BODY_" + UUID.randomUUID().toString().replace("-", ""); + Path invalidArchive = archiveWithNestedSkill(bodyMarker); + Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + ListAppender logs = new ListAppender<>(); + logs.start(); + root.addAppender(logs); + try { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, + namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long namespaceId = managementQueries.platformNamespaceId(); + long repositoryId = 600_000_000L + Integer.toUnsignedLong(UUID.randomUUID().hashCode()); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, repositoryId); + String ingestionId = runGoClient(taskId, null, COMMIT, invalidArchive); + ingestionTask.validatePending(); + + UUID id = UUID.fromString(ingestionId); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, id)).isEqualTo("FAILED"); + assertThat(jdbc.queryForObject(""" + SELECT last_error_code FROM github_sync_package_ingestion WHERE id = ? + """, String.class, id)).isEqualTo("PACKAGE_STRUCTURE_INVALID"); + assertThat(jdbc.queryForObject(""" + SELECT outcome FROM github_sync_path_result WHERE ingestion_id = ? + """, String.class, id)).isEqualTo("TERMINAL_FAILURE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE github_sync_ingestion_id = ? + """, Integer.class, id)).isZero(); + + String persisted = jdbc.queryForObject(""" + SELECT row_to_json(failure)::text FROM ( + SELECT task.snapshot_json, task.last_error_code AS task_error, + ingestion.last_error_code AS ingestion_error, path.reason + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + JOIN github_sync_path_result path ON path.ingestion_id = ingestion.id + WHERE ingestion.id = ? + ) failure + """, String.class, id); + String operatorView = objectMapper.writeValueAsString(managementQueries.runs(namespaceId, 0, 10)) + + objectMapper.writeValueAsString(managementQueries.sources(namespaceId, 0, 10)) + + objectMapper.writeValueAsString(managementQueries.skillPaths(namespaceId, sourceId, 0, 10)); + var admin = authentication(new UsernamePasswordAuthenticationToken( + new PlatformPrincipal("github-sync-publisher", "privacy-admin", null, + null, null, Set.of("SUPER_ADMIN")), null, + List.of(new SimpleGrantedAuthority("ROLE_SUPER_ADMIN")))); + String apiView = mockMvc.perform(get("/api/v1/admin/github-sync/runs").with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString() + + mockMvc.perform(get("/api/v1/admin/github-sync/sources").with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString() + + mockMvc.perform(get("/api/v1/admin/github-sync/sources/{sourceId}/skills", sourceId) + .with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + String logText = logs.list.stream().map(event -> event.getFormattedMessage() + + (event.getThrowableProxy() == null ? "" + : ThrowableProxyUtil.asString(event.getThrowableProxy())) + + event.getMDCPropertyMap()).collect(Collectors.joining("\n")); + String metricIds = meterRegistry.getMeters().stream().map(meter -> meter.getId().toString()) + .collect(Collectors.joining("\n")); + assertThat(persisted).doesNotContain(bodyMarker); + assertThat(operatorView).doesNotContain(bodyMarker); + assertThat(apiView).contains("PACKAGE_STRUCTURE_INVALID").doesNotContain(bodyMarker); + assertThat(logText).doesNotContain(bodyMarker); + assertThat(metricIds).doesNotContain(bodyMarker); + } finally { + root.detachAppender(logs); + logs.stop(); + Files.deleteIfExists(invalidArchive); + } + } + + @Test + void scannerHttpFailureDoesNotExposeSkillBodyInOperatorOutputs() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue(scanner != null, "requires the isolated scanner fixture"); + String bodyMarker = "SKILL_BODY_" + UUID.randomUUID().toString().replace("-", ""); + Path archive = revisionArchive(bodyMarker); + Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + ListAppender logs = new ListAppender<>(); + logs.start(); + root.addAppender(logs); + try { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, + namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + String namespaceSlug = "interop-scan-" + UUID.randomUUID().toString().substring(0, 8); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Interop Scanner Failure', 'TEAM') RETURNING id + """, Long.class, namespaceSlug); + long repositoryId = 700_000_000L + Integer.toUnsignedLong(UUID.randomUUID().hashCode()); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'openai', 'skills', 'NAMESPACE_ONLY', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, repositoryId); + UUID ingestionId = UUID.fromString(runGoClient(taskId, null, COMMIT, archive)); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + Long versionId = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion WHERE id = ? + """, Long.class, ingestionId); + assertThat(versionId).isNotNull(); + assertThat(jdbc.queryForObject("SELECT status FROM skill_version WHERE id = ?", + String.class, versionId)).isEqualTo("SCANNING"); + + scannerFailureMarker.set(bodyMarker); + scanTaskOutboxDispatcher.dispatch(); + awaitVersionStatus(versionId, "SCAN_FAILED"); + assertThat(jdbc.queryForObject(""" + SELECT failure_reason FROM security_audit WHERE skill_version_id = ? + """, String.class, versionId)).doesNotContain(bodyMarker); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill WHERE latest_version_id = ? + """, Integer.class, versionId)).isZero(); + + String persisted = jdbc.queryForObject(""" + SELECT row_to_json(failure)::text FROM ( + SELECT task.snapshot_json, task.last_error_code AS task_error, + ingestion.last_error_code AS ingestion_error, path.reason, + audit.failure_reason + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + JOIN github_sync_path_result path ON path.ingestion_id = ingestion.id + JOIN security_audit audit ON audit.skill_version_id = ingestion.published_version_id + WHERE ingestion.id = ? + ) failure + """, String.class, ingestionId); + String reviewerId = "privacy-scanner-" + UUID.randomUUID(); + jdbc.update("INSERT INTO user_account (id, display_name) VALUES (?, 'Privacy Scanner Reviewer')", + reviewerId); + jdbc.update(""" + INSERT INTO namespace_member (namespace_id, user_id, role, created_at, updated_at) + VALUES (?, ?, 'OWNER', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, namespaceId, reviewerId); + var admin = authentication(new UsernamePasswordAuthenticationToken( + new PlatformPrincipal(reviewerId, "privacy-admin", null, + null, null, Set.of("SUPER_ADMIN")), null, + List.of(new SimpleGrantedAuthority("ROLE_SUPER_ADMIN")))); + String apiView = mockMvc.perform(get("/api/v1/namespaces/{namespaceSlug}/github-sources/runs", + namespaceSlug).with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString() + + mockMvc.perform(get("/api/v1/namespaces/{namespaceSlug}/github-sources/resolved", + namespaceSlug).with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString() + + mockMvc.perform(get("/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills", + namespaceSlug, sourceId) + .with(admin)) + .andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + String logText = logs.list.stream().map(event -> event.getFormattedMessage() + + (event.getThrowableProxy() == null ? "" + : ThrowableProxyUtil.asString(event.getThrowableProxy())) + + event.getMDCPropertyMap()).collect(Collectors.joining("\n")); + String metricIds = meterRegistry.getMeters().stream().map(meter -> meter.getId().toString()) + .collect(Collectors.joining("\n")); + assertThat(persisted).doesNotContain(bodyMarker); + assertThat(apiView).doesNotContain(bodyMarker); + assertThat(logText).doesNotContain(bodyMarker); + assertThat(metricIds).doesNotContain(bodyMarker); + } finally { + scannerFailureMarker.set(null); + root.detachAppender(logs); + logs.stop(); + Files.deleteIfExists(archive); + } + } + + @Test + void goArchiveCanTravelThroughHttpReceiptAndPersistedIngestion() throws Exception { + byte[] archive = Files.readAllBytes(Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE"))); + assertThat(archive).isNotEmpty(); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(archive)); + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 10001, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID oversizedTaskId = S3_MODE ? UUID.randomUUID() : null; + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, priority, + snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, 100, + '{"repositoryId":10001,"owner":"openai","name":"skills"}'::jsonb, ?) + """, taskId, runId, namespaceId, sourceId, "repository:10001", COMMIT); + + if (S3_MODE) { + UUID oversizedRunId = UUID.randomUUID(); + long oversizedSourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 10002, 'openai', 'oversized-fixture', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, oversizedRunId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, priority, + snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, 90, + '{"repositoryId":10002,"owner":"openai","name":"skills"}'::jsonb, ?) + """, oversizedTaskId, oversizedRunId, namespaceId, oversizedSourceId, + "repository:10002", COMMIT); + } + + String ingestionId = runGoClient(taskId, oversizedTaskId); + if (S3_MODE) { + UUID uploadId = jdbc.queryForObject(""" + SELECT upload_id FROM github_sync_package_ingestion WHERE id = ? + """, UUID.class, UUID.fromString(ingestionId)); + assertThat(storage.exists("github-sync/staging/" + uploadId + ".zip.forbidden")) + .isFalse(); + String objectKey = jdbc.queryForObject(""" + SELECT object_key FROM github_sync_staged_upload WHERE id = ? + """, String.class, uploadId); + assertThat(storage.getMetadata(objectKey).size()).isEqualTo(archive.length); + StoragePrefixUsage stagedUsage = storage.listPrefixUsage("github-sync/staging/", 10_000) + .orElseThrow(); + assertThat(stagedUsage.complete()).isTrue(); + assertThat(stagedUsage.objectCount()).isPositive(); + assertThat(stagedUsage.bytes()).isGreaterThanOrEqualTo(archive.length); + capacityMetrics.sampleS3(); + assertThat(meterRegistry.get("skillhub.github.sync.staging.s3.used.bytes") + .gauge().value()).isEqualTo(stagedUsage.bytes()); + assertThat(meterRegistry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isEqualTo(1); + } + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE id = ? AND task_id = ? AND sha256 = ? AND size_bytes = ? + """, Integer.class, UUID.fromString(ingestionId), taskId, digest, archive.length)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND skill_path = ? + AND outcome = 'ACCEPTED' AND ingestion_id = ? + """, Integer.class, taskId, COMMIT, SKILL_PATH, UUID.fromString(ingestionId))).isEqualTo(1); + claimInSeparateJvm(ingestionId, IngestionClaimProcess.class); + Timestamp validationExpiry = jdbc.queryForObject(""" + SELECT validation_expires_at FROM github_sync_package_ingestion WHERE id = ? + """, Timestamp.class, UUID.fromString(ingestionId)); + UUID expiredToken = jdbc.queryForObject(""" + SELECT validation_token FROM github_sync_package_ingestion WHERE id = ? + """, UUID.class, UUID.fromString(ingestionId)); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(ingestionId))).isEqualTo("VALIDATING"); + Thread.sleep(Math.max(0, Duration.between(Instant.now(), + validationExpiry.toInstant()).toMillis() + 100)); + ingestionTask.validatePending(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE id = ? AND validated_at IS NOT NULL AND content_sha256 IS NOT NULL + AND status IN ('VALIDATED', 'READY_FOR_REVIEW') AND validation_attempts = 2 + """, Integer.class, UUID.fromString(ingestionId))).isEqualTo(1); + assertThat(ingestions.validated(UUID.fromString(ingestionId), expiredToken, + "0".repeat(64), 0, "{}", Instant.now())).isFalse(); + claimInSeparateJvm(ingestionId, PublicationClaimProcess.class); + Timestamp publicationExpiry = jdbc.queryForObject(""" + SELECT publication_expires_at FROM github_sync_package_ingestion WHERE id = ? + """, Timestamp.class, UUID.fromString(ingestionId)); + UUID expiredPublicationToken = jdbc.queryForObject(""" + SELECT publication_token FROM github_sync_package_ingestion WHERE id = ? + """, UUID.class, UUID.fromString(ingestionId)); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(ingestionId))).isEqualTo("VALIDATED"); + Thread.sleep(Math.max(0, Duration.between(Instant.now(), + publicationExpiry.toInstant()).toMillis() + 100)); + var expiredPublicationClaim = new GithubSyncPublicationRepository.ClaimedPublication( + UUID.fromString(ingestionId), expiredPublicationToken, namespaceId, "github", + 10001, sourceId, SKILL_PATH, "ACTIVE", "PUBLIC", "", archive.length, + digest, "0".repeat(64), null, 1); + assertThat(publications.noChange(expiredPublicationClaim, Instant.now())).isFalse(); + publicationTask.publishValidated(); + Long versionId = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion WHERE id = ? + """, Long.class, UUID.fromString(ingestionId)); + assertThat(versionId).isNotNull(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(ingestionId))).isEqualTo("READY_FOR_REVIEW"); + assertThat(jdbc.queryForObject(""" + SELECT publication_attempts FROM github_sync_package_ingestion WHERE id = ? + """, Integer.class, UUID.fromString(ingestionId))).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE skill_version_id = ? AND origin_type = 'GITHUB_SYNC' + AND github_sync_ingestion_id = ? AND status = 'PENDING' + """, Integer.class, versionId, UUID.fromString(ingestionId))).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE skill_version_id = ? AND origin_type = 'USER' + """, Integer.class, versionId)).isZero(); + assertThat(reviews.findByStatus(ReviewTaskStatus.PENDING, PageRequest.of(0, 10)) + .getTotalElements()).isZero(); + assertThat(reviews.findByOriginTypeAndStatus(ReviewOriginType.GITHUB_SYNC, + ReviewTaskStatus.PENDING, PageRequest.of(0, 10)).getTotalElements()).isPositive(); + assertThat(jdbc.queryForObject(""" + SELECT version.status FROM skill_version version WHERE version.id = ? + """, String.class, versionId)).isEqualTo("SCANNING"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM scan_task_outbox + WHERE version_id = ? AND status = 'PENDING' + """, Integer.class, versionId)).isEqualTo(1); + scanTaskOutboxDispatcher.dispatch(); + awaitVersionStatus(versionId, "PENDING_REVIEW"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM security_audit + WHERE skill_version_id = ? AND is_safe = TRUE AND scanned_at IS NOT NULL + """, Integer.class, versionId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill WHERE latest_version_id = ? + """, Integer.class, versionId)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + AND namespace_id = ? AND owner_id = 'github-sync-publisher' + AND github_sync_managed = TRUE AND visibility = 'PUBLIC' + """, Integer.class, versionId, namespaceId)).isEqualTo(1); + String platformSlug = skillSlug(versionId); + assertThat(anonymousSkillDetail("github", platformSlug).statusCode()).isEqualTo(400); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? AND latest_ingestion_id = ? + """, Integer.class, sourceId, SKILL_PATH, + UUID.fromString(ingestionId))).isEqualTo(1); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE github_sync_ingestion_id = ? + """, Integer.class, UUID.fromString(ingestionId))).isEqualTo(1); + if (S3_MODE) { + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, oversizedTaskId)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT upload.status FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + WHERE admission.task_id = ? + """, String.class, oversizedTaskId)).isEqualTo("FAILED"); + String oversizedObjectKey = jdbc.queryForObject(""" + SELECT upload.object_key FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + WHERE admission.task_id = ? + """, String.class, oversizedTaskId); + assertThat(storage.exists(oversizedObjectKey)).isFalse(); + } + + UUID repeatRunId = UUID.randomUUID(); + UUID repeatTaskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, repeatRunId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, priority, + snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, 100, + '{"repositoryId":10001,"owner":"openai","name":"skills"}'::jsonb, ?) + """, repeatTaskId, repeatRunId, namespaceId, sourceId, + "interop-repeat:" + repeatTaskId, COMMIT); + String repeatIngestionId = runGoClient(repeatTaskId, null); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(repeatIngestionId))).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill_version WHERE skill_id = + (SELECT skill_id FROM skill_version WHERE id = ?) + """, Integer.class, versionId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE origin_type = 'GITHUB_SYNC' AND namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(1); + + jdbc.update(""" + INSERT INTO user_account (id, display_name) + VALUES ('interop-reviewer', 'Interop Reviewer') + ON CONFLICT (id) DO NOTHING + """); + Long reviewId = jdbc.queryForObject(""" + SELECT id FROM review_task WHERE github_sync_ingestion_id = ? + """, Long.class, UUID.fromString(ingestionId)); + reviewService.approveReview(reviewId, "interop-reviewer", "License evidence checked", + Map.of(), Set.of("SUPER_ADMIN")); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(ingestionId))).isEqualTo("APPROVED"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM skill_version WHERE id = ? + """, String.class, versionId)).isEqualTo("PUBLISHED"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill WHERE latest_version_id = ? + """, Integer.class, versionId)).isEqualTo(1); + HttpResponse publicDetail = anonymousSkillDetail("github", platformSlug); + assertThat(publicDetail.statusCode()).isEqualTo(200); + var publicData = objectMapper.readTree(publicDetail.body()).path("data"); + assertThat(publicData.path("publishedVersion").path("status").asText()).isEqualTo("PUBLISHED"); + assertThat(publicData.path("githubSource").path("ownerLogin").asText()).isEqualTo("openai"); + assertThat(publicData.path("githubSource").path("commitSha").asText()).isEqualTo(COMMIT); + + long teamNamespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES ('interop-team', 'Interop Team', 'TEAM') RETURNING id + """, Long.class); + long teamSourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 10003, 'openai', 'skills', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, teamNamespaceId); + UUID teamTaskId = createRepositoryTask(teamNamespaceId, teamSourceId, 10003); + String teamIngestionId = runGoClient(teamTaskId, null); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + Long teamVersionId = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion WHERE id = ? + """, Long.class, UUID.fromString(teamIngestionId)); + assertThat(teamVersionId).isNotNull(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + AND namespace_id = ? AND owner_id = 'github-sync-publisher' + AND visibility = 'NAMESPACE_ONLY' AND latest_version_id IS NULL + """, Integer.class, teamVersionId, teamNamespaceId)).isEqualTo(1); + scanTaskOutboxDispatcher.dispatch(); + awaitVersionStatus(teamVersionId, "PENDING_REVIEW"); + Long teamReviewId = jdbc.queryForObject(""" + SELECT id FROM review_task WHERE github_sync_ingestion_id = ? + """, Long.class, UUID.fromString(teamIngestionId)); + reviewService.rejectReview(teamReviewId, "interop-reviewer", "Not ready for team use", + Map.of(), Set.of("SUPER_ADMIN")); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(teamIngestionId))).isEqualTo("REJECTED"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill WHERE latest_version_id = ? + """, Integer.class, teamVersionId)).isZero(); + + UUID teamRepeatTaskId = createRepositoryTask(teamNamespaceId, teamSourceId, 10003); + String teamRepeatIngestionId = runGoClient(teamRepeatTaskId, null); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(teamRepeatIngestionId))).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE namespace_id = ? AND origin_type = 'GITHUB_SYNC' + """, Integer.class, teamNamespaceId)).isEqualTo(1); + + assertThat(publications.requestRejectedRetry(teamNamespaceId, teamSourceId + 1, + SKILL_PATH, Instant.now())).isFalse(); + assertThat(publications.requestRejectedRetry(teamNamespaceId, teamSourceId, + SKILL_PATH, Instant.now())).isTrue(); + assertThat(sourceTasks.enqueueDue(Instant.now(), 100)).isPositive(); + UUID retryTaskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task + WHERE source_id = ? AND status = 'QUEUED' + ORDER BY created_at DESC LIMIT 1 + """, UUID.class, teamSourceId); + assertThat(jdbc.queryForObject("SELECT mode FROM github_sync_task WHERE id = ?", + String.class, retryTaskId)).isEqualTo("FULL"); + assertThat(jdbc.queryForObject(""" + SELECT retry_scheduled_at FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? + """, Timestamp.class, teamSourceId, SKILL_PATH)).isNotNull(); + jdbc.update("UPDATE github_sync_task SET status = 'FAILED' WHERE id = ?", retryTaskId); + sourceTasks.enqueueDue(Instant.now(), 100); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE source_id = ? AND status = 'QUEUED' + """, Integer.class, teamSourceId)).isZero(); + assertThat(publications.requestRejectedRetry(teamNamespaceId, teamSourceId, + SKILL_PATH, Instant.now())).isTrue(); + assertThat(sourceTasks.enqueueDue(Instant.now(), 100)).isPositive(); + retryTaskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task + WHERE source_id = ? AND status = 'QUEUED' + ORDER BY created_at DESC LIMIT 1 + """, UUID.class, teamSourceId); + jdbc.update("UPDATE github_sync_task SET fixed_commit_sha = ? WHERE id = ?", COMMIT, retryTaskId); + String retriedIngestionId = runGoClient(retryTaskId, null); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(retriedIngestionId))).isEqualTo("READY_FOR_REVIEW"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE namespace_id = ? AND origin_type = 'GITHUB_SYNC' + """, Integer.class, teamNamespaceId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT retry_requested_at FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? + """, Timestamp.class, teamSourceId, SKILL_PATH)).isNull(); + assertThat(jdbc.queryForObject(""" + SELECT retry_scheduled_at FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? + """, Timestamp.class, teamSourceId, SKILL_PATH)).isNull(); + + Path revisionA = revisionArchive("a"); + Path revisionB = revisionArchive("b"); + Path revisionC = revisionArchive("c"); + Path revisionD = revisionArchive("d"); + Path licenseRevision = revisionArchive("d", "v2"); + try { + UUID firstUpdateTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "a".repeat(40)); + String firstUpdateId = runGoClient(firstUpdateTask, null, "a".repeat(40), revisionA); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + Long firstUpdateReview = jdbc.queryForObject(""" + SELECT id FROM review_task WHERE github_sync_ingestion_id = ? + """, Long.class, UUID.fromString(firstUpdateId)); + assertThat(firstUpdateReview).isNotNull(); + + UUID secondUpdateTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "b".repeat(40)); + String secondUpdateId = runGoClient(secondUpdateTask, null, "b".repeat(40), revisionB); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + Long secondUpdateReview = jdbc.queryForObject(""" + SELECT id FROM review_task WHERE github_sync_ingestion_id = ? + """, Long.class, UUID.fromString(secondUpdateId)); + Long secondUpdateVersion = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion WHERE id = ? + """, Long.class, UUID.fromString(secondUpdateId)); + assertThat(jdbc.queryForObject("SELECT status FROM review_task WHERE id = ?", + String.class, firstUpdateReview)).isEqualTo("SUPERSEDED"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_ingestion WHERE id = ?", + String.class, UUID.fromString(firstUpdateId))).isEqualTo("SUPERSEDED"); + assertThat(jdbc.queryForObject("SELECT status FROM review_task WHERE id = ?", + String.class, secondUpdateReview)).isEqualTo("PENDING"); + assertThatThrownBy(() -> reviewService.approveReview(firstUpdateReview, + "interop-reviewer", "obsolete", Map.of(), Set.of("SUPER_ADMIN"))) + .hasMessageContaining("review.not_pending"); + scanTaskOutboxDispatcher.dispatch(); + awaitVersionStatus(secondUpdateVersion, "PENDING_REVIEW"); + reviewService.approveReview(secondUpdateReview, "interop-reviewer", "Current revision", + Map.of(), Set.of("SUPER_ADMIN")); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, secondUpdateVersion)).isEqualTo(secondUpdateVersion); + assertThat(anonymousSkillDetail("interop-team", skillSlug(secondUpdateVersion)) + .statusCode()).isEqualTo(403); + + UUID olderTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "c".repeat(40)); + String olderId = runGoClient(olderTask, null, "c".repeat(40), revisionC); + UUID newerTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "d".repeat(40)); + String newerId = runGoClient(newerTask, null, "d".repeat(40), revisionD); + ingestionTask.validatePending(); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET publication_retry_at = CURRENT_TIMESTAMP + INTERVAL '1 hour' + WHERE id = ? + """, UUID.fromString(olderId)); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT latest_ingestion_id FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? + """, UUID.class, teamSourceId, SKILL_PATH)).isEqualTo(UUID.fromString(newerId)); + jdbc.update(""" + UPDATE github_sync_package_ingestion SET publication_retry_at = NULL WHERE id = ? + """, UUID.fromString(olderId)); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(olderId))).isEqualTo("SUPERSEDED"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE github_sync_ingestion_id = ? + """, Integer.class, UUID.fromString(olderId))).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT latest_ingestion_id FROM github_source_skill_binding + WHERE source_id = ? AND skill_path = ? + """, UUID.class, teamSourceId, SKILL_PATH)).isEqualTo(UUID.fromString(newerId)); + assertThat(jdbc.queryForObject(""" + SELECT status FROM review_task WHERE github_sync_ingestion_id = ? + """, String.class, UUID.fromString(newerId))).isEqualTo("PENDING"); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, secondUpdateVersion)).isEqualTo(secondUpdateVersion); + + UUID sameContentTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "e".repeat(40)); + String sameContentId = runGoClient(sameContentTask, null, + "e".repeat(40), revisionD); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(sameContentId))).isEqualTo("NO_CHANGE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task WHERE github_sync_ingestion_id = ? + """, Integer.class, UUID.fromString(sameContentId))).isZero(); + + UUID licenseTask = createRepositoryTask(teamNamespaceId, teamSourceId, 10003, + "f".repeat(40)); + String licenseId = runGoClient(licenseTask, null, + "f".repeat(40), licenseRevision); + ingestionTask.validatePending(); + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(licenseId))).isEqualTo("READY_FOR_REVIEW"); + assertThat(jdbc.queryForObject(""" + SELECT content_sha256 FROM github_sync_package_ingestion WHERE id = ? + """, String.class, UUID.fromString(licenseId))).isNotEqualTo( + jdbc.queryForObject("SELECT content_sha256 FROM github_sync_package_ingestion WHERE id = ?", + String.class, UUID.fromString(newerId))); + assertThat(jdbc.queryForObject(""" + SELECT status FROM review_task WHERE github_sync_ingestion_id = ? + """, String.class, UUID.fromString(newerId))).isEqualTo("SUPERSEDED"); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, secondUpdateVersion)).isEqualTo(secondUpdateVersion); + } finally { + Files.deleteIfExists(revisionA); + Files.deleteIfExists(revisionB); + Files.deleteIfExists(revisionC); + Files.deleteIfExists(revisionD); + Files.deleteIfExists(licenseRevision); + } + } + + @Test + void separateGoProcessesResumeAcceptedPackageAfterLeaseExpiry() throws Exception { + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES ('interop-resume-team', 'Interop Resume Team', 'TEAM') RETURNING id + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 10004, 'openai', 'restart-fixture', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 10004); + jdbc.update("UPDATE github_sync_task SET priority = 1000 WHERE id = ?", taskId); + Path archive = Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE")); + + String ingestionId; + try (AutoCloseable ignored = pauseOtherSources(sourceId)) { + ingestionId = runGoClient(taskId, null, COMMIT, archive, "report-only", null); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("LEASED"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + jdbc.update(""" + UPDATE github_sync_task SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + + assertThat(runGoClient(taskId, null, COMMIT, archive, "resume-only", ingestionId)) + .isEqualTo(ingestionId); + } + assertThat(jdbc.queryForObject(""" + SELECT lease_generation FROM github_sync_task WHERE id = ? + """, Long.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + WHERE admission.task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE id = ? AND task_id = ? + """, Integer.class, UUID.fromString(ingestionId), taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + } + + private AutoCloseable pauseOtherSources(long activeSourceId) { + Map previousStatuses = new LinkedHashMap<>(); + jdbc.query(""" + SELECT id, status FROM github_source_binding + WHERE id <> ? AND status <> 'PAUSED' + """, result -> { + previousStatuses.put(result.getLong("id"), result.getString("status")); + }, activeSourceId); + jdbc.update(""" + UPDATE github_source_binding SET status = 'PAUSED' + WHERE id <> ? AND status <> 'PAUSED' + """, activeSourceId); + return () -> previousStatuses.forEach((id, status) -> jdbc.update(""" + UPDATE github_source_binding SET status = ? WHERE id = ? + """, status, id)); + } + + @Test + void repositoryWorkerResumesFailedPathWithoutReacquiringReceivedPackage() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 42, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + String commit = "c".repeat(40); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 42, commit); + Path archive = Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE")); + + String ingestionId = runGoClient(taskId, null, commit, archive, "report-only", null); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE id = ? AND task_id = ? + """, Integer.class, UUID.fromString(ingestionId), taskId)).isEqualTo(1); + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, reason) + VALUES (?, ?, 'skills/retry', 'RETRYABLE_FAILURE', 'FIXTURE_RETRY') + """, taskId, commit); + jdbc.update(""" + UPDATE github_sync_task + SET path_checkpoint = 'skills/retry', + lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + + runRepositoryWorker(taskId, "retry"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject(""" + SELECT retry_at FROM github_sync_task WHERE id = ? + """, Timestamp.class, taskId).toInstant()).isAfter(Instant.now()); + assertThat(jdbc.queryForObject(""" + SELECT outcome FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND skill_path = 'skills/retry' + """, String.class, taskId, commit)).isEqualTo("RETRYABLE_FAILURE"); + assertThat(jdbc.queryForObject(""" + SELECT fixed_commit_sha FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo(commit); + assertSingleReceivedPackage(taskId, ingestionId); + + jdbc.update(""" + UPDATE github_sync_task + SET retry_at = CURRENT_TIMESTAMP - INTERVAL '1 second', + due_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + runRepositoryWorker(taskId, "resolve"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("PARTIAL"); + assertThat(jdbc.queryForObject(""" + SELECT outcome FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND skill_path = 'skills/retry' + """, String.class, taskId, commit)).isEqualTo("TERMINAL_FAILURE"); + assertThat(jdbc.queryForObject(""" + SELECT lease_generation FROM github_sync_task WHERE id = ? + """, Long.class, taskId)).isEqualTo(3); + assertSingleReceivedPackage(taskId, ingestionId); + } + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_LIVE_WORKER", matches = "1") + void repositoryWorkerAcquiresTwoPublicSkillsThroughJava() throws Exception { + boolean realTree = "1".equals(System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE")); + long repositoryId = realTree ? 1103643573L : 987654350L; + byte[] referenceArchive = Files.readAllBytes(Path.of( + System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE"))); + String referenceDigest = HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest(referenceArchive)); + String namespaceSlug = "interop-multi-" + UUID.randomUUID().toString().substring(0, 8); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Interop Multi Skill', 'TEAM') RETURNING id + """, Long.class, namespaceSlug); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'openai', 'skills', 'NAMESPACE_ONLY', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, repositoryId); + + String workerOutput = runPublicMultiSkillWorker(taskId, false); + assertThat(workerOutput).contains("PASS"); + if (realTree) { + assertThat(workerOutput).contains("real Tree API: tree_calls=1"); + } + String lastErrorCode = jdbc.queryForObject( + "SELECT last_error_code FROM github_sync_task WHERE id = ?", String.class, taskId); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).as("lastErrorCode=%s", lastErrorCode) + .isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForList(""" + SELECT skill_path FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND outcome = 'ACCEPTED' + """, String.class, taskId, COMMIT)).containsExactlyInAnyOrder( + "skills/.system/skill-creator", "skills/.curated/define-goal"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'RECEIVED' + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT sha256 FROM github_sync_package_ingestion + WHERE task_id = ? AND skill_path = 'skills/.system/skill-creator' + """, String.class, taskId)).isEqualTo(referenceDigest); + + ingestionTask.validatePending(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status IN ('VALIDATED', 'READY_FOR_REVIEW') + AND validated_at IS NOT NULL AND content_sha256 IS NOT NULL + """, Integer.class, taskId)).isEqualTo(2); + + publicationTask.publishValidated(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'READY_FOR_REVIEW' + AND published_version_id IS NOT NULL + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE namespace_id = ? AND origin_type = 'GITHUB_SYNC' AND status = 'PENDING' + """, Integer.class, namespaceId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill + WHERE namespace_id = ? AND owner_id = 'github-sync-publisher' + AND visibility = 'NAMESPACE_ONLY' AND latest_version_id IS NULL + """, Integer.class, namespaceId)).isEqualTo(2); + + Long firstVersionId = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion + WHERE task_id = ? AND skill_path = 'skills/.system/skill-creator' + """, Long.class, taskId); + Long secondVersionId = jdbc.queryForObject(""" + SELECT published_version_id FROM github_sync_package_ingestion + WHERE task_id = ? AND skill_path = 'skills/.curated/define-goal' + """, Long.class, taskId); + assertThat(firstVersionId).isNotEqualTo(secondVersionId); + scanTaskOutboxDispatcher.dispatch(); + awaitVersionStatus(firstVersionId, "PENDING_REVIEW"); + awaitVersionStatus(secondVersionId, "PENDING_REVIEW"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM security_audit + WHERE skill_version_id IN (?, ?) AND is_safe = TRUE AND scanned_at IS NOT NULL + """, Integer.class, firstVersionId, secondVersionId)).isEqualTo(2); + + jdbc.update(""" + INSERT INTO user_account (id, display_name) + VALUES ('interop-reviewer', 'Interop Reviewer') + ON CONFLICT (id) DO NOTHING + """); + Long firstReviewId = jdbc.queryForObject(""" + SELECT review.id FROM review_task review + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = review.github_sync_ingestion_id + WHERE ingestion.task_id = ? AND ingestion.skill_path = 'skills/.system/skill-creator' + """, Long.class, taskId); + Long secondReviewId = jdbc.queryForObject(""" + SELECT review.id FROM review_task review + JOIN github_sync_package_ingestion ingestion + ON ingestion.id = review.github_sync_ingestion_id + WHERE ingestion.task_id = ? AND ingestion.skill_path = 'skills/.curated/define-goal' + """, Long.class, taskId); + reviewService.approveReview(firstReviewId, "interop-reviewer", "License evidence checked", + Map.of(), Set.of("SUPER_ADMIN")); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, firstVersionId)).isEqualTo(firstVersionId); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, secondVersionId)).isNull(); + reviewService.approveReview(secondReviewId, "interop-reviewer", "License evidence checked", + Map.of(), Set.of("SUPER_ADMIN")); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'APPROVED' + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill_version + WHERE id IN (?, ?) AND status = 'PUBLISHED' + """, Integer.class, firstVersionId, secondVersionId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT latest_version_id FROM skill + WHERE id = (SELECT skill_id FROM skill_version WHERE id = ?) + """, Long.class, secondVersionId)).isEqualTo(secondVersionId); + assertThat(anonymousSkillDetail(namespaceSlug, skillSlug(firstVersionId)).statusCode()) + .isEqualTo(403); + assertThat(anonymousSkillDetail(namespaceSlug, skillSlug(secondVersionId)).statusCode()) + .isEqualTo(403); + } + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_LIVE_ALL_PACKAGES_INTEROP", matches = "1") + void repositoryWorkerReportsEveryEligiblePublicSkillThroughJava() throws Exception { + if (!"1".equals(System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE"))) { + throw new IllegalStateException("full repository interop requires the real GitHub Tree API"); + } + String namespaceSlug = "interop-full-" + UUID.randomUUID().toString().substring(0, 8); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Interop Full Repository', 'TEAM') RETURNING id + """, Long.class, namespaceSlug); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 1103643573, 'openai', 'skills', 'NAMESPACE_ONLY', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 1103643573L); + + String workerOutput = runPublicMultiSkillWorker(taskId, true); + assertThat(workerOutput).contains("PASS", "real Tree API: tree_calls=1"); + Map taskState = jdbc.queryForMap(""" + SELECT status, last_error_code FROM github_sync_task WHERE id = ? + """, taskId); + List> pathReasons = jdbc.queryForList(""" + SELECT outcome, reason, count(*) AS paths FROM github_sync_path_result + WHERE task_id = ? GROUP BY outcome, reason ORDER BY outcome, reason + """, taskId); + for (int recoveryAttempt = 0; + recoveryAttempt < 2 && "RETRY_WAIT".equals(taskState.get("status")); + recoveryAttempt++) { + assertThat(taskState.get("last_error_code")).isEqualTo("BUDGET"); + assertThat(pathReasons).anySatisfy(reason -> { + assertThat(reason.get("outcome")).isEqualTo("RETRYABLE_FAILURE"); + assertThat(reason.get("reason")).isEqualTo("PACKAGE_ACQUISITION_FAILED"); + }); + jdbc.update(""" + UPDATE github_sync_task + SET retry_at = CURRENT_TIMESTAMP - INTERVAL '1 second', + due_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? AND status = 'RETRY_WAIT' + """, taskId); + assertThat(runPublicMultiSkillWorker(taskId, true)) + .contains("PASS", "real Tree API: tree_calls=1"); + taskState = jdbc.queryForMap(""" + SELECT status, last_error_code FROM github_sync_task WHERE id = ? + """, taskId); + pathReasons = jdbc.queryForList(""" + SELECT outcome, reason, count(*) AS paths FROM github_sync_path_result + WHERE task_id = ? GROUP BY outcome, reason ORDER BY outcome, reason + """, taskId); + } + assertThat(taskState.get("status")) + .as("lastErrorCode=%s, pathReasons=%s", taskState.get("last_error_code"), pathReasons) + .isEqualTo("PARTIAL"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND outcome = 'ACCEPTED' + """, Integer.class, taskId, COMMIT)).isEqualTo(43); + assertThat(jdbc.queryForList(""" + SELECT skill_path FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND outcome = 'TERMINAL_FAILURE' + AND reason = 'PACKAGE_TOO_LARGE' + """, String.class, taskId, COMMIT)).containsExactly( + "skills/.curated/cloudflare-deploy"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'RECEIVED' + """, Integer.class, taskId)).isEqualTo(43); + assertThat(jdbc.queryForObject(""" + SELECT count(DISTINCT skill_path) FROM github_sync_package_ingestion + WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(43); + + for (int batch = 0; batch < 6; batch++) { + ingestionTask.validatePending(); + } + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'RECEIVED' + """, Integer.class, taskId)).isZero(); + for (int batch = 0; batch < 11; batch++) { + publicationTask.publishValidated(); + } + List> ingestionStatuses = jdbc.queryForList(""" + SELECT status, last_error_code, count(*) AS packages + FROM github_sync_package_ingestion + WHERE task_id = ? GROUP BY status, last_error_code ORDER BY status, last_error_code + """, taskId); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'VALIDATED' + """, Integer.class, taskId)).as("ingestion statuses=%s", ingestionStatuses).isZero(); + Integer pendingReviews = jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE namespace_id = ? AND origin_type = 'GITHUB_SYNC' AND status = 'PENDING' + """, Integer.class, namespaceId); + Integer readyForReview = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'READY_FOR_REVIEW' + """, Integer.class, taskId); + assertThat(pendingReviews).as("ingestion statuses=%s", ingestionStatuses).isEqualTo(42); + assertThat(pendingReviews).isEqualTo(readyForReview); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE task_id = ? AND status = 'BLOCKED' AND last_error_code = 'SLUG_CONFLICT' + AND target_skill_slug IS NOT NULL + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_skill_binding + WHERE namespace_id = ? AND source_id = ? AND upstream_presence = 'PRESENT' + """, Integer.class, namespaceId, sourceId)).isEqualTo(42); + System.out.printf("Full public repository ingestion: statuses=%s pendingReviews=%d%n", + ingestionStatuses, pendingReviews); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM skill + WHERE namespace_id = ? AND latest_version_id IS NOT NULL + """, Integer.class, namespaceId)).isZero(); + } + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_LIVE_INTERRUPT_INTEROP", matches = "1") + void repositoryWorkerResumesAfterSigtermBetweenReceiptAndAcknowledgement() throws Exception { + String namespaceSlug = "interop-interrupted-" + UUID.randomUUID().toString().substring(0, 8); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Interrupted Worker', 'TEAM') RETURNING id + """, Long.class, namespaceSlug); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 987654350, 'openai', 'skills', 'NAMESPACE_ONLY', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 987654350L); + Path tempRoot = Files.createTempDirectory("skillhub-github-interrupted-"); + try { + ProcessBuilder builder = publicMultiSkillWorkerBuilder(taskId, false); + builder.environment().put("SKILLHUB_INTEROP_WORKER_INTERRUPT_AFTER_RECEIPT", "1"); + builder.environment().put("SKILLHUB_INTEROP_WORKER_TEMP_ROOT", tempRoot.toString()); + Process process = builder.start(); + if (!process.waitFor(180, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Interrupted Go worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Interrupted Go worker: %s", output).isZero(); + assertThat(output).contains("interrupted after Java package receipt", "PASS"); + try (var remaining = Files.list(tempRoot)) { + assertThat(remaining.toList()).isEmpty(); + } + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("LEASED"); + UUID firstIngestionId = jdbc.queryForObject(""" + SELECT id FROM github_sync_package_ingestion WHERE task_id = ? + """, UUID.class, taskId); + String firstPath = jdbc.queryForObject(""" + SELECT skill_path FROM github_sync_package_ingestion WHERE id = ? + """, String.class, firstIngestionId); + assertThat(jdbc.queryForObject(""" + SELECT ingestion_id FROM github_sync_path_result + WHERE task_id = ? AND skill_path = ? AND outcome = 'ACCEPTED' + """, UUID.class, taskId, firstPath)).isEqualTo(firstIngestionId); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission + WHERE task_id = ? AND skill_path = ? + """, Integer.class, taskId, firstPath)).isEqualTo(1); + + jdbc.update(""" + UPDATE github_sync_task + SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? AND status = 'LEASED' + """, taskId); + assertThat(runPublicMultiSkillWorker(taskId, false)).contains("PASS"); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id = ?", + Long.class, taskId)).isEqualTo(2L); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result + WHERE task_id = ? AND outcome = 'ACCEPTED' + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT id FROM github_sync_package_ingestion + WHERE task_id = ? AND skill_path = ? + """, UUID.class, taskId, firstPath)).isEqualTo(firstIngestionId); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission + WHERE task_id = ? AND skill_path = ? + """, Integer.class, taskId, firstPath)).isEqualTo(1); + } finally { + FileSystemUtils.deleteRecursively(tempRoot); + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_INTEROP_IMAGE", matches = ".+") + void serviceContainerResumesPublicRepositoryOnSmallTmpfs() throws Exception { + String token = publicReadTokenForInterop(); + UUID credentialId = createFixtureCredential("Container public read", token); + try { + String namespaceSlug = "interop-container-" + UUID.randomUUID().toString().substring(0, 8); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Container Interop', 'TEAM') RETURNING id + """, Long.class, namespaceSlug); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 1103643573, 'openai', 'skills', 'NAMESPACE_ONLY', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 1103643573L); + String image = System.getenv("SKILLHUB_GITHUB_SYNC_INTEROP_IMAGE"); + GenericContainer first = publicRepositoryServiceContainer(image); + try { + first.start(); + Instant firstDeadline = Instant.now().plusSeconds(120); + while (Instant.now().isBefore(firstDeadline) && jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId) == 0) { + assertThat(first.isRunning()).as("Go container exited before the first package receipt") + .isTrue(); + Thread.sleep(50); + } + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isBetween(1, 42); + } finally { + first.stop(); + } + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, taskId)).isEqualTo("LEASED"); + UUID firstIngestionId = jdbc.queryForObject(""" + SELECT id FROM github_sync_package_ingestion + WHERE task_id = ? ORDER BY created_at, id LIMIT 1 + """, UUID.class, taskId); + String firstPath = jdbc.queryForObject(""" + SELECT skill_path FROM github_sync_package_ingestion WHERE id = ? + """, String.class, firstIngestionId); + jdbc.update(""" + UPDATE github_sync_task + SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? AND status = 'LEASED' + """, taskId); + try (GenericContainer second = publicRepositoryServiceContainer(image)) { + second.start(); + Instant deadline = Instant.now().plusSeconds(600); + int recoveryAttempts = 0; + String status = ""; + while (Instant.now().isBefore(deadline)) { + Map state = jdbc.queryForMap(""" + SELECT status, last_error_code FROM github_sync_task WHERE id = ? + """, taskId); + status = (String) state.get("status"); + if ("PARTIAL".equals(status)) { + break; + } + if ("RETRY_WAIT".equals(status)) { + assertThat(state.get("last_error_code")).isEqualTo("BUDGET"); + assertThat(recoveryAttempts++).isLessThan(2); + jdbc.update(""" + UPDATE github_sync_task + SET retry_at = CURRENT_TIMESTAMP - INTERVAL '1 second', + due_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? AND status = 'RETRY_WAIT' + """, taskId); + } + assertThat(second.isRunning()).as("Go container exited before completion") + .isTrue(); + Thread.sleep(200); + } + assertThat(status).isEqualTo("PARTIAL"); + var processes = second.execInContainer("ps", "-o", "pid,ppid,stat,comm"); + assertThat(processes.getExitCode()).isZero(); + assertThat(processes.getStdout()).contains("tini"); + long zombies = processes.getStdout().lines().skip(1) + .filter(line -> line.trim().matches("[0-9]+\\s+[0-9]+\\s+Z\\S*\\s+.*")) + .count(); + assertThat(zombies).as("orphaned Git subprocesses").isZero(); + } + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result + WHERE task_id = ? AND outcome = 'ACCEPTED' + """, Integer.class, taskId)).isEqualTo(43); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(43); + assertThat(jdbc.queryForObject(""" + SELECT id FROM github_sync_package_ingestion + WHERE task_id = ? AND skill_path = ? + """, UUID.class, taskId, firstPath)).isEqualTo(firstIngestionId); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission + WHERE task_id = ? AND skill_path = ? + """, Integer.class, taskId, firstPath)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT lease_generation FROM github_sync_task WHERE id = ?", + Long.class, taskId)).isGreaterThanOrEqualTo(2L); + } finally { + systemCredentials.setEnabled(credentialId, false); + } + } + + private GenericContainer publicRepositoryServiceContainer(String image) { + GenericContainer container = new GenericContainer<>(DockerImageName.parse(image)) + .withNetworkMode("host") + .withTmpFs(Map.of("/var/lib/skillhub-github-sync", + "rw,size=100663296,mode=1777")) + .withEnv("SKILLHUB_GITHUB_SYNC_JAVA_URL", baseUrl() + "/internal/github-sync/v1") + .withEnv("SKILLHUB_GITHUB_SYNC_SERVICE_TOKEN", SERVICE_TOKEN) + .withEnv("SKILLHUB_GITHUB_SYNC_HEALTH_ADDR", "127.0.0.1:8090") + .withEnv("SKILLHUB_GITHUB_SYNC_POLL_INTERVAL", "1s") + .withEnv("SKILLHUB_GITHUB_SYNC_MAX_WORKSPACE_BYTES", "67108864") + .withEnv("SKILLHUB_GITHUB_SYNC_TOTAL_WORKSPACE_BYTES", "67108864") + .withEnv("SKILLHUB_GITHUB_SYNC_MIN_FREE_BYTES", "16777216"); + if (S3_MODE) { + container.withEnv("SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS", + URI.create(s3Endpoint).getAuthority()); + } + for (String proxy : List.of("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY")) { + String value = System.getenv(proxy); + if (value != null && !value.isBlank()) { + container.withEnv(proxy, value); + } + } + return container; + } + + private String publicReadTokenForInterop() throws Exception { + String token = System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_TOKEN"); + if (token != null && !token.isBlank()) { + return token.trim(); + } + Process process = new ProcessBuilder("gh", "auth", "token").start(); + if (!process.waitFor(10, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Public repository container test needs GitHub read access"); + } + if (process.exitValue() != 0) { + throw new IllegalStateException("Public repository container test needs GitHub read access"); + } + token = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + if (token.isBlank()) { + throw new IllegalStateException("Public repository container test has no GitHub read token"); + } + return token; + } + + private String runPublicMultiSkillWorker(UUID taskId, boolean allTree) throws Exception { + Process process = publicMultiSkillWorkerBuilder(taskId, allTree).start(); + if (!process.waitFor(allTree ? 600 : 180, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go public multi-Skill worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go public multi-Skill worker: %s", output).isZero(); + return output; + } + + private ProcessBuilder publicMultiSkillWorkerBuilder(UUID taskId, boolean allTree) { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropProcessRepositoryPublicMultiSkill$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_WORKER_PUBLIC_MULTI", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + if ("1".equals(System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_WORKER_REAL_TREE"))) { + builder.environment().put("SKILLHUB_INTEROP_WORKER_REAL_TREE", "1"); + } + if (allTree) { + builder.environment().put("SKILLHUB_INTEROP_WORKER_ALL_TREE", "1"); + } + if (S3_MODE) { + builder.environment().put("SKILLHUB_INTEROP_ALLOWED_S3_HOST", + URI.create(s3Endpoint).getAuthority()); + } + return builder; + } + + @Test + void repositoryWorkerReleasesAdmissionAndDefersWhenPackageCannotFit() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 987654330, 'openai', 'skills', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 987654330, "c".repeat(40)); + + assertThat(runCapacityWorker(taskId)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject(""" + SELECT last_error_code FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("CAPACITY"); + assertThat(jdbc.queryForObject(""" + SELECT retry_at FROM github_sync_task WHERE id = ? + """, Timestamp.class, taskId).toInstant()).isAfter(Instant.now().plus(Duration.ofHours(23))); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission + WHERE task_id = ? AND status = 'RELEASED' + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isZero(); + } + + private String runCapacityWorker(UUID taskId) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropProcessRepositoryCapacity$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_WORKER_CAPACITY", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go capacity worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go capacity worker: %s", output).isZero(); + return output; + } + + @Test + void highWatermarkMakesGoCheckHeadWithoutAcquiringPackages() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + Map previousPolicy = jdbc.queryForMap(""" + SELECT global_high, global_low, namespace_high, namespace_low, global_paused + FROM github_sync_admission_policy WHERE id = TRUE + """); + UUID blockerRun = UUID.randomUUID(); + UUID blockerTask = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, blockerRun, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, snapshot_json, + lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', 987654331), 1, ?, + CURRENT_TIMESTAMP + INTERVAL '5 minutes', 'capacity-blocker') + """, blockerTask, blockerRun, namespaceId, + "repository:987654331", "b".repeat(64)); + UUID blockerAdmission = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, + lease_generation) + VALUES (?, ?, ?, 987654331, ?, 'skills/blocker', 1) + """, blockerAdmission, blockerTask, namespaceId, "c".repeat(40)); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 987654332, 'old-owner', 'old-name', 'PUBLIC', + CURRENT_TIMESTAMP + INTERVAL '1 day') RETURNING id + """, Long.class, namespaceId); + UUID taskId = createRepositoryTask(namespaceId, sourceId, 987654332, "c".repeat(40)); + jdbc.update(""" + UPDATE github_sync_task + SET snapshot_json = snapshot_json || jsonb_build_object( + 'owner', 'old-owner', 'name', 'old-name') + WHERE id = ? + """, taskId); + + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1, global_low = 0, namespace_high = 100, namespace_low = 50, + global_paused = FALSE WHERE id = TRUE + """); + try { + assertThat(runHeadOnlyWorker(taskId)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT mode FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("HEAD_ONLY"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT last_observed_commit FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isEqualTo("c".repeat(40)); + assertThat(jdbc.queryForObject(""" + SELECT last_fully_processed_commit FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isNull(); + assertThat(jdbc.queryForObject(""" + SELECT owner_login || '/' || repository_name + FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isEqualTo("new-owner/new-name"); + assertThat(jdbc.queryForObject(""" + SELECT concat(snapshot_json ->> 'owner', '/', snapshot_json ->> 'name') + FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("new-owner/new-name"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE task_id = ? + """, Integer.class, taskId)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_admission WHERE id = ? + """, String.class, blockerAdmission)).isEqualTo("ACTIVE"); + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE id = ?", blockerAdmission); + assertThat(sourceTasks.enqueueDue(Instant.now(), 10)).isPositive(); + UUID resumedTask = jdbc.queryForObject("SELECT id FROM github_sync_task WHERE source_id = ? AND status = 'QUEUED'", + UUID.class, sourceId); + assertThat(jdbc.queryForObject("SELECT mode FROM github_sync_task WHERE id = ?", + String.class, resumedTask)).isEqualTo("FULL"); + assertThat(jdbc.queryForObject("SELECT fixed_commit_sha FROM github_sync_task WHERE id = ?", + String.class, resumedTask)).isNull(); + // The latest HEAD is obtained by Go; the old high-watermark observation must not pin this task. + String freshCommit = "d".repeat(40); + Instant resumedAt = Instant.now(); + String proof = "e".repeat(64); + var resumedLease = syncTasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, + "resumed-head-worker", proof, resumedAt, resumedAt.plusSeconds(300), + java.util.List.of("platform-pause/v1")).orElseThrow(); + assertThat(resumedLease.taskId()).isEqualTo(resumedTask); + var admitted = syncTasks.admitPackage(resumedTask, resumedLease.leaseGeneration(), proof, + freshCommit, "skills/fresh", resumedAt); + assertThat(admitted.status()).isEqualTo(GithubPackageAdmissionResult.Status.ADMITTED); + assertThat(jdbc.queryForObject("SELECT fixed_commit_sha FROM github_sync_task WHERE id = ?", + String.class, resumedTask)).isEqualTo(freshCommit); + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE task_id = ?", resumedTask); + jdbc.update("UPDATE github_sync_task SET status = 'FAILED', lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL WHERE id = ?", resumedTask); + } finally { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = ?, global_low = ?, namespace_high = ?, namespace_low = ?, + global_paused = ? WHERE id = TRUE + """, previousPolicy.get("global_high"), previousPolicy.get("global_low"), + previousPolicy.get("namespace_high"), previousPolicy.get("namespace_low"), + previousPolicy.get("global_paused")); + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE id = ?", + blockerAdmission); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'FAILED', lease_token_hash = NULL, lease_expires_at = NULL, + leased_by = NULL WHERE id = ? + """, blockerTask); + jdbc.update("UPDATE github_source_binding SET status = 'PAUSED' WHERE id = ?", sourceId); + } + } + + @Test + void renamedCandidateWithoutSkillStaysUnboundAfterGoValidation() throws Exception { + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type) + VALUES (?, 'Candidate Interop', 'TEAM') RETURNING id + """, Long.class, "candidate-interop-" + UUID.randomUUID().toString().substring(0, 8)); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + long candidateId = jdbc.queryForObject(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, next_validation_at) + VALUES (?, 987654333, 'old-owner', 'old-name', CURRENT_TIMESTAMP - INTERVAL '1 second') + RETURNING id + """, Long.class, namespaceId); + assertThat(candidateTasks.enqueueDue(Instant.now(), 10)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT validation_task_id FROM github_discovery_candidate WHERE id = ? + """, UUID.class, candidateId); + + assertThat(runEmptyCandidateWorker(taskId)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT owner_login || '/' || repository_name + FROM github_discovery_candidate WHERE id = ? + """, String.class, candidateId)).isEqualTo("new-owner/new-name"); + assertThat(jdbc.queryForObject(""" + SELECT concat(snapshot_json ->> 'owner', '/', snapshot_json ->> 'name') + FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("new-owner/new-name"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + WHERE namespace_id = ? AND repository_id = 987654333 + """, Integer.class, namespaceId)).isZero(); + assertThat(candidateTasks.reconcileCompleted(Instant.now(), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT status || '/' || last_error_code + FROM github_discovery_candidate WHERE id = ? + """, String.class, candidateId)).isEqualTo("REJECTED/NO_VALID_SKILL"); + assertThat(jdbc.queryForObject(""" + SELECT next_validation_at FROM github_discovery_candidate WHERE id = ? + """, Timestamp.class, candidateId).toInstant()).isAfter(Instant.now().plus(Duration.ofDays(6))); + } + + private String runEmptyCandidateWorker(UUID taskId) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropProcessRepositoryRenamedCandidateWithoutSkill$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_WORKER_EMPTY_CANDIDATE", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go candidate worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go candidate worker: %s", output).isZero(); + return output; + } + + private String runHeadOnlyWorker(UUID taskId) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropProcessRepositoryHeadOnly$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_WORKER_HEAD_ONLY", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go HEAD_ONLY worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go HEAD_ONLY worker: %s", output).isZero(); + return output; + } + + @Test + void separateGoProcessesResumeDiscoveryCheckpointAfterLeaseExpiry() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'DISCOVERY') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, dedupe_key, priority, snapshot_json) + VALUES (?, ?, ?, 'DISCOVERY', ?, 100, + jsonb_build_object('entryType', 'CURATED_OWNER', 'owner', 'interop-owner', + 'ruleVersion', 1, 'candidateBudget', 10)) + """, taskId, runId, namespaceId, "discovery-interop:" + taskId); + + int javaPort; + try (JavaApplicationProcess first = startJavaApplication(3_600_000)) { + javaPort = first.port(); + assertThat(runDiscoveryClient(taskId, "report", null, null, + "http://127.0.0.1:" + javaPort)).contains("REPORTED"); + // The restart exercise must not make this discovery lead due for another test's + // repository claim when the second application runs its startup scheduler tick. + assertThat(jdbc.update(""" + UPDATE github_discovery_candidate + SET next_validation_at = CURRENT_TIMESTAMP + INTERVAL '1 day' + WHERE namespace_id = ? AND repository_id = 424242 + """, namespaceId)).isEqualTo(1); + } + assertThat(jdbc.queryForObject(""" + SELECT discovery_checkpoint FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("interop-discovery-checkpoint"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND repository_id = 424242 + """, Integer.class, namespaceId)).isEqualTo(1); + jdbc.update(""" + UPDATE github_sync_task SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + + try (JavaApplicationProcess restarted = startJavaApplication(3_600_000, javaPort)) { + assertThat(runDiscoveryClient(taskId, "resume", null, null, + "http://127.0.0.1:" + restarted.port())).contains("COMPLETED"); + } + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT lease_generation FROM github_sync_task WHERE id = ? + """, Long.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + } + + @Test + void searchPreviewFlowsThroughJavaGoJavaWithoutCreatingSources() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + int candidatesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate WHERE namespace_id = ? + """, Integer.class, namespaceId); + int sourcesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE namespace_id = ? + """, Integer.class, namespaceId); + var request = new GithubSearchRuleRequest("Public skills", List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"), + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE")), null); + + UUID successId = searchPreviews.queue(request, "interop-admin").id(); + assertThat(runSearchPreviewWorker(successId, "success")).contains("PASS"); + var completed = searchPreviews.get(successId); + assertThat(completed.status()).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(completed.result().path("selectors").size()).isEqualTo(2); + assertThat(completed.result().path("selectors").get(0).path("kind").asText()) + .isEqualTo("KEYWORD"); + assertThat(completed.result().path("selectors").get(0).path("totalCount").asInt()) + .isEqualTo(1001); + assertThat(completed.result().path("selectors").get(0).path("incompleteResults").asBoolean()) + .isTrue(); + assertThat(completed.result().path("selectors").get(0).path("limitReached").asBoolean()) + .isTrue(); + assertThat(completed.result().path("selectors").get(0).path("samples").size()).isEqualTo(5); + + UUID limitedId = searchPreviews.queue(request, "interop-admin").id(); + assertThat(runSearchPreviewWorker(limitedId, "limit")).contains("PASS"); + var deferred = searchPreviews.get(limitedId); + assertThat(deferred.status()).isEqualTo("RETRY_WAIT"); + assertThat(deferred.errorCode()).isEqualTo("GITHUB_RATE_LIMITED"); + assertThat(deferred.retryAt()).isAfter(Instant.now()); + assertThat(deferred.result()).isNull(); + + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch + WHERE task_id IN (?, ?) + """, Integer.class, successId, limitedId)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate WHERE namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(candidatesBefore); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(sourcesBefore); + } + + @Test + @EnabledIfEnvironmentVariable(named = "SKILLHUB_GITHUB_SYNC_LIVE_SEARCH", matches = "1") + void publicGithubTwoSelectorPreviewCompletesWithoutCreatingSources() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + int candidatesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate WHERE namespace_id = ? + """, Integer.class, namespaceId); + int sourcesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE namespace_id = ? + """, Integer.class, namespaceId); + var request = new GithubSearchRuleRequest("Public skills", List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"), + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE")), null); + UUID taskId = searchPreviews.queue(request, "interop-admin").id(); + + assertThat(runSearchPreviewWorker(taskId, "live", null, "")).contains("PASS"); + var preview = searchPreviews.get(taskId); + assertThat(preview.status()).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(preview.result().path("selectors").size()).isEqualTo(2); + assertThat(preview.result().path("selectors").get(0).path("kind").asText()) + .isEqualTo("KEYWORD"); + assertThat(preview.result().path("selectors").get(1).path("kind").asText()) + .isEqualTo("TOPIC"); + assertThat(preview.result().path("selectors").get(0).path("totalCount").asInt()) + .isGreaterThanOrEqualTo(0); + assertThat(preview.result().path("selectors").get(1).path("totalCount").asInt()) + .isGreaterThanOrEqualTo(0); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch WHERE task_id = ? + """, Integer.class, taskId)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate WHERE namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(candidatesBefore); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(sourcesBefore); + } + + @Test + void curatedPublicRepositoryBecomesSourceAndQueuesRepositorySync() throws Exception { + var entry = discoveryEntries.createCuratedRepository("openai", "skills"); + assertThat(discoveryEntries.enqueueDue(Instant.now(), 10)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE dedupe_key = ? + """, UUID.class, "discovery:" + entry.id()); + assertThat(runCuratedRepositoryWorker(taskId)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + + long sourceId = jdbc.queryForObject(""" + SELECT source.id FROM github_source_binding source + JOIN github_source_origin origin ON origin.source_id = source.id + WHERE origin.origin_type = 'CURATED_REPOSITORY' AND origin.origin_key = ? + """, Long.class, entry.id().toString()); + assertThat(jdbc.queryForObject(""" + SELECT repository_id FROM github_source_binding WHERE id = ? + """, Long.class, sourceId)).isPositive(); + Map previousPolicy = jdbc.queryForMap(""" + SELECT global_high, global_low, namespace_high, namespace_low + FROM github_sync_admission_policy WHERE id = TRUE + """); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1000, global_low = 500, namespace_high = 100, namespace_low = 50 + WHERE id = TRUE + """); + try { + assertThat(sourceTasks.enqueueDue(Instant.now().plusSeconds(1), 10)) + .isGreaterThanOrEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE source_id = ? AND kind = 'REPOSITORY_SYNC' AND status = 'QUEUED' + """, Integer.class, sourceId)).isEqualTo(1); + } finally { + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = ?, global_low = ?, namespace_high = ?, namespace_low = ? + WHERE id = TRUE + """, previousPolicy.get("global_high"), previousPolicy.get("global_low"), + previousPolicy.get("namespace_high"), previousPolicy.get("namespace_low")); + jdbc.update(""" + UPDATE github_sync_task SET status = 'FAILED' + WHERE source_id = ? AND kind = 'REPOSITORY_SYNC' AND status = 'QUEUED' + """, sourceId); + } + } + + @Test + void versionedSearchRuleCreatesDeduplicatedLeadsWithoutSources() throws Exception { + verifySearchRuleDiscovery(false); + } + + @Test + void incompleteSearchSplitsFixedWindowBeforeCompleting() throws Exception { + verifySearchRuleDiscovery(true); + } + + @Test + void shortenedSearchPageResumesThroughJavaAndDeduplicatesRepositoryIds() throws Exception { + var request = new GithubSearchRuleRequest("Interop changing page", List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"), + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE")), null); + var audit = new AuditRequestContext("127.0.0.1", "interop-test"); + var entry = discoveryAdmin.createSearchRule(request, "github-sync-publisher", audit); + discoveryAdmin.setEnabled(entry.id(), true, "github-sync-publisher", audit); + assertThat(discoveryEntries.enqueueDue(Instant.now(), 10)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE dedupe_key = ? + """, UUID.class, "discovery:" + entry.id()); + jdbc.update(""" + UPDATE github_sync_task + SET snapshot_json = jsonb_set(snapshot_json, '{candidateBudget}', '3'::jsonb) + WHERE id = ? + """, taskId); + + assertThat(runChangingSearchPageWorker(taskId, "initial")).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("RETRY_WAIT"); + String checkpoint = jdbc.queryForObject(""" + SELECT discovery_checkpoint FROM github_sync_task WHERE id = ? + """, String.class, taskId); + assertThat(objectMapper.readTree(checkpoint).path("offset").asInt()).isEqualTo(3); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(3); + + jdbc.update(""" + UPDATE github_sync_task + SET retry_at = CURRENT_TIMESTAMP - INTERVAL '1 second', + due_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, taskId); + assertThat(runChangingSearchPageWorker(taskId, "shrunk")).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT lease_generation FROM github_sync_task WHERE id = ? + """, Long.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForList(""" + SELECT repository_id FROM github_discovery_task_hit + WHERE task_id = ? ORDER BY repository_id + """, Long.class, taskId)).containsExactly(9400001L, 9400002L, 9400003L, 9400006L); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit + WHERE task_id = ? AND repository_id = 9400002 + """, Integer.class, taskId)).isEqualTo(1); + assertThat(objectMapper.readTree(jdbc.queryForObject(""" + SELECT discovery_checkpoint FROM github_sync_task WHERE id = ? + """, String.class, taskId)).path("revisit").asInt()).isEqualTo(1); + } + + private String runChangingSearchPageWorker(UUID taskId, String phase) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropChangingSearchPage$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_SEARCH_PAGE_PHASE", phase); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go changing search page worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go changing search page worker: %s", output).isZero(); + return output; + } + + private void verifySearchRuleDiscovery(boolean saturated) throws Exception { + var request = new GithubSearchRuleRequest( + saturated ? "Interop saturated search" : "Interop search", List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"), + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE")), null); + var audit = new AuditRequestContext("127.0.0.1", "interop-test"); + var entry = discoveryAdmin.createSearchRule(request, "github-sync-publisher", audit); + var enabled = discoveryAdmin.setEnabled(entry.id(), true, "github-sync-publisher", audit); + assertThat(enabled.ruleVersion()).isEqualTo(2); + assertThat(discoveryEntries.enqueueDue(Instant.now(), 10)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE dedupe_key = ? + """, UUID.class, "discovery:" + entry.id()); + int sourcesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + """, Integer.class); + + assertThat(runSearchRuleWorker(taskId, saturated)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(saturated ? 5 : 3); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate_origin + WHERE origin_type = 'SEARCH_RULE' AND origin_key = ? + """, Integer.class, entry.id().toString())).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + """, Integer.class)).isEqualTo(sourcesBefore); + } + + private String runSearchRuleWorker(UUID taskId, boolean saturated) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropSearchRuleDiscovery$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_SEARCH_RULE", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + builder.environment().put("SKILLHUB_INTEROP_SEARCH_SATURATED", saturated ? "1" : "0"); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go search rule worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go search rule worker: %s", output).isZero(); + return output; + } + + @Test + void curatedUserAndOrganizationDiscoverPublicLeadsWithoutPublishing() throws Exception { + verifyCuratedOwnerDiscovery("fixture-user", "User", 987654324L); + verifyCuratedOwnerDiscovery("fixture-org", "Organization", 987654325L); + } + + private void verifyCuratedOwnerDiscovery(String owner, String ownerType, + long repositoryId) throws Exception { + var audit = new AuditRequestContext("127.0.0.1", "interop-test"); + var entry = discoveryAdmin.createCuratedOwner(new GithubCuratedOwnerRequest(owner, null), + "github-sync-publisher", audit); + assertThat(discoveryEntries.enqueueDue(Instant.now(), 10)).isEqualTo(1); + UUID taskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE dedupe_key = ? + """, UUID.class, "discovery:" + entry.id()); + int sourcesBefore = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + """, Integer.class); + + assertThat(runCuratedOwnerWorker(taskId, owner, ownerType)).contains("PASS"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("ACQUISITION_COMPLETE"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit + WHERE task_id = ? AND repository_id = ? + """, Integer.class, taskId, repositoryId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate_origin origin + JOIN github_discovery_candidate candidate ON candidate.id = origin.candidate_id + WHERE origin.origin_type = 'CURATED_OWNER' AND origin.origin_key = ? + AND candidate.repository_id = ? + """, Integer.class, owner, repositoryId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + """, Integer.class)).isEqualTo(sourcesBefore); + } + + private String runCuratedOwnerWorker(UUID taskId, String owner, String ownerType) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropCuratedOwnerDiscovery$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_OWNER_TYPE", ownerType); + builder.environment().put("SKILLHUB_INTEROP_OWNER_LOGIN", owner); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go curated owner worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go curated owner worker: %s", output).isZero(); + return output; + } + + private String runCuratedRepositoryWorker(UUID taskId) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropCuratedPublicRepository$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_CURATED_REPOSITORY", "1"); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + builder.environment().put("SKILLHUB_INTEROP_CURATED_MODE", + "1".equals(System.getenv("SKILLHUB_GITHUB_SYNC_LIVE_CURATED")) ? "live" : "fixture"); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go curated repository worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go curated repository worker: %s", output).isZero(); + return output; + } + + @Test + void credentialPoolFallsBackByQuotaResourceAcrossJavaAndGo() throws Exception { + String depletedToken = "fixture-public-reader-search-exhausted"; + String healthyToken = "fixture-public-reader-search-healthy"; + UUID depletedId = createFixtureCredential("Search exhausted", depletedToken); + UUID healthyId = createFixtureCredential("Search healthy", healthyToken); + Instant now = Instant.now(); + systemCredentials.recordQuota(depletedId, "SEARCH", 0, now.plusSeconds(3600), now); + try { + var request = new GithubSearchRuleRequest("Public skills", List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"), + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE")), null); + UUID healthyPreviewId = searchPreviews.queue(request, "interop-admin").id(); + assertThat(runSearchPreviewWorker(healthyPreviewId, "success", healthyId, healthyToken)) + .contains("PASS"); + assertThat(searchPreviews.get(healthyPreviewId).status()).isEqualTo("ACQUISITION_COMPLETE"); + + assertThat(systemCredentials.setEnabled(healthyId, false)).isTrue(); + UUID blockedPreviewId = searchPreviews.queue(request, "interop-admin").id(); + var unavailable = HttpClient.newHttpClient().send(HttpRequest.newBuilder( + URI.create(baseUrl() + "/internal/github-sync/v1/tasks/claim")) + .header("Authorization", "Bearer " + SERVICE_TOKEN) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"protocolMajor":1,"kind":"DISCOVERY","workerId":"credential-wait-worker", "capabilities":["platform-pause/v1"]} + """)).build(), HttpResponse.BodyHandlers.ofString()); + assertThat(unavailable.statusCode()).isEqualTo(204); + assertThat(unavailable.body()).isEmpty(); + assertThat(searchPreviews.get(blockedPreviewId).status()).isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject("SELECT official_pause_reason FROM github_sync_task WHERE id = ?", + String.class, blockedPreviewId)).isEqualTo("CREDENTIAL_UNAVAILABLE"); + jdbc.update("UPDATE github_sync_task SET status = 'FAILED' WHERE id = ?", blockedPreviewId); + + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + UUID runId = UUID.randomUUID(); + UUID curatedTaskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'DISCOVERY') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, dedupe_key, priority, snapshot_json) + VALUES (?, ?, ?, 'DISCOVERY', ?, 100, + jsonb_build_object('entryType', 'CURATED_OWNER', 'owner', 'interop-owner', + 'ruleVersion', 1, 'candidateBudget', 10)) + """, curatedTaskId, runId, namespaceId, "credential-core:" + curatedTaskId); + assertThat(runDiscoveryClient(curatedTaskId, "report", depletedId, depletedToken)) + .contains("REPORTED"); + jdbc.update(""" + UPDATE github_sync_task SET lease_expires_at = CURRENT_TIMESTAMP - INTERVAL '1 second' + WHERE id = ? + """, curatedTaskId); + assertThat(runDiscoveryClient(curatedTaskId, "resume", depletedId, depletedToken)) + .contains("COMPLETED"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, curatedTaskId)).isEqualTo("ACQUISITION_COMPLETE"); + String snapshot = jdbc.queryForObject(""" + SELECT snapshot_json::text FROM github_sync_task WHERE id = ? + """, String.class, curatedTaskId); + assertThat(snapshot).doesNotContain(depletedToken, healthyToken); + } finally { + // Remove only this test's credentials; disabled configured rows intentionally prevent anonymous claims. + jdbc.update("UPDATE github_sync_task SET credential_id = NULL WHERE credential_id IN (?, ?)", depletedId, healthyId); + jdbc.update("DELETE FROM github_system_credential WHERE id IN (?, ?)", depletedId, healthyId); + } + } + + private UUID createFixtureCredential(String label, String token) { + UUID id = UUID.randomUUID(); + var encrypted = credentialCipher.encrypt(id, token); + Instant now = Instant.now(); + systemCredentials.create(new GithubSystemCredential(id, label, + id.toString().replace("-", "").substring(0, 16), encrypted.keyVersion(), + encrypted.nonce(), encrypted.ciphertext(), "ACTIVE", null, null, now, now)); + return id; + } + + private String runSearchPreviewWorker(UUID taskId, String phase) throws Exception { + return runSearchPreviewWorker(taskId, phase, null, null); + } + + private String runSearchPreviewWorker(UUID taskId, String phase, + UUID credentialId, String token) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropSearchPreview$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_PREVIEW_PHASE", phase); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + if (credentialId != null) { + builder.environment().put("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID", credentialId.toString()); + builder.environment().put("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN", token); + } else if (token != null) { + builder.environment().put("SKILLHUB_INTEROP_EXPECT_ANONYMOUS", "1"); + } + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go search preview worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go search preview worker: %s", output).isZero(); + return output; + } + + private String runDiscoveryClient(UUID taskId, String phase) throws Exception { + return runDiscoveryClient(taskId, phase, null, null); + } + + private String runDiscoveryClient(UUID taskId, String phase, + UUID credentialId, String token) throws Exception { + return runDiscoveryClient(taskId, phase, credentialId, token, baseUrl()); + } + + private String runDiscoveryClient(UUID taskId, String phase, + UUID credentialId, String token, + String javaBaseUrl) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "run", + "./internal/controlplane/testdata/discoveryinterop"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_DISCOVERY_PHASE", phase); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", + javaBaseUrl + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + if (credentialId != null) { + builder.environment().put("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_ID", credentialId.toString()); + builder.environment().put("SKILLHUB_INTEROP_EXPECT_CREDENTIAL_TOKEN", token); + } + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go discovery HTTP client timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go discovery HTTP client: %s", output).isZero(); + return output; + } + + private void assertSingleReceivedPackage(UUID taskId, String ingestionId) { + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + WHERE admission.task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE id = ? AND task_id = ? + """, Integer.class, UUID.fromString(ingestionId), taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + } + + private void runRepositoryWorker(UUID taskId, String phase) throws Exception { + ProcessBuilder builder = new ProcessBuilder("go", "test", "./internal/worker", + "-run", "^TestInteropProcessRepositoryRecovery$", "-count=1", "-v"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_WORKER_PHASE", phase); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", baseUrl() + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + Process process = builder.start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go repository worker timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Go repository worker: %s", output).isZero(); + assertThat(output).contains("PASS"); + } + + private String runGoClient(UUID taskId, UUID oversizedTaskId) throws Exception { + return runGoClient(taskId, oversizedTaskId, COMMIT, + Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE"))); + } + + private String runGoClient(UUID taskId, int javaPort) throws Exception { + return runGoClient(taskId, null, COMMIT, + Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE")), + null, null, "http://127.0.0.1:" + javaPort); + } + + private void claimInSeparateJvm(String ingestionId, Class claimant) throws Exception { + ProcessBuilder builder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), + claimant.getName(), ingestionId); + builder.redirectErrorStream(true); + builder.environment().put("SKILLHUB_RESTART_JDBC_URL", POSTGRES.getJdbcUrl()); + builder.environment().put("SKILLHUB_RESTART_JDBC_USER", POSTGRES.getUsername()); + builder.environment().put("SKILLHUB_RESTART_JDBC_PASSWORD", POSTGRES.getPassword()); + Process process = builder.start(); + if (!process.waitFor(45, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Separate Java claimant timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Separate Java claimant: %s", output).isZero(); + assertThat(output).contains("CLAIMED"); + } + + private String runGoClient(UUID taskId, UUID oversizedTaskId, String commit, Path archivePath) + throws Exception { + return runGoClient(taskId, oversizedTaskId, commit, archivePath, null, null); + } + + private String runGoClient(UUID taskId, UUID oversizedTaskId, String commit, Path archivePath, + String mode, String expectedIngestionId) throws Exception { + return runGoClient(taskId, oversizedTaskId, commit, archivePath, mode, + expectedIngestionId, baseUrl()); + } + + private String runGoClient(UUID taskId, UUID oversizedTaskId, String commit, Path archivePath, + String mode, String expectedIngestionId, String javaBaseUrl) throws Exception { + Process process = goClientBuilder(taskId, oversizedTaskId, commit, archivePath, + mode, expectedIngestionId, javaBaseUrl).start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("Go HTTP interoperability client timed out"); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + assertThat(process.exitValue()).as("Go HTTP interoperability client: %s", output).isZero(); + return output; + } + + private ProcessBuilder goClientBuilder(UUID taskId, UUID oversizedTaskId, String commit, + Path archivePath, String mode, String expectedIngestionId, String javaBaseUrl) { + ProcessBuilder builder = new ProcessBuilder("go", "run", + "./internal/controlplane/testdata/httpinterop"); + builder.directory(findGoModule().toFile()).redirectErrorStream(true); + builder.environment().put("SKILLHUB_INTEROP_JAVA_URL", + javaBaseUrl + "/internal/github-sync/v1"); + builder.environment().put("SKILLHUB_INTEROP_SERVICE_TOKEN", SERVICE_TOKEN); + builder.environment().put("SKILLHUB_INTEROP_TASK_ID", taskId.toString()); + builder.environment().put("SKILLHUB_INTEROP_COMMIT_SHA", commit); + builder.environment().put("SKILLHUB_INTEROP_SKILL_PATH", SKILL_PATH); + builder.environment().put("SKILLHUB_INTEROP_ARCHIVE_PATH", archivePath.toAbsolutePath().toString()); + if (mode != null) { + builder.environment().put("SKILLHUB_INTEROP_MODE", mode); + } + if (expectedIngestionId != null) { + builder.environment().put("SKILLHUB_INTEROP_EXPECT_INGESTION_ID", expectedIngestionId); + } + if (S3_MODE) { + builder.environment().put("SKILLHUB_INTEROP_UPLOAD_MODE", "S3_PRESIGNED_PUT"); + builder.environment().put("SKILLHUB_INTEROP_ALLOWED_S3_HOST", + URI.create(s3Endpoint).getAuthority()); + if (oversizedTaskId != null) { + builder.environment().put("SKILLHUB_INTEROP_OVERSIZED_TASK_ID", oversizedTaskId.toString()); + } + } + return builder; + } + + private UUID createRepositoryTask(long namespaceId, long sourceId, long repositoryId) { + return createRepositoryTask(namespaceId, sourceId, repositoryId, COMMIT); + } + + private UUID createRepositoryTask(long namespaceId, long sourceId, long repositoryId, + String commit) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, priority, + snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, 100, + jsonb_build_object('repositoryId', ?, 'owner', 'openai', 'name', 'skills'), ?) + """, taskId, runId, namespaceId, sourceId, "repository:" + repositoryId, + repositoryId, commit); + return taskId; + } + + private Path revisionArchive(String marker) throws IOException { + return revisionArchive(marker, null); + } + + private Path archiveWithNestedSkill(String marker) throws IOException { + Path revision = revisionArchive(marker); + Path invalid = Files.createTempFile("skillhub-github-invalid-", ".zip"); + try (ZipInputStream input = new ZipInputStream(Files.newInputStream(revision)); + ZipOutputStream output = new ZipOutputStream(Files.newOutputStream(invalid))) { + ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + output.putNextEntry(new ZipEntry(entry.getName())); + input.transferTo(output); + output.closeEntry(); + input.closeEntry(); + } + output.putNextEntry(new ZipEntry("nested/SKILL.md")); + output.write(("# " + marker + "\n").getBytes(StandardCharsets.UTF_8)); + output.closeEntry(); + } catch (IOException | RuntimeException exception) { + Files.deleteIfExists(invalid); + throw exception; + } finally { + Files.deleteIfExists(revision); + } + return invalid; + } + + private Path revisionArchive(String marker, String licenseMarker) throws IOException { + Path variant = Files.createTempFile("skillhub-github-revision-", ".zip"); + boolean changed = false; + boolean licenseChanged = false; + try (InputStream input = Files.newInputStream( + Path.of(System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE"))); + ZipInputStream zipInput = new ZipInputStream(input); + OutputStream output = Files.newOutputStream(variant); + ZipOutputStream zipOutput = new ZipOutputStream(output)) { + ZipEntry entry; + while ((entry = zipInput.getNextEntry()) != null) { + zipOutput.putNextEntry(new ZipEntry(entry.getName())); + byte[] contents = zipInput.readAllBytes(); + zipOutput.write(contents); + if (entry.getName().equals("SKILL.md") || entry.getName().endsWith("/SKILL.md")) { + zipOutput.write(("\n\n") + .getBytes(StandardCharsets.UTF_8)); + changed = true; + } + if (licenseMarker != null && (entry.getName().equals("LICENSE.txt") + || entry.getName().endsWith("/LICENSE.txt"))) { + zipOutput.write(("\nLicense evidence revision " + licenseMarker + "\n") + .getBytes(StandardCharsets.UTF_8)); + licenseChanged = true; + } + zipOutput.closeEntry(); + zipInput.closeEntry(); + } + } + if (!changed || (licenseMarker != null && !licenseChanged)) { + Files.deleteIfExists(variant); + throw new IllegalStateException("Go archive has no SKILL.md for revision fixture"); + } + return variant; + } + + private Path findGoModule() { + for (Path current = Path.of("").toAbsolutePath(); current != null; current = current.getParent()) { + Path module = current.resolve("github-sync-go"); + if (Files.isRegularFile(module.resolve("go.mod"))) { + return module; + } + } + throw new IllegalStateException("Cannot locate github-sync-go module for HTTP interoperability test"); + } + + private void awaitVersionStatus(Long versionId, String expected) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + String actual; + do { + actual = jdbc.queryForObject("SELECT status FROM skill_version WHERE id = ?", + String.class, versionId); + if (expected.equals(actual)) { + return; + } + Thread.sleep(100); + } while (System.nanoTime() < deadline); + assertThat(actual).isEqualTo(expected); + } + + private String baseUrl() { + return "http://127.0.0.1:" + port; + } + + private String skillSlug(long versionId) { + return jdbc.queryForObject(""" + SELECT skill.slug FROM skill + JOIN skill_version version ON version.skill_id = skill.id + WHERE version.id = ? + """, String.class, versionId); + } + + private HttpResponse anonymousSkillDetail(String namespace, String slug) throws Exception { + HttpRequest request = HttpRequest.newBuilder(URI.create( + baseUrl() + "/api/web/skills/" + namespace + "/" + slug)).GET().build(); + return HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncInternalControllerSecurityTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncInternalControllerSecurityTest.java new file mode 100644 index 000000000..0ed765e77 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncInternalControllerSecurityTest.java @@ -0,0 +1,71 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.service.GithubCredentialLeaseAppService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.Optional; + +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +@TestPropertySource(properties = "skillhub.github-sync.service-tokens=" + + "fixture-service-token-one-32-characters,fixture-service-token-two-32-characters") +class GithubSyncInternalControllerSecurityTest { + private static final String CLAIM = "/internal/github-sync/v1/tasks/claim"; + private static final String REQUEST = """ + {"workerId":"fixture-worker","protocolMajor":1,"kind":"DISCOVERY"} + """; + + @Autowired private MockMvc mockMvc; + @MockBean private GithubCredentialLeaseAppService credentialLeases; + @MockBean private GithubSyncLeaseService leases; + + @Test + void anonymousAndWrongServiceTokensCannotClaim() throws Exception { + mockMvc.perform(post(CLAIM).contentType(MediaType.APPLICATION_JSON).content(REQUEST)) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("AUTH_INVALID")); + mockMvc.perform(post(CLAIM).header("Authorization", "Bearer wrong-service-token") + .contentType(MediaType.APPLICATION_JSON).content(REQUEST)) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("AUTH_INVALID")); + mockMvc.perform(post(CLAIM).with(user("human-user")) + .contentType(MediaType.APPLICATION_JSON).content(REQUEST)) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("AUTH_INVALID")); + verifyNoInteractions(credentialLeases); + } + + @Test + void bothConfiguredTokensReachProtocolButUnsupportedVersionIsRejected() throws Exception { + when(credentialLeases.claim(com.iflytek.skillhub.githubsync.GithubSyncTaskKind.DISCOVERY, + "fixture-worker")).thenReturn(Optional.empty()); + for (String token : new String[]{"fixture-service-token-one-32-characters", + "fixture-service-token-two-32-characters"}) { + mockMvc.perform(post(CLAIM).header("Authorization", "Bearer " + token) + .contentType(MediaType.APPLICATION_JSON).content(REQUEST)) + .andExpect(status().isNoContent()); + mockMvc.perform(post(CLAIM).header("Authorization", "Bearer " + token) + .contentType(MediaType.APPLICATION_JSON) + .content(REQUEST.replace("\"protocolMajor\":1", "\"protocolMajor\":2"))) + .andExpect(status().isUpgradeRequired()) + .andExpect(jsonPath("$.code").value("PROTOCOL_UNSUPPORTED")); + } + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncPathCapacityIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncPathCapacityIntegrationTest.java new file mode 100644 index 000000000..ed051fc0a --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncPathCapacityIntegrationTest.java @@ -0,0 +1,131 @@ +package com.iflytek.skillhub.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +@Testcontainers +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = { + "skillhub.builtin-skills.enabled=false", + "skillhub.github-sync.service-tokens=fixture-service-token-for-path-capacity-32-chars" +}) +class GithubSyncPathCapacityIntegrationTest { + private static final String SERVICE_TOKEN = "fixture-service-token-for-path-capacity-32-chars"; + private static final String COMMIT = "a".repeat(40); + private static final int PATHS = 10_000; + + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + @Autowired private ObjectMapper mapper; + @LocalServerPort private int port; + + @Test + void serviceContractAcceptsTenThousandRepositoryPathResults() throws Exception { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 20000, global_low = 10000, + namespace_high = 20000, namespace_low = 10000 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 3000001, 'fixture', 'many-skills', 'PUBLIC') RETURNING id + """, Long.class, namespaceId); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, + dedupe_key, snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'repository:3000001', + '{"repositoryId":3000001,"owner":"fixture","name":"many-skills"}'::jsonb, ?) + """, taskId, runId, namespaceId, sourceId, COMMIT); + + HttpClient client = HttpClient.newHttpClient(); + String baseUrl = "http://127.0.0.1:" + port + "/internal/github-sync/v1/tasks"; + var claim = client.send(HttpRequest.newBuilder(URI.create(baseUrl + "/claim")) + .header("Authorization", "Bearer " + SERVICE_TOKEN) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + "{\"workerId\":\"capacity-worker\",\"protocolMajor\":1,\"capabilities\":[\"platform-pause/v1\"],\"kind\":\"REPOSITORY_SYNC\"}")) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(claim.statusCode()).isEqualTo(200); + var leased = mapper.readTree(claim.body()); + assertThat(leased.path("taskId").asText()).isEqualTo(taskId.toString()); + assertThat(leased.path("mode").asText()).isEqualTo("FULL"); + + var completion = mapper.createObjectNode(); + completion.set("lease", leased.path("lease")); + completion.put("coverage", "COMPLETE"); + completion.put("observedCommitSha", COMMIT); + completion.put("pathCheckpoint", "skills/skill-09999"); + var results = completion.putArray("pathResults"); + for (int index = 0; index < PATHS; index++) { + var result = results.addObject(); + result.put("skillPath", "skills/skill-%05d".formatted(index)); + result.put("outcome", "TERMINAL_FAILURE"); + result.put("reason", "PACKAGE_TOO_LARGE"); + } + + byte[] payload = mapper.writeValueAsBytes(completion); + long started = System.nanoTime(); + var response = client.send(HttpRequest.newBuilder(URI.create(baseUrl + "/" + taskId + "/complete")) + .header("Authorization", "Bearer " + SERVICE_TOKEN) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofByteArray(payload)) + .build(), HttpResponse.BodyHandlers.ofString()); + long elapsedMillis = Duration.ofNanos(System.nanoTime() - started).toMillis(); + assertThat(response.statusCode()).isEqualTo(200); + assertThat(mapper.readTree(response.body()).path("status").asText()).isEqualTo("PARTIAL"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(PATHS); + System.out.printf("GitHub path HTTP baseline: paths=%d, requestBytes=%d, elapsedMs=%d%n", + PATHS, payload.length, elapsedMillis); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncUnifiedRepositoryIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncUnifiedRepositoryIntegrationTest.java new file mode 100644 index 000000000..40222f877 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/GithubSyncUnifiedRepositoryIntegrationTest.java @@ -0,0 +1,266 @@ +package com.iflytek.skillhub.controller; + +import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; +import com.iflytek.skillhub.dto.GithubSyncRepositoryResponse.*; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.repository.GithubSyncRepositoryQueryRepository; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import com.iflytek.skillhub.repository.GithubDiscoveryEntryStatusQueryRepository; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import java.time.Instant; +import java.sql.Timestamp; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.*; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.authentication; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +@Testcontainers +@Transactional +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestPropertySource(properties = "skillhub.builtin-skills.enabled=false") +class GithubSyncUnifiedRepositoryIntegrationTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + @DynamicPropertySource static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + @Autowired JdbcTemplate jdbc; + @Autowired GithubSyncRepositoryQueryRepository queries; + @Autowired GithubSyncManagementQueryRepository management; + @Autowired GithubDiscoveryEntryRepository entries; + @Autowired GithubDiscoveryEntryStatusQueryRepository entryStatus; + @Autowired MockMvc mvc; + long namespace; + static final Instant FIRST = Instant.parse("2026-01-01T00:00:00Z"); + @BeforeEach void setup() { namespace = management.platformNamespaceId(); } + + @Test void sourceCandidateOverlapAndOriginsNeverDuplicateIdentityOrInflateTotals() { + long a = source(namespace, 101, "ACTIVE"); + long b = candidate(namespace, 101, "BOUND", null); + sourceOrigin(a, "CURATED_REPOSITORY", UUID.randomUUID().toString()); + sourceOrigin(a, "SEARCH_RULE", "deleted-rule"); + candidateOrigin(b, "SEARCH_RULE", "deleted-rule"); + candidateOrigin(b, "OFFICIAL_DISCOVERY", "deleted-query"); + long onlyCandidate = candidate(namespace, 102, "PENDING", null); + candidateOrigin(onlyCandidate, "CURATED_OWNER", "owner"); + source(namespace, 103, "PAUSED"); + long other = otherNamespace(); source(other, 101, "ACTIVE"); candidate(other, 999, "PENDING", null); + jdbc.update("UPDATE github_source_binding SET owner_login='renamed' WHERE id=?", a); + var page = queries.repositories(0, 20, null, null, null, null, null); + assertThat(page.total()).isEqualTo(3); + assertThat(page.items()).extracting(Summary::repositoryId).containsExactly(103L, 102L, 101L); + assertThat(page.items().getLast().owner()).isEqualTo("renamed"); + assertThat(page.items().getLast().state()).isEqualTo(State.REGISTERED); + assertThat(page.items().getLast().originTypes()).containsExactly(OriginType.CURATED_REPOSITORY, OriginType.OFFICIAL_DISCOVERY, OriginType.SEARCH_RULE); + var origins = queries.origins(101, 0, 20, null); + assertThat(origins.total()).isEqualTo(3); + assertThat(origins.items()).filteredOn(o -> o.originType() == OriginType.SEARCH_RULE).singleElement() + .satisfies(o -> { assertThat(o.originKey()).isEqualTo("deleted-rule"); assertThat(o.entryId()).isNull(); }); + assertThat(origins.items()).filteredOn(o -> o.originType() == OriginType.CURATED_REPOSITORY).singleElement() + .satisfies(o -> assertThat(o.lastSeenAt()).isNull()); + assertThat(queries.detail(103).repository().sourceStatus()).isEqualTo(SourceStatus.PAUSED); + assertThatThrownBy(() -> queries.detail(999)).isInstanceOf(DomainNotFoundException.class); + assertThatThrownBy(() -> queries.paths(999, 0, 20, null)).isInstanceOf(DomainNotFoundException.class); + } + + @Test void allFiltersApplyBeforeStablePagingAndCountWithLiteralSearchAndAsOf() { + for (int i = 1; i <= 45; i++) { + long id = candidate(namespace, i, i % 2 == 0 ? "REJECTED" : "PENDING", i % 2 == 0 ? "NO_VALID_SKILL" : null); + candidateOrigin(id, "SEARCH_RULE", "rule"); + if (i % 3 == 0) source(namespace, i, i % 2 == 0 ? "PAUSED" : "ACTIVE"); + } + var p0 = queries.repositories(0, 20, null, null, OriginType.SEARCH_RULE, null, null); + var p1 = queries.repositories(1, 20, p0.asOf(), null, OriginType.SEARCH_RULE, null, null); + assertThat(p0.total()).isEqualTo(45); + assertThat(p0.items()).extracting(Summary::repositoryId).containsExactlyElementsOf(java.util.stream.LongStream.iterate(45, i -> i - 1).limit(20).boxed().toList()); + assertThat(p1.items()).extracting(Summary::repositoryId).containsExactlyElementsOf(java.util.stream.LongStream.iterate(25, i -> i - 1).limit(20).boxed().toList()); + assertThat(queries.repositories(3, 20, p0.asOf(), null, null, null, null).items()).isEmpty(); + for (State state : State.values()) for (SourceStatus status : SourceStatus.values()) { + var expected = queries.repositories(0, 100, p0.asOf(), null, null, null, null).items().stream() + .filter(item -> item.state() == state && item.sourceStatus() == status).toList(); + var actual = queries.repositories(0, 100, p0.asOf(), null, OriginType.SEARCH_RULE, state, status); + assertThat(actual.items()).isEqualTo(expected); assertThat(actual.total()).isEqualTo(expected.size()); + } + jdbc.update("UPDATE github_discovery_candidate SET repository_name='literal_%' WHERE repository_id=1 AND namespace_id=?", namespace); + assertThat(queries.repositories(0, 20, null, "_%", null, null, null).items()).extracting(Summary::repositoryId).containsExactly(1L); + candidate(namespace, 46, "PENDING", null); + jdbc.update("UPDATE github_discovery_candidate SET first_seen_at=? WHERE repository_id=46 AND namespace_id=?", Timestamp.from(p0.asOf().plusSeconds(1)), namespace); + assertThat(queries.repositories(0, 100, p0.asOf(), null, null, null, null).total()).isEqualTo(45); + } + + @Test void candidateStagesDoNotClaimRegistrationOrQuality() { + candidate(namespace, 1, "BOUND", null); + candidate(namespace, 2, "REJECTED", "NO_VALID_SKILL"); + candidate(namespace, 3, "REJECTED", "INVALID_MANIFEST"); + candidate(namespace, 4, "UNREACHABLE", "GITHUB_NOT_FOUND"); + long validating = candidate(namespace, 5, "PENDING", null); + UUID validation = task(run("REPOSITORY_SYNC", "QUEUED"), null, "REPOSITORY_SYNC", "FULL", "{\"repositoryId\":5}"); + jdbc.update("UPDATE github_discovery_candidate SET status='VALIDATING',validation_task_id=? WHERE id=?", validation, validating); + assertThat(queries.repositories(0, 20, null, null, null, null, null).items()).extracting(Summary::state) + .containsExactly(State.VALIDATING, State.UNREACHABLE, State.VALIDATION_FAILED, State.NO_VALID_SKILL, State.BOUND_WITHOUT_SOURCE); + } + + @Test void pureSourcePathsReuseSafeProjectionAndHidePurgedRunLinks() { + long source = source(namespace, 11, "ACTIVE"); + UUID run = run("REPOSITORY_SYNC", "ACQUISITION_COMPLETE"); + UUID task = task(run, source, "REPOSITORY_SYNC", "FULL", "{\"repositoryId\":11}"); + jdbc.update("UPDATE github_sync_task SET status='ACQUISITION_COMPLETE',completed_at=CURRENT_TIMESTAMP WHERE id=?", task); + jdbc.update("INSERT INTO github_sync_path_result(task_id,commit_sha,skill_path,outcome,reason) VALUES (?,?,'skills/a','IGNORED','IGNORE_PATTERN')", task, "a".repeat(40)); + var result = queries.paths(11, 0, 20, null); + assertThat(result.total()).isEqualTo(1); + assertThat(result.items().getFirst().runId()).isEqualTo(run); + assertThat(result.items().getFirst().reviewId()).isNull(); + assertThat(queries.detail(11).latestRunId()).isEqualTo(run); + assertThat(queries.detail(11).lastCheckedAt()).isNotNull(); + jdbc.update("UPDATE github_sync_run SET details_purged_at=CURRENT_TIMESTAMP WHERE id=?", run); + assertThat(queries.paths(11, 0, 20, null).items().getFirst().runId()).isNull(); + assertThat(queries.detail(11).latestRunId()).isNull(); + } + + @Test void repeatedDesignatedRepositoryPreservesDisabledEntryAndExecutionIsNotGuessed() { + var first = entries.createCuratedRepository("Owner", "Repo"); + entries.setEnabled(first.id(), false); + var disabled = entries.findPlatformEntry(first.id()).orElseThrow(); + var repeated = entries.createCuratedRepository("owner", "repo"); + assertThat(repeated.id()).isEqualTo(first.id()); + assertThat(repeated.enabled()).isFalse(); + assertThat(repeated.updatedAt()).isEqualTo(disabled.updatedAt()); + assertThat(repeated.nextDueAt()).isEqualTo(disabled.nextDueAt()); + assertThat(queries.repositories(0, 20, null, null, null, null, null).total()).isZero(); + assertThat(entryStatus.latest(namespace, first.id())).isNull(); + UUID run = run("DISCOVERY", "RETRY_WAIT"); + UUID task = task(run, null, "DISCOVERY", null, "{\"entryType\":\"CURATED_REPOSITORY\",\"ruleId\":\"" + first.id() + "\",\"owner\":\"owner\",\"repositoryName\":\"repo\"}"); + jdbc.update("UPDATE github_sync_task SET status='RETRY_WAIT',last_error_code='GITHUB_RATE_LIMITED',retry_at=CURRENT_TIMESTAMP+INTERVAL '1 hour' WHERE id=?", task); + var execution = entryStatus.latest(namespace, first.id()); + assertThat(execution.taskId()).isEqualTo(task); + assertThat(execution.errorCode()).isEqualTo("GITHUB_RATE_LIMITED"); + assertThat(entryStatus.latestForEntries(namespace, List.of(first.id())).get(first.id())).isEqualTo(execution); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE namespace_id=?", Integer.class, namespace)).isEqualTo(1); + } + + @Test + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.NOT_SUPPORTED) + void concurrentRepeatedRegistrationKeepsOneDisabledIdentity() throws Exception { + var original = entries.createCuratedRepository("concurrent-owner", "same-repo"); + entries.setEnabled(original.id(), false); + var ready = new java.util.concurrent.CountDownLatch(2); + var start = new java.util.concurrent.CountDownLatch(1); + try (var executor = java.util.concurrent.Executors.newFixedThreadPool(2)) { + java.util.concurrent.Callable register = () -> { + ready.countDown(); + assertThat(start.await(10, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + return entries.createCuratedRepository("CONCURRENT-owner", "SAME-repo"); + }; + var first = executor.submit(register); + var second = executor.submit(register); + try { assertThat(ready.await(10, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); } + finally { start.countDown(); } + for (var result : List.of(first, second)) { + var entry = result.get(15, java.util.concurrent.TimeUnit.SECONDS); + assertThat(entry.id()).isEqualTo(original.id()); + assertThat(entry.enabled()).isFalse(); + } + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_entry WHERE namespace_id=? AND lower(owner_login)='concurrent-owner' AND lower(repository_name)='same-repo'", Integer.class, namespace)).isEqualTo(1); + } finally { + jdbc.update("DELETE FROM github_discovery_entry WHERE id=?", original.id()); + } + } + + @Test void runFiltersAndTargetsUseRealRunsWithoutPreviewOrPurgedRows() { + UUID single = run("DISCOVERY", "QUEUED"); + String owner = "{\"entryType\":\"CURATED_OWNER\",\"owner\":\"sample\"}"; + task(single, null, "DISCOVERY", null, owner); task(single, null, "DISCOVERY", null, owner); + UUID multi = run("REPOSITORY_SYNC", "PARTIAL"); + task(multi, source(namespace, 1, "ACTIVE"), "REPOSITORY_SYNC", "HEAD_ONLY", "{}"); + task(multi, source(namespace, 2, "ACTIVE"), "REPOSITORY_SYNC", "FULL", "{}"); + UUID empty = run("DISCOVERY", "FAILED"); + UUID preview = run("DISCOVERY", "QUEUED"); task(preview, null, "DISCOVERY", null, "{\"entryType\":\"SEARCH_PREVIEW\"}"); + UUID purged = run("DISCOVERY", "QUEUED"); jdbc.update("UPDATE github_sync_run SET details_purged_at=CURRENT_TIMESTAMP WHERE id=?", purged); + var rows = management.runs(namespace, 0, 20); + assertThat(rows.total()).isEqualTo(3); + assertThat(rows.items()).filteredOn(r -> r.id().equals(single)).singleElement().satisfies(r -> { + assertThat(r.targetCount()).isEqualTo(1); assertThat(r.target()).isEqualTo(new Target("OWNER", "sample")); }); + assertThat(rows.items()).filteredOn(r -> r.id().equals(multi)).singleElement().satisfies(r -> { + assertThat(r.targetCount()).isEqualTo(2); assertThat(r.target()).isNull(); }); + assertThat(rows.items()).filteredOn(r -> r.id().equals(empty)).singleElement().satisfies(r -> { + assertThat(r.targetCount()).isZero(); assertThat(r.target()).isNull(); }); + for (RunKind kind : RunKind.values()) for (RunStatus status : RunStatus.values()) { + var expected = rows.items().stream().filter(r -> r.kind().equals(kind.name()) && r.status().equals(status.name())).toList(); + var actual = management.runs(namespace, 0, 1, kind, status); + assertThat(actual.total()).isEqualTo(expected.size()); assertThat(actual.items()).isEqualTo(expected.stream().limit(1).toList()); + assertThat(management.runs(namespace, 1, 1, kind, status).items()).isEqualTo(expected.stream().skip(1).limit(1).toList()); + } + } + + @Test void unboundCandidateRunUsesItsActualSnapshotRepositoryTarget() { + UUID run = run("REPOSITORY_SYNC", "QUEUED"); + UUID older = task(run, null, "REPOSITORY_SYNC", "FULL", "{\"repositoryId\":77,\"owner\":\"old-name\",\"name\":\"skills\"}"); + jdbc.update("UPDATE github_sync_task SET created_at=CURRENT_TIMESTAMP-INTERVAL '1 hour' WHERE id=?", older); + task(run, null, "REPOSITORY_SYNC", "FULL", "{\"repositoryId\":77,\"owner\":\"upstream\",\"name\":\"skills\"}"); + var item = management.runs(namespace, 0, 20).items().getFirst(); + assertThat(item.targetCount()).isEqualTo(1); + assertThat(item.target()).isEqualTo(new Target("REPOSITORY", "upstream/skills")); + } + + @Test void routesEnforceAdminScopeAndValidateFilters() throws Exception { + source(namespace, 9, "ACTIVE"); + for (String suffix : List.of("", "/9", "/9/origins", "/9/paths")) { + String path = "/api/v1/admin/github-sync/repositories" + suffix; + mvc.perform(get(path)).andExpect(status().isUnauthorized()); + mvc.perform(get(path).with(authentication(auth("USER")))).andExpect(status().isForbidden()); + mvc.perform(get(path).with(authentication(auth("SUPER_ADMIN")))).andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")); + } + for (String query : List.of("state=BOGUS", "originType=BOGUS", "sourceStatus=BOGUS", "size=101", "page=-1", "q=" + "a".repeat(201))) { + mvc.perform(get("/api/v1/admin/github-sync/repositories?" + query).with(authentication(auth("SUPER_ADMIN")))).andExpect(status().isBadRequest()); + } + mvc.perform(get("/api/v1/admin/github-sync/runs?kind=BOGUS").with(authentication(auth("SUPER_ADMIN")))).andExpect(status().isBadRequest()); + mvc.perform(get("/api/v1/admin/github-sync/runs?status=BOGUS").with(authentication(auth("SUPER_ADMIN")))).andExpect(status().isBadRequest()); + } + + private long source(long ns, long repo, String status) { + return jdbc.queryForObject("INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility,status,created_at) VALUES (?,?,'owner',?,'PUBLIC',?,?) RETURNING id", Long.class, ns, repo, "repo" + repo, status, Timestamp.from(FIRST)); + } + private long candidate(long ns, long repo, String status, String error) { + return jdbc.queryForObject("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name,status,last_error_code,first_seen_at) VALUES (?,?,'owner',?,?,?,?) RETURNING id", Long.class, ns, repo, "repo" + repo, status, error, Timestamp.from(FIRST)); + } + private void sourceOrigin(long source, String type, String key) { jdbc.update("INSERT INTO github_source_origin(source_id,origin_type,origin_key) VALUES (?,?,?)", source, type, key); } + private void candidateOrigin(long candidate, String type, String key) { jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) VALUES (?,?,?)", candidate, type, key); } + private long otherNamespace() { return jdbc.queryForObject("INSERT INTO namespace(slug,display_name,type) VALUES ('unified-other','Other','TEAM') RETURNING id", Long.class); } + private UUID run(String kind, String status) { UUID id = UUID.randomUUID(); jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind,status) VALUES (?,?,?,?)", id, namespace, kind, status); return id; } + private UUID task(UUID run, Long source, String kind, String mode, String snapshot) { + UUID id = UUID.randomUUID(); jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,source_id,kind,mode,dedupe_key,snapshot_json) VALUES (?,?,?,?,?,?,?,?::jsonb)", id, run, namespace, source, kind, mode, id.toString(), snapshot); return id; + } + private UsernamePasswordAuthenticationToken auth(String role) { + var principal = new PlatformPrincipal("unified-test", "Test", null, null, null, Set.of(role)); + return new UsernamePasswordAuthenticationToken(principal, null, List.of(new SimpleGrantedAuthority("ROLE_" + role))); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/ReviewPortalControllerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/ReviewPortalControllerTest.java index a942b481a..27710b68e 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/ReviewPortalControllerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/ReviewPortalControllerTest.java @@ -205,7 +205,7 @@ class ReviewPortalControllerTest { new com.iflytek.skillhub.dto.PageResponse<>(List.of(), 0, 0, 20), null ), - List.of(new SkillVersionResponse(100L, "1.2.0", "PENDING_REVIEW", null, 1, 10L, null, true, null)), + List.of(new SkillVersionResponse(100L, "1.2.0", null, "PENDING_REVIEW", null, 1, 10L, null, true, null)), List.of(new SkillFileResponse(1L, "README.md", 123L, "text/markdown", "sha")), "README.md", "# demo", diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/SkillControllerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/SkillControllerTest.java index a0d588972..01d25d8ef 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/SkillControllerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/SkillControllerTest.java @@ -9,6 +9,8 @@ import com.iflytek.skillhub.domain.skill.SkillVersion; import com.iflytek.skillhub.domain.skill.service.SkillDownloadService; import com.iflytek.skillhub.domain.skill.service.SkillQueryService; import com.iflytek.skillhub.dto.SkillSuiteReferenceResponse; +import com.iflytek.skillhub.dto.GithubSkillSourceResponse; +import com.iflytek.skillhub.repository.GithubSkillSourceQueryRepository; import com.iflytek.skillhub.service.SkillLabelAppService; import com.iflytek.skillhub.service.SkillSuiteAppService; import org.junit.jupiter.api.Test; @@ -26,11 +28,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.ArgumentMatchers.any; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -61,6 +65,56 @@ class SkillControllerTest { @MockBean private SkillSuiteAppService skillSuiteAppService; + @MockBean + private GithubSkillSourceQueryRepository githubSkillSourceQueryRepository; + + @Test + void getSkillDetailShouldExposeOnlyPublishedGithubSource() throws Exception { + when(skillQueryService.getSkillDetail(eq("team"), eq("demo"), eq((String) null), eq(Map.of()))) + .thenReturn(publishedGithubSkill()); + when(skillSuiteAppService.findVisibleMemberships( + eq(1L), eq((String) null), eq(Map.of()), anySet(), eq(0), eq(20))) + .thenReturn(new com.iflytek.skillhub.dto.PageResponse<>(List.of(), 0, 0, 20)); + when(githubSkillSourceQueryRepository.findPublishedVersion(10L)) + .thenReturn(Optional.of(new GithubSkillSourceResponse( + "upstream", "skills", "agents/demo", "a".repeat(40), + 2, "PRESENT_UNCLASSIFIED", List.of(new GithubSkillSourceResponse.LicenseDocument( + "REPOSITORY_ROOT", "LICENSE", true))))); + + mockMvc.perform(get("/api/web/skills/team/demo")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.githubSource.ownerLogin").value("upstream")) + .andExpect(jsonPath("$.data.githubSource.skillPath").value("agents/demo")) + .andExpect(jsonPath("$.data.githubSource.commitSha").value("a".repeat(40))) + .andExpect(jsonPath("$.data.githubSource.licenseDocuments[0].path").value("LICENSE")) + .andExpect(jsonPath("$.data.githubSource.ingestionStatus").doesNotExist()); + } + + @Test + void getSkillDetailShouldNotReadGithubSourceWhenTeamSkillIsForbidden() throws Exception { + when(skillQueryService.getSkillDetail(eq("team"), eq("demo"), eq((String) null), eq(Map.of()))) + .thenThrow(new DomainForbiddenException("error.skill.access.denied", "demo")); + + mockMvc.perform(get("/api/web/skills/team/demo")) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.data.githubSource").doesNotExist()); + verifyNoInteractions(githubSkillSourceQueryRepository); + } + + private SkillQueryService.SkillDetailDTO publishedGithubSkill() { + return new SkillQueryService.SkillDetailDTO( + 1L, "demo", "Demo", "owner-1", "Alice", "Synced skill", + "PRIVATE", "ACTIVE", 0L, 0, 0, null, 0, false, 1L, + Instant.parse("2026-03-15T10:00:00Z"), + Instant.parse("2026-03-15T10:00:00Z"), + false, false, true, true, + new com.iflytek.skillhub.domain.skill.service.SkillLifecycleProjectionService.VersionProjection( + 10L, "1.0.0", "PUBLISHED"), + new com.iflytek.skillhub.domain.skill.service.SkillLifecycleProjectionService.VersionProjection( + 10L, "1.0.0", "PUBLISHED"), + null, null, "PUBLISHED", null, null); + } + @Test void generatedOpenApiDescribesNativeRegistryFileAndDownloadContracts() throws Exception { var response = mockMvc.perform(get("/v3/api-docs")) diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCandidateEligibilityUpgradeTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCandidateEligibilityUpgradeTest.java new file mode 100644 index 000000000..4ac4f3449 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCandidateEligibilityUpgradeTest.java @@ -0,0 +1,41 @@ +package com.iflytek.skillhub.db; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubCandidateEligibilityUpgradeTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test void upgradingV117PreservesViewContractAndExistingEligibility() { + var source = new DriverManagerDataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + migrate(source, "117"); + var jdbc = new JdbcTemplate(source); + long namespace = jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id", Long.class); + jdbc.update("UPDATE github_deep_discovery_config SET enabled=TRUE,exclusions_json='[{\"kind\":\"OWNER\",\"value\":\"blocked\"}]' WHERE id"); + jdbc.update("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) VALUES (?,9101,'Blocked','synthetic')", namespace); + jdbc.update("INSERT INTO github_discovery_candidate_origin(candidate_id,origin_type,origin_key) SELECT id,'OFFICIAL_DISCOVERY',gen_random_uuid()::text FROM github_discovery_candidate"); + var before = jdbc.queryForList("SELECT * FROM github_candidate_discovery_eligibility ORDER BY candidate_id"); + String columns = "SELECT column_name,data_type,ordinal_position FROM information_schema.columns WHERE table_schema='public' AND table_name='github_candidate_discovery_eligibility' ORDER BY ordinal_position"; + var shape = jdbc.queryForList(columns); + migrate(source, "118"); + assertThat(jdbc.queryForList(columns)).isEqualTo(shape); + assertThat(jdbc.queryForList("SELECT * FROM github_candidate_discovery_eligibility ORDER BY candidate_id")).isEqualTo(before); + assertThat(jdbc.queryForObject("SELECT acquisition_allowed FROM github_candidate_discovery_eligibility WHERE repository_id=9101", Boolean.class)).isFalse(); + jdbc.update("UPDATE github_discovery_candidate SET owner_login='allowed' WHERE repository_id=9101"); + assertThat(jdbc.queryForObject("SELECT acquisition_allowed FROM github_candidate_discovery_eligibility WHERE repository_id=9101", Boolean.class)).isTrue(); + assertThat(jdbc.queryForObject("SELECT success FROM flyway_schema_history WHERE version='118'", Boolean.class)).isTrue(); + } + + private void migrate(DriverManagerDataSource source, String target) { + Flyway.configure().dataSource(source).locations("classpath:db/migration") + .target(MigrationVersion.fromVersion(target)).load().migrate(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCatalogNamespaceMigrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCatalogNamespaceMigrationTest.java new file mode 100644 index 000000000..53f8a05a7 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubCatalogNamespaceMigrationTest.java @@ -0,0 +1,116 @@ +package com.iflytek.skillhub.db; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.sql.DataSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Testcontainers +class GithubCatalogNamespaceMigrationTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void newDeploymentCreatesOwnedCatalogAndSystemActor() { + var fixture = beforeCatalogMigration("github_catalog_clean"); + + migrate(fixture.source(), fixture.schema(), "76"); + + assertThat(fixture.jdbc().queryForObject(""" + SELECT type FROM namespace WHERE slug = 'github' + """, String.class)).isEqualTo("CATALOG"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT created_by FROM namespace WHERE slug = 'github' + """, String.class)).isEqualTo("github-sync-publisher"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT system_account FROM user_account WHERE id = 'github-sync-publisher' + """, Boolean.class)).isTrue(); + assertThat(fixture.jdbc().queryForObject(""" + SELECT member.role FROM namespace_member member + JOIN namespace ON namespace.id = member.namespace_id + WHERE namespace.slug = 'github' AND member.user_id = 'github-sync-publisher' + """, String.class)).isEqualTo("OWNER"); + } + + @Test + void existingGithubNamespaceBlocksMigrationWithoutAdoptingUserData() { + var fixture = beforeCatalogMigration("github_slug_collision"); + fixture.jdbc().update(""" + INSERT INTO user_account (id, display_name) VALUES ('collision-owner', 'Existing Owner') + """); + fixture.jdbc().update(""" + INSERT INTO namespace (slug, display_name, type, created_by) + VALUES ('github', 'Existing Team', 'TEAM', 'collision-owner') + """); + + assertThatThrownBy(() -> migrate(fixture.source(), fixture.schema(), "76")) + .hasStackTraceContaining("Cannot create @github catalog: namespace slug github already exists"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT display_name FROM namespace WHERE slug = 'github' + """, String.class)).isEqualTo("Existing Team"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT type FROM namespace WHERE slug = 'github' + """, String.class)).isEqualTo("TEAM"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT count(*) FROM user_account WHERE id = 'github-sync-publisher' + """, Integer.class)).isZero(); + assertThat(fixture.jdbc().queryForObject(""" + SELECT count(*) FROM flyway_schema_history WHERE version = '76' AND success = TRUE + """, Integer.class)).isZero(); + } + + @Test + void existingActorBlocksMigrationWithoutPromotingThatAccount() { + var fixture = beforeCatalogMigration("github_actor_collision"); + fixture.jdbc().update(""" + INSERT INTO user_account (id, display_name, system_account) + VALUES ('github-sync-publisher', 'Existing User', FALSE) + """); + + assertThatThrownBy(() -> migrate(fixture.source(), fixture.schema(), "76")) + .hasStackTraceContaining("Cannot create GitHub catalog actor: user id github-sync-publisher already exists"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT display_name FROM user_account WHERE id = 'github-sync-publisher' + """, String.class)).isEqualTo("Existing User"); + assertThat(fixture.jdbc().queryForObject(""" + SELECT system_account FROM user_account WHERE id = 'github-sync-publisher' + """, Boolean.class)).isFalse(); + assertThat(fixture.jdbc().queryForObject(""" + SELECT count(*) FROM namespace WHERE slug = 'github' + """, Integer.class)).isZero(); + assertThat(fixture.jdbc().queryForObject(""" + SELECT count(*) FROM flyway_schema_history WHERE version = '76' AND success = TRUE + """, Integer.class)).isZero(); + } + + private Fixture beforeCatalogMigration(String schema) { + var admin = new JdbcTemplate(new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword())); + admin.execute("CREATE SCHEMA " + schema); + String url = POSTGRES.getJdbcUrl() + + (POSTGRES.getJdbcUrl().contains("?") ? "&" : "?") + + "currentSchema=" + schema; + DataSource source = new DriverManagerDataSource( + url, POSTGRES.getUsername(), POSTGRES.getPassword()); + migrate(source, schema, "75"); + return new Fixture(schema, source, new JdbcTemplate(source)); + } + + private void migrate(DataSource source, String schema, String target) { + Flyway.configure().dataSource(source).schemas(schema).defaultSchema(schema) + .locations("classpath:db/migration") + .target(MigrationVersion.fromVersion(target)).load().migrate(); + } + + private record Fixture(String schema, DataSource source, JdbcTemplate jdbc) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubContentRevisionUpgradeTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubContentRevisionUpgradeTest.java new file mode 100644 index 000000000..9ed5634c2 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubContentRevisionUpgradeTest.java @@ -0,0 +1,52 @@ +package com.iflytek.skillhub.db; + +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubContentRevisionUpgradeTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test void v106HistoryAABABackfillsConsecutiveContentRevisionsWithoutChangingTechnicalVersions() { + var source = new DriverManagerDataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + migrate(source, "106"); + var jdbc = new JdbcTemplate(source); + long namespace = jdbc.queryForObject("SELECT id FROM namespace WHERE slug='github'", Long.class); + long binding = jdbc.queryForObject("INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility) VALUES (?,9701,'fixture','history','PUBLIC') RETURNING id", Long.class, namespace); + long skill = jdbc.queryForObject("INSERT INTO skill(namespace_id,slug,owner_id,github_sync_managed) VALUES (?,'revision-history','github-sync-publisher',TRUE) RETURNING id", Long.class, namespace); + long ordinary = jdbc.queryForObject("INSERT INTO skill_version(skill_id,version,status) VALUES (?,'1.0.0','PUBLISHED') RETURNING id", Long.class, skill); + UUID latest = null; + for (int i=0;i<4;i++) { + String digest = (i==2 ? "b" : "a").repeat(64); + UUID run=UUID.randomUUID(), task=UUID.randomUUID(), admission=UUID.randomUUID(), upload=UUID.randomUUID(); + latest=UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run(id,namespace_id,kind) VALUES (?,?,'REPOSITORY_SYNC')",run,namespace); + jdbc.update("INSERT INTO github_sync_task(id,run_id,namespace_id,source_id,kind,mode,dedupe_key,snapshot_json) VALUES (?,?,?,?,'REPOSITORY_SYNC','FULL',?,'{}')",task,run,namespace,binding,task.toString()); + jdbc.update("INSERT INTO github_sync_package_admission(id,task_id,namespace_id,repository_id,commit_sha,skill_path,lease_generation) VALUES (?,?,?,9701,?,'skills/demo',1)",admission,task,namespace,"a".repeat(40)); + jdbc.update("INSERT INTO github_sync_staged_upload(id,admission_id,object_key,mode,ticket_hash,size_bytes,sha256,status,expires_at) VALUES (?,?,?,'LOCAL_STREAM',?,1,?,'ACCEPTED',now()+INTERVAL '1 hour')",upload,admission,upload.toString(),digest,digest); + jdbc.update("INSERT INTO github_sync_package_ingestion(id,task_id,admission_id,upload_id,namespace_id,repository_id,commit_sha,skill_path,object_key,size_bytes,sha256,content_sha256,status) VALUES (?,?,?,?,?,9701,?,'skills/demo',?,1,?,?,'SUPERSEDED')",latest,task,admission,upload,namespace,"a".repeat(40),upload.toString(),digest,digest); + jdbc.update("INSERT INTO skill_version(skill_id,version,status,github_sync_ingestion_id) VALUES (?,?,'REJECTED',?)",skill,"0.0.0-gfixture-"+i,latest); + } + jdbc.update("INSERT INTO github_source_skill_binding(source_id,namespace_id,repository_id,skill_path,skill_id,latest_content_sha256,latest_ingestion_id) VALUES (?,?,9701,'skills/demo',?,?,?)",binding,namespace,skill,"a".repeat(64),latest); + var versions=jdbc.queryForList("SELECT id,version FROM skill_version WHERE skill_id=? ORDER BY id",skill); + migrate(source,"107"); + assertThat(jdbc.queryForList("SELECT github_sync_revision FROM skill_version WHERE skill_id=? AND github_sync_ingestion_id IS NOT NULL ORDER BY id",Integer.class,skill)).containsExactly(1,1,2,3); + assertThat(jdbc.queryForObject("SELECT latest_revision FROM github_source_skill_binding WHERE skill_id=?",Integer.class,skill)).isEqualTo(3); + assertThat(jdbc.queryForObject("SELECT github_sync_revision FROM skill_version WHERE id=?",Integer.class,ordinary)).isNull(); + assertThat(jdbc.queryForList("SELECT id,version FROM skill_version WHERE skill_id=? ORDER BY id",skill)).isEqualTo(versions); + assertThat(jdbc.queryForObject("SELECT success FROM flyway_schema_history WHERE version='107'",Boolean.class)).isTrue(); + } + + private void migrate(DriverManagerDataSource source,String version) { + Flyway.configure().dataSource(source).locations("classpath:db/migration") + .target(MigrationVersion.fromVersion(version)).load().migrate(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubDiscoveryOutcomeUpgradeTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubDiscoveryOutcomeUpgradeTest.java new file mode 100644 index 000000000..4735aec8f --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubDiscoveryOutcomeUpgradeTest.java @@ -0,0 +1,47 @@ +package com.iflytek.skillhub.db; + +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryOutcomeRepository; +import java.time.Instant; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubDiscoveryOutcomeUpgradeTest { + @Container static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test void upgradingV114PreservesUnknownValidationForBothLegacyCandidatesAndSourceOnlyRepositories() { + var source = new DriverManagerDataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + migrate(source, "114"); + var jdbc = new JdbcTemplate(source); + long namespace = jdbc.queryForObject("SELECT namespace_id FROM github_deep_discovery_config WHERE id", Long.class); + jdbc.update("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) VALUES (?,9101,'legacy','candidate')", namespace); + jdbc.update("INSERT INTO github_source_binding(namespace_id,repository_id,owner_login,repository_name,visibility) VALUES (?,9102,'legacy','source','PUBLIC')", namespace); + jdbc.update("INSERT INTO github_discovery_candidate(namespace_id,repository_id,owner_login,repository_name) VALUES (?,9103,'legacy','proven')", namespace); + jdbc.update("INSERT INTO github_official_validated_seed(namespace_id,repository_id,github_owner_id,owner_login,ingestion_id,validated_at) VALUES (?,9103,93,'legacy',gen_random_uuid(),now()-INTERVAL '1 day')", namespace); + migrate(source, "115"); + assertThat(jdbc.queryForList("SELECT repository_id FROM github_discovery_legacy_unknown_validation ORDER BY repository_id", Long.class)).containsExactly(9101L, 9102L); + assertThat(jdbc.queryForObject("SELECT occurred_at IS NULL FROM github_discovery_outcome_event WHERE repository_id=9103 AND event_type='FIRST_VALID'", Boolean.class)).isTrue(); + var outcomes = new JdbcGithubDiscoveryOutcomeRepository(jdbc); + for (long repository : new long[]{9101, 9102, 9103, 9104}) { + outcomes.firstValid(namespace, repository, Instant.now()); + outcomes.firstValid(namespace, repository, Instant.now()); + } + assertThat(jdbc.queryForList("SELECT repository_id FROM github_discovery_outcome_event WHERE event_type='FIRST_VALID' AND NOT first_occurrence_known ORDER BY repository_id", Long.class)).containsExactly(9101L, 9102L); + assertThat(jdbc.queryForObject("SELECT first_occurrence_known AND occurred_at IS NOT NULL FROM github_discovery_outcome_event WHERE repository_id=9104 AND event_type='FIRST_VALID'", Boolean.class)).isTrue(); + assertThat(jdbc.queryForObject("SELECT occurred_at IS NULL AND NOT official_at_event FROM github_discovery_outcome_event WHERE repository_id=9103 AND event_type='FIRST_VALID'", Boolean.class)).isTrue(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_outcome_event WHERE event_type='FIRST_VALID'", Integer.class)).isEqualTo(4); + assertThat(jdbc.queryForObject("SELECT occurred_at IS NULL AND NOT official_at_event FROM github_discovery_outcome_event WHERE repository_id=9101 AND event_type='CANDIDATE_CREATED'", Boolean.class)).isTrue(); + } + + private void migrate(DriverManagerDataSource source, String target) { + Flyway.configure().dataSource(source).locations("classpath:db/migration") + .target(MigrationVersion.fromVersion(target)).load().migrate(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubMainUpgradeCompatibilityTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubMainUpgradeCompatibilityTest.java new file mode 100644 index 000000000..44d4a8a0c --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubMainUpgradeCompatibilityTest.java @@ -0,0 +1,52 @@ +package com.iflytek.skillhub.db; + +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Proves the released main schema and its existing authoring/review data survive the sync upgrade. */ +@Testcontainers +class GithubMainUpgradeCompatibilityTest { + @Container + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void releasedMainDataAndChecksumsSurviveAllSyncMigrations() { + var source = new DriverManagerDataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Flyway.configure().dataSource(source).locations("classpath:db/migration").target("75").load().migrate(); + var jdbc = new JdbcTemplate(source); + var releasedHistory = jdbc.queryForList("SELECT version,checksum FROM flyway_schema_history WHERE version::int <= 75 ORDER BY installed_rank"); + jdbc.update("INSERT INTO user_account(id,display_name) VALUES ('main-upgrade-owner','Main owner')"); + long namespace = jdbc.queryForObject("INSERT INTO namespace(slug,display_name,type,created_by) VALUES ('main-upgrade','Main team','TEAM','main-upgrade-owner') RETURNING id", Long.class); + long skill = jdbc.queryForObject("INSERT INTO skill(namespace_id,slug,owner_id,display_name,summary,visibility,active_avatar_id,active_avatar_audience) VALUES (?,'existing','main-upgrade-owner','Published title','Published summary','PRIVATE',777,'PRIVATE') RETURNING id", Long.class, namespace); + long version = jdbc.queryForObject("INSERT INTO skill_version(skill_id,version,status) VALUES (?,'1.2.3','PUBLISHED') RETURNING id", Long.class, skill); + jdbc.update("UPDATE skill SET latest_version_id=? WHERE id=?", version, skill); + long review = jdbc.queryForObject("INSERT INTO review_task(skill_id,namespace_id,subject_type,subject_id,subject_version,submitted_by,previous_visibility,requested_visibility,visibility_version_id) VALUES (?,?,'SKILL_METADATA',?,'visibility:main-fixture','main-upgrade-owner','PRIVATE','PUBLIC',?) RETURNING id", Long.class, skill, namespace, skill, version); + jdbc.update("INSERT INTO skill_draft(namespace_id,owner_id,name,content_digest) VALUES (?,'main-upgrade-owner','Existing draft',?)", namespace, "a".repeat(64)); + jdbc.update("INSERT INTO namespace_invitation_history(id,namespace_id,issued_by,issued_at,expires_at) VALUES ('main-invitation-history',?,'main-upgrade-owner',now(),now()+interval '1 day')", namespace); + jdbc.update("INSERT INTO namespace_invitation(namespace_id,token_hash,issued_by,issued_at,expires_at,history_id) VALUES (?,?,'main-upgrade-owner',now(),now()+interval '1 day','main-invitation-history')", namespace, "b".repeat(64)); + + var flyway = Flyway.configure().dataSource(source).locations("classpath:db/migration").load(); + flyway.migrate(); + flyway.validate(); + + assertThat(jdbc.queryForList("SELECT version,checksum FROM flyway_schema_history WHERE version::int <= 75 ORDER BY installed_rank")).isEqualTo(releasedHistory); + assertThat(jdbc.queryForObject("SELECT count(*) FROM flyway_schema_history WHERE success AND version::int BETWEEN 76 AND 119", Integer.class)).isEqualTo(44); + assertThat(jdbc.queryForObject("SELECT display_name FROM skill WHERE id=?", String.class, skill)).isEqualTo("Published title"); + assertThat(jdbc.queryForObject("SELECT active_avatar_id FROM skill WHERE id=?", Long.class, skill)).isEqualTo(777L); + assertThat(jdbc.queryForObject("SELECT latest_version_id FROM skill WHERE id=?", Long.class, skill)).isEqualTo(version); + assertThat(jdbc.queryForObject("SELECT github_sync_managed FROM skill WHERE id=?", Boolean.class, skill)).isFalse(); + assertThat(jdbc.queryForMap("SELECT previous_visibility,requested_visibility,visibility_version_id,origin_type FROM review_task WHERE id=?", review)) + .containsEntry("previous_visibility", "PRIVATE").containsEntry("requested_visibility", "PUBLIC") + .containsEntry("visibility_version_id", version).containsEntry("origin_type", "USER"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM skill_draft WHERE name='Existing draft'", Integer.class)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT history_id FROM namespace_invitation WHERE namespace_id=?", String.class, namespace)).isEqualTo("main-invitation-history"); + assertThat(jdbc.queryForObject("SELECT type FROM namespace WHERE slug='github'", String.class)).isEqualTo("CATALOG"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubReviewOriginMigrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubReviewOriginMigrationTest.java new file mode 100644 index 000000000..ccf8e074a --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/db/GithubReviewOriginMigrationTest.java @@ -0,0 +1,66 @@ +package com.iflytek.skillhub.db; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubReviewOriginMigrationTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void existingReviewsRemainUserReviewsAfterGithubOriginMigration() { + var dataSource = new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + migrate(dataSource, "82"); + var jdbc = new JdbcTemplate(dataSource); + jdbc.update("INSERT INTO user_account (id, display_name) VALUES ('review-author', 'Review Author')"); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, created_by) + VALUES ('review-origin-team', 'Review Origin Team', 'TEAM', 'review-author') + RETURNING id + """, Long.class); + long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id) + VALUES (?, 'existing-skill', 'review-author') RETURNING id + """, Long.class, namespaceId); + long versionId = jdbc.queryForObject(""" + INSERT INTO skill_version (skill_id, version, created_by) + VALUES (?, '1.0.0', 'review-author') RETURNING id + """, Long.class, skillId); + long reviewId = jdbc.queryForObject(""" + INSERT INTO review_task + (skill_version_id, skill_id, skill_version, namespace_id, + subject_type, subject_id, subject_version_id, subject_version, submitted_by) + VALUES (?, ?, '1.0.0', ?, 'SKILL_VERSION', ?, ?, '1.0.0', 'review-author') + RETURNING id + """, Long.class, versionId, skillId, namespaceId, skillId, versionId); + + migrate(dataSource, "83"); + + assertThat(jdbc.queryForObject(""" + SELECT origin_type FROM review_task WHERE id = ? + """, String.class, reviewId)).isEqualTo("USER"); + assertThat(jdbc.queryForObject(""" + SELECT github_sync_ingestion_id IS NULL FROM review_task WHERE id = ? + """, Boolean.class, reviewId)).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM review_task + WHERE namespace_id = ? AND status = 'PENDING' AND origin_type = 'USER' + """, Integer.class, namespaceId)).isEqualTo(1); + } + + private void migrate(DriverManagerDataSource dataSource, String target) { + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration") + .target(MigrationVersion.fromVersion(target)).load().migrate(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/exception/GlobalExceptionHandlerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/exception/GlobalExceptionHandlerTest.java index d931f1a56..8816832fc 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/exception/GlobalExceptionHandlerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/exception/GlobalExceptionHandlerTest.java @@ -137,15 +137,18 @@ class GlobalExceptionHandlerTest { try (RequestIdAccessor.Scope ignored = requestIdAccessor.open("request-123")) { ResponseEntity> response = handler.handleGlobalException( - new RuntimeException("boom"), request); + new RuntimeException("synthetic-token-and-skill-content-marker"), request); assertThat(response.getStatusCode()).isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR); } assertThat(loggedMessages()).anySatisfy(message -> assertThat(message) .contains("requestId=request-123") .contains("authentication=authenticated") + .contains("type=RuntimeException") .doesNotContain(STABLE_USER_ID) + .doesNotContain("synthetic-token-and-skill-content-marker") .doesNotContain("userId=")); + assertThat(appender.list).allSatisfy(event -> assertThat(event.getThrowableProxy()).isNull()); } @Test @@ -173,7 +176,8 @@ class GlobalExceptionHandlerTest { .thenReturn("/api/v1/skills/test/download"); StorageAccessException exception = new StorageAccessException( - "download", "skills/test.zip", new RuntimeException("unavailable")); + "download", "synthetic-object-key-secret", + new RuntimeException("synthetic-storage-token")); try (RequestIdAccessor.Scope ignored = requestIdAccessor.open("request-123")) { ResponseEntity> response = handler.handleStorageAccess(exception, request); @@ -184,7 +188,27 @@ class GlobalExceptionHandlerTest { .contains("requestId=request-123") .contains("authentication=authenticated") .doesNotContain(STABLE_USER_ID) + .doesNotContain("synthetic-object-key-secret", "synthetic-storage-token") .doesNotContain("userId=")); + assertThat(appender.list).allSatisfy(event -> assertThat(event.getThrowableProxy()).isNull()); + } + + @Test + void githubSyncStorageFailureOmitsUntrustedContentFromResponseAndLogs() { + String marker = "GITHUB_SYNC_STORAGE_WITH_SKILL_BODY_AND_TOKEN"; + when(request.getServletPath()).thenReturn("/internal/github-sync/v1/uploads/123/content"); + attachAppender(); + + ResponseEntity> response = handler.handleStorageAccess( + new StorageAccessException("putObject", "staging-key", + new IllegalStateException(marker)), request); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.SERVICE_UNAVAILABLE); + assertThat(response.getBody()).isNotNull(); + assertThat(response.getBody().toString()).doesNotContain(marker, "staging-key"); + assertThat(loggedMessages()).allSatisfy(message -> assertThat(message) + .doesNotContain(marker, "staging-key", "/internal/github-sync/v1/uploads/123/content")); + assertThat(appender.list).allSatisfy(event -> assertThat(event.getThrowableProxy()).isNull()); } @Test @@ -211,7 +235,7 @@ class GlobalExceptionHandlerTest { @Test void handleMvcBadRequest_shouldReturn400WithoutUnhandledErrorLog() { attachAppender(); - prepareClientErrorRequest("POST", "/api/v1/skills?bad=value"); + prepareClientErrorRequest("POST", "/api/v1/skills"); List exceptions = List.of( new MissingServletRequestParameterException("namespace", "String"), diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/filter/RequestLoggingFilterTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/filter/RequestLoggingFilterTest.java index 9b748099f..3a07bc3e1 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/filter/RequestLoggingFilterTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/filter/RequestLoggingFilterTest.java @@ -105,7 +105,7 @@ class RequestLoggingFilterTest { } @Test - void doFilterInternal_redactsOAuthCallbackQueryParameters() throws Exception { + void doFilterInternal_omitsOAuthCallbackQueryParameters() throws Exception { RequestLoggingFilter filter = new RequestLoggingFilter(); attachAppender(); @@ -119,11 +119,43 @@ class RequestLoggingFilterTest { .filter(entry -> entry.contains("GET /login/oauth2/code/feishu")) .findFirst() .orElseThrow(); - assertThat(message).contains("code=[REDACTED]"); - assertThat(message).contains("state=[REDACTED]"); - assertThat(message).contains("scope=contact:user.base:readonly"); - assertThat(message).doesNotContain("authorization-code"); - assertThat(message).doesNotContain("csrf-state"); + assertThat(message).doesNotContain("?", "code=", "state=", "scope=", + "authorization-code", "csrf-state"); + } + + @Test + void doFilterInternal_skipsGithubCredentialRoute() throws Exception { + RequestLoggingFilter filter = new RequestLoggingFilter(); + attachAppender(); + MockHttpServletRequest request = new MockHttpServletRequest( + "GET", "/api/v1/admin/github-sync/credentials"); + request.addHeader("Authorization", "Bearer header-secret"); + request.setQueryString("token=credential-value&github_token=another-value&api_key=third-value" + + "&authorization=query-secret&service_token=machine-secret&page=1"); + + filter.doFilter(request, new MockHttpServletResponse(), (req, res) -> {}); + + assertThat(loggedMessages()).isEmpty(); + } + + @Test + void doFilterInternal_omitsUntrustedQueryAndUserAgent() throws Exception { + RequestLoggingFilter filter = new RequestLoggingFilter(); + attachAppender(); + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/api/v1/skills"); + request.addHeader("Authorization", "Bearer synthetic-auth-secret"); + request.addHeader("User-Agent", "SKILL.md full-content-marker synthetic-agent-secret"); + request.addHeader("Content-Type", "application/json; synthetic-content-secret"); + request.setQueryString("custom=synthetic-query-secret&synthetic-key-secret=value&page=1"); + + filter.doFilter(request, new MockHttpServletResponse(), (req, res) -> {}); + + String message = loggedMessages().getFirst(); + assertThat(message).contains("GET /api/v1/skills"); + assertThat(message).doesNotContain("?", "synthetic-auth-secret", + "synthetic-agent-secret", "synthetic-content-secret", "synthetic-query-secret", + "synthetic-key-secret", + "full-content-marker", "page=1"); } @Test diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetricsTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetricsTest.java new file mode 100644 index 000000000..625ae1674 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/metrics/GithubSyncStagingCapacityMetricsTest.java @@ -0,0 +1,166 @@ +package com.iflytek.skillhub.metrics; + +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.storage.ObjectStorageService; +import com.iflytek.skillhub.storage.StorageProperties; +import com.iflytek.skillhub.storage.StoragePrefixUsage; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.nio.file.Path; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubSyncStagingCapacityMetricsTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + @TempDir Path storageRoot; + + @Test + void tracksUncleanedReservationsAndWarnsWhenLocalDiskCannotFitOnePackage() { + long namespaceId = jdbc.queryForObject("SELECT id FROM namespace WHERE slug = 'github'", + Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + String commit = "a".repeat(40); + String digest = "b".repeat(64); + Instant now = Instant.now(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, dedupe_key, snapshot_json) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, '{}'::jsonb) + """, taskId, runId, namespaceId, "staging-metric:" + taskId); + for (int index = 0; index < 3; index++) { + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + String path = "skills/demo-" + index; + String key = "github-sync/staging/" + uploadId + ".zip"; + int size = (index + 2) * 10; + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, + skill_path, lease_generation, status) + VALUES (?, ?, ?, 950001, ?, ?, 1, ?) + """, admissionId, taskId, namespaceId, commit, path, + index == 0 ? "ACTIVE" : "ACCEPTED"); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, + size_bytes, sha256, status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, ?, ?, ?, ?) + """, uploadId, admissionId, key, digest, size, digest, + index == 0 ? "ISSUED" : "ACCEPTED", Timestamp.from(now.plusSeconds(300))); + if (index > 0) { + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, + repository_id, commit_sha, skill_path, object_key, + size_bytes, sha256, status, object_cleaned_at) + VALUES (?, ?, ?, ?, ?, 950001, ?, ?, ?, ?, ?, 'READY_FOR_REVIEW', ?) + """, UUID.randomUUID(), taskId, admissionId, uploadId, namespaceId, + commit, path, key, size, digest, + index == 2 ? Timestamp.from(now) : null); + } + } + var properties = new StorageProperties(); + properties.getLocal().setBasePath(storageRoot.toString()); + var policy = new SkillPublishProperties(); + policy.setMaxPackageSize(Long.MAX_VALUE); + var registry = new SimpleMeterRegistry(); + var objectStorage = mock(ObjectStorageService.class); + var metrics = new GithubSyncStagingCapacityMetrics(jdbc, properties, objectStorage, + policy, registry, 100); + metrics.sample(); + + assertThat(registry.get("skillhub.github.sync.staging.reserved.bytes") + .gauge().value()).isEqualTo(50); + assertThat(registry.get("skillhub.github.sync.staging.local.available.bytes") + .gauge().value()).isPositive(); + assertThat(registry.get("skillhub.github.sync.staging.local.capacity.warning") + .gauge().value()).isEqualTo(1); + + properties.setProvider("s3"); + metrics.sample(); + assertThat(registry.get("skillhub.github.sync.staging.local.available.bytes") + .gauge().value()).isEqualTo(-1); + assertThat(registry.get("skillhub.github.sync.staging.local.capacity.warning") + .gauge().value()).isZero(); + when(objectStorage.listPrefixUsage("github-sync/staging/", 10_000)) + .thenReturn(Optional.of(new StoragePrefixUsage(80, 2, true))) + .thenReturn(Optional.of(new StoragePrefixUsage(120, 2, true))) + .thenReturn(Optional.of(new StoragePrefixUsage(120, 10_000, false))) + .thenReturn(Optional.empty()); + metrics.sampleS3(); + assertThat(registry.get("skillhub.github.sync.staging.s3.used.bytes") + .gauge().value()).isEqualTo(80); + assertThat(registry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isZero(); + metrics.sampleS3(); + assertThat(registry.get("skillhub.github.sync.staging.s3.used.bytes") + .gauge().value()).isEqualTo(120); + assertThat(registry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isEqualTo(1); + metrics.sampleS3(); + assertThat(registry.get("skillhub.github.sync.staging.s3.used.bytes") + .gauge().value()).isEqualTo(-1); + assertThat(registry.get("skillhub.github.sync.staging.s3.scan.complete") + .gauge().value()).isZero(); + metrics.sampleS3(); + assertThat(registry.get("skillhub.github.sync.staging.s3.objects") + .gauge().value()).isEqualTo(-1); + assertThat(registry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isEqualTo(1); + registry.close(); + + var unconfiguredRegistry = new SimpleMeterRegistry(); + var unconfigured = new GithubSyncStagingCapacityMetrics(jdbc, properties, objectStorage, + policy, unconfiguredRegistry, 0); + when(objectStorage.listPrefixUsage("github-sync/staging/", 10_000)) + .thenReturn(Optional.of(new StoragePrefixUsage(0, 0, true))) + .thenReturn(Optional.of(new StoragePrefixUsage(1, 1, true))); + unconfigured.sampleS3(); + assertThat(unconfiguredRegistry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isZero(); + unconfigured.sampleS3(); + assertThat(unconfiguredRegistry.get("skillhub.github.sync.staging.s3.capacity.warning") + .gauge().value()).isEqualTo(1); + unconfiguredRegistry.close(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/AesGcmGithubCredentialCipherTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/AesGcmGithubCredentialCipherTest.java new file mode 100644 index 000000000..749c52c72 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/AesGcmGithubCredentialCipherTest.java @@ -0,0 +1,69 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.dto.GithubCredentialResponse; +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.infra.jpa.AesGcmGithubCredentialCipher; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Base64; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AesGcmGithubCredentialCipherTest { + @Test + void missingKeyRejectsEncryptionAndDecryption() { + var cipher = new AesGcmGithubCredentialCipher("", ""); + UUID credentialId = UUID.randomUUID(); + + assertThat(cipher.configured()).isFalse(); + assertThatThrownBy(() -> cipher.encrypt(credentialId, "example-token")) + .isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> cipher.decrypt(credentialId, "v1", new byte[12], new byte[17])) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void newActiveKeyStillReadsCiphertextWrittenWithOldVersion() { + String oldKey = Base64.getEncoder().encodeToString(new byte[32]); + byte[] nextBytes = new byte[32]; + java.util.Arrays.fill(nextBytes, (byte) 1); + String nextKey = Base64.getEncoder().encodeToString(nextBytes); + UUID credentialId = UUID.randomUUID(); + var oldCipher = new AesGcmGithubCredentialCipher("v1=" + oldKey, "v1"); + var encrypted = oldCipher.encrypt(credentialId, "example-token"); + var rotatedCipher = new AesGcmGithubCredentialCipher( + "v1=" + oldKey + ",v2=" + nextKey, "v2"); + + assertThat(encrypted.keyVersion()).isEqualTo("v1"); + assertThat(rotatedCipher.decrypt(credentialId, encrypted.keyVersion(), + encrypted.nonce(), encrypted.ciphertext())).isEqualTo("example-token"); + assertThat(rotatedCipher.encrypt(credentialId, "next-token").keyVersion()).isEqualTo("v2"); + assertThatThrownBy(() -> rotatedCipher.decrypt(UUID.randomUUID(), encrypted.keyVersion(), + encrypted.nonce(), encrypted.ciphertext())).isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> new AesGcmGithubCredentialCipher("v2=" + nextKey, "v2") + .decrypt(credentialId, encrypted.keyVersion(), encrypted.nonce(), encrypted.ciphertext())) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void administratorResponseExposesOnlyCredentialMetadata() throws Exception { + String key = Base64.getEncoder().encodeToString(new byte[32]); + var cipher = new AesGcmGithubCredentialCipher("v1=" + key, "v1"); + UUID credentialId = UUID.randomUUID(); + var encrypted = cipher.encrypt(credentialId, "example-token"); + var credential = new GithubSystemCredential(credentialId, "Public reader", + "a".repeat(16), encrypted.keyVersion(), encrypted.nonce(), + encrypted.ciphertext(), "ACTIVE", null, null, Instant.now(), Instant.now()); + + String json = new ObjectMapper().findAndRegisterModules().writeValueAsString( + GithubCredentialResponse.from(credential, cipher, List.of())); + + assertThat(json).contains("fingerprint", "keyAvailable"); + assertThat(json).doesNotContain("example-token", "ciphertext", "nonce", "\"token\""); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubCandidateOriginRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubCandidateOriginRepositoryTest.java new file mode 100644 index 000000000..35836f278 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubCandidateOriginRepositoryTest.java @@ -0,0 +1,454 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatchResult; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCandidate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubCandidateTaskRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryEntryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncIngestionRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncHistoryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubCandidateOriginRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void syntheticThousandRepositoryCandidatesScheduleWithoutDuplicates() { + long namespaceId = platformNamespaceId(); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 20000, global_low = 10000, + namespace_high = 20000, namespace_low = 10000 + WHERE id = TRUE + """); + jdbc.update(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, next_validation_at) + SELECT ?, 2000000 + series.id, 'fixture-owner', 'fixture-repo-' || series.id, ? + FROM generate_series(1, 1000) AS series(id) + """, namespaceId, Timestamp.from(now)); + var candidates = new JdbcGithubCandidateTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + + long started = System.nanoTime(); + int scheduled = 0; + for (int batch = 0; batch < 10; batch++) { + scheduled += candidates.enqueueDue(now.plusSeconds(1), 100); + } + long elapsedMillis = Duration.ofNanos(System.nanoTime() - started).toMillis(); + assertThat(scheduled).isEqualTo(1000); + assertThat(candidates.enqueueDue(now.plusSeconds(1), 100)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND status = 'VALIDATING' + """, Integer.class, namespaceId)).isEqualTo(1000); + assertThat(jdbc.queryForObject(""" + SELECT count(DISTINCT validation_task_id) FROM github_discovery_candidate + WHERE namespace_id = ? + """, Integer.class, namespaceId)).isEqualTo(1000); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE namespace_id = ? AND kind = 'REPOSITORY_SYNC' AND mode = 'FULL' + """, Integer.class, namespaceId)).isEqualTo(1000); + System.out.printf("GitHub candidate scheduling baseline: repositories=1000, batches=10, elapsedMs=%d%n", + elapsedMillis); + } + + @Test + void existingSourceCanBeRediscoveredWhenCandidateBudgetIsFull() { + long namespaceId = platformNamespaceId(); + long repositoryId = 930010L; + jdbc.update(""" + UPDATE github_sync_admission_policy + SET candidate_pending_limit = 1, candidate_per_lease_limit = 1 + WHERE id = TRUE + """); + jdbc.update(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name) + VALUES (?, 930011, 'other-owner', 'unverified') + """, namespaceId); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'public-owner', 'skills', 'PUBLIC') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID taskId = leasedDiscovery(namespaceId, "SEARCH_RULE", ""); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var discovery = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + + assertThat(discovery.recordDiscoveryBatch(taskId, 1, "a".repeat(64), + new GithubDiscoveryBatch(null, "page-1", List.of( + new GithubDiscoveryCandidate(repositoryId, "public-owner", "skills", "main")), + 0, null), "b".repeat(64), Instant.now())) + .isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_discovery_candidate + WHERE namespace_id = ? AND repository_id = ? + """, String.class, namespaceId, repositoryId)).isEqualTo("BOUND"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND status IN ('PENDING', 'VALIDATING') + """, Integer.class, namespaceId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_origin + WHERE source_id = ? AND origin_type = 'SEARCH_RULE' + """, Integer.class, sourceId)).isEqualTo(1); + } + + @Test + void curatedRepositoryDeduplicatesWithinNamespaceButKeepsTeamBindingIndependent() { + long platformId = platformNamespaceId(); + long teamId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, created_by) + VALUES ('github-source-team', 'GitHub Source Team', 'TEAM', 'github-sync-publisher') + RETURNING id + """, Long.class); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var platformEntry = entries.createCuratedRepository("public-owner", "skills"); + var teamEntry = entries.createTeamRepository(teamId, "public-owner", "skills", "NAMESPACE_ONLY"); + var renamedEntry = entries.createCuratedRepository("renamed-owner", "skills"); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var discovery = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + long repositoryId = 930002L; + + recordCuratedRepository(discovery, platformId, platformEntry.id(), repositoryId, + "public-owner"); + recordCuratedRepository(discovery, teamId, teamEntry.id(), repositoryId, + "public-owner"); + recordCuratedRepository(discovery, platformId, renamedEntry.id(), repositoryId, + "renamed-owner"); + + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding WHERE repository_id = ? + """, Integer.class, repositoryId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT visibility FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, String.class, platformId, repositoryId)).isEqualTo("PUBLIC"); + assertThat(jdbc.queryForObject(""" + SELECT visibility FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, String.class, teamId, repositoryId)).isEqualTo("NAMESPACE_ONLY"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_origin origin + JOIN github_source_binding source ON source.id = origin.source_id + WHERE source.namespace_id = ? AND source.repository_id = ? + """, Integer.class, platformId, repositoryId)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_origin origin + JOIN github_source_binding source ON source.id = origin.source_id + WHERE source.namespace_id = ? AND source.repository_id = ? + AND origin.origin_type = 'TEAM_CONFIG' + """, Integer.class, teamId, repositoryId)).isEqualTo(1); + + jdbc.update("UPDATE github_source_binding SET status = 'PAUSED' WHERE namespace_id = ? AND repository_id = ?", + platformId, repositoryId); + recordCuratedRepository(discovery, platformId, platformEntry.id(), repositoryId, "renamed-owner"); + assertThat(jdbc.queryForObject("SELECT status FROM github_source_binding WHERE namespace_id = ? AND repository_id = ?", + String.class, platformId, repositoryId)).isEqualTo("PAUSED"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_source_binding WHERE repository_id = ?", + Integer.class, repositoryId)).isEqualTo(2); + + entries.setTeamRepositoryVisibility(teamId, teamEntry.id(), "PUBLIC"); + assertThat(jdbc.queryForObject(""" + SELECT visibility FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, String.class, teamId, repositoryId)).isEqualTo("PUBLIC"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_origin origin + JOIN github_source_binding source ON source.id = origin.source_id + WHERE source.namespace_id = ? AND source.repository_id = ? + """, Integer.class, platformId, repositoryId)).isEqualTo(2); + } + + private void recordCuratedRepository(JdbcGithubSyncTaskRepository discovery, long namespaceId, + UUID entryId, long repositoryId, String owner) { + UUID taskId = leasedCuratedDiscovery(namespaceId, entryId); + assertThat(discovery.recordDiscoveryBatch(taskId, 1, "a".repeat(64), + new GithubDiscoveryBatch(null, "page-1", List.of( + new GithubDiscoveryCandidate(repositoryId, owner, "skills", "main")), + 0, null), "b".repeat(64), Instant.now())) + .isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + } + + private UUID leasedCuratedDiscovery(long namespaceId, UUID entryId) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'DISCOVERY')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, snapshot_json, + lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'DISCOVERY', 'LEASED', ?, + jsonb_build_object('entryType', 'CURATED_REPOSITORY', + 'ruleId', ?::text, 'candidateBudget', 1), + 1, ?, ?, 'origin-test') + """, taskId, runId, namespaceId, "discovery:" + taskId, + entryId.toString(), "a".repeat(64), + Timestamp.from(Instant.now().plusSeconds(300))); + return taskId; + } + + @Test + void curatedOwnerPrioritySurvivesRemovalOfDiscoveryTaskHits() { + long namespaceId = platformNamespaceId(); + UUID taskId = leasedDiscovery(namespaceId, "CURATED_OWNER", "source-owner"); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var discovery = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + Instant now = Instant.now(); + assertThat(discovery.recordDiscoveryBatch(taskId, 1, "a".repeat(64), + new GithubDiscoveryBatch(null, "page-1", List.of( + new GithubDiscoveryCandidate(930001, "source-owner", "skills", "main")), + 0, null), "b".repeat(64), now)) + .isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + long candidateId = jdbc.queryForObject(""" + SELECT id FROM github_discovery_candidate WHERE repository_id = 930001 + """, Long.class); + assertThat(jdbc.queryForObject(""" + SELECT origin_key FROM github_discovery_candidate_origin + WHERE candidate_id = ? AND origin_type = 'CURATED_OWNER' + """, String.class, candidateId)).isEqualTo("source-owner"); + + jdbc.update("DELETE FROM github_discovery_task_hit WHERE task_id = ?", taskId); + jdbc.update("DELETE FROM github_discovery_batch WHERE task_id = ?", taskId); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50, + namespace_high = 100, namespace_low = 50 WHERE id = TRUE + """); + var candidates = new JdbcGithubCandidateTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + assertThat(candidates.enqueueDue(now.plusSeconds(1), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT task.priority FROM github_sync_task task + JOIN github_discovery_candidate candidate ON candidate.validation_task_id = task.id + WHERE candidate.id = ? + """, Integer.class, candidateId)).isEqualTo(70); + + UUID validationTaskId = jdbc.queryForObject(""" + SELECT validation_task_id FROM github_discovery_candidate WHERE id = ? + """, UUID.class, candidateId); + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + UUID validationToken = UUID.randomUUID(); + String commit = "c".repeat(40); + String digest = "d".repeat(64); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, 930001, ?, 'skills/demo', 1) + """, admissionId, validationTaskId, namespaceId, commit); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 1, ?, 'ACCEPTED', ?) + """, uploadId, admissionId, "origin-test/" + uploadId, digest, digest, + Timestamp.from(now.plusSeconds(3600))); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, + status, validation_token, validation_expires_at) + VALUES (?, ?, ?, ?, ?, 930001, ?, 'skills/demo', ?, 1, ?, + 'VALIDATING', ?, ?) + """, ingestionId, validationTaskId, admissionId, uploadId, namespaceId, + commit, "origin-test/" + uploadId, digest, validationToken, + Timestamp.from(now.plusSeconds(3600))); + var ingestions = new JdbcGithubSyncIngestionRepository(jdbc, + new JdbcGithubSyncSourceCheckpointRepository(jdbc), + new JdbcGithubSyncSchedulePolicyRepository(jdbc), Duration.ofHours(24)); + assertThat(ingestions.validated(ingestionId, validationToken, digest, 0, "{}", now)) + .isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT origin.origin_key FROM github_source_origin origin + JOIN github_source_binding source ON source.id = origin.source_id + WHERE source.namespace_id = ? AND source.repository_id = 930001 + AND origin.origin_type = 'CURATED_OWNER' + """, String.class, namespaceId)).isEqualTo("source-owner"); + + UUID discoveryRunId = jdbc.queryForObject(""" + SELECT run_id FROM github_sync_task WHERE id = ? + """, UUID.class, taskId); + Instant old = now.minus(Duration.ofDays(91)); + jdbc.update(""" + INSERT INTO github_discovery_task_hit + (task_id, repository_id, candidate_id, lease_generation) + VALUES (?, 930001, ?, 1) + """, taskId, candidateId); + jdbc.update(""" + INSERT INTO github_discovery_batch (task_id, checkpoint, request_sha256) + VALUES (?, 'old-page', ?) + """, taskId, "e".repeat(64)); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL + WHERE id = ? + """, Timestamp.from(old), taskId); + jdbc.update("UPDATE github_sync_run SET finished_at = ? WHERE id = ?", + Timestamp.from(old), discoveryRunId); + UUID unfinished = leasedDiscovery(namespaceId, "CURATED_OWNER", "another-owner"); + UUID recent = leasedDiscovery(namespaceId, "CURATED_OWNER", "recent-owner"); + UUID recentRun = jdbc.queryForObject("SELECT run_id FROM github_sync_task WHERE id = ?", + UUID.class, recent); + Instant recentFinished = now.minus(Duration.ofDays(89)); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL + WHERE id = ? + """, Timestamp.from(recentFinished), recent); + jdbc.update("UPDATE github_sync_run SET finished_at = ? WHERE id = ?", + Timestamp.from(recentFinished), recentRun); + var cleaned = new JdbcGithubSyncHistoryRepository(jdbc) + .deleteExpiredDiscoveryHistory(now.minus(Duration.ofDays(90)), 10); + assertThat(cleaned.tasks()).isEqualTo(1); + assertThat(cleaned.runs()).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE id = ?", + Integer.class, unfinished)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE id = ?", + Integer.class, recent)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ?", + Integer.class, taskId)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_batch WHERE task_id = ?", + Integer.class, taskId)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate_origin", + Integer.class)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_origin origin + JOIN github_source_binding source ON source.id = origin.source_id + WHERE source.namespace_id = ? AND source.repository_id = 930001 + """, Integer.class, namespaceId)).isEqualTo(1); + } + + private UUID leasedDiscovery(long namespaceId, String entryType, String owner) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'DISCOVERY')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, snapshot_json, + lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'DISCOVERY', 'LEASED', ?, + jsonb_build_object('entryType', ?::text, 'ruleId', ?::text, + 'owner', ?::text, 'candidateBudget', 10), + 1, ?, ?, 'origin-test') + """, taskId, runId, namespaceId, "discovery:" + taskId, + entryType, UUID.randomUUID().toString(), owner, + "a".repeat(64), Timestamp.from(Instant.now().plusSeconds(300))); + return taskId; + } + + @Test + void candidateWithoutValidSkillStaysUnboundAndIsRetriedAtLowFrequency() { + long namespaceId = platformNamespaceId(); + long repositoryId = 930009L; + Instant now = Instant.now(); + long candidateId = jdbc.queryForObject(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, + next_validation_at) + VALUES (?, ?, 'public-owner', 'no-skill-repo', ?) RETURNING id + """, Long.class, namespaceId, repositoryId, Timestamp.from(now)); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50, + namespace_high = 50, namespace_low = 25 WHERE id = TRUE + """); + var candidates = new JdbcGithubCandidateTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + + assertThat(candidates.enqueueDue(now.plusSeconds(1), 10)).isEqualTo(1); + UUID validationTaskId = jdbc.queryForObject(""" + SELECT validation_task_id FROM github_discovery_candidate WHERE id = ? + """, UUID.class, candidateId); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_discovery_candidate WHERE id = ? + """, String.class, candidateId)).isEqualTo("VALIDATING"); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ? WHERE id = ? + """, Timestamp.from(now.plusSeconds(2)), validationTaskId); + + assertThat(candidates.reconcileCompleted(now.plusSeconds(3), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_discovery_candidate WHERE id = ? + """, String.class, candidateId)).isEqualTo("REJECTED"); + assertThat(jdbc.queryForObject(""" + SELECT last_error_code FROM github_discovery_candidate WHERE id = ? + """, String.class, candidateId)).isEqualTo("NO_VALID_SKILL"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, Integer.class, namespaceId, repositoryId)).isZero(); + Instant retryAt = jdbc.queryForObject(""" + SELECT next_validation_at FROM github_discovery_candidate WHERE id = ? + """, Timestamp.class, candidateId).toInstant(); + assertThat(retryAt).isAfter(now.plus(Duration.ofDays(6))); + assertThat(candidates.enqueueDue(now.plus(Duration.ofDays(1)), 10)).isZero(); + assertThat(candidates.enqueueDue(retryAt.plusSeconds(1), 10)).isEqualTo(1); + } + + private long platformNamespaceId() { + return jdbc.queryForObject("SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'", + Long.class); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryHistoryCleanupTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryHistoryCleanupTest.java new file mode 100644 index 000000000..5f4175d7b --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryHistoryCleanupTest.java @@ -0,0 +1,252 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncHistoryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubRepositoryHistoryCleanupTest { + private static final String COMMIT = "a".repeat(40); + private static final String DIGEST = "b".repeat(64); + + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void sourceAndRunSummariesShowCurrentPackageFailureAfterAcquisitionCompletes() { + long namespaceId = platformNamespaceId(); + long sourceId = source(namespaceId, 950003); + Task task = completedTask(namespaceId, sourceId, 950003, Instant.now()); + UUID ingestionId = ingestion(task.id(), namespaceId, 950003, "skills/demo", "BLOCKED"); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET last_error_code = 'SLUG_CONFLICT' WHERE id = ? + """, ingestionId); + var management = new GithubSyncManagementQueryRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()), + new JdbcGithubSyncCapacityGate(jdbc)); + + assertThat(management.runs(namespaceId, 0, 10).items()) + .singleElement().satisfies(run -> { + assertThat(run.id()).isEqualTo(task.runId()); + assertThat(run.lastErrorCode()).isEqualTo("SLUG_CONFLICT"); + assertThat(run.blockedPackages()).isEqualTo(1); + }); + jdbc.update("UPDATE github_sync_package_ingestion SET status = 'FAILED' WHERE id = ?", ingestionId); + assertThat(management.sources(namespaceId, 0, 10).items()) + .singleElement().satisfies(source -> { + assertThat(source.blockedPackages()).isZero(); + assertThat(source.failedPackages()).isEqualTo(1); + }); + assertThat(management.runs(namespaceId, 0, 10).items()) + .singleElement().satisfies(run -> assertThat(run.failedPackages()).isEqualTo(1)); + Task retry = completedTask(namespaceId, sourceId, 950003, Instant.now().plusSeconds(1)); + UUID recoveredId = ingestion(retry.id(), namespaceId, 950003, + "skills/demo", "READY_FOR_REVIEW"); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET created_at = CURRENT_TIMESTAMP + INTERVAL '1 second' WHERE id = ? + """, recoveredId); + assertThat(management.sources(namespaceId, 0, 10).items()) + .singleElement().satisfies(source -> { + assertThat(source.failedPackages()).isZero(); + assertThat(source.pendingReviewPackages()).isEqualTo(1); + }); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'READY_FOR_REVIEW' WHERE id = ? + """, ingestionId); + assertThat(management.sources(namespaceId, 0, 10).items()) + .singleElement().satisfies(source -> { + assertThat(source.failedPackages()).isZero(); + assertThat(source.pendingReviewPackages()).isEqualTo(1); + }); + assertThat(management.runs(namespaceId, 0, 10).items()) + .filteredOn(run -> run.id().equals(task.runId())) + .singleElement().satisfies(run -> assertThat(run.lastErrorCode()).isNull()); + } + + @Test + void keepsAuditAndLatestSourcePathWhileRemovingExpiredOperationalDetails() { + Instant now = Instant.now(); + long namespaceId = platformNamespaceId(); + long sourceId = source(namespaceId, 950001); + Task old = completedTask(namespaceId, sourceId, 950001, now.minus(Duration.ofDays(91))); + Task recent = completedTask(namespaceId, sourceId, 950001, now.minus(Duration.ofDays(1))); + path(old.id(), "skills/demo"); + path(old.id(), "skills/only-old"); + path(recent.id(), "skills/demo"); + UUID oldIngestion = ingestion(old.id(), namespaceId, 950001, "skills/demo", "APPROVED"); + long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status) + VALUES (?, 'history-demo', 'github-sync-publisher', 'PUBLIC', 'ACTIVE') RETURNING id + """, Long.class, namespaceId); + long versionId = jdbc.queryForObject(""" + INSERT INTO skill_version + (skill_id, version, status, published_at, github_sync_ingestion_id) + VALUES (?, '1.0.0', 'PUBLISHED', CURRENT_TIMESTAMP, ?) RETURNING id + """, Long.class, skillId, oldIngestion); + jdbc.update(""" + UPDATE github_sync_package_ingestion SET published_version_id = ? WHERE id = ? + """, versionId, oldIngestion); + + long pendingSource = source(namespaceId, 950002); + Task pending = completedTask(namespaceId, pendingSource, 950002, + now.minus(Duration.ofDays(91))); + path(pending.id(), "skills/pending"); + ingestion(pending.id(), namespaceId, 950002, "skills/pending", "READY_FOR_REVIEW"); + + var cleaned = new JdbcGithubSyncHistoryRepository(jdbc).purgeExpiredRepositoryDetails( + now.minus(Duration.ofDays(90)), now, 10); + + assertThat(cleaned.tasks()).isEqualTo(1); + assertThat(cleaned.runs()).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT details_purged_at IS NOT NULL FROM github_sync_task WHERE id = ? + """, Boolean.class, old.id())).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT jsonb_exists(snapshot_json, 'ignorePatterns') + FROM github_sync_task WHERE id = ? + """, Boolean.class, old.id())).isFalse(); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'owner' FROM github_sync_task WHERE id = ? + """, String.class, old.id())).isEqualTo("oldowner"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result WHERE task_id = ? + """, Integer.class, old.id())).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT skill_path FROM github_sync_path_result WHERE task_id = ? + """, String.class, old.id())).isEqualTo("skills/only-old"); + assertThat(jdbc.queryForObject(""" + SELECT details_purged_at IS NULL FROM github_sync_task WHERE id = ? + """, Boolean.class, recent.id())).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT details_purged_at IS NULL FROM github_sync_task WHERE id = ? + """, Boolean.class, pending.id())).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT details_purged_at IS NOT NULL FROM github_sync_run WHERE id = ? + """, Boolean.class, old.runId())).isTrue(); + assertThat(new GithubSkillSourceQueryRepository(jdbc, new ObjectMapper()) + .findPublishedVersion(versionId)).isPresent(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion WHERE id = ? AND task_id = ? + """, Integer.class, oldIngestion, old.id())).isEqualTo(1); + var management = new GithubSyncManagementQueryRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()), + new JdbcGithubSyncCapacityGate(jdbc)); + assertThat(management.runs(namespaceId, 0, 10).items()) + .extracting(run -> run.id()).contains(recent.runId(), pending.runId()) + .doesNotContain(old.runId()); + assertThat(management.skillPaths(namespaceId, sourceId, 0, 10).items()) + .extracting(path -> path.skillPath()) + .containsExactlyInAnyOrder("skills/demo", "skills/only-old"); + } + + private Task completedTask(long namespaceId, long sourceId, long repositoryId, Instant completedAt) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, status, dedupe_key, + snapshot_json, fixed_commit_sha, completed_at) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'ACQUISITION_COMPLETE', ?, + jsonb_build_object('repositoryId', ?::bigint, 'owner', 'oldowner', + 'name', 'oldrepo', 'ignorePatterns', + jsonb_build_array('docs')), + ?, ?) + """, taskId, runId, namespaceId, sourceId, + "repository:" + repositoryId, repositoryId, COMMIT, Timestamp.from(completedAt)); + jdbc.update("UPDATE github_sync_run SET finished_at = ? WHERE id = ?", + Timestamp.from(completedAt), runId); + jdbc.update("UPDATE github_sync_task SET created_at = ? WHERE id = ?", + Timestamp.from(completedAt), taskId); + return new Task(taskId, runId); + } + + private void path(UUID taskId, String skillPath) { + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, sha256) + VALUES (?, ?, ?, 'ACCEPTED', ?) + """, taskId, COMMIT, skillPath, DIGEST); + } + + private UUID ingestion(UUID taskId, long namespaceId, long repositoryId, + String skillPath, String status) { + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, ?, ?, ?, 1) + """, admissionId, taskId, namespaceId, repositoryId, COMMIT, skillPath); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 1, ?, 'CLEANED', CURRENT_TIMESTAMP) + """, uploadId, admissionId, "history/" + uploadId, DIGEST, DIGEST); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, status, + object_cleaned_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, CURRENT_TIMESTAMP) + """, ingestionId, taskId, admissionId, uploadId, namespaceId, repositoryId, + COMMIT, skillPath, "history/" + uploadId, DIGEST, status); + return ingestionId; + } + + private long source(long namespaceId, long repositoryId) { + return jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'oldowner', 'oldrepo', 'PUBLIC') RETURNING id + """, Long.class, namespaceId, repositoryId); + } + + private long platformNamespaceId() { + return jdbc.queryForObject("SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'", + Long.class); + } + + private record Task(UUID id, UUID runId) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryMetadataRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryMetadataRepositoryTest.java new file mode 100644 index 000000000..0bc4d789f --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubRepositoryMetadataRepositoryTest.java @@ -0,0 +1,184 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubRepositoryMetadataResult; +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryEntryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubRepositoryMetadataRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void stableIdUpdatesRegisteredSourceButRejectsStaleLeaseAndReusedName() { + long namespaceId = platformNamespaceId(); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 920001, 'old-owner', 'old-name', 'PUBLIC') RETURNING id + """, Long.class, namespaceId); + UUID taskId = leasedTask(namespaceId, sourceId, 920001); + var repository = repository(); + Instant now = Instant.now(); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var entry = entries.createCuratedRepository("old-owner", "old-name"); + jdbc.update(""" + UPDATE github_discovery_entry SET repository_id = ?, next_due_at = ? WHERE id = ? + """, 920001L, Timestamp.from(now.minusSeconds(1)), entry.id()); + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + VALUES (?, 'CURATED_REPOSITORY', ?) + """, sourceId, entry.id().toString()); + + assertThat(repository.updateRepositoryMetadata(taskId, 1, "a".repeat(64), 920002, + "wrong-owner", "reused-name", "main", now)) + .isEqualTo(GithubRepositoryMetadataResult.REPOSITORY_MISMATCH); + assertThat(repository.updateRepositoryMetadata(taskId, 1, "b".repeat(64), 920001, + "new-owner", "new-name", "main", now)) + .isEqualTo(GithubRepositoryMetadataResult.LEASE_STALE); + assertThat(repository.updateRepositoryMetadata(taskId, 1, "a".repeat(64), 920001, + "new-owner", "new-name", "main", now)) + .isEqualTo(GithubRepositoryMetadataResult.UPDATED); + assertThat(jdbc.queryForObject(""" + SELECT owner_login || '/' || repository_name FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isEqualTo("new-owner/new-name"); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'owner' FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("new-owner"); + assertThat(entries.enqueueDue(now, 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'owner' FROM github_sync_task + WHERE dedupe_key = ? AND status = 'QUEUED' + """, String.class, "discovery:" + entry.id())).isEqualTo("new-owner"); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'repositoryName' FROM github_sync_task + WHERE dedupe_key = ? AND status = 'QUEUED' + """, String.class, "discovery:" + entry.id())).isEqualTo("new-name"); + } + + @Test + void stableIdUpdatesCandidateBeforeItBecomesARegisteredSource() { + long namespaceId = platformNamespaceId(); + UUID taskId = leasedTask(namespaceId, null, 920003); + jdbc.update(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, + status, validation_task_id, next_validation_at) + VALUES (?, 920003, 'old-owner', 'old-name', 'VALIDATING', ?, CURRENT_TIMESTAMP) + """, namespaceId, taskId); + + assertThat(repository().updateRepositoryMetadata(taskId, 1, "a".repeat(64), 920003, + "new-owner", "new-name", "main", Instant.now())) + .isEqualTo(GithubRepositoryMetadataResult.UPDATED); + assertThat(jdbc.queryForObject(""" + SELECT owner_login || '/' || repository_name FROM github_discovery_candidate + WHERE validation_task_id = ? + """, String.class, taskId)).isEqualTo("new-owner/new-name"); + } + + @Test + void pinnedRepositorySnapshotUsesOnlyItsNamespaceSourceCoordinate() { + long platformId = platformNamespaceId(); + long teamId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, created_by) + VALUES ('github-coordinate-team', 'GitHub Coordinate Team', 'TEAM', 'github-sync-publisher') + RETURNING id + """, Long.class); + long repositoryId = 920004L; + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'platform-owner', 'platform-name', 'PUBLIC'), + (?, ?, 'team-owner', 'team-name', 'NAMESPACE_ONLY') + """, platformId, repositoryId, teamId, repositoryId); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var platformEntry = entries.createCuratedRepository("old-owner", "old-name"); + var teamEntry = entries.createTeamRepository(teamId, "old-owner", "old-name", "NAMESPACE_ONLY"); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_discovery_entry + SET repository_id = ?, next_due_at = ? WHERE id IN (?, ?) + """, repositoryId, Timestamp.from(now.minusSeconds(1)), + platformEntry.id(), teamEntry.id()); + + assertThat(entries.enqueueDue(now, 10)).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'owner' FROM github_sync_task + WHERE namespace_id = ? AND dedupe_key = ? + """, String.class, platformId, "discovery:" + platformEntry.id())) + .isEqualTo("platform-owner"); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'owner' FROM github_sync_task + WHERE namespace_id = ? AND dedupe_key = ? + """, String.class, teamId, "discovery:" + teamEntry.id())) + .isEqualTo("team-owner"); + } + + private UUID leasedTask(long namespaceId, Long sourceId, long repositoryId) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, status, dedupe_key, + snapshot_json, lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', ?::bigint, 'owner', 'old-owner', + 'name', 'old-name'), 1, ?, ?, 'metadata-test') + """, taskId, runId, namespaceId, sourceId, "repository:" + repositoryId, + repositoryId, "a".repeat(64), Timestamp.from(Instant.now().plusSeconds(300))); + return taskId; + } + + private JdbcGithubSyncTaskRepository repository() { + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + return new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + } + + private long platformNamespaceId() { + return jdbc.queryForObject("SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'", + Long.class); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncAdmissionConcurrencyTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncAdmissionConcurrencyTest.java new file mode 100644 index 000000000..20e9ae78b --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncAdmissionConcurrencyTest.java @@ -0,0 +1,237 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubPackageAdmissionResult; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskMode; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceTaskRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubSyncAdmissionConcurrencyTest { + private static final String COMMIT = "a".repeat(40); + private static final String LEASE_TOKEN_HASH = "b".repeat(64); + + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void concurrentReservationsRespectNamespaceAndGlobalHighWatermarks() throws Exception { + var dataSource = new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration").load().migrate(); + var jdbc = new JdbcTemplate(dataSource); + var transactions = new TransactionTemplate(new DataSourceTransactionManager(dataSource)); + long catalogId = jdbc.queryForObject( + "SELECT id FROM namespace WHERE slug = 'github'", Long.class); + long teamId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, created_by) + VALUES ('admission-team', 'Admission Team', 'TEAM', 'github-sync-publisher') + RETURNING id + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 3, global_low = 1, namespace_high = 1, namespace_low = 0 + WHERE id = TRUE + """); + UUID first = leasedTask(jdbc, catalogId, 940001); + UUID second = leasedTask(jdbc, catalogId, 940002); + + List namespaceResults = concurrentAdmit( + dataSource, transactions, first, second); + assertThat(namespaceResults.stream().filter(result -> + result.status() == GithubPackageAdmissionResult.Status.ADMITTED)).hasSize(1); + assertThat(namespaceResults.stream().filter(result -> + result.status() == GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE)).hasSize(1); + assertThat(jdbc.queryForObject(""" + SELECT paused FROM github_sync_namespace_admission_state WHERE namespace_id = ? + """, Boolean.class, catalogId)).isTrue(); + + UUID teamTask = leasedTask(jdbc, teamId, 940003); + var teamResult = transactions.execute(status -> repository(new JdbcTemplate(dataSource)) + .admitPackage(teamTask, 1, LEASE_TOKEN_HASH, COMMIT, "skills/team", Instant.now())); + assertThat(teamResult.status()).isEqualTo(GithubPackageAdmissionResult.Status.ADMITTED); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE status = 'ACTIVE' + """, Integer.class)).isEqualTo(2); + jdbc.update(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, status) + VALUES (?, 940010, 'fixture', 'pending', 'PENDING'), + (?, 940011, 'fixture', 'rejected', 'REJECTED'), + (?, 940012, 'fixture', 'team-pending', 'PENDING') + """, catalogId, catalogId, teamId); + var overview = new GithubSyncManagementQueryRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()), + new JdbcGithubSyncCapacityGate(jdbc)).admissionSnapshot(); + assertThat(overview.globalUnresolvedCount()).isEqualTo(2); + assertThat(overview.platformUnresolvedCount()).isEqualTo(1); + assertThat(overview.platformPendingCandidateCount()).isEqualTo(1); + assertThat(overview.platformPaused()).isTrue(); + + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE status = 'ACTIVE'"); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 1, global_low = 0, namespace_high = 10, namespace_low = 0, + global_paused = FALSE WHERE id = TRUE + """); + jdbc.update("UPDATE github_sync_namespace_admission_state SET paused = FALSE"); + UUID globalFirst = leasedTask(jdbc, catalogId, 940004); + UUID globalSecond = leasedTask(jdbc, teamId, 940005); + + List globalResults = concurrentAdmit( + dataSource, transactions, globalFirst, globalSecond); + assertThat(globalResults.stream().filter(result -> + result.status() == GithubPackageAdmissionResult.Status.ADMITTED)).hasSize(1); + assertThat(globalResults.stream().filter(result -> + result.status() == GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE)).hasSize(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE status = 'ACTIVE' + """, Integer.class)).isEqualTo(1); + + assertThat(jdbc.queryForObject(""" + SELECT global_paused FROM github_sync_admission_policy WHERE id = TRUE + """, Boolean.class)).isTrue(); + + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 940006, 'public-owner', 'registered', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, teamId); + UUID registeredTask = queuedTask(jdbc, teamId, sourceId, 940006); + UUID candidateTask = queuedTask(jdbc, teamId, null, 940007); + var tasks = repository(new JdbcTemplate(dataSource)); + Instant claimTime = Instant.now(); + var headOnly = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "head-worker", + "c".repeat(64), claimTime, claimTime.plusSeconds(300)); + assertThat(headOnly).isPresent(); + assertThat(headOnly.orElseThrow().taskId()).isEqualTo(registeredTask); + assertThat(headOnly.orElseThrow().mode()).isEqualTo(GithubSyncTaskMode.HEAD_ONLY); + assertThat(tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "candidate-worker", + "d".repeat(64), claimTime, claimTime.plusSeconds(300))).isEmpty(); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_task WHERE id = ?", + String.class, candidateTask)).isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission WHERE status = 'ACTIVE' + """, Integer.class)).isEqualTo(1); + + String newestCommit = "e".repeat(40); + assertThat(tasks.completeHeadOnly(registeredTask, 1, "c".repeat(64), + newestCommit, claimTime.plusSeconds(1), claimTime.plusSeconds(86400))).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT last_observed_commit FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isEqualTo(newestCommit); + assertThat(jdbc.queryForObject(""" + SELECT last_fully_processed_commit FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isNull(); + + jdbc.update("UPDATE github_sync_package_admission SET status = 'RELEASED' WHERE status = 'ACTIVE'"); + var sourceScheduler = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + assertThat(sourceScheduler.enqueueDue(claimTime.plusSeconds(2), 10)).isEqualTo(1); + var resumed = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "resumed-worker", + "f".repeat(64), claimTime.plusSeconds(3), claimTime.plusSeconds(30 * 60)).orElseThrow(); + assertThat(resumed.mode()).isEqualTo(GithubSyncTaskMode.FULL); + assertThat(resumed.sourceId()).isEqualTo(sourceId); + Instant candidateRetry = claimTime.plusSeconds(20 * 60); + jdbc.update(""" + UPDATE github_sync_task SET lease_expires_at = ? WHERE status = 'LEASED' + """, Timestamp.from(candidateRetry.plusSeconds(3600))); + var recheckedCandidate = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, + "candidate-recheck-worker", "1".repeat(64), candidateRetry, + candidateRetry.plusSeconds(300)).orElseThrow(); + assertThat(recheckedCandidate.taskId()).isEqualTo(candidateTask); + assertThat(recheckedCandidate.mode()).isEqualTo(GithubSyncTaskMode.FULL); + assertThat(recheckedCandidate.fixedCommitSha()).isNull(); + } + + private List concurrentAdmit(DriverManagerDataSource dataSource, + TransactionTemplate transactions, UUID first, UUID second) throws Exception { + var start = new CountDownLatch(1); + try (var executor = Executors.newFixedThreadPool(2)) { + var firstResult = executor.submit(() -> { + start.await(); + return transactions.execute(status -> repository(new JdbcTemplate(dataSource)) + .admitPackage(first, 1, LEASE_TOKEN_HASH, COMMIT, "skills/first", Instant.now())); + }); + var secondResult = executor.submit(() -> { + start.await(); + return transactions.execute(status -> repository(new JdbcTemplate(dataSource)) + .admitPackage(second, 1, LEASE_TOKEN_HASH, COMMIT, "skills/second", Instant.now())); + }); + start.countDown(); + return List.of(firstResult.get(30, TimeUnit.SECONDS), + secondResult.get(30, TimeUnit.SECONDS)); + } + } + + private UUID leasedTask(JdbcTemplate jdbc, long namespaceId, long repositoryId) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, + snapshot_json, lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', ?::text, 'fixedCommitSha', ?::text), + 1, ?, ?, 'admission-worker') + """, taskId, runId, namespaceId, "repository:" + repositoryId, + Long.toString(repositoryId), COMMIT, LEASE_TOKEN_HASH, + Timestamp.from(Instant.now().plusSeconds(300))); + return taskId; + } + + private UUID queuedTask(JdbcTemplate jdbc, long namespaceId, Long sourceId, long repositoryId) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, status, + dedupe_key, snapshot_json) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'QUEUED', ?, + jsonb_build_object('repositoryId', ?::text)) + """, taskId, runId, namespaceId, sourceId, "repository:" + repositoryId, + Long.toString(repositoryId)); + return taskId; + } + + private JdbcGithubSyncTaskRepository repository(JdbcTemplate jdbc) { + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + return new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncLeaseConcurrencyTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncLeaseConcurrencyTest.java new file mode 100644 index 000000000..4a46ac4bf --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncLeaseConcurrencyTest.java @@ -0,0 +1,137 @@ +package com.iflytek.skillhub.repository; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatchResult; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCandidate; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskLease; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubSyncLeaseConcurrencyTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void concurrentWorkersClaimOnlyOneLeaseAndDiscoveryReplayDoesNotDuplicateRows() throws Exception { + var dataSource = new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration").load().migrate(); + var jdbc = new JdbcTemplate(dataSource); + long namespaceId = jdbc.queryForObject( + "SELECT id FROM namespace WHERE slug = 'github'", Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID ruleId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'DISCOVERY')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, snapshot_json) + VALUES (?, ?, ?, 'DISCOVERY', 'QUEUED', ?, + jsonb_build_object('entryType', 'SEARCH_RULE', 'ruleId', ?::text, + 'candidateBudget', 10)) + """, taskId, runId, namespaceId, "discovery:" + taskId, ruleId.toString()); + + var firstRepository = repository(new JdbcTemplate(dataSource)); + var secondRepository = repository(new JdbcTemplate(dataSource)); + Instant now = Instant.now(); + var start = new CountDownLatch(1); + try (var executor = Executors.newFixedThreadPool(2)) { + var first = executor.submit(() -> { + start.await(); + return firstRepository.claim(GithubSyncTaskKind.DISCOVERY, "worker-a", + "a".repeat(64), now, now.plusSeconds(300), platform(java.util.List.of())); + }); + var second = executor.submit(() -> { + start.await(); + return secondRepository.claim(GithubSyncTaskKind.DISCOVERY, "worker-b", + "b".repeat(64), now, now.plusSeconds(300), platform(java.util.List.of())); + }); + start.countDown(); + Optional firstLease = first.get(30, TimeUnit.SECONDS); + Optional secondLease = second.get(30, TimeUnit.SECONDS); + assertThat(firstLease.isPresent() == secondLease.isPresent()).isFalse(); + GithubSyncTaskLease lease = firstLease.orElseGet(secondLease::orElseThrow); + String token = firstLease.isPresent() ? "a".repeat(64) : "b".repeat(64); + assertThat(lease.taskId()).isEqualTo(taskId); + assertThat(lease.leaseGeneration()).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE id = ? AND status = 'LEASED' AND lease_generation = 1 + """, Integer.class, taskId)).isEqualTo(1); + + var batch = new GithubDiscoveryBatch(null, "page-1", List.of( + new GithubDiscoveryCandidate(930003, "public-owner", "skills", "main")), + 0, null); + assertThat(firstRepository.recordDiscoveryBatch(taskId, 1, token, batch, + "c".repeat(64), now)).isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + assertThat(secondRepository.recordDiscoveryBatch(taskId, 1, token, batch, + "c".repeat(64), now)).isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_batch WHERE task_id = ?", + Integer.class, taskId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND repository_id = 930003 + """, Integer.class, namespaceId)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(1); + + Instant afterExpiry = now.plusSeconds(301); + assertThat(firstRepository.renew(taskId, 1, token, afterExpiry, + afterExpiry.plusSeconds(300))).isEmpty(); + assertThat(firstRepository.recordDiscoveryBatch(taskId, 1, token, batch, + "c".repeat(64), afterExpiry)).isEqualTo(GithubDiscoveryBatchResult.LEASE_STALE); + var replacement = secondRepository.claim(GithubSyncTaskKind.DISCOVERY, "worker-replacement", + "d".repeat(64), afterExpiry, afterExpiry.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + assertThat(replacement.taskId()).isEqualTo(taskId); + assertThat(replacement.leaseGeneration()).isEqualTo(2); + assertThat(firstRepository.completeDiscovery(taskId, 1, token, "page-1", true, + null, null, afterExpiry.plusSeconds(1))) + .isEqualTo(GithubDiscoveryCompletionResult.LEASE_STALE); + assertThat(secondRepository.recordDiscoveryBatch(taskId, 2, "d".repeat(64), batch, + "c".repeat(64), afterExpiry.plusSeconds(1))) + .isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + assertThat(secondRepository.completeDiscovery(taskId, 2, "d".repeat(64), + "page-1", true, null, null, afterExpiry.plusSeconds(1))) + .isEqualTo(GithubDiscoveryCompletionResult.ACQUISITION_COMPLETE); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_batch WHERE task_id = ?", + Integer.class, taskId)).isEqualTo(1); + } + } + + private JdbcGithubSyncTaskRepository repository(JdbcTemplate jdbc) { + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + return new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncManualBindingRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncManualBindingRepositoryTest.java new file mode 100644 index 000000000..994d2955d --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncManualBindingRepositoryTest.java @@ -0,0 +1,287 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncManualBindingRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Import(JdbcGithubSyncManualBindingRepository.class) +@Testcontainers +class GithubSyncManualBindingRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void configurePostgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired + private JdbcTemplate jdbc; + + @Autowired + private JdbcGithubSyncManualBindingRepository bindings; + + @Test + void bindsOnlyTheChosenNamespaceSkillAndRetriesTheBlockedIngestion() { + Fixture fixture = fixture("manual-a", 1001L, "agents/demo", "demo"); + + var binding = bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId()); + + assertThat(binding.skillId()).isEqualTo(fixture.skillId()); + assertThat(binding.skillPath()).isEqualTo("agents/demo"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, fixture.ingestionId())).isEqualTo("VALIDATED"); + assertThat(jdbc.queryForObject(""" + SELECT latest_content_sha256 FROM github_source_skill_binding + WHERE namespace_id = ? AND skill_id = ? + """, String.class, fixture.namespaceId(), fixture.skillId())).isNull(); + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId())); + } + + @Test + void rejectsAnotherNamespaceAndAnUnrelatedSkillIdentity() { + Fixture fixture = fixture("manual-b", 1002L, "agents/demo", "demo"); + Long otherSkillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status) + VALUES (?, 'other', 'manual-owner', 'NAMESPACE_ONLY', 'ACTIVE') RETURNING id + """, Long.class, fixture.namespaceId()); + + assertThrows(DomainNotFoundException.class, + () -> bindings.bind(fixture.namespaceId() + 1000, fixture.ingestionId(), fixture.skillId())); + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), otherSkillId)); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_source_skill_binding WHERE namespace_id = ? + """, Integer.class, fixture.namespaceId())).isZero(); + } + + @Test + void confirmsMissingPathOnTheSameSourceAndRetriesIntoExistingSkill() { + PathFixture fixture = pathFixture("manual-path-a", 1101L); + + var result = bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId()); + + assertThat(result.previousPath()).isEqualTo("old/demo"); + assertThat(result.skillPath()).isEqualTo("new/demo"); + assertThat(jdbc.queryForObject(""" + SELECT skill_path FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, fixture.skillId())).isEqualTo("new/demo"); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, fixture.ingestionId())).isEqualTo("VALIDATED"); + assertThat(jdbc.queryForObject(""" + SELECT latest_content_sha256 FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, fixture.skillId())).isEqualTo("b".repeat(64)); + } + + @Test + void pathConfirmationRejectsOtherNamespaceAndOldPathStillPresent() { + PathFixture fixture = pathFixture("manual-path-b", 1102L); + Fixture other = fixture("manual-other", 1103L, "other/demo", "demo"); + + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), other.skillId())); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/demo', 'UNCHANGED') + """, fixture.taskId(), "a".repeat(40)); + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId())); + assertThat(jdbc.queryForObject(""" + SELECT skill_path FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, fixture.skillId())).isEqualTo("old/demo"); + } + + @Test + void pathConfirmationRejectsPartialScan() { + PathFixture fixture = pathFixture("manual-path-c", 1104L); + jdbc.update("UPDATE github_sync_task SET status = 'PARTIAL' WHERE id = ?", fixture.taskId()); + + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId())); + } + + @Test + void changedContentKeepsPreviousFingerprintUntilPublication() { + PathFixture fixture = pathFixture("manual-path-d", 1105L); + jdbc.update("UPDATE github_sync_package_ingestion SET content_sha256 = ? WHERE id = ?", + "d".repeat(64), fixture.ingestionId()); + + bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId()); + + assertThat(jdbc.queryForObject(""" + SELECT latest_content_sha256 FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, fixture.skillId())).isEqualTo("b".repeat(64)); + assertThat(jdbc.queryForObject(""" + SELECT latest_ingestion_id FROM github_source_skill_binding WHERE skill_id = ? + """, UUID.class, fixture.skillId())).isNull(); + } + + @Test + void pathConfirmationRejectsAnotherSourceInSameNamespace() { + PathFixture fixture = pathFixture("manual-path-e", 1106L); + Long otherSkillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status, github_sync_managed) + VALUES (?, 'other', 'github-sync-publisher', 'NAMESPACE_ONLY', 'ACTIVE', TRUE) + RETURNING id + """, Long.class, fixture.namespaceId()); + Long otherSourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 999999, 'other', 'skills', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, fixture.namespaceId()); + jdbc.update(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id, + latest_content_sha256) + VALUES (?, ?, 999999, 'old/other', ?, ?) + """, otherSourceId, fixture.namespaceId(), otherSkillId, "b".repeat(64)); + + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), otherSkillId)); + } + + @Test + void pathConfirmationRejectsAConflictFromAnOlderFullScan() { + PathFixture fixture = pathFixture("manual-path-f", 1107L); + Long sourceId = jdbc.queryForObject(""" + SELECT source_id FROM github_sync_task WHERE id = ? + """, Long.class, fixture.taskId()); + jdbc.update("UPDATE github_sync_task SET created_at = CURRENT_TIMESTAMP - INTERVAL '1 day' WHERE id = ?", + fixture.taskId()); + UUID runId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, fixture.namespaceId()); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, snapshot_json) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'new-scan', '{}'::jsonb) + """, UUID.randomUUID(), runId, fixture.namespaceId(), sourceId); + + assertThrows(DomainBadRequestException.class, + () -> bindings.bind(fixture.namespaceId(), fixture.ingestionId(), fixture.skillId())); + } + + private PathFixture pathFixture(String suffix, long repositoryId) { + Fixture fixture = fixture(suffix, repositoryId, "new/demo", "demo"); + Long sourceId = jdbc.queryForObject(""" + SELECT id FROM github_source_binding WHERE namespace_id = ? AND repository_id = ? + """, Long.class, fixture.namespaceId(), repositoryId); + UUID taskId = jdbc.queryForObject(""" + SELECT task_id FROM github_sync_package_ingestion WHERE id = ? + """, UUID.class, fixture.ingestionId()); + jdbc.update(""" + UPDATE skill SET github_sync_managed = TRUE, owner_id = 'github-sync-publisher' + WHERE id = ? + """, fixture.skillId()); + jdbc.update(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id, + latest_content_sha256) + VALUES (?, ?, ?, 'old/demo', ?, ?) + """, sourceId, fixture.namespaceId(), repositoryId, fixture.skillId(), "b".repeat(64)); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET last_error_code = 'PATH_MOVE_AMBIGUOUS', target_skill_slug = NULL + WHERE id = ? + """, fixture.ingestionId()); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', fixed_commit_sha = ? WHERE id = ? + """, "a".repeat(40), taskId); + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, ingestion_id) + VALUES (?, ?, 'new/demo', 'ACCEPTED', ?) + """, taskId, "a".repeat(40), fixture.ingestionId()); + return new PathFixture(fixture.namespaceId(), fixture.ingestionId(), + fixture.skillId(), taskId); + } + + private Fixture fixture(String suffix, long repositoryId, String path, String slug) { + jdbc.update(""" + INSERT INTO user_account (id, display_name) VALUES ('manual-owner', 'Owner') + ON CONFLICT (id) DO NOTHING + """); + Long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES (?, 'Team', 'TEAM', 'ACTIVE', 'manual-owner') RETURNING id + """, Long.class, suffix); + Long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status) + VALUES (?, ?, 'manual-owner', 'NAMESPACE_ONLY', 'ACTIVE') RETURNING id + """, Long.class, namespaceId, slug); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'upstream', 'skills', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, namespaceId, repositoryId); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + String commit = "a".repeat(40); + String digest = "b".repeat(64); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, snapshot_json) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, '{}'::jsonb) + """, taskId, runId, namespaceId, sourceId, suffix); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, ?, ?, ?, 1) + """, admissionId, taskId, namespaceId, repositoryId, commit, path); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 1, ?, 'ACCEPTED', CURRENT_TIMESTAMP + INTERVAL '1 day') + """, uploadId, admissionId, "manual/" + uploadId, "c".repeat(64), digest); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, + status, content_sha256, last_error_code, target_skill_slug) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, + 'BLOCKED', ?, 'SLUG_CONFLICT', ?) + """, ingestionId, taskId, admissionId, uploadId, namespaceId, repositoryId, + commit, path, "manual/" + uploadId, digest, digest, slug); + return new Fixture(namespaceId, ingestionId, skillId); + } + + private record Fixture(long namespaceId, UUID ingestionId, long skillId) {} + private record PathFixture(long namespaceId, UUID ingestionId, long skillId, UUID taskId) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathMoveRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathMoveRepositoryTest.java new file mode 100644 index 000000000..8ad6a80d4 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathMoveRepositoryTest.java @@ -0,0 +1,281 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPublicationRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Import(JdbcGithubSyncPublicationRepository.class) +@Testcontainers +class GithubSyncPathMoveRepositoryTest { + private static final String COMMIT = "a".repeat(40); + private static final String DIGEST = "b".repeat(64); + + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void configurePostgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + @Autowired private JdbcGithubSyncPublicationRepository publications; + + @Test + void movesOnlyAfterCompleteScanAndPreservesSkillIdentity() { + Fixture fixture = fixture("move-a"); + var claim = claim(); + assertThat(publications.inspectPathMove(claim).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.RETRY); + + complete(fixture); + var decision = publications.inspectPathMove(claim); + assertThat(decision.outcome()).isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.CANDIDATE); + assertThat(decision.skillId()).isEqualTo(fixture.skillId()); + assertThat(decision.previousPath()).isEqualTo("old/skill"); + assertThat(publications.movePath(claim, decision, Instant.now())).isTrue(); + assertThat(jdbc.queryForObject(""" + SELECT skill_path FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, fixture.skillId())).isEqualTo("new/skill"); + assertThat(jdbc.queryForObject(""" + SELECT published_skill_id FROM github_sync_package_ingestion WHERE id = ? + """, Long.class, fixture.ingestionId())).isEqualTo(fixture.skillId()); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_package_ingestion WHERE id = ? + """, String.class, fixture.ingestionId())).isEqualTo("NO_CHANGE"); + } + + @Test + void copyAtNewPathDoesNotMoveExistingPath() { + Fixture fixture = fixture("move-b"); + complete(fixture); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/skill', 'UNCHANGED') + """, fixture.taskId(), COMMIT); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.NONE); + } + + @Test + void twoNewPathsWithSameContentRemainAmbiguous() { + Fixture fixture = fixture("move-c"); + addIngestion(fixture.taskId(), fixture.namespaceId(), fixture.repositoryId(), "other/skill"); + complete(fixture); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.AMBIGUOUS); + } + + @Test + void terminalPartialScanDoesNotInferPathMove() { + Fixture fixture = fixture("move-d"); + jdbc.update("UPDATE github_sync_task SET status = 'PARTIAL' WHERE id = ?", fixture.taskId()); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.INCOMPLETE); + } + + @Test + void terminalFailureInAnotherPathDoesNotBlockObservedNewSkill() { + Fixture fixture = fixture("move-partial-observed"); + jdbc.update(""" + UPDATE github_sync_task SET status = 'PARTIAL', fixed_commit_sha = ?, + completed_at = CURRENT_TIMESTAMP, incomplete_subtrees = '[]'::jsonb + WHERE id = ? + """, COMMIT, fixture.taskId()); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/skill', 'UNCHANGED') + """, fixture.taskId(), COMMIT); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome, reason) + VALUES (?, ?, 'failed/skill', 'TERMINAL_FAILURE', 'PACKAGE_TOO_LARGE') + """, fixture.taskId(), COMMIT); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.NONE); + } + + @Test + void activeScanAllowsNewSkillAfterEveryExistingBindingWasObserved() { + Fixture fixture = fixture("move-active-observed"); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, + lease_token_hash = ?, lease_expires_at = CURRENT_TIMESTAMP + INTERVAL '1 hour', + leased_by = 'path-move-test' + WHERE id = ? + """, "c".repeat(64), fixture.taskId()); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/skill', 'UNCHANGED') + """, fixture.taskId(), COMMIT); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.NONE); + } + + @Test + void backpressuredScanAllowsObservedNewSkillButWaitsForMissingOldPath() { + Fixture fixture = fixture("move-paused-observed"); + jdbc.update("UPDATE github_sync_task SET status = 'RETRY_WAIT' WHERE id = ?", fixture.taskId()); + var publication = claim(); + + assertThat(publications.inspectPathMove(publication).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.RETRY); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/skill', 'UNCHANGED') + """, fixture.taskId(), COMMIT); + assertThat(publications.inspectPathMove(publication).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.NONE); + } + + @Test + void missingOldPathWithDifferentContentRequiresConfirmation() { + Fixture fixture = fixture("move-e"); + jdbc.update(""" + UPDATE github_sync_package_ingestion SET content_sha256 = ? WHERE id = ? + """, "d".repeat(64), fixture.ingestionId()); + complete(fixture); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.AMBIGUOUS); + } + + @Test + void identicalContentInAnotherNamespaceCannotBecomeMoveCandidate() { + Fixture fixture = fixture("move-f"); + var claim = claim(); + jdbc.update("DELETE FROM github_source_skill_binding WHERE skill_id = ?", fixture.skillId()); + fixture("move-g"); + complete(fixture); + + assertThat(publications.inspectPathMove(claim).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.NONE); + } + + @Test + void changedRulesMakeTheOldSnapshotIneligibleForAutomaticMove() { + Fixture fixture = fixture("move-h"); + complete(fixture); + jdbc.update(""" + UPDATE github_source_binding SET ignore_patterns = '["vendor"]'::jsonb + WHERE namespace_id = ? AND repository_id = ? + """, fixture.namespaceId(), fixture.repositoryId()); + + assertThat(publications.inspectPathMove(claim()).outcome()) + .isEqualTo(GithubSyncPublicationRepository.PathMoveOutcome.STALE); + } + + private GithubSyncPublicationRepository.ClaimedPublication claim() { + return publications.claim(UUID.randomUUID(), Instant.now(), Instant.now().plusSeconds(300)) + .orElseThrow(); + } + + private void complete(Fixture fixture) { + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', fixed_commit_sha = ? + WHERE id = ? + """, COMMIT, fixture.taskId()); + } + + private Fixture fixture(String namespaceSlug) { + jdbc.update(""" + INSERT INTO user_account (id, display_name) VALUES ('path-move-owner', 'Owner') + ON CONFLICT (id) DO NOTHING + """); + Long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES (?, 'Team', 'TEAM', 'ACTIVE', 'path-move-owner') RETURNING id + """, Long.class, namespaceSlug); + Long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status) + VALUES (?, 'skill', 'path-move-owner', 'NAMESPACE_ONLY', 'ACTIVE') RETURNING id + """, Long.class, namespaceId); + long repositoryId = Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000000) + 1; + Long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'upstream', 'skills', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, namespaceId, repositoryId); + jdbc.update(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id, latest_content_sha256) + VALUES (?, ?, ?, 'old/skill', ?, ?) + """, sourceId, namespaceId, repositoryId, skillId, DIGEST); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, snapshot_json) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, '{}'::jsonb) + """, taskId, runId, namespaceId, sourceId, namespaceSlug); + UUID ingestionId = addIngestion(taskId, namespaceId, repositoryId, "new/skill"); + return new Fixture(namespaceId, repositoryId, taskId, ingestionId, skillId); + } + + private UUID addIngestion(UUID taskId, long namespaceId, long repositoryId, String path) { + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, ?, ?, ?, 1) + """, admissionId, taskId, namespaceId, repositoryId, COMMIT, path); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 1, ?, 'ACCEPTED', CURRENT_TIMESTAMP + INTERVAL '1 day') + """, uploadId, admissionId, "path-move/" + uploadId, "c".repeat(64), DIGEST); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, + status, content_sha256, validated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 'VALIDATED', ?, CURRENT_TIMESTAMP) + """, ingestionId, taskId, admissionId, uploadId, namespaceId, repositoryId, + COMMIT, path, "path-move/" + uploadId, DIGEST, DIGEST); + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, ingestion_id) + VALUES (?, ?, ?, 'ACCEPTED', ?) + """, taskId, COMMIT, path, ingestionId); + return ingestionId; + } + + private record Fixture(long namespaceId, long repositoryId, UUID taskId, + UUID ingestionId, long skillId) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathPresenceRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathPresenceRepositoryTest.java new file mode 100644 index 000000000..24c2d5962 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncPathPresenceRepositoryTest.java @@ -0,0 +1,490 @@ +package com.iflytek.skillhub.repository; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubRepositoryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryPathResult; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubSyncPathPresenceRepositoryTest { + private static final String COMMIT = "a".repeat(40); + + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void configurePostgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void completedPartialScanRecordsRevisionUntilIgnoreRulesChange() { + Fixture fixture = fixture("presence-partial-checkpoint", "ACTIVE"); + UUID taskId = task(fixture, "QUEUED", List.of()); + String leaseHash = "e".repeat(64); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-test' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + + assertThat(tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, null, List.of(), List.of(new GithubRepositoryPathResult( + "large/skill", GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, + null, "PACKAGE_TOO_LARGE")), null, null, now, now.plusSeconds(86400))) + .isEqualTo(GithubRepositoryCompletionResult.PARTIAL); + assertThat(jdbc.queryForObject(""" + SELECT last_partial_commit FROM github_source_binding WHERE id = ? + """, String.class, fixture.sourceId())).isEqualTo(COMMIT); + assertThat(jdbc.queryForObject(""" + SELECT last_fully_processed_commit FROM github_source_binding WHERE id = ? + """, String.class, fixture.sourceId())).isNull(); + + assertThat(ignores.replace(fixture.namespaceId(), fixture.sourceId(), List.of("large"))) + .contains(List.of("large")); + assertThat(jdbc.queryForObject(""" + SELECT last_partial_commit FROM github_source_binding WHERE id = ? + """, String.class, fixture.sourceId())).isNull(); + } + + @Test + void ignoreRuleChangedDuringScanKeepsSourceDueForNewFullScan() { + Fixture fixture = fixture("presence-ignore-during-scan", "ACTIVE"); + UUID taskId = task(fixture, "QUEUED", List.of()); + String leaseHash = "e".repeat(64); + Instant now = Instant.now().truncatedTo(java.time.temporal.ChronoUnit.MICROS); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-test' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + assertThat(ignores.replace(fixture.namespaceId(), fixture.sourceId(), List.of("ignored"))) + .contains(List.of("ignored")); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + + assertThat(tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, null, List.of(), List.of(new GithubRepositoryPathResult( + "large/skill", GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, + null, "PACKAGE_TOO_LARGE")), null, null, now, now.plusSeconds(86400))) + .isEqualTo(GithubRepositoryCompletionResult.PARTIAL); + assertThat(jdbc.queryForObject(""" + SELECT last_partial_commit FROM github_source_binding WHERE id = ? + """, String.class, fixture.sourceId())).isNull(); + assertThat(jdbc.queryForObject(""" + SELECT next_check_at FROM github_source_binding WHERE id = ? + """, java.sql.Timestamp.class, fixture.sourceId()).toInstant()).isEqualTo(now); + } + + @Test + void syntheticTenThousandPathResultsPersistAtContractLimit() { + Fixture fixture = fixture("presence-capacity", "ACTIVE"); + UUID taskId = task(fixture, "QUEUED", List.of()); + String leaseHash = "f".repeat(64); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-capacity' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + List results = IntStream.range(0, 10_000) + .mapToObj(index -> new GithubRepositoryPathResult( + "skills/skill-%05d".formatted(index), + GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, + null, "PACKAGE_TOO_LARGE")) + .toList(); + + long started = System.nanoTime(); + var outcome = tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, results.getLast().skillPath(), List.of(), results, + null, null, now, now.plusSeconds(86400)); + long elapsedMillis = java.time.Duration.ofNanos(System.nanoTime() - started).toMillis(); + assertThat(outcome).isEqualTo(GithubRepositoryCompletionResult.PARTIAL); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result WHERE task_id = ? + """, Integer.class, taskId)).isEqualTo(10_000); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_task WHERE id = ? + """, String.class, taskId)).isEqualTo("PARTIAL"); + System.out.printf("GitHub path completion baseline: paths=10000, elapsedMs=%d%n", elapsedMillis); + } + + @Test + void completeScanRecordsMissingAndLaterRecoveryWithoutChangingTheSkill() { + Fixture fixture = fixture("presence-a", "ACTIVE"); + long skillId = binding(fixture, "old/demo", "demo"); + UUID firstSeenTask = task(fixture, "ACQUISITION_COMPLETE", List.of("old/demo")); + var presence = repository(); + + presence.reconcile(fixture.sourceId(), firstSeenTask, COMMIT, Instant.now()); + jdbc.update("UPDATE github_sync_task SET created_at = CURRENT_TIMESTAMP - INTERVAL '1 day' WHERE id = ?", + firstSeenTask); + UUID missingTask = task(fixture, "ACQUISITION_COMPLETE", List.of()); + presence.reconcile(fixture.sourceId(), missingTask, COMMIT, Instant.now()); + assertThat(state(skillId)).isEqualTo("MISSING"); + assertThat(jdbc.queryForObject(""" + SELECT missing_since IS NOT NULL FROM github_source_skill_binding WHERE skill_id = ? + """, Boolean.class, skillId)).isTrue(); + assertThat(jdbc.queryForObject("SELECT status FROM skill WHERE id = ?", String.class, skillId)) + .isEqualTo("ACTIVE"); + var management = new GithubSyncManagementQueryRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()), + new JdbcGithubSyncCapacityGate(jdbc)); + assertThat(management.skillPaths(fixture.namespaceId(), fixture.sourceId(), 0, 20) + .items().getFirst().upstreamPresence()).isEqualTo("MISSING"); + + jdbc.update("UPDATE github_sync_task SET created_at = CURRENT_TIMESTAMP - INTERVAL '12 hours' WHERE id = ?", + missingTask); + UUID recoveredTask = task(fixture, "ACQUISITION_COMPLETE", List.of("old/demo")); + presence.reconcile(fixture.sourceId(), recoveredTask, COMMIT, Instant.now()); + assertThat(state(skillId)).isEqualTo("PRESENT"); + assertThat(management.skillPaths(fixture.namespaceId(), fixture.sourceId(), 0, 20) + .items().getFirst().upstreamPresence()).isEqualTo("PRESENT"); + assertThat(jdbc.queryForObject(""" + SELECT missing_since IS NULL FROM github_source_skill_binding WHERE skill_id = ? + """, Boolean.class, skillId)).isTrue(); + } + + @Test + void partialScanAndPausedSourceNeverInferMissing() { + Fixture fixture = fixture("presence-b", "ACTIVE"); + long skillId = binding(fixture, "old/demo", "demo"); + UUID partial = task(fixture, "PARTIAL", List.of()); + repository().reconcile(fixture.sourceId(), partial, COMMIT, Instant.now()); + assertThat(state(skillId)).isEqualTo("UNKNOWN"); + + jdbc.update("UPDATE github_source_binding SET status = 'PAUSED' WHERE id = ?", fixture.sourceId()); + UUID complete = task(fixture, "ACQUISITION_COMPLETE", List.of()); + repository().reconcile(fixture.sourceId(), complete, COMMIT, Instant.now()); + assertThat(state(skillId)).isEqualTo("UNKNOWN"); + } + + @Test + void changedIgnoreRulesAndIgnoredPathsNeverInferMissing() { + Fixture fixture = fixture("presence-c", "ACTIVE"); + long ordinary = binding(fixture, "old/demo", "demo"); + long ignored = binding(fixture, "test/demo", "ignored"); + UUID task = task(fixture, "ACQUISITION_COMPLETE", List.of()); + jdbc.update("UPDATE github_source_binding SET ignore_patterns = '[\"vendor\"]'::jsonb WHERE id = ?", + fixture.sourceId()); + repository().reconcile(fixture.sourceId(), task, COMMIT, Instant.now()); + assertThat(state(ordinary)).isEqualTo("UNKNOWN"); + + jdbc.update("UPDATE github_source_binding SET ignore_patterns = '[]'::jsonb WHERE id = ?", + fixture.sourceId()); + repository().reconcile(fixture.sourceId(), task, COMMIT, Instant.now()); + assertThat(state(ordinary)).isEqualTo("MISSING"); + assertThat(state(ignored)).isEqualTo("UNKNOWN"); + } + + @Test + void olderSnapshotCannotOverwriteNewerPresence() { + Fixture fixture = fixture("presence-d", "ACTIVE"); + long skillId = binding(fixture, "old/demo", "demo"); + UUID oldTask = task(fixture, "ACQUISITION_COMPLETE", List.of("old/demo")); + jdbc.update("UPDATE github_sync_task SET created_at = CURRENT_TIMESTAMP - INTERVAL '1 day' WHERE id = ?", + oldTask); + UUID newTask = task(fixture, "ACQUISITION_COMPLETE", List.of()); + + repository().reconcile(fixture.sourceId(), newTask, COMMIT, Instant.now()); + repository().reconcile(fixture.sourceId(), oldTask, COMMIT, Instant.now()); + + assertThat(state(skillId)).isEqualTo("MISSING"); + } + + @Test + void completedScanRecoversUnreachableSourceWithoutRemovingPublishedSkill() { + Fixture fixture = fixture("presence-e", "UNREACHABLE"); + long skillId = binding(fixture, "old/demo", "demo"); + UUID taskId = task(fixture, "ACQUISITION_COMPLETE", List.of()); + String leaseHash = "f".repeat(64); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-test' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var taskRepository = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + + var result = taskRepository.completeRepository(taskId, 1, leaseHash, COMMIT, + true, null, List.of(), List.of(), null, null, now, now.plusSeconds(86400)); + + assertThat(result).isEqualTo(GithubRepositoryCompletionResult.ACQUISITION_COMPLETE); + assertThat(jdbc.queryForObject("SELECT status FROM github_source_binding WHERE id = ?", + String.class, fixture.sourceId())).isEqualTo("ACTIVE"); + assertThat(state(skillId)).isEqualTo("MISSING"); + assertThat(jdbc.queryForObject("SELECT status FROM skill WHERE id = ?", String.class, skillId)) + .isEqualTo("ACTIVE"); + } + + @Test + void failedPackageReleasesItsAdmissionOnlyWhenRepositoryCompletionIsValid() { + Fixture fixture = fixture("presence-f", "ACTIVE"); + UUID taskId = task(fixture, "QUEUED", List.of()); + String leaseHash = "f".repeat(64); + Instant now = Instant.now(); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-test' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + UUID failedAdmission = UUID.randomUUID(); + UUID unfinishedAdmission = UUID.randomUUID(); + for (var admission : List.of( + java.util.Map.entry(failedAdmission, "failed/demo"), + java.util.Map.entry(unfinishedAdmission, "unfinished/demo"))) { + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, 10001, ?, ?, 1) + """, admission.getKey(), taskId, fixture.namespaceId(), COMMIT, admission.getValue()); + } + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + var failedPath = new GithubRepositoryPathResult("failed/demo", + GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, null, "ARCHIVE_TOO_LARGE"); + + assertThat(tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, null, List.of(), List.of(failedPath), null, null, + now, now.plusSeconds(86400))).isEqualTo(GithubRepositoryCompletionResult.COVERAGE_CONFLICT); + assertThat(jdbc.queryForObject("SELECT status FROM github_sync_package_admission WHERE id = ?", + String.class, failedAdmission)).isEqualTo("ACTIVE"); + + var otherFailedPath = new GithubRepositoryPathResult("unfinished/demo", + GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, null, "ARCHIVE_TOO_LARGE"); + assertThat(tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, null, List.of(), List.of(failedPath, otherFailedPath), null, null, + now, now.plusSeconds(86400))).isEqualTo(GithubRepositoryCompletionResult.PARTIAL); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_admission + WHERE task_id = ? AND status = 'ACTIVE' + """, Integer.class, taskId)).isZero(); + } + + @Test + void terminalRepositoryCompletionWakesPublicationsDeferredDuringScan() { + Fixture fixture = fixture("presence-publication-wakeup", "ACTIVE"); + binding(fixture, "old/demo", "old-demo"); + UUID taskId = task(fixture, "QUEUED", List.of()); + String leaseHash = "f".repeat(64); + Instant now = Instant.now().truncatedTo(java.time.temporal.ChronoUnit.MICROS); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'LEASED', lease_generation = 1, lease_token_hash = ?, + lease_expires_at = ?, leased_by = 'presence-test' + WHERE id = ? + """, leaseHash, java.sql.Timestamp.from(now.plusSeconds(300)), taskId); + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, + lease_generation, status) + VALUES (?, ?, ?, 10001, ?, 'new/demo', 1, 'ACCEPTED') + """, admissionId, taskId, fixture.namespaceId(), COMMIT); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, size_bytes, sha256, status, expires_at) + VALUES (?, ?, ?, 'S3_PRESIGNED_PUT', 1, ?, 'ACCEPTED', ?) + """, uploadId, admissionId, "publication-wakeup/" + uploadId, + "a".repeat(64), java.sql.Timestamp.from(now.plusSeconds(300))); + Instant originalRetry = now.plusSeconds(900); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, status, + last_error_code, publication_retry_at) + VALUES (?, ?, ?, ?, ?, 10001, ?, 'new/demo', ?, 1, ?, 'VALIDATED', + 'PATH_MOVE_SCAN_INCOMPLETE', ?) + """, ingestionId, taskId, admissionId, uploadId, fixture.namespaceId(), + COMMIT, "publication-wakeup/" + uploadId, "a".repeat(64), + java.sql.Timestamp.from(originalRetry)); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'old/demo', 'UNCHANGED') + """, taskId, COMMIT); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + var failedPath = new GithubRepositoryPathResult("too-large/demo", + GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE, null, "PACKAGE_TOO_LARGE"); + + assertThat(tasks.completeRepository(taskId, 1, "0".repeat(64), COMMIT, + true, "too-large/demo", List.of(), List.of(failedPath), null, null, + now, now.plusSeconds(86400))).isEqualTo(GithubRepositoryCompletionResult.LEASE_STALE); + assertThat(jdbc.queryForObject(""" + SELECT publication_retry_at FROM github_sync_package_ingestion WHERE id = ? + """, java.sql.Timestamp.class, ingestionId).toInstant()).isEqualTo(originalRetry); + + assertThat(tasks.completeRepository(taskId, 1, leaseHash, COMMIT, + true, "too-large/demo", List.of(), List.of(failedPath), null, null, + now, now.plusSeconds(86400))).isEqualTo(GithubRepositoryCompletionResult.PARTIAL); + assertThat(jdbc.queryForObject(""" + SELECT publication_retry_at FROM github_sync_package_ingestion WHERE id = ? + """, java.sql.Timestamp.class, ingestionId).toInstant()).isEqualTo(now); + } + + @Test + void excludingOneSkillPathPreservesSiblingAndPublishedIdentityAndCanResume() { + Fixture fixture = fixture("presence-one-path-pause", "ACTIVE"); + long ignored = binding(fixture, "skills/paused", "paused"); + long sibling = binding(fixture, "skills/active", "active"); + long version = jdbc.queryForObject(""" + INSERT INTO skill_version (skill_id, version, status, created_by) + VALUES (?, '1.0.0', 'PUBLISHED', 'presence-owner') RETURNING id + """, Long.class, ignored); + jdbc.update("UPDATE skill SET latest_version_id = ? WHERE id = ?", version, ignored); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + assertThat(ignores.replace(fixture.namespaceId(), fixture.sourceId(), List.of("skills/paused"))) + .contains(List.of("skills/paused")); + assertThat(ignores.ignoredForSource(fixture.sourceId(), "skills/paused")).isTrue(); + assertThat(ignores.ignoredForSource(fixture.sourceId(), "skills/active")).isFalse(); + UUID scan = task(fixture, "ACQUISITION_COMPLETE", List.of("skills/active")); + jdbc.update("UPDATE github_sync_task SET snapshot_json = '{\"ignorePatterns\":[\"skills/paused\"]}'::jsonb WHERE id = ?", scan); + repository().reconcile(fixture.sourceId(), scan, COMMIT, Instant.now()); + assertThat(state(ignored)).isEqualTo("UNKNOWN"); + assertThat(state(sibling)).isEqualTo("PRESENT"); + assertThat(jdbc.queryForObject("SELECT latest_version_id FROM skill WHERE id = ?", + Long.class, ignored)).isEqualTo(version); + assertThat(jdbc.queryForObject("SELECT status FROM skill_version WHERE id = ?", + String.class, version)).isEqualTo("PUBLISHED"); + assertThat(jdbc.queryForObject("SELECT status FROM github_source_binding WHERE id = ?", + String.class, fixture.sourceId())).isEqualTo("ACTIVE"); + ignores.replace(fixture.namespaceId(), fixture.sourceId(), List.of()); + assertThat(ignores.ignoredForSource(fixture.sourceId(), "skills/paused")).isFalse(); + UUID resumed = task(fixture, "ACQUISITION_COMPLETE", List.of("skills/paused", "skills/active")); + repository().reconcile(fixture.sourceId(), resumed, COMMIT, Instant.now()); + assertThat(state(ignored)).isEqualTo("PRESENT"); + assertThat(jdbc.queryForObject("SELECT latest_version_id FROM skill WHERE id = ?", + Long.class, ignored)).isEqualTo(version); + } + + private JdbcGithubSyncPathPresenceRepository repository() { + return new JdbcGithubSyncPathPresenceRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper())); + } + + private String state(long skillId) { + return jdbc.queryForObject(""" + SELECT upstream_presence FROM github_source_skill_binding WHERE skill_id = ? + """, String.class, skillId); + } + + private Fixture fixture(String slug, String status) { + jdbc.update(""" + INSERT INTO user_account (id, display_name) VALUES ('presence-owner', 'Owner') + ON CONFLICT (id) DO NOTHING + """); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES (?, 'Team', 'TEAM', 'ACTIVE', 'presence-owner') RETURNING id + """, Long.class, slug); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, status) + VALUES (?, 10001, 'upstream', 'skills', 'NAMESPACE_ONLY', ?) RETURNING id + """, Long.class, namespaceId, status); + return new Fixture(namespaceId, sourceId); + } + + private long binding(Fixture fixture, String path, String slug) { + long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility, status) + VALUES (?, ?, 'presence-owner', 'NAMESPACE_ONLY', 'ACTIVE') RETURNING id + """, Long.class, fixture.namespaceId(), slug); + jdbc.update(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id) + VALUES (?, ?, 10001, ?, ?) + """, fixture.sourceId(), fixture.namespaceId(), path, skillId); + return skillId; + } + + private UUID task(Fixture fixture, String status, List presentPaths) { + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC') + """, runId, fixture.namespaceId()); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, status, + dedupe_key, snapshot_json, fixed_commit_sha) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, ?, + '{"ignorePatterns":[]}'::jsonb, ?) + """, taskId, runId, fixture.namespaceId(), fixture.sourceId(), + status, "presence:" + taskId, COMMIT); + for (String path : presentPaths) { + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, ?, 'UNCHANGED') + """, taskId, COMMIT, path); + } + return taskId; + } + + private record Fixture(long namespaceId, long sourceId) {} +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncSchedulePolicyRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncSchedulePolicyRepositoryTest.java new file mode 100644 index 000000000..3df222472 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncSchedulePolicyRepositoryTest.java @@ -0,0 +1,583 @@ +package com.iflytek.skillhub.repository; + +import static com.iflytek.skillhub.support.GithubSyncWorkerCapabilities.platform; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubSyncDeferralReason; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatchResult; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryEntryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceTaskRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncPathPresenceRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncSourceCheckpointRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncTaskRepository; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskMode; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubSyncSchedulePolicyRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void partialRevisionUsesHeadChecksUntilUpstreamCommitChanges() { + long namespaceId = platformNamespaceId(); + long repositoryId = 847204L; + String oldCommit = "a".repeat(40); + String newCommit = "b".repeat(40); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50, namespace_high = 50, namespace_low = 25 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + last_observed_commit, last_partial_commit, next_check_at) + VALUES (?, ?, 'source', 'partial', 'PUBLIC', ?, ?, CURRENT_TIMESTAMP - INTERVAL '1 hour') + RETURNING id + """, Long.class, namespaceId, repositoryId, oldCommit, oldCommit); + var sources = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + Instant now = Instant.now(); + + assertThat(sources.enqueueDue(now, 10)).isEqualTo(1); + UUID headTaskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE source_id = ? AND status = 'QUEUED' + """, UUID.class, sourceId); + assertThat(jdbc.queryForObject("SELECT mode FROM github_sync_task WHERE id = ?", + String.class, headTaskId)).isEqualTo("HEAD_ONLY"); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ? WHERE id = ? + """, Timestamp.from(now), headTaskId); + jdbc.update(""" + UPDATE github_source_binding SET next_check_at = ? WHERE id = ? + """, Timestamp.from(now.plusSeconds(86400)), sourceId); + assertThat(sources.enqueueDue(now.plusSeconds(1), 10)).isZero(); + + jdbc.update(""" + UPDATE github_source_binding SET last_observed_commit = ? WHERE id = ? + """, newCommit, sourceId); + assertThat(sources.enqueueDue(now.plusSeconds(2), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT mode FROM github_sync_task WHERE source_id = ? AND status = 'QUEUED' + """, String.class, sourceId)).isEqualTo("FULL"); + } + + @Test + void manualRescanDuringHeadCheckForcesOneFullScanAfterItFinishes() { + long namespaceId = platformNamespaceId(); + long repositoryId = 847205L; + String commit = "c".repeat(40); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50, namespace_high = 50, namespace_low = 25 + WHERE id = TRUE + """); + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + last_observed_commit, last_partial_commit, next_check_at) + VALUES (?, ?, 'source', 'manual-partial', 'PUBLIC', ?, ?, CURRENT_TIMESTAMP - INTERVAL '1 hour') + RETURNING id + """, Long.class, namespaceId, repositoryId, commit, commit); + var sources = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + Instant now = Instant.now(); + + assertThat(sources.enqueueDue(now, 10)).isEqualTo(1); + UUID headTaskId = jdbc.queryForObject(""" + SELECT id FROM github_sync_task WHERE source_id = ? AND status = 'QUEUED' + """, UUID.class, sourceId); + assertThat(sources.requestRescan(namespaceId, sourceId, now).orElseThrow().status()) + .isEqualTo("MERGED"); + assertThat(jdbc.queryForObject(""" + SELECT manual_rescan_requested_at IS NOT NULL FROM github_source_binding WHERE id = ? + """, Boolean.class, sourceId)).isTrue(); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ? WHERE id = ? + """, Timestamp.from(now), headTaskId); + jdbc.update(""" + UPDATE github_source_binding SET next_check_at = ? WHERE id = ? + """, Timestamp.from(now.plusSeconds(86400)), sourceId); + + assertThat(sources.enqueueDue(now.plusSeconds(1), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT mode FROM github_sync_task WHERE source_id = ? AND status = 'QUEUED' + """, String.class, sourceId)).isEqualTo("FULL"); + assertThat(jdbc.queryForObject(""" + SELECT manual_rescan_requested_at IS NOT NULL FROM github_source_binding WHERE id = ? + """, Boolean.class, sourceId)).isTrue(); + + // A full task may become HEAD_ONLY at claim time if the backlog fills meanwhile. + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = NULL, global_low = NULL WHERE id = TRUE + """); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + String leaseHash = "d".repeat(64); + var downgraded = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, + "schedule-worker", leaseHash, now.plusSeconds(2), now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + assertThat(downgraded.mode()).isEqualTo(GithubSyncTaskMode.HEAD_ONLY); + assertThat(jdbc.queryForObject(""" + SELECT manual_rescan_requested_at IS NOT NULL FROM github_source_binding WHERE id = ? + """, Boolean.class, sourceId)).isTrue(); + assertThat(tasks.completeHeadOnly(downgraded.taskId(), downgraded.leaseGeneration(), + leaseHash, commit, now.plusSeconds(3), now.plusSeconds(86400))).isTrue(); + + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50 WHERE id = TRUE + """); + assertThat(sources.enqueueDue(now.plusSeconds(4), 10)).isEqualTo(1); + var full = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, + "schedule-worker", "e".repeat(64), now.plusSeconds(5), now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + assertThat(full.mode()).isEqualTo(GithubSyncTaskMode.FULL); + assertThat(jdbc.queryForObject(""" + SELECT manual_rescan_requested_at FROM github_source_binding WHERE id = ? + """, Timestamp.class, sourceId)).isNull(); + } + + @Test + void categoryChangeUpdatesExistingAndNewEntriesWithoutDelayingEarlierWork() { + long namespaceId = platformNamespaceId(); + long repositoryId = 847201L; + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + status, next_check_at) + VALUES (?, ?, 'source', 'skills', 'PUBLIC', 'ACTIVE', CURRENT_TIMESTAMP + INTERVAL '20 days') + """, namespaceId, repositoryId); + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + status, next_check_at) + VALUES (?, ?, 'source', 'paused', 'PUBLIC', 'PAUSED', CURRENT_TIMESTAMP + INTERVAL '20 days') + """, namespaceId, repositoryId + 1); + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + status, next_check_at) + VALUES (?, ?, 'source', 'already-due', 'PUBLIC', 'ACTIVE', CURRENT_TIMESTAMP + INTERVAL '1 hour') + """, namespaceId, repositoryId + 2); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var before = entries.createCuratedRepository("source", "initial-skill-repo"); + var owner = entries.createCuratedOwner("source-owner", 24 * 3600); + Instant now = Instant.now(); + var repository = new JdbcGithubSyncSchedulePolicyRepository(jdbc); + + assertThat(repository.get().repositoryHours()).isNull(); + var updated = repository.update(6, 12, 48); + + assertThat(updated.repositoryHours()).isEqualTo(6); + assertThat(updated.curatedOwnerHours()).isEqualTo(12); + assertThat(updated.searchRuleHours()).isEqualTo(48); + assertThat(jdbc.queryForObject(""" + SELECT target_interval_seconds FROM github_discovery_entry WHERE id = ? + """, Integer.class, before.id())).isEqualTo(6 * 3600); + assertThat(entries.createCuratedRepository("source", "next-skill-repo") + .targetIntervalSeconds()).isEqualTo(6 * 3600); + assertThat(jdbc.queryForObject(""" + SELECT target_interval_seconds FROM github_discovery_entry WHERE id = ? + """, Integer.class, owner.id())).isEqualTo(12 * 3600); + assertThat(jdbc.queryForObject(""" + SELECT target_interval_seconds FROM github_discovery_entry + WHERE entry_type = 'SEARCH_RULE' LIMIT 1 + """, Integer.class)).isEqualTo(48 * 3600); + assertThat(jdbc.queryForObject(""" + SELECT next_check_at FROM github_source_binding WHERE repository_id = ? + """, Timestamp.class, repositoryId).toInstant()).isBefore(now.plusSeconds(7 * 3600)); + assertThat(jdbc.queryForObject(""" + SELECT next_check_at FROM github_source_binding WHERE repository_id = ? + """, Timestamp.class, repositoryId + 1).toInstant()).isAfter(now.plusSeconds(19 * 86400)); + assertThat(jdbc.queryForObject(""" + SELECT next_check_at FROM github_source_binding WHERE repository_id = ? + """, Timestamp.class, repositoryId + 2).toInstant()).isBefore(now.plusSeconds(2 * 3600)); + } + + @Test + void rescanMergesQueuedAndRetryingTaskWithoutChangingRetryTime() { + long namespaceId = platformNamespaceId(); + long repositoryId = 847203L; + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'source', 'rescan', 'PUBLIC', CURRENT_TIMESTAMP + INTERVAL '20 days') + RETURNING id + """, Long.class, namespaceId, repositoryId); + var sources = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + Instant now = Instant.now(); + + assertThat(sources.requestRescan(namespaceId, sourceId, now).orElseThrow().status()) + .isEqualTo("SCHEDULED"); + assertThat(sources.requestRescan(namespaceId + 1, sourceId, now)).isEmpty(); + assertThat(jdbc.queryForObject(""" + SELECT next_check_at FROM github_source_binding WHERE id = ? + """, Timestamp.class, sourceId).toInstant().toEpochMilli()).isEqualTo(now.toEpochMilli()); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, dedupe_key, + snapshot_json, due_at) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', 'HEAD_ONLY', ?, '{}'::jsonb, ?) + """, taskId, runId, namespaceId, sourceId, "repository:" + repositoryId, + Timestamp.from(now.plusSeconds(3600))); + + var merged = sources.requestRescan(namespaceId, sourceId, now).orElseThrow(); + assertThat(merged.status()).isEqualTo("MERGED"); + assertThat(merged.taskId()).isEqualTo(taskId); + assertThat(jdbc.queryForObject("SELECT priority FROM github_sync_task WHERE id = ?", + Integer.class, taskId)).isEqualTo(200); + Instant retryAt = now.plusSeconds(7200); + jdbc.update(""" + UPDATE github_sync_task SET status = 'RETRY_WAIT', retry_at = ? WHERE id = ? + """, Timestamp.from(retryAt), taskId); + + assertThat(sources.requestRescan(namespaceId, sourceId, now).orElseThrow().taskStatus()) + .isEqualTo("RETRY_WAIT"); + assertThat(jdbc.queryForObject("SELECT retry_at FROM github_sync_task WHERE id = ?", + Timestamp.class, taskId).toInstant().toEpochMilli()).isEqualTo(retryAt.toEpochMilli()); + } + + @Test + void sourceBatchAndLeasesGiveDueTeamSourceAPlaceAmongPlatformSources() { + long platform = platformNamespaceId(); + jdbc.update(""" + INSERT INTO user_account (id, display_name) VALUES ('fairness-owner', 'Owner') + ON CONFLICT (id) DO NOTHING + """); + long team = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES ('fairness-team', 'Team', 'TEAM', 'ACTIVE', 'fairness-owner') RETURNING id + """, Long.class); + for (int index = 0; index < 12; index++) { + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, ?, 'platform', ?, 'PUBLIC', CURRENT_TIMESTAMP - INTERVAL '2 days') + """, platform, 900000L + index, "repo-" + index); + } + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + next_check_at) + VALUES (?, 910000, 'team', 'repo', 'NAMESPACE_ONLY', CURRENT_TIMESTAMP - INTERVAL '1 day') + """, team); + var capacity = new JdbcGithubSyncCapacityGate(jdbc); + var scheduler = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), capacity); + Instant now = Instant.now(); + + assertThat(scheduler.enqueueDue(now, 10)).isEqualTo(10); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE namespace_id = ? AND status = 'QUEUED' + """, Integer.class, team)).isEqualTo(1); + + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), capacity, + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + var first = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "fairness-worker", "a".repeat(64), + now, now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + var second = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "fairness-worker", "b".repeat(64), + now.plusMillis(1), now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + + assertThat(java.util.List.of(first.namespaceId(), second.namespaceId())) + .containsExactlyInAnyOrder(platform, team); + assertThat(scheduler.enqueueDue(now.plusSeconds(86400), 10)).isEqualTo(3); + assertThat(scheduler.enqueueDue(now.plusSeconds(172800), 10)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task + WHERE kind = 'REPOSITORY_SYNC' AND status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') + """, Integer.class)).isEqualTo(13); + } + + @Test + void syntheticThousandRegisteredSourcesScheduleOnceInBatches() { + long namespaceId = platformNamespaceId(); + Instant now = Instant.now(); + String commit = "a".repeat(40); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 20000, global_low = 10000, + namespace_high = 20000, namespace_low = 10000 + WHERE id = TRUE + """); + jdbc.update(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, + last_observed_commit, next_check_at) + SELECT ?, 3000000 + series.id, 'fixture-owner', 'registered-' || series.id, + 'PUBLIC', ?, ? + FROM generate_series(1, 1000) AS series(id) + """, namespaceId, commit, Timestamp.from(now.minusSeconds(3600))); + var scheduler = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc)); + + long started = System.nanoTime(); + int scheduled = 0; + for (int batch = 0; batch < 10; batch++) { + scheduled += scheduler.enqueueDue(now, 100); + } + long elapsedMillis = java.time.Duration.ofNanos(System.nanoTime() - started).toMillis(); + assertThat(scheduled).isEqualTo(1000); + assertThat(scheduler.enqueueDue(now.plusSeconds(86400), 100)).isZero(); + assertThat(jdbc.queryForObject(""" + SELECT count(DISTINCT task.source_id) FROM github_sync_task task + WHERE task.namespace_id = ? AND task.kind = 'REPOSITORY_SYNC' + AND task.mode = 'FULL' AND task.status = 'QUEUED' + """, Integer.class, namespaceId)).isEqualTo(1000); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_task task + WHERE task.namespace_id = ? AND task.kind = 'REPOSITORY_SYNC' + AND task.status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') + """, Integer.class, namespaceId)).isEqualTo(1000); + System.out.printf("GitHub registered source scheduling baseline: sources=1000, batches=10, elapsedMs=%d%n", + elapsedMillis); + } + + @Test + void reclaimedRepositoryTaskReturnsReceivedAndRetryablePaths() { + long namespaceId = platformNamespaceId(); + long repositoryId = 920001L; + String commit = "a".repeat(40); + String digest = "b".repeat(64); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET global_high = 100, global_low = 50, namespace_high = 50, namespace_low = 25 + WHERE id = TRUE + """); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, + snapshot_json, fixed_commit_sha, lease_generation, lease_token_hash, + lease_expires_at, leased_by) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', ?::bigint, 'owner', 'example', 'name', 'repo'), + ?, 1, ?, CURRENT_TIMESTAMP - INTERVAL '1 minute', 'stopped-worker') + """, taskId, runId, namespaceId, "repository:" + repositoryId, + repositoryId, commit, "d".repeat(64)); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation, + status) + VALUES (?, ?, ?, ?, ?, 'skills/received', 1, 'ACCEPTED') + """, admissionId, taskId, namespaceId, repositoryId, commit); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 1, ?, 'ACCEPTED', CURRENT_TIMESTAMP) + """, uploadId, admissionId, "resume/" + uploadId, digest, digest); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256, status) + VALUES (?, ?, ?, ?, ?, ?, ?, 'skills/received', ?, 1, ?, 'RECEIVED') + """, ingestionId, taskId, admissionId, uploadId, namespaceId, repositoryId, + commit, "resume/" + uploadId, digest); + jdbc.update(""" + INSERT INTO github_sync_path_result (task_id, commit_sha, skill_path, outcome) + VALUES (?, ?, 'skills/retry', 'RETRYABLE_FAILURE') + """, taskId, commit); + var capacity = new JdbcGithubSyncCapacityGate(jdbc); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), capacity, + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + Instant now = Instant.now(); + + var claimed = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "resume-worker", + "c".repeat(64), now, now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + + assertThat(claimed.taskId()).isEqualTo(taskId); + assertThat(claimed.leaseGeneration()).isEqualTo(2); + assertThat(claimed.fixedCommitSha()).isEqualTo(commit); + assertThat(claimed.acceptedPaths()).containsExactly("skills/received"); + assertThat(claimed.retryPaths()).containsExactly("skills/retry"); + assertThat(tasks.renew(taskId, 1, "d".repeat(64), now, now.plusSeconds(300))) + .isEmpty(); + } + + @Test + void pausedDiscoveryEntryIsNotReclaimedUntilEnabledAgain() { + jdbc.update("INSERT INTO user_account (id, display_name) VALUES ('pause-claim-owner', 'Owner')"); + long namespaceId = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES ('pause-claim-team', 'Pause Claim Team', 'TEAM', 'ACTIVE', 'pause-claim-owner') + RETURNING id + """, Long.class); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var entry = entries.createTeamRepository(namespaceId, "pause-owner", "pause-repo", + "NAMESPACE_ONLY"); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'DISCOVERY')", + runId, entry.namespaceId()); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, + snapshot_json, due_at, retry_at) + VALUES (?, ?, ?, 'DISCOVERY', 'RETRY_WAIT', ?, + jsonb_build_object('entryType', 'CURATED_REPOSITORY', 'ruleId', ?::text), + CURRENT_TIMESTAMP - INTERVAL '1 minute', + CURRENT_TIMESTAMP - INTERVAL '1 minute') + """, taskId, runId, entry.namespaceId(), "discovery:" + entry.id(), entry.id().toString()); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), + new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + Instant now = Instant.now(); + + entries.setTeamRepositoryEnabled(namespaceId, entry.id(), false); + assertThat(tasks.claim(GithubSyncTaskKind.DISCOVERY, "paused-worker", + "a".repeat(64), now, now.plusSeconds(300), platform(java.util.List.of()))).isEmpty(); + + entries.setTeamRepositoryEnabled(namespaceId, entry.id(), true); + assertThat(tasks.claim(GithubSyncTaskKind.DISCOVERY, "resumed-worker", + "b".repeat(64), now, now.plusSeconds(300), platform(java.util.List.of()))) + .get().extracting(lease -> lease.taskId()).isEqualTo(taskId); + } + + @Test + void departedConfigurationAdministratorDoesNotOwnOrBlockTeamSourceWork() { + jdbc.update("INSERT INTO user_account (id, display_name) VALUES ('departed-sync-admin', 'Admin')"); + long namespace = jdbc.queryForObject(""" + INSERT INTO namespace (slug, display_name, type, status, created_by) + VALUES ('departed-sync-team', 'Team', 'TEAM', 'ACTIVE', 'departed-sync-admin') RETURNING id + """, Long.class); + jdbc.update(""" + INSERT INTO namespace_member (namespace_id, user_id, role) + VALUES (?, 'departed-sync-admin', 'ADMIN') + """, namespace); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()); + var entry = entries.createTeamRepository(namespace, "public-owner", "skills", "NAMESPACE_ONLY"); + long source = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility, next_check_at) + VALUES (?, 930007, 'public-owner', 'skills', 'NAMESPACE_ONLY', CURRENT_TIMESTAMP - INTERVAL '1 hour') + RETURNING id + """, Long.class, namespace); + jdbc.update("INSERT INTO github_source_origin (source_id, origin_type, origin_key) VALUES (?, 'TEAM_CONFIG', ?)", + source, entry.id().toString()); + jdbc.update("DELETE FROM namespace_member WHERE namespace_id = ? AND user_id = 'departed-sync-admin'", namespace); + var capacity = new JdbcGithubSyncCapacityGate(jdbc); + var scheduler = new JdbcGithubSourceTaskRepository(jdbc, new ObjectMapper(), capacity); + Instant now = Instant.now(); + assertThat(scheduler.enqueueDue(now, 10)).isEqualTo(1); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, new ObjectMapper(), capacity, + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + var lease = tasks.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "team-worker", "c".repeat(64), + now, now.plusSeconds(300), java.util.List.of()).orElseThrow(); + assertThat(lease.namespaceId()).isEqualTo(namespace); + assertThat(jdbc.queryForObject("SELECT source_id FROM github_sync_task WHERE id = ?", + Long.class, lease.taskId())).isEqualTo(source); + assertThat(jdbc.queryForObject("SELECT enabled FROM github_discovery_entry WHERE id = ?", + Boolean.class, entry.id())).isTrue(); + } + + @Test + void weeklyFullRecheckRateLimitKeepsFrozenWindowAndAcknowledgedPage() throws Exception { + var mapper = new ObjectMapper(); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, mapper); + var entry = entries.createSearchRule("Weekly recheck", "{\"selectors\":[{\"kind\":\"KEYWORD\",\"value\":\"skills\",\"priority\":\"CORE\"}]}", 86400); + entries.setEnabled(entry.id(), true); + Instant now = Instant.now(); + Instant previous = now.minusSeconds(8 * 86400L); + jdbc.update("UPDATE github_discovery_entry SET next_due_at = ? WHERE id = ?", Timestamp.from(previous), entry.id()); + assertThat(entries.enqueueDue(previous.plusSeconds(1), 10)).isEqualTo(1); + jdbc.update("UPDATE github_sync_task SET status = 'ACQUISITION_COMPLETE' WHERE dedupe_key = ?", "discovery:" + entry.id()); + assertThat(entries.enqueueDue(now, 10)).isEqualTo(1); + var ignores = new JdbcGithubSourceIgnoreRepository(jdbc, mapper); + var tasks = new JdbcGithubSyncTaskRepository(jdbc, mapper, new JdbcGithubSyncCapacityGate(jdbc), + new JdbcGithubSyncSourceCheckpointRepository(jdbc), ignores, + new JdbcGithubSyncPathPresenceRepository(jdbc, ignores)); + String token = "a".repeat(64); + var lease = tasks.claim(GithubSyncTaskKind.DISCOVERY, "weekly-worker", token, + now.plusSeconds(1), now.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + String snapshot = jdbc.queryForObject("SELECT snapshot_json::text FROM github_sync_task WHERE id = ?", String.class, lease.taskId()); + assertThat(mapper.readTree(snapshot).path("searchWindow").path("mode").asText()).isEqualTo("FULL_RECHECK"); + var batch = new GithubDiscoveryBatch(null, "page-1", java.util.List.of(), 0, null); + assertThat(tasks.recordDiscoveryBatch(lease.taskId(), lease.leaseGeneration(), token, batch, + "b".repeat(64), now.plusSeconds(2))) + .isEqualTo(GithubDiscoveryBatchResult.ACCEPTED); + Instant retry = now.plusSeconds(60); + assertThat(tasks.completeDiscovery(lease.taskId(), lease.leaseGeneration(), token, "page-1", false, + retry, GithubSyncDeferralReason.RATE_LIMIT, now.plusSeconds(3))) + .isEqualTo(GithubDiscoveryCompletionResult.RETRY_WAIT); + assertThat(entries.enqueueDue(now.plusSeconds(86400), 10)).isZero(); + assertThat(tasks.claim(GithubSyncTaskKind.DISCOVERY, "early-worker", "c".repeat(64), + now.plusSeconds(4), now.plusSeconds(300), platform(java.util.List.of()))).isEmpty(); + var resumed = tasks.claim(GithubSyncTaskKind.DISCOVERY, "resumed-weekly-worker", "d".repeat(64), + retry.plusSeconds(1), retry.plusSeconds(300), platform(java.util.List.of())).orElseThrow(); + assertThat(resumed.taskId()).isEqualTo(lease.taskId()); + assertThat(resumed.leaseGeneration()).isEqualTo(lease.leaseGeneration() + 1); + assertThat(jdbc.queryForObject("SELECT snapshot_json::text FROM github_sync_task WHERE id = ?", String.class, resumed.taskId())).isEqualTo(snapshot); + assertThat(jdbc.queryForObject("SELECT discovery_checkpoint FROM github_sync_task WHERE id = ?", String.class, resumed.taskId())).isEqualTo("page-1"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task WHERE dedupe_key = ?", Integer.class, "discovery:" + entry.id())).isEqualTo(2); + } + + private long platformNamespaceId() { + return jdbc.queryForObject("SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'", + Long.class); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncUploadCleanupRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncUploadCleanupRepositoryTest.java new file mode 100644 index 000000000..ebb2d17ee --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSyncUploadCleanupRepositoryTest.java @@ -0,0 +1,166 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.githubsync.GithubSyncUploadRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncUploadRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubSyncUploadCleanupRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + + @Test + void localTicketCannotSelectAnotherUploadObjectKey() { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID firstAdmission = UUID.randomUUID(); + UUID secondAdmission = UUID.randomUUID(); + UUID firstUpload = UUID.randomUUID(); + UUID secondUpload = UUID.randomUUID(); + String firstKey = "github-sync/staging/" + firstUpload + ".zip"; + String secondKey = "github-sync/staging/" + secondUpload + ".zip"; + String leaseHash = "a".repeat(64); + String commit = "b".repeat(40); + String digest = "c".repeat(64); + Instant now = Instant.now(); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, + snapshot_json, lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'LEASED', ?, + jsonb_build_object('repositoryId', 930002), 1, ?, ?, 'upload-ticket-test') + """, taskId, runId, namespaceId, "repository:930002", leaseHash, + Timestamp.from(now.plusSeconds(300))); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, 930002, ?, 'skills/first', 1), + (?, ?, ?, 930002, ?, 'skills/second', 1) + """, firstAdmission, taskId, namespaceId, commit, + secondAdmission, taskId, namespaceId, commit); + var repository = new JdbcGithubSyncUploadRepository(jdbc); + assertThat(repository.issue(taskId, 1, leaseHash, firstAdmission, commit, + "skills/first", firstUpload, firstKey, "LOCAL_STREAM", "d".repeat(64), + 12, digest, now.plusSeconds(120), now)).isTrue(); + assertThat(repository.issue(taskId, 1, leaseHash, secondAdmission, commit, + "skills/second", secondUpload, secondKey, "LOCAL_STREAM", "e".repeat(64), + 12, digest, now.plusSeconds(120), now)).isTrue(); + + assertThat(repository.beginLocalUpload(secondUpload, "d".repeat(64), now.plusSeconds(1))) + .isEmpty(); + assertThat(repository.beginLocalUpload(UUID.randomUUID(), "d".repeat(64), + now.plusSeconds(1))).isEmpty(); + assertThat(repository.beginLocalUpload(firstUpload, "d".repeat(64), now.plusSeconds(1))) + .contains(new GithubSyncUploadRepository.LocalUpload( + firstKey, 12, digest)); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_staged_upload WHERE id = ? + """, String.class, secondUpload)).isEqualTo("ISSUED"); + } + + @Test + void completedLocalUploadCanBeFailedAndReclaimedForCleanup() { + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + UUID admissionId = UUID.randomUUID(); + UUID uploadId = UUID.randomUUID(); + String objectKey = "github-sync/staging/" + uploadId + ".zip"; + String commit = "a".repeat(40); + String digest = "b".repeat(64); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'REPOSITORY_SYNC')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, dedupe_key, snapshot_json) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', ?, + jsonb_build_object('repositoryId', 930001, 'owner', 'example', 'name', 'repo')) + """, taskId, runId, namespaceId, "repository:930001"); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, 930001, ?, 'skills/demo', 1) + """, admissionId, taskId, namespaceId, commit); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 12, ?, 'UPLOADED', CURRENT_TIMESTAMP) + """, uploadId, admissionId, objectKey, digest, digest); + var repository = new JdbcGithubSyncUploadRepository(jdbc); + Instant now = Instant.now(); + + assertThat(repository.failUpload(uploadId, now)).isTrue(); + var claimed = repository.claimCleanup(now, Duration.ofHours(1), 10); + + assertThat(claimed).hasSize(1); + assertThat(claimed.getFirst().uploadId()).isEqualTo(uploadId); + assertThat(claimed.getFirst().objectKey()).isEqualTo(objectKey); + repository.markCleaned(uploadId, now); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_staged_upload WHERE id = ? + """, String.class, uploadId)).isEqualTo("CLEANED"); + assertThat(repository.failUpload(uploadId, now)).isFalse(); + + UUID acceptedAdmissionId = UUID.randomUUID(); + UUID acceptedUploadId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation, + status) + VALUES (?, ?, ?, 930001, ?, 'skills/accepted', 1, 'ACCEPTED') + """, acceptedAdmissionId, taskId, namespaceId, commit); + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, + status, expires_at) + VALUES (?, ?, ?, 'LOCAL_STREAM', ?, 12, ?, 'ACCEPTED', CURRENT_TIMESTAMP) + """, acceptedUploadId, acceptedAdmissionId, + "github-sync/staging/" + acceptedUploadId + ".zip", digest, digest); + + assertThat(repository.failUpload(acceptedUploadId, now)).isFalse(); + assertThat(jdbc.queryForObject(""" + SELECT status FROM github_sync_staged_upload WHERE id = ? + """, String.class, acceptedUploadId)).isEqualTo("ACCEPTED"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSystemCredentialRepositoryTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSystemCredentialRepositoryTest.java new file mode 100644 index 000000000..c4c079e46 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/repository/GithubSystemCredentialRepositoryTest.java @@ -0,0 +1,88 @@ +package com.iflytek.skillhub.repository; + +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSystemCredentialRepository; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +class GithubSystemCredentialRepositoryTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void quotaExhaustionDoesNotInvalidateCredentialButAuthenticationFailureDoes() { + var dataSource = new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration").load().migrate(); + var jdbc = new JdbcTemplate(dataSource); + var credentials = new JdbcGithubSystemCredentialRepository(jdbc); + Instant now = Instant.now(); + UUID credentialId = UUID.randomUUID(); + String fingerprint = "a".repeat(16); + credentials.create(new GithubSystemCredential(credentialId, "Public GitHub reader", + fingerprint, "v1", new byte[12], new byte[17], "ACTIVE", + null, null, now, now)); + long namespaceId = jdbc.queryForObject( + "SELECT id FROM namespace WHERE slug = 'github'", Long.class); + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + String leaseTokenHash = "b".repeat(64); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind) VALUES (?, ?, 'DISCOVERY')", + runId, namespaceId); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, snapshot_json, + lease_generation, lease_token_hash, lease_expires_at, leased_by) + VALUES (?, ?, ?, 'DISCOVERY', 'LEASED', ?, '{}'::jsonb, + 1, ?, ?, 'credential-worker') + """, taskId, runId, namespaceId, "discovery:" + taskId, + leaseTokenHash, Timestamp.from(now.plusSeconds(300))); + assertThat(credentials.assignToLease(taskId, 1, credentialId, fingerprint, now)).isTrue(); + + Instant resetAt = now.plusSeconds(3600); + assertThat(credentials.markRateLimited(taskId, 1, leaseTokenHash, + credentialId, resetAt, now)).isTrue(); + assertThat(credentials.recordQuotaForLease(taskId, 1, leaseTokenHash, + credentialId, "SEARCH", 0, resetAt, now)).isTrue(); + assertThat(credentials.find(credentialId).orElseThrow().status()).isEqualTo("ACTIVE"); + assertThat(credentials.availableForClaim(now, "SEARCH")).isEmpty(); + assertThat(credentials.quotas(credentialId)).singleElement() + .satisfies(quota -> assertThat(quota.remaining()).isZero()); + + assertThat(credentials.markInvalid(taskId, 1, "c".repeat(64), credentialId, now)).isFalse(); + assertThat(credentials.markInvalid(taskId, 1, leaseTokenHash, credentialId, now)).isTrue(); + assertThat(credentials.find(credentialId).orElseThrow().status()).isEqualTo("INVALID"); + assertThat(credentials.availableForClaim(resetAt.plusSeconds(1), "SEARCH")).isEmpty(); + + UUID depletedId = UUID.randomUUID(); + UUID healthyId = UUID.randomUUID(); + credentials.create(new GithubSystemCredential(depletedId, "Search exhausted", + "d".repeat(16), "v1", new byte[12], new byte[17], "ACTIVE", + null, null, now, now)); + credentials.create(new GithubSystemCredential(healthyId, "Healthy reader", + "e".repeat(16), "v1", new byte[12], new byte[17], "ACTIVE", + null, null, now, now)); + credentials.recordQuota(depletedId, "SEARCH", 0, resetAt, now); + assertThat(credentials.availableForClaim(now, "SEARCH").map(GithubSystemCredential::id)) + .contains(healthyId); + assertThat(credentials.setEnabled(healthyId, false)).isTrue(); + assertThat(credentials.availableForClaim(now, "SEARCH")).isEmpty(); + assertThat(credentials.availableForClaim(now, "CORE").map(GithubSystemCredential::id)) + .contains(depletedId); + assertThat(credentials.availableForClaim(resetAt.plusSeconds(1), "SEARCH") + .map(GithubSystemCredential::id)).contains(depletedId); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/security/SensitiveLogSanitizerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/security/SensitiveLogSanitizerTest.java index f68e819e8..3cda51cee 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/security/SensitiveLogSanitizerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/security/SensitiveLogSanitizerTest.java @@ -3,18 +3,17 @@ package com.iflytek.skillhub.security; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; class SensitiveLogSanitizerTest { private final SensitiveLogSanitizer sanitizer = new SensitiveLogSanitizer(); @Test - void shouldRedactSensitiveQueryParameters() { - String sanitized = sanitizer.sanitizeQuery("returnTo=%2Fdashboard&token=abc123&password=secret&code=xyz"); + void shouldOmitAllQueryParametersFromLoggedRequestTarget() { + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/api/v1/skills"); + request.setQueryString("returnTo=%2Fdashboard&token=abc123&custom=skill-content-marker"); - assertThat(sanitized).contains("returnTo=%2Fdashboard"); - assertThat(sanitized).contains("token=[REDACTED]"); - assertThat(sanitized).contains("password=[REDACTED]"); - assertThat(sanitized).contains("code=[REDACTED]"); + assertThat(sanitizer.sanitizeRequestTarget(request)).isEqualTo("/api/v1/skills"); } } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppServiceTest.java new file mode 100644 index 000000000..142192ccc --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubCredentialLeaseAppServiceTest.java @@ -0,0 +1,103 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import com.iflytek.skillhub.githubsync.GithubOfficialQuotaRepository; +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskLease; +import com.iflytek.skillhub.githubsync.GithubSyncTaskMode; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class GithubCredentialLeaseAppServiceTest { + @Test + void selectsQuotaResourceForEachTaskType() { + assertResource(GithubSyncTaskKind.DISCOVERY, "SEARCH_RULE", "SEARCH"); + assertResource(GithubSyncTaskKind.DISCOVERY, "SEARCH_PREVIEW", "SEARCH"); + assertResource(GithubSyncTaskKind.DISCOVERY, "CURATED_OWNER", "CORE"); + assertResource(GithubSyncTaskKind.DISCOVERY, "CURATED_REPOSITORY", "CORE"); + assertResource(GithubSyncTaskKind.REPOSITORY_SYNC, "", "CORE"); + } + + @Test + void configuredPoolUnavailableDefersWithCredentialReasonThenClearsAfterRecovery() { + var leases = mock(GithubSyncLeaseService.class); + var credentials = mock(GithubSystemCredentialRepository.class); + var cipher = mock(GithubCredentialCipher.class); + var quota = mock(GithubOfficialQuotaRepository.class); + var service = new GithubCredentialLeaseAppService(leases, credentials, cipher, new ObjectMapper(), quota); + var task = repositoryTask(); + when(leases.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "worker")) + .thenReturn(Optional.of(new GithubSyncLeaseService.LeasedTask(task, "fixture-lease"))); + when(credentials.hasConfiguredCredentials()).thenReturn(true); + when(credentials.availableForClaim(any(), eq("CORE"))).thenReturn(Optional.empty()); + + assertThat(service.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "worker")).isEmpty(); + verify(quota).deferClaim(eq(task.taskId()), eq(1L), any(), any(), eq("CREDENTIAL_UNAVAILABLE")); + verify(quota, never()).clearClaimWait(any(), org.mockito.ArgumentMatchers.anyLong()); + + var credential = new GithubSystemCredential(UUID.randomUUID(), "fixture", "fingerprint", "v1", + new byte[12], new byte[16], "ACTIVE", null, null, Instant.now(), Instant.now()); + when(credentials.availableForClaim(any(), eq("CORE"))).thenReturn(Optional.of(credential)); + when(cipher.decrypt(eq(credential.id()), eq("v1"), any(), any())).thenReturn("fixture-token"); + when(credentials.assignToLease(eq(task.taskId()), eq(1L), eq(credential.id()), eq("fingerprint"), any())).thenReturn(true); + assertThat(service.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "worker")).isPresent(); + verify(quota).clearClaimWait(task.taskId(), 1L); + } + + @Test + void removalOfConfiguredCredentialsAllowsAnonymousClaimAndClearsOldWait() { + var leases = mock(GithubSyncLeaseService.class); + var credentials = mock(GithubSystemCredentialRepository.class); + var quota = mock(GithubOfficialQuotaRepository.class); + var service = new GithubCredentialLeaseAppService(leases, credentials, mock(GithubCredentialCipher.class), new ObjectMapper(), quota); + var task = repositoryTask(); + when(leases.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "worker")) + .thenReturn(Optional.of(new GithubSyncLeaseService.LeasedTask(task, "fixture-lease"))); + when(credentials.availableForClaim(any(), eq("CORE"))).thenReturn(Optional.empty()); + when(credentials.hasConfiguredCredentials()).thenReturn(false); + assertThat(service.claim(GithubSyncTaskKind.REPOSITORY_SYNC, "worker")).isPresent(); + verify(quota).clearClaimWait(task.taskId(), 1L); + verify(quota, never()).deferClaim(any(), org.mockito.ArgumentMatchers.anyLong(), any(), any(), any()); + } + + private GithubSyncTaskLease repositoryTask() { + return new GithubSyncTaskLease(UUID.randomUUID(), UUID.randomUUID(), GithubSyncTaskKind.REPOSITORY_SYNC, + GithubSyncTaskMode.FULL, 1, null, "{}", null, null, null, "[]", List.of(), List.of(), 1, + Instant.now().plusSeconds(300)); + } + + private void assertResource(GithubSyncTaskKind kind, String entryType, String resource) { + var leases = mock(GithubSyncLeaseService.class); + var credentials = mock(GithubSystemCredentialRepository.class); + var cipher = mock(GithubCredentialCipher.class); + var service = new GithubCredentialLeaseAppService(leases, credentials, cipher, + new ObjectMapper()); + var task = new GithubSyncTaskLease(UUID.randomUUID(), UUID.randomUUID(), kind, + kind == GithubSyncTaskKind.REPOSITORY_SYNC ? GithubSyncTaskMode.FULL : null, + 1, null, "{\"entryType\":\"" + entryType + "\"}", null, null, null, + "[]", List.of(), List.of(), 1, Instant.now().plusSeconds(300)); + when(leases.claim(eq(kind), eq("fixture-worker"))) + .thenReturn(Optional.of(new GithubSyncLeaseService.LeasedTask(task, "fixture-lease"))); + when(credentials.availableForClaim(any(Instant.class), eq(resource))) + .thenReturn(Optional.empty()); + + assertThat(service.claim(kind, "fixture-worker")).isPresent(); + verify(credentials).availableForClaim(any(Instant.class), eq(resource)); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDeepDiscoveryCredentialWaitTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDeepDiscoveryCredentialWaitTest.java new file mode 100644 index 000000000..e5a4f0b81 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDeepDiscoveryCredentialWaitTest.java @@ -0,0 +1,68 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.dto.GithubDeepDiscoveryReadResponse.*; +import com.iflytek.skillhub.dto.GithubSyncWorkerStatusResponse; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfigurationRepository; +import com.iflytek.skillhub.repository.GithubDeepDiscoveryQueryRepository; +import com.iflytek.skillhub.repository.GithubSyncWorkerPresenceRepository; +import java.time.Instant; +import java.util.List; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class GithubDeepDiscoveryCredentialWaitTest { + @Test + void pacingWaitDoesNotStopInFlightDiscoveryButWorkerOutageDoes() { + var queries=mock(GithubDeepDiscoveryQueryRepository.class); + var configurations=mock(GithubDeepDiscoveryConfigurationRepository.class); + var workers=mock(GithubSyncWorkerPresenceRepository.class); + var service=new GithubDeepDiscoveryReadAppService(queries,configurations,workers); + Instant now=Instant.now(); + when(configurations.get()).thenReturn(new GithubDeepDiscoveryConfiguration(1,1,true, + GithubDeepDiscoveryConfiguration.BudgetProfile.BALANCED,List.of(),"fixture",now,null)); + when(workers.status()).thenReturn(new GithubSyncWorkerStatusResponse(now,true)); + when(queries.compatibleWorker(Channel.REPOSITORY_METADATA)).thenReturn(true); + when(queries.compatibleWorker(Channel.OWNER_EXPANSION)).thenReturn(true); + when(queries.discoveryBudgetWait()).thenReturn(new GithubDeepDiscoveryQueryRepository.BudgetWait(true,now.plusSeconds(200))); + when(queries.progressingWork(1,null,null)).thenReturn(List.of("NEW_DISCOVERY","SOURCE_UPDATE")); + assertThat(service.summary(now.minusSeconds(3600),now.minusSeconds(1)).waits()).anySatisfy(wait -> { + assertThat(wait.reason()).isEqualTo("BUDGET"); + assertThat(wait.continuingWork()).contains("NEW_DISCOVERY","SOURCE_UPDATE"); + }); + when(workers.status()).thenReturn(new GithubSyncWorkerStatusResponse(now,false)); + assertThat(service.summary(now.minusSeconds(3600),now.minusSeconds(1)).waits()).allSatisfy(wait -> assertThat(wait.continuingWork()).isEmpty()); + } + + @Test + void currentPoolStateOverridesStaleTaskReasonBeforeAnotherClaim() { + var queries = mock(GithubDeepDiscoveryQueryRepository.class); + var configurations = mock(GithubDeepDiscoveryConfigurationRepository.class); + var workers = mock(GithubSyncWorkerPresenceRepository.class); + var service = new GithubDeepDiscoveryReadAppService(queries, configurations, workers); + Instant now = Instant.now(); + when(configurations.get()).thenReturn(new GithubDeepDiscoveryConfiguration(1, 1, true, + GithubDeepDiscoveryConfiguration.BudgetProfile.BALANCED, List.of(), "fixture", now, null)); + when(workers.status()).thenReturn(new GithubSyncWorkerStatusResponse(now, true)); + var repository = new RepositorySummary(99, "fixture", "repo", now, now, + RepositoryStatus.PENDING, List.of(Channel.README), null, false, null, 3); + when(queries.detail(1, 99)).thenReturn(new RepositoryDetail(repository, null, now, + "NOT_CHECKED", List.of(), null, null)); + when(queries.repositoryWaits(1, 99L)).thenReturn(List.of(new GithubDeepDiscoveryQueryRepository.WaitEvidence( + "CREDENTIAL_UNAVAILABLE", "CANDIDATE_VALIDATION", now.plusSeconds(60)))); + when(queries.credentialPoolUnavailable("CORE")).thenReturn(true); + + var blocked = service.repository(99); + assertThat(blocked.waits()).anySatisfy(wait -> { + assertThat(wait.reason()).isEqualTo("CREDENTIAL_UNAVAILABLE"); + assertThat(wait.action()).isEqualTo("MANAGE_CREDENTIALS"); + assertThat(wait.retryAt()).isNull(); + }); + assertThat(blocked.waits()).noneMatch(wait -> wait.reason().equals("PRIMARY_RATE_LIMIT")); + + when(queries.credentialPoolUnavailable("CORE")).thenReturn(false); + assertThat(service.repository(99).waits()).noneMatch(wait -> wait.reason().equals("CREDENTIAL_UNAVAILABLE")); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDiscoveryConfigurationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDiscoveryConfigurationTest.java new file mode 100644 index 000000000..d289cb22e --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubDiscoveryConfigurationTest.java @@ -0,0 +1,134 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.dto.GithubSearchSelectorRequest; +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryEntryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSearchPreviewRepository; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Testcontainers +class GithubDiscoveryConfigurationTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @Test + void disabledInitialRuleAndImmutableQueuedSnapshotWithSideEffectFreePreview() throws Exception { + var dataSource = new DriverManagerDataSource( + POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()); + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration").load().migrate(); + var jdbc = new JdbcTemplate(dataSource); + var mapper = new ObjectMapper(); + var entries = new JdbcGithubDiscoveryEntryRepository(jdbc, mapper); + var previews = new JdbcGithubSearchPreviewRepository(jdbc, mapper); + var admin = new GithubDiscoveryAdminAppService(null, null, mapper, null, null, null); + String criteria = admin.serializeSelectors(List.of( + new GithubSearchSelectorRequest("KEYWORD", "agent skills", "CORE"))); + String nextCriteria = admin.serializeSelectors(List.of( + new GithubSearchSelectorRequest("TOPIC", "agent-skills", "EXTENSION"))); + + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_entry + WHERE entry_type = 'SEARCH_RULE' AND enabled = FALSE + """, Integer.class)).isEqualTo(1); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_sync_task", Integer.class)).isZero(); + assertThatThrownBy(() -> admin.serializeSelectors(List.of( + new GithubSearchSelectorRequest("KEYWORD", "stars:>100", "CORE")))) + .isInstanceOf(DomainBadRequestException.class); + + var created = entries.createSearchRule("Independent rule", criteria, 86400); + assertThat(created.enabled()).isFalse(); + assertThat(created.ruleVersion()).isEqualTo(1); + var enabled = entries.setEnabled(created.id(), true); + assertThat(enabled.ruleVersion()).isEqualTo(2); + assertThat(entries.enqueueDue(Instant.now().plusSeconds(1), 10)).isEqualTo(1); + String snapshot = jdbc.queryForObject(""" + SELECT snapshot_json::text FROM github_sync_task + WHERE dedupe_key = ? + """, String.class, "discovery:" + created.id()); + var queued = mapper.readTree(snapshot); + assertThat(queued.path("ruleVersion").asLong()).isEqualTo(2); + assertThat(queued.path("criteria").path("selectors").get(0).path("value").asText()) + .isEqualTo("agent skills"); + assertThat(queued.path("searchWindow").path("until").asText()).isNotBlank(); + + var edited = entries.updateSearchRule(created.id(), "Independent rule", + nextCriteria, 86400, true); + assertThat(edited.ruleVersion()).isEqualTo(3); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json::text FROM github_sync_task + WHERE dedupe_key = ? + """, String.class, "discovery:" + created.id())).isEqualTo(snapshot); + + UUID previewId = previews.queue(nextCriteria, "github-sync-publisher", Instant.now()); + assertThat(previews.find(previewId)).isPresent(); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json ->> 'entryType' FROM github_sync_task WHERE id = ? + """, String.class, previewId)).isEqualTo("SEARCH_PREVIEW"); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_source_binding", + Integer.class)).isZero(); + assertThat(jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate", + Integer.class)).isZero(); + + var budgetRule = entries.createSearchRule("Budget rule", nextCriteria, 86400); + entries.setEnabled(budgetRule.id(), true); + long namespaceId = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' + """, Long.class); + jdbc.update(""" + UPDATE github_sync_admission_policy + SET candidate_pending_limit = 1, candidate_per_lease_limit = 1 + WHERE id = TRUE + """); + long candidateId = jdbc.queryForObject(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name) + VALUES (?, 930005, 'public-owner', 'skills') RETURNING id + """, Long.class, namespaceId); + assertThat(entries.enqueueDue(Instant.now().plusSeconds(1), 10)).isZero(); + jdbc.update("UPDATE github_discovery_candidate SET status = 'BOUND' WHERE id = ?", candidateId); + assertThat(entries.enqueueDue(Instant.now().plusSeconds(1), 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT (snapshot_json ->> 'candidateBudget')::integer + FROM github_sync_task WHERE dedupe_key = ? + """, Integer.class, "discovery:" + budgetRule.id())).isEqualTo(1); + + jdbc.update(""" + UPDATE github_sync_task SET status = 'ACQUISITION_COMPLETE' + WHERE dedupe_key = ? AND status = 'QUEUED' + """, "discovery:" + budgetRule.id()); + Instant incrementalCheck = Instant.now().plusSeconds(86400 + 5); + assertThat(entries.enqueueDue(incrementalCheck, 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json #>> '{searchWindow,mode}' + FROM github_sync_task WHERE dedupe_key = ? + ORDER BY created_at DESC, id DESC LIMIT 1 + """, String.class, "discovery:" + budgetRule.id())).isEqualTo("INCREMENTAL"); + + jdbc.update(""" + UPDATE github_sync_task SET status = 'ACQUISITION_COMPLETE' + WHERE dedupe_key = ? AND status = 'QUEUED' + """, "discovery:" + budgetRule.id()); + Instant fullRecheck = Instant.now().plusSeconds(8 * 86400L); + assertThat(entries.enqueueDue(fullRecheck, 10)).isEqualTo(1); + assertThat(jdbc.queryForObject(""" + SELECT snapshot_json #>> '{searchWindow,mode}' + FROM github_sync_task WHERE dedupe_key = ? + ORDER BY created_at DESC, id DESC LIMIT 1 + """, String.class, "discovery:" + budgetRule.id())).isEqualTo("FULL_RECHECK"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncArchiveValidatorInteropTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncArchiveValidatorInteropTest.java new file mode 100644 index 000000000..f8ac4114a --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncArchiveValidatorInteropTest.java @@ -0,0 +1,68 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.domain.skill.metadata.SkillMetadataParser; +import com.iflytek.skillhub.domain.skill.validation.SkillPackageValidator; +import com.iflytek.skillhub.storage.ObjectStorageService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Assumptions; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class GithubSyncArchiveValidatorInteropTest { + + @Test + void validatesArchiveProducedByGoAtFixedPublicCommit() throws IOException, NoSuchAlgorithmException { + String fixturePath = System.getenv("SKILLHUB_GITHUB_SYNC_ARCHIVE_FIXTURE"); + Assumptions.assumeTrue(fixturePath != null && !fixturePath.isBlank(), + "requires an explicit Go-produced archive path"); + Path archive = Path.of(fixturePath); + assertThat(archive).isRegularFile(); + byte[] bytes = Files.readAllBytes(archive); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + + ObjectStorageService storage = mock(ObjectStorageService.class); + when(storage.getObject("go-produced-archive")) + .thenAnswer(invocation -> Files.newInputStream(archive)); + SkillPublishProperties policy = new SkillPublishProperties(); + var validator = new GithubSyncArchiveValidator(storage, policy, + new SkillPackageValidator(new SkillMetadataParser(), policy.getMaxFileCount(), + policy.getMaxSingleFileSize(), policy.getMaxPackageSize(), + policy.getAllowedFileExtensions())); + + var first = validator.validate("go-produced-archive", bytes.length, digest, + "skills/.system/skill-creator"); + var second = validator.validate("go-produced-archive", bytes.length, digest, + "skills/.system/skill-creator"); + + assertThat(first.contentSha256()).matches("[0-9a-f]{64}") + .isEqualTo(second.contentSha256()); + assertThat(first.evidenceJson()).contains("LICENSE.txt", "PRESENT_UNCLASSIFIED"); + + AtomicBoolean published = new AtomicBoolean(); + assertThatThrownBy(() -> validator.withValidatedEntries("go-produced-archive", bytes.length, + digest, "skills/.system/skill-creator", "0".repeat(64), + (validation, entries) -> { + published.set(true); + return null; + })) + .isInstanceOf(GithubSyncArchiveValidator.ValidationFailure.class) + .hasMessage("PACKAGE_CONTENT_CHANGED"); + assertThat(published).isFalse(); + assertThatThrownBy(() -> validator.validate("go-produced-archive", bytes.length, + "0".repeat(64), "skills/.system/skill-creator")) + .isInstanceOf(GithubSyncArchiveValidator.ValidationFailure.class) + .hasMessage("ARCHIVE_DIGEST_MISMATCH"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncLeaseFailureDetailTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncLeaseFailureDetailTest.java new file mode 100644 index 000000000..b8a08f9f2 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncLeaseFailureDetailTest.java @@ -0,0 +1,52 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.githubsync.GithubSyncFailureReason; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicyRepository; +import com.iflytek.skillhub.githubsync.GithubSyncTaskRepository; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +class GithubSyncLeaseFailureDetailTest { + private final GithubSyncTaskRepository tasks = mock(GithubSyncTaskRepository.class); + private final GithubSyncLeaseService leases = new GithubSyncLeaseService(tasks, + mock(GithubSystemCredentialRepository.class), + mock(GithubSyncSchedulePolicyRepository.class), Duration.ofMinutes(5), Duration.ofHours(24)); + private final UUID taskId = UUID.randomUUID(); + private final String token = "a".repeat(64); + + @Test + void rejectsArbitraryDiagnosticTextBeforeItReachesTaskPersistence() { + assertThatThrownBy(() -> leases.fail(taskId, 1, token, + GithubSyncFailureReason.RETRYABLE_ACQUISITION, "SYNTHETIC_SECRET_123", null, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("detail must be a supported diagnostic code"); + verifyNoInteractions(tasks); + } + + @Test + void acceptsOnlyWorkerAndKnownGithubDiagnostics() { + leases.fail(taskId, 1, token, GithubSyncFailureReason.RETRYABLE_ACQUISITION, + "WORKER_ERROR", null, null); + leases.fail(taskId, 1, token, GithubSyncFailureReason.RETRYABLE_ACQUISITION, + "GITHUB_HTTP_403:GITHUB_RATE_LIMITED", null, null); + + verify(tasks).fail(eq(taskId), eq(1L), anyString(), + eq(GithubSyncFailureReason.RETRYABLE_ACQUISITION), eq("WORKER_ERROR"), isNull(), any()); + verify(tasks).fail(eq(taskId), eq(1L), anyString(), + eq(GithubSyncFailureReason.RETRYABLE_ACQUISITION), + eq("GITHUB_HTTP_403:GITHUB_RATE_LIMITED"), isNull(), any()); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppServiceTest.java new file mode 100644 index 000000000..487962983 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncReviewQueueAppServiceTest.java @@ -0,0 +1,79 @@ +package com.iflytek.skillhub.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.dto.PageResponse; +import com.iflytek.skillhub.dto.ReviewTaskResponse; +import com.iflytek.skillhub.repository.GithubSyncReviewSourceQueryRepository; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class GithubSyncReviewQueueAppServiceTest { + @Mock private ReviewPortalAppService reviews; + @Mock private GithubSyncReviewSourceQueryRepository sources; + + @Test + void addsSourceMetadataOnlyToAuthorizedReviewPage() { + ReviewTaskResponse review = review(17L); + PageResponse visible = new PageResponse<>(List.of(review), 1, 0, 20); + UUID ingestionId = UUID.randomUUID(); + UUID runId = UUID.randomUUID(); + Map roles = Map.of(42L, NamespaceRole.ADMIN); + when(reviews.listSyncReviews("PENDING", 42L, 0, 20, "DESC", "reviewer", roles)) + .thenReturn(visible); + when(sources.findQueueSources(List.of(17L))).thenReturn(Map.of(17L, + new GithubSyncReviewSourceQueryRepository.QueueSource(ingestionId, runId, + 981L, "owner", "repository", "skills/example", "a".repeat(40)))); + + var page = new GithubSyncReviewQueueAppService(reviews, sources).list( + "PENDING", 42L, 0, 20, "DESC", "reviewer", roles); + + assertThat(page.total()).isEqualTo(1); + assertThat(page.items()).hasSize(1); + assertThat(page.items().getFirst().review()).isSameAs(review); + assertThat(page.items().getFirst().runId()).isEqualTo(runId); + assertThat(page.items().getFirst().skillPath()).isEqualTo("skills/example"); + verify(sources).findQueueSources(List.of(17L)); + } + + @Test + void doesNotQuerySourceMetadataForEmptyPage() { + when(reviews.listSyncReviews("PENDING", null, 0, 20, "DESC", "reviewer", Map.of())) + .thenReturn(new PageResponse<>(List.of(), 0, 0, 20)); + + var page = new GithubSyncReviewQueueAppService(reviews, sources).list( + "PENDING", null, 0, 20, "DESC", "reviewer", Map.of()); + + assertThat(page.items()).isEmpty(); + verify(sources).findQueueSources(List.of()); + } + + @Test + void rejectsReviewWithoutItsIngestionSource() { + when(reviews.listSyncReviews("PENDING", null, 0, 20, "DESC", "reviewer", Map.of())) + .thenReturn(new PageResponse<>(List.of(review(17L)), 1, 0, 20)); + when(sources.findQueueSources(List.of(17L))).thenReturn(Map.of()); + + assertThatThrownBy(() -> new GithubSyncReviewQueueAppService(reviews, sources).list( + "PENDING", null, 0, 20, "DESC", "reviewer", Map.of())) + .isInstanceOf(IllegalStateException.class); + } + + private ReviewTaskResponse review(long id) { + return new ReviewTaskResponse(id, 9L, "team", "example", "1.0.0", "PENDING", + "github-sync-publisher", null, null, null, null, + Instant.parse("2026-09-29T00:00:00Z"), null, + "SKILL_VERSION", 8L, 9L, "example", "GITHUB_SYNC"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncRootEvidenceValidationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncRootEvidenceValidationTest.java new file mode 100644 index 000000000..d27c2df6c --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncRootEvidenceValidationTest.java @@ -0,0 +1,123 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.domain.skill.metadata.SkillMetadataParser; +import com.iflytek.skillhub.domain.skill.validation.SkillPackageValidator; +import com.iflytek.skillhub.storage.ObjectStorageService; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class GithubSyncRootEvidenceValidationTest { + @Test + void rootLicenseAndNoticeAreRehashedAndRecordedWithRootScope() throws Exception { + byte[] license = "Root license evidence".getBytes(StandardCharsets.UTF_8); + byte[] notice = "Root notice evidence".getBytes(StandardCharsets.UTF_8); + var result = validate(List.of(evidence("LICENSE", license, true), evidence("NOTICE", notice, true)), + Map.of("LICENSE", license, "NOTICE", notice)); + assertThat(result.evidenceJson()).contains("REPOSITORY_ROOT", "LICENSE", "NOTICE", "PRESENT_UNCLASSIFIED"); + var changed = validate(List.of(evidence("LICENSE", notice, true)), Map.of("LICENSE", notice)); + assertThat(changed.contentSha256()).isNotEqualTo(result.contentSha256()); + } + + @Test + void tamperedDigestAndUndeclaredRootFileAreRejected() throws Exception { + byte[] original = "Original".getBytes(StandardCharsets.UTF_8); + assertThatThrownBy(() -> validate(List.of(evidence("LICENSE", original, true)), + Map.of("LICENSE", "Tampered".getBytes(StandardCharsets.UTF_8)))) + .hasMessage("LICENSE_EVIDENCE_INVALID"); + assertThatThrownBy(() -> validate(List.of(), Map.of("NOTICE", original))) + .hasMessage("LICENSE_EVIDENCE_INVALID"); + } + + @Test + void sixteenRootDocumentsAreAllowedAndSeventeenthIsRejected() throws Exception { + var records = new ArrayList>(); + for (int i = 0; i < 16; i++) records.add(evidence("LICENSE-" + i, new byte[65537], false)); + assertThat(validate(records, Map.of()).evidenceJson()).contains("PRESENT_UNPREVIEWABLE"); + records.add(evidence("LICENSE-16", new byte[65537], false)); + assertThatThrownBy(() -> validate(records, Map.of())).hasMessage("LICENSE_EVIDENCE_INVALID"); + } + + @Test + void singleRootFileBoundaryRequiresCorrectBundlingDecision() throws Exception { + byte[] exact = new byte[65536]; + Arrays.fill(exact, (byte) 'x'); + assertThat(validate(List.of(evidence("LICENSE", exact, true)), Map.of("LICENSE", exact)).evidenceJson()) + .contains("PRESENT_UNCLASSIFIED"); + byte[] over = Arrays.copyOf(exact, 65537); + assertThat(validate(List.of(evidence("LICENSE", over, false)), Map.of()).evidenceJson()) + .contains("PRESENT_UNPREVIEWABLE"); + assertThatThrownBy(() -> validate(List.of(evidence("LICENSE", over, true)), Map.of("LICENSE", over))) + .isInstanceOf(GithubSyncArchiveValidator.ValidationFailure.class); + assertThatThrownBy(() -> validate(List.of(evidence("LICENSE", exact, false)), Map.of())) + .hasMessage("LICENSE_EVIDENCE_INVALID"); + } + + @Test + void totalRootEvidenceBoundaryIsIndependentOfSkillPackageAllowance() throws Exception { + byte[] content = new byte[65536]; + var records = new ArrayList>(); + var files = new LinkedHashMap(); + for (int i = 0; i < 4; i++) { + records.add(evidence("LICENSE-" + i, content, true)); + files.put("LICENSE-" + i, content); + } + assertThat(validate(records, files).evidenceJson()).contains("PRESENT_UNCLASSIFIED"); + records.add(evidence("NOTICE", new byte[]{1}, false)); + assertThat(validate(records, files).evidenceJson()).contains("PRESENT_UNPREVIEWABLE"); + records.set(4, evidence("NOTICE", new byte[]{1}, true)); + files.put("NOTICE", new byte[]{1}); + assertThatThrownBy(() -> validate(records, files)).hasMessage("LICENSE_EVIDENCE_LIMIT"); + } + + private Map evidence(String name, byte[] bytes, boolean bundled) throws Exception { + var digest = MessageDigest.getInstance("SHA-1"); + digest.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.UTF_8)); + return Map.of("path", name, "gitSha", HexFormat.of().formatHex(digest.digest(bytes)), + "size", bytes.length, "bundled", bundled); + } + + private GithubSyncArchiveValidator.ArchiveValidation validate(List> records, + Map files) throws Exception { + var output = new ByteArrayOutputStream(); + try (var zip = new ZipOutputStream(output)) { + entry(zip, "SKILL.md", "---\nname: root-evidence\ndescription: Root evidence validation fixture\n---\n# Fixture\n".getBytes(StandardCharsets.UTF_8)); + entry(zip, "__skillhub_sync_evidence__.json", new ObjectMapper().writeValueAsBytes( + Map.of("formatVersion", 1, "rootFiles", records))); + for (var file : files.entrySet()) entry(zip, "__skillhub_sync_evidence__/" + file.getKey(), file.getValue()); + } + byte[] archive = output.toByteArray(); + var storage = mock(ObjectStorageService.class); + when(storage.getObject("fixture")).thenAnswer(ignored -> new ByteArrayInputStream(archive)); + var policy = new SkillPublishProperties(); + var validator = new GithubSyncArchiveValidator(storage, policy, + new SkillPackageValidator(new SkillMetadataParser(), policy.getMaxFileCount(), + policy.getMaxSingleFileSize(), policy.getMaxPackageSize(), policy.getAllowedFileExtensions())); + return validator.validate("fixture", archive.length, + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(archive)), "skills/root-evidence"); + } + + private void entry(ZipOutputStream zip, String path, byte[] bytes) throws Exception { + zip.putNextEntry(new ZipEntry(path)); + zip.write(bytes); + zip.closeEntry(); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncUploadAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncUploadAppServiceTest.java new file mode 100644 index 000000000..b169a19b7 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubSyncUploadAppServiceTest.java @@ -0,0 +1,287 @@ +package com.iflytek.skillhub.service; + +import com.iflytek.skillhub.config.SkillPublishProperties; +import com.iflytek.skillhub.githubsync.GithubPackageReportResult; +import com.iflytek.skillhub.githubsync.GithubSyncUploadRepository; +import com.iflytek.skillhub.storage.ObjectMetadata; +import com.iflytek.skillhub.storage.ObjectStorageService; +import com.iflytek.skillhub.storage.PresignedUpload; +import com.iflytek.skillhub.storage.StorageAccessException; +import com.iflytek.skillhub.storage.StorageProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GithubSyncUploadAppServiceTest { + private static final String COMMIT = "a".repeat(40); + private static final String DIGEST = "b".repeat(64); + private static final String TOKEN = "c".repeat(64); + private static final String OBJECT_KEY = "github-sync/staging/package.zip"; + + @Mock private GithubSyncUploadRepository uploads; + @Mock private ObjectStorageService storage; + + @Test + void localStorageFailureDoesNotPropagateUntrustedExceptionContent() throws IOException { + String marker = "STORAGE_ERROR_WITH_SKILL_BODY_AND_TOKEN"; + UUID uploadId = UUID.randomUUID(); + String ticket = "d".repeat(64); + when(uploads.beginLocalUpload(eq(uploadId), any(), any())).thenReturn(Optional.of( + new GithubSyncUploadRepository.LocalUpload(OBJECT_KEY, 2, DIGEST))); + when(uploads.failUpload(eq(uploadId), any())).thenReturn(true); + doThrow(new StorageAccessException("putObject", OBJECT_KEY, + new IllegalStateException(marker))).when(storage).putObject( + eq(OBJECT_KEY), any(InputStream.class), eq(2L), eq("application/zip")); + + assertThatThrownBy(() -> service(new SkillPublishProperties()).uploadLocal( + uploadId, ticket, new ByteArrayInputStream(new byte[] {1, 2}))) + .isInstanceOf(StorageAccessException.class) + .hasMessageNotContaining(marker) + .hasNoCause(); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + verify(storage).deleteObject(OBJECT_KEY); + } + + @Test + void s3SigningFailureDoesNotPropagateUntrustedExceptionContent() { + String marker = "S3_ERROR_WITH_SKILL_BODY_AND_TOKEN"; + var properties = new StorageProperties(); + properties.setProvider("s3"); + when(uploads.issue(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), any(), any(), eq("S3_PRESIGNED_PUT"), isNull(), + eq(12L), eq(DIGEST), any(), any())).thenReturn(true); + when(storage.presignPut(any(), eq(Duration.ofMinutes(5)))) + .thenThrow(new StorageAccessException("presignPut", OBJECT_KEY, + new IllegalStateException(marker))); + var service = new GithubSyncUploadAppService(uploads, storage, properties, + new SkillPublishProperties(), Duration.ofMinutes(5)); + + assertThatThrownBy(() -> service.authorize(UUID.randomUUID(), 1, TOKEN, + UUID.randomUUID(), COMMIT, "skills/demo", 12, DIGEST)) + .isInstanceOf(StorageAccessException.class) + .hasMessageNotContaining(marker) + .hasNoCause(); + verify(uploads).failUpload(any(), any(Instant.class)); + } + + @Test + void reportStorageFailureDoesNotPropagateUntrustedExceptionContent() { + String marker = "METADATA_ERROR_WITH_SKILL_BODY_AND_TOKEN"; + UUID uploadId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(12L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, null, OBJECT_KEY)); + when(storage.exists(OBJECT_KEY)).thenReturn(true); + when(storage.getMetadata(OBJECT_KEY)).thenThrow(new IllegalStateException(marker)); + + assertThatThrownBy(() -> service(new SkillPublishProperties()).report( + UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), COMMIT, "skills/demo", + uploadId, 12, DIGEST)) + .isInstanceOf(StorageAccessException.class) + .hasMessageNotContaining(marker) + .hasNoCause(); + verify(uploads, never()).acceptReport(any(), anyLong(), any(), any(), any(), + any(), any(), anyLong(), any(), any()); + } + + @Test + void s3AuthorizationSignsOnlyTheServerGeneratedStagingKey() { + UUID taskId = UUID.randomUUID(); + UUID admissionId = UUID.randomUUID(); + var properties = new StorageProperties(); + properties.setProvider("s3"); + when(uploads.issue(eq(taskId), eq(1L), any(), eq(admissionId), eq(COMMIT), + eq("skills/demo"), any(), any(), eq("S3_PRESIGNED_PUT"), isNull(), + eq(12L), eq(DIGEST), any(), any())).thenReturn(true); + when(storage.presignPut(any(), eq(Duration.ofMinutes(5)))) + .thenReturn(Optional.of(new PresignedUpload("https://storage.example/upload", Map.of()))); + var service = new GithubSyncUploadAppService(uploads, storage, properties, + new SkillPublishProperties(), Duration.ofMinutes(5)); + + var authorization = service.authorize(taskId, 1, TOKEN, admissionId, COMMIT, + "skills/demo", 12, DIGEST).orElseThrow(); + + ArgumentCaptor uploadIds = ArgumentCaptor.forClass(UUID.class); + ArgumentCaptor objectKeys = ArgumentCaptor.forClass(String.class); + verify(uploads).issue(eq(taskId), eq(1L), any(), eq(admissionId), eq(COMMIT), + eq("skills/demo"), uploadIds.capture(), objectKeys.capture(), + eq("S3_PRESIGNED_PUT"), isNull(), eq(12L), eq(DIGEST), any(), any()); + assertThat(uploadIds.getValue()).isEqualTo(authorization.uploadId()); + assertThat(objectKeys.getValue()).isEqualTo( + "github-sync/staging/" + authorization.uploadId() + ".zip"); + verify(storage).presignPut(objectKeys.getValue(), Duration.ofMinutes(5)); + } + + @Test + void localUploadCannotUseTicketForAnotherUploadId() throws IOException { + UUID wrongUploadId = UUID.randomUUID(); + String ticket = "d".repeat(64); + when(uploads.beginLocalUpload(eq(wrongUploadId), any(), any())) + .thenReturn(Optional.empty()); + + var result = service(new SkillPublishProperties()).uploadLocal( + wrongUploadId, ticket, new ByteArrayInputStream(new byte[] {1, 2})); + + assertThat(result).isEqualTo(GithubSyncUploadAppService.LocalUploadResult.TICKET_INVALID); + verifyNoInteractions(storage); + } + + @Test + void removesTicketObjectWhenPackagePolicyShrinksBeforeReport() { + SkillPublishProperties policy = new SkillPublishProperties(); + policy.setMaxPackageSize(10); + UUID uploadId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(12L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, null, OBJECT_KEY)); + when(uploads.failUpload(eq(uploadId), any())).thenReturn(true); + + var result = service(policy).report(UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), + COMMIT, "skills/demo", uploadId, 12, DIGEST); + + assertThat(result.status()).isEqualTo(GithubPackageReportResult.Status.TOO_LARGE); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + verify(storage).deleteObject(OBJECT_KEY); + verify(uploads, never()).acceptReport(any(), anyLong(), any(), any(), any(), + any(), any(), anyLong(), any(), any()); + } + + @Test + void removesOversizedS3ObjectBeforeAcceptingReport() { + SkillPublishProperties policy = new SkillPublishProperties(); + policy.setMaxPackageSize(10); + UUID uploadId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(8L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, null, OBJECT_KEY)); + when(uploads.failUpload(eq(uploadId), any())).thenReturn(true); + when(storage.exists(OBJECT_KEY)).thenReturn(true); + when(storage.getMetadata(OBJECT_KEY)).thenReturn( + new ObjectMetadata(12, "application/zip", Instant.now())); + + var result = service(policy).report(UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), + COMMIT, "skills/demo", uploadId, 8, DIGEST); + + assertThat(result.status()).isEqualTo(GithubPackageReportResult.Status.TOO_LARGE); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + verify(storage).deleteObject(OBJECT_KEY); + } + + @Test + void existingAcceptedReportRemainsIdempotentAfterPolicyShrinks() { + SkillPublishProperties policy = new SkillPublishProperties(); + policy.setMaxPackageSize(10); + UUID uploadId = UUID.randomUUID(); + UUID ingestionId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(12L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.RECEIVED, ingestionId, null)); + + var result = service(policy).report(UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), + COMMIT, "skills/demo", uploadId, 12, DIGEST); + + assertThat(result.ingestionId()).isEqualTo(ingestionId); + verifyNoInteractions(storage); + } + + @Test + void failedImmediateDeleteLeavesOversizedUploadForCompensation() { + SkillPublishProperties policy = new SkillPublishProperties(); + policy.setMaxPackageSize(10); + UUID uploadId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(12L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, null, OBJECT_KEY)); + when(uploads.failUpload(eq(uploadId), any())).thenReturn(true); + doThrow(new IllegalStateException("temporary storage failure")) + .when(storage).deleteObject(OBJECT_KEY); + + var result = service(policy).report(UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), + COMMIT, "skills/demo", uploadId, 12, DIGEST); + + assertThat(result.status()).isEqualTo(GithubPackageReportResult.Status.TOO_LARGE); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + } + + @Test + void acceptedUploadIsNeverDeletedByConcurrentOversizeReport() { + SkillPublishProperties policy = new SkillPublishProperties(); + policy.setMaxPackageSize(10); + UUID uploadId = UUID.randomUUID(); + when(uploads.lookupReport(any(), eq(1L), any(), any(), eq(COMMIT), + eq("skills/demo"), eq(uploadId), eq(12L), eq(DIGEST), any())) + .thenReturn(new GithubPackageReportResult( + GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, null, OBJECT_KEY)); + + var result = service(policy).report(UUID.randomUUID(), 1, TOKEN, UUID.randomUUID(), + COMMIT, "skills/demo", uploadId, 12, DIGEST); + + assertThat(result.status()).isEqualTo(GithubPackageReportResult.Status.TOO_LARGE); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + verifyNoInteractions(storage); + } + + @Test + void localStreamRejectsBytesBeyondTicketSize() throws IOException { + UUID uploadId = UUID.randomUUID(); + String ticket = "d".repeat(64); + when(uploads.beginLocalUpload(eq(uploadId), any(), any())).thenReturn(Optional.of( + new GithubSyncUploadRepository.LocalUpload(OBJECT_KEY, 2, DIGEST))); + when(uploads.failUpload(eq(uploadId), any())).thenReturn(true); + doAnswer(invocation -> { + InputStream body = invocation.getArgument(1); + try { + body.transferTo(OutputStream.nullOutputStream()); + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + return null; + }).when(storage).putObject(eq(OBJECT_KEY), any(InputStream.class), eq(2L), + eq("application/zip")); + + var result = service(new SkillPublishProperties()).uploadLocal(uploadId, ticket, + new ByteArrayInputStream(new byte[] {1, 2, 3})); + + assertThat(result).isEqualTo(GithubSyncUploadAppService.LocalUploadResult.TOO_LARGE); + verify(uploads).failUpload(eq(uploadId), any(Instant.class)); + verify(storage).deleteObject(OBJECT_KEY); + verify(uploads, never()).finishLocalUpload(eq(uploadId), any()); + } + + private GithubSyncUploadAppService service(SkillPublishProperties policy) { + return new GithubSyncUploadAppService(uploads, storage, new StorageProperties(), + policy, Duration.ofMinutes(5)); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubTeamSourceAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubTeamSourceAppServiceTest.java new file mode 100644 index 000000000..ef585d4a2 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/GithubTeamSourceAppServiceTest.java @@ -0,0 +1,216 @@ +package com.iflytek.skillhub.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.domain.audit.AuditLogService; +import com.iflytek.skillhub.domain.namespace.Namespace; +import com.iflytek.skillhub.domain.namespace.NamespaceMember; +import com.iflytek.skillhub.domain.namespace.NamespaceRole; +import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.dto.GithubTeamRepositoryRequest; +import com.iflytek.skillhub.infra.jpa.JdbcGithubDiscoveryEntryRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSourceIgnoreRepository; +import com.iflytek.skillhub.infra.jpa.JdbcGithubSyncCapacityGate; +import com.iflytek.skillhub.infra.jpa.NamespaceJpaRepository; +import com.iflytek.skillhub.infra.jpa.NamespaceMemberJpaRepository; +import com.iflytek.skillhub.observability.RequestIdAccessor; +import com.iflytek.skillhub.repository.GithubSyncManagementQueryRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +@DataJpaTest +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@ActiveProfiles("test") +@Testcontainers +class GithubTeamSourceAppServiceTest { + @Container + private static final PostgreSQLContainer POSTGRES = + new PostgreSQLContainer<>("postgres:16-alpine"); + + @DynamicPropertySource + static void postgres(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect"); + registry.add("spring.flyway.enabled", () -> true); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); + } + + @Autowired private JdbcTemplate jdbc; + @Autowired private NamespaceJpaRepository namespaces; + @Autowired private NamespaceMemberJpaRepository members; + + @Test + void teamRunsAndSourcesAreVisibleOnlyToThatTeamAdministrators() { + jdbc.update(""" + INSERT INTO user_account (id, display_name) + VALUES ('github-run-owner-one', 'Owner One'), + ('github-run-member-one', 'Member One'), + ('github-run-owner-two', 'Owner Two') + """); + Namespace first = namespaces.saveAndFlush(new Namespace( + "github-runs-first", "GitHub Runs First", "github-run-owner-one")); + Namespace second = namespaces.saveAndFlush(new Namespace( + "github-runs-second", "GitHub Runs Second", "github-run-owner-two")); + members.saveAndFlush(new NamespaceMember(first.getId(), "github-run-owner-one", NamespaceRole.OWNER)); + members.saveAndFlush(new NamespaceMember(first.getId(), "github-run-member-one", NamespaceRole.MEMBER)); + members.saveAndFlush(new NamespaceMember(second.getId(), "github-run-owner-two", NamespaceRole.OWNER)); + long firstSource = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 930005, 'first-owner', 'first-repository', 'NAMESPACE_ONLY') + RETURNING id + """, Long.class, first.getId()); + long secondSource = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, 930006, 'second-owner', 'second-repository', 'NAMESPACE_ONLY') + RETURNING id + """, Long.class, second.getId()); + UUID firstRun = UUID.randomUUID(); + UUID secondRun = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind) + VALUES (?, ?, 'REPOSITORY_SYNC'), (?, ?, 'DISCOVERY') + """, firstRun, first.getId(), secondRun, second.getId()); + UUID discoveryTask = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, snapshot_json, + retry_at, last_error_code) + VALUES (?, ?, ?, 'DISCOVERY', 'RETRY_WAIT', 'discovery-run-error', '{}'::jsonb, + CURRENT_TIMESTAMP, 'GITHUB_RATE_LIMITED') + """, discoveryTask, secondRun, second.getId()); + var management = new GithubSyncManagementQueryRepository(jdbc, + new JdbcGithubSourceIgnoreRepository(jdbc, new ObjectMapper()), + new JdbcGithubSyncCapacityGate(jdbc)); + var service = new GithubTeamSourceAppService(namespaces, members, + new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()), management, + mock(AuditLogService.class), mock(RequestIdAccessor.class)); + + assertThat(service.resolvedSources(first.getSlug(), "github-run-owner-one", 0, 20).items()) + .singleElement().satisfies(source -> assertThat(source.id()).isEqualTo(firstSource)); + assertThat(service.resolvedSources(second.getSlug(), "github-run-owner-two", 0, 20).items()) + .singleElement().satisfies(source -> assertThat(source.id()).isEqualTo(secondSource)); + assertThatThrownBy(() -> service.resolvedSources(first.getSlug(), "github-run-member-one", 0, 20)) + .isInstanceOf(DomainForbiddenException.class); + assertThatThrownBy(() -> service.resolvedSources(first.getSlug(), "github-run-owner-two", 0, 20)) + .isInstanceOf(DomainForbiddenException.class); + assertThat(service.skillPaths(first.getSlug(), "github-run-owner-one", firstSource, 0, 20) + .items()).isEmpty(); + assertThatThrownBy(() -> service.skillPaths(first.getSlug(), "github-run-owner-one", + secondSource, 0, 20)).isInstanceOf(DomainNotFoundException.class); + assertThatThrownBy(() -> service.skillPaths(first.getSlug(), "github-run-member-one", + firstSource, 0, 20)).isInstanceOf(DomainForbiddenException.class); + assertThat(service.runs(first.getSlug(), "github-run-owner-one", 0, 20).items()) + .singleElement().satisfies(run -> assertThat(run.id()).isEqualTo(firstRun)); + assertThat(service.runs(second.getSlug(), "github-run-owner-two", 0, 20).items()) + .singleElement().satisfies(run -> { + assertThat(run.id()).isEqualTo(secondRun); + assertThat(run.lastErrorCode()).isEqualTo("GITHUB_RATE_LIMITED"); + assertThat(run.nextRetryAt()).isNotNull(); + }); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', retry_at = NULL + WHERE id = ? + """, discoveryTask); + assertThat(service.runs(second.getSlug(), "github-run-owner-two", 0, 20).items()) + .singleElement().satisfies(run -> assertThat(run.lastErrorCode()).isNull()); + assertThatThrownBy(() -> service.runs(first.getSlug(), "github-run-member-one", 0, 20)) + .isInstanceOf(DomainForbiddenException.class); + assertThatThrownBy(() -> service.runs(first.getSlug(), "github-run-owner-two", 0, 20)) + .isInstanceOf(DomainForbiddenException.class); + } + + @Test + void teamAdminControlsSourceWhilePublishedSkillKeepsItsVisibility() { + jdbc.update(""" + INSERT INTO user_account (id, display_name) + VALUES ('github-team-owner', 'Owner'), ('github-team-member', 'Member') + """); + Namespace team = namespaces.saveAndFlush(new Namespace( + "github-managed-team", "GitHub Managed Team", "github-team-owner")); + members.saveAndFlush(new NamespaceMember(team.getId(), "github-team-owner", NamespaceRole.OWNER)); + members.saveAndFlush(new NamespaceMember(team.getId(), "github-team-member", NamespaceRole.MEMBER)); + var service = new GithubTeamSourceAppService(namespaces, members, + new JdbcGithubDiscoveryEntryRepository(jdbc, new ObjectMapper()), null, + mock(AuditLogService.class), mock(RequestIdAccessor.class)); + var context = new AuditRequestContext("127.0.0.1", "test"); + + assertThatThrownBy(() -> service.create(team.getSlug(), + new GithubTeamRepositoryRequest("public-owner", "skills", null), + "github-team-member", context)).isInstanceOf(DomainForbiddenException.class); + assertThatThrownBy(() -> service.list(team.getSlug(), "github-team-member")) + .isInstanceOf(DomainForbiddenException.class); + assertThat(jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_entry WHERE namespace_id = ? + """, Integer.class, team.getId())).isZero(); + + var configured = service.create(team.getSlug(), + new GithubTeamRepositoryRequest("public-owner", "skills", null), + "github-team-owner", context); + assertThat(configured.sourceVisibility()).isEqualTo("NAMESPACE_ONLY"); + assertThat(service.list(team.getSlug(), "github-team-owner")) + .singleElement().satisfies(entry -> assertThat(entry.id()).isEqualTo(configured.id())); + long repositoryId = 930004L; + long sourceId = jdbc.queryForObject(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, visibility) + VALUES (?, ?, 'public-owner', 'skills', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, team.getId(), repositoryId); + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + VALUES (?, 'TEAM_CONFIG', ?) + """, sourceId, configured.id().toString()); + long skillId = jdbc.queryForObject(""" + INSERT INTO skill (namespace_id, slug, owner_id, visibility) + VALUES (?, 'published-skill', 'github-team-owner', 'NAMESPACE_ONLY') RETURNING id + """, Long.class, team.getId()); + long versionId = jdbc.queryForObject(""" + INSERT INTO skill_version (skill_id, version, status, created_by) + VALUES (?, '1.0.0', 'PUBLISHED', 'github-team-owner') RETURNING id + """, Long.class, skillId); + jdbc.update("UPDATE skill SET latest_version_id = ? WHERE id = ?", versionId, skillId); + + var updated = service.setVisibility(team.getSlug(), configured.id(), "PUBLIC", + "github-team-owner", context); + assertThat(updated.sourceVisibility()).isEqualTo("PUBLIC"); + assertThat(jdbc.queryForObject(""" + SELECT visibility FROM github_source_binding WHERE id = ? + """, String.class, sourceId)).isEqualTo("PUBLIC"); + assertThat(jdbc.queryForObject("SELECT visibility FROM skill WHERE id = ?", + String.class, skillId)).isEqualTo("NAMESPACE_ONLY"); + assertThat(jdbc.queryForObject("SELECT latest_version_id FROM skill WHERE id = ?", + Long.class, skillId)).isEqualTo(versionId); + + UUID entryId = configured.id(); + assertThatThrownBy(() -> service.setEnabled(team.getSlug(), entryId, false, + "github-team-member", context)).isInstanceOf(DomainForbiddenException.class); + service.setEnabled(team.getSlug(), entryId, false, "github-team-owner", context); + assertThat(jdbc.queryForObject("SELECT status FROM github_source_binding WHERE id = ?", + String.class, sourceId)).isEqualTo("PAUSED"); + service.setEnabled(team.getSlug(), entryId, true, "github-team-owner", context); + assertThat(jdbc.queryForObject("SELECT status FROM github_source_binding WHERE id = ?", + String.class, sourceId)).isEqualTo("ACTIVE"); + assertThat(jdbc.queryForObject("SELECT visibility FROM skill WHERE id = ?", + String.class, skillId)).isEqualTo("NAMESPACE_ONLY"); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/ReviewSkillDetailAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/ReviewSkillDetailAppServiceTest.java index 936c59c29..e48dbc625 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/ReviewSkillDetailAppServiceTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/ReviewSkillDetailAppServiceTest.java @@ -74,6 +74,7 @@ class ReviewSkillDetailAppServiceTest { Namespace namespace = createNamespace(20L, "team-a"); Skill skill = createSkill(101L, 20L, "skill-a"); SkillVersion pending = createVersion(101L, 101L, "1.2.0", SkillVersionStatus.PENDING_REVIEW); + setField(pending, "githubSyncRevision", 3); pending.setParsedMetadataJson(""" {"name":"skill-a","description":"待审摘要","displayNameOverride":"网页新标题","complianceSnapshot":{"schemaVersion":"1.0","items":[{"standard":"mitre-attack","version":"v19.1","controlId":"T1059","evidence":[]}],"digest":"sha256:demo"}} """); @@ -103,6 +104,9 @@ class ReviewSkillDetailAppServiceTest { assertThat(response.skill().displayName()).isEqualTo("网页新标题"); assertThat(response.skill().summary()).isEqualTo("待审摘要"); assertThat(response.skill().ownerPreviewVersion().displayName()).isEqualTo("网页新标题"); + assertThat(response.versions().get(0).syncRevision()).isEqualTo(3); + assertThat(response.versions().get(1).syncRevision()).isNull(); + assertThat(response.downloadUrl()).isEqualTo("/api/v1/reviews/42/download"); assertThat(response.documentationPath()).isEqualTo("README.md"); assertThat(response.documentationContent()).isEqualTo("# demo"); @@ -150,6 +154,7 @@ class ReviewSkillDetailAppServiceTest { Namespace namespace = createNamespace(20L, "team-a"); Skill skill = createSkill(101L, 20L, "skill-a"); SkillVersion pending = createVersion(101L, 101L, "1.2.0", SkillVersionStatus.PENDING_REVIEW); + setField(pending, "githubSyncRevision", 3); SkillDownloadService.DownloadResult result = new SkillDownloadService.DownloadResult( () -> new ByteArrayInputStream("zip".getBytes()), "skill-a-1.2.0.zip", @@ -163,7 +168,7 @@ class ReviewSkillDetailAppServiceTest { given(namespaceRepository.findById(20L)).willReturn(Optional.of(namespace)); given(rbacService.getUserRoleCodes("admin")).willReturn(Set.of("SKILL_ADMIN")); given(reviewService.canViewReview(task, "admin", namespace.getType(), Map.of(), Set.of("SKILL_ADMIN"))).willReturn(true); - given(skillQueryService.getReviewSkillSnapshot(101L)).willReturn( + given(skillQueryService.getReviewSkillSnapshot(101L, false)).willReturn( new SkillQueryService.ReviewSkillSnapshotDTO( skill, "Owner", @@ -202,8 +207,10 @@ class ReviewSkillDetailAppServiceTest { given(namespaceRepository.findById(20L)).willReturn(Optional.of(namespace)); given(rbacService.getUserRoleCodes("admin")).willReturn(Set.of("SKILL_ADMIN")); given(reviewService.canViewReview(task, "admin", namespace.getType(), Map.of(), Set.of("SKILL_ADMIN"))).willReturn(true); - given(skillQueryService.getReviewSkillSnapshot(100L)).willReturn(new SkillQueryService.ReviewSkillSnapshotDTO( - skill, "Owner", published, published, List.of(published), List.of(createFile(5L, 100L, "SKILL.md")), "SKILL.md", "private package")); + var snapshot = new SkillQueryService.ReviewSkillSnapshotDTO( + skill, "Owner", published, published, List.of(published), List.of(createFile(5L, 100L, "SKILL.md")), "SKILL.md", "private package"); + given(skillQueryService.getReviewSkillSnapshot(100L)).willReturn(snapshot); + given(skillQueryService.getReviewSkillSnapshot(100L, false)).willReturn(snapshot); given(skillQueryService.getFileContentByVersionId(100L, "SKILL.md")) .willReturn(new ByteArrayInputStream("private package".getBytes())); var download = new SkillDownloadService.DownloadResult(() -> new ByteArrayInputStream(new byte[]{1}), "review.zip", 1, "application/zip", null, true); @@ -214,6 +221,7 @@ class ReviewSkillDetailAppServiceTest { assertThat(detail.visibilityChange().scanStatus()).isEqualTo("PASSED"); assertThat(service.getReviewFileContent(42L, "SKILL.md", "admin", Map.of()).readAllBytes()).isEqualTo("private package".getBytes()); assertThat(service.downloadReviewPackage(42L, "admin", Map.of()).filename()).isEqualTo("review.zip"); + assertThat(service.getReviewSkillDetail(42L, "admin", Map.of(), false).visibilityChange().requestedVisibility()).isEqualTo("PUBLIC"); } @Test diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/SkillSearchAppServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/SkillSearchAppServiceTest.java index fe98f52ce..51f36230a 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/SkillSearchAppServiceTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/SkillSearchAppServiceTest.java @@ -175,6 +175,7 @@ class SkillSearchAppServiceTest { second.setLatestVersionId(102L); SkillVersion firstVersion = publishedVersion(10L, 101L, "1.0.0"); + setField(firstVersion, "githubSyncRevision", 2); SkillVersion secondVersion = publishedVersion(11L, 102L, "2.0.0"); Namespace namespace = new Namespace("team-a", "Team A", "owner-1"); @@ -191,7 +192,9 @@ class SkillSearchAppServiceTest { assertEquals(2, response.items().size()); assertEquals("1.0.0", response.items().get(0).publishedVersion().version()); + assertEquals(2, response.items().get(0).publishedVersion().syncRevision()); assertEquals("2.0.0", response.items().get(1).publishedVersion().version()); + assertNull(response.items().get(1).publishedVersion().syncRevision()); verify(skillVersionRepository, times(1)).findByIdIn(List.of(101L, 102L)); verify(skillVersionRepository, times(0)) .findBySkillIdInAndStatus(List.of(10L, 11L), com.iflytek.skillhub.domain.skill.SkillVersionStatus.PUBLISHED); diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/stream/ScanTaskConsumerLoggingTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/stream/ScanTaskConsumerLoggingTest.java index c388fcbf8..2b4239c22 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/stream/ScanTaskConsumerLoggingTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/stream/ScanTaskConsumerLoggingTest.java @@ -3,6 +3,7 @@ package com.iflytek.skillhub.stream; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; import ch.qos.logback.core.read.ListAppender; import com.fasterxml.jackson.databind.ObjectMapper; import com.iflytek.skillhub.domain.security.ScanTaskProducer; @@ -13,10 +14,14 @@ import com.iflytek.skillhub.domain.security.SecurityScanner; import com.iflytek.skillhub.domain.skill.SkillVersion; import com.iflytek.skillhub.domain.skill.SkillVersionRepository; import com.iflytek.skillhub.domain.skill.SkillVersionStatus; +import com.iflytek.skillhub.infra.http.HttpClientException; +import com.iflytek.skillhub.infra.scanner.SecurityScanException; import com.iflytek.skillhub.observability.MessageObservationSupport; import com.iflytek.skillhub.observability.RequestIdAccessor; import com.iflytek.skillhub.storage.ObjectMetadata; import com.iflytek.skillhub.storage.ObjectStorageService; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -30,11 +35,13 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -81,12 +88,13 @@ class ScanTaskConsumerLoggingTest { @Test void handleMessage_logsBundleStageFailureBeforeRetry() { + String marker = "STORAGE_ERROR_WITH_SKILL_BODY_AND_TOKEN"; TestableLoggingConsumer consumer = new TestableLoggingConsumer( new FailingScanner(null), new NoOpSecurityScanService(), new EmptySkillVersionRepository(), new TestProducer(), - new InMemoryObjectStorageService(new IllegalStateException("bundle missing")) + new InMemoryObjectStorageService(new IllegalStateException(marker)) ); attachAppender(); @@ -100,15 +108,23 @@ class ScanTaskConsumerLoggingTest { assertThat(loggedMessages()).anyMatch(message -> message.contains( "Failed to stage security scan bundle: taskId=task-2, versionId=43, bundleKey=packages/8/43/bundle.zip" )); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + assertThat(appender.list).allSatisfy(event -> { + if (event.getThrowableProxy() != null) { + assertThat(ThrowableProxyUtil.asString(event.getThrowableProxy())) + .doesNotContain(marker); + } + }); } @Test void handleMessage_logsFinalFailureAfterRetriesExhausted() { + String marker = "SCANNER_ERROR_WITH_SKILL_BODY_AND_TOKEN"; SkillVersion version = new SkillVersion(8L, "1.0.0", "publisher-1"); setVersionId(version, 44L); version.setStatus(SkillVersionStatus.SCANNING); TestableLoggingConsumer consumer = new TestableLoggingConsumer( - new FailingScanner(new IllegalStateException("scanner unavailable")), + new FailingScanner(new IllegalStateException(marker)), new NoOpSecurityScanService(), new SingleSkillVersionRepository(version), new TestProducer(), @@ -127,6 +143,116 @@ class ScanTaskConsumerLoggingTest { assertThat(loggedMessages()).anyMatch(message -> message.contains( "Security scan task failed permanently: taskId=task-3, versionId=44, scanner=SKILL_SCANNER" )); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + } + + @Test + void retryPublishFailureDoesNotExposeProducerException() { + String marker = "REDIS_ERROR_WITH_SKILL_BODY_AND_TOKEN"; + ScanTaskProducer failingProducer = task -> { + throw new IllegalStateException(marker); + }; + TestableLoggingConsumer consumer = new TestableLoggingConsumer( + new FailingScanner(new IllegalStateException("scanner unavailable")), + new NoOpSecurityScanService(), + new EmptySkillVersionRepository(), + failingProducer, + new InMemoryObjectStorageService(Map.of("packages/8/45/bundle.zip", "zip".getBytes())) + ); + attachAppender(); + + assertThatThrownBy(() -> consumer.handleMessage(new StreamMessageId(4, 0), Map.of( + "taskId", "task-retry-publish", + "versionId", "45", + "bundleKey", "packages/8/45/bundle.zip", + "scannerType", ScannerType.SKILL_SCANNER.getValue() + ))).isInstanceOf(IllegalStateException.class) + .hasMessage("Security scan retry publication failed") + .hasNoCause(); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + } + + @Test + void scanFailureDoesNotExposeExceptionContentToObservation() { + String marker = "OBSERVATION_ERROR_WITH_SKILL_BODY_AND_TOKEN"; + ObservationRegistry registry = ObservationRegistry.create(); + List observedErrors = new ArrayList<>(); + registry.observationConfig().observationHandler(new ObservationHandler() { + @Override + public void onStop(Observation.Context context) { + if (context.getError() != null) { + observedErrors.add(context.getError()); + } + } + + @Override + public boolean supportsContext(Observation.Context context) { + return true; + } + }); + TestableLoggingConsumer consumer = new TestableLoggingConsumer( + new FailingScanner(null), + new NoOpSecurityScanService(), + new EmptySkillVersionRepository(), + new TestProducer(), + new InMemoryObjectStorageService(new IllegalStateException(marker)), + registry + ); + + consumer.handleMessage(new StreamMessageId(5, 0), Map.of( + "taskId", "task-observed", + "versionId", "46", + "bundleKey", "packages/8/46/bundle.zip", + "scannerType", ScannerType.SKILL_SCANNER.getValue() + )); + + assertThat(observedErrors).hasSize(1); + assertThat(observedErrors.get(0).getMessage()).doesNotContain(marker); + assertThat(observedErrors.get(0).getCause()).isNull(); + } + + @Test + void unavailableScannerDoesNotExposeResponseContentWhileDeferring() { + String marker = "SCANNER_503_WITH_SKILL_BODY_AND_TOKEN"; + ObservationRegistry registry = ObservationRegistry.create(); + List observedErrors = new ArrayList<>(); + registry.observationConfig().observationHandler(new ObservationHandler() { + @Override + public void onStop(Observation.Context context) { + if (context.getError() != null) { + observedErrors.add(context.getError()); + } + } + + @Override + public boolean supportsContext(Observation.Context context) { + return true; + } + }); + TestableLoggingConsumer consumer = new TestableLoggingConsumer( + new FailingScanner(new SecurityScanException(marker, new HttpClientException(503, marker))), + new NoOpSecurityScanService(), + new EmptySkillVersionRepository(), + new TestProducer(), + new InMemoryObjectStorageService(Map.of("packages/8/47/bundle.zip", "zip".getBytes())), + registry + ); + attachAppender(); + + consumer.handleMessage(new StreamMessageId(6, 0), Map.of( + "taskId", "task-scanner-unavailable", + "versionId", "47", + "bundleKey", "packages/8/47/bundle.zip", + "scannerType", ScannerType.SKILL_SCANNER.getValue(), + "createdAtMillis", Long.toString(System.currentTimeMillis()) + )); + + assertThat(loggedMessages()).anyMatch(message -> message.contains( + "Scanner unavailable; keeping task pending for later recovery")); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + assertThat(observedErrors).hasSize(1); + assertThat(observedErrors.get(0).getMessage()).doesNotContain(marker); + assertThat(observedErrors.get(0).getCause()).isNull(); } private void attachAppender() { @@ -169,6 +295,16 @@ class ScanTaskConsumerLoggingTest { SkillVersionRepository skillVersionRepository, ScanTaskProducer scanTaskProducer, ObjectStorageService objectStorageService) { + this(securityScanner, securityScanService, skillVersionRepository, scanTaskProducer, + objectStorageService, ObservationRegistry.NOOP); + } + + private TestableLoggingConsumer(SecurityScanner securityScanner, + SecurityScanService securityScanService, + SkillVersionRepository skillVersionRepository, + ScanTaskProducer scanTaskProducer, + ObjectStorageService objectStorageService, + ObservationRegistry observationRegistry) { super( redissonClientWithAvailableProcessingLock(), "skillhub:scan:requests", @@ -178,7 +314,7 @@ class ScanTaskConsumerLoggingTest { skillVersionRepository, scanTaskProducer, objectStorageService, - new MessageObservationSupport(ObservationRegistry.NOOP, new RequestIdAccessor()) + new MessageObservationSupport(observationRegistry, new RequestIdAccessor()) ); } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/support/GithubSyncWorkerCapabilities.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/support/GithubSyncWorkerCapabilities.java new file mode 100644 index 000000000..74faf77c5 --- /dev/null +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/support/GithubSyncWorkerCapabilities.java @@ -0,0 +1,15 @@ +package com.iflytek.skillhub.support; + +import com.iflytek.skillhub.githubsync.GithubPlatformControlRepository; +import java.util.LinkedHashSet; +import java.util.List; + +/** Existing worker fixtures opt in to platform checkpoints without changing production capability gates. */ +public final class GithubSyncWorkerCapabilities { + private GithubSyncWorkerCapabilities() {} + public static List platform(List capabilities) { + var result = new LinkedHashSet<>(capabilities); + result.add(GithubPlatformControlRepository.CAPABILITY); + return List.copyOf(result); + } +} diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcherTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcherTest.java index eafa3b6f3..0a1a37d88 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcherTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/task/ScanTaskOutboxDispatcherTest.java @@ -1,5 +1,9 @@ package com.iflytek.skillhub.task; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import ch.qos.logback.core.read.ListAppender; import com.iflytek.skillhub.domain.security.ScanTask; import com.iflytek.skillhub.domain.security.ScanTaskOutbox; import com.iflytek.skillhub.domain.security.ScanTaskOutboxRepository; @@ -13,6 +17,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.LoggerFactory; +import org.springframework.test.util.ReflectionTestUtils; import java.time.Clock; import java.time.Duration; @@ -20,6 +26,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -36,6 +43,41 @@ class ScanTaskOutboxDispatcherTest { @Mock ScanTaskProducer producer; @Mock SkillVersionRepository versionRepository; + @Test + void publisherFailureDoesNotPersistOrLogExceptionContent() { + String marker = "SKILL_BODY_AND_TOKEN_SENTINEL"; + ScanTaskOutbox outbox = outbox("task-private-error", 7L); + SkillVersion version = new SkillVersion(9L, "1.0.0", "user"); + version.setStatus(SkillVersionStatus.SCANNING); + given(repository.findDispatchable(any(), any(Integer.class))).willReturn(List.of(outbox)); + given(versionRepository.findById(7L)).willReturn(Optional.of(version)); + doThrow(new IllegalStateException(marker)).when(producer).publishScanTask(any()); + Logger logger = (Logger) LoggerFactory.getLogger(ScanTaskOutboxDispatcher.class); + ListAppender logs = new ListAppender<>(); + logs.start(); + logger.addAppender(logs); + try { + dispatcher(2).dispatch(); + assertThat(outbox.getStatus()).isEqualTo(ScanTaskOutboxStatus.PENDING); + assertThat(ReflectionTestUtils.getField(outbox, "lastError")) + .isEqualTo("SCAN_TASK_PUBLISH_FAILED:IllegalStateException"); + + dispatcher(2).dispatch(); + assertThat(outbox.getStatus()).isEqualTo(ScanTaskOutboxStatus.FAILED); + assertThat(version.getStatus()).isEqualTo(SkillVersionStatus.SCAN_FAILED); + assertThat(ReflectionTestUtils.getField(outbox, "lastError")) + .isEqualTo("SCAN_TASK_PUBLISH_FAILED:IllegalStateException"); + String logText = logs.list.stream().map(event -> event.getFormattedMessage() + + (event.getThrowableProxy() == null ? "" + : ThrowableProxyUtil.asString(event.getThrowableProxy()))) + .collect(Collectors.joining("\n")); + assertThat(logText).doesNotContain(marker); + } finally { + logger.detachAppender(logs); + logs.stop(); + } + } + @Test void failedRedisPublishLeavesTaskPendingForRetry() { ScanTaskOutbox outbox = outbox("task-1", 1L); diff --git a/server/skillhub-app/src/test/resources/application-test.yml b/server/skillhub-app/src/test/resources/application-test.yml index 8749a3557..c95ada164 100644 --- a/server/skillhub-app/src/test/resources/application-test.yml +++ b/server/skillhub-app/src/test/resources/application-test.yml @@ -37,6 +37,8 @@ spring: user-info-uri: https://api.github.com/user skillhub: + scheduling: + enabled: false auth: enforce-active-user-check: false access-policy: diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicy.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicy.java index 59e867c04..89cc502f5 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicy.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicy.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; public class NamespaceAccessPolicy { public boolean isImmutable(Namespace namespace) { - return namespace.getType() == NamespaceType.GLOBAL; + return namespace.getType() != NamespaceType.TEAM; } public boolean canMutateSettings(Namespace namespace) { diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceService.java index a7ed681e0..930ecbb5a 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceService.java @@ -53,6 +53,9 @@ public class NamespaceService { public Namespace createNamespace(String slug, String displayName, String description, String creatorUserId) { userAccountRepository.findLockedById(creatorUserId); SlugValidator.validate(slug); + if ("github".equals(slug)) { + throw new DomainBadRequestException("error.slug.reserved", slug); + } if (namespaceRepository.findBySlug(slug).isPresent()) { throw new DomainBadRequestException("error.namespace.slug.exists", slug); diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceType.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceType.java index 4db5f1fe5..645d04167 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceType.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/NamespaceType.java @@ -2,5 +2,6 @@ package com.iflytek.skillhub.domain.namespace; public enum NamespaceType { GLOBAL, + CATALOG, TEAM } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/PersonalNamespaceProvisioningService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/PersonalNamespaceProvisioningService.java index 8efd576a7..36e180c4a 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/PersonalNamespaceProvisioningService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/namespace/PersonalNamespaceProvisioningService.java @@ -100,7 +100,7 @@ public class PersonalNamespaceProvisioningService { .filter(member -> member.getRole() == NamespaceRole.OWNER) .map(member -> namespaceRepository.findById(member.getNamespaceId())) .flatMap(Optional::stream) - .anyMatch(namespace -> namespace.getType() != NamespaceType.GLOBAL); + .anyMatch(namespace -> namespace.getType() == NamespaceType.TEAM); } /** diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewOriginType.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewOriginType.java new file mode 100644 index 000000000..95cf9969c --- /dev/null +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewOriginType.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.domain.review; + +public enum ReviewOriginType { + USER, + GITHUB_SYNC +} diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewPermissionChecker.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewPermissionChecker.java index df86ec752..2edbd190b 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewPermissionChecker.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewPermissionChecker.java @@ -67,7 +67,7 @@ public class ReviewPermissionChecker { if (hasPlatformReviewRole(platformRoles)) { return true; } - if (namespaceType == NamespaceType.GLOBAL) { + if (namespaceType != NamespaceType.TEAM) { return false; } @@ -140,7 +140,7 @@ public class ReviewPermissionChecker { private boolean canSelfReviewNamespace(Long namespaceId, NamespaceType namespaceType, Map userNamespaceRoles) { - if (namespaceType == NamespaceType.GLOBAL) { + if (namespaceType != NamespaceType.TEAM) { return false; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewService.java index 497f1e782..c18159ff2 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewService.java @@ -107,9 +107,15 @@ public class ReviewService { Set platformRoles) { SkillVersion skillVersion = skillVersionRepository.findById(skillVersionId) .orElseThrow(() -> new DomainNotFoundException("skill_version.not_found", skillVersionId)); + if (skillVersion.getGithubSyncIngestionId() != null) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", skillVersionId); + } Skill skill = skillRepository.findById(skillVersion.getSkillId()) .orElseThrow(() -> new DomainNotFoundException("skill.not_found", skillVersion.getSkillId())); + if (skill.isGithubSyncManaged()) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", skillVersionId); + } Namespace namespace = namespaceRepository.findById(skill.getNamespaceId()) .orElseThrow(() -> new DomainNotFoundException("namespace.not_found", skill.getNamespaceId())); assertNamespaceActive(namespace); @@ -157,9 +163,15 @@ public class ReviewService { Map userNamespaceRoles) { SkillVersion skillVersion = skillVersionRepository.findById(skillVersionId) .orElseThrow(() -> new DomainNotFoundException("skill_version.not_found", skillVersionId)); + if (skillVersion.getGithubSyncIngestionId() != null) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", skillVersionId); + } Skill skill = skillRepository.findById(skillVersion.getSkillId()) .orElseThrow(() -> new DomainNotFoundException("skill.not_found", skillVersion.getSkillId())); + if (skill.isGithubSyncManaged()) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", skillVersionId); + } Namespace namespace = namespaceRepository.findById(skill.getNamespaceId()) .orElseThrow(() -> new DomainNotFoundException("namespace.not_found", skill.getNamespaceId())); assertNamespaceActive(namespace); diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTask.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTask.java index abb5eb0d3..9534161c1 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTask.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTask.java @@ -2,6 +2,7 @@ package com.iflytek.skillhub.domain.review; import jakarta.persistence.*; import java.time.Instant; +import java.util.UUID; @Entity @Table(name = "review_task") @@ -24,6 +25,13 @@ public class ReviewTask { @Column(name = "subject_type", nullable = false, length = 32) private ReviewSubjectType subjectType; + @Enumerated(EnumType.STRING) + @Column(name = "origin_type", nullable = false, length = 24) + private ReviewOriginType originType = ReviewOriginType.USER; + + @Column(name = "github_sync_ingestion_id") + private UUID githubSyncIngestionId; + @Column(name = "subject_id", nullable = false) private Long subjectId; @@ -161,6 +169,18 @@ public class ReviewTask { } public boolean isAvatarRemoval() { return avatarRemoval; } public void setAvatarRemoval(boolean removal) { this.avatarRemoval = removal; } + public static ReviewTask forGithubSyncSkillVersion(Long skillVersionId, Long skillId, + Long namespaceId, String skillVersion, + String systemActorId, UUID ingestionId) { + if (ingestionId == null) { + throw new IllegalArgumentException("GitHub sync ingestion ID is required"); + } + ReviewTask task = new ReviewTask(skillVersionId, skillId, namespaceId, skillVersion, systemActorId); + task.originType = ReviewOriginType.GITHUB_SYNC; + task.githubSyncIngestionId = ingestionId; + return task; + } + public Long getId() { return id; } @@ -172,6 +192,10 @@ public class ReviewTask { public ReviewSubjectType getSubjectType() { return subjectType; } + public ReviewOriginType getOriginType() { return originType; } + + public UUID getGithubSyncIngestionId() { return githubSyncIngestionId; } + public Long getSubjectId() { return subjectId; } public Long getSubjectVersionId() { return subjectVersionId; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskRepository.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskRepository.java index 1b7f05a4c..d70b66c88 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskRepository.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskRepository.java @@ -16,12 +16,24 @@ public interface ReviewTaskRepository { ReviewTask save(ReviewTask reviewTask); Optional findById(Long id); Optional findBySkillVersionIdAndStatus(Long skillVersionId, ReviewTaskStatus status); + default List findBySkillIdAndStatus(Long skillId, ReviewTaskStatus status) { + throw new UnsupportedOperationException("Skill-scoped review queries are not supported"); + } default Optional findBySubjectTypeAndSubjectVersionIdAndStatus( ReviewSubjectType subjectType, Long subjectVersionId, ReviewTaskStatus status) { throw new UnsupportedOperationException("Typed review subjects are not supported by this repository"); } Page findByStatus(ReviewTaskStatus status, Pageable pageable); Page findByNamespaceIdAndStatus(Long namespaceId, ReviewTaskStatus status, Pageable pageable); + default Page findByOriginTypeAndStatus(ReviewOriginType originType, + ReviewTaskStatus status, Pageable pageable) { + throw new UnsupportedOperationException("Origin-specific review queries are not supported"); + } + default Page findByNamespaceIdAndOriginTypeAndStatus(Long namespaceId, + ReviewOriginType originType, + ReviewTaskStatus status, Pageable pageable) { + throw new UnsupportedOperationException("Origin-specific review queries are not supported"); + } Page findBySubmittedByAndStatus(String submittedBy, ReviewTaskStatus status, Pageable pageable); List findBySubmittedByAndSkillIdAndSkillVersionOrderBySubmittedAtDescIdDesc( String submittedBy, Long skillId, String skillVersion); diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskStatus.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskStatus.java index 758dab45b..c0f7e4d98 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskStatus.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/review/ReviewTaskStatus.java @@ -3,5 +3,6 @@ package com.iflytek.skillhub.domain.review; public enum ReviewTaskStatus { PENDING, APPROVED, - REJECTED + REJECTED, + SUPERSEDED } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/Skill.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/Skill.java index 7367345fd..196a7b662 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/Skill.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/Skill.java @@ -31,6 +31,9 @@ public class Skill { @Column(name = "source_skill_id") private Long sourceSkillId; + @Column(name = "github_sync_managed", nullable = false) + private boolean githubSyncManaged; + @Enumerated(EnumType.STRING) @Column(nullable = false, length = 20) private SkillVisibility visibility; @@ -139,6 +142,19 @@ public class Skill { return sourceSkillId; } + public boolean isGithubSyncManaged() { + return githubSyncManaged; + } + + /** Converts an administrator-confirmed existing Skill to namespace-owned GitHub sync custody. */ + public void assignGithubSyncOwnership(String systemActorId) { + if (!"github-sync-publisher".equals(systemActorId)) { + throw new IllegalArgumentException("GitHub sync ownership requires the reserved system actor"); + } + ownerId = systemActorId; + githubSyncManaged = true; + } + public SkillVisibility getVisibility() { return visibility; } @@ -218,6 +234,10 @@ public class Skill { this.sourceSkillId = sourceSkillId; } + public void setGithubSyncManaged(boolean githubSyncManaged) { + this.githubSyncManaged = githubSyncManaged; + } + public void setVisibility(SkillVisibility visibility) { this.visibility = visibility; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillFileRepository.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillFileRepository.java index a704944fb..a2d3c925e 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillFileRepository.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillFileRepository.java @@ -1,12 +1,14 @@ package com.iflytek.skillhub.domain.skill; import java.util.List; +import java.util.Optional; /** * Domain repository contract for files belonging to one published or draft skill version. */ public interface SkillFileRepository { List findByVersionId(Long versionId); + Optional findByVersionIdAndFilePath(Long versionId, String filePath); List findByVersionIdIn(List versionIds); SkillFile save(SkillFile file); List saveAll(Iterable files); diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java index ec576e194..f15c8c52d 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java @@ -3,6 +3,7 @@ package com.iflytek.skillhub.domain.skill; import jakarta.persistence.*; import java.time.Clock; import java.time.Instant; +import java.util.UUID; import org.hibernate.annotations.JdbcTypeCode; import org.hibernate.type.SqlTypes; @@ -39,6 +40,12 @@ public class SkillVersion { @Column(name = "requested_visibility", length = 20) private SkillVisibility requestedVisibility; + @Column(name = "github_sync_ingestion_id") + private UUID githubSyncIngestionId; + + @Column(name = "github_sync_revision") + private Integer githubSyncRevision; + @Column(name = "file_count", nullable = false) private Integer fileCount = 0; @@ -117,6 +124,18 @@ public class SkillVersion { return requestedVisibility; } + public UUID getGithubSyncIngestionId() { + return githubSyncIngestionId; + } + + public Integer getGithubSyncRevision() { + return githubSyncRevision; + } + + public void setGithubSyncIngestionId(UUID githubSyncIngestionId) { + this.githubSyncIngestionId = githubSyncIngestionId; + } + public Integer getFileCount() { return fileCount; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillLifecycleProjectionService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillLifecycleProjectionService.java index 14d44bfc3..359a2f94e 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillLifecycleProjectionService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillLifecycleProjectionService.java @@ -38,13 +38,14 @@ public class SkillLifecycleProjectionService { String status, String parsedMetadataJson, String displayName, - String summary + String summary, + Integer syncRevision ) { public VersionProjection(Long id, String version, String status, String parsedMetadataJson) { - this(id, version, status, parsedMetadataJson, null, null); + this(id, version, status, parsedMetadataJson, null, null, null); } public VersionProjection(Long id, String version, String status) { - this(id, version, status, null, null, null); + this(id, version, status, null, null, null, null); } } @@ -191,7 +192,8 @@ public class SkillLifecycleProjectionService { version.getStatus().name(), version.getParsedMetadataJson(), metadata == null ? null : metadata.displayName(), - metadata == null ? null : metadata.description() + metadata == null ? null : metadata.description(), + version.getGithubSyncRevision() ); } } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java index 37c3427e8..d6043acf0 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java @@ -11,6 +11,7 @@ import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository; import com.iflytek.skillhub.domain.namespace.NamespaceRepository; import com.iflytek.skillhub.domain.namespace.NamespaceRole; import com.iflytek.skillhub.domain.namespace.NamespaceStatus; +import com.iflytek.skillhub.domain.namespace.NamespaceType; import com.iflytek.skillhub.domain.namespace.SlugValidator; import com.iflytek.skillhub.domain.review.ReviewTaskStatus; import com.iflytek.skillhub.domain.review.ReviewTask; @@ -57,6 +58,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -86,7 +88,6 @@ public class SkillPublishService { @org.springframework.beans.factory.annotation.Autowired public void setVisibilityService(SkillVisibilityService service) { this.visibilityService = service; } - @Autowired public void setAvatarParticipant(ReviewAvatarParticipant participant) { this.avatarParticipant = participant; @@ -100,8 +101,13 @@ public class SkillPublishService { public record PublishResult( Long skillId, String slug, - SkillVersion version - ) {} + SkillVersion version, + boolean newSkill + ) { + public PublishResult(Long skillId, String slug, SkillVersion version) { + this(skillId, slug, version, false); + } + } private record StagedPackageFile( Path path, @@ -331,7 +337,27 @@ public class SkillPublishService { SkillVisibility visibility, Set platformRoles, boolean confirmWarnings, String displayName) { return publishFromEntriesInternal(namespaceSlug, entries, publisherId, visibility, platformRoles, - confirmWarnings, false, false, null, displayName); + confirmWarnings, false, false, null, null, displayName); + } + + /** Publishes a validated GitHub package into the dedicated sync review queue. */ + @Transactional + public PublishResult publishGithubSyncFromEntries( + String namespaceSlug, List entries, SkillVisibility visibility, + UUID ingestionId, String contentSha256, Long expectedSkillId) { + if (ingestionId == null || contentSha256 == null || !contentSha256.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Validated GitHub ingestion and content digest are required"); + } + if (visibility != SkillVisibility.PUBLIC && visibility != SkillVisibility.NAMESPACE_ONLY) { + throw new IllegalArgumentException("GitHub sync requires a reviewable visibility"); + } + PublishResult result = publishFromEntriesInternal(namespaceSlug, entries, "github-sync-publisher", visibility, + Set.of(), true, false, true, null, + new GithubSyncPublicationTarget(expectedSkillId, ingestionId, contentSha256), null); + // The publication repository assigns the revision with JDBC in the same transaction. + // Flush pending JPA updates (including SCANNING) first so they cannot overwrite it. + skillVersionRepository.flush(); + return result; } /** @@ -391,10 +417,16 @@ public class SkillPublishService { boolean confirmWarnings, java.util.Set platformRoles) { Skill skill = skillRepository.findById(skillId) .orElseThrow(() -> new DomainBadRequestException("error.skill.notFound", skillId)); + if (skill.isGithubSyncManaged()) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", skillId); + } assertCanManageLifecycle(skill, publisherId, userNamespaceRoles, platformRoles); SkillVersion publishedVersion = skillVersionRepository.findBySkillIdAndVersion(skillId, sourceVersion) .orElseThrow(() -> new DomainBadRequestException("error.skill.version.notFound", sourceVersion)); + if (publishedVersion.getGithubSyncIngestionId() != null) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", publishedVersion.getId()); + } if (publishedVersion.getStatus() != SkillVersionStatus.PUBLISHED) { throw new DomainBadRequestException("error.skill.version.notPublished", sourceVersion); } @@ -417,6 +449,7 @@ public class SkillPublishService { false, // forceAutoPublish=false: respect visibility rules true, null, + null, displayNameOverrideForRerelease(publishedVersion) ); } @@ -458,19 +491,28 @@ public class SkillPublishService { boolean bypassMembershipCheck, BundlePublicationTarget bundleTarget) { return publishFromEntriesInternal(namespaceSlug, entries, publisherId, visibility, platformRoles, - confirmWarnings, forceAutoPublish, bypassMembershipCheck, bundleTarget, null); + confirmWarnings, forceAutoPublish, bypassMembershipCheck, bundleTarget, null, null); } private PublishResult publishFromEntriesInternal( String namespaceSlug, List entries, String publisherId, SkillVisibility visibility, Set platformRoles, boolean confirmWarnings, boolean forceAutoPublish, boolean bypassMembershipCheck, - BundlePublicationTarget bundleTarget, String displayName) { + BundlePublicationTarget bundleTarget, GithubSyncPublicationTarget syncTarget, String displayName) { // 1. Find namespace by slug Namespace namespace = namespaceRepository.findBySlug(namespaceSlug) .orElseThrow(() -> new DomainBadRequestException("error.namespace.slug.notFound", namespaceSlug)); assertNamespaceWritable(namespace); + if (syncTarget != null && namespace.getType() != NamespaceType.CATALOG + && namespace.getType() != NamespaceType.TEAM) { + throw new DomainBadRequestException("error.skill.publish.githubSync.namespaceInvalid"); + } + if (syncTarget != null + && namespace.getType() == NamespaceType.CATALOG + && visibility != SkillVisibility.PUBLIC) { + throw new DomainBadRequestException("error.skill.publish.githubSync.namespaceInvalid"); + } boolean isSuperAdmin = platformRoles.contains("SUPER_ADMIN"); @@ -496,7 +538,15 @@ public class SkillPublishService { String skillMdContent = new String(skillMd.content()); SkillMetadata metadata = skillMetadataParser.parse(skillMdContent).withDisplayNameOverride(displayName); - if (metadata.version() == null || metadata.version().isBlank()) { + String upstreamVersion = syncTarget == null ? null : metadata.version(); + if (syncTarget != null) { + // A reverted upstream package is a new review target after an intervening update. + // The ingestion suffix keeps that version distinct without using the commit as identity. + String syncVersion = "0.0.0-g" + syncTarget.contentSha256().substring(0, 40) + + "-" + syncTarget.ingestionId().toString().replace("-", "").substring(0, 16); + metadata = new SkillMetadata(metadata.name(), metadata.description(), syncVersion, + metadata.body(), metadata.frontmatter()); + } else if (metadata.version() == null || metadata.version().isBlank()) { String autoVersion = AUTO_VERSION_FORMATTER.format(currentTime()); metadata = new SkillMetadata(metadata.name(), metadata.description(), autoVersion, metadata.body(), metadata.frontmatter(), metadata.displayNameOverride()); } @@ -533,13 +583,35 @@ public class SkillPublishService { // 6. Find or create Skill record (with owner isolation) List existingSkills = skillRepository.findByNamespaceIdAndSlug(namespace.getId(), skillSlug); + boolean exactBoundSkill = syncTarget != null && syncTarget.expectedSkillId() != null + && existingSkills.stream().anyMatch(existing -> + Objects.equals(existing.getId(), syncTarget.expectedSkillId())); + if (syncTarget != null) { + if ((!existingSkills.isEmpty() && !exactBoundSkill) + || (syncTarget.expectedSkillId() != null && !exactBoundSkill)) { + throw new DomainBadRequestException("error.skill.publish.githubSync.slugConflict", skillSlug); + } + } // Check if any other owner's skill has published versions // Only PUBLISHED status blocks same-name publishing (UPLOADED/PENDING_REVIEW allowed) for (Skill existing : existingSkills) { - boolean isBoundTarget = bundleTarget != null - && Objects.equals(existing.getId(), bundleTarget.expectedSkillId()); + if (syncTarget == null && existing.isGithubSyncManaged()) { + throw new DomainBadRequestException( + "error.skill.publish.githubSync.slugConflict", skillSlug); + } + boolean isBoundTarget = (bundleTarget != null + && Objects.equals(existing.getId(), bundleTarget.expectedSkillId())) + || (syncTarget != null + && Objects.equals(existing.getId(), syncTarget.expectedSkillId())); + if (syncTarget != null && exactBoundSkill && !isBoundTarget) { + continue; + } if (!existing.getOwnerId().equals(publisherId) && !isBoundTarget) { + if (existing.getOwnerId().equals("github-sync-publisher")) { + throw new DomainBadRequestException( + "error.skill.publish.githubSync.slugConflict", skillSlug); + } boolean hasPublished = !skillVersionRepository .findBySkillIdAndStatus(existing.getId(), SkillVersionStatus.PUBLISHED) .isEmpty(); @@ -554,10 +626,25 @@ public class SkillPublishService { } } - Skill skill = bundleTarget == null - ? findOrCreateOwnedSkill(namespace, skillSlug, publisherId, visibility) - : resolveBundleTargetSkill(namespace, existingSkills, publisherId, visibility, - platformRoles, bundleTarget); + Skill skill; + boolean newSkill = false; + if (syncTarget != null && syncTarget.expectedSkillId() != null) { + skill = existingSkills.stream() + .filter(existing -> Objects.equals(existing.getId(), syncTarget.expectedSkillId())) + .findFirst().orElseThrow(); + } else if (bundleTarget == null) { + var owned = skillRepository.findByNamespaceIdAndSlugAndOwnerId(namespace.getId(), skillSlug, publisherId); + if (owned.isPresent()) { + skill = owned.get(); + } else { + skill = createSkill(namespace, skillSlug, publisherId, visibility); + newSkill = true; + } + } else { + skill = resolveBundleTargetSkill(namespace, existingSkills, publisherId, visibility, + platformRoles, bundleTarget); + newSkill = bundleTarget.expectedSkillId() == null; + } if (visibilityService != null) { visibilityService.lockedSkill(skill.getId()); @@ -574,7 +661,29 @@ public class SkillPublishService { if (bundleTarget != null && !pendingVersions.isEmpty()) { throw bundleStateChanged(); } - if (bundleTarget == null) { + if (syncTarget != null) { + for (ReviewTask task : reviewTaskRepository.findBySkillIdAndStatus( + skill.getId(), ReviewTaskStatus.PENDING)) { + if (task.getGithubSyncIngestionId() == null) { + throw new DomainBadRequestException( + "error.skill.publish.githubSync.pendingUserReview", skillSlug); + } + int updated = reviewTaskRepository.updateStatusWithVersion( + task.getId(), ReviewTaskStatus.SUPERSEDED, + publisherId, "Newer GitHub content received", task.getVersion()); + if (updated != 1) { + throw new DomainBadRequestException( + "error.skill.publish.concurrentConflict", skillSlug); + } + SkillVersion previous = skillVersionRepository.findById(task.getSkillVersionId()) + .orElseThrow(() -> new IllegalStateException("Sync review has no Skill version")); + if (previous.getStatus() == SkillVersionStatus.PENDING_REVIEW + || previous.getStatus() == SkillVersionStatus.SCANNING) { + previous.setStatus(SkillVersionStatus.UPLOADED); + skillVersionRepository.save(previous); + } + } + } else if (bundleTarget == null) { for (SkillVersion pending : pendingVersions) { reviewTaskRepository.findBySkillVersionIdAndStatus(pending.getId(), ReviewTaskStatus.PENDING) .ifPresent(this::withdrawTask); @@ -586,6 +695,9 @@ public class SkillPublishService { // 7. Check version doesn't already exist java.util.Optional existingVersion = skillVersionRepository.findBySkillIdAndVersion(skill.getId(), metadata.version()); if (existingVersion.isPresent()) { + if (syncTarget != null) { + throw new DomainBadRequestException("error.skill.publish.githubSync.versionExists", skillSlug); + } if (bundleTarget != null) { throw bundleStateChanged(); } @@ -599,6 +711,9 @@ public class SkillPublishService { // 8. Create SkillVersion SkillVersion version = new SkillVersion(skill.getId(), metadata.version(), publisherId); version.setRequestedVisibility(visibility); + if (syncTarget != null) { + version.setGithubSyncIngestionId(syncTarget.ingestionId()); + } boolean autoPublish = forceAutoPublish || isSuperAdmin; if (autoPublish) { version.setStatus(SkillVersionStatus.PUBLISHED); @@ -615,7 +730,8 @@ public class SkillPublishService { // Store metadata as JSON try { - String metadataJson = objectMapper.writeValueAsString(buildParsedMetadata(metadata, complianceSnapshot)); + String metadataJson = objectMapper.writeValueAsString( + buildParsedMetadata(metadata, complianceSnapshot, syncTarget, upstreamVersion)); version.setParsedMetadataJson(metadataJson); version.setManifestJson(objectMapper.writeValueAsString(buildManifest(entries))); } catch (Exception e) { @@ -745,8 +861,12 @@ public class SkillPublishService { // Create review task for PUBLIC/NAMESPACE_ONLY (not PRIVATE) if (!autoPublish && visibility != SkillVisibility.PRIVATE) { - ReviewTask reviewTask = new ReviewTask( - version.getId(), skill.getId(), namespace.getId(), version.getVersion(), publisherId); + ReviewTask reviewTask = syncTarget == null + ? new ReviewTask(version.getId(), skill.getId(), namespace.getId(), + version.getVersion(), publisherId) + : ReviewTask.forGithubSyncSkillVersion(version.getId(), skill.getId(), + namespace.getId(), version.getVersion(), publisherId, + syncTarget.ingestionId()); ReviewTask savedReviewTask = reviewTaskRepository.save(reviewTask); eventPublisher.publishEvent(new ReviewSubmittedEvent( savedReviewTask.getId(), @@ -768,6 +888,9 @@ public class SkillPublishService { skill.setDisplayName(metadata.displayName()); skill.setSummary(metadata.description()); } + if (syncTarget != null) { + skill.assignGithubSyncOwnership(publisherId); + } if (autoPublish || visibility == SkillVisibility.PRIVATE) { // Update latestVersionId for autoPublish or PRIVATE skill (UPLOADED status) skill.setLatestVersionId(version.getId()); @@ -781,17 +904,7 @@ public class SkillPublishService { } // 13. Return identifiers for the created version - return new PublishResult(skill.getId(), skill.getSlug(), version); - } - - private Skill findOrCreateOwnedSkill( - Namespace namespace, - String skillSlug, - String publisherId, - SkillVisibility visibility - ) { - return skillRepository.findByNamespaceIdAndSlugAndOwnerId(namespace.getId(), skillSlug, publisherId) - .orElseGet(() -> createSkill(namespace, skillSlug, publisherId, visibility)); + return new PublishResult(skill.getId(), skill.getSlug(), version, newSkill); } private Skill resolveBundleTargetSkill( @@ -936,7 +1049,7 @@ public class SkillPublishService { } } -private void assertCanManageLifecycle(Skill skill, + private void assertCanManageLifecycle(Skill skill, String actorUserId, Map userNamespaceRoles, Set platformRoles) { @@ -970,6 +1083,9 @@ private void assertCanManageLifecycle(Skill skill, } } + private record GithubSyncPublicationTarget(Long expectedSkillId, UUID ingestionId, + String contentSha256) {} + private Instant currentTime() { return Instant.now(clock); } @@ -1023,7 +1139,10 @@ private void assertCanManageLifecycle(Skill skill, .toList(); } - private Map buildParsedMetadata(SkillMetadata metadata, ComplianceSnapshot complianceSnapshot) { + private Map buildParsedMetadata(SkillMetadata metadata, + ComplianceSnapshot complianceSnapshot, + GithubSyncPublicationTarget syncTarget, + String upstreamVersion) { Map parsedMetadata = new LinkedHashMap<>(); parsedMetadata.put("name", metadata.name()); parsedMetadata.put("description", metadata.description()); @@ -1031,6 +1150,11 @@ private void assertCanManageLifecycle(Skill skill, parsedMetadata.put("body", metadata.body()); parsedMetadata.put("frontmatter", metadata.frontmatter()); parsedMetadata.put("displayNameOverride", metadata.displayNameOverride()); + if (syncTarget != null) { + parsedMetadata.put("upstreamVersion", upstreamVersion); + parsedMetadata.put("githubContentSha256", syncTarget.contentSha256()); + } + parsedMetadata.put(ComplianceMetadataService.SNAPSHOT_FIELD_NAME, complianceSnapshot); return parsedMetadata; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java index 2e13a6a73..7b7c1b28b 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java @@ -24,6 +24,8 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URLEncoder; @@ -40,6 +42,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Read-side domain service for skill detail, version browsing, and packaged @@ -52,6 +55,8 @@ import java.util.stream.Collectors; public class SkillQueryService { private final SkillOwnerAccessPolicy ownerAccess; + private static final int MAX_REVIEW_PREVIEW_BYTES = 256 * 1024; + private final NamespaceRepository namespaceRepository; private final SkillRepository skillRepository; private final SkillVersionRepository skillVersionRepository; @@ -603,6 +608,10 @@ public class SkillQueryService { } public ReviewSkillSnapshotDTO getReviewSkillSnapshot(Long skillVersionId) { + return getReviewSkillSnapshot(skillVersionId, true); + } + + public ReviewSkillSnapshotDTO getReviewSkillSnapshot(Long skillVersionId, boolean includeDocumentation) { SkillVersion activeVersion = skillVersionRepository.findById(skillVersionId) .orElseThrow(() -> new DomainBadRequestException("error.skill.version.notFound", skillVersionId)); Skill skill = skillRepository.findById(activeVersion.getSkillId()) @@ -629,11 +638,18 @@ public class SkillQueryService { Comparator.nullsLast(Comparator.reverseOrder())) .thenComparing(SkillVersion::getId, Comparator.reverseOrder())) .toList(); - List files = availableFiles(activeVersion.getId()); - String documentationPath = resolveDocumentationPath(files); - String documentationContent = documentationPath == null - ? null - : readTextContent(findFile(activeVersion, documentationPath)); + // The review file tree uses stored package metadata. Checking every object here would + // turn one page load into up to MAX_FILE_COUNT remote storage requests. + List files = skillFileRepository.findByVersionId(activeVersion.getId()); + SkillFile documentationFile = Stream.of("README.md", "SKILL.md") + .map(path -> files.stream().filter(file -> file.getFilePath().equals(path)).findFirst().orElse(null)) + .filter(Objects::nonNull) + .filter(file -> !includeDocumentation || objectStorageService.exists(file.getStorageKey())) + .findFirst() + .orElse(null); + String documentationPath = documentationFile == null ? null : documentationFile.getFilePath(); + String documentationContent = !includeDocumentation || documentationFile == null + ? null : readTextContent(documentationFile); SkillVersion publishedVersion = versions.stream() .filter(version -> version.getStatus() == SkillVersionStatus.PUBLISHED) .findFirst() @@ -829,11 +845,32 @@ public class SkillQueryService { return readFileContent(file); } + /** Reads at most one review preview into memory before returning any response bytes. */ + public InputStream getFilePreviewContentByVersionId(Long versionId, String filePath) { + SkillVersion version = skillVersionRepository.findById(versionId) + .orElseThrow(() -> new DomainBadRequestException("error.skill.version.notFound", versionId)); + SkillFile file = findFile(version, filePath); + if (file.getFileSize() != null && file.getFileSize() > MAX_REVIEW_PREVIEW_BYTES) { + throw new DomainBadRequestException("error.skill.file.previewTooLarge", filePath); + } + try (InputStream content = readFileContent(file)) { + byte[] bytes = content.readNBytes(MAX_REVIEW_PREVIEW_BYTES + 1); + if (bytes.length > MAX_REVIEW_PREVIEW_BYTES) { + throw new DomainBadRequestException("error.skill.file.previewTooLarge", filePath); + } + return new ByteArrayInputStream(bytes); + } catch (IOException e) { + throw new IllegalStateException("Failed to read skill file preview", e); + } + } + private SkillFile findFile(SkillVersion skillVersion, String filePath) { - return availableFiles(skillVersion.getId()).stream() - .filter(f -> f.getFilePath().equals(filePath)) - .findFirst() + SkillFile file = skillFileRepository.findByVersionIdAndFilePath(skillVersion.getId(), filePath) .orElseThrow(() -> new DomainBadRequestException("error.skill.file.notFound", filePath)); + if (!objectStorageService.exists(file.getStorageKey())) { + throw new DomainBadRequestException("error.skill.file.notFound", filePath); + } + return file; } private List availableFiles(Long versionId) { @@ -991,7 +1028,8 @@ public class SkillQueryService { String currentUserId, Map userNsRoles, Set platformRoles) { - if (namespace.getType() == NamespaceType.GLOBAL) { + if (namespace.getType() != NamespaceType.TEAM) { + return false; } if (namespace.getStatus() != NamespaceStatus.ACTIVE || skill.getStatus() != SkillStatus.ACTIVE) { diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillReviewSubmitService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillReviewSubmitService.java index e874f364d..c3e641ada 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillReviewSubmitService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillReviewSubmitService.java @@ -87,8 +87,14 @@ public class SkillReviewSubmitService { String actorUserId, Map userNamespaceRoles, java.util.Set platformRoles) { Skill skill = skillRepository.findById(skillId) .orElseThrow(() -> new DomainBadRequestException("error.skill.notFound", skillId)); + if (skill.isGithubSyncManaged()) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", versionId); + } SkillVersion version = skillVersionRepository.findById(versionId) .orElseThrow(() -> new DomainBadRequestException("error.skill.version.notFound", versionId)); + if (version.getGithubSyncIngestionId() != null) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", versionId); + } if (visibilityService != null) { skill = visibilityService.lockedSkill(skill.getId()); @@ -142,8 +148,14 @@ public class SkillReviewSubmitService { Set platformRoles) { Skill skill = skillRepository.findById(skillId) .orElseThrow(() -> new DomainBadRequestException("error.skill.notFound", skillId)); + if (skill.isGithubSyncManaged()) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", versionId); + } SkillVersion version = skillVersionRepository.findById(versionId) .orElseThrow(() -> new DomainBadRequestException("error.skill.version.notFound", versionId)); + if (version.getGithubSyncIngestionId() != null) { + throw new DomainBadRequestException("review.submit.githubSyncManaged", versionId); + } if (visibilityService != null) { skill = visibilityService.lockedSkill(skill.getId()); diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicyTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicyTest.java index 245849e79..fe147e6df 100644 --- a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicyTest.java +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceAccessPolicyTest.java @@ -19,6 +19,18 @@ class NamespaceAccessPolicyTest { assertThat(policy.canTransferOwnership(namespace)).isFalse(); } + @Test + void githubCatalogNamespaceIsImmutable() { + Namespace namespace = new Namespace("github", "GitHub Skills", "github-sync-publisher"); + namespace.setType(NamespaceType.CATALOG); + + assertThat(policy.isImmutable(namespace)).isTrue(); + assertThat(policy.canMutateSettings(namespace)).isFalse(); + assertThat(policy.canManageMembers(namespace)).isFalse(); + assertThat(policy.canTransferOwnership(namespace)).isFalse(); + assertThat(policy.canDelete(namespace, NamespaceRole.OWNER)).isFalse(); + } + @Test void activeTeamNamespaceAllowsAdminAndOwnerToFreezeButNotMember() { Namespace namespace = new Namespace("team-a", "Team A", "owner"); diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceServiceTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceServiceTest.java index b2827094f..0fa6e7dd8 100644 --- a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceServiceTest.java +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/namespace/NamespaceServiceTest.java @@ -79,6 +79,15 @@ class NamespaceServiceTest { namespaceService.createNamespace(slug, "Name", "Desc", "user-1")); } + @Test + void createNamespace_rejectsReservedGithubSlugBeforeLookingUpExistingNamespaces() { + DomainBadRequestException error = assertThrows(DomainBadRequestException.class, () -> + namespaceService.createNamespace("github", "Other", "Desc", "user-1")); + + assertEquals("error.slug.reserved", error.messageCode()); + verifyNoInteractions(namespaceRepository, namespaceMemberRepository); + } + @Test void createNamespace_shouldThrowExceptionForInvalidSlug() { assertThrows(DomainBadRequestException.class, () -> diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/SkillGithubSyncOwnershipTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/SkillGithubSyncOwnershipTest.java new file mode 100644 index 000000000..a4cb9819f --- /dev/null +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/SkillGithubSyncOwnershipTest.java @@ -0,0 +1,23 @@ +package com.iflytek.skillhub.domain.skill; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SkillGithubSyncOwnershipTest { + @Test + void transfersTechnicalOwnershipOnlyToTheReservedSystemActor() { + Skill skill = new Skill(10L, "demo", "original-owner", SkillVisibility.NAMESPACE_ONLY); + + assertThrows(IllegalArgumentException.class, () -> skill.assignGithubSyncOwnership("admin")); + assertEquals("original-owner", skill.getOwnerId()); + assertFalse(skill.isGithubSyncManaged()); + + skill.assignGithubSyncOwnership("github-sync-publisher"); + assertEquals("github-sync-publisher", skill.getOwnerId()); + assertTrue(skill.isGithubSyncManaged()); + } +} diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java index bc4057dca..79449c2c7 100644 --- a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java @@ -507,7 +507,7 @@ class SkillQueryServiceTest { when(namespaceRepository.findBySlug(namespaceSlug)).thenReturn(Optional.of(namespace)); when(skillRepository.findByNamespaceIdAndSlug(1L, skillSlug)).thenReturn(List.of(skill)); when(skillVersionRepository.findBySkillIdAndVersion(1L, version)).thenReturn(Optional.of(skillVersion)); - when(skillFileRepository.findByVersionId(1L)).thenReturn(List.of(file)); + when(skillFileRepository.findByVersionIdAndFilePath(1L, filePath)).thenReturn(Optional.of(file)); when(objectStorageService.exists(file.getStorageKey())).thenReturn(true); when(objectStorageService.getObject(file.getStorageKey())) .thenThrow(new UncheckedIOException(new java.io.FileNotFoundException(file.getStorageKey()))); @@ -517,6 +517,48 @@ class SkillQueryServiceTest { assertEquals("error.skill.file.notFound", ex.messageCode()); assertArrayEquals(new Object[]{filePath}, ex.messageArgs()); + verify(skillFileRepository, never()).findByVersionId(1L); + } + + @Test + void testGetFileContentByVersionId_ShouldReadOnlySelectedFile() throws Exception { + SkillVersion version = new SkillVersion(1L, "1.0.0", "owner"); + setId(version, 1L); + SkillFile file = new SkillFile(1L, "SKILL.md", 12L, "text/markdown", "hash", "skill-key"); + when(skillVersionRepository.findById(1L)).thenReturn(Optional.of(version)); + when(skillFileRepository.findByVersionIdAndFilePath(1L, "SKILL.md")).thenReturn(Optional.of(file)); + when(objectStorageService.exists("skill-key")).thenReturn(true); + when(objectStorageService.getObject("skill-key")).thenReturn(new ByteArrayInputStream("# Skill".getBytes())); + + try (InputStream content = service.getFileContentByVersionId(1L, "SKILL.md")) { + assertEquals("# Skill", new String(content.readAllBytes())); + } + verify(skillFileRepository, never()).findByVersionId(1L); + verify(objectStorageService, times(1)).exists("skill-key"); + } + + @Test + void testGetReviewSkillSnapshot_ShouldNotCheckEveryStoredFile() throws Exception { + Skill skill = new Skill(1L, "sample", "owner", SkillVisibility.PUBLIC); + setId(skill, 1L); + SkillVersion version = new SkillVersion(1L, "1.0.0", "owner"); + setId(version, 2L); + version.setStatus(SkillVersionStatus.PENDING_REVIEW); + SkillFile documentation = new SkillFile(2L, "README.md", 8L, "text/markdown", "hash", "readme-key"); + SkillFile other = new SkillFile(2L, "references/example.md", 8L, "text/markdown", "hash", "other-key"); + when(skillVersionRepository.findById(2L)).thenReturn(Optional.of(version)); + when(skillRepository.findById(1L)).thenReturn(Optional.of(skill)); + when(userAccountRepository.findById("owner")).thenReturn(Optional.empty()); + when(skillVersionRepository.findBySkillId(1L)).thenReturn(List.of(version)); + when(skillFileRepository.findByVersionId(2L)).thenReturn(List.of(documentation, other)); + when(objectStorageService.exists("readme-key")).thenReturn(true); + when(objectStorageService.getObject("readme-key")).thenReturn(new ByteArrayInputStream("# Sample".getBytes())); + + SkillQueryService.ReviewSkillSnapshotDTO snapshot = service.getReviewSkillSnapshot(2L); + + assertEquals(2, snapshot.files().size()); + assertEquals("# Sample", snapshot.documentationContent()); + verify(objectStorageService, never()).exists("other-key"); } @Test @@ -1226,6 +1268,26 @@ class SkillQueryServiceTest { service.getSkillDetail(namespaceSlug, skillSlug, userId, Map.of(), Set.of("SUPER_ADMIN"))); } + @Test + void testGetSkillDetail_ShouldRejectAnonymousTeamPrivateAndNamespaceOnlySkills() throws Exception { + Namespace namespace = new Namespace("team", "Team", "owner-1"); + setId(namespace, 1L); + Skill privateSkill = new Skill(1L, "private", "owner-1", SkillVisibility.PRIVATE); + privateSkill.setLatestVersionId(11L); + Skill namespaceOnlySkill = new Skill(1L, "team-only", "owner-1", SkillVisibility.NAMESPACE_ONLY); + namespaceOnlySkill.setLatestVersionId(12L); + + when(namespaceRepository.findBySlug("team")).thenReturn(Optional.of(namespace)); + when(skillRepository.findByNamespaceIdAndSlug(1L, "private")).thenReturn(List.of(privateSkill)); + when(skillRepository.findByNamespaceIdAndSlug(1L, "team-only")).thenReturn(List.of(namespaceOnlySkill)); + + assertThrows(DomainForbiddenException.class, () -> + service.getSkillDetail("team", "private", null, Map.of())); + assertThrows(DomainForbiddenException.class, () -> + service.getSkillDetail("team", "team-only", null, Map.of())); + verifyNoInteractions(skillVersionRepository); + } + @Test void testGetSkillDetail_ShouldPreferPendingVersionForOwnerPreview() throws Exception { String namespaceSlug = "test-ns"; diff --git a/server/skillhub-github-sync/pom.xml b/server/skillhub-github-sync/pom.xml new file mode 100644 index 000000000..b20913d25 --- /dev/null +++ b/server/skillhub-github-sync/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + com.iflytek.skillhub + skillhub-parent + 0.1.0 + + skillhub-github-sync + + + com.iflytek.skillhub + skillhub-domain + + + com.iflytek.skillhub + skillhub-storage + + + diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCandidateTaskRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCandidateTaskRepository.java new file mode 100644 index 000000000..fbeee092c --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCandidateTaskRepository.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +public interface GithubCandidateTaskRepository { + int enqueueDue(Instant now, int maxCandidates); + + int reconcileCompleted(Instant now, int maxCandidates); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialCipher.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialCipher.java new file mode 100644 index 000000000..aaa332e13 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialCipher.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.UUID; + +public interface GithubCredentialCipher { + boolean configured(); + boolean supports(String keyVersion); + EncryptedValue encrypt(UUID credentialId, String token); + String decrypt(UUID credentialId, String keyVersion, byte[] nonce, byte[] ciphertext); + + record EncryptedValue(String keyVersion, byte[] nonce, byte[] ciphertext) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialQuota.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialQuota.java new file mode 100644 index 000000000..b4fdb2e2e --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubCredentialQuota.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +public record GithubCredentialQuota(String resource, int remaining, + Instant resetAt, Instant observedAt) {} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfiguration.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfiguration.java new file mode 100644 index 000000000..c5cef01b3 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfiguration.java @@ -0,0 +1,50 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +/** Administrator intent for official discovery, independent of worker availability. */ +public record GithubDeepDiscoveryConfiguration( + long namespaceId, + long version, + boolean enabled, + BudgetProfile budgetProfile, + List exclusions, + String strategyVersion, + Instant updatedAt, + String updatedBy) { + + public GithubDeepDiscoveryConfiguration { + exclusions = List.copyOf(exclusions); + } + + public enum BudgetProfile { + ECONOMY, BALANCED, ACCELERATED + } + + public enum ExclusionKind { + OWNER, REPOSITORY + } + + public record Exclusion(ExclusionKind kind, String value) { + private static final Pattern OWNER = Pattern.compile("^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$"); + private static final Pattern REPOSITORY = Pattern.compile("^[A-Za-z0-9_.-]{1,100}$"); + + public Exclusion { + if (kind == null || value == null || value.length() > 140) { + throw new IllegalArgumentException("Invalid GitHub discovery exclusion"); + } + value = value.trim().toLowerCase(Locale.ROOT); + String[] parts = value.split("/", -1); + if (!OWNER.matcher(parts[0]).matches() + || kind == ExclusionKind.OWNER && parts.length != 1 + || kind == ExclusionKind.REPOSITORY && (parts.length != 2 + || !REPOSITORY.matcher(parts[1]).matches() + || parts[1].equals(".") || parts[1].equals(".."))) { + throw new IllegalArgumentException("Invalid GitHub discovery exclusion"); + } + } + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfigurationRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfigurationRepository.java new file mode 100644 index 000000000..1d9ba0d0e --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDeepDiscoveryConfigurationRepository.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.List; +import java.util.Optional; + +public interface GithubDeepDiscoveryConfigurationRepository { + GithubDeepDiscoveryConfiguration get(); + + Optional update(long expectedVersion, boolean enabled, + GithubDeepDiscoveryConfiguration.BudgetProfile budgetProfile, + List exclusions, String actorId); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatch.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatch.java new file mode 100644 index 000000000..347b0fce8 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatch.java @@ -0,0 +1,26 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.HashSet; +import java.util.List; + +public record GithubDiscoveryBatch(String previousCheckpoint, String checkpoint, + List repositories, int deferredCount, + GithubSyncDeferralReason deferralReason) { + public GithubDiscoveryBatch { + if (checkpoint == null || checkpoint.isBlank() || checkpoint.length() > 4096 + || checkpoint.chars().anyMatch(Character::isISOControl) + || (previousCheckpoint != null && (previousCheckpoint.length() > 4096 + || previousCheckpoint.chars().anyMatch(Character::isISOControl))) + || repositories == null || repositories.size() > 100 || deferredCount < 0 + || (deferredCount > 0 && deferralReason == null)) { + throw new IllegalArgumentException("invalid GitHub discovery batch"); + } + var seen = new HashSet(); + for (GithubDiscoveryCandidate repository : repositories) { + if (repository == null || !seen.add(repository.repositoryId())) { + throw new IllegalArgumentException("invalid or duplicate repository in discovery batch"); + } + } + repositories = List.copyOf(repositories); + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatchResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatchResult.java new file mode 100644 index 000000000..9e1c08eae --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryBatchResult.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubDiscoveryBatchResult { + ACCEPTED, + LEASE_STALE, + CHECKPOINT_CONFLICT, + BUDGET_EXCEEDED, + RESULT_CONFLICT +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCandidate.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCandidate.java new file mode 100644 index 000000000..e791067ae --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCandidate.java @@ -0,0 +1,31 @@ +package com.iflytek.skillhub.githubsync; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.regex.Pattern; +import java.util.List; + +/** A public repository lead; it is not yet a verified source binding. */ +public record GithubDiscoveryCandidate(long repositoryId, String owner, String name, String defaultBranch, + @JsonInclude(JsonInclude.Include.NON_NULL) List officialProvenance) { + public GithubDiscoveryCandidate(long repositoryId, String owner, String name, String defaultBranch) { + this(repositoryId, owner, name, defaultBranch, null); + } + private static final Pattern OWNER = Pattern.compile("[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?"); + private static final Pattern NAME = Pattern.compile("[A-Za-z0-9_.-]{1,100}"); + + public GithubDiscoveryCandidate { + if (officialProvenance != null) { + if (officialProvenance.isEmpty() || officialProvenance.size() > 16) { + throw new IllegalArgumentException("Invalid official provenance count"); + } + officialProvenance = List.copyOf(officialProvenance); + } + if (repositoryId <= 0 || owner == null || !OWNER.matcher(owner).matches() + || name == null || !NAME.matcher(name).matches() || name.equals(".") || name.equals("..") + || (defaultBranch != null && (defaultBranch.isBlank() || defaultBranch.length() > 256 + || defaultBranch.chars().anyMatch(Character::isISOControl)))) { + throw new IllegalArgumentException("invalid public GitHub repository candidate"); + } + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCompletionResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCompletionResult.java new file mode 100644 index 000000000..a6a61e4b0 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryCompletionResult.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubDiscoveryCompletionResult { + ACQUISITION_COMPLETE, + RETRY_WAIT, + LEASE_STALE, + CHECKPOINT_CONFLICT +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntry.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntry.java new file mode 100644 index 000000000..a85b0822d --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntry.java @@ -0,0 +1,22 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.UUID; + +public record GithubDiscoveryEntry( + UUID id, + long namespaceId, + String entryType, + String title, + String ownerLogin, + String repositoryName, + Long repositoryId, + String sourceVisibility, + String criteriaJson, + long ruleVersion, + boolean enabled, + int targetIntervalSeconds, + Instant nextDueAt, + Instant createdAt, + Instant updatedAt) { +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntryRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntryRepository.java new file mode 100644 index 000000000..7f062f50d --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryEntryRepository.java @@ -0,0 +1,37 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.time.Instant; + +public interface GithubDiscoveryEntryRepository { + List listPlatformEntries(); + + List listTeamRepositories(long namespaceId); + + Optional findTeamRepository(long namespaceId, UUID id); + + Optional findPlatformEntry(UUID id); + + GithubDiscoveryEntry createSearchRule(String title, String criteriaJson, + int targetIntervalSeconds); + + GithubDiscoveryEntry updateSearchRule(UUID id, String title, String criteriaJson, + int targetIntervalSeconds, boolean enabled); + + GithubDiscoveryEntry createCuratedOwner(String ownerLogin, int targetIntervalSeconds); + + GithubDiscoveryEntry createCuratedRepository(String ownerLogin, String repositoryName); + + GithubDiscoveryEntry createTeamRepository(long namespaceId, String ownerLogin, + String repositoryName, String visibility); + + GithubDiscoveryEntry setTeamRepositoryEnabled(long namespaceId, UUID id, boolean enabled); + + GithubDiscoveryEntry setTeamRepositoryVisibility(long namespaceId, UUID id, String visibility); + + GithubDiscoveryEntry setEnabled(UUID id, boolean enabled); + + int enqueueDue(Instant now, int maxEntries); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryOutcomeRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryOutcomeRepository.java new file mode 100644 index 000000000..7ef12ef65 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubDiscoveryOutcomeRepository.java @@ -0,0 +1,14 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.UUID; + +/** Durable factual outcomes, written in the same transaction as their business transition. */ +public interface GithubDiscoveryOutcomeRepository { + void candidateCreated(long namespaceId, long repositoryId, Instant now); + void firstValid(long namespaceId, long repositoryId, Instant now); + void publication(long namespaceId, long repositoryId, long skillId, long versionId, + boolean newSkill, Instant now); + void provenance(long candidateId, UUID queryId, String strategyVersion, UUID shardId, + UUID runId, Long seedRepositoryId); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBatch.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBatch.java new file mode 100644 index 000000000..b861725e9 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBatch.java @@ -0,0 +1,16 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record GithubOfficialDiscoveryBatch(UUID shardId, String coverage, Integer requestsUsed, + List childRanges, String gapReason, String ownerLogin) { + public GithubOfficialDiscoveryBatch(UUID shardId, String coverage, Integer requestsUsed, + List childRanges, String gapReason) { + this(shardId, coverage, requestsUsed, childRanges, gapReason, null); + } + public record Range(Instant since, Instant until, String partition) {} + public record Provenance(String channel, UUID queryId, long queryVersion, String strategyVersion, + UUID shardId, Long seedRepositoryId) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBudgetRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBudgetRepository.java new file mode 100644 index 000000000..fda5933d5 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryBudgetRepository.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.UUID; + +/** Finalize only a lease that can be delivered, inside the original claim transaction. */ +public interface GithubOfficialDiscoveryBudgetRepository { + void reserveSuccessfulClaim(UUID taskId, long generation); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryCatalog.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryCatalog.java new file mode 100644 index 000000000..5de832b39 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryCatalog.java @@ -0,0 +1,54 @@ +package com.iflytek.skillhub.githubsync; + +import org.springframework.stereotype.Component; + +import java.util.List; + +/** Versioned Java-owned query definitions. Go compiles snapshots, never a second query catalog. */ +@Component +public class GithubOfficialDiscoveryCatalog { + public static final long GENERATION = 3; + public static final String VERSION = "official-search-v3"; + public static final List REQUIRED_CAPABILITIES = List.of( + "official-discovery/v1", "official-checkpoint/v1", "official-pause/v1"); + + public static final String OWNER_CAPABILITY = "official-owner/v1"; + public static final List OWNER_REQUIRED_CAPABILITIES = List.of( + "official-discovery/v1", "official-checkpoint/v1", "official-pause/v1", OWNER_CAPABILITY); + + public List definitions() { + return List.of( + new Definition("metadata-agent-skills", 1, "REPOSITORY_METADATA", "CORE", "INCREMENTAL", + new Query("KEYWORD", "agent skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("topic-agent-skills", 1, "REPOSITORY_METADATA", "CORE", "INCREMENTAL", + new Query("TOPIC", "agent-skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("readme-skill-manifest", 1, "README", "CORE", "INCREMENTAL", + new Query("KEYWORD", "SKILL.md", "README")), + new Definition("metadata-claude-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "INCREMENTAL", + new Query("KEYWORD", "claude skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("metadata-claude-code-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "INCREMENTAL", + new Query("KEYWORD", "claude code skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("topic-claude-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("TOPIC", "claude-skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("metadata-codex-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "INCREMENTAL", + new Query("KEYWORD", "codex skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("topic-codex-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("TOPIC", "codex-skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("metadata-openclaw-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "INCREMENTAL", + new Query("KEYWORD", "openclaw skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("topic-openclaw-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("TOPIC", "openclaw-skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("metadata-gemini-skills", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("KEYWORD", "gemini skills", "NAME_DESCRIPTION_TOPICS")), + new Definition("metadata-skill-registry", 1, "REPOSITORY_METADATA", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("KEYWORD", "skill registry", "NAME_DESCRIPTION_TOPICS")), + new Definition("readme-agent-skills", 1, "README", "ECOSYSTEM", "HISTORY_RECHECK", + new Query("KEYWORD", "agent skills", "README")), + new Definition("readme-skill-workflow", 1, "README", "BROAD", "HISTORY_RECHECK", + new Query("KEYWORD", "skill workflow", "README"))); + } + + public record Definition(String key, long version, String channel, String tier, + String cadence, Query query) {} + public record Query(String kind, String value, String scope) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryRepository.java new file mode 100644 index 000000000..1bd7283c5 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialDiscoveryRepository.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +public interface GithubOfficialDiscoveryRepository { + int enqueueDue(Instant now, int maxQueries); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialLifecycleRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialLifecycleRepository.java new file mode 100644 index 000000000..6ab5064c5 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialLifecycleRepository.java @@ -0,0 +1,27 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import java.util.UUID; +import java.util.List; + +public interface GithubOfficialLifecycleRepository { + boolean isOfficialDiscovery(UUID taskId); + void report(UUID taskId, long generation, String token, GithubDiscoveryBatch batch, + GithubOfficialDiscoveryBatch officialBatch); + Control control(UUID taskId, long generation, String token); + void pauseDiscovery(UUID taskId, long generation, String token, String checkpoint); + void pauseRepository(UUID taskId, long generation, String token, RepositoryProgress progress); + record RepositoryProgress(String observedCommitSha, String pathCheckpoint, + List incompleteSubtrees, List pathResults) { + @JsonAnySetter + public void rejectUnknown(String name, Object value) { + throw new IllegalArgumentException("Unknown repository pause progress field: " + name); + } + } + UUID recover(UUID taskId, long generation, String token, String reason, String rejectedCheckpoint); + String complete(UUID taskId, long generation, String token, String checkpoint, + String coverage, Instant retryAt); + + record Control(String action, long configurationVersion, String reason, Instant retryAt) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialProtocolException.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialProtocolException.java new file mode 100644 index 000000000..c6c989c18 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialProtocolException.java @@ -0,0 +1,16 @@ +package com.iflytek.skillhub.githubsync; + +/** Controlled internal protocol failure; carries no upstream body or credential details. */ +public class GithubOfficialProtocolException extends RuntimeException { + private final int status; + private final String code; + + public GithubOfficialProtocolException(int status, String code) { + super(code); + this.status = status; + this.code = code; + } + + public int status() { return status; } + public String code() { return code; } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialQuotaRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialQuotaRepository.java new file mode 100644 index 000000000..837365388 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubOfficialQuotaRepository.java @@ -0,0 +1,19 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.UUID; + +/** Shared conservative gates, without adding balances across unknown GitHub identities. */ +public interface GithubOfficialQuotaRepository { + record Observation(String resource, Instant observedAt, String state, Integer remaining, + Instant resetAt, Integer retryAfterSeconds) {} + boolean record(UUID taskId, long generation, String leaseTokenHash, UUID expectedCredentialId, + Observation observation, Instant now); + Instant cooldown(String resource, Instant now); + Instant cooldownForLease(UUID taskId, Instant now); + default void deferClaim(UUID taskId, long generation, Instant due, Instant now) { + deferClaim(taskId, generation, due, now, "QUOTA_COOLDOWN"); + } + void deferClaim(UUID taskId, long generation, Instant due, Instant now, String reason); + void clearClaimWait(UUID taskId, long generation); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageAdmissionResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageAdmissionResult.java new file mode 100644 index 000000000..92abe527a --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageAdmissionResult.java @@ -0,0 +1,13 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.UUID; + +public record GithubPackageAdmissionResult(Status status, UUID admissionId) { + public enum Status { + ADMITTED, LEASE_STALE, COMMIT_MISMATCH, BACKPRESSURE_ACTIVE, PATH_IGNORED + } + + public static GithubPackageAdmissionResult rejected(Status status) { + return new GithubPackageAdmissionResult(status, null); + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageReportResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageReportResult.java new file mode 100644 index 000000000..d903e6987 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPackageReportResult.java @@ -0,0 +1,14 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.UUID; + +public record GithubPackageReportResult(Status status, UUID ingestionId, String objectKey) { + public enum Status { + READY_TO_CHECK_OBJECT, RECEIVED, LEASE_STALE, RESULT_CONFLICT, + UPLOAD_NOT_READY, OBJECT_MISSING, OBJECT_MISMATCH, TOO_LARGE + } + + public static GithubPackageReportResult rejected(Status status) { + return new GithubPackageReportResult(status, null, null); + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPlatformControlRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPlatformControlRepository.java new file mode 100644 index 000000000..1e5fe3123 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubPlatformControlRepository.java @@ -0,0 +1,18 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.UUID; + +/** Platform-only acquisition control; accepted ingestions are not paused. */ +public interface GithubPlatformControlRepository { + String CAPABILITY = "platform-pause/v1"; + record State(long namespaceId, long version, boolean desiredPaused, String state, + long pendingStopCount, long expiredUnconfirmedCount, Instant updatedAt) {} + record Control(String action, long platformVersion) {} + State get(); + State update(long expectedVersion, boolean paused, String actor); + Control control(UUID taskId, long generation, String token); + void pause(UUID taskId, long generation, String token, long observedVersion, + String kind, String discoveryCheckpoint, + GithubOfficialLifecycleRepository.RepositoryProgress progress); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryCompletionResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryCompletionResult.java new file mode 100644 index 000000000..3cfe2efdf --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryCompletionResult.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubRepositoryCompletionResult { + ACQUISITION_COMPLETE, + PARTIAL, + RETRY_WAIT, + LEASE_STALE, + COMMIT_MISMATCH, + CHECKPOINT_CONFLICT, + RESULT_CONFLICT, + COVERAGE_CONFLICT +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryMetadataResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryMetadataResult.java new file mode 100644 index 000000000..e4539cede --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryMetadataResult.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubRepositoryMetadataResult { + UPDATED, + LEASE_STALE, + REPOSITORY_MISMATCH +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryPathResult.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryPathResult.java new file mode 100644 index 000000000..b38d925dc --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubRepositoryPathResult.java @@ -0,0 +1,12 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.UUID; + +public record GithubRepositoryPathResult(String skillPath, Outcome outcome, UUID ingestionId, String reason) { + public enum Outcome { + ACCEPTED, + IGNORED, + RETRYABLE_FAILURE, + TERMINAL_FAILURE + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSearchPreviewRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSearchPreviewRepository.java new file mode 100644 index 000000000..ba73c360c --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSearchPreviewRepository.java @@ -0,0 +1,14 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSearchPreviewRepository { + UUID queue(String criteriaJson, String actorId, Instant now); + + Optional find(UUID id); + + record Preview(UUID id, String status, String errorCode, Instant retryAt, + String snapshotJson, Instant createdAt, Instant updatedAt) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSkillPathPolicy.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSkillPathPolicy.java new file mode 100644 index 000000000..ff214c49d --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSkillPathPolicy.java @@ -0,0 +1,60 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Candidate root rules; files inside a parent Skill package are unaffected. */ +public final class GithubSkillPathPolicy { + private static final Set DEFAULT_IGNORED = Set.of( + ".git", "node_modules", ".venv", "venv", "dist", "build", "target", + "__pycache__", "test", "tests"); + + private GithubSkillPathPolicy() {} + + public static List normalizePatterns(List requested) { + if (requested == null || requested.size() > 64) { + throw new IllegalArgumentException("Invalid GitHub source ignore patterns"); + } + Set unique = new HashSet<>(); + for (String pattern : requested) { + if (pattern == null || pattern.isBlank() || pattern.length() > 1024 + || !validPath(pattern) || !unique.add(pattern)) { + throw new IllegalArgumentException("Invalid GitHub source ignore patterns"); + } + } + return unique.stream().sorted().toList(); + } + + public static boolean ignored(String skillPath, List additionalPatterns) { + if (skillPath == null || (!".".equals(skillPath) && !validPath(skillPath))) { + throw new IllegalArgumentException("Invalid GitHub Skill path"); + } + if (".".equals(skillPath)) { + return false; + } + for (String segment : skillPath.split("/")) { + if (DEFAULT_IGNORED.contains(segment)) { + return true; + } + } + for (String prefix : additionalPatterns) { + if (skillPath.equals(prefix) || skillPath.startsWith(prefix + "/")) { + return true; + } + } + return false; + } + + private static boolean validPath(String path) { + if (path.startsWith("/") || path.endsWith("/") || path.indexOf('\\') >= 0) { + return false; + } + for (String segment : path.split("/", -1)) { + if (segment.isEmpty() || segment.equals(".") || segment.equals("..")) { + return false; + } + } + return path.codePoints().noneMatch(Character::isISOControl); + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceBinding.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceBinding.java new file mode 100644 index 000000000..6dd0362ad --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceBinding.java @@ -0,0 +1,133 @@ +package com.iflytek.skillhub.githubsync; + +import com.iflytek.skillhub.domain.skill.SkillVisibility; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.PrePersist; +import jakarta.persistence.PreUpdate; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; + +import java.time.Instant; + +@Entity +@Table(name = "github_source_binding", uniqueConstraints = + @UniqueConstraint(name = "uq_github_source_namespace_repository", columnNames = {"namespace_id", "repository_id"})) +public class GithubSourceBinding { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "namespace_id", nullable = false) + private Long namespaceId; + + @Column(name = "repository_id", nullable = false) + private Long repositoryId; + + @Column(name = "owner_login", nullable = false, length = 128) + private String ownerLogin; + + @Column(name = "repository_name", nullable = false, length = 256) + private String repositoryName; + + @Column(name = "default_branch", length = 256) + private String defaultBranch; + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 20) + private SkillVisibility visibility; + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 20) + private GithubSourceStatus status = GithubSourceStatus.ACTIVE; + + @Column(name = "last_observed_commit", length = 40) + private String lastObservedCommit; + + @Column(name = "last_fully_processed_commit", length = 40) + private String lastFullyProcessedCommit; + + @Column(name = "next_check_at") + private Instant nextCheckAt; + + @Column(name = "unreachable_attempts", nullable = false) + private int unreachableAttempts; + + @Column(name = "last_error_code", length = 64) + private String lastErrorCode; + + @Column(name = "created_at", nullable = false, updatable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + protected GithubSourceBinding() {} + + public GithubSourceBinding(long namespaceId, long repositoryId, String ownerLogin, + String repositoryName, SkillVisibility visibility) { + if (namespaceId <= 0 || repositoryId <= 0) { + throw new IllegalArgumentException("namespace and repository IDs must be positive"); + } + if (ownerLogin == null || ownerLogin.isBlank() || repositoryName == null || repositoryName.isBlank()) { + throw new IllegalArgumentException("repository owner and name are required"); + } + if (visibility != SkillVisibility.PUBLIC && visibility != SkillVisibility.NAMESPACE_ONLY) { + throw new IllegalArgumentException("GitHub source visibility must require review"); + } + this.namespaceId = namespaceId; + this.repositoryId = repositoryId; + this.ownerLogin = ownerLogin; + this.repositoryName = repositoryName; + this.visibility = visibility; + } + + public void updateRepositoryLocation(String ownerLogin, String repositoryName, String defaultBranch) { + if (ownerLogin == null || ownerLogin.isBlank() || repositoryName == null || repositoryName.isBlank()) { + throw new IllegalArgumentException("repository owner and name are required"); + } + this.ownerLogin = ownerLogin; + this.repositoryName = repositoryName; + this.defaultBranch = defaultBranch; + } + + public void pause() { + this.status = GithubSourceStatus.PAUSED; + } + + public void resume() { + this.status = GithubSourceStatus.ACTIVE; + this.unreachableAttempts = 0; + this.lastErrorCode = null; + } + + @PrePersist + void prePersist() { + createdAt = Instant.now(); + updatedAt = createdAt; + } + + @PreUpdate + void preUpdate() { + updatedAt = Instant.now(); + } + + public Long getId() { return id; } + public Long getNamespaceId() { return namespaceId; } + public Long getRepositoryId() { return repositoryId; } + public String getOwnerLogin() { return ownerLogin; } + public String getRepositoryName() { return repositoryName; } + public String getDefaultBranch() { return defaultBranch; } + public SkillVisibility getVisibility() { return visibility; } + public GithubSourceStatus getStatus() { return status; } + public String getLastObservedCommit() { return lastObservedCommit; } + public String getLastFullyProcessedCommit() { return lastFullyProcessedCommit; } + public Instant getNextCheckAt() { return nextCheckAt; } + public int getUnreachableAttempts() { return unreachableAttempts; } + public String getLastErrorCode() { return lastErrorCode; } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceIgnoreRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceIgnoreRepository.java new file mode 100644 index 000000000..ff2f16207 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceIgnoreRepository.java @@ -0,0 +1,10 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.List; +import java.util.Optional; + +public interface GithubSourceIgnoreRepository { + Optional> find(long namespaceId, long sourceId); + Optional> replace(long namespaceId, long sourceId, List patterns); + boolean ignoredForSource(long sourceId, String skillPath); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOrigin.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOrigin.java new file mode 100644 index 000000000..222e6e7fe --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOrigin.java @@ -0,0 +1,58 @@ +package com.iflytek.skillhub.githubsync; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; + +import java.time.Instant; + +@Entity +@Table(name = "github_source_origin", uniqueConstraints = + @UniqueConstraint(name = "uq_github_source_origin", columnNames = {"source_id", "origin_type", "origin_key"})) +public class GithubSourceOrigin { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "source_id", nullable = false) + private Long sourceId; + + @Enumerated(EnumType.STRING) + @Column(name = "origin_type", nullable = false, length = 32) + private GithubSourceOriginType originType; + + @Column(name = "origin_key", nullable = false, length = 256) + private String originKey; + + @Column(name = "created_at", nullable = false, updatable = false) + private Instant createdAt; + + protected GithubSourceOrigin() {} + + public GithubSourceOrigin(long sourceId, GithubSourceOriginType originType, String originKey) { + if (sourceId <= 0 || originType == null || originKey == null || originKey.isBlank()) { + throw new IllegalArgumentException("source and origin are required"); + } + this.sourceId = sourceId; + this.originType = originType; + this.originKey = originKey; + } + + @PrePersist + void prePersist() { + createdAt = Instant.now(); + } + + public Long getId() { return id; } + public Long getSourceId() { return sourceId; } + public GithubSourceOriginType getOriginType() { return originType; } + public String getOriginKey() { return originKey; } + public Instant getCreatedAt() { return createdAt; } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginRepository.java new file mode 100644 index 000000000..22c1b9f3b --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginRepository.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.githubsync; + +public interface GithubSourceOriginRepository { + boolean existsBySourceIdAndOriginTypeAndOriginKey( + Long sourceId, GithubSourceOriginType originType, String originKey); + GithubSourceOrigin save(GithubSourceOrigin origin); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginType.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginType.java new file mode 100644 index 000000000..8e33ec4be --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceOriginType.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSourceOriginType { + SEARCH_RULE, + CURATED_REPOSITORY, + CURATED_OWNER, + TEAM_CONFIG +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceRepository.java new file mode 100644 index 000000000..367e54cd7 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceRepository.java @@ -0,0 +1,8 @@ +package com.iflytek.skillhub.githubsync; + +import java.util.Optional; + +public interface GithubSourceRepository { + Optional findByNamespaceIdAndRepositoryId(Long namespaceId, Long repositoryId); + GithubSourceBinding save(GithubSourceBinding source); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceStatus.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceStatus.java new file mode 100644 index 000000000..9a3b1fb25 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceStatus.java @@ -0,0 +1,7 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSourceStatus { + ACTIVE, + PAUSED, + UNREACHABLE +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceTaskRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceTaskRepository.java new file mode 100644 index 000000000..30b37b57c --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSourceTaskRepository.java @@ -0,0 +1,13 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSourceTaskRepository { + int enqueueDue(Instant now, int maxSources); + + Optional requestRescan(long namespaceId, long sourceId, Instant now); + + record RescanRequest(String status, UUID taskId, String taskStatus, Instant nextCheckAt) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicy.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicy.java new file mode 100644 index 000000000..a036ea363 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicy.java @@ -0,0 +1,15 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +/** Configured high/low watermarks for unresolved GitHub sync work. */ +public record GithubSyncAdmissionPolicy( + Integer globalHigh, + Integer globalLow, + Integer namespaceHigh, + Integer namespaceLow, + int candidatePendingLimit, + int candidatePerLeaseLimit, + boolean globalPaused, + Instant updatedAt) { +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicyRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicyRepository.java new file mode 100644 index 000000000..bbf54c2f2 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncAdmissionPolicyRepository.java @@ -0,0 +1,10 @@ +package com.iflytek.skillhub.githubsync; + +public interface GithubSyncAdmissionPolicyRepository { + GithubSyncAdmissionPolicy get(); + + GithubSyncAdmissionPolicy getForUpdate(); + + GithubSyncAdmissionPolicy update(int globalHigh, int globalLow, int namespaceHigh, int namespaceLow, + int candidatePendingLimit, int candidatePerLeaseLimit); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncDeferralReason.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncDeferralReason.java new file mode 100644 index 000000000..e52d40f17 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncDeferralReason.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSyncDeferralReason { + BUDGET, + RATE_LIMIT, + BACKPRESSURE, + CAPACITY, + LOCAL_DISK_BUDGET +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncFailureReason.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncFailureReason.java new file mode 100644 index 000000000..7c4978dc4 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncFailureReason.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSyncFailureReason { + RETRYABLE_ACQUISITION, + GITHUB_RATE_LIMITED, + SOURCE_UNREADABLE, + INCOMPLETE_TREE, + INTERNAL_FAILURE +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncHistoryRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncHistoryRepository.java new file mode 100644 index 000000000..b0cac6003 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncHistoryRepository.java @@ -0,0 +1,11 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +public interface GithubSyncHistoryRepository { + CleanupResult deleteExpiredDiscoveryHistory(Instant cutoff, int batchSize); + + CleanupResult purgeExpiredRepositoryDetails(Instant cutoff, Instant now, int batchSize); + + record CleanupResult(int tasks, int runs) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncIngestionRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncIngestionRepository.java new file mode 100644 index 000000000..27a0c27c6 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncIngestionRepository.java @@ -0,0 +1,25 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSyncIngestionRepository { + Optional claim(UUID validationToken, Instant now, Instant expiresAt); + + boolean validated(UUID ingestionId, UUID validationToken, String contentSha256, + int warningCount, String evidenceJson, Instant now); + + boolean failed(UUID ingestionId, UUID validationToken, String reason, + Instant retryAt, String pathOutcome, Instant now); + + List claimCleanup(Instant now, Duration signedUrlGrace, int limit); + + void markCleaned(UUID ingestionId, Instant now); + + record ClaimedIngestion(UUID id, UUID validationToken, String objectKey, long sizeBytes, + String sha256, String skillPath, Long sourceId, int attempt) {} + record CleanupIngestion(UUID id, String objectKey) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncLeaseService.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncLeaseService.java new file mode 100644 index 000000000..3993e8ab0 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncLeaseService.java @@ -0,0 +1,325 @@ +package com.iflytek.skillhub.githubsync; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Duration; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Pattern; + +@Service +public class GithubSyncLeaseService { + private static final Pattern SAFE_FAILURE_DETAIL = Pattern.compile( + "WORKER_(?:TIMEOUT|ERROR)|GITHUB_HTTP_[1-5][0-9]{2}:" + + "(?:RETRYABLE_FAILURE|AUTH_INVALID|SOURCE_UNREADABLE|GITHUB_RATE_LIMITED)"); + private final GithubSyncTaskRepository tasks; + private final GithubSystemCredentialRepository credentials; + private final GithubSyncSchedulePolicyRepository schedulePolicies; + private final Duration leaseDuration; + private final Duration repositoryCheckInterval; + private final SecureRandom random = new SecureRandom(); + + public GithubSyncLeaseService(GithubSyncTaskRepository tasks, + GithubSystemCredentialRepository credentials, + GithubSyncSchedulePolicyRepository schedulePolicies, + @Value("${skillhub.github-sync.lease-duration:PT5M}") Duration leaseDuration, + @Value("${skillhub.github-sync.repository-check-interval:PT24H}") + Duration repositoryCheckInterval) { + if (leaseDuration.isNegative() || leaseDuration.isZero() + || repositoryCheckInterval.isNegative() || repositoryCheckInterval.isZero()) { + throw new IllegalArgumentException("GitHub sync intervals must be positive"); + } + this.tasks = tasks; + this.credentials = credentials; + this.schedulePolicies = schedulePolicies; + this.leaseDuration = leaseDuration; + this.repositoryCheckInterval = repositoryCheckInterval; + } + + @Transactional + public Optional claim(GithubSyncTaskKind kind, String workerId) { + return claim(kind, workerId, List.of()); + } + + @Transactional + public Optional claim(GithubSyncTaskKind kind, String workerId, List capabilities) { + if (capabilities == null || capabilities.size() > 32 || capabilities.stream().anyMatch( + value -> value == null || value.isBlank() || value.length() > 128 + || value.chars().anyMatch(Character::isISOControl))) { + throw new IllegalArgumentException("Invalid GitHub worker capabilities"); + } + if (kind == null) { + throw new IllegalArgumentException("task kind is required"); + } + if (workerId == null || workerId.isBlank() || workerId.length() > 128 + || workerId.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("workerId must contain 1 to 128 printable characters"); + } + byte[] tokenBytes = new byte[32]; + random.nextBytes(tokenBytes); + String token = HexFormat.of().formatHex(tokenBytes); + Instant now = Instant.now(); + Optional claimed = capabilities.isEmpty() + ? tasks.claim(kind, workerId, hash(token), now, now.plus(leaseDuration)) + : tasks.claim(kind, workerId, hash(token), now, now.plus(leaseDuration), capabilities); + return claimed + .map(task -> new LeasedTask(task, token)); + } + + @Transactional + public Optional renew(UUID taskId, long generation, String token) { + if (!validLease(taskId, generation, token)) { + return Optional.empty(); + } + Instant now = Instant.now(); + return tasks.renew(taskId, generation, hash(token), now, now.plus(leaseDuration)); + } + + @Transactional + public GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long generation, + String token, long repositoryId, String owner, String name, String defaultBranch) { + return updateRepositoryMetadata(taskId, generation, token, repositoryId, owner, name, defaultBranch, null); + } + + @Transactional + public GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long generation, + String token, long repositoryId, String owner, String name, String defaultBranch, Long ownerId) { + if (!validLease(taskId, generation, token)) { + return GithubRepositoryMetadataResult.LEASE_STALE; + } + if (repositoryId <= 0 || ownerId != null && ownerId <= 0 || owner == null + || !owner.matches("[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?") + || name == null || !name.matches("[A-Za-z0-9_.-]{1,100}") + || name.equals(".") || name.equals("..") + || defaultBranch == null || defaultBranch.isBlank() || defaultBranch.length() > 256 + || defaultBranch.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("invalid public GitHub repository metadata"); + } + return tasks.updateRepositoryMetadata(taskId, generation, hash(token), repositoryId, + owner, name, defaultBranch, ownerId, Instant.now()); + } + + @Transactional + public GithubPackageAdmissionResult admitPackage(UUID taskId, long generation, String token, + String commitSha, String skillPath) { + if (!validLease(taskId, generation, token)) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.LEASE_STALE); + } + if (commitSha == null || !commitSha.matches("[0-9a-f]{40}")) { + throw new IllegalArgumentException("commitSha must be a lowercase Git commit SHA"); + } + // Full coordinate validation is repeated after Java reads the repository ID from its task snapshot. + if (skillPath == null || skillPath.isEmpty()) { + throw new IllegalArgumentException("skillPath is required"); + } + return tasks.admitPackage(taskId, generation, hash(token), commitSha, skillPath, Instant.now()); + } + + @Transactional + public boolean completeHeadOnly(UUID taskId, long generation, String token, String observedCommitSha) { + if (!validLease(taskId, generation, token)) { + return false; + } + if (observedCommitSha == null || !observedCommitSha.matches("[0-9a-f]{40}")) { + throw new IllegalArgumentException("observedCommitSha must be a lowercase Git commit SHA"); + } + Instant now = Instant.now(); + return tasks.completeHeadOnly(taskId, generation, hash(token), observedCommitSha, + now, now.plus(repositoryInterval())); + } + + @Transactional + public Optional fail(UUID taskId, long generation, String token, + GithubSyncFailureReason reason, String detail, + Quota quota, Instant retryAt) { + if (!validLease(taskId, generation, token)) { + return Optional.empty(); + } + if (reason == null) { + throw new IllegalArgumentException("failure reason is required"); + } + Instant now = Instant.now(); + if (reason == GithubSyncFailureReason.GITHUB_RATE_LIMITED + && (retryAt == null || !retryAt.isAfter(now))) { + throw new IllegalArgumentException("GitHub rate limit requires a future retryAt"); + } + if (reason == GithubSyncFailureReason.SOURCE_UNREADABLE && retryAt != null) { + throw new IllegalArgumentException("SOURCE_UNREADABLE retry schedule is owned by Java"); + } + if (retryAt != null && !retryAt.isAfter(now)) { + throw new IllegalArgumentException("retryAt must be in the future"); + } + if (detail != null && !SAFE_FAILURE_DETAIL.matcher(detail).matches()) { + throw new IllegalArgumentException("detail must be a supported diagnostic code"); + } + if (quota != null) { + if (quota.resource() == null + || !java.util.Set.of("CORE", "SEARCH", "GRAPHQL", "OTHER").contains(quota.resource()) + || quota.remaining() < 0 || quota.resetAt() == null) { + throw new IllegalArgumentException("invalid GitHub quota observation"); + } + credentials.recordQuotaForTaskLease(taskId, generation, hash(token), quota.resource(), + quota.remaining(), quota.resetAt(), now); + } + return tasks.fail(taskId, generation, hash(token), reason, detail, retryAt, now); + } + + @Transactional + public GithubDiscoveryBatchResult reportDiscoveryBatch(UUID taskId, long generation, + String token, GithubDiscoveryBatch batch) { + if (!validLease(taskId, generation, token)) { + return GithubDiscoveryBatchResult.LEASE_STALE; + } + if (batch == null) { + throw new IllegalArgumentException("discovery batch is required"); + } + return tasks.recordDiscoveryBatch(taskId, generation, hash(token), batch, + hashDiscoveryBatch(batch), Instant.now()); + } + + @Transactional + public GithubDiscoveryCompletionResult completeDiscovery(UUID taskId, long generation, String token, + String checkpoint, boolean complete, Instant retryAt, GithubSyncDeferralReason deferralReason) { + if (!validLease(taskId, generation, token)) { + return GithubDiscoveryCompletionResult.LEASE_STALE; + } + if (checkpoint == null || checkpoint.isBlank() || checkpoint.length() > 4096 + || checkpoint.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("a persisted discovery checkpoint is required"); + } + Instant now = Instant.now(); + if ((complete && (retryAt != null || deferralReason != null)) + || (!complete && (retryAt == null || !retryAt.isAfter(now)))) { + throw new IllegalArgumentException("discovery coverage and retry schedule disagree"); + } + return tasks.completeDiscovery(taskId, generation, hash(token), checkpoint, + complete, retryAt, deferralReason, now); + } + + @Transactional + public boolean completePreview(UUID taskId, long generation, String token, String resultJson) { + if (!validLease(taskId, generation, token) || resultJson == null || resultJson.length() > 32768) { + return false; + } + return tasks.completePreview(taskId, generation, hash(token), resultJson, Instant.now()); + } + + @Transactional + public GithubRepositoryCompletionResult completeRepository(UUID taskId, long generation, String token, + String commitSha, boolean complete, String pathCheckpoint, List incompleteSubtrees, + List pathResults, Instant retryAt, + GithubSyncDeferralReason deferralReason) { + if (!validLease(taskId, generation, token)) { + return GithubRepositoryCompletionResult.LEASE_STALE; + } + if (commitSha == null || !commitSha.matches("[0-9a-f]{40}")) { + throw new IllegalArgumentException("observedCommitSha must be a lowercase Git commit SHA"); + } + if (pathCheckpoint != null) { + new RepositorySkillCoordinate(1, 1, pathCheckpoint); + } + List subtrees = incompleteSubtrees == null ? List.of() : incompleteSubtrees; + List results = pathResults == null ? List.of() : pathResults; + if (subtrees.size() > 1000 || results.size() > 10000) { + throw new IllegalArgumentException("repository completion exceeds path budget"); + } + var seen = new HashSet(); + for (String subtree : subtrees) { + new RepositorySkillCoordinate(1, 1, subtree); + if (!seen.add(subtree)) { + throw new IllegalArgumentException("duplicate incomplete subtree"); + } + } + seen.clear(); + boolean retryable = false; + for (GithubRepositoryPathResult result : results) { + if (result == null || result.outcome() == null) { + throw new IllegalArgumentException("path outcome is required"); + } + new RepositorySkillCoordinate(1, 1, result.skillPath()); + if (!seen.add(result.skillPath()) || result.reason() != null && result.reason().length() > 500 + || result.outcome() != GithubRepositoryPathResult.Outcome.ACCEPTED + && result.ingestionId() != null + || result.outcome() == GithubRepositoryPathResult.Outcome.ACCEPTED + && result.ingestionId() == null) { + throw new IllegalArgumentException("invalid repository path result"); + } + retryable |= result.outcome() == GithubRepositoryPathResult.Outcome.RETRYABLE_FAILURE; + } + Instant now = Instant.now(); + if (complete && (retryAt != null || deferralReason != null || retryable || !subtrees.isEmpty()) + || !complete && (retryAt == null || !retryAt.isAfter(now))) { + throw new IllegalArgumentException("repository coverage and retry schedule disagree"); + } + return tasks.completeRepository(taskId, generation, hash(token), commitSha, complete, + pathCheckpoint, subtrees, results, retryAt, deferralReason, now, + now.plus(repositoryInterval())); + } + + private Duration repositoryInterval() { + Integer hours = schedulePolicies.get().repositoryHours(); + return hours == null ? repositoryCheckInterval : Duration.ofHours(hours); + } + + private String hashDiscoveryBatch(GithubDiscoveryBatch batch) { + MessageDigest digest = newDigest(); + updateDigest(digest, batch.previousCheckpoint()); + updateDigest(digest, batch.checkpoint()); + updateDigest(digest, Integer.toString(batch.deferredCount())); + updateDigest(digest, batch.deferralReason() == null ? null : batch.deferralReason().name()); + batch.repositories().stream().sorted(Comparator.comparingLong(GithubDiscoveryCandidate::repositoryId)) + .forEach(repository -> { + updateDigest(digest, Long.toString(repository.repositoryId())); + updateDigest(digest, repository.owner()); + updateDigest(digest, repository.name()); + updateDigest(digest, repository.defaultBranch()); + }); + return HexFormat.of().formatHex(digest.digest()); + } + + private void updateDigest(MessageDigest digest, String value) { + if (value == null) { + digest.update(ByteBuffer.allocate(4).putInt(-1).array()); + return; + } + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(4).putInt(bytes.length).array()); + digest.update(bytes); + } + + private boolean validLease(UUID taskId, long generation, String token) { + return taskId != null && generation >= 1 && token != null && token.length() == 64 + && token.chars().allMatch(value -> (value >= '0' && value <= '9') + || (value >= 'a' && value <= 'f')); + } + + private String hash(String token) { + return HexFormat.of().formatHex(newDigest().digest(token.getBytes(StandardCharsets.UTF_8))); + } + + private MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + public record LeasedTask(GithubSyncTaskLease task, String token) { + } + + public record Quota(String resource, int remaining, Instant resetAt) { + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncPublicationRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncPublicationRepository.java new file mode 100644 index 000000000..d4fd92546 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncPublicationRepository.java @@ -0,0 +1,68 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +/** Durable handoff from a validated staged package to a Skill version and sync review. */ +public interface GithubSyncPublicationRepository { + Optional claim(UUID token, Instant now, Instant expiresAt); + + /** Locks the claim row for the publication transaction. */ + boolean lockAndVerify(ClaimedPublication claim, Instant now); + + /** Reads the current path target after the publication transaction has acquired its path lock. */ + CurrentTarget currentTarget(ClaimedPublication claim); + + boolean superseded(ClaimedPublication claim, Instant now); + + /** Returns false if a newer worker owns the lease; callers must roll back their transaction. */ + boolean readyForReview(ClaimedPublication claim, Long skillId, Long versionId, Instant now); + + /** Records actual newly created entities only, in the successful publication transaction. */ + void recordOutcome(ClaimedPublication claim, Long skillId, Long versionId, + boolean newSkill, Instant now); + + boolean noChange(ClaimedPublication claim, Instant now); + + /** Marks one rejected path for a fresh acquisition and review. */ + boolean requestRejectedRetry(long namespaceId, long sourceId, String skillPath, Instant now); + + /** Classifies an unbound path against the completed repository snapshot. */ + PathMoveDecision inspectPathMove(ClaimedPublication claim); + + /** Atomically moves the source binding and settles the identical package without a new version. */ + boolean movePath(ClaimedPublication claim, PathMoveDecision decision, Instant now); + + boolean ignored(ClaimedPublication claim, Instant now); + + boolean blocked(ClaimedPublication claim, String reason, Instant now); + + /** Persists the canonical slug when a namespace coordinate needs an administrator decision. */ + boolean blockedSlugConflict(ClaimedPublication claim, String skillSlug, Instant now); + + boolean retry(ClaimedPublication claim, String reason, Instant retryAt, Instant now); + + enum PathMoveOutcome { NONE, RETRY, INCOMPLETE, STALE, AMBIGUOUS, CANDIDATE } + + record CurrentTarget(Long skillId, String contentSha256, boolean retryRequested, + boolean newerAcquisition) {} + + record PathMoveDecision(PathMoveOutcome outcome, Long skillId, String previousPath) { + public static PathMoveDecision of(PathMoveOutcome outcome) { + return new PathMoveDecision(outcome, null, null); + } + } + + record ClaimedPublication(UUID id, UUID token, Long namespaceId, String namespaceSlug, + long repositoryId, Long sourceId, String skillPath, + String sourceStatus, String visibility, String objectKey, long sizeBytes, + String archiveSha256, String contentSha256, + Long expectedSkillId, int attempt) { + public ClaimedPublication withExpectedSkillId(Long skillId) { + return new ClaimedPublication(id, token, namespaceId, namespaceSlug, repositoryId, + sourceId, skillPath, sourceStatus, visibility, objectKey, sizeBytes, + archiveSha256, contentSha256, skillId, attempt); + } + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicy.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicy.java new file mode 100644 index 000000000..8cfdf152f --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicy.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; + +public record GithubSyncSchedulePolicy(Integer repositoryHours, int curatedOwnerHours, + int searchRuleHours, Instant updatedAt) {} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicyRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicyRepository.java new file mode 100644 index 000000000..af44fa04e --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncSchedulePolicyRepository.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +public interface GithubSyncSchedulePolicyRepository { + GithubSyncSchedulePolicy get(); + + GithubSyncSchedulePolicy getForUpdate(); + + GithubSyncSchedulePolicy update(int repositoryHours, int curatedOwnerHours, int searchRuleHours); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskKind.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskKind.java new file mode 100644 index 000000000..3e860fc7c --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskKind.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSyncTaskKind { + DISCOVERY, + REPOSITORY_SYNC +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskLease.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskLease.java new file mode 100644 index 000000000..a7ad3e347 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskLease.java @@ -0,0 +1,49 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +/** Persisted task facts returned by an atomic claim; the raw lease token is never stored here. */ +public record GithubSyncTaskLease( + UUID taskId, + UUID runId, + GithubSyncTaskKind kind, + GithubSyncTaskMode mode, + long namespaceId, + Long sourceId, + String snapshotJson, + String discoveryCheckpoint, + String fixedCommitSha, + String pathCheckpoint, + String incompleteSubtreesJson, + List retryPaths, + List acceptedPaths, + long leaseGeneration, + Instant leaseExpiresAt, + List requiredCapabilities, Long platformControlVersion) { + + public GithubSyncTaskLease { + requiredCapabilities = List.copyOf(requiredCapabilities); + } + + public GithubSyncTaskLease(UUID taskId, UUID runId, GithubSyncTaskKind kind, + GithubSyncTaskMode mode, long namespaceId, Long sourceId, String snapshotJson, + String discoveryCheckpoint, String fixedCommitSha, String pathCheckpoint, + String incompleteSubtreesJson, List retryPaths, List acceptedPaths, + long leaseGeneration, Instant leaseExpiresAt, List requiredCapabilities) { + this(taskId, runId, kind, mode, namespaceId, sourceId, snapshotJson, discoveryCheckpoint, + fixedCommitSha, pathCheckpoint, incompleteSubtreesJson, retryPaths, acceptedPaths, + leaseGeneration, leaseExpiresAt, requiredCapabilities, null); + } + + public GithubSyncTaskLease(UUID taskId, UUID runId, GithubSyncTaskKind kind, + GithubSyncTaskMode mode, long namespaceId, Long sourceId, String snapshotJson, + String discoveryCheckpoint, String fixedCommitSha, String pathCheckpoint, + String incompleteSubtreesJson, List retryPaths, List acceptedPaths, + long leaseGeneration, Instant leaseExpiresAt) { + this(taskId, runId, kind, mode, namespaceId, sourceId, snapshotJson, discoveryCheckpoint, + fixedCommitSha, pathCheckpoint, incompleteSubtreesJson, retryPaths, acceptedPaths, + leaseGeneration, leaseExpiresAt, List.of()); + } +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskMode.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskMode.java new file mode 100644 index 000000000..af26aa772 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskMode.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.githubsync; + +public enum GithubSyncTaskMode { + FULL, + HEAD_ONLY +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskRepository.java new file mode 100644 index 000000000..76dc2f420 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncTaskRepository.java @@ -0,0 +1,54 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSyncTaskRepository { + Optional claim(GithubSyncTaskKind kind, String workerId, + String leaseTokenHash, Instant now, Instant expiresAt); + + default Optional claim(GithubSyncTaskKind kind, String workerId, + String leaseTokenHash, Instant now, Instant expiresAt, List capabilities) { + return claim(kind, workerId, leaseTokenHash, now, expiresAt); + } + + Optional renew(UUID taskId, long leaseGeneration, String leaseTokenHash, + Instant now, Instant expiresAt); + + GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long leaseGeneration, + String leaseTokenHash, long repositoryId, String owner, String name, + String defaultBranch, Instant now); + + default GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long leaseGeneration, + String leaseTokenHash, long repositoryId, String owner, String name, + String defaultBranch, Long ownerId, Instant now) { + return updateRepositoryMetadata(taskId, leaseGeneration, leaseTokenHash, repositoryId, owner, name, defaultBranch, now); + } + + GithubPackageAdmissionResult admitPackage(UUID taskId, long leaseGeneration, + String leaseTokenHash, String commitSha, String skillPath, Instant now); + + boolean completeHeadOnly(UUID taskId, long leaseGeneration, String leaseTokenHash, + String observedCommitSha, Instant now, Instant nextCheckAt); + + Optional fail(UUID taskId, long leaseGeneration, String leaseTokenHash, + GithubSyncFailureReason reason, String detail, + Instant requestedRetryAt, Instant now); + + GithubDiscoveryBatchResult recordDiscoveryBatch(UUID taskId, long leaseGeneration, + String leaseTokenHash, GithubDiscoveryBatch batch, String requestSha256, Instant now); + + GithubDiscoveryCompletionResult completeDiscovery(UUID taskId, long leaseGeneration, + String leaseTokenHash, String checkpoint, boolean complete, + Instant retryAt, GithubSyncDeferralReason deferralReason, Instant now); + + boolean completePreview(UUID taskId, long leaseGeneration, String leaseTokenHash, + String resultJson, Instant now); + + GithubRepositoryCompletionResult completeRepository(UUID taskId, long leaseGeneration, + String leaseTokenHash, String commitSha, boolean complete, String pathCheckpoint, + List incompleteSubtrees, List pathResults, + Instant retryAt, GithubSyncDeferralReason deferralReason, Instant now, Instant nextCheckAt); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncUploadRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncUploadRepository.java new file mode 100644 index 000000000..f257d4cd6 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSyncUploadRepository.java @@ -0,0 +1,34 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSyncUploadRepository { + boolean issue(UUID taskId, long generation, String leaseTokenHash, UUID admissionId, + String commitSha, String skillPath, UUID uploadId, String objectKey, + String mode, String ticketHash, long sizeBytes, String sha256, Instant expiresAt, Instant now); + + Optional beginLocalUpload(UUID uploadId, String ticketHash, Instant now); + + boolean finishLocalUpload(UUID uploadId, Instant now); + + boolean failUpload(UUID uploadId, Instant now); + + List claimCleanup(Instant now, Duration signedUrlGrace, int limit); + + void markCleaned(UUID uploadId, Instant now); + + GithubPackageReportResult lookupReport(UUID taskId, long generation, String leaseTokenHash, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256, Instant now); + + GithubPackageReportResult acceptReport(UUID taskId, long generation, String leaseTokenHash, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256, Instant now); + + record LocalUpload(String objectKey, long sizeBytes, String sha256) {} + record CleanupUpload(UUID uploadId, String objectKey) {} +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredential.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredential.java new file mode 100644 index 000000000..7897af288 --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredential.java @@ -0,0 +1,9 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.UUID; + +public record GithubSystemCredential(UUID id, String label, String fingerprint, + String keyVersion, byte[] nonce, byte[] ciphertext, String status, + Instant rateLimitedUntil, Instant lastUsedAt, Instant createdAt, Instant updatedAt) { +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredentialRepository.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredentialRepository.java new file mode 100644 index 000000000..a9a4cd35b --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/GithubSystemCredentialRepository.java @@ -0,0 +1,28 @@ +package com.iflytek.skillhub.githubsync; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface GithubSystemCredentialRepository { + List list(); + List quotas(UUID credentialId); + Optional find(UUID id); + default boolean hasConfiguredCredentials() { return !list().isEmpty(); } + Optional availableForClaim(Instant now, String resource); + void create(GithubSystemCredential credential); + boolean replace(UUID id, String label, String fingerprint, String keyVersion, byte[] nonce, byte[] ciphertext); + boolean setEnabled(UUID id, boolean enabled); + boolean assignToLease(UUID taskId, long generation, UUID credentialId, + String fingerprint, Instant now); + boolean markInvalid(UUID taskId, long generation, String leaseTokenHash, UUID credentialId, Instant now); + boolean markRateLimited(UUID taskId, long generation, String leaseTokenHash, + UUID credentialId, Instant retryAt, Instant now); + boolean recordQuotaForLease(UUID taskId, long generation, String leaseTokenHash, + UUID credentialId, String resource, int remaining, + Instant resetAt, Instant now); + boolean recordQuotaForTaskLease(UUID taskId, long generation, String leaseTokenHash, + String resource, int remaining, Instant resetAt, Instant now); + void recordQuota(UUID credentialId, String resource, int remaining, Instant resetAt, Instant now); +} diff --git a/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/RepositorySkillCoordinate.java b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/RepositorySkillCoordinate.java new file mode 100644 index 000000000..47dbe0f5c --- /dev/null +++ b/server/skillhub-github-sync/src/main/java/com/iflytek/skillhub/githubsync/RepositorySkillCoordinate.java @@ -0,0 +1,26 @@ +package com.iflytek.skillhub.githubsync; + +/** + * Identity of one upstream Skill within a SkillHub namespace. Repository names and commits are + * deliberately excluded: both can change without changing the source binding. + */ +public record RepositorySkillCoordinate(long namespaceId, long repositoryId, String skillPath) { + + public RepositorySkillCoordinate { + if (namespaceId <= 0 || repositoryId <= 0) { + throw new IllegalArgumentException("namespaceId and repositoryId must be positive"); + } + if (skillPath == null || skillPath.isEmpty() || skillPath.length() > 1024 + || skillPath.startsWith("/") || skillPath.endsWith("/") + || skillPath.indexOf('\\') >= 0 || skillPath.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("skillPath must be a normalized relative POSIX directory"); + } + if (!skillPath.equals(".")) { + for (String segment : skillPath.split("/", -1)) { + if (segment.isEmpty() || segment.equals(".") || segment.equals("..")) { + throw new IllegalArgumentException("skillPath must be a normalized relative POSIX directory"); + } + } + } + } +} diff --git a/server/skillhub-infra/pom.xml b/server/skillhub-infra/pom.xml index 96d63ecd3..2027d17e0 100644 --- a/server/skillhub-infra/pom.xml +++ b/server/skillhub-infra/pom.xml @@ -23,6 +23,10 @@ com.iflytek.skillhub skillhub-notification + + com.iflytek.skillhub + skillhub-github-sync + org.springframework.boot spring-boot-starter-webflux diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/AesGcmGithubCredentialCipher.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/AesGcmGithubCredentialCipher.java new file mode 100644 index 000000000..fa02bd8f3 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/AesGcmGithubCredentialCipher.java @@ -0,0 +1,121 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubCredentialCipher; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** Key material is supplied by the deployment secret, never persisted alongside ciphertext. */ +@Component +public class AesGcmGithubCredentialCipher implements GithubCredentialCipher { + private static final int NONCE_BYTES = 12; + private static final int TAG_BITS = 128; + private final Map keys; + private final String activeVersion; + private final SecureRandom random = new SecureRandom(); + + public AesGcmGithubCredentialCipher( + @Value("${skillhub.github-sync.credential-keys:}") String configuredKeys, + @Value("${skillhub.github-sync.active-credential-key-version:}") String activeVersion) { + Map parsed = new HashMap<>(); + if (!configuredKeys.isBlank()) { + for (String item : configuredKeys.split(",", -1)) { + int separator = item.indexOf('='); + if (separator < 1 || separator == item.length() - 1) { + throw new IllegalArgumentException("Invalid GitHub credential key configuration"); + } + String version = item.substring(0, separator).trim(); + if (!version.matches("[A-Za-z0-9._-]{1,40}") || parsed.containsKey(version)) { + throw new IllegalArgumentException("Invalid GitHub credential key version"); + } + byte[] decoded; + try { + decoded = Base64.getDecoder().decode(item.substring(separator + 1).trim()); + } catch (IllegalArgumentException exception) { + throw new IllegalArgumentException("Invalid GitHub credential key encoding"); + } + if (decoded.length != 32) { + throw new IllegalArgumentException("GitHub credential keys must be 32 bytes"); + } + parsed.put(version, new SecretKeySpec(decoded, "AES")); + Arrays.fill(decoded, (byte) 0); + } + } + this.activeVersion = activeVersion.trim(); + if (parsed.isEmpty() != this.activeVersion.isEmpty() + || !parsed.isEmpty() && !parsed.containsKey(this.activeVersion)) { + throw new IllegalArgumentException("Active GitHub credential key version is missing"); + } + this.keys = Map.copyOf(parsed); + } + + @Override + public boolean configured() { + return !keys.isEmpty(); + } + + @Override + public boolean supports(String keyVersion) { + return keys.containsKey(keyVersion); + } + + @Override + public EncryptedValue encrypt(UUID credentialId, String token) { + if (!configured()) { + throw new IllegalStateException("GitHub credential encryption key is not configured"); + } + byte[] nonce = new byte[NONCE_BYTES]; + random.nextBytes(nonce); + try { + Cipher cipher = init(Cipher.ENCRYPT_MODE, credentialId, activeVersion, nonce); + byte[] plaintext = token.getBytes(StandardCharsets.UTF_8); + try { + return new EncryptedValue(activeVersion, nonce, cipher.doFinal(plaintext)); + } finally { + Arrays.fill(plaintext, (byte) 0); + } + } catch (GeneralSecurityException exception) { + throw new IllegalStateException("Could not encrypt GitHub credential", exception); + } + } + + @Override + public String decrypt(UUID credentialId, String keyVersion, byte[] nonce, byte[] ciphertext) { + if (!keys.containsKey(keyVersion)) { + throw new IllegalStateException("GitHub credential key version is unavailable: " + keyVersion); + } + if (nonce == null || nonce.length != NONCE_BYTES || ciphertext == null) { + throw new IllegalStateException("GitHub credential ciphertext is malformed"); + } + try { + Cipher cipher = init(Cipher.DECRYPT_MODE, credentialId, keyVersion, nonce); + byte[] plaintext = cipher.doFinal(ciphertext); + try { + return new String(plaintext, StandardCharsets.UTF_8); + } finally { + Arrays.fill(plaintext, (byte) 0); + } + } catch (GeneralSecurityException exception) { + throw new IllegalStateException("Could not decrypt GitHub credential", exception); + } + } + + private Cipher init(int mode, UUID credentialId, String version, byte[] nonce) + throws GeneralSecurityException { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(mode, keys.get(version), new GCMParameterSpec(TAG_BITS, nonce)); + cipher.updateAAD((credentialId + ":" + version).getBytes(StandardCharsets.UTF_8)); + return cipher; + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceJpaRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceJpaRepository.java new file mode 100644 index 000000000..279c9bace --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceJpaRepository.java @@ -0,0 +1,10 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSourceBinding; +import com.iflytek.skillhub.githubsync.GithubSourceRepository; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GithubSourceJpaRepository extends JpaRepository, GithubSourceRepository { +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceOriginJpaRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceOriginJpaRepository.java new file mode 100644 index 000000000..5105d17db --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/GithubSourceOriginJpaRepository.java @@ -0,0 +1,11 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSourceOrigin; +import com.iflytek.skillhub.githubsync.GithubSourceOriginRepository; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GithubSourceOriginJpaRepository + extends JpaRepository, GithubSourceOriginRepository { +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubCandidateTaskRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubCandidateTaskRepository.java new file mode 100644 index 000000000..3edabe7a6 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubCandidateTaskRepository.java @@ -0,0 +1,166 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubCandidateTaskRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +@Repository +public class JdbcGithubCandidateTaskRepository implements GithubCandidateTaskRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper objectMapper; + private final JdbcGithubSyncCapacityGate capacityGate; + + public JdbcGithubCandidateTaskRepository(JdbcTemplate jdbc, ObjectMapper objectMapper, + JdbcGithubSyncCapacityGate capacityGate) { + this.jdbc = jdbc; + this.objectMapper = objectMapper; + this.capacityGate = capacityGate; + } + + @Override + @Transactional + public int enqueueDue(Instant now, int maxCandidates) { + requireBatchSize(maxCandidates); + jdbc.queryForObject("SELECT enabled FROM github_deep_discovery_config WHERE id = TRUE FOR SHARE", Boolean.class); + List due = jdbc.query(""" + SELECT candidate.id, candidate.namespace_id, candidate.repository_id, + candidate.owner_login, candidate.repository_name, eligibility.official_only, + EXISTS ( + SELECT 1 FROM github_discovery_candidate_origin origin + WHERE origin.candidate_id = candidate.id + AND origin.origin_type = 'CURATED_OWNER' + ) AS curated_owner + FROM github_discovery_candidate candidate + JOIN github_candidate_discovery_eligibility eligibility ON eligibility.candidate_id = candidate.id + WHERE candidate.status IN ('PENDING', 'REJECTED') + AND candidate.next_validation_at <= ? + AND eligibility.acquisition_allowed + ORDER BY candidate.next_validation_at, candidate.id + LIMIT ? FOR UPDATE OF candidate SKIP LOCKED + """, (result, row) -> new Candidate( + result.getLong("id"), result.getLong("namespace_id"), + result.getLong("repository_id"), result.getString("owner_login"), + result.getString("repository_name"), result.getBoolean("curated_owner"), result.getBoolean("official_only")), + Timestamp.from(now), maxCandidates); + int queued = 0; + for (Candidate candidate : due) { + Integer source = jdbc.query(""" + SELECT 1 FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, result -> result.next() ? 1 : null, + candidate.namespaceId(), candidate.repositoryId()); + if (source != null) { + jdbc.update(""" + UPDATE github_discovery_candidate + SET status = 'BOUND', last_error_code = NULL WHERE id = ? + """, candidate.id()); + continue; + } + if (!capacityGate.allowsFullAcquisition(candidate.namespaceId(), now)) { + continue; + } + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + String dedupeKey = "repository:" + candidate.repositoryId(); + ObjectNode snapshot = objectMapper.createObjectNode(); + snapshot.put("candidateId", candidate.id()); + snapshot.put("repositoryId", candidate.repositoryId()); + snapshot.put("owner", candidate.owner()); + snapshot.put("name", candidate.name()); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind, status, created_at) + VALUES (?, ?, 'REPOSITORY_SYNC', 'QUEUED', ?) + """, runId, candidate.namespaceId(), Timestamp.from(now)); + int inserted = jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, mode, status, dedupe_key, + priority, snapshot_json, due_at, created_at, updated_at, required_capabilities) + VALUES (?, ?, ?, 'REPOSITORY_SYNC', 'FULL', 'QUEUED', ?, ?, ?::jsonb, ?, ?, ?, CASE WHEN ? THEN ARRAY['official-pause/v1'] ELSE ARRAY[]::text[] END) + ON CONFLICT (namespace_id, dedupe_key) + WHERE status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') DO NOTHING + """, taskId, runId, candidate.namespaceId(), dedupeKey, + candidate.curatedOwner() ? 70 : 40, snapshot.toString(), + Timestamp.from(now), Timestamp.from(now), Timestamp.from(now), candidate.officialOnly()); + if (inserted == 0) { + jdbc.update("DELETE FROM github_sync_run WHERE id = ?", runId); + continue; + } + jdbc.update(""" + UPDATE github_discovery_candidate + SET status = 'VALIDATING', validation_task_id = ?, last_error_code = NULL + WHERE id = ? + """, taskId, candidate.id()); + queued++; + } + return queued; + } + + @Override + @Transactional + public int reconcileCompleted(Instant now, int maxCandidates) { + requireBatchSize(maxCandidates); + List bound = jdbc.query(""" + SELECT candidate.id FROM github_discovery_candidate candidate + WHERE candidate.status IN ('VALIDATING', 'UNREACHABLE') + AND EXISTS (SELECT 1 FROM github_source_binding source + WHERE source.namespace_id = candidate.namespace_id + AND source.repository_id = candidate.repository_id) + ORDER BY candidate.id + LIMIT ? FOR UPDATE OF candidate SKIP LOCKED + """, (result, row) -> result.getLong("id"), maxCandidates); + for (Long id : bound) { + jdbc.update(""" + UPDATE github_discovery_candidate + SET status = 'BOUND', last_error_code = NULL WHERE id = ? + """, id); + } + List completed = jdbc.query(""" + SELECT candidate.id, task.status FROM github_discovery_candidate candidate + JOIN github_sync_task task ON task.id = candidate.validation_task_id + WHERE candidate.status IN ('VALIDATING', 'UNREACHABLE') + AND task.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND NOT EXISTS ( + SELECT 1 FROM github_sync_package_ingestion ingestion + WHERE ingestion.task_id = task.id + AND ingestion.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', + 'VALIDATED', 'READY_FOR_REVIEW', 'BLOCKED')) + AND NOT EXISTS ( + SELECT 1 FROM github_source_binding source + WHERE source.namespace_id = candidate.namespace_id + AND source.repository_id = candidate.repository_id) + ORDER BY candidate.id + LIMIT ? FOR UPDATE OF candidate SKIP LOCKED + """, (result, row) -> new Completed(result.getLong("id"), result.getString("status")), + maxCandidates - bound.size()); + for (Completed candidate : completed) { + jdbc.update(""" + UPDATE github_discovery_candidate + SET status = 'REJECTED', next_validation_at = ?, + last_error_code = ? WHERE id = ? + """, Timestamp.from(now.plus(Duration.ofDays(7))), + "FAILED".equals(candidate.taskStatus()) ? "REPOSITORY_FAILED" : "NO_VALID_SKILL", + candidate.id()); + } + return bound.size() + completed.size(); + } + + private void requireBatchSize(int value) { + if (value < 1 || value > 100) { + throw new IllegalArgumentException("GitHub candidate batch size must be 1 to 100"); + } + } + + private record Candidate(long id, long namespaceId, long repositoryId, + String owner, String name, boolean curatedOwner, boolean officialOnly) {} + + private record Completed(long id, String taskStatus) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDeepDiscoveryConfigurationRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDeepDiscoveryConfigurationRepository.java new file mode 100644 index 000000000..eade7564b --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDeepDiscoveryConfigurationRepository.java @@ -0,0 +1,76 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfiguration; +import com.iflytek.skillhub.githubsync.GithubDeepDiscoveryConfigurationRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Optional; + +/** JDBC keeps the version comparison and whole-configuration replacement in one SQL statement. */ +@Repository +public class JdbcGithubDeepDiscoveryConfigurationRepository implements GithubDeepDiscoveryConfigurationRepository { + private static final TypeReference> EXCLUSIONS = + new TypeReference<>() {}; + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + + public JdbcGithubDeepDiscoveryConfigurationRepository(JdbcTemplate jdbc, ObjectMapper mapper) { + this.jdbc = jdbc; + this.mapper = mapper; + } + + @Override + public GithubDeepDiscoveryConfiguration get() { + return jdbc.query("SELECT * FROM github_deep_discovery_config WHERE id = TRUE", result -> { + if (!result.next()) { + throw new IllegalStateException("GitHub deep discovery configuration is missing"); + } + return map(result); + }); + } + + @Override + public Optional update(long expectedVersion, boolean enabled, + GithubDeepDiscoveryConfiguration.BudgetProfile budgetProfile, + List exclusions, String actorId) { + String exclusionsJson; + try { + exclusionsJson = mapper.writeValueAsString(exclusions); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Could not serialize GitHub discovery exclusions", exception); + } + return jdbc.query(""" + UPDATE github_deep_discovery_config + SET enabled = ?, budget_profile = ?, exclusions_json = ?::jsonb, + config_version = config_version + 1, updated_at = CURRENT_TIMESTAMP, updated_by = ? + WHERE id = TRUE AND config_version = ? + RETURNING * + """, statement -> { + statement.setBoolean(1, enabled); + statement.setString(2, budgetProfile.name()); + statement.setString(3, exclusionsJson); + statement.setString(4, actorId); + statement.setLong(5, expectedVersion); + }, result -> result.next() ? Optional.of(map(result)) : Optional.empty()); + } + + private GithubDeepDiscoveryConfiguration map(ResultSet result) throws SQLException { + try { + return new GithubDeepDiscoveryConfiguration(result.getLong("namespace_id"), + result.getLong("config_version"), result.getBoolean("enabled"), + GithubDeepDiscoveryConfiguration.BudgetProfile.valueOf(result.getString("budget_profile")), + mapper.readValue(result.getString("exclusions_json"), EXCLUSIONS), + result.getString("strategy_version"), result.getTimestamp("updated_at").toInstant(), + result.getString("updated_by")); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted GitHub discovery exclusions", exception); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryEntryRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryEntryRepository.java new file mode 100644 index 000000000..1c476de25 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryEntryRepository.java @@ -0,0 +1,360 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntry; +import com.iflytek.skillhub.githubsync.GithubDiscoveryEntryRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +/** Discovery configuration uses JSONB snapshots and PostgreSQL RETURNING in one statement. */ +@Repository +public class JdbcGithubDiscoveryEntryRepository implements GithubDiscoveryEntryRepository { + private static final String COLUMNS = " " + """ + id, namespace_id, entry_type, title, owner_login, repository_name, repository_id, + source_visibility, + criteria_json::text AS criteria_json, + rule_version, enabled, target_interval_seconds, next_due_at, created_at, updated_at + """; + + private final JdbcTemplate jdbc; + private final ObjectMapper objectMapper; + + public JdbcGithubDiscoveryEntryRepository(JdbcTemplate jdbc, ObjectMapper objectMapper) { + this.jdbc = jdbc; + this.objectMapper = objectMapper; + } + + @Override + public List listPlatformEntries() { + return jdbc.query("SELECT " + COLUMNS + " FROM github_discovery_entry " + + "WHERE namespace_id = ? ORDER BY entry_type, title, id", + (result, row) -> map(result), platformNamespaceId()); + } + + @Override + public List listTeamRepositories(long namespaceId) { + return jdbc.query("SELECT " + COLUMNS + " FROM github_discovery_entry " + + "WHERE namespace_id = ? AND entry_type = 'CURATED_REPOSITORY' " + + "ORDER BY lower(owner_login), lower(repository_name), id", + (result, row) -> map(result), namespaceId); + } + + @Override + public Optional findTeamRepository(long namespaceId, UUID id) { + return jdbc.query("SELECT " + COLUMNS + " FROM github_discovery_entry " + + "WHERE namespace_id = ? AND id = ? AND entry_type = 'CURATED_REPOSITORY'", + result -> result.next() ? Optional.of(map(result)) : Optional.empty(), namespaceId, id); + } + + @Override + public Optional findPlatformEntry(UUID id) { + return jdbc.query("SELECT " + COLUMNS + " FROM github_discovery_entry " + + "WHERE id = ? AND namespace_id = ?", + result -> result.next() ? Optional.of(map(result)) : Optional.empty(), + id, platformNamespaceId()); + } + + @Override + public GithubDiscoveryEntry createSearchRule(String title, String criteriaJson, + int targetIntervalSeconds) { + return returning(""" + INSERT INTO github_discovery_entry + (id, namespace_id, entry_type, title, criteria_json, + enabled, target_interval_seconds) + VALUES (?, ?, 'SEARCH_RULE', ?, ?::jsonb, FALSE, ?) + RETURNING """ + COLUMNS, + UUID.randomUUID(), platformNamespaceId(), title, criteriaJson, targetIntervalSeconds); + } + + @Override + public GithubDiscoveryEntry updateSearchRule(UUID id, String title, String criteriaJson, + int targetIntervalSeconds, boolean enabled) { + return returning(""" + UPDATE github_discovery_entry + SET title = ?, criteria_json = ?::jsonb, target_interval_seconds = ?, + enabled = ?, rule_version = rule_version + 1, + next_due_at = CASE WHEN ? THEN CURRENT_TIMESTAMP ELSE next_due_at END, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND namespace_id = ? AND entry_type = 'SEARCH_RULE' + RETURNING """ + COLUMNS, + title, criteriaJson, targetIntervalSeconds, enabled, enabled, + id, platformNamespaceId()); + } + + @Override + public GithubDiscoveryEntry createCuratedOwner(String ownerLogin, int targetIntervalSeconds) { + return returning(""" + INSERT INTO github_discovery_entry + (id, namespace_id, entry_type, title, owner_login, + enabled, target_interval_seconds) + VALUES (?, ?, 'CURATED_OWNER', ?, ?, TRUE, ?) + RETURNING """ + COLUMNS, + UUID.randomUUID(), platformNamespaceId(), ownerLogin, ownerLogin, + targetIntervalSeconds); + } + + @Override + public GithubDiscoveryEntry createCuratedRepository(String ownerLogin, String repositoryName) { + return returning(""" + INSERT INTO github_discovery_entry + (id, namespace_id, entry_type, title, owner_login, repository_name, + source_visibility, enabled, target_interval_seconds) + VALUES (?, ?, 'CURATED_REPOSITORY', ?, ?, ?, 'PUBLIC', TRUE, + COALESCE((SELECT repository_hours * 3600 FROM github_sync_schedule_policy WHERE id = TRUE FOR SHARE), 86400)) + ON CONFLICT (namespace_id, lower(owner_login), lower(repository_name)) + WHERE entry_type='CURATED_REPOSITORY' + DO UPDATE SET id=github_discovery_entry.id + RETURNING """ + COLUMNS, + UUID.randomUUID(), platformNamespaceId(), repositoryName, + ownerLogin, repositoryName); + } + + @Override + public GithubDiscoveryEntry createTeamRepository(long namespaceId, String ownerLogin, + String repositoryName, String visibility) { + return returning(""" + INSERT INTO github_discovery_entry + (id, namespace_id, entry_type, title, owner_login, repository_name, + source_visibility, enabled, target_interval_seconds) + VALUES (?, ?, 'CURATED_REPOSITORY', ?, ?, ?, ?, TRUE, + COALESCE((SELECT repository_hours * 3600 FROM github_sync_schedule_policy WHERE id = TRUE FOR SHARE), 86400)) + RETURNING """ + COLUMNS, + UUID.randomUUID(), namespaceId, repositoryName, ownerLogin, repositoryName, visibility); + } + + @Override + @Transactional + public GithubDiscoveryEntry setTeamRepositoryEnabled(long namespaceId, UUID id, boolean enabled) { + GithubDiscoveryEntry entry = returning(""" + UPDATE github_discovery_entry + SET enabled = ?, rule_version = rule_version + 1, + next_due_at = CASE WHEN ? THEN CURRENT_TIMESTAMP ELSE next_due_at END, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND namespace_id = ? AND entry_type = 'CURATED_REPOSITORY' + RETURNING """ + COLUMNS, + enabled, enabled, id, namespaceId); + jdbc.update(""" + UPDATE github_source_binding source + SET status = CASE WHEN ? THEN 'ACTIVE' ELSE 'PAUSED' END, + next_check_at = CASE WHEN ? THEN CURRENT_TIMESTAMP ELSE next_check_at END, + updated_at = CURRENT_TIMESTAMP + FROM github_source_origin origin + WHERE origin.source_id = source.id + AND origin.origin_type = 'TEAM_CONFIG' AND origin.origin_key = ? + AND source.namespace_id = ? + AND (source.status = 'PAUSED' OR ? = FALSE) + """, enabled, enabled, id.toString(), namespaceId, enabled); + return entry; + } + + @Override + @Transactional + public GithubDiscoveryEntry setTeamRepositoryVisibility(long namespaceId, UUID id, String visibility) { + GithubDiscoveryEntry entry = returning(""" + UPDATE github_discovery_entry + SET source_visibility = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND namespace_id = ? AND entry_type = 'CURATED_REPOSITORY' + RETURNING """ + COLUMNS, + visibility, id, namespaceId); + jdbc.update(""" + UPDATE github_source_binding source + SET visibility = ?, updated_at = CURRENT_TIMESTAMP + FROM github_source_origin origin + WHERE origin.source_id = source.id + AND origin.origin_type = 'TEAM_CONFIG' AND origin.origin_key = ? + AND source.namespace_id = ? + """, visibility, id.toString(), namespaceId); + return entry; + } + + @Override + public GithubDiscoveryEntry setEnabled(UUID id, boolean enabled) { + return returning(""" + UPDATE github_discovery_entry + SET enabled = ?, rule_version = rule_version + 1, + next_due_at = CASE WHEN ? THEN CURRENT_TIMESTAMP ELSE next_due_at END, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND namespace_id = ? + RETURNING """ + COLUMNS, + enabled, enabled, id, platformNamespaceId()); + } + + @Override + @Transactional + public int enqueueDue(Instant now, int maxEntries) { + if (maxEntries < 1 || maxEntries > 100) { + throw new IllegalArgumentException("Discovery scheduler batch must contain 1 to 100 entries"); + } + List due = jdbc.query(""" + SELECT entry.id, entry.namespace_id, entry.entry_type, entry.title, + entry.owner_login, entry.repository_name, entry.repository_id, + entry.source_visibility, + entry.criteria_json::text AS criteria_json, + entry.rule_version, entry.enabled, entry.target_interval_seconds, + entry.next_due_at, entry.created_at, entry.updated_at + FROM github_discovery_entry entry + WHERE entry.enabled = TRUE AND entry.next_due_at <= ? + AND NOT EXISTS ( + SELECT 1 FROM github_sync_task task + WHERE task.namespace_id = entry.namespace_id + AND task.dedupe_key = 'discovery:' || entry.id::text + AND task.status IN ('QUEUED', 'LEASED', 'RETRY_WAIT')) + ORDER BY entry.next_due_at, entry.id + LIMIT ? FOR UPDATE OF entry SKIP LOCKED + """, (result, row) -> map(result), Timestamp.from(now), maxEntries); + int queued = 0; + for (GithubDiscoveryEntry entry : due) { + Integer budget = "CURATED_REPOSITORY".equals(entry.entryType()) + ? 1 : availableCandidateBudget(entry.namespaceId()); + if (budget == null || budget < 1) { + continue; + } + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + String dedupeKey = "discovery:" + entry.id(); + String snapshot = snapshotJson(entry, dedupeKey, budget, now); + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind, status, created_at) + VALUES (?, ?, 'DISCOVERY', 'QUEUED', ?) + """, runId, entry.namespaceId(), Timestamp.from(now)); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, priority, + snapshot_json, due_at, created_at, updated_at) + VALUES (?, ?, ?, 'DISCOVERY', 'QUEUED', ?, ?, ?::jsonb, ?, ?, ?) + """, taskId, runId, entry.namespaceId(), dedupeKey, + "CURATED_REPOSITORY".equals(entry.entryType()) ? 90 + : "CURATED_OWNER".equals(entry.entryType()) ? 80 : 20, + snapshot, Timestamp.from(now), Timestamp.from(now), Timestamp.from(now)); + jdbc.update(""" + UPDATE github_discovery_entry + SET next_due_at = ?, updated_at = ? WHERE id = ? + """, Timestamp.from(now.plusSeconds(entry.targetIntervalSeconds())), + Timestamp.from(now), entry.id()); + queued++; + } + return queued; + } + + private Integer availableCandidateBudget(long namespaceId) { + return jdbc.query(""" + SELECT least(policy.candidate_per_lease_limit, + policy.candidate_pending_limit - ( + SELECT count(*) FROM github_discovery_candidate candidate + WHERE candidate.namespace_id = ? + AND candidate.status IN ('PENDING', 'VALIDATING'))) AS budget + FROM github_sync_admission_policy policy WHERE policy.id = TRUE + """, result -> result.next() ? result.getInt("budget") : null, namespaceId); + } + + private String snapshotJson(GithubDiscoveryEntry entry, String dedupeKey, + int candidateBudget, Instant now) { + ObjectNode snapshot = objectMapper.createObjectNode(); + snapshot.put("entryType", entry.entryType()); + snapshot.put("ruleId", entry.id().toString()); + snapshot.put("ruleVersion", entry.ruleVersion()); + snapshot.put("candidateBudget", candidateBudget); + if ("CURATED_OWNER".equals(entry.entryType())) { + snapshot.put("owner", entry.ownerLogin()); + } else if ("CURATED_REPOSITORY".equals(entry.entryType())) { + String owner = entry.ownerLogin(); + String name = entry.repositoryName(); + if (entry.repositoryId() != null) { + List current = jdbc.query(""" + SELECT owner_login, repository_name FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, (result, row) -> new RepositoryCoordinate( + result.getString("owner_login"), result.getString("repository_name")), + entry.namespaceId(), entry.repositoryId()); + if (!current.isEmpty()) { + owner = current.getFirst().owner(); + name = current.getFirst().name(); + } + } + snapshot.put("owner", owner); + snapshot.put("repositoryName", name); + } else { + try { + snapshot.set("criteria", objectMapper.readTree(entry.criteriaJson())); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Persisted GitHub search rule is invalid", exception); + } + SuccessfulWindows windows = successfulWindows(entry.namespaceId(), dedupeKey, + entry.ruleVersion()); + ObjectNode window = snapshot.putObject("searchWindow"); + if (windows.lastFull() == null || windows.lastFull().isBefore(now.minus(Duration.ofDays(7))) + || windows.lastSuccess() == null) { + window.put("mode", "FULL_RECHECK"); + } else { + window.put("mode", "INCREMENTAL"); + Instant since = windows.lastSuccess().minus(Duration.ofHours(1)); + window.put("since", since.isAfter(now) ? now.toString() : since.toString()); + } + window.put("until", now.toString()); + } + return snapshot.toString(); + } + + private SuccessfulWindows successfulWindows(long namespaceId, String dedupeKey, long ruleVersion) { + return jdbc.query(""" + SELECT max((snapshot_json #>> '{searchWindow,until}')::timestamptz) AS last_success, + max(CASE WHEN snapshot_json #>> '{searchWindow,mode}' = 'FULL_RECHECK' + THEN (snapshot_json #>> '{searchWindow,until}')::timestamptz END) AS last_full + FROM github_sync_task + WHERE namespace_id = ? AND dedupe_key = ? + AND status = 'ACQUISITION_COMPLETE' + AND (snapshot_json ->> 'ruleVersion')::bigint = ? + """, result -> result.next() + ? new SuccessfulWindows( + result.getTimestamp("last_success") == null ? null + : result.getTimestamp("last_success").toInstant(), + result.getTimestamp("last_full") == null ? null + : result.getTimestamp("last_full").toInstant()) + : new SuccessfulWindows(null, null), namespaceId, dedupeKey, ruleVersion); + } + + private record SuccessfulWindows(Instant lastSuccess, Instant lastFull) {} + + private record RepositoryCoordinate(String owner, String name) {} + + private GithubDiscoveryEntry returning(String sql, Object... args) { + return Objects.requireNonNull(jdbc.query(sql, + result -> result.next() ? map(result) : null, args), + "GitHub discovery entry was not found"); + } + + private long platformNamespaceId() { + Long id = jdbc.queryForObject(""" + SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG' + """, Long.class); + return Objects.requireNonNull(id, "@github catalog namespace is missing"); + } + + private static GithubDiscoveryEntry map(ResultSet result) throws SQLException { + long repositoryId = result.getLong("repository_id"); + Long nullableRepositoryId = result.wasNull() ? null : repositoryId; + return new GithubDiscoveryEntry( + result.getObject("id", UUID.class), result.getLong("namespace_id"), + result.getString("entry_type"), result.getString("title"), + result.getString("owner_login"), result.getString("repository_name"), + nullableRepositoryId, result.getString("source_visibility"), + result.getString("criteria_json"), + result.getLong("rule_version"), result.getBoolean("enabled"), + result.getInt("target_interval_seconds"), result.getTimestamp("next_due_at").toInstant(), + result.getTimestamp("created_at").toInstant(), + result.getTimestamp("updated_at").toInstant()); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryOutcomeRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryOutcomeRepository.java new file mode 100644 index 000000000..842836f70 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubDiscoveryOutcomeRepository.java @@ -0,0 +1,49 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubDiscoveryOutcomeRepository; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +public class JdbcGithubDiscoveryOutcomeRepository implements GithubDiscoveryOutcomeRepository { + private final JdbcTemplate jdbc; + public JdbcGithubDiscoveryOutcomeRepository(JdbcTemplate jdbc) { this.jdbc = jdbc; } + + @Override public void candidateCreated(long namespaceId, long repositoryId, Instant now) { + record(namespaceId, repositoryId, "CANDIDATE_CREATED", repositoryId, now); + } + @Override public void firstValid(long namespaceId, long repositoryId, Instant now) { + record(namespaceId, repositoryId, "FIRST_VALID", repositoryId, now); + } + @Override public void publication(long namespaceId, long repositoryId, long skillId, + long versionId, boolean newSkill, Instant now) { + record(namespaceId, repositoryId, newSkill ? "NEW_SKILL" : "UPDATED_VERSION", + newSkill ? skillId : versionId, now); + } + private void record(long namespaceId, long repositoryId, String type, long subject, Instant now) { + jdbc.update(""" + INSERT INTO github_discovery_outcome_event + (namespace_id,repository_id,event_type,subject_key,occurred_at,official_at_event,query_ids,first_occurrence_known) + SELECT ?,?,?,?,?, cardinality(origins.ids)>0, origins.ids, + NOT (?='FIRST_VALID' AND EXISTS(SELECT 1 FROM github_discovery_legacy_unknown_validation + WHERE namespace_id=? AND repository_id=?)) + FROM (SELECT COALESCE(array_agg(DISTINCT q.id), ARRAY[]::uuid[]) ids + FROM github_discovery_candidate c + JOIN github_discovery_candidate_origin o ON o.candidate_id=c.id + JOIN github_deep_discovery_query q ON q.id::text=o.origin_key + WHERE c.namespace_id=? AND c.repository_id=? AND o.origin_type='OFFICIAL_DISCOVERY') origins + ON CONFLICT (namespace_id,event_type,subject_key) DO NOTHING + """, namespaceId, repositoryId, type, subject, Timestamp.from(now), type, namespaceId, repositoryId, namespaceId, repositoryId); + } + @Override public void provenance(long candidateId, UUID queryId, String strategyVersion, + UUID shardId, UUID runId, Long seedRepositoryId) { + jdbc.update(""" + UPDATE github_discovery_candidate_origin SET recent_shard_id=?,recent_run_id=?, + recent_seed_repository_id=?,latest_strategy_version=? + WHERE candidate_id=? AND origin_type='OFFICIAL_DISCOVERY' AND origin_key=? + """, shardId, runId, seedRepositoryId, strategyVersion, candidateId, queryId.toString()); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryBudgetRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryBudgetRepository.java new file mode 100644 index 000000000..25dfcf796 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryBudgetRepository.java @@ -0,0 +1,55 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryBudgetRepository; +import java.util.UUID; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +public class JdbcGithubOfficialDiscoveryBudgetRepository implements GithubOfficialDiscoveryBudgetRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubOfficialDiscoveryBudgetRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public void reserveSuccessfulClaim(UUID taskId, long generation) { + // Claim already holds config -> budget -> task -> credential locks in this transaction. + // The reservation key makes retries idempotent; task retention may clean the audit row without resetting the global pacing timestamp. + boolean official=Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS(SELECT 1 FROM github_sync_task WHERE id=? AND official_query_id IS NOT NULL)",Boolean.class,taskId)); + if (!official) return; + int reserved=jdbc.update(""" + WITH lease_grant AS MATERIALIZED ( + SELECT task.id, task.lease_generation, task.official_query_id, + config.budget_profile, + (task.snapshot_json #>> '{officialDiscovery,budget,maxRequests}')::integer AS requests, + clock_timestamp() AS granted_at + FROM github_sync_task task + JOIN github_deep_discovery_config config ON config.namespace_id=task.namespace_id + WHERE task.id=? AND task.lease_generation=? AND task.status='LEASED' + AND task.kind='DISCOVERY' AND task.official_query_id IS NOT NULL + AND task.lease_expires_at > clock_timestamp() + ), reservation AS ( + INSERT INTO github_official_discovery_reservation + (task_id,lease_generation,budget_profile,reserved_requests,granted_at,next_lease_eligible_at) + SELECT id,lease_generation,budget_profile,requests,granted_at, + granted_at + make_interval(secs => requests * 3600.0 / CASE budget_profile + WHEN 'ECONOMY' THEN 60 WHEN 'BALANCED' THEN 180 ELSE 360 END) + FROM lease_grant + ON CONFLICT DO NOTHING + RETURNING granted_at,next_lease_eligible_at + ), paced AS ( + UPDATE github_official_discovery_budget budget + SET next_lease_eligible_at=reservation.next_lease_eligible_at + FROM reservation WHERE budget.id + ) + UPDATE github_deep_discovery_query query + SET last_successful_claim_at=reservation.granted_at + FROM lease_grant,reservation WHERE query.id=lease_grant.official_query_id + """, taskId, generation); + if (reserved == 0 && !Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS(SELECT 1 FROM github_official_discovery_reservation WHERE task_id=? AND lease_generation=?)",Boolean.class,taskId,generation))) { + throw new IllegalStateException("Official discovery lease expired before budget reservation"); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryRepository.java new file mode 100644 index 000000000..c4b5e4049 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialDiscoveryRepository.java @@ -0,0 +1,267 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryCatalog; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** The configuration lock serializes catalog reconciliation, scheduling and administrator changes. */ +@Repository +public class JdbcGithubOfficialDiscoveryRepository implements GithubOfficialDiscoveryRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + private final GithubOfficialDiscoveryCatalog catalog; + private final JdbcGithubOfficialShardRepository shards; + private final JdbcGithubOfficialOwnerRepository owners; + + public JdbcGithubOfficialDiscoveryRepository(JdbcTemplate jdbc, ObjectMapper mapper, + GithubOfficialDiscoveryCatalog catalog, JdbcGithubOfficialShardRepository shards, JdbcGithubOfficialOwnerRepository owners) { + this.jdbc = jdbc; + this.mapper = mapper; + this.catalog = catalog; + this.shards = shards; + this.owners = owners; + } + + @Override + @Transactional + public int enqueueDue(Instant now, int maxQueries) { + if (maxQueries < 1 || maxQueries > 100) { + throw new IllegalArgumentException("Official discovery scheduler batch must contain 1 to 100 queries"); + } + Configuration configuration = jdbc.query(""" + SELECT namespace_id, enabled, config_version, budget_profile, strategy_generation, strategy_version + FROM github_deep_discovery_config WHERE id = TRUE FOR UPDATE + """, result -> { + if (!result.next()) { + throw new IllegalStateException("GitHub deep discovery configuration is missing"); + } + return new Configuration(result.getLong("namespace_id"), result.getBoolean("enabled"), + result.getLong("config_version"), result.getString("budget_profile"), + result.getLong("strategy_generation"), result.getString("strategy_version")); + }); + if (configuration.strategyGeneration() > GithubOfficialDiscoveryCatalog.GENERATION) { + return 0; + } + if (configuration.strategyGeneration() == GithubOfficialDiscoveryCatalog.GENERATION + && !GithubOfficialDiscoveryCatalog.VERSION.equals(configuration.strategyVersion())) { + throw new IllegalStateException("Official strategy changes require a new catalog generation"); + } + reconcile(configuration.namespaceId(), now); + if (!configuration.enabled()) { + return 0; + } + owners.reconcile(configuration.namespaceId(), now); + Integer available = jdbc.query(""" + SELECT least(candidate_per_lease_limit, candidate_pending_limit - ( + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND status IN ('PENDING', 'VALIDATING'))) AS budget + FROM github_sync_admission_policy WHERE id = TRUE + """, result -> result.next() ? result.getInt("budget") : null, configuration.namespaceId()); + if (available == null || available < 1) { + return 0; + } + shards.replayFailed(configuration.namespaceId()); + // A terminal acquisition failure is a replay boundary, not successful coverage. + jdbc.update(""" + UPDATE github_deep_discovery_query query SET active_task_id = NULL, checkpoint = NULL, + next_due_at = GREATEST(query.next_due_at, task.updated_at + INTERVAL '1 hour'), + wait_reason = 'FAILED_REPLAY' + FROM github_sync_task task WHERE query.active_task_id = task.id AND task.status = 'FAILED' + AND query.namespace_id = ? + """, configuration.namespaceId()); + List due = jdbc.query(""" + SELECT id, definition_json::text, covered_until, github_owner_id, owner_login, seed_repository_id + FROM github_deep_discovery_query + WHERE namespace_id = ? AND (retired_at IS NULL OR EXISTS ( + SELECT 1 FROM github_official_discovery_shard shard + WHERE shard.query_id = github_deep_discovery_query.id AND shard.state IN ('PENDING', 'GAP'))) + AND NOT EXISTS (SELECT 1 FROM github_official_owner_eligibility owner + WHERE owner.query_id = github_deep_discovery_query.id AND owner.excluded) + AND next_due_at <= ? AND active_task_id IS NULL + ORDER BY next_due_at, query_key LIMIT ? FOR UPDATE SKIP LOCKED + """, (result, row) -> new QueryProgress(result.getObject("id", UUID.class), + definition(result.getString("definition_json")), + result.getTimestamp("covered_until") == null ? null : result.getTimestamp("covered_until").toInstant(), + (Long) result.getObject("github_owner_id"), result.getString("owner_login"), + (Long) result.getObject("seed_repository_id")), configuration.namespaceId(), Timestamp.from(now), maxQueries); + Instant cutoff = now.truncatedTo(ChronoUnit.SECONDS); + int queued = 0; + for (QueryProgress progress : due) { + Instant since = progress.coveredUntil() == null || "HISTORY_RECHECK".equals(progress.definition().cadence()) ? Instant.EPOCH + : progress.coveredUntil().minus(Duration.ofHours(1)).truncatedTo(ChronoUnit.SECONDS); + if (since.isBefore(Instant.EPOCH)) { + since = Instant.EPOCH; + } + if (!since.isBefore(cutoff)) { + continue; + } + UUID proposedTaskId = UUID.randomUUID(); + UUID runId = UUID.randomUUID(); + ObjectNode freshSnapshot = snapshot(configuration, progress, proposedTaskId, since, cutoff, available); + boolean owner = progress.ownerId() != null; + var work = owner ? new JdbcGithubOfficialShardRepository.Work(null, proposedTaskId, false, freshSnapshot, null, 0) + : shards.next(progress.id(), proposedTaskId, freshSnapshot, available, now); + List capabilities = owner ? GithubOfficialDiscoveryCatalog.OWNER_REQUIRED_CAPABILITIES + : GithubOfficialDiscoveryCatalog.REQUIRED_CAPABILITIES; + UUID taskId = work.taskId(); + ObjectNode snapshot = work.snapshot(); + if (work.reuse()) { + int locked = jdbc.queryForObject("SELECT count(*) FROM (SELECT id FROM github_sync_task WHERE id = ? AND status = 'PARTIAL' FOR UPDATE) task", Integer.class, taskId); + if (locked != 1) { + throw new IllegalStateException("GAP replay requires a terminal PARTIAL task"); + } + jdbc.update("DELETE FROM github_discovery_batch WHERE task_id = ?", taskId); + jdbc.update("DELETE FROM github_discovery_task_hit WHERE task_id = ?", taskId); + jdbc.update(""" + UPDATE github_sync_task SET status = 'QUEUED', snapshot_json = ?::jsonb, + due_at = ?, retry_at = NULL, completed_at = NULL, lease_token_hash = NULL, + lease_expires_at = NULL, leased_by = NULL, official_pause_reason = NULL, updated_at = ? + WHERE id = ? + """, snapshot.toString(), Timestamp.from(now), Timestamp.from(now), taskId); + jdbc.update(""" + UPDATE github_sync_run SET status = 'QUEUED', finished_at = NULL + WHERE id = (SELECT run_id FROM github_sync_task WHERE id = ?) + """, taskId); + } else { + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind, status, created_at) + VALUES (?, ?, 'DISCOVERY', 'QUEUED', ?) + """, runId, configuration.namespaceId(), Timestamp.from(now)); + jdbc.update(connection -> { + var statement = connection.prepareStatement(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, priority, snapshot_json, + due_at, created_at, updated_at, required_capabilities, official_query_id) + VALUES (?, ?, ?, 'DISCOVERY', 'QUEUED', ?, 10, ?::jsonb, ?, ?, ?, ?, ?) + """); + statement.setObject(1, taskId); + statement.setObject(2, runId); + statement.setLong(3, configuration.namespaceId()); + statement.setString(4, "official-discovery:" + progress.id()); + statement.setString(5, snapshot.toString()); + statement.setTimestamp(6, Timestamp.from(now)); + statement.setTimestamp(7, Timestamp.from(now)); + statement.setTimestamp(8, Timestamp.from(now)); + statement.setArray(9, connection.createArrayOf("text", capabilities.toArray())); + statement.setObject(10, progress.id()); + return statement; + }); + jdbc.update("UPDATE github_sync_task SET lease_generation = ? WHERE id = ?", work.generation(), taskId); + } + jdbc.update("UPDATE github_sync_task SET discovery_checkpoint = ? WHERE id = ?", work.checkpoint(), taskId); + if (!owner) { + shards.assigned(work); + } + jdbc.update(""" + UPDATE github_deep_discovery_query + SET active_task_id = ?, checkpoint = ?, wait_reason = 'AWAITING_RESULT' WHERE id = ? + """, taskId, work.checkpoint(), progress.id()); + queued++; + } + return queued; + } + + private void reconcile(long namespaceId, Instant now) { + List current = new ArrayList<>(); + for (var definition : catalog.definitions()) { + String json = mapper.valueToTree(definition).toString(); + jdbc.update(""" + INSERT INTO github_deep_discovery_query + (id, namespace_id, query_key, query_version, strategy_version, definition_json, next_due_at) + VALUES (?, ?, ?, ?, ?, ?::jsonb, ?) + ON CONFLICT (namespace_id, query_key, query_version) DO NOTHING + """, UUID.randomUUID(), namespaceId, definition.key(), definition.version(), + GithubOfficialDiscoveryCatalog.VERSION, json, Timestamp.from(now)); + UUID id = jdbc.query(""" + SELECT id, definition_json::text FROM github_deep_discovery_query + WHERE namespace_id = ? AND query_key = ? AND query_version = ? + """, result -> { + if (!result.next()) { + throw new IllegalStateException("Official query is missing"); + } + var previous = definition(result.getString("definition_json")); + if (!previous.channel().equals(definition.channel()) || !previous.query().equals(definition.query())) { + throw new IllegalStateException("Official query changes require a new query version"); + } + return result.getObject("id", UUID.class); + }, namespaceId, definition.key(), definition.version()); + current.add(id); + jdbc.update(""" + UPDATE github_deep_discovery_query + SET strategy_version = ?, definition_json = ?::jsonb, retired_at = NULL WHERE id = ? + AND (strategy_version IS DISTINCT FROM ? OR definition_json <> ?::jsonb OR retired_at IS NOT NULL) + """, GithubOfficialDiscoveryCatalog.VERSION, json, id, GithubOfficialDiscoveryCatalog.VERSION, json); + } + jdbc.update(connection -> { + var statement = connection.prepareStatement(""" + UPDATE github_deep_discovery_query SET retired_at = ? + WHERE namespace_id = ? AND query_source = 'CATALOG' AND retired_at IS NULL AND NOT (id = ANY (?)) + """); + statement.setTimestamp(1, Timestamp.from(now)); + statement.setLong(2, namespaceId); + statement.setArray(3, connection.createArrayOf("uuid", current.toArray())); + return statement; + }); + jdbc.update("UPDATE github_deep_discovery_config SET strategy_version = ?, strategy_generation = ? " + + "WHERE id = TRUE AND strategy_generation < ?", GithubOfficialDiscoveryCatalog.VERSION, + GithubOfficialDiscoveryCatalog.GENERATION, GithubOfficialDiscoveryCatalog.GENERATION); + } + + private ObjectNode snapshot(Configuration configuration, QueryProgress progress, UUID taskId, + Instant since, Instant cutoff, int candidateBudget) { + ObjectNode root = mapper.createObjectNode(); + ObjectNode official = root.putObject("officialDiscovery"); + official.put("formatVersion", 1); + official.put("configurationVersion", configuration.version()); + official.put("strategyVersion", GithubOfficialDiscoveryCatalog.VERSION); + official.put("queryId", progress.id().toString()); + official.put("queryVersion", progress.definition().version()); + official.put("shardId", taskId.toString()); + official.put("channel", progress.definition().channel()); + official.put("mode", since.equals(Instant.EPOCH) ? "HISTORICAL" : "INCREMENTAL"); + official.set("query", mapper.valueToTree(progress.definition().query())); + official.putObject("range").put("since", since.toString()).put("until", cutoff.toString()); + official.put("fixedCutoffAt", cutoff.toString()); + // Search page request bounds only; not tree/owner work or calibrated profile allocations. + official.putObject("budget").put("profile", configuration.profile()) + .put("maxRequests", 10).put("maxPages", 10).put("candidateBudget", candidateBudget) + .put("maxSplitDepth", JdbcGithubOfficialShardRepository.MAX_DEPTH) + .put("maxChildShards", JdbcGithubOfficialShardRepository.MAX_CHILDREN); + if (progress.ownerId() != null) { + official.put("mode", "OWNER_RECHECK"); + official.put("ownerId", progress.ownerId()); + official.put("ownerLogin", progress.ownerLogin()); + ((ObjectNode) official.path("query")).put("value", progress.ownerLogin()); + official.put("seedRepositoryId", progress.seedRepositoryId()); + official.put("expansionDepth", 1); + ((ObjectNode) official.path("budget")).put("maxSplitDepth", 0).put("maxChildShards", 0); + } + return root; + } + + private GithubOfficialDiscoveryCatalog.Definition definition(String json) { + try { + return mapper.readValue(json, GithubOfficialDiscoveryCatalog.Definition.class); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted official query definition", exception); + } + } + + private record Configuration(long namespaceId, boolean enabled, long version, String profile, + long strategyGeneration, String strategyVersion) {} + private record QueryProgress(UUID id, GithubOfficialDiscoveryCatalog.Definition definition, + Instant coveredUntil, Long ownerId, String ownerLogin, Long seedRepositoryId) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialLifecycleRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialLifecycleRepository.java new file mode 100644 index 000000000..fd00b21cc --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialLifecycleRepository.java @@ -0,0 +1,573 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCandidate; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubOfficialLifecycleRepository; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import com.iflytek.skillhub.githubsync.GithubRepositoryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubSyncDeferralReason; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.HashSet; +import java.util.Locale; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +/** Official page acceptance and durable progress share the lease transaction. */ +@Repository +@Transactional +public class JdbcGithubOfficialLifecycleRepository implements GithubOfficialLifecycleRepository { + private final JdbcTemplate jdbc; + private final JdbcGithubDiscoveryOutcomeRepository outcomes; + private final ObjectMapper mapper; + private final GithubSyncLeaseService leases; + private final JdbcGithubSyncCapacityGate capacity; + private final JdbcGithubOfficialShardRepository shards; + private final JdbcGithubOfficialQuotaRepository quota; + + public JdbcGithubOfficialLifecycleRepository(JdbcTemplate jdbc, ObjectMapper mapper, + GithubSyncLeaseService leases, JdbcGithubSyncCapacityGate capacity, + JdbcGithubOfficialShardRepository shards) { + this.jdbc = jdbc; + this.outcomes = new JdbcGithubDiscoveryOutcomeRepository(jdbc); + this.mapper = mapper; + this.leases = leases; + this.capacity = capacity; + this.shards = shards; + this.quota = new JdbcGithubOfficialQuotaRepository(jdbc); + } + + @Override + public boolean isOfficialDiscovery(UUID taskId) { + return Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS (SELECT 1 FROM github_sync_task " + + "WHERE id = ? AND official_query_id IS NOT NULL)", Boolean.class, taskId)); + } + + @Override + public void report(UUID taskId, long generation, String token, GithubDiscoveryBatch batch, + GithubOfficialDiscoveryBatch metadata) { + Configuration config = configuration(); + Lease lease = lease(taskId, generation, token); + JsonNode checkpoint = checkpoint(batch.checkpoint(), lease.snapshot()); + validateMetadata(metadata, checkpoint, lease); + List reportedRepositories = batch.repositories().stream() + .sorted(Comparator.comparingLong(GithubDiscoveryCandidate::repositoryId)).toList(); + for (var candidate : reportedRepositories) { + validateProvenance(candidate, lease); + } + String digest = hash(json(List.of(batch.previousCheckpoint() == null ? "" : batch.previousCheckpoint(), + batch.checkpoint(), reportedRepositories, batch.deferredCount(), + batch.deferralReason() == null ? "" : batch.deferralReason().name(), metadata))); + String prior = jdbc.query("SELECT request_sha256 FROM github_discovery_batch WHERE task_id = ? AND checkpoint = ?", + result -> result.next() ? result.getString(1) : null, taskId, batch.checkpoint()); + if (prior != null) { + require(prior.equals(digest), 409, "RESULT_CONFLICT"); + return; + } + if (metadata.ownerLogin() != null) { + jdbc.update("UPDATE github_deep_discovery_query SET owner_login = ? WHERE id = ? AND query_source = 'OWNER'", + metadata.ownerLogin(), lease.queryId()); + } + boolean ownerBlocked = ownerExcluded(lease); + boolean controlAllowed = config.enabled() && !ownerBlocked; + if (!controlAllowed) { + boolean alreadyAccepted = Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_discovery_batch WHERE task_id = ? + AND official_metadata ->> 'generation' = ? + AND official_metadata ->> 'configurationVersion' = ? + AND COALESCE(official_metadata ->> 'controlAllowed', official_metadata ->> 'configurationEnabled') = 'false') + """, Boolean.class, taskId, Long.toString(generation), Long.toString(config.version()))); + require(!alreadyAccepted, 409, "BACKPRESSURE_ACTIVE"); + } + require(Objects.equals(lease.checkpoint(), batch.previousCheckpoint()), 409, "CHECKPOINT_CONFLICT"); + // A completed shard cannot be turned back into an unfinished page by a later report. + if (lease.checkpoint() != null) { + require(!List.of("COMPLETE", "SPLIT").contains(checkpoint(lease.checkpoint(), lease.snapshot()).path("coverage").asText()), + 409, "CHECKPOINT_CONFLICT"); + } + int requests = jdbc.queryForObject(""" + SELECT COALESCE(sum((official_metadata ->> 'requestsUsed')::integer), 0) + FROM github_discovery_batch WHERE task_id = ? AND official_metadata ->> 'generation' = ? + """, Integer.class, taskId, Long.toString(generation)); + require(requests + metadata.requestsUsed() <= lease.snapshot().path("budget").path("maxRequests").asInt(), + 422, "CANDIDATE_BUDGET_EXCEEDED"); + var reportedIds = new HashSet<>(jdbc.query(""" + SELECT value::bigint AS repository_id + FROM github_discovery_batch batch, + jsonb_array_elements_text(COALESCE(batch.official_metadata -> 'reportedRepositoryIds', '[]'::jsonb)) + WHERE task_id = ? AND official_metadata ->> 'generation' = ? + UNION SELECT repository_id FROM github_discovery_task_hit WHERE task_id = ? AND lease_generation = ? + """, (result, row) -> result.getLong("repository_id"), taskId, Long.toString(generation), taskId, generation)); + reportedRepositories.forEach(candidate -> reportedIds.add(candidate.repositoryId())); + require(reportedIds.size() <= lease.snapshot().path("budget").path("candidateBudget").asInt(), + 422, "CANDIDATE_BUDGET_EXCEEDED"); + // Advance the page for a valid response while excluding new official discovery at current receipt policy. + List repositories = reportedRepositories.stream() + .filter(candidate -> !ownerBlocked && !excluded(candidate, config.exclusions())).toList(); + // Lock namespace before candidates, matching the legacy discovery acceptance order. + jdbc.queryForObject("SELECT id FROM namespace WHERE id = ? FOR UPDATE", Long.class, lease.namespaceId()); + int pendingLimit = jdbc.queryForObject("SELECT candidate_pending_limit FROM github_sync_admission_policy WHERE id = TRUE", Integer.class); + long pending = jdbc.queryForObject("SELECT count(*) FROM github_discovery_candidate WHERE namespace_id = ? " + + "AND status IN ('PENDING','VALIDATING')", Long.class, lease.namespaceId()); + long hits = jdbc.queryForObject("SELECT count(*) FROM github_discovery_task_hit WHERE task_id = ? AND lease_generation = ?", + Long.class, taskId, generation); + int newHits = 0; + int newPending = 0; + for (var candidate : repositories) { + if (!Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS (SELECT 1 FROM github_discovery_task_hit " + + "WHERE task_id = ? AND repository_id = ? AND lease_generation = ?)", Boolean.class, taskId, candidate.repositoryId(), generation))) { + newHits++; + } + if (!Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_discovery_candidate WHERE namespace_id = ? AND repository_id = ?) + OR EXISTS (SELECT 1 FROM github_source_binding WHERE namespace_id = ? AND repository_id = ?) + """, Boolean.class, lease.namespaceId(), candidate.repositoryId(), lease.namespaceId(), candidate.repositoryId()))) { + newPending++; + } + } + require(hits + newHits <= lease.snapshot().path("budget").path("candidateBudget").asInt() + && (newPending == 0 || pending + newPending <= pendingLimit), + 422, "CANDIDATE_BUDGET_EXCEEDED"); + Instant now = Instant.now(); + for (var candidate : reportedRepositories) { + if (excluded(candidate, config.exclusions())) { + // Keep an existing lead's current coordinates usable by live exclusion checks, without + // creating a candidate or granting it a new official origin. + jdbc.update(""" + UPDATE github_discovery_candidate SET owner_login = ?, repository_name = ?, default_branch = ?, + last_seen_at = ? WHERE namespace_id = ? AND repository_id = ? + """, candidate.owner(), candidate.name(), candidate.defaultBranch(), Timestamp.from(now), + lease.namespaceId(), candidate.repositoryId()); + } + } + for (var candidate : repositories) { + Long id = jdbc.query(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, default_branch, first_seen_at, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (namespace_id, repository_id) DO NOTHING RETURNING id + """, result -> result.next() ? result.getLong("id") : null, + lease.namespaceId(), candidate.repositoryId(), candidate.owner(), candidate.name(), + candidate.defaultBranch(), Timestamp.from(now), Timestamp.from(now)); + boolean created = id != null; + if (!created) { + id = jdbc.queryForObject(""" + UPDATE github_discovery_candidate SET owner_login = ?, repository_name = ?, default_branch = ?, + last_seen_at = ? WHERE namespace_id = ? AND repository_id = ? RETURNING id + """, Long.class, candidate.owner(), candidate.name(), candidate.defaultBranch(), Timestamp.from(now), + lease.namespaceId(), candidate.repositoryId()); + } + jdbc.update(""" + INSERT INTO github_discovery_task_hit (task_id, repository_id, candidate_id, lease_generation, created_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT (task_id, repository_id) DO UPDATE SET lease_generation = EXCLUDED.lease_generation + """, taskId, candidate.repositoryId(), id, generation, Timestamp.from(now)); + jdbc.update(""" + INSERT INTO github_discovery_candidate_origin + (candidate_id, origin_type, origin_key, first_seen_at, last_seen_at) + VALUES (?, 'OFFICIAL_DISCOVERY', ?, ?, ?) + ON CONFLICT (candidate_id, origin_type, origin_key) DO UPDATE SET last_seen_at = EXCLUDED.last_seen_at + """, id, lease.queryId().toString(), Timestamp.from(now), Timestamp.from(now)); + outcomes.provenance(id, lease.queryId(), lease.snapshot().path("strategyVersion").asText(), + taskId, lease.runId(), owner(lease.snapshot()) ? lease.snapshot().path("seedRepositoryId").asLong() : null); + if (created) { + outcomes.candidateCreated(lease.namespaceId(), candidate.repositoryId(), now); + } + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + SELECT id, 'OFFICIAL_DISCOVERY', ? FROM github_source_binding WHERE namespace_id = ? AND repository_id = ? + ON CONFLICT (source_id, origin_type, origin_key) DO NOTHING + """, lease.queryId().toString(), lease.namespaceId(), candidate.repositoryId()); + jdbc.update(""" + UPDATE github_discovery_candidate SET status = 'BOUND', last_error_code = NULL WHERE id = ? + AND EXISTS (SELECT 1 FROM github_source_binding WHERE namespace_id = ? AND repository_id = ?) + """, id, lease.namespaceId(), candidate.repositoryId()); + } + if ("SPLIT".equals(metadata.coverage())) { + shards.split(taskId, lease.queryId(), lease.snapshot(), metadata.childRanges()); + } + ObjectNode persistedMetadata = mapper.valueToTree(metadata); + persistedMetadata.put("generation", generation); + persistedMetadata.put("configurationVersion", config.version()); + persistedMetadata.put("configurationEnabled", config.enabled()); + persistedMetadata.put("controlAllowed", controlAllowed); + persistedMetadata.put("excludedCount", reportedRepositories.size() - repositories.size()); + var reportedIdsJson = persistedMetadata.putArray("reportedRepositoryIds"); + reportedRepositories.forEach(candidate -> reportedIdsJson.add(candidate.repositoryId())); + jdbc.update(""" + INSERT INTO github_discovery_batch (task_id, checkpoint, request_sha256, deferred_count, + deferral_reason, official_metadata, created_at) VALUES (?, ?, ?, ?, ?, ?::jsonb, ?) + """, taskId, batch.checkpoint(), digest, batch.deferredCount(), + batch.deferralReason() == null ? null : batch.deferralReason().name(), persistedMetadata.toString(), Timestamp.from(now)); + jdbc.update("UPDATE github_sync_task SET discovery_checkpoint = ?, last_progress_at = ?, updated_at = ? WHERE id = ?", + batch.checkpoint(), Timestamp.from(now), Timestamp.from(now), taskId); + jdbc.update("UPDATE github_deep_discovery_query SET checkpoint = ?, last_progress_at = ?, wait_reason = ?, gap_reason = COALESCE(?, gap_reason) WHERE id = ?", + batch.checkpoint(), Timestamp.from(now), config.enabled() ? ("GAP".equals(metadata.coverage()) ? metadata.gapReason() : metadata.coverage()) : "DISABLED", metadata.gapReason(), lease.queryId()); + } + + @Override + public Control control(UUID taskId, long generation, String token) { + Configuration config = configuration(); + if (isOfficialDiscovery(taskId)) { + Lease discovery = lease(taskId, generation, token); + String reason = !config.enabled() ? "DISABLED" : ownerExcluded(discovery) ? "EXCLUDED" : null; + Control cooldown = quotaControl(taskId, config); + return reason == null && cooldown != null ? cooldown + : new Control(reason == null ? "CONTINUE" : "PAUSE", config.version(), reason, null); + } + RepositoryLease repository = repositoryLease(taskId, generation, token); + return repositoryControl(taskId, repository, config); + } + + @Override + public void pauseDiscovery(UUID taskId, long generation, String token, String checkpoint) { + Configuration config = configuration(); + Lease lease = lease(taskId, generation, token); + String reason = !config.enabled() ? "DISABLED" : ownerExcluded(lease) ? "EXCLUDED" : null; + Control cooldown = quotaControl(taskId, config); + if (reason == null && cooldown != null) { + reason = cooldown.reason(); + } + require(reason != null, 409, "CONTROL_CHANGED"); + require(Objects.equals(checkpoint, lease.checkpoint()), 409, "CHECKPOINT_CONFLICT"); + boolean deferred = "QUOTA_COOLDOWN".equals(reason); + release(taskId, deferred ? "RETRY_WAIT" : "QUEUED", deferred ? cooldown.retryAt() : Instant.now(), reason); + jdbc.update("UPDATE github_deep_discovery_query SET wait_reason = ? WHERE id = ?", reason, lease.queryId()); + } + + @Override + public void pauseRepository(UUID taskId, long generation, String token, RepositoryProgress progress) { + Configuration config = configuration(); + // Same order as package admission; no new admission can slip into a paused lease. + jdbc.queryForObject("SELECT id FROM github_sync_admission_policy WHERE id = TRUE FOR UPDATE", Boolean.class); + RepositoryLease repository = repositoryLease(taskId, generation, token); + Control control = repositoryControl(taskId, repository, config); + require("PAUSE".equals(control.action()), 409, "CONTROL_CHANGED"); + require(progress != null && progress.incompleteSubtrees() != null && progress.pathResults() != null, + 422, "RESULT_CONFLICT"); + if (progress.observedCommitSha() == null) { + require(progress.pathCheckpoint() == null && progress.incompleteSubtrees().isEmpty() + && progress.pathResults().isEmpty() && repository.fixedCommit() == null + && repository.snapshotCommit() == null && repository.pathCheckpoint() == null, + 409, "COMMIT_MISMATCH"); + boolean started = Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_sync_package_admission WHERE task_id = ?) + OR EXISTS (SELECT 1 FROM github_sync_package_ingestion WHERE task_id = ?) + OR EXISTS (SELECT 1 FROM github_sync_path_result WHERE task_id = ?) + """, Boolean.class, taskId, taskId, taskId)); + require(!started, 409, "COMMIT_MISMATCH"); + } else { + // Reuse established commit/path/ingestion validation. This internal future retry is replaced + // by the paused state in the same transaction; old complete API semantics are unchanged. + GithubRepositoryCompletionResult result = leases.completeRepository(taskId, generation, token, + progress.observedCommitSha(), false, progress.pathCheckpoint(), progress.incompleteSubtrees(), + progress.pathResults(), Instant.now().plusSeconds(60), GithubSyncDeferralReason.CAPACITY); + require(result == GithubRepositoryCompletionResult.RETRY_WAIT, 409, result.name()); + } + boolean deferred = "QUOTA_COOLDOWN".equals(control.reason()); + release(taskId, deferred ? "RETRY_WAIT" : "QUEUED", deferred ? control.retryAt() : Instant.now(), control.reason()); + } + + private RepositoryLease repositoryLease(UUID taskId, long generation, String token) { + require(token != null && !token.isBlank() && generation > 0, 409, "LEASE_STALE"); + RepositoryLease repository = jdbc.query(""" + SELECT namespace_id, source_id, (snapshot_json ->> 'repositoryId')::bigint AS repository_id, + fixed_commit_sha, snapshot_json ->> 'fixedCommitSha' AS snapshot_commit, path_checkpoint + FROM github_sync_task WHERE id = ? AND kind = 'REPOSITORY_SYNC' AND mode = 'FULL' + AND status = 'LEASED' AND lease_generation = ? AND lease_token_hash = ? AND lease_expires_at > now() + AND required_capabilities @> ARRAY['official-pause/v1'] FOR UPDATE + """, result -> result.next() ? new RepositoryLease(result.getLong("namespace_id"), + (Long) result.getObject("source_id"), result.getLong("repository_id"), result.getString("fixed_commit_sha"), + result.getString("snapshot_commit"), result.getString("path_checkpoint")) : null, + taskId, generation, hash(token)); + require(repository != null, 409, "LEASE_STALE"); + return repository; + } + + private Control repositoryControl(UUID taskId, RepositoryLease repository, Configuration config) { + // Include a source created after this candidate task was leased. + String sourceStatus = jdbc.query(""" + SELECT status FROM github_source_binding WHERE namespace_id = ? AND repository_id = ? FOR SHARE + """, result -> result.next() ? result.getString(1) : null, repository.namespaceId(), repository.repositoryId()); + if ("PAUSED".equals(sourceStatus)) { + return new Control("PAUSE", config.version(), "SOURCE_PAUSED", null); + } + Control cooldown = quotaControl(taskId, config); + if (cooldown != null) { + return cooldown; + } + if (sourceStatus != null) { + return new Control("CONTINUE", config.version(), null, null); + } + Eligibility eligibility = jdbc.query(""" + SELECT has_official, has_independent, configuration_enabled, excluded + FROM github_candidate_discovery_eligibility WHERE namespace_id = ? AND repository_id = ? + """, result -> result.next() ? new Eligibility(result.getBoolean(1), result.getBoolean(2), + result.getBoolean(3), result.getBoolean(4)) : null, repository.namespaceId(), repository.repositoryId()); + require(eligibility != null && eligibility.official(), 409, "CONTROL_CHANGED"); + if (eligibility.independent()) { + return new Control("CONTINUE", config.version(), null, null); + } + String reason = !eligibility.enabled() ? "DISABLED" : eligibility.excluded() ? "EXCLUDED" + : !capacity.allowsFullAcquisition(repository.namespaceId(), Instant.now()) ? "REVIEW_BACKPRESSURE" : null; + return new Control(reason == null ? "CONTINUE" : "PAUSE", config.version(), reason, null); + } + + @Override + public UUID recover(UUID taskId, long generation, String token, String reason, String rejectedCheckpoint) { + configuration(); + Lease lease = lease(taskId, generation, token); + require(List.of("MALFORMED", "FORMAT_UNSUPPORTED", "IDENTITY_MISMATCH").contains(reason == null ? "" : reason) + && rejectedCheckpoint != null, 422, "RESULT_CONFLICT"); + require(Objects.equals(rejectedCheckpoint, lease.checkpoint()), 409, "CHECKPOINT_CONFLICT"); + UUID replacement = UUID.randomUUID(); + UUID replacementRun = UUID.randomUUID(); + ObjectNode snapshot = lease.snapshot().deepCopy(); + snapshot.put("shardId", replacement.toString()); + snapshot.remove("checkpoint"); + ObjectNode wrapper = mapper.createObjectNode(); + wrapper.set("officialDiscovery", snapshot); + // Release the partial unique open-work key before inserting the replacement in this transaction. + release(taskId, "FAILED", Instant.now(), "CHECKPOINT_RECOVERY"); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind, status) VALUES (?, ?, 'DISCOVERY', 'QUEUED')", + replacementRun, lease.namespaceId()); + jdbc.update(""" + INSERT INTO github_sync_task (id, run_id, namespace_id, kind, status, dedupe_key, priority, + snapshot_json, due_at, created_at, updated_at, required_capabilities, official_query_id) + SELECT ?, ?, namespace_id, kind, 'QUEUED', dedupe_key, priority, ?::jsonb, + now(), now(), now(), required_capabilities, official_query_id FROM github_sync_task WHERE id = ? + """, replacement, replacementRun, wrapper.toString(), taskId); + if (!owner(lease.snapshot())) { + shards.recover(taskId, replacement, lease.queryId(), lease.snapshot()); + } + jdbc.update("UPDATE github_deep_discovery_query SET active_task_id = ?, checkpoint = NULL, wait_reason = 'CHECKPOINT_RECOVERY' WHERE id = ?", + replacement, lease.queryId()); + return replacement; + } + + @Override + public String complete(UUID taskId, long generation, String token, String checkpoint, + String coverage, Instant retryAt) { + configuration(); + Lease lease = lease(taskId, generation, token); + require(checkpoint != null && Objects.equals(checkpoint, lease.checkpoint()), 409, "CHECKPOINT_CONFLICT"); + String savedCoverage = checkpoint(checkpoint, lease.snapshot()).path("coverage").asText(); + Instant now = Instant.now(); + if ("COMPLETE".equals(coverage)) { + require(List.of("COMPLETE", "SPLIT").contains(savedCoverage) && retryAt == null, 409, "RESULT_CONFLICT"); + if (owner(lease.snapshot())) { + int hours = jdbc.queryForObject("SELECT search_rule_hours FROM github_sync_schedule_policy WHERE id = TRUE", Integer.class); + release(taskId, "ACQUISITION_COMPLETE", now, null); + jdbc.update(""" + UPDATE github_deep_discovery_query SET active_task_id = NULL, checkpoint = NULL, + last_full_recheck_at = ?, next_due_at = ?, last_progress_at = ?, wait_reason = NULL, gap_reason = NULL + WHERE id = ? + """, Timestamp.from(now), Timestamp.from(now.plusSeconds(hours * 3600L)), Timestamp.from(now), lease.queryId()); + return "ACQUISITION_COMPLETE"; + } + var root = shards.finish(taskId, lease.queryId(), lease.snapshot(), "SPLIT".equals(savedCoverage)); + if (root == null) { + release(taskId, "ACQUISITION_COMPLETE", now, "SHARD_HANDOFF"); + jdbc.update(""" + UPDATE github_deep_discovery_query SET active_task_id = NULL, checkpoint = NULL, + next_due_at = ?, last_progress_at = ?, wait_reason = 'SHARD_PENDING' WHERE id = ? + """, Timestamp.from(shards.nextDue(lease.queryId(), now)), Timestamp.from(now), lease.queryId()); + return "ACQUISITION_COMPLETE"; + } + Instant until = root.until(); + int hours = jdbc.queryForObject("SELECT search_rule_hours FROM github_sync_schedule_policy WHERE id = TRUE", Integer.class); + release(taskId, "ACQUISITION_COMPLETE", now, null); + jdbc.update(""" + UPDATE github_deep_discovery_query SET covered_until = GREATEST(covered_until, ?), + active_task_id = NULL, checkpoint = NULL, next_due_at = ?, last_progress_at = ?, wait_reason = NULL, gap_reason = NULL, + last_full_recheck_at = CASE WHEN ? THEN ? ELSE last_full_recheck_at END WHERE id = ? + """, Timestamp.from(until), Timestamp.from(now.plusSeconds(hours * 3600L)), Timestamp.from(now), + root.historical(), Timestamp.from(now), lease.queryId()); + return "ACQUISITION_COMPLETE"; + } + require("INCOMPLETE".equals(coverage) && !List.of("COMPLETE", "SPLIT").contains(savedCoverage) + && retryAt != null && retryAt.isAfter(now), 422, "RESULT_CONFLICT"); + String waitReason = "GAP".equals(savedCoverage) + ? jdbc.queryForObject("SELECT official_metadata ->> 'gapReason' FROM github_discovery_batch WHERE task_id = ? AND checkpoint = ?", + String.class, taskId, checkpoint) : savedCoverage; + if ("GAP".equals(savedCoverage) && !owner(lease.snapshot())) { + shards.gap(taskId, lease.queryId(), lease.snapshot(), checkpoint, generation, retryAt, waitReason); + release(taskId, "PARTIAL", retryAt, waitReason); + jdbc.update(""" + UPDATE github_deep_discovery_query SET active_task_id = NULL, checkpoint = NULL, + next_due_at = ?, last_progress_at = ?, wait_reason = ? WHERE id = ? + """, Timestamp.from(shards.nextDue(lease.queryId(), now)), Timestamp.from(now), waitReason, lease.queryId()); + return "PARTIAL"; + } + release(taskId, "RETRY_WAIT", retryAt, savedCoverage); + jdbc.update("UPDATE github_deep_discovery_query SET wait_reason = ? WHERE id = ?", waitReason, lease.queryId()); + return "RETRY_WAIT"; + } + + private void release(UUID taskId, String status, Instant due, String reason) { + jdbc.update(""" + UPDATE github_sync_task SET status = ?, due_at = ?, retry_at = ?, official_pause_reason = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + completed_at = CASE WHEN ? IN ('FAILED', 'ACQUISITION_COMPLETE', 'PARTIAL') THEN now() ELSE NULL END, + updated_at = now() WHERE id = ? + """, status, Timestamp.from(due), "RETRY_WAIT".equals(status) ? Timestamp.from(due) : null, reason, status, taskId); + } + + private boolean excluded(GithubDiscoveryCandidate candidate, JsonNode exclusions) { + String owner = candidate.owner().toLowerCase(Locale.ROOT); + String repository = owner + "/" + candidate.name().toLowerCase(Locale.ROOT); + for (JsonNode exclusion : exclusions) { + if ("OWNER".equals(exclusion.path("kind").asText()) && owner.equals(exclusion.path("value").asText()) + || "REPOSITORY".equals(exclusion.path("kind").asText()) && repository.equals(exclusion.path("value").asText())) { + return true; + } + } + return false; + } + + private JsonNode parseExclusions(String json) { + try { + return mapper.readTree(json); + } catch (Exception exception) { + throw new IllegalStateException("Invalid stored official exclusions", exception); + } + } + + private Configuration configuration() { + return jdbc.query("SELECT enabled, config_version, exclusions_json::text FROM github_deep_discovery_config WHERE id = TRUE FOR SHARE", + result -> { + if (!result.next()) { + throw new IllegalStateException("Official discovery configuration is missing"); + } + return new Configuration(result.getBoolean(1), result.getLong(2), parseExclusions(result.getString(3))); + }); + } + + private Lease lease(UUID taskId, long generation, String token) { + require(token != null && !token.isBlank() && generation > 0, 409, "LEASE_STALE"); + Lease lease = jdbc.query(""" + SELECT task.namespace_id, task.run_id, task.official_query_id, task.snapshot_json::text, + task.discovery_checkpoint FROM github_sync_task task + JOIN github_deep_discovery_query query ON query.id = task.official_query_id AND query.active_task_id = task.id + WHERE task.id = ? AND task.kind = 'DISCOVERY' AND task.status = 'LEASED' AND task.lease_generation = ? + AND task.lease_token_hash = ? AND task.lease_expires_at > now() FOR UPDATE OF task + """, result -> result.next() ? new Lease(result.getLong(1), result.getObject(2, UUID.class), + result.getObject(3, UUID.class), parse(result.getString(4)).path("officialDiscovery"), result.getString(5)) : null, + taskId, generation, hash(token)); + require(lease != null, 409, "LEASE_STALE"); + return lease; + } + + private JsonNode checkpoint(String value, JsonNode snapshot) { + require(value != null && value.length() <= 4096 && value.getBytes(StandardCharsets.UTF_8).length <= 4096, + 422, "RESULT_CONFLICT"); + JsonNode result = parse(value); + require(result.path("formatVersion").isIntegralNumber() && result.path("formatVersion").asInt() == 1 + && result.path("nextPage").isIntegralNumber() && result.path("nextPage").canConvertToInt() + && result.path("nextPage").asInt() >= 1 && (owner(snapshot) || result.path("nextPage").asInt() <= 11) + && (!result.has("cursor") || result.path("cursor").isTextual() && result.path("cursor").asText().length() <= 1024) + && List.of("IN_PROGRESS", "COMPLETE", "GAP", "SPLIT").contains(result.path("coverage").asText()), + 422, "RESULT_CONFLICT"); + for (String field : List.of("strategyVersion", "queryId", "queryVersion", "shardId")) { + require(Objects.equals(snapshot.get(field), result.get(field)), 409, "CHECKPOINT_CONFLICT"); + } + require(!owner(snapshot) || !"SPLIT".equals(result.path("coverage").asText()), 422, "RESULT_CONFLICT"); + return result; + } + + private void validateMetadata(GithubOfficialDiscoveryBatch metadata, JsonNode checkpoint, Lease lease) { + require(metadata != null && metadata.shardId() != null + && metadata.shardId().toString().equals(lease.snapshot().path("shardId").asText()) + && Objects.equals(metadata.coverage(), checkpoint.path("coverage").asText()) + && metadata.requestsUsed() != null && metadata.requestsUsed() >= 0 && metadata.requestsUsed() <= 10 + && ("SPLIT".equals(metadata.coverage()) + ? metadata.childRanges() != null && metadata.childRanges().size() == 2 + : metadata.childRanges() == null || metadata.childRanges().isEmpty()), 422, "RESULT_CONFLICT"); + require(metadata.ownerLogin() == null || owner(lease.snapshot()) + && metadata.ownerLogin().matches("[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?"), 422, "RESULT_CONFLICT"); + require("GAP".equals(metadata.coverage()) + ? List.of("RESULT_LIMIT", "INCOMPLETE_RESULTS", "UNSPLITTABLE", "WORK_BUDGET").contains(metadata.gapReason() == null ? "" : metadata.gapReason()) + : metadata.gapReason() == null, + 422, "RESULT_CONFLICT"); + } + + private void validateProvenance(GithubDiscoveryCandidate candidate, Lease lease) { + require(candidate.officialProvenance() != null && candidate.officialProvenance().size() == 1, 422, "RESULT_CONFLICT"); + var provenance = candidate.officialProvenance().getFirst(); + JsonNode snapshot = lease.snapshot(); + require(Objects.equals(provenance.queryId(), lease.queryId()) + && provenance.queryVersion() == snapshot.path("queryVersion").asLong() + && Objects.equals(provenance.strategyVersion(), snapshot.path("strategyVersion").asText()) + && Objects.equals(provenance.channel(), snapshot.path("channel").asText()) + && provenance.shardId() != null && provenance.shardId().toString().equals(snapshot.path("shardId").asText()) + && (owner(snapshot) ? Objects.equals(provenance.seedRepositoryId(), snapshot.path("seedRepositoryId").asLong()) + : provenance.seedRepositoryId() == null), 422, "RESULT_CONFLICT"); + } + + private Control quotaControl(UUID taskId, Configuration config) { + Instant until = quota.cooldownForLease(taskId, Instant.now()); + return until == null ? null : new Control("PAUSE", config.version(), "QUOTA_COOLDOWN", until); + } + + private boolean owner(JsonNode snapshot) { + return "OWNER_EXPANSION".equals(snapshot.path("channel").asText()); + } + + private boolean ownerExcluded(Lease lease) { + return owner(lease.snapshot()) && Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_official_owner_eligibility WHERE query_id = ? AND excluded) + """, Boolean.class, lease.queryId())); + } + + private JsonNode parse(String value) { + try { + JsonNode parsed = mapper.readTree(value); + require(parsed != null && parsed.isObject(), 422, "RESULT_CONFLICT"); + return parsed; + } catch (Exception exception) { + throw new GithubOfficialProtocolException(422, "RESULT_CONFLICT"); + } + } + + private String json(Object value) { + try { + return mapper.writeValueAsString(value); + } catch (Exception exception) { + throw new IllegalArgumentException("Invalid official discovery result", exception); + } + } + + private static String hash(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 unavailable", exception); + } + } + + private static void require(boolean valid, int status, String code) { + if (!valid) { + throw new GithubOfficialProtocolException(status, code); + } + } + + private record RepositoryLease(long namespaceId, Long sourceId, long repositoryId, + String fixedCommit, String snapshotCommit, String pathCheckpoint) {} + private record Eligibility(boolean official, boolean independent, boolean enabled, boolean excluded) {} + private record Configuration(boolean enabled, long version, JsonNode exclusions) {} + private record Lease(long namespaceId, UUID runId, UUID queryId, JsonNode snapshot, String checkpoint) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialOwnerRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialOwnerRepository.java new file mode 100644 index 000000000..25aa07941 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialOwnerRepository.java @@ -0,0 +1,55 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryCatalog; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +/** One-hop owner queries derive from durable validation evidence and independent official discovery. */ +@Repository +public class JdbcGithubOfficialOwnerRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + + public JdbcGithubOfficialOwnerRepository(JdbcTemplate jdbc, ObjectMapper mapper) { + this.jdbc = jdbc; + this.mapper = mapper; + } + + public void reconcile(long namespaceId, Instant now) { + var seeds = jdbc.query(""" + SELECT DISTINCT ON (seed.github_owner_id) seed.github_owner_id, seed.owner_login, seed.repository_id + FROM github_official_validated_seed seed + JOIN github_discovery_candidate candidate ON candidate.namespace_id = seed.namespace_id + AND candidate.repository_id = seed.repository_id + WHERE seed.namespace_id = ? AND EXISTS ( + SELECT 1 FROM github_discovery_candidate_origin origin + JOIN github_deep_discovery_query query ON query.id::text = origin.origin_key + AND query.namespace_id = seed.namespace_id AND query.query_source = 'CATALOG' + WHERE origin.candidate_id = candidate.id AND origin.origin_type = 'OFFICIAL_DISCOVERY') + AND NOT EXISTS (SELECT 1 FROM github_deep_discovery_query existing + WHERE existing.namespace_id = seed.namespace_id AND existing.github_owner_id = seed.github_owner_id + AND existing.query_source = 'OWNER') + ORDER BY seed.github_owner_id, seed.validated_at, seed.repository_id LIMIT 100 + """, (result, row) -> new Seed(result.getLong(1), result.getString(2), result.getLong(3)), namespaceId); + for (Seed seed : seeds) { + var definition = new GithubOfficialDiscoveryCatalog.Definition("owner-" + seed.ownerId(), 1, + "OWNER_EXPANSION", "ECOSYSTEM", "OWNER_RECHECK", + new GithubOfficialDiscoveryCatalog.Query("OWNER", seed.login(), "OWNER_REPOSITORIES")); + jdbc.update(""" + INSERT INTO github_deep_discovery_query + (id, namespace_id, query_key, query_version, strategy_version, definition_json, + next_due_at, query_source, github_owner_id, owner_login, seed_repository_id) + VALUES (?, ?, ?, 1, ?, ?::jsonb, ?, 'OWNER', ?, ?, ?) + ON CONFLICT (namespace_id, github_owner_id) WHERE query_source = 'OWNER' DO NOTHING + """, UUID.randomUUID(), namespaceId, definition.key(), GithubOfficialDiscoveryCatalog.VERSION, + mapper.valueToTree(definition).toString(), Timestamp.from(now), seed.ownerId(), seed.login(), seed.repositoryId()); + } + } + + private record Seed(long ownerId, String login, long repositoryId) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialQuotaRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialQuotaRepository.java new file mode 100644 index 000000000..3b1348d2b --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialQuotaRepository.java @@ -0,0 +1,196 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubOfficialQuotaRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +@Repository +public class JdbcGithubOfficialQuotaRepository implements GithubOfficialQuotaRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubOfficialQuotaRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean record(UUID taskId, long generation, String leaseTokenHash, UUID expectedCredentialId, + Observation observation, Instant now) { + validate(observation, now); + var leases = jdbc.query(""" + SELECT task.credential_id, task.credential_fingerprint, CASE WHEN task.kind = 'REPOSITORY_SYNC' + OR task.snapshot_json -> 'officialDiscovery' ->> 'channel' = 'OWNER_EXPANSION' + OR task.snapshot_json ->> 'entryType' IN ('CURATED_OWNER', 'CURATED_REPOSITORY') + THEN 'CORE' ELSE 'SEARCH' END AS resource + FROM github_sync_task task + LEFT JOIN github_system_credential credential ON credential.id = task.credential_id + WHERE task.id = ? AND task.status = 'LEASED' AND task.lease_generation = ? + AND task.lease_token_hash = ? AND task.lease_expires_at > CASE WHEN task.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + AND (?::uuid IS NULL OR task.credential_id = ?) + AND (?::uuid IS NOT NULL OR task.required_capabilities @> ARRAY['official-pause/v1']) + AND (task.credential_id IS NULL OR task.credential_fingerprint = credential.fingerprint) + FOR UPDATE OF task + """, (result, row) -> new Lease(result.getObject("credential_id", UUID.class), result.getString("credential_fingerprint"), result.getString("resource")), + taskId, generation, leaseTokenHash, Timestamp.from(now), expectedCredentialId, expectedCredentialId, expectedCredentialId); + if (leases.isEmpty()) { + return false; + } + Lease lease = leases.getFirst(); + if (lease.credentialId() != null) { + // Lock task, then credential: replacement never locks tasks. Recheck after waiting + // so a report accepted against the old fingerprint cannot poison a replacement. + var fingerprints = jdbc.query("SELECT fingerprint FROM github_system_credential WHERE id=? FOR UPDATE", + (result, row) -> result.getString(1), lease.credentialId()); + if (fingerprints.isEmpty() || !java.util.Objects.equals(lease.fingerprint(), fingerprints.getFirst())) { + return false; + } + } + if (!lease.resource().equals(observation.resource()) + && !("UNKNOWN".equals(observation.state()) && "OTHER".equals(observation.resource()))) { + throw new IllegalArgumentException("Quota resource differs from leased GitHub operation"); + } + Instant validUntil = observation.observedAt().plusSeconds(300); + if (validUntil.isAfter(now.plusSeconds(300))) { + validUntil = now.plusSeconds(300); + } + if (observation.resetAt() != null && observation.resetAt().isBefore(validUntil)) { + validUntil = observation.resetAt(); + } + String key = lease.credentialId() == null ? "anonymous" : "credential:" + lease.credentialId(); + jdbc.update(""" + INSERT INTO github_official_quota_observation + (observation_key, resource, credential_id, state, remaining, reset_at, observed_at, received_at, valid_until) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (observation_key, resource) DO UPDATE SET state = EXCLUDED.state, + remaining = EXCLUDED.remaining, reset_at = EXCLUDED.reset_at, observed_at = EXCLUDED.observed_at, + received_at = EXCLUDED.received_at, valid_until = EXCLUDED.valid_until + WHERE github_official_quota_observation.observed_at <= EXCLUDED.observed_at + """, key, observation.resource(), lease.credentialId(), observation.state(), observation.remaining(), + timestamp(observation.resetAt()), Timestamp.from(observation.observedAt()), Timestamp.from(now), Timestamp.from(validUntil)); + // Authenticated primary limits belong to the leased credential and API resource. + // Distinct configured tokens may share an upstream identity; this permits trying an + // unexhausted token, not summing or assuming their allowances are independent. + if (lease.credentialId() != null && List.of("OBSERVED", "PRIMARY_EXHAUSTED").contains(observation.state())) { + jdbc.update(""" + INSERT INTO github_system_credential_quota + (credential_id, resource, remaining, reset_at, observed_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (credential_id, resource) DO UPDATE SET remaining=EXCLUDED.remaining, + reset_at=EXCLUDED.reset_at, observed_at=EXCLUDED.observed_at + WHERE github_system_credential_quota.observed_at <= EXCLUDED.observed_at + """, lease.credentialId(), observation.resource(), observation.remaining(), + timestamp(observation.resetAt()), Timestamp.from(observation.observedAt())); + } + if (lease.credentialId() == null && ("PRIMARY_EXHAUSTED".equals(observation.state()) + || "OBSERVED".equals(observation.state()) && Integer.valueOf(0).equals(observation.remaining()) + && observation.resetAt().isAfter(now))) { + jdbc.update(""" + INSERT INTO github_official_resource_cooldown (resource, primary_until) VALUES (?, ?) + ON CONFLICT (resource) DO UPDATE SET primary_until = GREATEST( + github_official_resource_cooldown.primary_until, EXCLUDED.primary_until) + """, observation.resource(), Timestamp.from(observation.resetAt())); + } + if ("SECONDARY_COOLDOWN".equals(observation.state()) + || "UNKNOWN".equals(observation.state()) && observation.retryAfterSeconds() != null + || "AUTH_INVALID".equals(observation.state()) && lease.credentialId() == null) { + Instant until = observation.observedAt().plusSeconds(observation.retryAfterSeconds() == null ? 60 : Math.max(1, observation.retryAfterSeconds())); + jdbc.update("UPDATE github_official_secondary_cooldown SET cooldown_until = GREATEST(cooldown_until, ?) WHERE id = TRUE", + Timestamp.from(until)); + } + if ("AUTH_INVALID".equals(observation.state()) && lease.credentialId() != null) { + jdbc.update("UPDATE github_system_credential SET status = 'INVALID', rate_limited_until = NULL, updated_at = ? WHERE id = ?", + Timestamp.from(now), lease.credentialId()); + } + return true; + } + + @Override + public Instant cooldown(String resource, Instant now) { + return jdbc.query(""" + SELECT max(blocked_until) FROM ( + SELECT cooldown_until AS blocked_until FROM github_official_secondary_cooldown WHERE id = TRUE + UNION ALL SELECT primary_until FROM github_official_resource_cooldown WHERE resource = ? + AND NOT EXISTS (SELECT 1 FROM github_system_credential) + ) cooldown WHERE blocked_until > ? + """, result -> result.next() && result.getTimestamp(1) != null ? result.getTimestamp(1).toInstant() : null, + resource, Timestamp.from(now)); + } + + @Override + public Instant cooldownForLease(UUID taskId, Instant now) { + return jdbc.query(""" + WITH leased AS ( + SELECT credential_id, CASE WHEN kind='REPOSITORY_SYNC' + OR snapshot_json -> 'officialDiscovery' ->> 'channel'='OWNER_EXPANSION' + OR snapshot_json ->> 'entryType' IN ('CURATED_OWNER','CURATED_REPOSITORY') + THEN 'CORE' ELSE 'SEARCH' END AS resource + FROM github_sync_task WHERE id=? + ) SELECT max(blocked_until) FROM ( + SELECT cooldown_until AS blocked_until FROM github_official_secondary_cooldown WHERE id + UNION ALL SELECT c.primary_until FROM github_official_resource_cooldown c + JOIN leased l ON l.credential_id IS NULL AND l.resource=c.resource + UNION ALL SELECT q.reset_at FROM github_system_credential_quota q + JOIN leased l ON l.credential_id=q.credential_id AND l.resource=q.resource + WHERE q.remaining=0 + ) cooldown WHERE blocked_until>? + """, result -> result.next() && result.getTimestamp(1)!=null ? result.getTimestamp(1).toInstant() : null, + taskId, Timestamp.from(now)); + } + + @Override + public void deferClaim(UUID taskId, long generation, Instant due, Instant now, String reason) { + if (!List.of("QUOTA_COOLDOWN", "CREDENTIAL_UNAVAILABLE").contains(reason)) { + throw new IllegalArgumentException("Invalid credential deferral reason"); + } + int count = jdbc.update(""" + UPDATE github_sync_task SET status = 'RETRY_WAIT', due_at = ?, retry_at = ?, official_pause_reason = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, updated_at = ? + WHERE id = ? AND status = 'LEASED' AND lease_generation = ? + """, Timestamp.from(due), Timestamp.from(due), reason, Timestamp.from(now), taskId, generation); + if (count != 1) { + throw new IllegalStateException("Credential deferral lost its newly claimed task"); + } + } + + @Override + public void clearClaimWait(UUID taskId, long generation) { + jdbc.update(""" + UPDATE github_sync_task SET official_pause_reason = NULL + WHERE id = ? AND status = 'LEASED' AND lease_generation = ? + AND official_pause_reason IN ('QUOTA_COOLDOWN','CREDENTIAL_UNAVAILABLE') + """, taskId, generation); + jdbc.update(""" + UPDATE github_deep_discovery_query q SET wait_reason = NULL + FROM github_sync_task t WHERE t.id = ? AND t.status = 'LEASED' + AND t.lease_generation = ? AND q.active_task_id = t.id + AND q.wait_reason IN ('QUOTA_COOLDOWN','CREDENTIAL_UNAVAILABLE') + """, taskId, generation); + } + + private void validate(Observation value, Instant now) { + if (value == null || value.resource() == null || !List.of("CORE", "SEARCH", "GRAPHQL", "OTHER").contains(value.resource()) + || value.state() == null || !List.of("OBSERVED", "UNKNOWN", "PRIMARY_EXHAUSTED", "SECONDARY_COOLDOWN", "AUTH_INVALID").contains(value.state()) + || value.observedAt() == null || value.observedAt().isBefore(now.minusSeconds(300)) || value.observedAt().isAfter(now.plusSeconds(300)) + || value.remaining() != null && value.remaining() < 0 + || value.resetAt() != null && value.resetAt().isAfter(now.plusSeconds(86400)) + || value.retryAfterSeconds() != null && (value.retryAfterSeconds() < 0 || value.retryAfterSeconds() > 86400)) { + throw new IllegalArgumentException("Invalid official GitHub quota observation"); + } + if (("OBSERVED".equals(value.state()) || "PRIMARY_EXHAUSTED".equals(value.state())) + && (value.remaining() == null || value.resetAt() == null) + || "PRIMARY_EXHAUSTED".equals(value.state()) && (!Integer.valueOf(0).equals(value.remaining()) || !value.resetAt().isAfter(now)) + || "UNKNOWN".equals(value.state()) && (value.remaining() != null || value.resetAt() != null)) { + throw new IllegalArgumentException("Quota state and values disagree"); + } + } + + private Timestamp timestamp(Instant instant) { + return instant == null ? null : Timestamp.from(instant); + } + + private record Lease(UUID credentialId, String fingerprint, String resource) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialShardRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialShardRepository.java new file mode 100644 index 000000000..e614370c5 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubOfficialShardRepository.java @@ -0,0 +1,217 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubOfficialDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +/** Bounded sequential binary trees. Callers hold configuration/query lease locks in one transaction. */ +@Repository +public class JdbcGithubOfficialShardRepository { + public static final int MAX_DEPTH = 32; + public static final int MAX_CHILDREN = 126; + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + + public JdbcGithubOfficialShardRepository(JdbcTemplate jdbc, ObjectMapper mapper) { + this.jdbc = jdbc; + this.mapper = mapper; + } + + public Work next(UUID queryId, UUID proposedTaskId, ObjectNode freshSnapshot, int candidateBudget, Instant now) { + List pending = jdbc.query(""" + SELECT * FROM github_official_discovery_shard WHERE query_id = ? + AND (state = 'PENDING' OR (state = 'GAP' AND retry_at <= ?)) + ORDER BY CASE WHEN state = 'PENDING' THEN 0 ELSE 1 END, range_since, depth, id LIMIT 1 FOR UPDATE + """, (result, row) -> row(result), queryId, Timestamp.from(now)); + if (pending.isEmpty()) { + boolean unfinished = Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_official_discovery_shard + WHERE query_id = ? AND parent_id IS NULL AND state <> 'COMPLETE') + """, Boolean.class, queryId)); + if (unfinished) { + throw new IllegalStateException("Official shard tree has no resumable leaf"); + } + jdbc.update("DELETE FROM github_official_discovery_shard WHERE query_id = ? AND parent_id IS NULL AND state = 'COMPLETE'", queryId); + JsonNode official = freshSnapshot.path("officialDiscovery"); + UUID rootId = UUID.randomUUID(); + insert(rootId, queryId, rootId, null, 0, official, "PENDING", null); + pending = jdbc.query("SELECT * FROM github_official_discovery_shard WHERE id = ?", + (result, row) -> row(result), rootId); + } + Shard node = pending.getFirst(); + Shard root = root(node.rootId()); + int descendants = jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE root_id = ? AND parent_id IS NOT NULL", Integer.class, root.id()); + UUID taskId = node.taskId() == null ? proposedTaskId : node.taskId(); + boolean reuse = node.taskId() != null; + ObjectNode official = node.snapshot().deepCopy(); + official.put("shardId", taskId.toString()); + official.remove("checkpoint"); + ObjectNode budget = (ObjectNode) official.path("budget"); + budget.put("maxSplitDepth", Math.min(budget.path("maxSplitDepth").asInt(), Math.max(0, root.snapshot().path("budget").path("maxSplitDepth").asInt() - node.depth()))); + budget.put("maxChildShards", Math.min(budget.path("maxChildShards").asInt(), Math.max(0, root.snapshot().path("budget").path("maxChildShards").asInt() - descendants))); + budget.put("candidateBudget", Math.min(budget.path("candidateBudget").asInt(), candidateBudget)); + // Task is inserted by the scheduler before task_id is assigned (FK order). + ObjectNode wrapper = mapper.createObjectNode(); + wrapper.set("officialDiscovery", official); + String checkpoint = node.checkpoint(); + if (checkpoint != null && !reuse) { + try { + ObjectNode saved = (ObjectNode) mapper.readTree(checkpoint); + saved.put("shardId", taskId.toString()); + checkpoint = saved.toString(); + } catch (com.fasterxml.jackson.core.JsonProcessingException exception) { + throw new IllegalStateException("Invalid durable GAP checkpoint", exception); + } + } + return new Work(node.id(), taskId, reuse, wrapper, checkpoint, node.generation()); + } + + public void assigned(Work work) { + int updated = jdbc.update(""" + UPDATE github_official_discovery_shard SET state = 'ACTIVE', task_id = ?, snapshot_json = ?::jsonb + WHERE id = ? AND state IN ('PENDING', 'GAP') + """, work.taskId(), work.snapshot().path("officialDiscovery").toString(), work.shardId()); + require(updated == 1); + } + + public void gap(UUID taskId, UUID queryId, JsonNode snapshot, String checkpoint, + long generation, Instant retryAt, String reason) { + Shard node = ensure(taskId, queryId, snapshot); + require("ACTIVE".equals(node.state())); + jdbc.update(""" + UPDATE github_official_discovery_shard SET state = 'GAP', retry_at = ?, gap_reason = ?, + gap_checkpoint = ?, last_lease_generation = ?, snapshot_json = ?::jsonb WHERE id = ? + """, Timestamp.from(retryAt), reason, checkpoint, generation, snapshot.toString(), node.id()); + } + + public Instant nextDue(UUID queryId, Instant now) { + return jdbc.queryForObject(""" + SELECT CASE WHEN bool_or(state = 'PENDING') THEN ? ELSE min(retry_at) FILTER (WHERE state = 'GAP') END + FROM github_official_discovery_shard WHERE query_id = ? + """, (result, row) -> result.getTimestamp(1).toInstant(), Timestamp.from(now), queryId); + } + + /** Lazily represents tasks created before V102, keeping their immutable zero-split budget. */ + public Shard ensure(UUID taskId, UUID queryId, JsonNode snapshot) { + List nodes = jdbc.query("SELECT * FROM github_official_discovery_shard WHERE task_id = ? FOR UPDATE", + (result, row) -> row(result), taskId); + if (!nodes.isEmpty()) { + return nodes.getFirst(); + } + UUID id = UUID.randomUUID(); + insert(id, queryId, id, null, 0, snapshot, "ACTIVE", taskId); + return jdbc.queryForObject("SELECT * FROM github_official_discovery_shard WHERE id = ?", (result, row) -> row(result), id); + } + + public void split(UUID taskId, UUID queryId, JsonNode snapshot, List ranges) { + Shard parent = ensure(taskId, queryId, snapshot); + Shard root = root(parent.rootId()); + require("ACTIVE".equals(parent.state()) && ranges != null && ranges.size() == 2); + int descendants = jdbc.queryForObject("SELECT count(*) FROM github_official_discovery_shard WHERE root_id = ? AND parent_id IS NOT NULL", Integer.class, root.id()); + int maxDepth = Math.min(MAX_DEPTH, root.snapshot().path("budget").path("maxSplitDepth").asInt()); + int maxChildren = Math.min(MAX_CHILDREN, root.snapshot().path("budget").path("maxChildShards").asInt()); + require(parent.depth() < maxDepth && descendants + 2 <= maxChildren + && snapshot.path("budget").path("maxSplitDepth").asInt() > 0 + && snapshot.path("budget").path("maxChildShards").asInt() >= 2); + var left = ranges.get(0); + var right = ranges.get(1); + require(valid(left) && valid(right) && parent.since().equals(left.since()) + && left.until().equals(right.since()) && parent.until().equals(right.until())); + Instant cutoff = Instant.parse(snapshot.path("fixedCutoffAt").asText()); + require(!right.until().isAfter(cutoff)); + for (var range : ranges) { + ObjectNode child = parent.snapshot().deepCopy(); + child.remove("checkpoint"); + child.putObject("range").put("since", range.since().toString()).put("until", range.until().toString()); + insert(UUID.randomUUID(), queryId, root.id(), parent.id(), parent.depth() + 1, child, "PENDING", null); + } + jdbc.update("UPDATE github_official_discovery_shard SET state = 'SPLIT' WHERE id = ?", parent.id()); + } + + public Completion finish(UUID taskId, UUID queryId, JsonNode snapshot, boolean split) { + Shard node = ensure(taskId, queryId, snapshot); + Shard root = root(node.rootId()); + require(split ? "SPLIT".equals(node.state()) : "ACTIVE".equals(node.state())); + if (!split) { + jdbc.update("UPDATE github_official_discovery_shard SET state = 'COMPLETE' WHERE id = ?", node.id()); + } + boolean unfinished = Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_official_discovery_shard + WHERE root_id = ? AND state NOT IN ('SPLIT', 'COMPLETE')) + """, Boolean.class, root.id())); + if (unfinished) { + return null; + } + jdbc.update("UPDATE github_official_discovery_shard SET state = 'COMPLETE', completed_at = now() WHERE id = ?", root.id()); + return new Completion(root.until(), "HISTORICAL".equals(root.snapshot().path("mode").asText())); + } + + public void recover(UUID taskId, UUID replacement, UUID queryId, JsonNode snapshot) { + Shard node = ensure(taskId, queryId, snapshot); + require(!"SPLIT".equals(node.state())); + jdbc.update("UPDATE github_official_discovery_shard SET task_id = ? WHERE id = ?", replacement, node.id()); + } + + public void replayFailed(long namespaceId) { + jdbc.update(""" + UPDATE github_official_discovery_shard shard SET task_id = NULL, + state = CASE WHEN shard.state = 'ACTIVE' THEN + CASE WHEN shard.gap_checkpoint IS NULL THEN 'PENDING' ELSE 'GAP' END ELSE shard.state END, + last_lease_generation = GREATEST(shard.last_lease_generation, task.lease_generation) + FROM github_sync_task task WHERE shard.task_id = task.id AND task.status = 'FAILED' AND task.namespace_id = ? + """, namespaceId); + } + + private Shard root(UUID id) { + return jdbc.queryForObject("SELECT * FROM github_official_discovery_shard WHERE id = ? FOR UPDATE", + (result, row) -> row(result), id); + } + + private boolean valid(GithubOfficialDiscoveryBatch.Range range) { + return range != null && range.since() != null && range.until() != null + && range.since().getNano() == 0 && range.until().getNano() == 0 && range.since().isBefore(range.until()) + && (range.partition() == null || range.partition().isEmpty()); + } + + private void insert(UUID id, UUID queryId, UUID rootId, UUID parentId, int depth, JsonNode snapshot, + String state, UUID taskId) { + jdbc.update(""" + INSERT INTO github_official_discovery_shard + (id, query_id, root_id, parent_id, depth, range_since, range_until, snapshot_json, state, task_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?) + """, id, queryId, rootId, parentId, depth, + Timestamp.from(Instant.parse(snapshot.path("range").path("since").asText())), + Timestamp.from(Instant.parse(snapshot.path("range").path("until").asText())), snapshot.toString(), state, taskId); + } + + private Shard row(java.sql.ResultSet result) throws java.sql.SQLException { + try { + return new Shard(result.getObject("id", UUID.class), result.getObject("root_id", UUID.class), + result.getInt("depth"), result.getTimestamp("range_since").toInstant(), + result.getTimestamp("range_until").toInstant(), (ObjectNode) mapper.readTree(result.getString("snapshot_json")), + result.getString("state"), result.getObject("task_id", UUID.class), + result.getString("gap_checkpoint"), result.getLong("last_lease_generation")); + } catch (com.fasterxml.jackson.core.JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted official shard snapshot", exception); + } + } + + private static void require(boolean valid) { + if (!valid) { + throw new GithubOfficialProtocolException(409, "RESULT_CONFLICT"); + } + } + + public record Shard(UUID id, UUID rootId, int depth, Instant since, Instant until, ObjectNode snapshot, String state, UUID taskId, String checkpoint, long generation) {} + public record Work(UUID shardId, UUID taskId, boolean reuse, ObjectNode snapshot, String checkpoint, long generation) {} + public record Completion(Instant until, boolean historical) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubPlatformControlRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubPlatformControlRepository.java new file mode 100644 index 000000000..987e07c31 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubPlatformControlRepository.java @@ -0,0 +1,185 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubOfficialLifecycleRepository; +import com.iflytek.skillhub.githubsync.GithubOfficialProtocolException; +import com.iflytek.skillhub.githubsync.GithubPlatformControlRepository; +import com.iflytek.skillhub.githubsync.GithubRepositoryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubSyncDeferralReason; +import com.iflytek.skillhub.githubsync.GithubSyncLeaseService; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Objects; +import java.util.UUID; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +@Repository +@Transactional +public class JdbcGithubPlatformControlRepository implements GithubPlatformControlRepository { + private final JdbcTemplate jdbc; + private final GithubSyncLeaseService leases; + + public JdbcGithubPlatformControlRepository(JdbcTemplate jdbc, GithubSyncLeaseService leases) { + this.jdbc = jdbc; + this.leases = leases; + } + + @Override + public State get() { + return jdbc.queryForObject(""" + SELECT c.*, count(t.id) AS pending, + count(t.id) FILTER (WHERE t.lease_expires_at IS NULL OR t.lease_expires_at <= clock_timestamp()) AS expired + FROM github_platform_collection_control c + LEFT JOIN github_sync_task t ON t.namespace_id=c.namespace_id + AND t.platform_pause_version=c.version AND t.platform_stop_ack_at IS NULL + GROUP BY c.namespace_id + """, (r, n) -> new State(r.getLong("namespace_id"), r.getLong("version"), r.getBoolean("paused"), + !r.getBoolean("paused") ? "RUNNING" : r.getLong("pending") == 0 ? "PAUSED" : "PAUSING", + r.getBoolean("paused") ? r.getLong("pending") : 0, + r.getBoolean("paused") ? r.getLong("expired") : 0, r.getTimestamp("updated_at").toInstant())); + } + + @Override + public State update(long expectedVersion, boolean paused, String actor) { + var current = lock(false); + if (current.version() != expectedVersion) { + // A retry of the transition just acknowledged by this client is harmless. + require(current.version() == expectedVersion + 1 && current.desiredPaused() == paused, "CONTROL_CHANGED"); + return get(); + } + if (current.desiredPaused() == paused) { + return get(); + } + jdbc.update("UPDATE github_platform_collection_control SET paused=?,version=version+1,updated_at=clock_timestamp(),updated_by=? WHERE namespace_id=?", + paused, actor, current.namespaceId()); + if (paused) { + jdbc.update(""" + UPDATE github_sync_task SET platform_pause_version=?,platform_pause_generation=lease_generation, + platform_pause_token_hash=lease_token_hash,platform_stop_ack_at=NULL + WHERE namespace_id=? AND status='LEASED' + """, current.version() + 1, current.namespaceId()); + } + return get(); + } + + @Override + public Control control(UUID taskId, long generation, String token) { + var state = lock(true); + var task = task(taskId); + require(task.namespace() == state.namespaceId(), "PLATFORM_SCOPE_MISMATCH"); + verifyProof(task, generation, hash(token)); + require("LEASED".equals(task.status()) && task.unexpired() + || state.desiredPaused() && task.pauseVersion() != null && task.pauseVersion() == state.version() + && task.pauseGeneration() != null && task.pauseGeneration() == generation, + "LEASE_STALE"); + return new Control(state.desiredPaused() ? "PAUSE" : "CONTINUE", state.version()); + } + + @Override + public void pause(UUID taskId, long generation, String token, long observedVersion, + String kind, String checkpoint, GithubOfficialLifecycleRepository.RepositoryProgress progress) { + var state = lock(true); + require(state.desiredPaused() && state.version() == observedVersion, "CONTROL_CHANGED"); + // Match the admission -> task lock order when validating received repository paths. + jdbc.queryForObject("SELECT id FROM github_sync_admission_policy WHERE id FOR UPDATE", Boolean.class); + var task = task(taskId); + String digest = hash(token); + require(task.namespace() == state.namespaceId() && task.pauseVersion() != null + && task.pauseVersion() == observedVersion && task.pauseGeneration() != null + && task.pauseGeneration() == generation && Objects.equals(task.pauseToken(), digest) + && task.generation() == generation, "LEASE_STALE"); + require(Objects.equals(kind, task.kind()), "RESULT_CONFLICT"); + if (task.acknowledged()) { + return; + } + boolean active = "LEASED".equals(task.status()) && task.unexpired(); + if (active) { + require(Objects.equals(task.token(), digest), "LEASE_STALE"); + if ("DISCOVERY".equals(kind)) { + require(progress == null && Objects.equals(checkpoint, task.checkpoint()), "CHECKPOINT_CONFLICT"); + } else { + require(checkpoint == null && progress != null && progress.incompleteSubtrees() != null + && progress.pathResults() != null, "RESULT_CONFLICT"); + if (progress.observedCommitSha() == null) { + require(task.commit() == null && progress.pathCheckpoint() == null + && progress.incompleteSubtrees().isEmpty() && progress.pathResults().isEmpty(), "COMMIT_MISMATCH"); + require(!Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS(SELECT 1 FROM github_sync_package_admission WHERE task_id=?)", Boolean.class, taskId)), "COMMIT_MISMATCH"); + } else if ("FULL".equals(task.mode())) { + var result = leases.completeRepository(taskId, generation, token, progress.observedCommitSha(), false, + progress.pathCheckpoint(), progress.incompleteSubtrees(), progress.pathResults(), + Instant.now().plusSeconds(60), GithubSyncDeferralReason.CAPACITY); + require(result == GithubRepositoryCompletionResult.RETRY_WAIT, "RESULT_CONFLICT"); + } else { + require(progress.pathCheckpoint() == null && progress.pathResults().isEmpty() + && progress.incompleteSubtrees().isEmpty(), "RESULT_CONFLICT"); + require(task.commit() == null || Objects.equals(task.commit(), progress.observedCommitSha()), "COMMIT_MISMATCH"); + } + } + } + jdbc.update(""" + UPDATE github_sync_package_admission SET status='RELEASED',updated_at=clock_timestamp() + WHERE task_id=? AND lease_generation=? AND status='ACTIVE' + """, taskId, generation); + // Expired or already-settled work only acknowledges stopping. Never write its proposed progress. + jdbc.update("UPDATE github_sync_task SET platform_stop_ack_at=clock_timestamp() WHERE id=?", taskId); + if ("LEASED".equals(task.status())) { + jdbc.update(""" + UPDATE github_sync_task SET status='QUEUED',lease_token_hash=NULL,lease_expires_at=NULL, + leased_by=NULL,credential_id=NULL,credential_fingerprint=NULL,due_at=clock_timestamp(), + retry_at=NULL,updated_at=clock_timestamp() WHERE id=? + """, taskId); + + } + } + + private State lock(boolean shared) { + return jdbc.queryForObject( + "SELECT * FROM github_platform_collection_control FOR " + (shared ? "SHARE" : "UPDATE"), + (result, row) -> new State(result.getLong("namespace_id"), result.getLong("version"), + result.getBoolean("paused"), null, 0, 0, result.getTimestamp("updated_at").toInstant())); + } + + private record Task(long namespace, String kind, String mode, String status, long generation, String token, + String checkpoint, String commit, boolean unexpired, Long pauseVersion, Long pauseGeneration, + String pauseToken, boolean acknowledged) {} + + private Task task(UUID id) { + return jdbc.query(""" + SELECT *, lease_expires_at > clock_timestamp() AS unexpired + FROM github_sync_task WHERE id = ? FOR UPDATE + """, result -> { + require(result.next(), "LEASE_STALE"); + return new Task(result.getLong("namespace_id"), result.getString("kind"), result.getString("mode"), + result.getString("status"), result.getLong("lease_generation"), result.getString("lease_token_hash"), + result.getString("discovery_checkpoint"), result.getString("fixed_commit_sha"), + result.getBoolean("unexpired"), result.getObject("platform_pause_version", Long.class), + result.getObject("platform_pause_generation", Long.class), result.getString("platform_pause_token_hash"), + result.getTimestamp("platform_stop_ack_at") != null); + }, id); + } + + private void verifyProof(Task task, long generation, String digest) { + require(task.generation() == generation && (Objects.equals(task.token(), digest) + || task.pauseGeneration() != null && task.pauseGeneration() == generation + && Objects.equals(task.pauseToken(), digest)), "LEASE_STALE"); + } + + private static String hash(String token) { + require(token != null && token.matches("[0-9a-fA-F]{64}"), "LEASE_STALE"); + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.UTF_8))); + } catch (java.security.NoSuchAlgorithmException exception) { + throw new IllegalStateException(exception); + } + } + + private static void require(boolean valid, String code) { + if (!valid) { + throw new GithubOfficialProtocolException(409, code); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSearchPreviewRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSearchPreviewRepository.java new file mode 100644 index 000000000..b730d1d5e --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSearchPreviewRepository.java @@ -0,0 +1,77 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.iflytek.skillhub.githubsync.GithubSearchPreviewRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSearchPreviewRepository implements GithubSearchPreviewRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + + public JdbcGithubSearchPreviewRepository(JdbcTemplate jdbc, ObjectMapper mapper) { + this.jdbc = jdbc; + this.mapper = mapper; + } + + @Override + public UUID queue(String criteriaJson, String actorId, Instant now) { + Long namespaceId = jdbc.query("SELECT id FROM namespace WHERE slug = 'github' AND type = 'CATALOG'", + result -> result.next() ? result.getLong(1) : null); + if (namespaceId == null) { + throw new IllegalStateException("GitHub catalog namespace is missing"); + } + UUID id = UUID.randomUUID(); + UUID runId = UUID.randomUUID(); + ObjectNode snapshot = mapper.createObjectNode(); + snapshot.put("entryType", "SEARCH_PREVIEW"); + snapshot.put("ruleVersion", 1); + snapshot.put("candidateBudget", 1); + snapshot.put("actorId", actorId); + snapshot.set("criteria", readCriteria(criteriaJson)); + ObjectNode window = snapshot.putObject("searchWindow"); + window.put("mode", "FULL_RECHECK"); + window.put("until", now.toString()); + jdbc.update("INSERT INTO github_sync_run (id, namespace_id, kind, status, created_at) " + + "VALUES (?, ?, 'DISCOVERY', 'QUEUED', ?)", runId, namespaceId, Timestamp.from(now)); + jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, kind, status, dedupe_key, priority, + snapshot_json, due_at, created_at, updated_at) + VALUES (?, ?, ?, 'DISCOVERY', 'QUEUED', ?, 10, ?::jsonb, ?, ?, ?) + """, id, runId, namespaceId, "preview:" + id, snapshot.toString(), + Timestamp.from(now), Timestamp.from(now), Timestamp.from(now)); + return id; + } + + @Override + public Optional find(UUID id) { + return jdbc.query(""" + SELECT id, status, last_error_code, retry_at, snapshot_json::text AS snapshot_json, + created_at, updated_at + FROM github_sync_task + WHERE id = ? AND kind = 'DISCOVERY' AND snapshot_json ->> 'entryType' = 'SEARCH_PREVIEW' + """, result -> result.next() ? Optional.of(new Preview( + result.getObject("id", UUID.class), result.getString("status"), + result.getString("last_error_code"), + result.getTimestamp("retry_at") == null ? null : result.getTimestamp("retry_at").toInstant(), + result.getString("snapshot_json"), + result.getTimestamp("created_at").toInstant(), + result.getTimestamp("updated_at").toInstant())) : Optional.empty(), id); + } + + private com.fasterxml.jackson.databind.JsonNode readCriteria(String value) { + try { + return mapper.readTree(value); + } catch (com.fasterxml.jackson.core.JsonProcessingException exception) { + throw new IllegalStateException("Invalid validated search criteria", exception); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceIgnoreRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceIgnoreRepository.java new file mode 100644 index 000000000..ebff30fc4 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceIgnoreRepository.java @@ -0,0 +1,75 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubSkillPathPolicy; +import com.iflytek.skillhub.githubsync.GithubSourceIgnoreRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Optional; + +@Repository +public class JdbcGithubSourceIgnoreRepository implements GithubSourceIgnoreRepository { + private static final TypeReference> PATTERNS = new TypeReference<>() {}; + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + + public JdbcGithubSourceIgnoreRepository(JdbcTemplate jdbc, ObjectMapper mapper) { + this.jdbc = jdbc; + this.mapper = mapper; + } + + @Override + public Optional> find(long namespaceId, long sourceId) { + return jdbc.query("SELECT ignore_patterns::text FROM github_source_binding " + + "WHERE namespace_id = ? AND id = ?", + (result, row) -> parse(result.getString(1)), namespaceId, sourceId) + .stream().findFirst(); + } + + @Override + @Transactional + public Optional> replace(long namespaceId, long sourceId, List requested) { + List patterns = GithubSkillPathPolicy.normalizePatterns(requested); + Optional> current = jdbc.query("SELECT ignore_patterns::text FROM github_source_binding " + + "WHERE namespace_id = ? AND id = ? FOR UPDATE", + (result, row) -> parse(result.getString(1)), namespaceId, sourceId) + .stream().findFirst(); + if (current.isEmpty() || current.get().equals(patterns)) { + return current; + } + try { + jdbc.update("UPDATE github_source_binding SET ignore_patterns = ?::jsonb, " + + "last_fully_processed_commit = NULL, last_partial_commit = NULL, " + + "next_check_at = CURRENT_TIMESTAMP, " + + "updated_at = CURRENT_TIMESTAMP WHERE namespace_id = ? AND id = ?", + mapper.writeValueAsString(patterns), namespaceId, sourceId); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Could not serialize GitHub ignore patterns", exception); + } + return Optional.of(patterns); + } + + @Override + @Transactional + public boolean ignoredForSource(long sourceId, String skillPath) { + List patterns = jdbc.query("SELECT ignore_patterns::text FROM github_source_binding " + + "WHERE id = ? FOR SHARE", + (result, row) -> parse(result.getString(1)), sourceId) + .stream().findFirst() + .orElseThrow(() -> new IllegalStateException("GitHub source is missing")); + return GithubSkillPathPolicy.ignored(skillPath, patterns); + } + + private List parse(String json) { + try { + return mapper.readValue(json, PATTERNS); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted GitHub ignore patterns", exception); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceTaskRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceTaskRepository.java new file mode 100644 index 000000000..d191d1123 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSourceTaskRepository.java @@ -0,0 +1,211 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.iflytek.skillhub.githubsync.GithubSourceTaskRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSourceTaskRepository implements GithubSourceTaskRepository { + private final JdbcTemplate jdbc; + private final ObjectMapper objectMapper; + private final JdbcGithubSyncCapacityGate capacityGate; + + public JdbcGithubSourceTaskRepository(JdbcTemplate jdbc, ObjectMapper objectMapper, + JdbcGithubSyncCapacityGate capacityGate) { + this.jdbc = jdbc; + this.objectMapper = objectMapper; + this.capacityGate = capacityGate; + } + + @Override + @Transactional + public int enqueueDue(Instant now, int maxSources) { + if (maxSources < 1 || maxSources > 100) { + throw new IllegalArgumentException("GitHub source scheduler batch must be 1 to 100"); + } + List sources = jdbc.query(""" + WITH ranked AS ( + SELECT source.id, + row_number() OVER ( + PARTITION BY source.namespace_id + ORDER BY CASE WHEN COALESCE(source.next_check_at, source.created_at) <= ? + THEN 0 ELSE 1 END, + source.next_check_at NULLS FIRST, source.id) AS namespace_position + FROM github_source_binding source + WHERE source.status IN ('ACTIVE', 'UNREACHABLE') + AND (COALESCE(source.next_check_at, source.created_at) <= ? + OR source.manual_rescan_requested_at IS NOT NULL + OR (source.last_observed_commit IS DISTINCT FROM source.last_fully_processed_commit + AND source.last_observed_commit IS DISTINCT FROM source.last_partial_commit + AND (SELECT task.mode FROM github_sync_task task + WHERE task.source_id = source.id + AND task.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + ORDER BY task.completed_at DESC NULLS LAST, task.id DESC LIMIT 1) + = 'HEAD_ONLY') + OR EXISTS (SELECT 1 FROM github_source_skill_binding binding + WHERE binding.source_id = source.id + AND binding.retry_requested_at IS NOT NULL + AND binding.retry_scheduled_at IS NULL)) + AND NOT EXISTS ( + SELECT 1 FROM github_sync_task task + WHERE task.namespace_id = source.namespace_id + AND task.dedupe_key = 'repository:' || source.repository_id::text + AND task.status IN ('QUEUED', 'LEASED', 'RETRY_WAIT')) + ) + SELECT source.id, source.namespace_id, source.repository_id, + source.owner_login, source.repository_name, + source.ignore_patterns::text AS ignore_patterns, + source.last_observed_commit, source.last_fully_processed_commit, + source.last_partial_commit, + source.manual_rescan_requested_at IS NOT NULL AS manual_rescan_requested, + EXISTS (SELECT 1 FROM github_source_skill_binding binding + WHERE binding.source_id = source.id + AND binding.retry_requested_at IS NOT NULL + AND binding.retry_scheduled_at IS NULL) AS retry_requested, + COALESCE(source.next_check_at, source.created_at) <= ? AS due + FROM github_source_binding source + JOIN ranked ON ranked.id = source.id + ORDER BY ranked.namespace_position, + source.next_check_at NULLS FIRST, source.id + LIMIT ? FOR UPDATE OF source SKIP LOCKED + """, (result, row) -> new Source( + result.getLong("id"), result.getLong("namespace_id"), + result.getLong("repository_id"), result.getString("owner_login"), + result.getString("repository_name"), result.getString("ignore_patterns"), + result.getBoolean("due"), + result.getString("last_observed_commit"), + result.getString("last_fully_processed_commit"), + result.getString("last_partial_commit"), + result.getBoolean("manual_rescan_requested"), + result.getBoolean("retry_requested")), + Timestamp.from(now), Timestamp.from(now), Timestamp.from(now), maxSources); + int queued = 0; + for (Source source : sources) { + boolean full = capacityGate.allowsFullAcquisition(source.namespaceId(), now) + && !capacityGate.hasUnfinishedIngestion(source.namespaceId(), source.repositoryId()) + && (source.manualRescanRequested() || source.retryRequested() + || (!Objects.equals(source.observedCommit(), source.processedCommit()) + && !Objects.equals(source.observedCommit(), source.partialCommit()))); + if (!source.due() && !full) { + continue; + } + UUID runId = UUID.randomUUID(); + UUID taskId = UUID.randomUUID(); + String dedupeKey = "repository:" + source.repositoryId(); + ObjectNode snapshot = objectMapper.createObjectNode(); + snapshot.put("repositoryId", source.repositoryId()); + snapshot.put("owner", source.owner()); + snapshot.put("name", source.name()); + try { + snapshot.set("ignorePatterns", objectMapper.readTree(source.ignorePatterns())); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Invalid persisted GitHub ignore patterns", exception); + } + jdbc.update(""" + INSERT INTO github_sync_run (id, namespace_id, kind, status, created_at) + VALUES (?, ?, 'REPOSITORY_SYNC', 'QUEUED', ?) + """, runId, source.namespaceId(), Timestamp.from(now)); + int inserted = jdbc.update(""" + INSERT INTO github_sync_task + (id, run_id, namespace_id, source_id, kind, mode, status, + dedupe_key, priority, snapshot_json, due_at, created_at, updated_at) + VALUES (?, ?, ?, ?, 'REPOSITORY_SYNC', ?, 'QUEUED', ?, 100, + ?::jsonb, ?, ?, ?) + ON CONFLICT (namespace_id, dedupe_key) + WHERE status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') DO NOTHING + """, taskId, runId, source.namespaceId(), source.id(), + full ? "FULL" : "HEAD_ONLY", dedupeKey, snapshot.toString(), + Timestamp.from(now), Timestamp.from(now), Timestamp.from(now)); + if (inserted == 0) { + jdbc.update("DELETE FROM github_sync_run WHERE id = ?", runId); + continue; + } + if (full) { + jdbc.update(""" + UPDATE github_source_skill_binding + SET retry_scheduled_at = ?, updated_at = ? + WHERE source_id = ? AND retry_requested_at IS NOT NULL + AND retry_scheduled_at IS NULL + """, Timestamp.from(now), Timestamp.from(now), source.id()); + } + queued++; + } + return queued; + } + + @Override + @Transactional + public Optional requestRescan(long namespaceId, long sourceId, Instant now) { + List sources = jdbc.query(""" + SELECT status, next_check_at FROM github_source_binding + WHERE namespace_id = ? AND id = ? FOR UPDATE + """, (result, row) -> new RescanSource(result.getString("status"), + result.getTimestamp("next_check_at") == null ? null + : result.getTimestamp("next_check_at").toInstant()), namespaceId, sourceId); + if (sources.isEmpty()) { + return Optional.empty(); + } + RescanSource source = sources.getFirst(); + if ("PAUSED".equals(source.status())) { + return Optional.of(new RescanRequest("PAUSED", null, null, source.nextCheckAt())); + } + List openTasks = jdbc.query(""" + SELECT id, status, mode FROM github_sync_task + WHERE namespace_id = ? AND source_id = ? + AND status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') + ORDER BY created_at DESC, id DESC LIMIT 1 FOR UPDATE + """, (result, row) -> new OpenTask(result.getObject("id", UUID.class), + result.getString("status"), result.getString("mode")), namespaceId, sourceId); + if (!openTasks.isEmpty()) { + OpenTask task = openTasks.getFirst(); + if ("QUEUED".equals(task.status())) { + jdbc.update(""" + UPDATE github_sync_task + SET priority = GREATEST(priority, 200), due_at = LEAST(due_at, ?), + updated_at = ? + WHERE id = ? AND status = 'QUEUED' + """, Timestamp.from(now), Timestamp.from(now), task.id()); + } + if ("HEAD_ONLY".equals(task.mode()) || "QUEUED".equals(task.status())) { + jdbc.update(""" + UPDATE github_source_binding + SET manual_rescan_requested_at = COALESCE(manual_rescan_requested_at, ?), + updated_at = ? + WHERE namespace_id = ? AND id = ? + """, Timestamp.from(now), Timestamp.from(now), namespaceId, sourceId); + } + return Optional.of(new RescanRequest("MERGED", task.id(), task.status(), source.nextCheckAt())); + } + jdbc.update(""" + UPDATE github_source_binding + SET manual_rescan_requested_at = COALESCE(manual_rescan_requested_at, ?), + next_check_at = LEAST(COALESCE(next_check_at, ?), ?), updated_at = ? + WHERE namespace_id = ? AND id = ? + """, Timestamp.from(now), Timestamp.from(now), Timestamp.from(now), + Timestamp.from(now), namespaceId, sourceId); + return Optional.of(new RescanRequest("SCHEDULED", null, null, + source.nextCheckAt() == null || source.nextCheckAt().isAfter(now) + ? now : source.nextCheckAt())); + } + + private record Source(long id, long namespaceId, long repositoryId, + String owner, String name, String ignorePatterns, boolean due, + String observedCommit, String processedCommit, String partialCommit, + boolean manualRescanRequested, + boolean retryRequested) {} + + private record RescanSource(String status, Instant nextCheckAt) {} + + private record OpenTask(UUID id, String status, String mode) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncAdmissionPolicyRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncAdmissionPolicyRepository.java new file mode 100644 index 000000000..acb986443 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncAdmissionPolicyRepository.java @@ -0,0 +1,78 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicy; +import com.iflytek.skillhub.githubsync.GithubSyncAdmissionPolicyRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Objects; + +/** The single policy row is the same lock used by package admission. */ +@Repository +public class JdbcGithubSyncAdmissionPolicyRepository implements GithubSyncAdmissionPolicyRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncAdmissionPolicyRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public GithubSyncAdmissionPolicy get() { + return Objects.requireNonNull(jdbc.query(""" + SELECT global_high, global_low, namespace_high, namespace_low, + candidate_pending_limit, candidate_per_lease_limit, + global_paused, updated_at + FROM github_sync_admission_policy WHERE id = TRUE + """, result -> result.next() ? map(result) : null), + "GitHub sync admission policy is missing"); + } + + @Override + public GithubSyncAdmissionPolicy getForUpdate() { + return Objects.requireNonNull(jdbc.query(""" + SELECT global_high, global_low, namespace_high, namespace_low, + candidate_pending_limit, candidate_per_lease_limit, + global_paused, updated_at + FROM github_sync_admission_policy WHERE id = TRUE FOR UPDATE + """, result -> result.next() ? map(result) : null), + "GitHub sync admission policy is missing"); + } + + @Override + public GithubSyncAdmissionPolicy update(int globalHigh, int globalLow, + int namespaceHigh, int namespaceLow, + int candidatePendingLimit, int candidatePerLeaseLimit) { + return Objects.requireNonNull(jdbc.query(""" + UPDATE github_sync_admission_policy + SET global_high = ?, global_low = ?, namespace_high = ?, namespace_low = ?, + candidate_pending_limit = ?, candidate_per_lease_limit = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = TRUE + RETURNING global_high, global_low, namespace_high, namespace_low, + candidate_pending_limit, candidate_per_lease_limit, + global_paused, updated_at + """, statement -> { + statement.setInt(1, globalHigh); + statement.setInt(2, globalLow); + statement.setInt(3, namespaceHigh); + statement.setInt(4, namespaceLow); + statement.setInt(5, candidatePendingLimit); + statement.setInt(6, candidatePerLeaseLimit); + }, result -> result.next() ? map(result) : null), + "GitHub sync admission policy is missing"); + } + + private static GithubSyncAdmissionPolicy map(ResultSet result) throws SQLException { + return new GithubSyncAdmissionPolicy( + (Integer) result.getObject("global_high"), + (Integer) result.getObject("global_low"), + (Integer) result.getObject("namespace_high"), + (Integer) result.getObject("namespace_low"), + result.getInt("candidate_pending_limit"), + result.getInt("candidate_per_lease_limit"), + result.getBoolean("global_paused"), + result.getTimestamp("updated_at").toInstant()); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncCapacityGate.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncCapacityGate.java new file mode 100644 index 000000000..e86aec468 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncCapacityGate.java @@ -0,0 +1,79 @@ +package com.iflytek.skillhub.infra.jpa; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +import java.sql.Timestamp; +import java.time.Instant; + +/** Shared unresolved-coordinate count for task admission and pre-acquisition scheduling. */ +@Component +public class JdbcGithubSyncCapacityGate { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncCapacityGate(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public boolean allowsFullAcquisition(long namespaceId, Instant now) { + Policy policy = jdbc.query(""" + SELECT global_high, global_low, namespace_high, namespace_low, global_paused + FROM github_sync_admission_policy WHERE id = TRUE + """, result -> result.next() ? new Policy( + (Integer) result.getObject("global_high"), + (Integer) result.getObject("global_low"), + (Integer) result.getObject("namespace_high"), + (Integer) result.getObject("namespace_low"), + result.getBoolean("global_paused")) : null); + if (policy == null || policy.globalHigh() == null || policy.namespaceHigh() == null) { + return false; + } + Boolean namespacePaused = jdbc.query(""" + SELECT paused FROM github_sync_namespace_admission_state WHERE namespace_id = ? + """, result -> result.next() ? result.getBoolean("paused") : false, namespaceId); + return belowResumeBoundary(activeCount(null, now), policy.globalHigh(), policy.globalLow(), + policy.globalPaused()) + && belowResumeBoundary(activeCount(namespaceId, now), + policy.namespaceHigh(), policy.namespaceLow(), Boolean.TRUE.equals(namespacePaused)); + } + + public int activeCount(Long namespaceId, Instant now) { + String sql = """ + SELECT count(*) FROM ( + SELECT admission.namespace_id, admission.repository_id, admission.skill_path + FROM github_sync_package_admission admission + JOIN github_sync_task task ON task.id = admission.task_id + WHERE admission.status = 'ACTIVE' AND task.status = 'LEASED' + AND task.lease_generation = admission.lease_generation + AND task.lease_expires_at > ? + UNION + SELECT ingestion.namespace_id, ingestion.repository_id, ingestion.skill_path + FROM github_sync_package_ingestion ingestion + WHERE ingestion.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', + 'VALIDATED', 'READY_FOR_REVIEW', 'BLOCKED') + ) active_coordinates + """ + (namespaceId == null ? "" : " WHERE active_coordinates.namespace_id = ?"); + Integer count = namespaceId == null + ? jdbc.queryForObject(sql, Integer.class, Timestamp.from(now)) + : jdbc.queryForObject(sql, Integer.class, Timestamp.from(now), namespaceId); + return count == null ? 0 : count; + } + + /** Pending reviews and conflicts do not prevent checking for a newer upstream revision. */ + public boolean hasUnfinishedIngestion(long namespaceId, long repositoryId) { + Boolean unfinished = jdbc.queryForObject(""" + SELECT EXISTS ( + SELECT 1 FROM github_sync_package_ingestion + WHERE namespace_id = ? AND repository_id = ? + AND status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', 'VALIDATED')) + """, Boolean.class, namespaceId, repositoryId); + return Boolean.TRUE.equals(unfinished); + } + + private boolean belowResumeBoundary(int count, int high, int low, boolean paused) { + return paused ? count <= low : count < high; + } + + private record Policy(Integer globalHigh, Integer globalLow, + Integer namespaceHigh, Integer namespaceLow, boolean globalPaused) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncHistoryRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncHistoryRepository.java new file mode 100644 index 000000000..64b3e8df9 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncHistoryRepository.java @@ -0,0 +1,163 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncHistoryRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@Repository +public class JdbcGithubSyncHistoryRepository implements GithubSyncHistoryRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncHistoryRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @Transactional + public CleanupResult deleteExpiredDiscoveryHistory(Instant cutoff, int batchSize) { + requireBatchSize(batchSize); + // Exclude scheduler GAP-task reuse while cleanup locks tasks and clears ledger FKs. + jdbc.query("SELECT id FROM github_deep_discovery_config WHERE id = TRUE FOR SHARE", result -> {}); + // Retired queries never schedule another root; expire their completed diagnostic trees too. + jdbc.update(""" + WITH picked AS ( + SELECT id FROM github_official_discovery_shard + WHERE parent_id IS NULL AND state = 'COMPLETE' AND completed_at < ? + ORDER BY completed_at, id LIMIT ? FOR UPDATE SKIP LOCKED + ) + DELETE FROM github_official_discovery_shard shard USING picked WHERE shard.id = picked.id + """, Timestamp.from(cutoff), batchSize); + var deletedTasks = jdbc.query(""" + WITH picked AS ( + SELECT task.id FROM github_sync_task task + JOIN github_sync_run run ON run.id = task.run_id + WHERE task.kind = 'DISCOVERY' + AND NOT EXISTS (SELECT 1 FROM github_platform_collection_control platform + WHERE platform.namespace_id=task.namespace_id AND platform.paused + AND platform.version=task.platform_pause_version AND task.platform_stop_ack_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM github_deep_discovery_query progress + WHERE progress.active_task_id = task.id) + AND task.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND task.completed_at < ? + AND run.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND run.finished_at < ? + AND NOT EXISTS (SELECT 1 FROM github_sync_package_ingestion ingestion + WHERE ingestion.task_id = task.id) + AND NOT EXISTS (SELECT 1 FROM github_sync_package_admission admission + WHERE admission.task_id = task.id) + AND NOT EXISTS (SELECT 1 FROM github_sync_path_result path + WHERE path.task_id = task.id) + AND NOT EXISTS (SELECT 1 FROM github_discovery_candidate candidate + WHERE candidate.validation_task_id = task.id) + ORDER BY task.completed_at, task.id + LIMIT ? FOR UPDATE OF task SKIP LOCKED + ) + DELETE FROM github_sync_task task USING picked + WHERE task.id = picked.id RETURNING task.id + """, (result, row) -> result.getObject("id"), + Timestamp.from(cutoff), Timestamp.from(cutoff), batchSize).size(); + var deletedRuns = jdbc.query(""" + WITH picked AS ( + SELECT run.id FROM github_sync_run run + WHERE run.kind = 'DISCOVERY' + AND run.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND run.finished_at < ? + AND NOT EXISTS (SELECT 1 FROM github_sync_task task WHERE task.run_id = run.id) + ORDER BY run.finished_at, run.id + LIMIT ? FOR UPDATE OF run SKIP LOCKED + ) + DELETE FROM github_sync_run run USING picked + WHERE run.id = picked.id RETURNING run.id + """, (result, row) -> result.getObject("id"), + Timestamp.from(cutoff), batchSize).size(); + return new CleanupResult(deletedTasks, deletedRuns); + } + + @Override + @Transactional + public CleanupResult purgeExpiredRepositoryDetails(Instant cutoff, Instant now, int batchSize) { + requireBatchSize(batchSize); + List taskIds = jdbc.query(""" + SELECT task.id FROM github_sync_task task + JOIN github_sync_run run ON run.id = task.run_id + WHERE task.kind = 'REPOSITORY_SYNC' + AND task.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND task.completed_at < ? AND task.details_purged_at IS NULL + AND run.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND run.finished_at < ? + AND NOT EXISTS ( + SELECT 1 FROM github_sync_package_ingestion ingestion + WHERE ingestion.task_id = task.id + AND ingestion.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', + 'VALIDATED', 'READY_FOR_REVIEW', 'BLOCKED')) + AND NOT EXISTS ( + SELECT 1 FROM github_discovery_candidate candidate + WHERE candidate.validation_task_id = task.id + AND candidate.status IN ('VALIDATING', 'UNREACHABLE')) + ORDER BY task.completed_at, task.id + LIMIT ? FOR UPDATE OF task SKIP LOCKED + """, (result, row) -> result.getObject("id", UUID.class), + Timestamp.from(cutoff), Timestamp.from(cutoff), batchSize); + if (!taskIds.isEmpty()) { + String placeholders = String.join(",", Collections.nCopies(taskIds.size(), "?")); + Object[] ids = taskIds.toArray(); + jdbc.update(""" + DELETE FROM github_sync_path_result path + USING github_sync_task task + WHERE path.task_id = task.id AND task.id IN (%s) + AND (task.source_id IS NULL OR EXISTS ( + SELECT 1 FROM github_sync_path_result newer_path + JOIN github_sync_task newer_task ON newer_task.id = newer_path.task_id + WHERE newer_task.source_id = task.source_id + AND newer_path.skill_path = path.skill_path + AND (newer_task.created_at, newer_task.id, + newer_path.updated_at, newer_path.id) + > (task.created_at, task.id, path.updated_at, path.id))) + """.formatted(placeholders), ids); + Object[] updateArgs = new Object[taskIds.size() + 1]; + updateArgs[0] = Timestamp.from(now); + System.arraycopy(ids, 0, updateArgs, 1, ids.length); + jdbc.update(""" + UPDATE github_sync_task + SET snapshot_json = jsonb_strip_nulls(jsonb_build_object( + 'repositoryId', snapshot_json -> 'repositoryId', + 'owner', snapshot_json -> 'owner', + 'name', snapshot_json -> 'name')), + path_checkpoint = NULL, discovery_checkpoint = NULL, + incomplete_subtrees = '[]'::jsonb, last_error_detail = NULL, + credential_id = NULL, credential_fingerprint = NULL, + details_purged_at = ? + WHERE id IN (%s) + """.formatted(placeholders), updateArgs); + } + int runs = jdbc.update(""" + WITH picked AS ( + SELECT run.id FROM github_sync_run run + WHERE run.kind = 'REPOSITORY_SYNC' + AND run.status IN ('ACQUISITION_COMPLETE', 'PARTIAL', 'FAILED') + AND run.finished_at < ? AND run.details_purged_at IS NULL + AND NOT EXISTS (SELECT 1 FROM github_sync_task task + WHERE task.run_id = run.id AND task.details_purged_at IS NULL) + ORDER BY run.finished_at, run.id + LIMIT ? FOR UPDATE OF run SKIP LOCKED + ) + UPDATE github_sync_run run + SET details_purged_at = ? + FROM picked WHERE run.id = picked.id + """, Timestamp.from(cutoff), batchSize, Timestamp.from(now)); + return new CleanupResult(taskIds.size(), runs); + } + + private void requireBatchSize(int batchSize) { + if (batchSize < 1 || batchSize > 1000) { + throw new IllegalArgumentException("GitHub history cleanup batch must be 1 to 1000"); + } + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncIngestionRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncIngestionRepository.java new file mode 100644 index 000000000..5ad01c466 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncIngestionRepository.java @@ -0,0 +1,293 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncIngestionRepository; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicyRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSyncIngestionRepository implements GithubSyncIngestionRepository { + private final JdbcTemplate jdbc; + private final JdbcGithubDiscoveryOutcomeRepository outcomes; + private final JdbcGithubSyncSourceCheckpointRepository checkpoints; + private final GithubSyncSchedulePolicyRepository schedulePolicies; + private final Duration repositoryCheckInterval; + + public JdbcGithubSyncIngestionRepository(JdbcTemplate jdbc, + JdbcGithubSyncSourceCheckpointRepository checkpoints, + GithubSyncSchedulePolicyRepository schedulePolicies, + @Value("${skillhub.github-sync.repository-check-interval:PT24H}") + Duration repositoryCheckInterval) { + this.jdbc = jdbc; + this.outcomes = new JdbcGithubDiscoveryOutcomeRepository(jdbc); + this.checkpoints = checkpoints; + this.schedulePolicies = schedulePolicies; + if (repositoryCheckInterval.isZero() || repositoryCheckInterval.isNegative()) { + throw new IllegalArgumentException("GitHub repository check interval must be positive"); + } + this.repositoryCheckInterval = repositoryCheckInterval; + } + + @Override + @Transactional + public Optional claim(UUID validationToken, Instant now, Instant expiresAt) { + return jdbc.query(""" + WITH picked AS ( + SELECT id FROM github_sync_package_ingestion + WHERE status = 'RECEIVED' + OR (status = 'RETRY_WAIT' AND retry_at <= ?) + OR (status = 'VALIDATING' AND validation_expires_at <= ?) + ORDER BY created_at, id + LIMIT 1 FOR UPDATE SKIP LOCKED + ) + UPDATE github_sync_package_ingestion AS ingestion + SET status = 'VALIDATING', validation_token = ?, validation_expires_at = ?, + validation_attempts = ingestion.validation_attempts + 1, + retry_at = NULL, last_error_code = NULL, updated_at = ? + FROM picked WHERE ingestion.id = picked.id + RETURNING ingestion.id, ingestion.object_key, ingestion.size_bytes, + ingestion.sha256, ingestion.skill_path, ingestion.task_id, + ingestion.validation_attempts + """, statement -> { + statement.setTimestamp(1, Timestamp.from(now)); + statement.setTimestamp(2, Timestamp.from(now)); + statement.setObject(3, validationToken); + statement.setTimestamp(4, Timestamp.from(expiresAt)); + statement.setTimestamp(5, Timestamp.from(now)); + }, result -> result.next() ? Optional.of(new ClaimedIngestion( + result.getObject("id", UUID.class), validationToken, + result.getString("object_key"), result.getLong("size_bytes"), + result.getString("sha256"), result.getString("skill_path"), + jdbc.queryForObject("SELECT source_id FROM github_sync_task WHERE id = ?", + Long.class, result.getObject("task_id", UUID.class)), + result.getInt("validation_attempts"))) : Optional.empty()); + } + + @Override + @Transactional + public boolean validated(UUID ingestionId, UUID validationToken, String contentSha256, + int warningCount, String evidenceJson, Instant now) { + lockTaskForIngestion(ingestionId); + int updated = jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'VALIDATED', validation_token = NULL, validation_expires_at = NULL, + validated_at = ?, content_sha256 = ?, validation_warning_count = ?, + license_evidence_json = ?::jsonb, + last_error_code = NULL, retry_at = NULL, updated_at = ? + WHERE id = ? AND status = 'VALIDATING' AND validation_token = ? + AND validation_expires_at > ? + """, Timestamp.from(now), contentSha256, warningCount, evidenceJson, + Timestamp.from(now), ingestionId, validationToken, Timestamp.from(now)); + if (updated != 1) { + return false; + } + jdbc.update(""" + UPDATE github_sync_path_result path + SET outcome = 'ACCEPTED', reason = NULL, retry_at = NULL, updated_at = ? + FROM github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND path.task_id = ingestion.task_id + AND path.commit_sha = ingestion.commit_sha AND path.skill_path = ingestion.skill_path + """, Timestamp.from(now), ingestionId); + bindValidatedCandidate(ingestionId, now); + jdbc.query("SELECT namespace_id, repository_id FROM github_sync_package_ingestion WHERE id = ?", + result -> { + if (result.next()) { + outcomes.firstValid(result.getLong("namespace_id"), result.getLong("repository_id"), now); + } + return null; + }, ingestionId); + jdbc.update(""" + INSERT INTO github_official_validated_seed + (namespace_id, repository_id, github_owner_id, owner_login, ingestion_id, validated_at) + SELECT ingestion.namespace_id, ingestion.repository_id, + (task.snapshot_json ->> 'ownerId')::bigint, task.snapshot_json ->> 'owner', ingestion.id, ? + FROM github_sync_package_ingestion ingestion JOIN github_sync_task task ON task.id = ingestion.task_id + JOIN github_deep_discovery_config config ON config.namespace_id = ingestion.namespace_id + WHERE ingestion.id = ? AND (task.snapshot_json ->> 'ownerId')::bigint > 0 + ON CONFLICT (namespace_id, repository_id, github_owner_id) DO UPDATE + SET owner_login = EXCLUDED.owner_login + """, Timestamp.from(now), ingestionId); + jdbc.update(""" + UPDATE github_source_binding source SET github_owner_id = (task.snapshot_json ->> 'ownerId')::bigint + FROM github_sync_package_ingestion ingestion JOIN github_sync_task task ON task.id = ingestion.task_id + WHERE ingestion.id = ? AND source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id AND task.snapshot_json ->> 'ownerId' IS NOT NULL + """, ingestionId); + jdbc.query(""" + SELECT task.id, ingestion.commit_sha + FROM github_sync_package_ingestion ingestion + JOIN github_sync_task task ON task.id = ingestion.task_id + WHERE ingestion.id = ? + """, result -> { + if (result.next()) { + checkpoints.advanceIfComplete(result.getObject("id", UUID.class), + result.getString("commit_sha"), now); + } + return null; + }, ingestionId); + return true; + } + + private void lockTaskForIngestion(UUID ingestionId) { + // Repository pause/completion takes task before source/path locks. Validation must use the same + // order so candidate-to-source promotion cannot deadlock an in-flight pause. + jdbc.queryForList(""" + SELECT task.id FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + WHERE ingestion.id = ? FOR UPDATE OF task + """, UUID.class, ingestionId); + } + + /** Search hits become sources only after a real Skill package passes Java validation. */ + private void bindValidatedCandidate(UUID ingestionId, Instant now) { + Long sourceId = jdbc.query(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, + default_branch, visibility, last_observed_commit, next_check_at) + SELECT candidate.namespace_id, candidate.repository_id, + candidate.owner_login, candidate.repository_name, + candidate.default_branch, + CASE WHEN namespace.type = 'CATALOG' THEN 'PUBLIC' + ELSE 'NAMESPACE_ONLY' END, + ingestion.commit_sha, ? + FROM github_sync_package_ingestion ingestion + JOIN github_discovery_candidate candidate + ON candidate.namespace_id = ingestion.namespace_id + AND candidate.repository_id = ingestion.repository_id + JOIN namespace ON namespace.id = candidate.namespace_id + WHERE ingestion.id = ? AND candidate.status IN ('PENDING', 'VALIDATING', 'BOUND', 'UNREACHABLE') + ON CONFLICT (namespace_id, repository_id) DO UPDATE + SET owner_login = EXCLUDED.owner_login, + repository_name = EXCLUDED.repository_name, + default_branch = COALESCE(EXCLUDED.default_branch, + github_source_binding.default_branch), + updated_at = CURRENT_TIMESTAMP + RETURNING id + """, statement -> { + Integer hours = schedulePolicies.get().repositoryHours(); + statement.setTimestamp(1, Timestamp.from(now.plus( + hours == null ? repositoryCheckInterval : Duration.ofHours(hours)))); + statement.setObject(2, ingestionId); + }, result -> result.next() ? result.getLong("id") : null); + if (sourceId == null) { + return; + } + jdbc.update(""" + UPDATE github_discovery_candidate candidate + SET status = 'BOUND', unreachable_attempts = 0, + last_error_code = NULL, last_seen_at = ? + FROM github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND candidate.namespace_id = ingestion.namespace_id + AND candidate.repository_id = ingestion.repository_id + """, Timestamp.from(now), ingestionId); + jdbc.update(""" + UPDATE github_sync_task task + SET source_id = ?, updated_at = ? + FROM github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND task.id = ingestion.task_id AND task.source_id IS NULL + """, sourceId, Timestamp.from(now), ingestionId); + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + SELECT ?, origin.origin_type, origin.origin_key + FROM github_discovery_candidate_origin origin + JOIN github_discovery_candidate candidate ON candidate.id = origin.candidate_id + JOIN github_sync_package_ingestion ingestion ON ingestion.id = ? + WHERE candidate.namespace_id = ingestion.namespace_id + AND candidate.repository_id = ingestion.repository_id + ON CONFLICT (source_id, origin_type, origin_key) DO NOTHING + """, sourceId, ingestionId); + } + + @Override + @Transactional + public boolean failed(UUID ingestionId, UUID validationToken, String reason, + Instant retryAt, String pathOutcome, Instant now) { + if (pathOutcome != null && !pathOutcome.equals("RETRYABLE_FAILURE") + && !pathOutcome.equals("TERMINAL_FAILURE") && !pathOutcome.equals("IGNORED")) { + throw new IllegalArgumentException("invalid GitHub sync path failure outcome"); + } + lockTaskForIngestion(ingestionId); + String status = retryAt == null ? "FAILED" : "RETRY_WAIT"; + int updated = jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = ?, validation_token = NULL, validation_expires_at = NULL, + last_error_code = ?, retry_at = ?, updated_at = ? + WHERE id = ? AND status = 'VALIDATING' AND validation_token = ? + AND validation_expires_at > ? + """, status, reason, retryAt == null ? null : Timestamp.from(retryAt), + Timestamp.from(now), ingestionId, validationToken, Timestamp.from(now)); + if (updated != 1) { + return false; + } + if (pathOutcome != null) { + jdbc.update(""" + UPDATE github_sync_path_result path + SET outcome = ?, reason = ?, retry_at = ?, updated_at = ? + FROM github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND path.task_id = ingestion.task_id + AND path.commit_sha = ingestion.commit_sha AND path.skill_path = ingestion.skill_path + """, pathOutcome, reason, retryAt == null ? null : Timestamp.from(retryAt), + Timestamp.from(now), ingestionId); + jdbc.update(""" + UPDATE github_source_binding source + SET last_fully_processed_commit = NULL, updated_at = ? + FROM github_sync_task task, github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND task.id = ingestion.task_id AND source.id = task.source_id + AND source.last_fully_processed_commit = ingestion.commit_sha + """, Timestamp.from(now), ingestionId); + } + return true; + } + + @Override + @Transactional + public List claimCleanup(Instant now, Duration signedUrlGrace, int limit) { + if (limit < 1 || limit > 100) { + throw new IllegalArgumentException("cleanup batch limit must be between 1 and 100"); + } + return jdbc.query(""" + WITH picked AS ( + SELECT ingestion.id FROM github_sync_package_ingestion ingestion + JOIN github_sync_staged_upload upload ON upload.id = ingestion.upload_id + WHERE ingestion.status IN ('FAILED', 'NO_CHANGE', 'READY_FOR_REVIEW', + 'APPROVED', 'REJECTED', 'SUPERSEDED') + AND ingestion.object_cleaned_at IS NULL + AND (ingestion.object_cleanup_claimed_at IS NULL + OR ingestion.object_cleanup_claimed_at < ?) + AND (upload.mode = 'LOCAL_STREAM' OR upload.expires_at < ?) + ORDER BY ingestion.updated_at, ingestion.id + LIMIT ? FOR UPDATE OF ingestion SKIP LOCKED + ) + UPDATE github_sync_package_ingestion ingestion + SET object_cleanup_claimed_at = ? + FROM picked WHERE ingestion.id = picked.id + RETURNING ingestion.id, ingestion.object_key + """, statement -> { + statement.setTimestamp(1, Timestamp.from(now.minus(Duration.ofMinutes(15)))); + statement.setTimestamp(2, Timestamp.from(now.minus(signedUrlGrace))); + statement.setInt(3, limit); + statement.setTimestamp(4, Timestamp.from(now)); + }, (result, row) -> new CleanupIngestion(result.getObject("id", UUID.class), + result.getString("object_key"))); + } + + @Override + public void markCleaned(UUID ingestionId, Instant now) { + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET object_cleaned_at = ?, object_cleanup_claimed_at = NULL + WHERE id = ? AND status IN ('FAILED', 'NO_CHANGE', 'READY_FOR_REVIEW', + 'APPROVED', 'REJECTED', 'SUPERSEDED') + AND object_cleaned_at IS NULL + """, Timestamp.from(now), ingestionId); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncManualBindingRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncManualBindingRepository.java new file mode 100644 index 000000000..e22407be3 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncManualBindingRepository.java @@ -0,0 +1,210 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; +import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +/** Locks the blocked ingestion and target identity while reserving a namespace-local binding. */ +@Repository +public class JdbcGithubSyncManualBindingRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncManualBindingRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public record Binding(UUID ingestionId, long sourceId, long namespaceId, long repositoryId, + String skillPath, String skillSlug, long skillId, String previousPath) {} + + @Transactional(propagation = Propagation.MANDATORY) + public Binding bind(long namespaceId, UUID ingestionId, long targetSkillId) { + Conflict conflict = jdbc.query(""" + SELECT ingestion.namespace_id, ingestion.repository_id, ingestion.task_id, + ingestion.commit_sha, ingestion.content_sha256, ingestion.skill_path, + ingestion.target_skill_slug, ingestion.status, ingestion.last_error_code, + source.id AS source_id + FROM github_sync_package_ingestion ingestion + JOIN github_source_binding source + ON source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id + WHERE ingestion.id = ? AND ingestion.namespace_id = ? + FOR UPDATE OF ingestion + """, result -> result.next() ? new Conflict( + result.getLong("namespace_id"), result.getLong("repository_id"), + result.getObject("task_id", UUID.class), result.getString("commit_sha"), + result.getString("content_sha256"), result.getString("skill_path"), + result.getString("target_skill_slug"), + result.getString("status"), result.getString("last_error_code"), + result.getLong("source_id")) : null, ingestionId, namespaceId); + if (conflict == null) { + throw new DomainNotFoundException("error.githubSync.binding.conflictNotFound"); + } + if ("BLOCKED".equals(conflict.status()) && "PATH_MOVE_AMBIGUOUS".equals(conflict.errorCode())) { + return bindPathMove(conflict, ingestionId, targetSkillId); + } + if (!"BLOCKED".equals(conflict.status()) || !"SLUG_CONFLICT".equals(conflict.errorCode()) + || conflict.skillSlug() == null) { + throw new DomainBadRequestException("error.githubSync.binding.notResolvable"); + } + Target target = jdbc.query(""" + SELECT id, slug, status, github_sync_managed FROM skill + WHERE id = ? AND namespace_id = ? FOR UPDATE + """, result -> result.next() ? new Target( + result.getLong("id"), result.getString("slug"), + result.getString("status"), result.getBoolean("github_sync_managed")) + : null, targetSkillId, namespaceId); + if (target == null || !target.slug().equals(conflict.skillSlug()) + || !"ACTIVE".equals(target.status()) || target.syncManaged()) { + throw new DomainBadRequestException("error.githubSync.binding.invalidTarget"); + } + Boolean pendingUserReview = jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM review_task + WHERE skill_id = ? AND status = 'PENDING' AND origin_type = 'USER') + """, Boolean.class, targetSkillId); + if (Boolean.TRUE.equals(pendingUserReview)) { + throw new DomainBadRequestException("error.githubSync.binding.pendingReview"); + } + Long boundSource = jdbc.query(""" + SELECT source_id FROM github_source_skill_binding + WHERE namespace_id = ? AND skill_id = ? FOR UPDATE + """, result -> result.next() ? result.getLong("source_id") : null, + namespaceId, targetSkillId); + if (boundSource != null) { + throw new DomainBadRequestException("error.githubSync.binding.alreadyBound"); + } + Long boundPath = jdbc.query(""" + SELECT id FROM github_source_skill_binding + WHERE namespace_id = ? AND repository_id = ? AND skill_path = ? FOR UPDATE + """, result -> result.next() ? result.getLong("id") : null, + namespaceId, conflict.repositoryId(), conflict.skillPath()); + if (boundPath != null) { + throw new DomainBadRequestException("error.githubSync.binding.alreadyBound"); + } + try { + jdbc.update(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id, + latest_content_sha256, latest_ingestion_id) + VALUES (?, ?, ?, ?, ?, NULL, NULL) + """, conflict.sourceId(), namespaceId, conflict.repositoryId(), + conflict.skillPath(), targetSkillId); + } catch (DataIntegrityViolationException exception) { + throw new DomainBadRequestException("error.githubSync.binding.alreadyBound"); + } + int resumed = jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'VALIDATED', last_error_code = NULL, target_skill_slug = NULL, + publication_retry_at = NULL, updated_at = ? + WHERE id = ? AND status = 'BLOCKED' AND last_error_code = 'SLUG_CONFLICT' + """, Timestamp.from(Instant.now()), ingestionId); + if (resumed != 1) { + throw new IllegalStateException("GitHub sync conflict changed while binding"); + } + return new Binding(ingestionId, conflict.sourceId(), namespaceId, + conflict.repositoryId(), conflict.skillPath(), conflict.skillSlug(), targetSkillId, null); + } + + private Binding bindPathMove(Conflict conflict, UUID ingestionId, long targetSkillId) { + Long lockedSource = jdbc.query(""" + SELECT id FROM github_source_binding WHERE id = ? AND namespace_id = ? + AND repository_id = ? FOR UPDATE + """, result -> result.next() ? result.getLong("id") : null, + conflict.sourceId(), conflict.namespaceId(), conflict.repositoryId()); + if (lockedSource == null) { + throw new DomainBadRequestException("error.githubSync.binding.notResolvable"); + } + Boolean complete = jdbc.query(""" + SELECT task.mode = 'FULL' AND task.status = 'ACQUISITION_COMPLETE' + AND task.fixed_commit_sha = ? + AND COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns + AND NOT EXISTS (SELECT 1 FROM github_sync_task newer + WHERE newer.source_id = task.source_id + AND newer.kind = 'REPOSITORY_SYNC' AND newer.mode = 'FULL' + AND (newer.created_at, newer.id) > (task.created_at, task.id)) + AS complete + FROM github_sync_task task + JOIN github_source_binding source ON source.id = task.source_id + WHERE task.id = ? AND task.source_id = ? + AND task.namespace_id = ? + """, result -> result.next() && result.getBoolean("complete"), + conflict.commitSha(), conflict.taskId(), conflict.sourceId(), conflict.namespaceId()); + Boolean currentAccepted = jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND skill_path = ? + AND outcome = 'ACCEPTED' AND ingestion_id = ?) + """, Boolean.class, conflict.taskId(), conflict.commitSha(), conflict.skillPath(), ingestionId); + if (!Boolean.TRUE.equals(complete) || !Boolean.TRUE.equals(currentAccepted)) { + throw new DomainBadRequestException("error.githubSync.binding.notResolvable"); + } + PathTarget target = jdbc.query(""" + SELECT binding.skill_path, skill.slug + FROM github_source_skill_binding binding + JOIN skill ON skill.id = binding.skill_id + WHERE binding.source_id = ? AND binding.namespace_id = ? + AND binding.repository_id = ? AND binding.skill_id = ? + AND binding.skill_path <> ? AND binding.latest_content_sha256 IS NOT NULL + AND skill.namespace_id = ? AND skill.status = 'ACTIVE' + AND skill.github_sync_managed = TRUE + FOR UPDATE OF binding, skill + """, result -> result.next() ? new PathTarget(result.getString("skill_path"), + result.getString("slug")) : null, + conflict.sourceId(), conflict.namespaceId(), conflict.repositoryId(), + targetSkillId, conflict.skillPath(), conflict.namespaceId()); + if (target == null) { + throw new DomainBadRequestException("error.githubSync.binding.invalidTarget"); + } + Boolean oldPathPresent = jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND skill_path = ?) + """, Boolean.class, conflict.taskId(), conflict.commitSha(), target.previousPath()); + if (Boolean.TRUE.equals(oldPathPresent)) { + throw new DomainBadRequestException("error.githubSync.binding.invalidTarget"); + } + int moved = jdbc.update(""" + UPDATE github_source_skill_binding + SET skill_path = ?, upstream_presence = 'PRESENT', + presence_commit_sha = ?, missing_since = NULL, + latest_ingestion_id = CASE WHEN latest_content_sha256 = ? + THEN ? ELSE latest_ingestion_id END, + updated_at = ? + WHERE source_id = ? AND namespace_id = ? AND repository_id = ? + AND skill_path = ? AND skill_id = ? + AND NOT EXISTS (SELECT 1 FROM github_source_skill_binding occupied + WHERE occupied.namespace_id = ? AND occupied.repository_id = ? + AND occupied.skill_path = ?) + """, conflict.skillPath(), conflict.commitSha(), conflict.contentSha256(), ingestionId, + Timestamp.from(Instant.now()), conflict.sourceId(), conflict.namespaceId(), + conflict.repositoryId(), target.previousPath(), targetSkillId, + conflict.namespaceId(), conflict.repositoryId(), conflict.skillPath()); + if (moved != 1) { + throw new DomainBadRequestException("error.githubSync.binding.alreadyBound"); + } + int resumed = jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'VALIDATED', last_error_code = NULL, + publication_retry_at = NULL, updated_at = ? + WHERE id = ? AND status = 'BLOCKED' AND last_error_code = 'PATH_MOVE_AMBIGUOUS' + """, Timestamp.from(Instant.now()), ingestionId); + if (resumed != 1) { + throw new IllegalStateException("GitHub sync path conflict changed while binding"); + } + return new Binding(ingestionId, conflict.sourceId(), conflict.namespaceId(), + conflict.repositoryId(), conflict.skillPath(), target.skillSlug(), + targetSkillId, target.previousPath()); + } + + private record Conflict(long namespaceId, long repositoryId, UUID taskId, String commitSha, + String contentSha256, String skillPath, String skillSlug, + String status, String errorCode, long sourceId) {} + private record Target(long id, String slug, String status, boolean syncManaged) {} + private record PathTarget(String previousPath, String skillSlug) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPathPresenceRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPathPresenceRepository.java new file mode 100644 index 000000000..58f058f1d --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPathPresenceRepository.java @@ -0,0 +1,96 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSkillPathPolicy; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +/** Reconciles source path presence only from a complete, current-rule repository snapshot. */ +@Repository +public class JdbcGithubSyncPathPresenceRepository { + private final JdbcTemplate jdbc; + private final JdbcGithubSourceIgnoreRepository ignores; + + public JdbcGithubSyncPathPresenceRepository(JdbcTemplate jdbc, JdbcGithubSourceIgnoreRepository ignores) { + this.jdbc = jdbc; + this.ignores = ignores; + } + + @Transactional(propagation = Propagation.MANDATORY) + public void reconcile(long sourceId, UUID taskId, String commitSha, Instant now) { + Snapshot snapshot = jdbc.query(""" + SELECT source.namespace_id, source.status, + COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns AS rules_match + FROM github_source_binding source + JOIN github_sync_task task ON task.source_id = source.id + WHERE source.id = ? AND task.id = ? AND task.kind = 'REPOSITORY_SYNC' + AND task.mode = 'FULL' AND task.status = 'ACQUISITION_COMPLETE' + AND task.fixed_commit_sha = ? + AND NOT EXISTS (SELECT 1 FROM github_sync_task newer + WHERE newer.source_id = task.source_id + AND newer.kind = 'REPOSITORY_SYNC' AND newer.mode = 'FULL' + AND (newer.created_at, newer.id) > (task.created_at, task.id)) + """, result -> result.next() ? new Snapshot(result.getLong("namespace_id"), + result.getString("status"), result.getBoolean("rules_match")) : null, + sourceId, taskId, commitSha); + if (snapshot == null || "PAUSED".equals(snapshot.status()) || !snapshot.rulesMatch()) { + return; + } + List patterns = ignores.find(snapshot.namespaceId(), sourceId) + .orElseThrow(() -> new IllegalStateException("GitHub source disappeared during path reconciliation")); + jdbc.update(""" + UPDATE github_source_skill_binding binding + SET upstream_presence = 'PRESENT', presence_commit_sha = ?, + missing_since = NULL, updated_at = ? + WHERE binding.source_id = ? AND EXISTS ( + SELECT 1 FROM github_sync_path_result path + WHERE path.task_id = ? AND path.commit_sha = ? + AND path.skill_path = binding.skill_path) + """, commitSha, Timestamp.from(now), sourceId, taskId, commitSha); + List absent = jdbc.query(""" + SELECT binding.id, binding.skill_path + FROM github_source_skill_binding binding + WHERE binding.source_id = ? AND NOT EXISTS ( + SELECT 1 FROM github_sync_path_result path + WHERE path.task_id = ? AND path.commit_sha = ? + AND path.skill_path = binding.skill_path) + """, (result, row) -> new PathBinding(result.getLong("id"), + result.getString("skill_path")), sourceId, taskId, commitSha); + List missing = absent.stream() + .filter(binding -> !GithubSkillPathPolicy.ignored(binding.skillPath(), patterns)) + .toList(); + jdbc.batchUpdate(""" + UPDATE github_source_skill_binding + SET upstream_presence = 'MISSING', presence_commit_sha = ?, + missing_since = COALESCE(missing_since, ?), updated_at = ? + WHERE id = ? AND source_id = ? + """, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement statement, int index) throws SQLException { + statement.setString(1, commitSha); + statement.setTimestamp(2, Timestamp.from(now)); + statement.setTimestamp(3, Timestamp.from(now)); + statement.setLong(4, missing.get(index).id()); + statement.setLong(5, sourceId); + } + + @Override + public int getBatchSize() { + return missing.size(); + } + }); + } + + private record Snapshot(long namespaceId, String status, boolean rulesMatch) {} + private record PathBinding(long id, String skillPath) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPublicationRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPublicationRepository.java new file mode 100644 index 000000000..38907c70e --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncPublicationRepository.java @@ -0,0 +1,517 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncPublicationRepository; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSyncPublicationRepository implements GithubSyncPublicationRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncPublicationRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @Transactional + public Optional claim(UUID token, Instant now, Instant expiresAt) { + UUID id = jdbc.query(""" + WITH picked AS ( + SELECT ingestion.id + FROM github_sync_package_ingestion ingestion + WHERE ingestion.status = 'VALIDATED' + AND (ingestion.publication_retry_at IS NULL OR ingestion.publication_retry_at <= ?) + AND (ingestion.publication_token IS NULL OR ingestion.publication_expires_at <= ?) + ORDER BY ingestion.validated_at, ingestion.id + LIMIT 1 FOR UPDATE OF ingestion SKIP LOCKED + ) + UPDATE github_sync_package_ingestion ingestion + SET publication_token = ?, publication_expires_at = ?, + publication_attempts = ingestion.publication_attempts + 1, + publication_retry_at = NULL, updated_at = ? + FROM picked WHERE ingestion.id = picked.id + RETURNING ingestion.id + """, statement -> { + statement.setTimestamp(1, Timestamp.from(now)); + statement.setTimestamp(2, Timestamp.from(now)); + statement.setObject(3, token); + statement.setTimestamp(4, Timestamp.from(expiresAt)); + statement.setTimestamp(5, Timestamp.from(now)); + }, result -> result.next() ? result.getObject("id", UUID.class) : null); + if (id == null) { + return Optional.empty(); + } + return jdbc.query(""" + SELECT ingestion.id, ingestion.namespace_id, namespace.slug AS namespace_slug, + ingestion.repository_id, source.id AS source_id, + source.status AS source_status, ingestion.skill_path, + COALESCE(source.visibility, + CASE WHEN namespace.type = 'CATALOG' THEN 'PUBLIC' + ELSE 'NAMESPACE_ONLY' END) AS visibility, + ingestion.object_key, ingestion.size_bytes, + ingestion.sha256 AS archive_sha256, ingestion.content_sha256, + binding.skill_id AS expected_skill_id, + ingestion.publication_attempts + FROM github_sync_package_ingestion ingestion + JOIN namespace ON namespace.id = ingestion.namespace_id + LEFT JOIN github_source_binding source + ON source.namespace_id = ingestion.namespace_id + AND source.repository_id = ingestion.repository_id + LEFT JOIN github_source_skill_binding binding + ON binding.namespace_id = ingestion.namespace_id + AND binding.repository_id = ingestion.repository_id + AND binding.skill_path = ingestion.skill_path + WHERE ingestion.id = ? + """, result -> result.next() ? Optional.of(new ClaimedPublication( + result.getObject("id", UUID.class), token, + result.getLong("namespace_id"), result.getString("namespace_slug"), + result.getLong("repository_id"), result.getObject("source_id", Long.class), + result.getString("skill_path"), result.getString("source_status"), + result.getString("visibility"), + result.getString("object_key"), result.getLong("size_bytes"), + result.getString("archive_sha256"), result.getString("content_sha256"), + result.getObject("expected_skill_id", Long.class), + result.getInt("publication_attempts"))) : Optional.empty(), id); + } + + @Override + @Transactional + public boolean requestRejectedRetry(long namespaceId, long sourceId, String skillPath, Instant now) { + int updated = jdbc.update(""" + UPDATE github_source_skill_binding binding + SET retry_requested_at = ?, + retry_scheduled_at = NULL, + updated_at = ? + FROM github_sync_package_ingestion ingestion + WHERE binding.namespace_id = ? AND binding.source_id = ? + AND binding.skill_path = ? + AND ingestion.id = binding.latest_ingestion_id + AND ingestion.status = 'REJECTED' + AND (binding.retry_requested_at IS NULL + OR (binding.retry_scheduled_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM github_sync_task task + WHERE task.source_id = binding.source_id + AND task.mode = 'FULL' + AND task.status IN ('QUEUED', 'LEASED', 'RETRY_WAIT') + AND task.updated_at >= binding.retry_scheduled_at) + AND NOT EXISTS (SELECT 1 FROM github_sync_package_ingestion pending + WHERE pending.namespace_id = binding.namespace_id + AND pending.repository_id = binding.repository_id + AND pending.skill_path = binding.skill_path + AND pending.created_at >= binding.retry_requested_at + AND pending.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT', + 'VALIDATED', 'READY_FOR_REVIEW'))) + ) + """, Timestamp.from(now), Timestamp.from(now), namespaceId, sourceId, skillPath); + if (updated == 1) { + int queued = jdbc.update(""" + UPDATE github_sync_task + SET mode = 'FULL', updated_at = ? + WHERE namespace_id = ? AND source_id = ? + AND kind = 'REPOSITORY_SYNC' AND status = 'QUEUED' + """, Timestamp.from(now), namespaceId, sourceId); + if (queued > 0) { + jdbc.update(""" + UPDATE github_source_skill_binding + SET retry_scheduled_at = ?, updated_at = ? + WHERE namespace_id = ? AND source_id = ? AND skill_path = ? + """, Timestamp.from(now), Timestamp.from(now), + namespaceId, sourceId, skillPath); + } + } + return updated == 1; + } + + @Override + @Transactional + public boolean lockAndVerify(ClaimedPublication claim, Instant now) { + // Serializes only one Skill path. A late worker must not restore an older target. + jdbc.query("SELECT pg_advisory_xact_lock(hashtext(?), hashtext(?))", + result -> { result.next(); return null; }, + claim.namespaceId() + ":" + claim.repositoryId(), claim.skillPath()); + return Boolean.TRUE.equals(jdbc.query(""" + SELECT publication_token = ? AND publication_expires_at > ? AS valid + FROM github_sync_package_ingestion + WHERE id = ? AND status = 'VALIDATED' + FOR UPDATE + """, result -> result.next() && result.getBoolean("valid"), + claim.token(), Timestamp.from(now), claim.id())); + } + + @Override + @Transactional + public CurrentTarget currentTarget(ClaimedPublication claim) { + return jdbc.query(""" + SELECT binding.skill_id, binding.latest_content_sha256, + (binding.retry_requested_at IS NOT NULL + AND claimed.created_at >= binding.retry_requested_at) AS retry_requested, + COALESCE((latest_task.created_at, latest_task.id) + > (claimed_task.created_at, claimed_task.id), FALSE) AS newer_acquisition + FROM github_sync_package_ingestion claimed + JOIN github_sync_task claimed_task ON claimed_task.id = claimed.task_id + LEFT JOIN github_source_skill_binding binding + ON binding.namespace_id = claimed.namespace_id + AND binding.repository_id = claimed.repository_id + AND binding.skill_path = claimed.skill_path + LEFT JOIN github_sync_package_ingestion latest + ON latest.id = binding.latest_ingestion_id + LEFT JOIN github_sync_task latest_task ON latest_task.id = latest.task_id + WHERE claimed.id = ? + """, result -> result.next() + ? new CurrentTarget(result.getObject("skill_id", Long.class), + result.getString("latest_content_sha256"), + result.getBoolean("retry_requested"), + result.getBoolean("newer_acquisition")) + : new CurrentTarget(null, null, false, false), claim.id()); + } + + @Override + @Transactional + public boolean superseded(ClaimedPublication claim, Instant now) { + return settle(claim, "SUPERSEDED", null, null, now); + } + + @Override + public void recordOutcome(ClaimedPublication claim, Long skillId, Long versionId, + boolean newSkill, Instant now) { + new JdbcGithubDiscoveryOutcomeRepository(jdbc).publication(claim.namespaceId(), + claim.repositoryId(), skillId, versionId, newSkill, now); + } + + @Override + @Transactional + public boolean readyForReview(ClaimedPublication claim, Long skillId, Long versionId, Instant now) { + int updated = jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'READY_FOR_REVIEW', publication_token = NULL, + publication_expires_at = NULL, publication_retry_at = NULL, + published_skill_id = ?, published_version_id = ?, + last_error_code = NULL, updated_at = ? + WHERE id = ? AND status = 'VALIDATED' AND publication_token = ? + """, skillId, versionId, Timestamp.from(now), claim.id(), claim.token()); + if (updated != 1) { + return false; + } + record BindingRevision(long id, int revision) {} + BindingRevision binding = jdbc.query(""" + INSERT INTO github_source_skill_binding + (source_id, namespace_id, repository_id, skill_path, skill_id, + latest_content_sha256, latest_ingestion_id, latest_revision) + VALUES (?, ?, ?, ?, ?, ?, ?, 1) + ON CONFLICT (namespace_id, repository_id, skill_path) DO UPDATE + SET latest_content_sha256 = EXCLUDED.latest_content_sha256, + latest_ingestion_id = EXCLUDED.latest_ingestion_id, + latest_revision = CASE + WHEN github_source_skill_binding.latest_revision = 0 THEN 1 + WHEN github_source_skill_binding.latest_content_sha256 = EXCLUDED.latest_content_sha256 + THEN github_source_skill_binding.latest_revision + ELSE github_source_skill_binding.latest_revision + 1 END, + retry_requested_at = NULL, + retry_scheduled_at = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE github_source_skill_binding.skill_id = EXCLUDED.skill_id + AND github_source_skill_binding.source_id = EXCLUDED.source_id + RETURNING id, latest_revision + """, statement -> { + statement.setLong(1, claim.sourceId()); + statement.setLong(2, claim.namespaceId()); + statement.setLong(3, claim.repositoryId()); + statement.setString(4, claim.skillPath()); + statement.setLong(5, skillId); + statement.setString(6, claim.contentSha256()); + statement.setObject(7, claim.id()); + }, result -> result.next() + ? new BindingRevision(result.getLong("id"), result.getInt("latest_revision")) : null); + if (binding == null) { + throw new IllegalStateException("GitHub Skill path binding changed while publishing"); + } + if (jdbc.update(""" + UPDATE skill_version SET github_sync_revision = ? + WHERE id = ? AND skill_id = ? AND github_sync_ingestion_id = ? + """, binding.revision(), versionId, skillId, claim.id()) != 1) { + throw new IllegalStateException("GitHub Skill version changed while assigning content revision"); + } + jdbc.update(""" + UPDATE github_source_skill_binding binding + SET upstream_presence = 'PRESENT', presence_commit_sha = ingestion.commit_sha, + missing_since = NULL, updated_at = ? + FROM github_sync_package_ingestion ingestion + JOIN github_sync_task task ON task.id = ingestion.task_id + JOIN github_source_binding source ON source.id = task.source_id + WHERE binding.id = ? AND ingestion.id = ? + AND binding.upstream_presence = 'UNKNOWN' + AND task.mode = 'FULL' + AND task.status IN ('ACQUISITION_COMPLETE', 'PARTIAL') + AND task.completed_at IS NOT NULL + AND task.incomplete_subtrees = '[]'::jsonb + AND task.fixed_commit_sha = ingestion.commit_sha + AND source.status <> 'PAUSED' + AND COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns + AND NOT EXISTS (SELECT 1 FROM github_sync_task newer + WHERE newer.source_id = task.source_id + AND newer.kind = 'REPOSITORY_SYNC' AND newer.mode = 'FULL' + AND (newer.created_at, newer.id) > (task.created_at, task.id)) + """, Timestamp.from(now), binding.id(), claim.id()); + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'SUPERSEDED', updated_at = ? + WHERE namespace_id = ? AND repository_id = ? AND skill_path = ? + AND id <> ? AND status IN ('READY_FOR_REVIEW', 'BLOCKED') + """, Timestamp.from(now), claim.namespaceId(), claim.repositoryId(), + claim.skillPath(), claim.id()); + return true; + } + + @Override + @Transactional + public boolean noChange(ClaimedPublication claim, Instant now) { + return settle(claim, "NO_CHANGE", null, null, now); + } + + @Override + @Transactional + public PathMoveDecision inspectPathMove(ClaimedPublication claim) { + if (claim.expectedSkillId() != null) { + return PathMoveDecision.of(PathMoveOutcome.NONE); + } + // Serializes path decisions for this source, including two new paths with the same hash. + boolean sourceExists = Boolean.TRUE.equals(jdbc.query( + "SELECT id FROM github_source_binding WHERE id = ? FOR UPDATE", + (ResultSetExtractor) result -> result.next(), claim.sourceId())); + if (!sourceExists) { + return PathMoveDecision.of(PathMoveOutcome.RETRY); + } + Integer boundPaths = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_skill_binding + WHERE source_id = ? AND namespace_id = ? AND repository_id = ? + """, Integer.class, claim.sourceId(), claim.namespaceId(), claim.repositoryId()); + if (boundPaths == null || boundPaths == 0) { + return PathMoveDecision.of(PathMoveOutcome.NONE); + } + String taskStatus = jdbc.query(""" + SELECT CASE WHEN COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + <> source.ignore_patterns + OR EXISTS (SELECT 1 FROM github_sync_task newer + WHERE newer.source_id = task.source_id + AND newer.kind = 'REPOSITORY_SYNC' AND newer.mode = 'FULL' + AND (newer.created_at, newer.id) > (task.created_at, task.id)) + THEN 'STALE' + WHEN task.mode = 'FULL' AND task.status = 'PARTIAL' + AND task.completed_at IS NOT NULL + AND task.incomplete_subtrees = '[]'::jsonb + AND task.fixed_commit_sha = ingestion.commit_sha + THEN 'TERMINAL_PARTIAL' + WHEN task.mode = 'FULL' AND task.status = 'ACQUISITION_COMPLETE' + AND task.fixed_commit_sha = ingestion.commit_sha + THEN 'COMPLETE' ELSE task.status END AS scan_status + FROM github_sync_task task + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = task.id + JOIN github_source_binding source ON source.id = task.source_id + WHERE ingestion.id = ? AND task.namespace_id = ? AND task.source_id = ? + """, result -> result.next() ? result.getString("scan_status") : null, + claim.id(), claim.namespaceId(), claim.sourceId()); + Integer currentPathPresent = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result path + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = path.task_id + WHERE ingestion.id = ? AND path.commit_sha = ingestion.commit_sha + AND path.skill_path = ingestion.skill_path AND path.outcome = 'ACCEPTED' + """, Integer.class, claim.id()); + if ("TERMINAL_PARTIAL".equals(taskStatus)) { + // A terminal failure in another Skill does not invalidate an accepted + // new path when every existing binding was also observed. Do not infer + // a move or a deletion from this partial result. + return PathMoveDecision.of(currentPathPresent != null && currentPathPresent == 1 + && unobservedBoundPaths(claim) == 0 + ? PathMoveOutcome.NONE : PathMoveOutcome.INCOMPLETE); + } + if (("LEASED".equals(taskStatus) || "RETRY_WAIT".equals(taskStatus)) + && currentPathPresent != null && currentPathPresent == 1 + && unobservedBoundPaths(claim) == 0) { + // Every existing binding has already appeared in this fixed-revision + // result. This new path cannot be a move, even if scanning is paused. + return PathMoveDecision.of(PathMoveOutcome.NONE); + } + if (!"COMPLETE".equals(taskStatus)) { + return PathMoveDecision.of("STALE".equals(taskStatus) ? PathMoveOutcome.STALE + : "PARTIAL".equals(taskStatus) || "FAILED".equals(taskStatus) + ? PathMoveOutcome.INCOMPLETE + : PathMoveOutcome.RETRY); + } + if (currentPathPresent == null || currentPathPresent != 1) { + return PathMoveDecision.of(PathMoveOutcome.AMBIGUOUS); + } + List matches = jdbc.query(""" + SELECT skill_id, skill_path FROM github_source_skill_binding + WHERE source_id = ? AND namespace_id = ? AND repository_id = ? + AND latest_content_sha256 = ? AND skill_path <> ? + """, (result, row) -> new PathMoveDecision(PathMoveOutcome.CANDIDATE, + result.getLong("skill_id"), result.getString("skill_path")), + claim.sourceId(), claim.namespaceId(), claim.repositoryId(), + claim.contentSha256(), claim.skillPath()); + if (matches.isEmpty()) { + return PathMoveDecision.of(unobservedBoundPaths(claim) > 0 + ? PathMoveOutcome.AMBIGUOUS : PathMoveOutcome.NONE); + } + if (matches.size() != 1) { + return PathMoveDecision.of(PathMoveOutcome.AMBIGUOUS); + } + PathMoveDecision match = matches.getFirst(); + Integer oldPathPresent = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result path + JOIN github_sync_package_ingestion ingestion ON ingestion.task_id = path.task_id + WHERE ingestion.id = ? AND path.commit_sha = ingestion.commit_sha + AND path.skill_path = ? + """, Integer.class, claim.id(), match.previousPath()); + if (oldPathPresent != null && oldPathPresent > 0) { + return PathMoveDecision.of(PathMoveOutcome.NONE); + } + Integer unsettled = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion candidate + JOIN github_sync_package_ingestion current ON current.task_id = candidate.task_id + WHERE current.id = ? AND candidate.commit_sha = current.commit_sha + AND candidate.status IN ('RECEIVED', 'VALIDATING', 'RETRY_WAIT') + """, Integer.class, claim.id()); + if (unsettled != null && unsettled > 0) { + return PathMoveDecision.of(PathMoveOutcome.RETRY); + } + Integer failed = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion candidate + JOIN github_sync_package_ingestion current ON current.task_id = candidate.task_id + WHERE current.id = ? AND candidate.commit_sha = current.commit_sha + AND candidate.status = 'FAILED' + """, Integer.class, claim.id()); + Integer sameContent = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion candidate + JOIN github_sync_package_ingestion current ON current.task_id = candidate.task_id + WHERE current.id = ? AND candidate.commit_sha = current.commit_sha + AND candidate.content_sha256 = ? AND candidate.skill_path <> ? + """, Integer.class, claim.id(), claim.contentSha256(), match.previousPath()); + if (failed != null && failed > 0 || sameContent == null || sameContent != 1) { + return PathMoveDecision.of(PathMoveOutcome.AMBIGUOUS); + } + return match; + } + + private int unobservedBoundPaths(ClaimedPublication claim) { + Integer missing = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_skill_binding binding + JOIN github_sync_package_ingestion ingestion ON ingestion.id = ? + WHERE binding.source_id = ? AND binding.namespace_id = ? + AND binding.repository_id = ? AND binding.skill_path <> ? + AND NOT EXISTS ( + SELECT 1 FROM github_sync_path_result path + WHERE path.task_id = ingestion.task_id + AND path.commit_sha = ingestion.commit_sha + AND path.skill_path = binding.skill_path + AND path.outcome <> 'RETRYABLE_FAILURE') + """, Integer.class, claim.id(), claim.sourceId(), claim.namespaceId(), + claim.repositoryId(), claim.skillPath()); + return missing == null ? 1 : missing; + } + + @Override + @Transactional + public boolean movePath(ClaimedPublication claim, PathMoveDecision decision, Instant now) { + if (decision.outcome() != PathMoveOutcome.CANDIDATE) { + throw new IllegalArgumentException("GitHub path move requires a unique candidate"); + } + int updated = jdbc.update(""" + UPDATE github_source_skill_binding + SET skill_path = ?, latest_ingestion_id = ?, + upstream_presence = 'PRESENT', + presence_commit_sha = (SELECT commit_sha FROM github_sync_package_ingestion + WHERE id = ?), + missing_since = NULL, updated_at = ? + WHERE source_id = ? AND namespace_id = ? AND repository_id = ? + AND skill_path = ? AND skill_id = ? AND latest_content_sha256 = ? + AND NOT EXISTS ( + SELECT 1 FROM github_source_skill_binding occupied + WHERE occupied.namespace_id = ? AND occupied.repository_id = ? + AND occupied.skill_path = ?) + """, claim.skillPath(), claim.id(), claim.id(), Timestamp.from(now), claim.sourceId(), + claim.namespaceId(), claim.repositoryId(), decision.previousPath(), decision.skillId(), + claim.contentSha256(), claim.namespaceId(), claim.repositoryId(), claim.skillPath()); + if (updated != 1) { + throw new IllegalStateException("GitHub Skill path binding changed while moving"); + } + return jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'NO_CHANGE', published_skill_id = ?, publication_token = NULL, + publication_expires_at = NULL, publication_retry_at = NULL, + last_error_code = NULL, updated_at = ? + WHERE id = ? AND status = 'VALIDATED' AND publication_token = ? + AND publication_expires_at > ? + """, decision.skillId(), Timestamp.from(now), claim.id(), claim.token(), + Timestamp.from(now)) == 1; + } + + @Override + @Transactional + public boolean ignored(ClaimedPublication claim, Instant now) { + if (!settle(claim, "FAILED", "SKILL_PATH_IGNORED", null, now)) { + return false; + } + jdbc.update(""" + UPDATE github_sync_path_result path + SET outcome = 'IGNORED', reason = 'SKILL_PATH_IGNORED', + retry_at = NULL, updated_at = ? + FROM github_sync_package_ingestion ingestion + WHERE ingestion.id = ? AND path.task_id = ingestion.task_id + AND path.commit_sha = ingestion.commit_sha AND path.skill_path = ingestion.skill_path + """, Timestamp.from(now), claim.id()); + return true; + } + + @Override + @Transactional + public boolean blocked(ClaimedPublication claim, String reason, Instant now) { + return settle(claim, "BLOCKED", reason, null, now); + } + + @Override + @Transactional + public boolean blockedSlugConflict(ClaimedPublication claim, String skillSlug, Instant now) { + if (skillSlug == null || skillSlug.isBlank() || skillSlug.length() > 128) { + throw new IllegalArgumentException("GitHub sync conflict requires a valid Skill slug"); + } + return jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = 'BLOCKED', publication_token = NULL, publication_expires_at = NULL, + publication_retry_at = NULL, last_error_code = 'SLUG_CONFLICT', + target_skill_slug = ?, updated_at = ? + WHERE id = ? AND status = 'VALIDATED' AND publication_token = ? + AND publication_expires_at > ? + """, skillSlug, Timestamp.from(now), claim.id(), claim.token(), + Timestamp.from(now)) == 1; + } + + @Override + @Transactional + public boolean retry(ClaimedPublication claim, String reason, Instant retryAt, Instant now) { + if (retryAt == null || !retryAt.isAfter(now)) { + throw new IllegalArgumentException("GitHub publication retry must be in the future"); + } + return settle(claim, "VALIDATED", reason, retryAt, now); + } + + private boolean settle(ClaimedPublication claim, String status, String reason, + Instant retryAt, Instant now) { + return jdbc.update(""" + UPDATE github_sync_package_ingestion + SET status = ?, publication_token = NULL, publication_expires_at = NULL, + publication_retry_at = ?, last_error_code = ?, updated_at = ? + WHERE id = ? AND status = 'VALIDATED' AND publication_token = ? + AND publication_expires_at > ? + """, status, retryAt == null ? null : Timestamp.from(retryAt), reason, + Timestamp.from(now), claim.id(), claim.token(), Timestamp.from(now)) == 1; + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSchedulePolicyRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSchedulePolicyRepository.java new file mode 100644 index 000000000..d2ac41d86 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSchedulePolicyRepository.java @@ -0,0 +1,75 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicy; +import com.iflytek.skillhub.githubsync.GithubSyncSchedulePolicyRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +public class JdbcGithubSyncSchedulePolicyRepository implements GithubSyncSchedulePolicyRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncSchedulePolicyRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public GithubSyncSchedulePolicy get() { + return read("SELECT * FROM github_sync_schedule_policy WHERE id = TRUE"); + } + + @Override + public GithubSyncSchedulePolicy getForUpdate() { + return read("SELECT * FROM github_sync_schedule_policy WHERE id = TRUE FOR UPDATE"); + } + + @Override + public GithubSyncSchedulePolicy update(int repositoryHours, int curatedOwnerHours, int searchRuleHours) { + jdbc.update(""" + UPDATE github_sync_schedule_policy + SET repository_hours = ?, curated_owner_hours = ?, search_rule_hours = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = TRUE + """, repositoryHours, curatedOwnerHours, searchRuleHours); + jdbc.update(""" + UPDATE github_discovery_entry + SET target_interval_seconds = ?, + next_due_at = LEAST(next_due_at, CURRENT_TIMESTAMP + (? * INTERVAL '1 second')), + updated_at = CURRENT_TIMESTAMP + WHERE entry_type = 'CURATED_REPOSITORY' + """, repositoryHours * 3600, repositoryHours * 3600); + jdbc.update(""" + UPDATE github_discovery_entry + SET target_interval_seconds = ?, + next_due_at = LEAST(next_due_at, CURRENT_TIMESTAMP + (? * INTERVAL '1 second')), + updated_at = CURRENT_TIMESTAMP + WHERE entry_type = 'CURATED_OWNER' + """, curatedOwnerHours * 3600, curatedOwnerHours * 3600); + jdbc.update(""" + UPDATE github_discovery_entry + SET target_interval_seconds = ?, + next_due_at = LEAST(next_due_at, CURRENT_TIMESTAMP + (? * INTERVAL '1 second')), + updated_at = CURRENT_TIMESTAMP + WHERE entry_type = 'SEARCH_RULE' + """, searchRuleHours * 3600, searchRuleHours * 3600); + jdbc.update(""" + UPDATE github_source_binding + SET next_check_at = LEAST(COALESCE(next_check_at, CURRENT_TIMESTAMP), + CURRENT_TIMESTAMP + (? * INTERVAL '1 second')), + updated_at = CURRENT_TIMESTAMP + WHERE status = 'ACTIVE' + """, repositoryHours * 3600); + return get(); + } + + private GithubSyncSchedulePolicy read(String sql) { + return jdbc.query(sql, result -> { + if (!result.next()) { + throw new IllegalStateException("GitHub sync schedule policy is missing"); + } + return new GithubSyncSchedulePolicy(result.getObject("repository_hours", Integer.class), + result.getInt("curated_owner_hours"), result.getInt("search_rule_hours"), + result.getTimestamp("updated_at").toInstant()); + }); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSourceCheckpointRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSourceCheckpointRepository.java new file mode 100644 index 000000000..0f572dcff --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncSourceCheckpointRepository.java @@ -0,0 +1,42 @@ +package com.iflytek.skillhub.infra.jpa; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.UUID; + +/** Only complete, validated acquisition may advance a source's full-processing marker. */ +@Repository +public class JdbcGithubSyncSourceCheckpointRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncSourceCheckpointRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public void advanceIfComplete(UUID taskId, String commitSha, Instant now) { + jdbc.update(""" + UPDATE github_source_binding source + SET last_fully_processed_commit = ?, updated_at = ? + FROM github_sync_task task + WHERE task.id = ? AND task.source_id = source.id + AND task.kind = 'REPOSITORY_SYNC' AND task.mode = 'FULL' + AND task.status = 'ACQUISITION_COMPLETE' AND task.fixed_commit_sha = ? + AND source.last_observed_commit = ? + AND COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns + AND NOT EXISTS ( + SELECT 1 FROM github_sync_package_ingestion ingestion + WHERE ingestion.task_id = task.id + AND ingestion.status NOT IN ('VALIDATED', 'NO_CHANGE', + 'READY_FOR_REVIEW', 'APPROVED', + 'REJECTED', 'SUPERSEDED', 'BLOCKED')) + AND NOT EXISTS ( + SELECT 1 FROM github_sync_path_result path + WHERE path.task_id = task.id AND path.commit_sha = ? + AND path.outcome IN ('RETRYABLE_FAILURE', 'TERMINAL_FAILURE')) + """, commitSha, Timestamp.from(now), taskId, commitSha, commitSha, commitSha); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncTaskRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncTaskRepository.java new file mode 100644 index 000000000..f9c495964 --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncTaskRepository.java @@ -0,0 +1,1269 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iflytek.skillhub.githubsync.GithubSyncFailureReason; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatch; +import com.iflytek.skillhub.githubsync.GithubDiscoveryBatchResult; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCandidate; +import com.iflytek.skillhub.githubsync.GithubDiscoveryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubSyncDeferralReason; +import com.iflytek.skillhub.githubsync.GithubSyncTaskKind; +import com.iflytek.skillhub.githubsync.GithubSyncTaskLease; +import com.iflytek.skillhub.githubsync.GithubSyncTaskMode; +import com.iflytek.skillhub.githubsync.GithubSyncTaskRepository; +import com.iflytek.skillhub.githubsync.GithubPackageAdmissionResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryCompletionResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryMetadataResult; +import com.iflytek.skillhub.githubsync.GithubRepositoryPathResult; +import com.iflytek.skillhub.githubsync.RepositorySkillCoordinate; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.Duration; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +/** + * Claims require PostgreSQL's SKIP LOCKED and a single UPDATE/RETURNING statement: a read followed + * by a JPA save would allow two application instances to issue leases for the same task. + */ +@Repository +public class JdbcGithubSyncTaskRepository implements GithubSyncTaskRepository { + private static final String CLAIM_SQL = """ + WITH picked AS ( + SELECT candidate.id, capability.required_capabilities FROM github_sync_task candidate + LEFT JOIN github_platform_collection_control platform ON platform.namespace_id=candidate.namespace_id + LEFT JOIN github_deep_discovery_query fair_query ON fair_query.id=candidate.official_query_id + LEFT JOIN github_sync_claim_cursor cursor + ON cursor.kind = candidate.kind AND cursor.namespace_id = candidate.namespace_id + LEFT JOIN github_candidate_discovery_eligibility eligibility + ON candidate.kind = 'REPOSITORY_SYNC' AND candidate.source_id IS NULL + AND eligibility.namespace_id = candidate.namespace_id + AND eligibility.repository_id::text = candidate.snapshot_json ->> 'repositoryId' + CROSS JOIN LATERAL (SELECT array_remove(CASE + WHEN candidate.kind = 'REPOSITORY_SYNC' AND candidate.source_id IS NOT NULL + THEN array_remove(candidate.required_capabilities, 'official-pause/v1') + WHEN eligibility.has_official THEN + array_remove(candidate.required_capabilities, 'official-pause/v1') || + CASE WHEN eligibility.official_only THEN ARRAY['official-pause/v1'] ELSE ARRAY[]::text[] END + ELSE candidate.required_capabilities END, 'platform-pause/v1') || CASE WHEN platform.namespace_id IS NOT NULL + THEN ARRAY['platform-pause/v1'] ELSE ARRAY[]::text[] END AS required_capabilities) capability + WHERE candidate.kind = ? AND ((candidate.status IN ('QUEUED', 'RETRY_WAIT') + AND candidate.due_at <= CASE WHEN candidate.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END) + OR (candidate.status = 'LEASED' AND candidate.lease_expires_at <= CASE WHEN candidate.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END)) + AND (platform.namespace_id IS NULL OR NOT platform.paused) + AND capability.required_capabilities <@ ?::text[] + AND (candidate.official_query_id IS NULL OR candidate.kind <> 'DISCOVERY' OR ( + (SELECT next_lease_eligible_at <= clock_timestamp() FROM github_official_discovery_budget WHERE id) + AND (SELECT count(*) FROM github_discovery_candidate pending + WHERE pending.namespace_id=candidate.namespace_id + AND pending.status IN ('PENDING','VALIDATING')) + < (SELECT candidate_pending_limit FROM github_sync_admission_policy WHERE id) + AND NOT EXISTS (SELECT 1 FROM github_sync_task active + WHERE active.kind='DISCOVERY' AND active.official_query_id IS NOT NULL + AND active.status='LEASED' AND active.lease_expires_at > clock_timestamp()))) + AND (eligibility.candidate_id IS NULL OR eligibility.acquisition_allowed) + AND (candidate.official_query_id IS NULL OR EXISTS ( + SELECT 1 FROM github_deep_discovery_query official + JOIN github_deep_discovery_config config ON config.namespace_id = official.namespace_id + WHERE official.id = candidate.official_query_id + AND official.namespace_id = candidate.namespace_id + AND official.active_task_id = candidate.id + AND config.enabled = TRUE + AND NOT EXISTS (SELECT 1 FROM github_official_owner_eligibility owner + WHERE owner.query_id = official.id AND owner.excluded))) + AND (candidate.source_id IS NULL OR EXISTS ( + SELECT 1 FROM github_source_binding source + WHERE source.id = candidate.source_id AND source.status <> 'PAUSED')) + AND (candidate.kind <> 'DISCOVERY' OR NOT EXISTS ( + SELECT 1 FROM github_discovery_entry entry + WHERE entry.id = (candidate.snapshot_json->>'ruleId')::uuid + AND entry.namespace_id = candidate.namespace_id + AND entry.enabled = FALSE)) + ORDER BY CASE WHEN candidate.source_id IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN candidate.official_query_id IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN candidate.source_id IS NULL AND candidate.official_query_id IS NULL THEN candidate.priority ELSE 0 END DESC, + fair_query.last_successful_claim_at NULLS FIRST, + cursor.last_claimed_at NULLS FIRST, + candidate.priority DESC, candidate.due_at, candidate.id + LIMIT 1 FOR UPDATE OF candidate SKIP LOCKED + ) + UPDATE github_sync_task AS task + SET status = 'LEASED', + required_capabilities = picked.required_capabilities, + lease_generation = task.lease_generation + 1, + lease_token_hash = ?, + lease_expires_at = CASE WHEN task.official_query_id IS NOT NULL + THEN clock_timestamp() + (?::timestamptz - ?::timestamptz) ELSE ? END, + leased_by = ?, + credential_id = NULL, + credential_fingerprint = NULL, + updated_at = ? + FROM picked + WHERE task.id = picked.id + RETURNING task.id, task.run_id, task.kind, task.mode, task.namespace_id, + task.source_id, task.snapshot_json::text AS snapshot_json, + task.discovery_checkpoint, task.fixed_commit_sha, + task.path_checkpoint, task.incomplete_subtrees::text AS incomplete_subtrees, + ARRAY(SELECT path.skill_path FROM github_sync_path_result path + WHERE path.task_id = task.id AND path.commit_sha = task.fixed_commit_sha + AND path.outcome = 'RETRYABLE_FAILURE' + ORDER BY path.skill_path) AS retry_paths, + ARRAY(SELECT ingestion.skill_path FROM github_sync_package_ingestion ingestion + WHERE ingestion.task_id = task.id + AND ingestion.commit_sha = task.fixed_commit_sha + ORDER BY ingestion.skill_path) AS accepted_paths, + task.lease_generation, task.lease_expires_at, task.required_capabilities, + (SELECT version FROM github_platform_collection_control WHERE namespace_id=task.namespace_id) AS platform_control_version + """; + + private static final String RENEW_SQL = """ + UPDATE github_sync_task + SET lease_expires_at = CASE WHEN official_query_id IS NOT NULL + THEN clock_timestamp() + (?::timestamptz - ?::timestamptz) ELSE ? END, updated_at = ? + WHERE id = ? AND status = 'LEASED' AND lease_generation = ? + AND lease_token_hash = ? AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL + THEN clock_timestamp() ELSE ? END + RETURNING lease_expires_at + """; + + private static final String LOCK_ADMISSION_TASK_SQL = """ + SELECT task.namespace_id, task.source_id, task.kind, task.mode, task.fixed_commit_sha, + task.snapshot_json ->> 'repositoryId' AS snapshot_repository_id, + task.snapshot_json ->> 'fixedCommitSha' AS snapshot_commit_sha, + source.repository_id AS source_repository_id + FROM github_sync_task task + LEFT JOIN github_source_binding source ON source.id = task.source_id + WHERE task.id = ? AND task.status = 'LEASED' AND task.lease_generation = ? + AND task.lease_token_hash = ? AND task.lease_expires_at > CASE WHEN task.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE OF task + """; + + private static final String LOCK_LEASE_SQL = """ + SELECT source_id, kind, mode FROM github_sync_task + WHERE id = ? AND status = 'LEASED' AND lease_generation = ? + AND lease_token_hash = ? AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE + """; + + private static final String LOCK_DISCOVERY_LEASE_SQL = """ + SELECT namespace_id, discovery_checkpoint, + COALESCE(snapshot_json ->> 'candidateBudget', + snapshot_json #>> '{officialDiscovery,budget,candidateBudget}') AS candidate_budget, + CASE WHEN official_query_id IS NOT NULL THEN 'OFFICIAL_DISCOVERY' + ELSE snapshot_json ->> 'entryType' END AS entry_type, + snapshot_json ->> 'ruleId' AS entry_id + FROM github_sync_task + WHERE id = ? AND kind = 'DISCOVERY' AND status = 'LEASED' + AND lease_generation = ? AND lease_token_hash = ? AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE + """; + + private static final String LOCK_REPOSITORY_LEASE_SQL = """ + SELECT source_id, fixed_commit_sha, path_checkpoint, + snapshot_json ->> 'fixedCommitSha' AS snapshot_commit_sha + FROM github_sync_task + WHERE id = ? AND kind = 'REPOSITORY_SYNC' AND mode = 'FULL' + AND status = 'LEASED' AND lease_generation = ? + AND lease_token_hash = ? AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE + """; + + private final JdbcTemplate jdbc; + private final JdbcGithubDiscoveryOutcomeRepository outcomes; + private final ObjectMapper objectMapper; + private final JdbcGithubSyncCapacityGate capacityGate; + private final JdbcGithubSyncSourceCheckpointRepository checkpoints; + private final JdbcGithubSourceIgnoreRepository ignores; + private final JdbcGithubSyncPathPresenceRepository presence; + + public JdbcGithubSyncTaskRepository(JdbcTemplate jdbc, ObjectMapper objectMapper, + JdbcGithubSyncCapacityGate capacityGate, + JdbcGithubSyncSourceCheckpointRepository checkpoints, + JdbcGithubSourceIgnoreRepository ignores, + JdbcGithubSyncPathPresenceRepository presence) { + this.jdbc = jdbc; + this.outcomes = new JdbcGithubDiscoveryOutcomeRepository(jdbc); + this.objectMapper = objectMapper; + this.capacityGate = capacityGate; + this.checkpoints = checkpoints; + this.ignores = ignores; + this.presence = presence; + } + + @Override + public Optional claim(GithubSyncTaskKind kind, String workerId, String leaseTokenHash, + Instant now, Instant expiresAt) { + return claim(kind, workerId, leaseTokenHash, now, expiresAt, List.of()); + } + + @Override + public Optional claim(GithubSyncTaskKind kind, String workerId, String leaseTokenHash, + Instant now, Instant expiresAt, List capabilities) { + jdbc.queryForList("SELECT namespace_id FROM github_platform_collection_control FOR SHARE"); + if (kind == GithubSyncTaskKind.DISCOVERY || kind == GithubSyncTaskKind.REPOSITORY_SYNC) { + // Hold through the lease transaction so a completed disable cannot race an official claim. + jdbc.queryForObject("SELECT enabled FROM github_deep_discovery_config WHERE id = TRUE FOR SHARE", + Boolean.class); + } + if (kind == GithubSyncTaskKind.DISCOVERY) { + // Serialize official grant decisions across all workers. Legacy candidates remain eligible. + jdbc.queryForObject("SELECT id FROM github_official_discovery_budget WHERE id FOR UPDATE", Boolean.class); + } + Optional claimed = jdbc.query(connection -> { + var statement = connection.prepareStatement(CLAIM_SQL); + statement.setString(1, kind.name()); + statement.setTimestamp(2, Timestamp.from(now)); + statement.setTimestamp(3, Timestamp.from(now)); + statement.setArray(4, connection.createArrayOf("text", capabilities.toArray())); + statement.setString(5, leaseTokenHash); + statement.setTimestamp(6, Timestamp.from(expiresAt)); + statement.setTimestamp(7, Timestamp.from(now)); + statement.setTimestamp(8, Timestamp.from(expiresAt)); + statement.setString(9, workerId); + statement.setTimestamp(10, Timestamp.from(now)); + return statement; + }, result -> result.next() ? Optional.of(mapLease(result)) : Optional.empty()); + claimed.ifPresent(task -> jdbc.update(""" + INSERT INTO github_sync_claim_cursor (kind, namespace_id, last_claimed_at) + VALUES (?, ?, ?) + ON CONFLICT (kind, namespace_id) DO UPDATE + SET last_claimed_at = GREATEST(github_sync_claim_cursor.last_claimed_at, + EXCLUDED.last_claimed_at) + """, kind.name(), task.namespaceId(), Timestamp.from(now))); + claimed.ifPresent(task -> jdbc.update(""" + UPDATE github_sync_package_admission + SET status = 'RELEASED', updated_at = ? + WHERE task_id = ? AND lease_generation < ? AND status = 'ACTIVE' + """, Timestamp.from(now), task.taskId(), task.leaseGeneration())); + if (claimed.isPresent() && claimed.get().mode() == GithubSyncTaskMode.FULL + && !capacityGate.allowsFullAcquisition(claimed.get().namespaceId(), now)) { + GithubSyncTaskLease task = claimed.get(); + if (task.sourceId() != null) { + jdbc.update("UPDATE github_sync_task SET mode = 'HEAD_ONLY', updated_at = ? WHERE id = ?", + Timestamp.from(now), task.taskId()); + return Optional.of(new GithubSyncTaskLease(task.taskId(), task.runId(), task.kind(), + GithubSyncTaskMode.HEAD_ONLY, task.namespaceId(), task.sourceId(), + task.snapshotJson(), task.discoveryCheckpoint(), task.fixedCommitSha(), + task.pathCheckpoint(), task.incompleteSubtreesJson(), task.retryPaths(), + task.acceptedPaths(), + task.leaseGeneration(), task.leaseExpiresAt(), task.requiredCapabilities(), task.platformControlVersion())); + } + Instant retryAt = now.plus(Duration.ofMinutes(15)); + jdbc.update(""" + UPDATE github_sync_task + SET status = 'RETRY_WAIT', due_at = ?, retry_at = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + last_error_code = 'BACKPRESSURE', updated_at = ? + WHERE id = ? + """, Timestamp.from(retryAt), Timestamp.from(retryAt), + Timestamp.from(now), task.taskId()); + return Optional.empty(); + } + if (claimed.isPresent() && claimed.get().mode() == GithubSyncTaskMode.FULL + && claimed.get().sourceId() != null) { + jdbc.update(""" + UPDATE github_source_binding + SET manual_rescan_requested_at = NULL, updated_at = ? + WHERE id = ? AND manual_rescan_requested_at IS NOT NULL + """, Timestamp.from(now), claimed.get().sourceId()); + } + return claimed; + } + + @Override + public Optional renew(UUID taskId, long leaseGeneration, String leaseTokenHash, + Instant now, Instant expiresAt) { + return jdbc.query(connection -> { + var statement = connection.prepareStatement(RENEW_SQL); + statement.setTimestamp(1, Timestamp.from(expiresAt)); + statement.setTimestamp(2, Timestamp.from(now)); + statement.setTimestamp(3, Timestamp.from(expiresAt)); + statement.setTimestamp(4, Timestamp.from(now)); + statement.setObject(5, taskId); + statement.setLong(6, leaseGeneration); + statement.setString(7, leaseTokenHash); + statement.setTimestamp(8, Timestamp.from(now)); + return statement; + }, result -> result.next() + ? Optional.of(result.getTimestamp("lease_expires_at").toInstant()) + : Optional.empty()); + } + + @Override + public GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long leaseGeneration, + String leaseTokenHash, long repositoryId, String owner, String name, + String defaultBranch, Instant now) { + return updateRepositoryMetadata(taskId, leaseGeneration, leaseTokenHash, repositoryId, owner, name, defaultBranch, null, now); + } + + @Override + public GithubRepositoryMetadataResult updateRepositoryMetadata(UUID taskId, long leaseGeneration, + String leaseTokenHash, long repositoryId, String owner, String name, + String defaultBranch, Long ownerId, Instant now) { + List leased = jdbc.query(""" + SELECT task.source_id, (task.snapshot_json ->> 'repositoryId')::bigint AS repository_id + FROM github_sync_task task + WHERE task.id = ? AND task.kind = 'REPOSITORY_SYNC' AND task.status = 'LEASED' + AND task.lease_generation = ? AND task.lease_token_hash = ? + AND task.lease_expires_at > CASE WHEN task.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE OF task + """, (result, row) -> new RepositoryMetadataTask( + (Long) result.getObject("source_id"), result.getLong("repository_id")), + taskId, leaseGeneration, leaseTokenHash, Timestamp.from(now)); + if (leased.isEmpty()) { + return GithubRepositoryMetadataResult.LEASE_STALE; + } + RepositoryMetadataTask task = leased.getFirst(); + if (task.repositoryId() != repositoryId) { + return GithubRepositoryMetadataResult.REPOSITORY_MISMATCH; + } + if (task.sourceId() != null) { + int updated = jdbc.update(""" + UPDATE github_source_binding + SET owner_login = ?, repository_name = ?, default_branch = ?, updated_at = ? + WHERE id = ? AND repository_id = ? + """, owner, name, defaultBranch, Timestamp.from(now), task.sourceId(), repositoryId); + if (updated != 1) { + return GithubRepositoryMetadataResult.REPOSITORY_MISMATCH; + } + } else { + int updated = jdbc.update(""" + UPDATE github_discovery_candidate + SET owner_login = ?, repository_name = ?, default_branch = ?, last_seen_at = ? + WHERE validation_task_id = ? AND repository_id = ? + """, owner, name, defaultBranch, Timestamp.from(now), taskId, repositoryId); + if (updated != 1) { + return GithubRepositoryMetadataResult.REPOSITORY_MISMATCH; + } + } + jdbc.update(""" + UPDATE github_sync_task + SET snapshot_json = snapshot_json || jsonb_build_object( + 'owner', ?, 'name', ?, 'defaultBranch', ?), updated_at = ? + WHERE id = ? + """, owner, name, defaultBranch, Timestamp.from(now), taskId); + if (ownerId == null) { + jdbc.update("UPDATE github_sync_task SET snapshot_json = snapshot_json - 'ownerId' WHERE id = ?", taskId); + jdbc.update(""" + UPDATE github_discovery_candidate candidate SET github_owner_id = NULL + FROM github_sync_task task WHERE task.id = ? AND candidate.namespace_id = task.namespace_id + AND candidate.repository_id = ? + """, taskId, repositoryId); + jdbc.update(""" + UPDATE github_source_binding source SET github_owner_id = NULL + FROM github_sync_task task WHERE task.id = ? AND source.namespace_id = task.namespace_id + AND source.repository_id = ? + """, taskId, repositoryId); + } + if (ownerId != null) { + jdbc.update("UPDATE github_sync_task SET snapshot_json = snapshot_json || jsonb_build_object('ownerId', ?::bigint) WHERE id = ?", ownerId, taskId); + jdbc.update(""" + UPDATE github_discovery_candidate candidate SET github_owner_id = ?, owner_login = ?, + repository_name = ?, default_branch = ? + FROM github_sync_task task WHERE task.id = ? AND candidate.namespace_id = task.namespace_id + AND candidate.repository_id = ? + """, ownerId, owner, name, defaultBranch, taskId, repositoryId); + jdbc.update(""" + UPDATE github_source_binding source SET github_owner_id = ? + FROM github_sync_task task WHERE task.id = ? AND source.namespace_id = task.namespace_id + AND source.repository_id = ? + """, ownerId, taskId, repositoryId); + } + return GithubRepositoryMetadataResult.UPDATED; + } + + private record RepositoryMetadataTask(Long sourceId, long repositoryId) {} + + @Override + public GithubPackageAdmissionResult admitPackage(UUID taskId, long leaseGeneration, + String leaseTokenHash, String commitSha, String skillPath, Instant now) { + jdbc.queryForList("SELECT namespace_id FROM github_platform_collection_control FOR SHARE"); + jdbc.queryForObject("SELECT enabled FROM github_deep_discovery_config WHERE id = TRUE FOR SHARE", Boolean.class); + // This single row serializes capacity checks and reservations across all application instances. + AdmissionPolicy policy = jdbc.query(""" + SELECT global_high, global_low, namespace_high, namespace_low, global_paused + FROM github_sync_admission_policy WHERE id = TRUE FOR UPDATE + """, result -> result.next() ? new AdmissionPolicy( + (Integer) result.getObject("global_high"), (Integer) result.getObject("global_low"), + (Integer) result.getObject("namespace_high"), (Integer) result.getObject("namespace_low"), + result.getBoolean("global_paused")) : null); + if (policy == null) { + throw new IllegalStateException("GitHub sync admission policy is missing"); + } + AdmissionTask task = jdbc.query(connection -> { + var statement = connection.prepareStatement(LOCK_ADMISSION_TASK_SQL); + statement.setObject(1, taskId); + statement.setLong(2, leaseGeneration); + statement.setString(3, leaseTokenHash); + statement.setTimestamp(4, Timestamp.from(now)); + return statement; + }, result -> result.next() ? new AdmissionTask( + result.getLong("namespace_id"), (Long) result.getObject("source_id"), + result.getString("kind"), result.getString("mode"), + result.getString("fixed_commit_sha"), result.getString("snapshot_repository_id"), + result.getString("snapshot_commit_sha"), + (Long) result.getObject("source_repository_id")) : null); + if (task == null) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.LEASE_STALE); + } + if (!"REPOSITORY_SYNC".equals(task.kind()) || !"FULL".equals(task.mode())) { + throw new IllegalArgumentException("package admission requires a FULL repository task"); + } + if (Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS(SELECT 1 FROM github_platform_collection_control WHERE namespace_id=? AND paused)", Boolean.class,task.namespaceId()))) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + UUID previous = jdbc.query(""" + SELECT id FROM github_sync_package_admission + WHERE task_id = ? AND lease_generation = ? AND commit_sha = ? AND skill_path = ? + """, statement -> { + statement.setObject(1, taskId); + statement.setLong(2, leaseGeneration); + statement.setString(3, commitSha); + statement.setString(4, skillPath); + }, result -> result.next() ? result.getObject("id", UUID.class) : null); + if (previous != null) { + return new GithubPackageAdmissionResult(GithubPackageAdmissionResult.Status.ADMITTED, previous); + } + if (new JdbcGithubOfficialQuotaRepository(jdbc).cooldown("CORE", now) != null) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + if (task.sourceId() != null) { + String sourceStatus = jdbc.queryForObject(""" + SELECT status FROM github_source_binding WHERE id = ? FOR UPDATE + """, String.class, task.sourceId()); + if ("PAUSED".equals(sourceStatus)) { + return GithubPackageAdmissionResult.rejected( + GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + } + long repositoryId; + try { + repositoryId = Long.parseLong(task.snapshotRepositoryId()); + } catch (NumberFormatException exception) { + throw new IllegalStateException("Repository task snapshot has no valid repository ID", exception); + } + Long effectiveSourceId = task.sourceId(); + if (effectiveSourceId == null) { + effectiveSourceId = jdbc.query(""" + SELECT id, status FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? FOR SHARE + """, result -> result.next() ? result.getLong("id") : null, task.namespaceId(), repositoryId); + if (effectiveSourceId != null && "PAUSED".equals(jdbc.queryForObject( + "SELECT status FROM github_source_binding WHERE id = ?", String.class, effectiveSourceId))) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + } + if (task.sourceId() == null && Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS (SELECT 1 FROM github_candidate_discovery_eligibility eligibility + WHERE namespace_id = ? AND repository_id = ? + AND (NOT acquisition_allowed OR (official_only AND NOT EXISTS ( + SELECT 1 FROM github_sync_task held WHERE held.id = ? + AND held.required_capabilities @> ARRAY['official-pause/v1'])))) + """, Boolean.class, task.namespaceId(), repositoryId, taskId))) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + new RepositorySkillCoordinate(task.namespaceId(), repositoryId, skillPath); + if (effectiveSourceId != null && ignores.ignoredForSource(effectiveSourceId, skillPath)) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.PATH_IGNORED); + } + if (task.sourceRepositoryId() != null && task.sourceRepositoryId() != repositoryId) { + throw new IllegalStateException("Repository task source and snapshot disagree"); + } + if ((task.fixedCommitSha() != null && !task.fixedCommitSha().equals(commitSha)) + || (task.snapshotCommitSha() != null && !task.snapshotCommitSha().equals(commitSha))) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.COMMIT_MISMATCH); + } + if (policy.globalHigh() == null || policy.namespaceHigh() == null) { + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + int globalCount = capacityGate.activeCount(null, now); + if (policy.globalPaused() && globalCount <= policy.globalLow()) { + jdbc.update("UPDATE github_sync_admission_policy SET global_paused = FALSE, updated_at = ? WHERE id = TRUE", + Timestamp.from(now)); + } else if (policy.globalPaused() || globalCount >= policy.globalHigh()) { + jdbc.update("UPDATE github_sync_admission_policy SET global_paused = TRUE, updated_at = ? WHERE id = TRUE", + Timestamp.from(now)); + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + int namespaceCount = capacityGate.activeCount(task.namespaceId(), now); + jdbc.update(""" + INSERT INTO github_sync_namespace_admission_state (namespace_id) + VALUES (?) ON CONFLICT (namespace_id) DO NOTHING + """, task.namespaceId()); + Boolean namespacePaused = jdbc.queryForObject(""" + SELECT paused FROM github_sync_namespace_admission_state WHERE namespace_id = ? + """, Boolean.class, task.namespaceId()); + if (Boolean.TRUE.equals(namespacePaused) && namespaceCount <= policy.namespaceLow()) { + jdbc.update(""" + UPDATE github_sync_namespace_admission_state SET paused = FALSE, updated_at = ? + WHERE namespace_id = ? + """, Timestamp.from(now), task.namespaceId()); + } else if (Boolean.TRUE.equals(namespacePaused) || namespaceCount >= policy.namespaceHigh()) { + jdbc.update(""" + UPDATE github_sync_namespace_admission_state SET paused = TRUE, updated_at = ? + WHERE namespace_id = ? + """, Timestamp.from(now), task.namespaceId()); + return GithubPackageAdmissionResult.rejected(GithubPackageAdmissionResult.Status.BACKPRESSURE_ACTIVE); + } + UUID admissionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_admission + (id, task_id, namespace_id, repository_id, commit_sha, skill_path, lease_generation) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, admissionId, taskId, task.namespaceId(), repositoryId, commitSha, skillPath, + leaseGeneration); + jdbc.update(""" + UPDATE github_sync_task SET fixed_commit_sha = ?, updated_at = ? WHERE id = ? + """, commitSha, Timestamp.from(now), taskId); + if (capacityGate.activeCount(null, now) >= policy.globalHigh()) { + jdbc.update("UPDATE github_sync_admission_policy SET global_paused = TRUE, updated_at = ? WHERE id = TRUE", + Timestamp.from(now)); + } + if (capacityGate.activeCount(task.namespaceId(), now) >= policy.namespaceHigh()) { + jdbc.update(""" + UPDATE github_sync_namespace_admission_state + SET paused = TRUE, updated_at = ? WHERE namespace_id = ? + """, Timestamp.from(now), task.namespaceId()); + } + return new GithubPackageAdmissionResult(GithubPackageAdmissionResult.Status.ADMITTED, admissionId); + } + + private record AdmissionPolicy(Integer globalHigh, Integer globalLow, Integer namespaceHigh, + Integer namespaceLow, boolean globalPaused) {} + private record AdmissionTask(long namespaceId, Long sourceId, String kind, String mode, + String fixedCommitSha, + String snapshotRepositoryId, String snapshotCommitSha, + Long sourceRepositoryId) {} + + @Override + public boolean completeHeadOnly(UUID taskId, long leaseGeneration, String leaseTokenHash, + String observedCommitSha, Instant now, Instant nextCheckAt) { + LockedTask task = lockLease(taskId, leaseGeneration, leaseTokenHash, now); + if (task == null) { + return false; + } + if (!"REPOSITORY_SYNC".equals(task.kind()) || !"HEAD_ONLY".equals(task.mode()) + || task.sourceId() == null) { + throw new IllegalArgumentException("HEAD_ONLY completion requires a registered repository source"); + } + int updated = jdbc.update(""" + UPDATE github_source_binding + SET last_observed_commit = ?, unreachable_attempts = 0, last_error_code = NULL, + status = CASE WHEN status = 'PAUSED' THEN status ELSE 'ACTIVE' END, + next_check_at = ?, updated_at = ? + WHERE id = ? + """, observedCommitSha, Timestamp.from(nextCheckAt), Timestamp.from(now), task.sourceId()); + if (updated != 1) { + throw new IllegalStateException("GitHub source disappeared during task completion"); + } + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', fixed_commit_sha = ?, completed_at = ?, + retry_at = NULL, lease_token_hash = NULL, lease_expires_at = NULL, + leased_by = NULL, last_error_code = NULL, last_error_detail = NULL, updated_at = ? + WHERE id = ? + """, observedCommitSha, Timestamp.from(now), Timestamp.from(now), taskId); + return true; + } + + @Override + public Optional fail(UUID taskId, long leaseGeneration, String leaseTokenHash, + GithubSyncFailureReason reason, String detail, + Instant requestedRetryAt, Instant now) { + LockedTask task = lockLease(taskId, leaseGeneration, leaseTokenHash, now); + if (task == null) { + return Optional.empty(); + } + Instant retryAt = requestedRetryAt; + if (reason == GithubSyncFailureReason.SOURCE_UNREADABLE) { + Integer attempts = task.sourceId() == null + ? jdbc.query(""" + SELECT unreachable_attempts FROM github_discovery_candidate + WHERE validation_task_id = ? FOR UPDATE + """, result -> result.next() ? result.getInt("unreachable_attempts") : null, + taskId) + : jdbc.query(""" + SELECT unreachable_attempts FROM github_source_binding WHERE id = ? FOR UPDATE + """, statement -> statement.setLong(1, task.sourceId()), result -> + result.next() ? result.getInt("unreachable_attempts") : null); + if (attempts == null) { + throw new IllegalStateException("GitHub repository source or candidate disappeared"); + } + int nextAttempts = attempts + 1; + Duration backoff = nextAttempts == 1 ? Duration.ofDays(1) + : nextAttempts == 2 ? Duration.ofDays(3) : Duration.ofDays(7); + retryAt = now.plus(backoff); + if (task.sourceId() == null) { + jdbc.update(""" + UPDATE github_discovery_candidate + SET unreachable_attempts = ?, last_error_code = ?, next_validation_at = ?, + status = CASE WHEN ? >= 3 THEN 'UNREACHABLE' ELSE status END + WHERE validation_task_id = ? + """, nextAttempts, reason.name(), Timestamp.from(retryAt), + nextAttempts, taskId); + } else { + jdbc.update(""" + UPDATE github_source_binding + SET unreachable_attempts = ?, last_error_code = ?, next_check_at = ?, + status = CASE WHEN status = 'PAUSED' THEN status + WHEN ? >= 3 THEN 'UNREACHABLE' ELSE status END, + updated_at = ? + WHERE id = ? + """, nextAttempts, reason.name(), Timestamp.from(retryAt), + nextAttempts, Timestamp.from(now), task.sourceId()); + } + } + if (retryAt == null) { + retryAt = now.plus(reason == GithubSyncFailureReason.INCOMPLETE_TREE + ? Duration.ofMinutes(30) : Duration.ofMinutes(15)); + } + jdbc.update(""" + UPDATE github_sync_task + SET status = 'RETRY_WAIT', retry_at = ?, due_at = ?, last_error_code = ?, + last_error_detail = ?, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + updated_at = ? + WHERE id = ? + """, Timestamp.from(retryAt), Timestamp.from(retryAt), reason.name(), detail, + Timestamp.from(now), taskId); + jdbc.update(""" + UPDATE github_sync_package_admission + SET status = 'RELEASED', updated_at = ? + WHERE task_id = ? AND lease_generation = ? AND status = 'ACTIVE' + """, Timestamp.from(now), taskId, leaseGeneration); + return Optional.of(retryAt); + } + + @Override + public GithubDiscoveryBatchResult recordDiscoveryBatch(UUID taskId, long leaseGeneration, + String leaseTokenHash, GithubDiscoveryBatch batch, String requestSha256, Instant now) { + DiscoveryLease lease = lockDiscoveryLease(taskId, leaseGeneration, leaseTokenHash, now); + if (lease == null) { + return GithubDiscoveryBatchResult.LEASE_STALE; + } + if (!List.of("SEARCH_RULE", "CURATED_OWNER", "CURATED_REPOSITORY") + .contains(lease.entryType())) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + String priorDigest = jdbc.query(""" + SELECT request_sha256 FROM github_discovery_batch + WHERE task_id = ? AND checkpoint = ? + """, statement -> { + statement.setObject(1, taskId); + statement.setString(2, batch.checkpoint()); + }, result -> result.next() ? result.getString("request_sha256") : null); + if (priorDigest != null) { + return priorDigest.equals(requestSha256) + ? GithubDiscoveryBatchResult.ACCEPTED : GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + if (!Objects.equals(lease.checkpoint(), batch.previousCheckpoint())) { + return GithubDiscoveryBatchResult.CHECKPOINT_CONFLICT; + } + // Serialize retention checks across discovery tasks for this namespace. + jdbc.queryForObject("SELECT id FROM namespace WHERE id = ? FOR UPDATE", + Long.class, lease.namespaceId()); + if ("CURATED_REPOSITORY".equals(lease.entryType())) { + if (batch.repositories().size() > 1 || lease.entryId() == null) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + String namespaceType = jdbc.queryForObject( + "SELECT type FROM namespace WHERE id = ?", String.class, lease.namespaceId()); + if (!"CATALOG".equals(namespaceType) && !"TEAM".equals(namespaceType)) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + ConfiguredRepository configured = jdbc.query(""" + SELECT source_visibility, enabled FROM github_discovery_entry + WHERE id = ? AND namespace_id = ? AND entry_type = 'CURATED_REPOSITORY' + FOR UPDATE + """, result -> result.next() + ? new ConfiguredRepository(result.getString("source_visibility"), + result.getBoolean("enabled")) : null, + UUID.fromString(lease.entryId()), lease.namespaceId()); + if (configured == null || "TEAM".equals(namespaceType) + && !List.of("PUBLIC", "NAMESPACE_ONLY").contains(configured.visibility())) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + Integer priorRepository = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch + WHERE task_id = ? AND curated_repository_id IS NOT NULL + """, Integer.class, taskId); + if (!batch.repositories().isEmpty() && priorRepository != null && priorRepository > 0) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + for (GithubDiscoveryCandidate candidate : batch.repositories()) { + if ("TEAM".equals(namespaceType)) { + Integer conflictingOrigin = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding source + JOIN github_source_origin origin ON origin.source_id = source.id + WHERE source.namespace_id = ? AND source.repository_id = ? + AND origin.origin_type = 'TEAM_CONFIG' AND origin.origin_key <> ? + """, Integer.class, lease.namespaceId(), candidate.repositoryId(), + lease.entryId()); + if (conflictingOrigin != null && conflictingOrigin > 0) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + } + int pinned = jdbc.update(""" + UPDATE github_discovery_entry + SET repository_id = COALESCE(repository_id, ?), updated_at = ? + WHERE id = ? AND namespace_id = ? AND entry_type = 'CURATED_REPOSITORY' + AND (repository_id IS NULL OR repository_id = ?) + """, candidate.repositoryId(), Timestamp.from(now), + UUID.fromString(lease.entryId()), lease.namespaceId(), candidate.repositoryId()); + if (pinned != 1) { + return GithubDiscoveryBatchResult.RESULT_CONFLICT; + } + Long sourceId = jdbc.query(connection -> { + var statement = connection.prepareStatement(""" + INSERT INTO github_source_binding + (namespace_id, repository_id, owner_login, repository_name, + default_branch, visibility, status, next_check_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (namespace_id, repository_id) DO UPDATE + SET owner_login = EXCLUDED.owner_login, + repository_name = EXCLUDED.repository_name, + default_branch = EXCLUDED.default_branch, + updated_at = CURRENT_TIMESTAMP + RETURNING id + """); + statement.setLong(1, lease.namespaceId()); + statement.setLong(2, candidate.repositoryId()); + statement.setString(3, candidate.owner()); + statement.setString(4, candidate.name()); + statement.setString(5, candidate.defaultBranch()); + statement.setString(6, "CATALOG".equals(namespaceType) + ? "PUBLIC" : configured.visibility()); + statement.setString(7, "TEAM".equals(namespaceType) && !configured.enabled() + ? "PAUSED" : "ACTIVE"); + statement.setTimestamp(8, Timestamp.from(now)); + return statement; + }, result -> result.next() ? result.getLong("id") : null); + if (sourceId == null) { + throw new IllegalStateException("Could not persist curated GitHub repository"); + } + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + VALUES (?, ?, ?) + ON CONFLICT (source_id, origin_type, origin_key) DO NOTHING + """, sourceId, + "TEAM".equals(namespaceType) ? "TEAM_CONFIG" : "CURATED_REPOSITORY", + lease.entryId()); + } + Long repositoryId = batch.repositories().isEmpty() + ? null : batch.repositories().get(0).repositoryId(); + recordDiscoveryCheckpoint(taskId, batch, requestSha256, now, repositoryId); + return GithubDiscoveryBatchResult.ACCEPTED; + } + var previousHits = new HashSet<>(jdbc.queryForList( + "SELECT repository_id FROM github_discovery_task_hit WHERE task_id = ?", + Long.class, taskId)); + long newCount = batch.repositories().stream() + .filter(candidate -> !previousHits.contains(candidate.repositoryId())).count(); + Integer leaseHits = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_task_hit + WHERE task_id = ? AND lease_generation = ? + """, Integer.class, taskId, leaseGeneration); + Integer pendingCount = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_candidate + WHERE namespace_id = ? AND status IN ('PENDING', 'VALIDATING') + """, Integer.class, lease.namespaceId()); + Integer pendingLimit = jdbc.queryForObject(""" + SELECT candidate_pending_limit FROM github_sync_admission_policy WHERE id = TRUE + """, Integer.class); + int newPendingCount = 0; + for (GithubDiscoveryCandidate candidate : batch.repositories()) { + String existingStatus = jdbc.query(""" + SELECT status FROM github_discovery_candidate + WHERE namespace_id = ? AND repository_id = ? + """, statement -> { + statement.setLong(1, lease.namespaceId()); + statement.setLong(2, candidate.repositoryId()); + }, result -> result.next() ? result.getString("status") : null); + Integer existingSource = jdbc.queryForObject(""" + SELECT count(*) FROM github_source_binding + WHERE namespace_id = ? AND repository_id = ? + """, Integer.class, lease.namespaceId(), candidate.repositoryId()); + if (existingStatus == null && existingSource == 0) { + newPendingCount++; + } + } + if (newCount > 0 && (leaseHits == null ? 0 : leaseHits) + newCount > lease.budget() + || newPendingCount > 0 + && (pendingCount == null ? 0 : pendingCount) + newPendingCount > pendingLimit) { + return GithubDiscoveryBatchResult.BUDGET_EXCEEDED; + } + var createdRepositories = new java.util.ArrayList(); + for (GithubDiscoveryCandidate candidate : batch.repositories().stream() + .sorted(Comparator.comparingLong(GithubDiscoveryCandidate::repositoryId)).toList()) { + Long candidateId = jdbc.query(connection -> { + var statement = connection.prepareStatement(""" + INSERT INTO github_discovery_candidate + (namespace_id, repository_id, owner_login, repository_name, + default_branch, first_seen_at, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (namespace_id, repository_id) DO NOTHING + RETURNING id + """); + statement.setLong(1, lease.namespaceId()); + statement.setLong(2, candidate.repositoryId()); + statement.setString(3, candidate.owner()); + statement.setString(4, candidate.name()); + statement.setString(5, candidate.defaultBranch()); + statement.setTimestamp(6, Timestamp.from(now)); + statement.setTimestamp(7, Timestamp.from(now)); + return statement; + }, result -> result.next() ? result.getLong("id") : null); + if (candidateId != null) { + createdRepositories.add(candidate.repositoryId()); + } else { + candidateId = jdbc.queryForObject(""" + UPDATE github_discovery_candidate SET owner_login = ?, repository_name = ?, default_branch = ?, + last_seen_at = ? WHERE namespace_id = ? AND repository_id = ? RETURNING id + """, Long.class, candidate.owner(), candidate.name(), candidate.defaultBranch(), Timestamp.from(now), + lease.namespaceId(), candidate.repositoryId()); + } + jdbc.update(""" + INSERT INTO github_discovery_task_hit + (task_id, repository_id, candidate_id, lease_generation, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (task_id, repository_id) DO NOTHING + """, taskId, candidate.repositoryId(), candidateId, + leaseGeneration, Timestamp.from(now)); + } + jdbc.update(""" + INSERT INTO github_discovery_candidate_origin + (candidate_id, origin_type, origin_key, first_seen_at, last_seen_at) + SELECT hit.candidate_id, task.snapshot_json ->> 'entryType', + CASE WHEN task.snapshot_json ->> 'entryType' = 'CURATED_OWNER' + THEN task.snapshot_json ->> 'owner' + ELSE task.snapshot_json ->> 'ruleId' END, + ?, ? + FROM github_discovery_task_hit hit + JOIN github_sync_task task ON task.id = hit.task_id + WHERE hit.task_id = ? + AND task.snapshot_json ->> 'entryType' IN ('SEARCH_RULE', 'CURATED_OWNER') + AND CASE WHEN task.snapshot_json ->> 'entryType' = 'CURATED_OWNER' + THEN task.snapshot_json ->> 'owner' + ELSE task.snapshot_json ->> 'ruleId' END IS NOT NULL + ON CONFLICT (candidate_id, origin_type, origin_key) DO UPDATE + SET last_seen_at = GREATEST(github_discovery_candidate_origin.last_seen_at, + EXCLUDED.last_seen_at) + """, Timestamp.from(now), Timestamp.from(now), taskId); + createdRepositories.forEach(repositoryId -> outcomes.candidateCreated(lease.namespaceId(), repositoryId, now)); + jdbc.update(""" + UPDATE github_discovery_candidate candidate + SET status = 'BOUND', last_error_code = NULL + WHERE candidate.namespace_id = ? + AND candidate.id IN ( + SELECT hit.candidate_id FROM github_discovery_task_hit hit + WHERE hit.task_id = ?) + AND EXISTS (SELECT 1 FROM github_source_binding source + WHERE source.namespace_id = candidate.namespace_id + AND source.repository_id = candidate.repository_id) + """, lease.namespaceId(), taskId); + jdbc.update(""" + INSERT INTO github_source_origin (source_id, origin_type, origin_key) + SELECT source.id, origin.origin_type, origin.origin_key + FROM github_discovery_task_hit hit + JOIN github_discovery_candidate candidate ON candidate.id = hit.candidate_id + JOIN github_source_binding source + ON source.namespace_id = candidate.namespace_id + AND source.repository_id = candidate.repository_id + JOIN github_discovery_candidate_origin origin ON origin.candidate_id = candidate.id + WHERE hit.task_id = ? + ON CONFLICT (source_id, origin_type, origin_key) DO NOTHING + """, taskId); + recordDiscoveryCheckpoint(taskId, batch, requestSha256, now, null); + return GithubDiscoveryBatchResult.ACCEPTED; + } + + private void recordDiscoveryCheckpoint(UUID taskId, GithubDiscoveryBatch batch, + String requestSha256, Instant now, Long curatedRepositoryId) { + jdbc.update(""" + INSERT INTO github_discovery_batch + (task_id, checkpoint, request_sha256, deferred_count, deferral_reason, + curated_repository_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, taskId, batch.checkpoint(), requestSha256, batch.deferredCount(), + batch.deferralReason() == null ? null : batch.deferralReason().name(), + curatedRepositoryId, Timestamp.from(now)); + jdbc.update(""" + UPDATE github_sync_task SET discovery_checkpoint = ?, updated_at = ? WHERE id = ? + """, batch.checkpoint(), Timestamp.from(now), taskId); + } + + + @Override + public GithubDiscoveryCompletionResult completeDiscovery(UUID taskId, long leaseGeneration, + String leaseTokenHash, String checkpoint, boolean complete, + Instant retryAt, GithubSyncDeferralReason deferralReason, Instant now) { + DiscoveryLease lease = lockDiscoveryLease(taskId, leaseGeneration, leaseTokenHash, now); + if (lease == null) { + return GithubDiscoveryCompletionResult.LEASE_STALE; + } + if (!List.of("SEARCH_RULE", "CURATED_OWNER", "CURATED_REPOSITORY") + .contains(lease.entryType())) { + return GithubDiscoveryCompletionResult.CHECKPOINT_CONFLICT; + } + if (!Objects.equals(lease.checkpoint(), checkpoint)) { + return GithubDiscoveryCompletionResult.CHECKPOINT_CONFLICT; + } + Integer persisted = jdbc.queryForObject(""" + SELECT count(*) FROM github_discovery_batch WHERE task_id = ? AND checkpoint = ? + """, Integer.class, taskId, checkpoint); + if (persisted == null || persisted == 0) { + return GithubDiscoveryCompletionResult.CHECKPOINT_CONFLICT; + } + if (complete) { + jdbc.update(""" + UPDATE github_sync_task + SET status = 'ACQUISITION_COMPLETE', completed_at = ?, retry_at = NULL, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + last_error_code = NULL, last_error_detail = NULL, updated_at = ? + WHERE id = ? + """, Timestamp.from(now), Timestamp.from(now), taskId); + return GithubDiscoveryCompletionResult.ACQUISITION_COMPLETE; + } + jdbc.update(""" + UPDATE github_sync_task + SET status = 'RETRY_WAIT', retry_at = ?, due_at = ?, + last_error_code = ?, last_error_detail = NULL, + lease_token_hash = NULL, lease_expires_at = NULL, + leased_by = NULL, updated_at = ? + WHERE id = ? + """, Timestamp.from(retryAt), Timestamp.from(retryAt), + deferralReason == null ? "DISCOVERY_INCOMPLETE" : deferralReason.name(), + Timestamp.from(now), taskId); + return GithubDiscoveryCompletionResult.RETRY_WAIT; + } + + @Override + public boolean completePreview(UUID taskId, long leaseGeneration, String leaseTokenHash, + String resultJson, Instant now) { + int updated = jdbc.update(""" + UPDATE github_sync_task + SET snapshot_json = jsonb_set(snapshot_json, '{previewResult}', ?::jsonb), + status = 'ACQUISITION_COMPLETE', completed_at = ?, retry_at = NULL, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + last_error_code = NULL, last_error_detail = NULL, updated_at = ? + WHERE id = ? AND kind = 'DISCOVERY' AND status = 'LEASED' + AND snapshot_json ->> 'entryType' = 'SEARCH_PREVIEW' + AND lease_generation = ? AND lease_token_hash = ? AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + """, resultJson, Timestamp.from(now), Timestamp.from(now), taskId, + leaseGeneration, leaseTokenHash, Timestamp.from(now)); + if (updated == 1) { + jdbc.update(""" + UPDATE github_sync_run SET status = 'ACQUISITION_COMPLETE', finished_at = ? + WHERE id = (SELECT run_id FROM github_sync_task WHERE id = ?) + """, Timestamp.from(now), taskId); + } + return updated == 1; + } + + @Override + public GithubRepositoryCompletionResult completeRepository(UUID taskId, long leaseGeneration, + String leaseTokenHash, String commitSha, boolean complete, String pathCheckpoint, + List incompleteSubtrees, List pathResults, + Instant retryAt, GithubSyncDeferralReason deferralReason, Instant now, Instant nextCheckAt) { + RepositoryLease lease = jdbc.query(connection -> { + var statement = connection.prepareStatement(LOCK_REPOSITORY_LEASE_SQL); + statement.setObject(1, taskId); + statement.setLong(2, leaseGeneration); + statement.setString(3, leaseTokenHash); + statement.setTimestamp(4, Timestamp.from(now)); + return statement; + }, result -> result.next() ? new RepositoryLease((Long) result.getObject("source_id"), + result.getString("fixed_commit_sha"), result.getString("snapshot_commit_sha"), + result.getString("path_checkpoint")) : null); + if (lease == null) { + return GithubRepositoryCompletionResult.LEASE_STALE; + } + if (lease.fixedCommitSha() != null && !lease.fixedCommitSha().equals(commitSha) + || lease.snapshotCommitSha() != null && !lease.snapshotCommitSha().equals(commitSha)) { + return GithubRepositoryCompletionResult.COMMIT_MISMATCH; + } + if (lease.pathCheckpoint() != null && pathCheckpoint != null + && comparePathCheckpoints(pathCheckpoint, lease.pathCheckpoint()) < 0) { + return GithubRepositoryCompletionResult.CHECKPOINT_CONFLICT; + } + Map persistedPaths = new HashMap<>(); + jdbc.query(""" + SELECT skill_path, outcome, ingestion_id, reason FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? FOR UPDATE + """, (result, rowNumber) -> Map.entry(result.getString("skill_path"), + new PathResult(result.getString("outcome"), + result.getObject("ingestion_id", UUID.class), result.getString("reason"))), + taskId, commitSha).forEach(entry -> persistedPaths.put(entry.getKey(), entry.getValue())); + for (GithubRepositoryPathResult path : pathResults) { + PathResult persisted = persistedPaths.get(path.skillPath()); + if (path.outcome() == GithubRepositoryPathResult.Outcome.ACCEPTED) { + if (persisted == null || !"ACCEPTED".equals(persisted.outcome()) + || !path.ingestionId().equals(persisted.ingestionId())) { + return GithubRepositoryCompletionResult.RESULT_CONFLICT; + } + Integer ingestion = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_package_ingestion + WHERE id = ? AND task_id = ? AND commit_sha = ? AND skill_path = ? + AND status NOT IN ('FAILED', 'NO_CHANGE') + """, Integer.class, path.ingestionId(), taskId, commitSha, path.skillPath()); + if (ingestion == null || ingestion != 1) { + return GithubRepositoryCompletionResult.RESULT_CONFLICT; + } + } else if (persisted != null && !"RETRYABLE_FAILURE".equals(persisted.outcome()) + && (!persisted.outcome().equals(path.outcome().name()) + || !Objects.equals(persisted.reason(), path.reason()))) { + return GithubRepositoryCompletionResult.RESULT_CONFLICT; + } + } + if (complete) { + var resolvedPaths = new HashSet(); + var failedAdmissions = new HashSet(); + for (GithubRepositoryPathResult path : pathResults) { + if (path.outcome() != GithubRepositoryPathResult.Outcome.RETRYABLE_FAILURE) { + resolvedPaths.add(path.skillPath()); + } + if (path.outcome() == GithubRepositoryPathResult.Outcome.TERMINAL_FAILURE + || path.outcome() == GithubRepositoryPathResult.Outcome.IGNORED) { + failedAdmissions.add(path.skillPath()); + } + } + for (Map.Entry persisted : persistedPaths.entrySet()) { + if ("RETRYABLE_FAILURE".equals(persisted.getValue().outcome()) + && !resolvedPaths.contains(persisted.getKey())) { + return GithubRepositoryCompletionResult.COVERAGE_CONFLICT; + } + } + List activePaths = jdbc.query(""" + SELECT skill_path FROM github_sync_package_admission + WHERE task_id = ? AND lease_generation = ? AND status = 'ACTIVE' + FOR UPDATE + """, (result, row) -> result.getString("skill_path"), taskId, leaseGeneration); + if (activePaths.stream().anyMatch(path -> !failedAdmissions.contains(path))) { + return GithubRepositoryCompletionResult.COVERAGE_CONFLICT; + } + for (String path : activePaths) { + jdbc.update(""" + UPDATE github_sync_package_admission + SET status = 'RELEASED', updated_at = ? + WHERE task_id = ? AND lease_generation = ? AND skill_path = ? AND status = 'ACTIVE' + """, Timestamp.from(now), taskId, leaseGeneration, path); + } + } + for (GithubRepositoryPathResult path : pathResults) { + if (path.outcome() == GithubRepositoryPathResult.Outcome.ACCEPTED) { + continue; + } + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, reason, retry_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (task_id, commit_sha, skill_path) DO UPDATE + SET outcome = EXCLUDED.outcome, reason = EXCLUDED.reason, + retry_at = EXCLUDED.retry_at, updated_at = CURRENT_TIMESTAMP + WHERE github_sync_path_result.outcome = 'RETRYABLE_FAILURE' + """, taskId, commitSha, path.skillPath(), path.outcome().name(), + path.reason(), path.outcome() == GithubRepositoryPathResult.Outcome.RETRYABLE_FAILURE + ? Timestamp.from(retryAt) : null); + } + String subtreesJson; + try { + subtreesJson = objectMapper.writeValueAsString(incompleteSubtrees); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Could not serialize incomplete GitHub subtrees", exception); + } + if (complete) { + Integer terminal = jdbc.queryForObject(""" + SELECT count(*) FROM github_sync_path_result + WHERE task_id = ? AND commit_sha = ? AND outcome = 'TERMINAL_FAILURE' + """, Integer.class, taskId, commitSha); + String status = terminal != null && terminal > 0 ? "PARTIAL" : "ACQUISITION_COMPLETE"; + if (lease.sourceId() != null) { + int updated = jdbc.update(""" + UPDATE github_source_binding source + SET last_observed_commit = ?, + last_partial_commit = CASE + WHEN ? = 'PARTIAL' + AND COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns + THEN ? ELSE NULL END, + unreachable_attempts = 0, last_error_code = NULL, + status = CASE WHEN source.status = 'PAUSED' THEN source.status ELSE 'ACTIVE' END, + next_check_at = CASE + WHEN COALESCE(task.snapshot_json -> 'ignorePatterns', '[]'::jsonb) + = source.ignore_patterns THEN ?::timestamptz ELSE ?::timestamptz END, + updated_at = ? + FROM github_sync_task task + WHERE source.id = ? AND task.id = ? + """, commitSha, status, commitSha, Timestamp.from(nextCheckAt), + Timestamp.from(now), Timestamp.from(now), lease.sourceId(), taskId); + if (updated != 1) { + throw new IllegalStateException("GitHub source disappeared during repository completion"); + } + } + jdbc.update(""" + UPDATE github_sync_task + SET status = ?, fixed_commit_sha = ?, path_checkpoint = COALESCE(?, path_checkpoint), + incomplete_subtrees = '[]'::jsonb, completed_at = ?, retry_at = NULL, + lease_token_hash = NULL, lease_expires_at = NULL, leased_by = NULL, + last_error_code = ?, last_error_detail = NULL, updated_at = ? + WHERE id = ? + """, status, commitSha, pathCheckpoint, Timestamp.from(now), + terminal != null && terminal > 0 ? "PATH_TERMINAL_FAILURE" : null, + Timestamp.from(now), taskId); + // A publication deferred while the tree was still being scanned can be decided now. + jdbc.update(""" + UPDATE github_sync_package_ingestion + SET publication_retry_at = ?, updated_at = ? + WHERE task_id = ? AND status = 'VALIDATED' + AND last_error_code = 'PATH_MOVE_SCAN_INCOMPLETE' + AND publication_token IS NULL AND publication_retry_at > ? + """, Timestamp.from(now), Timestamp.from(now), taskId, Timestamp.from(now)); + if ("ACQUISITION_COMPLETE".equals(status)) { + if (lease.sourceId() != null) { + presence.reconcile(lease.sourceId(), taskId, commitSha, now); + } + checkpoints.advanceIfComplete(taskId, commitSha, now); + } + return terminal != null && terminal > 0 + ? GithubRepositoryCompletionResult.PARTIAL + : GithubRepositoryCompletionResult.ACQUISITION_COMPLETE; + } + jdbc.update(""" + UPDATE github_sync_task + SET status = 'RETRY_WAIT', fixed_commit_sha = ?, + path_checkpoint = COALESCE(?, path_checkpoint), + incomplete_subtrees = ?::jsonb, retry_at = ?, due_at = ?, + last_error_code = ?, last_error_detail = NULL, + lease_token_hash = NULL, lease_expires_at = NULL, + leased_by = NULL, updated_at = ? + WHERE id = ? + """, commitSha, pathCheckpoint, subtreesJson, Timestamp.from(retryAt), + Timestamp.from(retryAt), deferralReason == null ? "REPOSITORY_INCOMPLETE" + : deferralReason.name(), Timestamp.from(now), taskId); + jdbc.update(""" + UPDATE github_sync_package_admission + SET status = 'RELEASED', updated_at = ? + WHERE task_id = ? AND lease_generation = ? AND status = 'ACTIVE' + """, Timestamp.from(now), taskId, leaseGeneration); + if (lease.sourceId() != null) { + jdbc.update(""" + UPDATE github_source_binding SET last_observed_commit = ?, updated_at = ? WHERE id = ? + """, commitSha, Timestamp.from(now), lease.sourceId()); + } + return GithubRepositoryCompletionResult.RETRY_WAIT; + } + + private record RepositoryLease(Long sourceId, String fixedCommitSha, String snapshotCommitSha, + String pathCheckpoint) {} + private record PathResult(String outcome, UUID ingestionId, String reason) {} + + // Go sorts UTF-8 paths by code point. String.compareTo uses UTF-16 code units and can + // disagree when a path contains supplementary Unicode characters. + private int comparePathCheckpoints(String left, String right) { + int leftOffset = 0; + int rightOffset = 0; + while (leftOffset < left.length() && rightOffset < right.length()) { + int leftCodePoint = left.codePointAt(leftOffset); + int rightCodePoint = right.codePointAt(rightOffset); + if (leftCodePoint != rightCodePoint) { + return Integer.compare(leftCodePoint, rightCodePoint); + } + leftOffset += Character.charCount(leftCodePoint); + rightOffset += Character.charCount(rightCodePoint); + } + return Integer.compare(left.length() - leftOffset, right.length() - rightOffset); + } + + private DiscoveryLease lockDiscoveryLease(UUID taskId, long leaseGeneration, + String leaseTokenHash, Instant now) { + return jdbc.query(connection -> { + var statement = connection.prepareStatement(LOCK_DISCOVERY_LEASE_SQL); + statement.setObject(1, taskId); + statement.setLong(2, leaseGeneration); + statement.setString(3, leaseTokenHash); + statement.setTimestamp(4, Timestamp.from(now)); + return statement; + }, result -> { + if (!result.next()) { + return null; + } + String budgetText = result.getString("candidate_budget"); + if (budgetText == null) { + throw new IllegalStateException("Discovery task has no candidate budget"); + } + int budget; + try { + budget = Integer.parseInt(budgetText); + } catch (NumberFormatException exception) { + throw new IllegalStateException("Discovery task has an invalid candidate budget", exception); + } + if (budget < 1) { + throw new IllegalStateException("Discovery candidate budget must be positive"); + } + return new DiscoveryLease(result.getLong("namespace_id"), + result.getString("discovery_checkpoint"), budget, + result.getString("entry_type"), result.getString("entry_id")); + }); + } + + private record DiscoveryLease(long namespaceId, String checkpoint, int budget, + String entryType, String entryId) {} + + private record ConfiguredRepository(String visibility, boolean enabled) {} + + private LockedTask lockLease(UUID taskId, long generation, String tokenHash, Instant now) { + return jdbc.query(connection -> { + var statement = connection.prepareStatement(LOCK_LEASE_SQL); + statement.setObject(1, taskId); + statement.setLong(2, generation); + statement.setString(3, tokenHash); + statement.setTimestamp(4, Timestamp.from(now)); + return statement; + }, result -> { + if (!result.next()) { + return null; + } + long sourceId = result.getLong("source_id"); + return new LockedTask(result.wasNull() ? null : sourceId, + result.getString("kind"), result.getString("mode")); + }); + } + + private record LockedTask(Long sourceId, String kind, String mode) {} + + private GithubSyncTaskLease mapLease(ResultSet result) throws SQLException { + long sourceId = result.getLong("source_id"); + Long nullableSourceId = result.wasNull() ? null : sourceId; + String mode = result.getString("mode"); + return new GithubSyncTaskLease( + result.getObject("id", UUID.class), + result.getObject("run_id", UUID.class), + GithubSyncTaskKind.valueOf(result.getString("kind")), + mode == null ? null : GithubSyncTaskMode.valueOf(mode), + result.getLong("namespace_id"), + nullableSourceId, + result.getString("snapshot_json"), + result.getString("discovery_checkpoint"), + result.getString("fixed_commit_sha"), + result.getString("path_checkpoint"), + result.getString("incomplete_subtrees"), + List.of((String[]) result.getArray("retry_paths").getArray()), + List.of((String[]) result.getArray("accepted_paths").getArray()), + result.getLong("lease_generation"), + result.getTimestamp("lease_expires_at").toInstant(), + List.of((String[]) result.getArray("required_capabilities").getArray()), + result.getObject("platform_control_version",Long.class)); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncUploadRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncUploadRepository.java new file mode 100644 index 000000000..bd395b28d --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSyncUploadRepository.java @@ -0,0 +1,271 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSyncUploadRepository; +import com.iflytek.skillhub.githubsync.GithubPackageReportResult; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSyncUploadRepository implements GithubSyncUploadRepository { + private final JdbcTemplate jdbc; + + public JdbcGithubSyncUploadRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @Transactional + public boolean issue(UUID taskId, long generation, String leaseTokenHash, UUID admissionId, + String commitSha, String skillPath, UUID uploadId, String objectKey, + String mode, String ticketHash, long sizeBytes, String sha256, Instant expiresAt, Instant now) { + Integer valid = jdbc.query(""" + SELECT 1 FROM github_sync_package_admission admission + JOIN github_sync_task task ON task.id = admission.task_id + WHERE admission.id = ? AND admission.task_id = ? AND admission.lease_generation = ? + AND admission.commit_sha = ? AND admission.skill_path = ? AND admission.status = 'ACTIVE' + AND task.status = 'LEASED' AND task.lease_generation = ? + AND task.lease_token_hash = ? AND task.lease_expires_at > ? + FOR UPDATE OF task, admission + """, statement -> { + statement.setObject(1, admissionId); + statement.setObject(2, taskId); + statement.setLong(3, generation); + statement.setString(4, commitSha); + statement.setString(5, skillPath); + statement.setLong(6, generation); + statement.setString(7, leaseTokenHash); + statement.setTimestamp(8, Timestamp.from(now)); + }, result -> result.next() ? 1 : null); + if (valid == null) { + return false; + } + jdbc.update(""" + INSERT INTO github_sync_staged_upload + (id, admission_id, object_key, mode, ticket_hash, size_bytes, sha256, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, uploadId, admissionId, objectKey, mode, ticketHash, + sizeBytes, sha256, Timestamp.from(expiresAt)); + return true; + } + + @Override + @Transactional + public Optional beginLocalUpload(UUID uploadId, String ticketHash, Instant now) { + LocalUpload upload = jdbc.query(""" + SELECT upload.object_key, upload.size_bytes, upload.sha256 + FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + JOIN github_sync_task task ON task.id = admission.task_id + WHERE upload.id = ? AND upload.mode = 'LOCAL_STREAM' AND upload.status = 'ISSUED' + AND upload.ticket_hash = ? AND upload.expires_at > ? + AND admission.status = 'ACTIVE' AND task.status = 'LEASED' + AND task.lease_generation = admission.lease_generation AND task.lease_expires_at > ? + FOR UPDATE OF upload + """, statement -> { + statement.setObject(1, uploadId); + statement.setString(2, ticketHash); + statement.setTimestamp(3, Timestamp.from(now)); + statement.setTimestamp(4, Timestamp.from(now)); + }, result -> result.next() ? new LocalUpload(result.getString("object_key"), + result.getLong("size_bytes"), result.getString("sha256")) : null); + if (upload == null) { + return Optional.empty(); + } + jdbc.update(""" + UPDATE github_sync_staged_upload SET status = 'UPLOADING', updated_at = ? WHERE id = ? + """, Timestamp.from(now), uploadId); + return Optional.of(upload); + } + + @Override + @Transactional + public boolean finishLocalUpload(UUID uploadId, Instant now) { + return jdbc.update(""" + UPDATE github_sync_staged_upload AS upload + SET status = 'UPLOADED', updated_at = ? + FROM github_sync_package_admission admission, github_sync_task task + WHERE upload.id = ? AND upload.status = 'UPLOADING' + AND admission.id = upload.admission_id AND admission.status = 'ACTIVE' + AND task.id = admission.task_id AND task.status = 'LEASED' + AND task.lease_generation = admission.lease_generation AND task.lease_expires_at > ? + """, Timestamp.from(now), uploadId, Timestamp.from(now)) == 1; + } + + @Override + public boolean failUpload(UUID uploadId, Instant now) { + return jdbc.update(""" + UPDATE github_sync_staged_upload SET status = 'FAILED', updated_at = ? + WHERE id = ? AND status IN ('ISSUED', 'UPLOADING', 'UPLOADED') + """, Timestamp.from(now), uploadId) == 1; + } + + @Override + @Transactional + public List claimCleanup(Instant now, Duration signedUrlGrace, int limit) { + if (limit < 1 || limit > 100) { + throw new IllegalArgumentException("cleanup batch limit must be between 1 and 100"); + } + return jdbc.query(""" + WITH picked AS ( + SELECT upload.id FROM github_sync_staged_upload upload + JOIN github_sync_package_admission admission ON admission.id = upload.admission_id + JOIN github_sync_task task ON task.id = admission.task_id + WHERE upload.status IN ('ISSUED', 'UPLOADING', 'UPLOADED', 'FAILED', 'DELETING') + AND (upload.mode = 'LOCAL_STREAM' OR upload.expires_at < ?) + AND ( + (upload.status = 'FAILED') + OR (upload.status = 'DELETING' AND upload.updated_at < ?) + OR (upload.status IN ('ISSUED', 'UPLOADING', 'UPLOADED') + AND (task.status <> 'LEASED' OR task.lease_generation <> admission.lease_generation + OR task.lease_expires_at <= ?) + AND (upload.status <> 'UPLOADING' OR upload.updated_at < ?)) + ) + ORDER BY upload.expires_at, upload.id + LIMIT ? FOR UPDATE OF upload SKIP LOCKED + ) + UPDATE github_sync_staged_upload upload + SET status = 'DELETING', updated_at = ? + FROM picked WHERE upload.id = picked.id + RETURNING upload.id, upload.object_key + """, statement -> { + statement.setTimestamp(1, Timestamp.from(now.minus(signedUrlGrace))); + statement.setTimestamp(2, Timestamp.from(now.minus(Duration.ofMinutes(15)))); + statement.setTimestamp(3, Timestamp.from(now)); + statement.setTimestamp(4, Timestamp.from(now.minus(Duration.ofHours(1)))); + statement.setInt(5, limit); + statement.setTimestamp(6, Timestamp.from(now)); + }, (result, row) -> new CleanupUpload(result.getObject("id", UUID.class), + result.getString("object_key"))); + } + + @Override + public void markCleaned(UUID uploadId, Instant now) { + jdbc.update(""" + UPDATE github_sync_staged_upload + SET status = 'CLEANED', updated_at = ? WHERE id = ? AND status = 'DELETING' + """, Timestamp.from(now), uploadId); + } + + @Override + public GithubPackageReportResult lookupReport(UUID taskId, long generation, String leaseTokenHash, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256, Instant now) { + return readReport(taskId, generation, leaseTokenHash, admissionId, commitSha, skillPath, + uploadId, sizeBytes, sha256, now, false); + } + + @Override + @Transactional + public GithubPackageReportResult acceptReport(UUID taskId, long generation, String leaseTokenHash, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256, Instant now) { + GithubPackageReportResult checked = readReport(taskId, generation, leaseTokenHash, + admissionId, commitSha, skillPath, uploadId, sizeBytes, sha256, now, true); + if (checked.status() != GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT) { + return checked; + } + ReportCoordinate coordinate = jdbc.query(""" + SELECT namespace_id, repository_id FROM github_sync_package_admission WHERE id = ? + """, statement -> statement.setObject(1, admissionId), result -> + result.next() ? new ReportCoordinate(result.getLong("namespace_id"), + result.getLong("repository_id")) : null); + if (coordinate == null) { + throw new IllegalStateException("GitHub sync package admission disappeared"); + } + UUID ingestionId = UUID.randomUUID(); + jdbc.update(""" + INSERT INTO github_sync_package_ingestion + (id, task_id, admission_id, upload_id, namespace_id, repository_id, + commit_sha, skill_path, object_key, size_bytes, sha256) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ingestionId, taskId, admissionId, uploadId, coordinate.namespaceId(), + coordinate.repositoryId(), commitSha, skillPath, checked.objectKey(), sizeBytes, sha256); + jdbc.update(""" + UPDATE github_sync_package_admission SET status = 'ACCEPTED', updated_at = ? WHERE id = ? + """, Timestamp.from(now), admissionId); + jdbc.update(""" + UPDATE github_sync_staged_upload SET status = 'ACCEPTED', updated_at = ? WHERE id = ? + """, Timestamp.from(now), uploadId); + jdbc.update(""" + INSERT INTO github_sync_path_result + (task_id, commit_sha, skill_path, outcome, ingestion_id, sha256) + VALUES (?, ?, ?, 'ACCEPTED', ?, ?) + ON CONFLICT (task_id, commit_sha, skill_path) DO UPDATE + SET outcome = 'ACCEPTED', ingestion_id = EXCLUDED.ingestion_id, + sha256 = EXCLUDED.sha256, reason = NULL, retry_at = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE github_sync_path_result.outcome <> 'ACCEPTED' + """, taskId, commitSha, skillPath, ingestionId, sha256); + return new GithubPackageReportResult(GithubPackageReportResult.Status.RECEIVED, ingestionId, null); + } + + private GithubPackageReportResult readReport(UUID taskId, long generation, String leaseTokenHash, + UUID admissionId, String commitSha, String skillPath, UUID uploadId, + long sizeBytes, String sha256, Instant now, boolean lock) { + String sql = """ + SELECT admission.status AS admission_status, upload.status AS upload_status, + upload.mode, upload.object_key, upload.size_bytes, upload.sha256 AS upload_sha256, + ingestion.id AS ingestion_id, ingestion.size_bytes AS ingestion_size, + ingestion.sha256 AS ingestion_sha256 + FROM github_sync_package_admission admission + JOIN github_sync_task task ON task.id = admission.task_id + JOIN github_sync_staged_upload upload ON upload.admission_id = admission.id + LEFT JOIN github_sync_package_ingestion ingestion + ON ingestion.task_id = task.id AND ingestion.commit_sha = admission.commit_sha + AND ingestion.skill_path = admission.skill_path + WHERE admission.id = ? AND admission.task_id = ? AND admission.lease_generation = ? + AND admission.commit_sha = ? AND admission.skill_path = ? AND upload.id = ? + AND task.status = 'LEASED' AND task.lease_generation = ? + AND task.lease_token_hash = ? AND task.lease_expires_at > ? + """ + (lock ? " FOR UPDATE OF task, admission, upload" : ""); + ReportRow row = jdbc.query(sql, statement -> { + statement.setObject(1, admissionId); + statement.setObject(2, taskId); + statement.setLong(3, generation); + statement.setString(4, commitSha); + statement.setString(5, skillPath); + statement.setObject(6, uploadId); + statement.setLong(7, generation); + statement.setString(8, leaseTokenHash); + statement.setTimestamp(9, Timestamp.from(now)); + }, result -> result.next() ? new ReportRow( + result.getString("admission_status"), result.getString("upload_status"), + result.getString("mode"), result.getString("object_key"), + result.getLong("size_bytes"), result.getString("upload_sha256"), + result.getObject("ingestion_id", UUID.class), + (Long) result.getObject("ingestion_size"), result.getString("ingestion_sha256")) : null); + if (row == null) { + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.LEASE_STALE); + } + if (row.ingestionId() != null) { + if (row.ingestionSize() == sizeBytes && row.ingestionSha256().equals(sha256)) { + return new GithubPackageReportResult(GithubPackageReportResult.Status.RECEIVED, + row.ingestionId(), null); + } + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.RESULT_CONFLICT); + } + if (!"ACTIVE".equals(row.admissionStatus()) || row.sizeBytes() != sizeBytes + || !row.uploadSha256().equals(sha256)) { + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.RESULT_CONFLICT); + } + if (("LOCAL_STREAM".equals(row.mode()) && !"UPLOADED".equals(row.uploadStatus())) + || ("S3_PRESIGNED_PUT".equals(row.mode()) && !"ISSUED".equals(row.uploadStatus()))) { + return GithubPackageReportResult.rejected(GithubPackageReportResult.Status.UPLOAD_NOT_READY); + } + return new GithubPackageReportResult(GithubPackageReportResult.Status.READY_TO_CHECK_OBJECT, + null, row.objectKey()); + } + + private record ReportCoordinate(long namespaceId, long repositoryId) {} + private record ReportRow(String admissionStatus, String uploadStatus, String mode, + String objectKey, long sizeBytes, String uploadSha256, + UUID ingestionId, Long ingestionSize, String ingestionSha256) {} +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSystemCredentialRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSystemCredentialRepository.java new file mode 100644 index 000000000..bbcefa04e --- /dev/null +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/JdbcGithubSystemCredentialRepository.java @@ -0,0 +1,224 @@ +package com.iflytek.skillhub.infra.jpa; + +import com.iflytek.skillhub.githubsync.GithubSystemCredential; +import com.iflytek.skillhub.githubsync.GithubCredentialQuota; +import com.iflytek.skillhub.githubsync.GithubSystemCredentialRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class JdbcGithubSystemCredentialRepository implements GithubSystemCredentialRepository { + private static final String FIELDS = "id, label, fingerprint, key_version, nonce, ciphertext, " + + "status, rate_limited_until, last_used_at, created_at, updated_at"; + private final JdbcTemplate jdbc; + + public JdbcGithubSystemCredentialRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public List list() { + return jdbc.query("SELECT " + FIELDS + " FROM github_system_credential ORDER BY created_at DESC, id DESC", + JdbcGithubSystemCredentialRepository::map); + } + + @Override + public List quotas(UUID credentialId) { + return jdbc.query("SELECT resource, remaining, reset_at, observed_at " + + "FROM github_system_credential_quota WHERE credential_id = ? ORDER BY resource", + (result, row) -> new GithubCredentialQuota(result.getString("resource"), + result.getInt("remaining"), instant(result, "reset_at"), + instant(result, "observed_at")), credentialId); + } + + @Override + public Optional find(UUID id) { + return jdbc.query("SELECT " + FIELDS + " FROM github_system_credential WHERE id = ?", + JdbcGithubSystemCredentialRepository::map, id).stream().findFirst(); + } + + @Override + public boolean hasConfiguredCredentials() { + return Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS (SELECT 1 FROM github_system_credential)", Boolean.class)); + } + + @Override + public Optional availableForClaim(Instant now, String resource) { + if (resource == null || !List.of("CORE", "SEARCH", "GRAPHQL", "OTHER").contains(resource)) { + throw new IllegalArgumentException("Invalid GitHub quota resource"); + } + return jdbc.query("SELECT " + FIELDS + " FROM github_system_credential credential " + + "WHERE status = 'ACTIVE' AND (rate_limited_until IS NULL OR rate_limited_until <= ?) " + + "AND NOT EXISTS (SELECT 1 FROM github_system_credential_quota quota " + + "WHERE quota.credential_id = credential.id AND quota.resource = ? " + + "AND quota.remaining = 0 AND quota.reset_at > ?) " + + "ORDER BY last_used_at ASC NULLS FIRST, id LIMIT 1 FOR UPDATE SKIP LOCKED", + JdbcGithubSystemCredentialRepository::map, Timestamp.from(now), resource, + Timestamp.from(now)).stream().findFirst(); + } + + @Override + public void create(GithubSystemCredential credential) { + jdbc.update("INSERT INTO github_system_credential " + + "(id, label, fingerprint, key_version, nonce, ciphertext, status) " + + "VALUES (?, ?, ?, ?, ?, ?, 'ACTIVE')", + credential.id(), credential.label(), credential.fingerprint(), credential.keyVersion(), + credential.nonce(), credential.ciphertext()); + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean replace(UUID id, String label, String fingerprint, String keyVersion, byte[] nonce, byte[] ciphertext) { + var previous = jdbc.query("SELECT fingerprint FROM github_system_credential WHERE id=? FOR UPDATE", + (result, row) -> result.getString(1), id); + if (previous.isEmpty()) { + return false; + } + jdbc.update("UPDATE github_system_credential SET label = ?, fingerprint = ?, key_version = ?, " + + "nonce = ?, ciphertext = ?, status = 'ACTIVE', rate_limited_until = NULL, " + + "updated_at = CURRENT_TIMESTAMP WHERE id = ?", + label, fingerprint, keyVersion, nonce, ciphertext, id); + if (!previous.getFirst().equals(fingerprint)) { + jdbc.update("DELETE FROM github_system_credential_quota WHERE credential_id=?", id); + jdbc.update("DELETE FROM github_official_quota_observation WHERE credential_id=?", id); + } + return true; + } + + @Override + public boolean setEnabled(UUID id, boolean enabled) { + return jdbc.update("UPDATE github_system_credential SET status = ?, updated_at = CURRENT_TIMESTAMP " + + "WHERE id = ? AND status IN ('ACTIVE', 'DISABLED')", + enabled ? "ACTIVE" : "DISABLED", id) == 1; + } + + @Override + public boolean assignToLease(UUID taskId, long generation, UUID credentialId, + String fingerprint, Instant now) { + int updated = jdbc.update("UPDATE github_sync_task SET credential_id = ?, credential_fingerprint = ? " + + "WHERE id = ? AND lease_generation = ? AND status = 'LEASED'", + credentialId, fingerprint, taskId, generation); + if (updated == 1) { + jdbc.update("UPDATE github_system_credential SET last_used_at = ? WHERE id = ?", + Timestamp.from(now), credentialId); + } + return updated == 1; + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean markInvalid(UUID taskId, long generation, String leaseTokenHash, + UUID credentialId, Instant now) { + if (lockCredentialForLease(taskId, generation, leaseTokenHash, credentialId, now) == null) { + return false; + } + return jdbc.update("UPDATE github_system_credential AS credential " + + "SET status = 'INVALID', rate_limited_until = NULL, updated_at = ? " + + "FROM github_sync_task AS task " + + "WHERE credential.id = ? AND credential.status = 'ACTIVE' " + + "AND task.id = ? AND task.status = 'LEASED' " + + "AND task.credential_id = credential.id " + + "AND task.credential_fingerprint = credential.fingerprint " + + "AND task.lease_generation = ? " + + "AND task.lease_token_hash = ? AND task.lease_expires_at > CASE WHEN task.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END", + Timestamp.from(now), credentialId, taskId, generation, leaseTokenHash, + Timestamp.from(now)) == 1; + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean markRateLimited(UUID taskId, long generation, String leaseTokenHash, + UUID credentialId, Instant retryAt, Instant now) { + if (lockCredentialForLease(taskId, generation, leaseTokenHash, credentialId, now) == null) { + return false; + } + return jdbc.update("UPDATE github_system_credential AS credential " + + "SET rate_limited_until = GREATEST(COALESCE(credential.rate_limited_until, ?), ?), " + + "updated_at = ? FROM github_sync_task AS task " + + "WHERE credential.id = ? AND credential.status = 'ACTIVE' " + + "AND task.id = ? AND task.status = 'LEASED' " + + "AND task.credential_id = credential.id " + + "AND task.credential_fingerprint = credential.fingerprint " + + "AND task.lease_generation = ? " + + "AND task.lease_token_hash = ? AND task.lease_expires_at > CASE WHEN task.official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END", + Timestamp.from(retryAt), Timestamp.from(retryAt), Timestamp.from(now), + credentialId, taskId, generation, leaseTokenHash, Timestamp.from(now)) == 1; + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean recordQuotaForLease(UUID taskId, long generation, String leaseTokenHash, + UUID credentialId, String resource, int remaining, + Instant resetAt, Instant now) { + UUID id = lockCredentialForLease(taskId, generation, leaseTokenHash, credentialId, now); + if (id == null) return false; + return writeQuota(id, resource, remaining, resetAt, now); + } + + @Override + @org.springframework.transaction.annotation.Transactional + public boolean recordQuotaForTaskLease(UUID taskId, long generation, String leaseTokenHash, + String resource, int remaining, Instant resetAt, Instant now) { + UUID id = lockCredentialForLease(taskId, generation, leaseTokenHash, null, now); + if (id == null) return false; + return writeQuota(id, resource, remaining, resetAt, now); + } + + private UUID lockCredentialForLease(UUID taskId, long generation, String leaseTokenHash, + UUID expectedId, Instant now) { + var tasks = jdbc.query(""" + SELECT credential_id, credential_fingerprint FROM github_sync_task + WHERE id=? AND status='LEASED' AND lease_generation=? AND lease_token_hash=? + AND credential_id IS NOT NULL AND (?::uuid IS NULL OR credential_id=?) + AND lease_expires_at > CASE WHEN official_query_id IS NOT NULL THEN clock_timestamp() ELSE ? END + FOR UPDATE + """, (result, row) -> new LeaseCredential(result.getObject("credential_id", UUID.class), + result.getString("credential_fingerprint")), + taskId, generation, leaseTokenHash, expectedId, expectedId, Timestamp.from(now)); + if (tasks.isEmpty()) return null; + var lease = tasks.getFirst(); + var current = jdbc.query("SELECT fingerprint,status FROM github_system_credential WHERE id=? FOR UPDATE", + (result, row) -> new String[]{result.getString(1), result.getString(2)}, lease.id()); + if (current.isEmpty() || !"ACTIVE".equals(current.getFirst()[1]) + || !java.util.Objects.equals(lease.fingerprint(), current.getFirst()[0])) return null; + return lease.id(); + } + + private record LeaseCredential(UUID id, String fingerprint) {} + + @Override + public void recordQuota(UUID credentialId, String resource, int remaining, + Instant resetAt, Instant now) { + writeQuota(credentialId, resource, remaining, resetAt, now); + } + + private boolean writeQuota(UUID credentialId, String resource, int remaining, Instant resetAt, Instant now) { + return jdbc.update("INSERT INTO github_system_credential_quota " + + "(credential_id, resource, remaining, reset_at, observed_at) VALUES (?, ?, ?, ?, ?) " + + "ON CONFLICT (credential_id, resource) DO UPDATE SET remaining = EXCLUDED.remaining, " + + "reset_at = EXCLUDED.reset_at, observed_at = EXCLUDED.observed_at " + + "WHERE github_system_credential_quota.observed_at <= EXCLUDED.observed_at", + credentialId, resource, remaining, Timestamp.from(resetAt), Timestamp.from(now)) == 1; + } + + private static GithubSystemCredential map(ResultSet result, int row) throws SQLException { + return new GithubSystemCredential(result.getObject("id", UUID.class), result.getString("label"), + result.getString("fingerprint"), result.getString("key_version"), + result.getBytes("nonce"), result.getBytes("ciphertext"), result.getString("status"), + instant(result, "rate_limited_until"), instant(result, "last_used_at"), + instant(result, "created_at"), instant(result, "updated_at")); + } + + private static Instant instant(ResultSet result, String name) throws SQLException { + Timestamp value = result.getTimestamp(name); + return value == null ? null : value.toInstant(); + } +} diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/ReviewTaskJpaRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/ReviewTaskJpaRepository.java index 6f25b6e61..e6059d27c 100644 --- a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/ReviewTaskJpaRepository.java +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/ReviewTaskJpaRepository.java @@ -4,6 +4,7 @@ import com.iflytek.skillhub.domain.review.ReviewTask; import com.iflytek.skillhub.domain.review.ReviewTaskRepository; import com.iflytek.skillhub.domain.review.ReviewTaskStatus; import com.iflytek.skillhub.domain.review.ReviewSubjectType; +import com.iflytek.skillhub.domain.review.ReviewOriginType; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -26,13 +27,31 @@ public interface ReviewTaskJpaRepository extends JpaRepository Optional findBySkillVersionIdAndStatus(Long skillVersionId, ReviewTaskStatus status); + List findBySkillIdAndStatus(Long skillId, ReviewTaskStatus status); + @Override Optional findBySubjectTypeAndSubjectVersionIdAndStatus( ReviewSubjectType subjectType, Long subjectVersionId, ReviewTaskStatus status); - Page findByStatus(ReviewTaskStatus status, Pageable pageable); + @Override + default Page findByStatus(ReviewTaskStatus status, Pageable pageable) { + return findByOriginTypeAndStatus(ReviewOriginType.USER, status, pageable); + } - Page findByNamespaceIdAndStatus(Long namespaceId, ReviewTaskStatus status, Pageable pageable); + @Override + default Page findByNamespaceIdAndStatus(Long namespaceId, + ReviewTaskStatus status, Pageable pageable) { + return findByNamespaceIdAndOriginTypeAndStatus(namespaceId, ReviewOriginType.USER, status, pageable); + } + + @Override + Page findByOriginTypeAndStatus(ReviewOriginType originType, ReviewTaskStatus status, + Pageable pageable); + + @Override + Page findByNamespaceIdAndOriginTypeAndStatus(Long namespaceId, + ReviewOriginType originType, + ReviewTaskStatus status, Pageable pageable); Page findBySubmittedByAndStatus(String submittedBy, ReviewTaskStatus status, Pageable pageable); diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillFileJpaRepository.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillFileJpaRepository.java index 3a356bff5..75c9d8217 100644 --- a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillFileJpaRepository.java +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/jpa/SkillFileJpaRepository.java @@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import java.util.List; +import java.util.Optional; /** * JPA-backed repository for package files attached to one skill version. @@ -13,6 +14,7 @@ import java.util.List; @Repository public interface SkillFileJpaRepository extends JpaRepository, SkillFileRepository { List findByVersionId(Long versionId); + Optional findByVersionIdAndFilePath(Long versionId, String filePath); List findByVersionIdIn(List versionIds); void deleteByVersionId(Long versionId); boolean existsByStorageKeyAndVersionIdNotIn(String storageKey, List versionIds); diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapter.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapter.java index 6dd46fcd0..0d87e95d9 100644 --- a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapter.java +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapter.java @@ -39,8 +39,10 @@ public class SkillScannerAdapter implements SecurityScanner { : skillScannerService.scanUpload(Path.of(request.skillPackagePath()), scanOptions); return mapToResponse(apiResponse); } catch (HttpClientException e) { - log.error("Security scan failed for versionId={}: {}", request.skillVersionId(), e.getMessage()); - throw new SecurityScanException("Security scan request failed: " + e.getMessage(), e); + log.error("Security scan failed for versionId={}, status={}", + request.skillVersionId(), e.getStatusCode()); + throw new SecurityScanException("Security scan request failed", + new HttpClientException(e.getStatusCode(), null)); } } @@ -55,9 +57,8 @@ public class SkillScannerAdapter implements SecurityScanner { } private SecurityScanResponse mapToResponse(SkillScannerApiResponse apiResponse) { - log.info("Scanner API raw response: scanId={}, skillName={}, isSafe={}, maxSeverity={}, findingsCount={}, duration={}s", - apiResponse.scanId(), apiResponse.skillName(), apiResponse.isSafe(), - apiResponse.maxSeverity(), apiResponse.findingsCount(), apiResponse.scanDurationSeconds()); + log.info("Scanner API response: isSafe={}, findingsCount={}", + apiResponse.isSafe(), apiResponse.findingsCount()); SecurityScanResponse response = new SecurityScanResponse( apiResponse.scanId(), @@ -68,14 +69,8 @@ public class SkillScannerAdapter implements SecurityScanner { apiResponse.scanDurationSeconds() != null ? apiResponse.scanDurationSeconds() : 0.0 ); - log.info("Mapped response: scanId={}, verdict={}, findingsCount={}, maxSeverity={}", - response.scanId(), response.verdict(), response.findingsCount(), response.maxSeverity()); - if (!response.findings().isEmpty()) { - for (SecurityFinding f : response.findings()) { - log.debug("Mapped finding: ruleId={}, severity={}, category={}, filePath={}, lineNumber={}", - f.ruleId(), f.severity(), f.category(), f.filePath(), f.lineNumber()); - } - } + log.info("Mapped scanner response: verdict={}, findingsCount={}", + response.verdict(), response.findingsCount()); return response; } diff --git a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerService.java b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerService.java index aa75a018a..5dcf3e92c 100644 --- a/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerService.java +++ b/server/skillhub-infra/src/main/java/com/iflytek/skillhub/infra/scanner/SkillScannerService.java @@ -34,7 +34,7 @@ public class SkillScannerService { public SkillScannerApiResponse scanDirectory(String skillDirectory, ScanOptions options) { String uri = baseUrl + "/scan"; - log.info("Scanning local directory via scanner: {} -> {}", skillDirectory, uri); + log.info("Scanning local directory via scanner"); Map body = buildScanRequestBody(skillDirectory, options); try { @@ -47,7 +47,7 @@ public class SkillScannerService { public SkillScannerApiResponse scanUpload(Path skillPackagePath, ScanOptions options) { String uri = buildUploadUri(options); - log.info("Uploading skill package to scanner: {}", uri); + log.info("Uploading skill package to scanner"); MultiValueMap parts = new LinkedMultiValueMap<>(); parts.add("file", new FileSystemResource(skillPackagePath)); @@ -132,7 +132,6 @@ public class SkillScannerService { if (exception.getStatusCode() > 0) { return new HttpClientException(exception.getStatusCode(), null); } - Throwable cause = exception.getCause() == null ? exception : exception.getCause(); - return new HttpClientException("Scanner API request failed", cause); + return new HttpClientException(0, null); } } diff --git a/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapterTest.java b/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapterTest.java index 2de339188..7fceed573 100644 --- a/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapterTest.java +++ b/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerAdapterTest.java @@ -99,7 +99,7 @@ class SkillScannerAdapterTest { assertThatThrownBy(() -> adapter.scan(new SecurityScanRequest("task-1", 42L, "/tmp/skill", Map.of()))) .isInstanceOf(SecurityScanException.class) - .hasMessage("Security scan request failed: HTTP " + statusCode + ": scanner unavailable") + .hasMessage("Security scan request failed") .satisfies(error -> assertThat(((SecurityScanException) error).isScannerUnavailable()).isTrue()); } diff --git a/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerLoggingTest.java b/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerLoggingTest.java index 8fc29a412..a9aa4fc30 100644 --- a/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerLoggingTest.java +++ b/server/skillhub-infra/src/test/java/com/iflytek/skillhub/infra/scanner/SkillScannerLoggingTest.java @@ -3,19 +3,29 @@ package com.iflytek.skillhub.infra.scanner; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; import ch.qos.logback.core.read.ListAppender; import com.iflytek.skillhub.domain.security.SecurityScanRequest; import com.iflytek.skillhub.infra.http.HttpClient; import com.iflytek.skillhub.infra.http.HttpClientException; +import com.iflytek.skillhub.infra.http.WebClientHttpClient; +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.slf4j.LoggerFactory; import org.springframework.http.HttpHeaders; import org.springframework.util.MultiValueMap; +import org.springframework.web.reactive.function.client.WebClient; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -36,8 +46,9 @@ class SkillScannerLoggingTest { } @Test - void scanUpload_truncatesLoggedScannerErrorBody() { - HttpClient httpClient = new ThrowingHttpClient("sensitive-body-".repeat(100)); + void scanUpload_doesNotPropagateScannerErrorBody() { + String marker = "sensitive-body-".repeat(100); + HttpClient httpClient = new ThrowingHttpClient(new HttpClientException(500, marker)); SkillScannerService service = new SkillScannerService( httpClient, "http://scanner.test", @@ -47,30 +58,48 @@ class SkillScannerLoggingTest { attachAppender(serviceLogger); assertThatThrownBy(() -> service.scanUpload(Path.of("/tmp/demo.zip"), ScanOptions.disabled())) - .isInstanceOf(HttpClientException.class); + .isInstanceOf(HttpClientException.class) + .hasMessageNotContaining(marker); List messages = loggedMessages(); assertThat(messages).anyMatch(message -> message.contains("Scanner API error: status=500")); - assertThat(messages).noneMatch(message -> message.contains("sensitive-body-".repeat(20))); + assertThat(messages).noneMatch(message -> message.contains("sensitive-body-")); } @Test - void scan_logsSummaryWithoutDumpingFindingDetailsAtInfo() { + void scanUpload_doesNotPropagateNetworkCause() { + String marker = "private-response-body"; + HttpClient httpClient = new ThrowingHttpClient(new HttpClientException( + "Scanner connection failed", new IllegalStateException(marker))); + SkillScannerService service = new SkillScannerService( + httpClient, "http://scanner.test", "/scan-upload", "/health"); + attachAppender(serviceLogger); + + assertThatThrownBy(() -> service.scanUpload(Path.of("/tmp/demo.zip"), ScanOptions.disabled())) + .isInstanceOf(HttpClientException.class) + .hasMessageNotContaining(marker) + .satisfies(error -> assertThat(error.getCause()).isNull()); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + } + + @Test + void scan_logsOnlyVerdictAndCountsEvenAtDebug() { + String marker = "private-skill-body-marker"; StubSkillScannerService service = new StubSkillScannerService(); service.directoryResponse = new SkillScannerApiResponse( - "scan-9", - "skill", + "scan-" + marker, + "skill-" + marker, false, "HIGH", 1, List.of(new SkillScannerApiResponse.Finding( "ID-1", - "RULE-1", + "RULE-" + marker, "HIGH", "code-execution", "Danger title", - "Very sensitive description", - "src/main.py", + "Very sensitive description " + marker, + "src/" + marker + ".py", 7, "eval(secret)", "Use safe api", @@ -82,14 +111,95 @@ class SkillScannerLoggingTest { ); SkillScannerAdapter adapter = new SkillScannerAdapter(service, "local", ScanOptions.disabled()); attachAppender(adapterLogger); + adapterLogger.setLevel(Level.DEBUG); adapter.scan(new SecurityScanRequest("task-1", 1L, "/tmp/skill", Map.of())); List messages = loggedMessages(); - assertThat(messages).anyMatch(message -> message.contains("Scanner API raw response")); - assertThat(messages).noneMatch(message -> message.contains("Very sensitive description")); + assertThat(messages).anyMatch(message -> message.contains("Mapped scanner response: verdict=DANGEROUS")); + assertThat(messages).noneMatch(message -> message.contains(marker)); assertThat(messages).noneMatch(message -> message.contains("eval(secret)")); - assertThat(messages).noneMatch(message -> message.contains("Mapped finding:")); + } + + @Test + void scan_doesNotLogOrRetainScannerErrorBody() { + String marker = "private-scanner-error-body"; + StubSkillScannerService service = new StubSkillScannerService(); + service.directoryException = new HttpClientException(503, marker); + SkillScannerAdapter adapter = new SkillScannerAdapter(service, "local", ScanOptions.disabled()); + attachAppender(adapterLogger); + + assertThatThrownBy(() -> adapter.scan(new SecurityScanRequest( + "task-1", 1L, "/tmp/skill", Map.of()))) + .isInstanceOf(SecurityScanException.class) + .hasMessageNotContaining(marker) + .satisfies(error -> assertThat(error.getCause()).hasMessageNotContaining(marker)); + assertThat(loggedMessages()).noneMatch(message -> message.contains(marker)); + } + + @Test + void realHttpScannerFailureAndResponseStayOutOfLogs(@TempDir Path tempDir) throws Exception { + String marker = "scanner-private-body-marker"; + AtomicBoolean fail = new AtomicBoolean(true); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/scan-upload", exchange -> { + exchange.getRequestBody().readAllBytes(); + boolean unavailable = fail.get(); + String response = unavailable + ? "{\"detail\":\"" + marker + "\"}" + : "{\"scan_id\":\"" + marker + "\",\"skill_name\":\"" + marker + + "\",\"is_safe\":true,\"max_severity\":\"LOW\",\"findings_count\":0," + + "\"findings\":[],\"scan_duration_seconds\":0.01}"; + byte[] bytes = response.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(unavailable ? 503 : 200, bytes.length); + try (var output = exchange.getResponseBody()) { + output.write(bytes); + } + }); + server.createContext("/health", exchange -> { + byte[] bytes = marker.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(503, bytes.length); + try (var output = exchange.getResponseBody()) { + output.write(bytes); + } + }); + server.start(); + Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + ListAppender allLogs = new ListAppender<>(); + allLogs.start(); + root.addAppender(allLogs); + try { + Path archive = Files.writeString(tempDir.resolve("demo.zip"), "fixture"); + SkillScannerService service = new SkillScannerService( + new WebClientHttpClient(WebClient.builder().build()), + "http://127.0.0.1:" + server.getAddress().getPort(), "/scan-upload", "/health"); + SkillScannerAdapter adapter = new SkillScannerAdapter( + service, "upload", ScanOptions.disabled()); + SecurityScanRequest request = new SecurityScanRequest("task-1", 1L, + archive.toString(), Map.of()); + + assertThat(service.isHealthy()).isFalse(); + assertThatThrownBy(() -> adapter.scan(request)) + .isInstanceOf(SecurityScanException.class) + .hasMessage("Security scan request failed") + .satisfies(error -> { + assertThat(error.getCause()).hasMessageNotContaining(marker); + assertThat(((SecurityScanException) error).isScannerUnavailable()).isTrue(); + }); + fail.set(false); + assertThat(adapter.scan(request).verdict().name()).isEqualTo("SAFE"); + + String output = allLogs.list.stream().map(event -> event.getFormattedMessage() + + (event.getThrowableProxy() == null ? "" + : ThrowableProxyUtil.asString(event.getThrowableProxy()))) + .collect(Collectors.joining("\n")); + assertThat(output).doesNotContain(marker); + } finally { + root.detachAppender(allLogs); + allLogs.stop(); + server.stop(0); + } } private void attachAppender(Logger logger) { @@ -106,10 +216,10 @@ class SkillScannerLoggingTest { } private static final class ThrowingHttpClient implements HttpClient { - private final String body; + private final HttpClientException failure; - private ThrowingHttpClient(String body) { - this.body = body; + private ThrowingHttpClient(HttpClientException failure) { + this.failure = failure; } @Override @@ -137,7 +247,7 @@ class SkillScannerLoggingTest { MultiValueMap parts, HttpHeaders headers, Class responseType) { - throw new HttpClientException(500, body); + throw failure; } @Override @@ -148,6 +258,7 @@ class SkillScannerLoggingTest { private static final class StubSkillScannerService extends SkillScannerService { private SkillScannerApiResponse directoryResponse; + private HttpClientException directoryException; private StubSkillScannerService() { super(new NoOpHttpClient(), "http://scanner.test", "/scan-upload", "/health"); @@ -155,6 +266,9 @@ class SkillScannerLoggingTest { @Override public SkillScannerApiResponse scanDirectory(String skillDirectory, ScanOptions options) { + if (directoryException != null) { + throw directoryException; + } return directoryResponse; } } diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java index 86b6b5901..a511f3ccb 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java @@ -30,15 +30,23 @@ public class LocalFileStorageService implements ObjectStorageService { */ @Override public void putObject(String key, InputStream data, long size, String contentType) { + Path tmp = null; try { Path target = resolve(key); Files.createDirectories(target.getParent()); - Path tmp = target.resolveSibling(target.getFileName() + ".tmp"); + tmp = target.resolveSibling(target.getFileName() + ".tmp"); try (OutputStream out = Files.newOutputStream(tmp, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { data.transferTo(out); } Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { throw new StorageAccessException("putObject", key, e); } + } catch (IOException e) { + throw new StorageAccessException("putObject", key, e); + } finally { + if (tmp != null) { + try { Files.deleteIfExists(tmp); } + catch (IOException ignored) { /* Best effort after a failed or completed move. */ } + } + } } /** diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java index 1ba1b45af..308129598 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java @@ -3,6 +3,7 @@ package com.iflytek.skillhub.storage; import java.io.InputStream; import java.time.Duration; import java.util.List; +import java.util.Optional; /** * Storage abstraction for binary skill assets and bundles regardless of the backing provider. @@ -15,4 +16,12 @@ public interface ObjectStorageService { boolean exists(String key); ObjectMetadata getMetadata(String key); String generatePresignedUrl(String key, Duration expiry, String downloadFilename); + + default Optional presignPut(String key, Duration expiry) { + return Optional.empty(); + } + + default Optional listPrefixUsage(String prefix, int maxObjects) { + return Optional.empty(); + } } diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/PresignedUpload.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/PresignedUpload.java new file mode 100644 index 000000000..92ac46cf8 --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/PresignedUpload.java @@ -0,0 +1,6 @@ +package com.iflytek.skillhub.storage; + +import java.util.Map; + +public record PresignedUpload(String url, Map headers) { +} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index ba0250bf7..c1cc02492 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -1,6 +1,7 @@ package com.iflytek.skillhub.storage; import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -17,7 +18,9 @@ import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; import java.io.InputStream; import java.io.IOException; @@ -28,6 +31,9 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; /** * S3-compatible object storage implementation used for persisted skill packages and generated @@ -41,6 +47,7 @@ public class S3StorageService implements ObjectStorageService { private final Object bucketPreparationLock = new Object(); private S3Client s3Client; private S3Presigner s3Presigner; + private S3Presigner s3UploadPresigner; private volatile boolean bucketPrepared; public S3StorageService(S3StorageProperties properties) { this.properties = properties; } @@ -52,6 +59,7 @@ public class S3StorageService implements ObjectStorageService { .connectionAcquisitionTimeout(properties.getConnectionAcquisitionTimeout()); this.s3Client = buildS3Client(httpClientBuilder); this.s3Presigner = buildPresigner(); + this.s3UploadPresigner = buildUploadPresigner(); String authMode = isStaticCredentials() ? "static credentials" : "IAM credentials (default provider chain)"; if (properties.isAutoCreateBucket()) { log.info("Initialized S3 storage client for bucket '{}' using {} (bucket auto-creation is deferred until first write)", @@ -93,20 +101,48 @@ public class S3StorageService implements ObjectStorageService { } S3Presigner buildPresigner() { + String endpoint = properties.getPublicEndpoint(); + if (endpoint == null || endpoint.isBlank()) { + endpoint = properties.getEndpoint(); + } + return buildPresignerAtEndpoint(endpoint); + } + + private S3Presigner buildUploadPresigner() { + String endpoint = properties.getEndpoint(); + if (endpoint == null || endpoint.isBlank() + || endpoint.equals(properties.getPublicEndpoint())) { + return s3Presigner; + } + return buildPresignerAtEndpoint(endpoint); + } + + private S3Presigner buildPresignerAtEndpoint(String endpoint) { var presignerBuilder = S3Presigner.builder() .region(Region.of(properties.getRegion())) .credentialsProvider(buildCredentialsProvider()) .serviceConfiguration(S3Configuration.builder() .pathStyleAccessEnabled(properties.isForcePathStyle()) .build()); - if (properties.getPublicEndpoint() != null && !properties.getPublicEndpoint().isBlank()) { - presignerBuilder.endpointOverride(URI.create(properties.getPublicEndpoint())); - } else if (properties.getEndpoint() != null && !properties.getEndpoint().isBlank()) { - presignerBuilder.endpointOverride(URI.create(properties.getEndpoint())); + if (endpoint != null && !endpoint.isBlank()) { + presignerBuilder.endpointOverride(URI.create(endpoint)); } return presignerBuilder.build(); } + @PreDestroy + void close() { + if (s3UploadPresigner != null && s3UploadPresigner != s3Presigner) { + s3UploadPresigner.close(); + } + if (s3Presigner != null) { + s3Presigner.close(); + } + if (s3Client != null) { + s3Client.close(); + } + } + private void ensureBucketPrepared() { if (!properties.isAutoCreateBucket() || bucketPrepared) { return; @@ -220,6 +256,10 @@ public class S3StorageService implements ObjectStorageService { return true; } catch (NoSuchKeyException e) { return false; } + catch (S3Exception e) { + if (e.statusCode() == 404) return false; + throw new StorageAccessException("exists", key, e); + } catch (RuntimeException e) { throw new StorageAccessException("exists", key, e); } } @@ -232,6 +272,48 @@ public class S3StorageService implements ObjectStorageService { } } + @Override + public Optional listPrefixUsage(String prefix, int maxObjects) { + if (prefix == null || prefix.isBlank() || maxObjects < 1) { + throw new IllegalArgumentException("A prefix and positive object limit are required"); + } + long bytes = 0; + long objects = 0; + String continuationToken = null; + int pages = 0; + try { + while (objects < maxObjects && pages < 100) { + var response = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(properties.getBucket()) + .prefix(prefix) + .continuationToken(continuationToken) + .maxKeys((int) Math.min(1000L, maxObjects - objects)) + .build()); + pages++; + for (S3Object object : response.contents()) { + bytes = Math.addExact(bytes, object.size()); + objects++; + } + if (!response.isTruncated()) { + return Optional.of(new StoragePrefixUsage(bytes, objects, true)); + } + String nextToken = response.nextContinuationToken(); + if (nextToken == null || nextToken.isBlank() || nextToken.equals(continuationToken)) { + return Optional.of(new StoragePrefixUsage(bytes, objects, false)); + } + continuationToken = nextToken; + } + return Optional.of(new StoragePrefixUsage(bytes, objects, false)); + } catch (NoSuchBucketException exception) { + if (properties.isAutoCreateBucket() && !bucketPrepared) { + return Optional.of(new StoragePrefixUsage(0, 0, true)); + } + throw new StorageAccessException("listPrefixUsage", prefix, exception); + } catch (RuntimeException exception) { + throw new StorageAccessException("listPrefixUsage", prefix, exception); + } + } + @Override public String generatePresignedUrl(String key, Duration expiry, String downloadFilename) { Duration signatureDuration = expiry != null ? expiry : properties.getPresignExpiry(); @@ -255,4 +337,29 @@ public class S3StorageService implements ObjectStorageService { throw new StorageAccessException("generatePresignedUrl", key, e); } } + + @Override + public Optional presignPut(String key, Duration expiry) { + try { + // A signed PUT is sent by the worker directly to S3, so no Java putObject call + // can create a missing bucket before the first GitHub sync upload. + ensureBucketPrepared(); + PresignedPutObjectRequest request = s3UploadPresigner.presignPutObject( + PutObjectPresignRequest.builder() + .signatureDuration(expiry) + .putObjectRequest(PutObjectRequest.builder() + .bucket(properties.getBucket()) + .key(key) + .contentType("application/zip") + .build()) + .build()); + Map headers = request.signedHeaders().entrySet().stream() + .filter(entry -> !entry.getKey().equalsIgnoreCase("host")) + .collect(Collectors.toMap(Map.Entry::getKey, + entry -> String.join(",", entry.getValue()))); + return Optional.of(new PresignedUpload(request.url().toString(), headers)); + } catch (RuntimeException exception) { + throw new StorageAccessException("presignPut", key, exception); + } + } } diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StoragePrefixUsage.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StoragePrefixUsage.java new file mode 100644 index 000000000..b832c5720 --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StoragePrefixUsage.java @@ -0,0 +1,4 @@ +package com.iflytek.skillhub.storage; + +/** Actual object bytes under one storage prefix, bounded by the caller's scan limit. */ +public record StoragePrefixUsage(long bytes, long objectCount, boolean complete) {} diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java index 5c89448bf..f828b071a 100644 --- a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java @@ -13,6 +13,9 @@ import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.CreateBucketResponse; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.presigner.S3Presigner; @@ -128,6 +131,64 @@ class S3StorageServiceTest { verify(client).getObject(any(GetObjectRequest.class)); } + @Test + void prefixUsageCountsActualObjectsAcrossPagesWithoutCreatingBucket() { + S3Client client = mock(S3Client.class); + S3Presigner presigner = mock(S3Presigner.class); + when(client.listObjectsV2(any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().isTruncated(true) + .nextContinuationToken("page-2") + .contents(S3Object.builder().key("github-sync/staging/a.zip").size(30L).build()) + .build()) + .thenReturn(ListObjectsV2Response.builder().isTruncated(false) + .contents(S3Object.builder().key("github-sync/staging/b.zip").size(50L).build()) + .build()); + TestableS3StorageService service = new TestableS3StorageService(properties(false), client, presigner); + service.init(); + + assertThat(service.listPrefixUsage("github-sync/staging/", 10_000)) + .contains(new StoragePrefixUsage(80, 2, true)); + verify(client, times(2)).listObjectsV2(any(ListObjectsV2Request.class)); + verify(client, never()).createBucket(any(CreateBucketRequest.class)); + } + + @Test + void prefixUsageReportsIncompleteWhenObjectBudgetIsReached() { + S3Client client = mock(S3Client.class); + S3Presigner presigner = mock(S3Presigner.class); + when(client.listObjectsV2(any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().isTruncated(true) + .nextContinuationToken("page-2") + .contents(S3Object.builder().key("github-sync/staging/a.zip").size(30L).build()) + .build()); + TestableS3StorageService service = new TestableS3StorageService(properties(false), client, presigner); + service.init(); + + assertThat(service.listPrefixUsage("github-sync/staging/", 1)) + .contains(new StoragePrefixUsage(30, 1, false)); + verify(client, times(1)).listObjectsV2(any(ListObjectsV2Request.class)); + } + + @Test + void missingBucketIsEmptyOnlyBeforeTheFirstStagedWrite() { + S3Client client = mock(S3Client.class); + S3Presigner presigner = mock(S3Presigner.class); + when(client.listObjectsV2(any(ListObjectsV2Request.class))) + .thenThrow(NoSuchBucketException.builder().message("missing").build()); + when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))) + .thenReturn(PutObjectResponse.builder().build()); + TestableS3StorageService service = new TestableS3StorageService(properties(true), client, presigner); + service.init(); + + assertThat(service.listPrefixUsage("github-sync/staging/", 10)) + .contains(new StoragePrefixUsage(0, 0, true)); + byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + service.putObject("github-sync/staging/demo.zip", new ByteArrayInputStream(content), + content.length, "application/zip"); + assertThatThrownBy(() -> service.listPrefixUsage("github-sync/staging/", 10)) + .isInstanceOf(StorageAccessException.class); + } + @Test void putObjectShouldCreateBucketAndRetryWhenMissing() { S3Client client = mock(S3Client.class); diff --git a/web/Dockerfile b/web/Dockerfile index 74a45aa43..3d8fd99d0 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -5,7 +5,8 @@ FROM --platform=$BUILDPLATFORM node:22-alpine AS build RUN corepack enable WORKDIR /app COPY package.json pnpm-lock.yaml ./ -RUN pnpm install --frozen-lockfile +# Corepack uses Node fetch; explicitly honor the build proxy for its download. +RUN NODE_USE_ENV_PROXY=1 pnpm install --frozen-lockfile COPY . . ARG VITE_BASE_PATH=/__SKILLHUB_WEB_BASE_PATH__/ ENV VITE_BASE_PATH=$VITE_BASE_PATH diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 9b4cf985b..f92af8763 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -18,6 +18,8 @@ import type { ReviewSkillDetail, ReviewProgressPage, ReviewTask, + GithubSyncReviewSource, + GithubSyncReviewQueueItem, PromotionSortBy, PromotionSortDirection, PromotionStatus, @@ -178,6 +180,416 @@ export function getCsrfHeaders(headers?: HeadersInit): HeadersInit { return withCsrf(headers) } +type GithubSyncPage = { items?: T[]; total?: number; page?: number; size?: number } +export type GithubSyncSource = components['schemas']['GithubSyncSourceSummaryResponse'] +export type GithubSyncRun = components['schemas']['GithubSyncRunSummaryResponse'] +export type GithubSyncRunQuery = NonNullable +export type GithubSyncRepository = components['schemas']['GithubSyncRepositorySummary'] +export type GithubSyncRepositoryDetail = components['schemas']['GithubSyncRepositoryDetail'] +export type GithubSyncRepositoryPage = components['schemas']['PageGithubSyncRepositorySummary'] +export type GithubSyncRepositoryOriginPage = components['schemas']['PageGithubSyncRepositoryOrigin'] +export type GithubSyncRepositoryPathPage = components['schemas']['PageGithubDeepDiscoveryPathResult'] +export type GithubSyncRepositoryQuery = NonNullable +export type GithubPlatformControl = components['schemas']['GithubPlatformControlState'] +export type GithubPlatformControlUpdateInput = components['schemas']['GithubPlatformControlUpdateRequest'] +export type GithubSyncRunDetail = components['schemas']['GithubSyncRunDetailResponse'] & { + id: string; kind: string; status: string; createdAt: string; finishedAt?: string | null + taskCount: number; pathCount: number; nextRetryAt?: string | null + receivedPackages: number; processingPackages: number; pendingReviewPackages: number + blockedPackages: number; failedPackages: number; lastErrorCode?: string | null +} +export type GithubSyncRunTask = components['schemas']['GithubSyncRunTaskResponse'] & { + id: string; sourceId?: number | null; repository?: string | null; kind: string; mode?: string | null + status: string; fixedCommitSha?: string | null; lastErrorCode?: string | null + retryAt?: string | null; createdAt: string; completedAt?: string | null +} +export type GithubSyncRunPath = components['schemas']['GithubSyncRunPathResponse'] & { + taskId: string; sourceId?: number | null; repository?: string | null; skillPath: string + commitSha: string; outcome: string; reason?: string | null; ingestionStatus?: string | null + ingestionErrorCode?: string | null + reviewStatus?: string | null; retryAt?: string | null; updatedAt: string +} +export type GithubSyncSkillPath = components['schemas']['GithubSyncSkillPathResponse'] & { + latestBindingIngestionStatus?: string | null + retryRequested?: boolean + retryInProgress?: boolean +} +export type GithubSyncAdmissionPolicy = components['schemas']['GithubSyncAdmissionPolicyResponse'] +export type GithubSyncWorkerStatus = components['schemas']['GithubSyncWorkerStatusResponse'] +export type GithubSyncAdmissionPolicyInput = components['schemas']['GithubSyncAdmissionPolicyUpdateRequest'] +export type GithubSyncSchedulePolicy = components['schemas']['GithubSyncSchedulePolicyResponse'] +export type GithubSyncSchedulePolicyInput = components['schemas']['GithubSyncSchedulePolicyRequest'] +export type GithubDiscoveryEntry = components['schemas']['GithubDiscoveryEntryResponse'] +export type GithubSearchRuleInput = components['schemas']['GithubSearchRuleRequest'] +export type GithubSearchPreview = { + id: string + status: string + errorCode?: string | null + retryAt?: string | null + criteria: { selectors: Array<{ kind: string; value: string; priority: string }> } + searchedUntil: string + result?: { selectors: Array<{ + kind: string; value: string; priority: string; totalCount: number + incompleteResults: boolean; limitReached: boolean + samples: Array<{ repositoryId: number; owner: string; name: string }> + }> } | null + createdAt: string + updatedAt: string +} +export type GithubCredential = components['schemas']['GithubCredentialResponse'] +export type GithubSourceIgnore = components['schemas']['GithubSourceIgnoreResponse'] +export type GithubSourceRescan = components['schemas']['GithubSourceRescanResponse'] +export type GithubSyncManualBinding = components['schemas']['GithubSyncManualBindingResponse'] + +export type GithubDeepDiscoveryConfiguration = components['schemas']['GithubDeepDiscoveryConfigurationResponse'] +export type GithubDeepDiscoveryConfigurationInput = components['schemas']['GithubDeepDiscoveryConfigurationRequest'] +export type GithubDeepDiscoverySummary = components['schemas']['GithubDeepDiscoverySummary'] +export type GithubDeepDiscoveryProgress = components['schemas']['GithubDeepDiscoveryProgress'] +export type GithubDeepDiscoveryRepository = components['schemas']['GithubDeepDiscoveryRepositorySummary'] +export type GithubDeepDiscoveryRepositoryDetail = components['schemas']['GithubDeepDiscoveryRepositoryDetail'] +export type GithubDeepDiscoveryProvenance = components['schemas']['GithubDeepDiscoveryProvenance'] +export type GithubDeepDiscoveryPathResult = components['schemas']['GithubDeepDiscoveryPathResult'] +export type GithubDeepDiscoveryWait = components['schemas']['GithubDeepDiscoveryWait'] +export type GithubDeepDiscoveryRepositoryPage = components['schemas']['DiscoveryPageGithubDeepDiscoveryRepositorySummary'] +export type GithubDeepDiscoveryProvenancePage = components['schemas']['DiscoveryPageGithubDeepDiscoveryProvenance'] +export type GithubDeepDiscoveryPathPage = components['schemas']['DiscoveryPageGithubDeepDiscoveryPathResult'] +export type GithubDeepDiscoveryRepositoryQuery = NonNullable + +export const githubDeepDiscoveryApi = { + async configuration(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/configuration', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateConfiguration(body: GithubDeepDiscoveryConfigurationInput): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/deep-discovery/configuration', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async summary(from: string, until: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/summary', { + params: { query: { from, until } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async progress(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/progress', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repositories(query: GithubDeepDiscoveryRepositoryQuery, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/repositories', { + params: { query }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repository(repositoryId: number, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}', { + params: { path: { repositoryId } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async provenance(repositoryId: number, page: number, asOf?: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}/provenance', { + params: { path: { repositoryId }, query: { page, size: 20, asOf } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async paths(repositoryId: number, page: number, asOf?: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}/paths', { + params: { path: { repositoryId }, query: { page, size: 20, asOf } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, +} + +export const githubSyncAdminApi = { + async platformControl(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/platform-control', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updatePlatformControl(body: GithubPlatformControlUpdateInput): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/platform-control', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repositories(query: GithubSyncRepositoryQuery, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/repositories', { + params: { query }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repository(repositoryId: number, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/repositories/{repositoryId}', { + params: { path: { repositoryId } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repositoryOrigins(repositoryId: number, page: number, size: number, asOf?: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/repositories/{repositoryId}/origins', { + params: { path: { repositoryId }, query: { page, size, asOf } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async repositoryPaths(repositoryId: number, page: number, size: number, asOf?: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/repositories/{repositoryId}/paths', { + params: { path: { repositoryId }, query: { page, size, asOf } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async workerStatus(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/worker-status', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async schedulePolicy(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/schedule-policy', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateSchedulePolicy(body: GithubSyncSchedulePolicyInput): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/schedule-policy', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async rescan(sourceId: number): Promise { + const result = await client.POST('/api/v1/admin/github-sync/sources/{sourceId}/rescan', { + params: { path: { sourceId } }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async retryRejectedSkill(sourceId: number, skillPath: string): Promise { + return fetchJson(`/api/v1/admin/github-sync/sources/${sourceId}/retry-rejected-skill`, { + method: 'POST', headers: await ensureCsrfHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ skillPath }), + }) + }, + async sources(page: number, size: number, signal?: AbortSignal): Promise> { + const result = await client.GET('/api/v1/admin/github-sync/sources', { + params: { query: { page, size } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async runs(page: number, size: number, signal?: AbortSignal, kind?: GithubSyncRunQuery['kind'], status?: GithubSyncRunQuery['status']): Promise> { + const result = await client.GET('/api/v1/admin/github-sync/runs', { + params: { query: { page, size, kind, status } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async runDetail(runId: string, signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/runs/{runId}', { + params: { path: { runId } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async runTasks(runId: string, page: number, size: number, signal?: AbortSignal): Promise> { + const result = await client.GET('/api/v1/admin/github-sync/runs/{runId}/tasks', { + params: { path: { runId }, query: { page, size } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async runPaths(runId: string, page: number, size: number, signal?: AbortSignal): Promise> { + const result = await client.GET('/api/v1/admin/github-sync/runs/{runId}/paths', { + params: { path: { runId }, query: { page, size } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async skillPaths(sourceId: number, page: number, size: number, signal?: AbortSignal): Promise> { + const result = await client.GET('/api/v1/admin/github-sync/sources/{sourceId}/skills', { + params: { path: { sourceId }, query: { page, size } }, headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async bindExistingSkill(ingestionId: string, skillId: number, reason: string): Promise { + const result = await client.POST('/api/v1/admin/github-sync/conflicts/{ingestionId}/bind-skill', { + params: { path: { ingestionId } }, body: { skillId, reason }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async ignorePatterns(sourceId: number): Promise { + const result = await client.GET('/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns', { + params: { path: { sourceId } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateIgnorePatterns(sourceId: number, ignorePatterns: string[]): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns', { + params: { path: { sourceId } }, body: { ignorePatterns }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async admissionPolicy(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/admission-policy', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateAdmissionPolicy(body: GithubSyncAdmissionPolicyInput): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/admission-policy', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async discoveryEntries(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/discovery-entries', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async addRepository(ownerLogin: string, repositoryName: string): Promise { + const result = await client.POST('/api/v1/admin/github-sync/discovery-entries/curated-repositories', { + body: { ownerLogin, repositoryName }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async addOwner(ownerLogin: string, targetHours?: number): Promise { + const result = await client.POST('/api/v1/admin/github-sync/discovery-entries/curated-owners', { + body: { ownerLogin, targetHours }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async addSearchRule(body: GithubSearchRuleInput): Promise { + const result = await client.POST('/api/v1/admin/github-sync/discovery-entries/search-rules', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async previewSearchRule(body: GithubSearchRuleInput): Promise { + const result = await client.POST('/api/v1/admin/github-sync/discovery-entries/search-rules/preview', { + body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async searchRulePreview(id: string): Promise { + const result = await client.GET('/api/v1/admin/github-sync/discovery-entries/search-rules/previews/{id}', { + params: { path: { id } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateSearchRule(id: string, body: GithubSearchRuleInput): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/discovery-entries/search-rules/{id}', { + params: { path: { id } }, body, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async setDiscoveryEnabled(id: string, enabled: boolean): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/discovery-entries/{id}/enabled', { + params: { path: { id } }, body: { enabled }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async credentials(signal?: AbortSignal): Promise { + const result = await client.GET('/api/v1/admin/github-sync/credentials', { + headers: withRequestHeaders(), signal, + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async createCredential(label: string, token: string): Promise { + const result = await client.POST('/api/v1/admin/github-sync/credentials', { + body: { label, token }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async replaceCredential(id: string, label: string, token: string): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/credentials/{id}', { + params: { path: { id } }, body: { label, token }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async setCredentialEnabled(id: string, enabled: boolean): Promise { + const result = await client.PUT('/api/v1/admin/github-sync/credentials/{id}/enabled', { + params: { path: { id } }, body: { enabled }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, +} + +export const githubTeamSourcesApi = { + async rescan(namespaceSlug: string, sourceId: number): Promise { + const result = await client.POST('/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/rescan', { + params: { path: { namespaceSlug, sourceId } }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async retryRejectedSkill(namespaceSlug: string, sourceId: number, skillPath: string): Promise { + return fetchJson(`/api/v1/namespaces/${encodeURIComponent(namespaceSlug)}/github-sources/resolved/${sourceId}/retry-rejected-skill`, { + method: 'POST', headers: await ensureCsrfHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ skillPath }), + }) + }, + async entries(namespaceSlug: string): Promise { + const result = await client.GET('/api/v1/namespaces/{namespaceSlug}/github-sources', { + params: { path: { namespaceSlug } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async resolved(namespaceSlug: string, page: number, size: number): Promise> { + const result = await client.GET('/api/v1/namespaces/{namespaceSlug}/github-sources/resolved', { + params: { path: { namespaceSlug }, query: { page, size } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async runs(namespaceSlug: string, page: number, size: number): Promise> { + const result = await client.GET('/api/v1/namespaces/{namespaceSlug}/github-sources/runs', { + params: { path: { namespaceSlug }, query: { page, size } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async skillPaths(namespaceSlug: string, sourceId: number, page: number, size: number): Promise> { + const result = await client.GET('/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills', { + params: { path: { namespaceSlug, sourceId }, query: { page, size } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse>(result.data, result.error, result.response) + }, + async bindExistingSkill(namespaceSlug: string, ingestionId: string, skillId: number, reason: string): Promise { + const result = await client.POST('/api/v1/namespaces/{namespaceSlug}/github-sources/conflicts/{ingestionId}/bind-skill', { + params: { path: { namespaceSlug, ingestionId } }, body: { skillId, reason }, + headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async ignorePatterns(namespaceSlug: string, sourceId: number): Promise { + const result = await client.GET('/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns', { + params: { path: { namespaceSlug, sourceId } }, headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async updateIgnorePatterns(namespaceSlug: string, sourceId: number, ignorePatterns: string[]): Promise { + const result = await client.PUT('/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns', { + params: { path: { namespaceSlug, sourceId } }, body: { ignorePatterns }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async create(namespaceSlug: string, ownerLogin: string, repositoryName: string, visibility: string): Promise { + const result = await client.POST('/api/v1/namespaces/{namespaceSlug}/github-sources', { + params: { path: { namespaceSlug } }, body: { ownerLogin, repositoryName, visibility }, + headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async setEnabled(namespaceSlug: string, entryId: string, enabled: boolean): Promise { + const result = await client.PUT('/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/enabled', { + params: { path: { namespaceSlug, entryId } }, body: { enabled }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, + async setVisibility(namespaceSlug: string, entryId: string, visibility: string): Promise { + const result = await client.PUT('/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/visibility', { + params: { path: { namespaceSlug, entryId } }, body: { visibility }, headers: await ensureCsrfHeaders(), + }) + return unwrapOpenApiResponse(result.data, result.error, result.response) + }, +} + export type SessionBootstrapRuntimeConfig = { enabled: boolean provider?: string @@ -1161,6 +1573,19 @@ export const suiteApi = { } export const reviewApi = { + async listSync(params: { status: string; namespaceId?: number; page?: number; size?: number; sortDirection?: 'ASC' | 'DESC' }) { + const result = await client.GET('/api/v1/reviews/sync', { + params: { query: { status: params.status, namespaceId: params.namespaceId, + page: params.page ?? 0, size: params.size ?? 20, + sortDirection: params.sortDirection ?? 'DESC' } }, + headers: withRequestHeaders(), + }) + return unwrapOpenApiResponse<{ items: GithubSyncReviewQueueItem[]; total: number; page: number; size: number }>( + result.data, result.error, result.response) + }, + async getSyncSource(id: number): Promise { + return fetchJson(`${WEB_API_PREFIX}/reviews/sync/${id}/source`) + }, async list(params: { status: string; namespaceId?: number; page?: number; size?: number; sortDirection?: 'ASC' | 'DESC' }) { const searchParams = new URLSearchParams() searchParams.set('status', params.status) @@ -1199,8 +1624,8 @@ export const reviewApi = { return fetchJson(`${WEB_API_PREFIX}/reviews/${reviewTaskId}/attempts`) }, - async getSkillDetail(id: number): Promise { - return fetchJson(`${WEB_API_PREFIX}/reviews/${id}/skill-detail`) + async getSkillDetail(id: number, includeDocumentation = true): Promise { + return fetchJson(`${WEB_API_PREFIX}/reviews/${id}/skill-detail?includeDocumentation=${includeDocumentation}`) }, async approve(id: number, comment?: string): Promise { diff --git a/web/src/api/generated/schema.d.ts b/web/src/api/generated/schema.d.ts index 6273465b7..dae609305 100644 --- a/web/src/api/generated/schema.d.ts +++ b/web/src/api/generated/schema.d.ts @@ -426,6 +426,110 @@ export interface paths { patch?: never; trace?: never; }; + "/api/web/namespaces/{namespaceSlug}/github-sources/{entryId}/visibility": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 设置团队来源后续导入内容的可见范围 */ + put: operations["setVisibility"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/visibility": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 设置团队来源后续导入内容的可见范围 */ + put: operations["setVisibility_1"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/{entryId}/enabled": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 暂停或恢复团队 GitHub 来源同步 */ + put: operations["setEnabled"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/{entryId}/enabled": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 暂停或恢复团队 GitHub 来源同步 */ + put: operations["setEnabled_1"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看团队 GitHub 来源的候选路径忽略规则 */ + get: operations["ignorePatterns"]; + /** 设置团队 GitHub 来源的候选路径忽略规则 */ + put: operations["updateIgnorePatterns"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/ignore-patterns": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看团队 GitHub 来源的候选路径忽略规则 */ + get: operations["ignorePatterns_1"]; + /** 设置团队 GitHub 来源的候选路径忽略规则 */ + put: operations["updateIgnorePatterns_1"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/web/authoring/drafts/{draftId}/files": { parameters: { query?: never; @@ -608,6 +712,167 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/admin/github-sync/sources/{sourceId}/ignore-patterns": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看平台 GitHub 来源的候选路径忽略规则 */ + get: operations["ignorePatterns_2"]; + /** 设置平台 GitHub 来源的候选路径忽略规则 */ + put: operations["updateIgnorePatterns_2"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/schedule-policy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 读取 GitHub 同步三类任务目标周期 */ + get: operations["schedulePolicy"]; + /** 按类别调整 GitHub 同步任务目标周期 */ + put: operations["updateSchedulePolicy"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/platform-control": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 读取平台技能采集控制状态 */ + get: operations["get_2"]; + /** 暂停或恢复平台技能采集 */ + put: operations["update_1"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/{id}/enabled": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 启用或停用 GitHub 发现入口 */ + put: operations["setEnabled_2"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/search-rules/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 编辑仓库搜索规则并生成新版本 */ + put: operations["updateSearchRule"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/configuration": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * 读取官方 GitHub 深度探索配置 + * @description 配置启用与采集服务在线及执行状态独立 + */ + get: operations["get_3"]; + /** 按配置版本更新官方 GitHub 深度探索设置 */ + put: operations["update_2"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/credentials/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 替换系统级公开 GitHub 凭证并恢复健康状态 */ + put: operations["replace"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/credentials/{id}/enabled": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** 启用或停用系统级公开 GitHub 凭证 */ + put: operations["setEnabled_3"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/admission-policy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 读取 GitHub 同步准入水位与当前积压 */ + get: operations["getAdmissionPolicy"]; + /** 更新 GitHub 同步准入水位与候选预算 */ + put: operations["updateAdmissionPolicy"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/suites/{suiteId}/versions/{versionId}/yank": { parameters: { query?: never; @@ -2306,6 +2571,144 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/retry-rejected-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 重新采集并送审一个已拒绝的团队 GitHub Skill */ + post: operations["retryRejectedSkill"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/retry-rejected-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 重新采集并送审一个已拒绝的团队 GitHub Skill */ + post: operations["retryRejectedSkill_1"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/rescan": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 按现有任务队列安排团队 GitHub 来源重新检查 */ + post: operations["rescan"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/rescan": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 按现有任务队列安排团队 GitHub 来源重新检查 */ + post: operations["rescan_1"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/conflicts/{ingestionId}/bind-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 确认团队 GitHub 来源与既有 Skill ID 的绑定 */ + post: operations["bindExistingSkill"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/conflicts/{ingestionId}/bind-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 确认团队 GitHub 来源与既有 Skill ID 的绑定 */ + post: operations["bindExistingSkill_1"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看团队 GitHub 公开仓库来源配置 */ + get: operations["list_6"]; + put?: never; + /** 为团队登记 GitHub 公开仓库 */ + post: operations["create_2"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看团队 GitHub 公开仓库来源配置 */ + get: operations["list_7"]; + put?: never; + /** 为团队登记 GitHub 公开仓库 */ + post: operations["create_3"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/web/namespaces": { parameters: { query?: never; @@ -2547,9 +2950,9 @@ export interface paths { path?: never; cookie?: never; }; - get: operations["list_4"]; + get: operations["list_8"]; put?: never; - post: operations["create_2"]; + post: operations["create_4"]; delete?: never; options?: never; head?: never; @@ -3181,6 +3584,143 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/admin/github-sync/sources/{sourceId}/retry-rejected-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 重新采集并送审一个已拒绝的 GitHub Skill */ + post: operations["retryRejectedSkill_2"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/sources/{sourceId}/rescan": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 按现有任务队列安排平台 GitHub 来源重新检查 */ + post: operations["rescan_2"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/search-rules": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 创建初始停用的仓库搜索规则 */ + post: operations["createSearchRule"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/search-rules/preview": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 预览搜索规则的仓库命中情况，不登记来源 */ + post: operations["previewSearchRule"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/curated-repositories": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 登记精选 GitHub 公开仓库 */ + post: operations["createCuratedRepository"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/curated-owners": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 登记精选 GitHub 用户或组织 */ + post: operations["createCuratedOwner"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/credentials": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看系统级公开 GitHub 凭证的脱敏状态 */ + get: operations["list_11"]; + put?: never; + /** 加密保存系统级公开 GitHub 凭证 */ + post: operations["create_5"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/conflicts/{ingestionId}/bind-skill": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** 确认平台 GitHub 来源与既有 Skill ID 的绑定 */ + post: operations["bindExistingSkill_2"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/account/merge/verify": { parameters: { query?: never; @@ -4315,6 +4855,108 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/reviews/sync/{id}/source": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看同步审核的 GitHub 来源和许可证证据 */ + get: operations["getSource"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/reviews/sync/{id}/source": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看同步审核的 GitHub 来源和许可证证据 */ + get: operations["getSource_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/reviews/sync/{id}/base-file": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 预览同步审核基线版本的变更文件 */ + get: operations["getBaseFile"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/reviews/sync/{id}/base-file": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 预览同步审核基线版本的变更文件 */ + get: operations["getBaseFile_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/reviews/sync": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 按审核权限查询 GitHub 同步审核队列 */ + get: operations["list_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/reviews/sync": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 按审核权限查询 GitHub 同步审核队列 */ + get: operations["list_3"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/web/reviews/pending": { parameters: { query?: never; @@ -4740,7 +5382,7 @@ export interface paths { path?: never; cookie?: never; }; - get: operations["list_2"]; + get: operations["list_4"]; put?: never; post?: never; delete?: never; @@ -4756,7 +5398,7 @@ export interface paths { path?: never; cookie?: never; }; - get: operations["list_3"]; + get: operations["list_5"]; put?: never; post?: never; delete?: never; @@ -4829,6 +5471,108 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队 GitHub 来源的发现与采集运行 */ + get: operations["runs"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队 GitHub 来源的发现与采集运行 */ + get: operations["runs_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队 GitHub 来源的逐 Skill 路径状态 */ + get: operations["skillPaths"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/resolved/{sourceId}/skills": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队 GitHub 来源的逐 Skill 路径状态 */ + get: operations["skillPaths_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/web/namespaces/{namespaceSlug}/github-sources/resolved": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队已解析 GitHub 来源与摄入进度 */ + get: operations["resolved"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/namespaces/{namespaceSlug}/github-sources/resolved": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看团队已解析 GitHub 来源与摄入进度 */ + get: operations["resolved_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/web/me/suites/workspace": { parameters: { query?: never; @@ -5734,7 +6478,7 @@ export interface paths { path?: never; cookie?: never; }; - get: operations["list_5"]; + get: operations["list_9"]; put?: never; post?: never; delete?: never; @@ -5791,6 +6535,329 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/admin/github-sync/worker-status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看 GitHub 采集服务最后在线时间 */ + get: operations["workerStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/sources": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台 GitHub 来源及摄入进度 */ + get: operations["sources"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/sources/{sourceId}/skills": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台 GitHub 来源的逐 Skill 路径状态 */ + get: operations["skillPaths_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台 GitHub 发现与仓库采集运行 */ + get: operations["runs_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/runs/{runId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看平台 GitHub 同步单次运行详情 */ + get: operations["runDetail"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/runs/{runId}/tasks": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台 GitHub 同步单次运行的任务 */ + get: operations["runTasks"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/runs/{runId}/paths": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台 GitHub 同步单次运行的逐 Skill 路径结果 */ + get: operations["runPaths"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/repositories": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看平台全部发现方式的仓库 */ + get: operations["repositories"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/repositories/{repositoryId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看统一仓库登记和最近检查摘要 */ + get: operations["repository"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/repositories/{repositoryId}/paths": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看统一仓库技能路径及审核链接 */ + get: operations["paths"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/repositories/{repositoryId}/origins": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查看仓库全部持久出处 */ + get: operations["origins"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看 GitHub 发现规则与精选入口 */ + get: operations["list_10"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/discovery-entries/search-rules/previews/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看搜索规则预览结果 */ + get: operations["searchRulePreview"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/summary": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看官方探索近期成果与当前积压 */ + get: operations["summary_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/repositories": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查询官方探索命中的仓库 */ + get: operations["repositories_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 按需读取仓库检查摘要 */ + get: operations["repository_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}/provenance": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查询仓库命中出处 */ + get: operations["provenance"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/repositories/{repositoryId}/paths": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 分页查询仓库技能路径最新结果 */ + get: operations["paths_1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/github-sync/deep-discovery/progress": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** 查看官方探索渠道进度与等待原因 */ + get: operations["progress"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/admin/audit-logs": { parameters: { query?: never; @@ -6354,7 +7421,7 @@ export interface components { status?: "ACTIVE" | "FROZEN" | "ARCHIVED"; description?: string; /** @enum {string} */ - type?: "GLOBAL" | "TEAM"; + type?: "GLOBAL" | "CATALOG" | "TEAM"; avatarUrl?: string; createdBy?: string; /** Format: date-time */ @@ -6362,6 +7429,75 @@ export interface components { /** Format: date-time */ updatedAt?: string; }; + GithubTeamSourceVisibilityRequest: { + visibility: string; + }; + ApiResponseGithubDiscoveryEntryResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDiscoveryEntryResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + Execution: { + /** Format: uuid */ + taskId?: string; + /** Format: uuid */ + runId?: string; + status?: string; + errorCode?: string; + /** Format: date-time */ + retryAt?: string; + }; + GithubDiscoveryEntryResponse: { + /** Format: uuid */ + id?: string; + entryType?: string; + title?: string; + ownerLogin?: string; + repositoryName?: string; + /** Format: int64 */ + repositoryId?: number; + sourceVisibility?: string; + selectors?: components["schemas"]["GithubSearchSelectorRequest"][]; + /** Format: int64 */ + ruleVersion?: number; + enabled?: boolean; + /** Format: int32 */ + targetHours?: number; + /** Format: date-time */ + nextDueAt?: string; + /** Format: date-time */ + updatedAt?: string; + latestExecution?: components["schemas"]["Execution"]; + }; + GithubSearchSelectorRequest: { + kind?: string; + value?: string; + priority?: string; + }; + GithubDiscoveryEntryEnabledRequest: { + enabled: boolean; + }; + GithubSourceIgnoreRequest: { + ignorePatterns: string[]; + }; + ApiResponseGithubSourceIgnoreResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSourceIgnoreResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSourceIgnoreResponse: { + /** Format: int64 */ + sourceId?: number; + ignorePatterns?: string[]; + }; SaveDraftFileRequest: { path: string; content: string; @@ -6592,6 +7728,198 @@ export interface components { timestamp?: string; requestId?: string; }; + GithubSyncSchedulePolicyRequest: { + /** Format: int32 */ + repositoryHours?: number; + /** Format: int32 */ + curatedOwnerHours?: number; + /** Format: int32 */ + searchRuleHours?: number; + }; + ApiResponseGithubSyncSchedulePolicyResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncSchedulePolicyResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncSchedulePolicyResponse: { + /** Format: int32 */ + repositoryHours?: number; + /** Format: int32 */ + curatedOwnerHours?: number; + /** Format: int32 */ + searchRuleHours?: number; + effectiveRepositoryInterval?: string; + /** Format: date-time */ + updatedAt?: string; + }; + GithubPlatformControlUpdateRequest: { + /** Format: int64 */ + expectedVersion: number; + paused: boolean; + }; + ApiResponseGithubPlatformControlState: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubPlatformControlState"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubPlatformControlState: { + /** Format: int64 */ + namespaceId?: number; + /** Format: int64 */ + version?: number; + desiredPaused?: boolean; + state?: string; + /** Format: int64 */ + pendingStopCount?: number; + /** Format: int64 */ + expiredUnconfirmedCount?: number; + /** Format: date-time */ + updatedAt?: string; + }; + GithubSearchRuleRequest: { + title: string; + selectors: components["schemas"]["GithubSearchSelectorRequest"][]; + /** Format: int32 */ + targetHours?: number; + }; + ExclusionRequest: { + /** @enum {string} */ + kind: "OWNER" | "REPOSITORY"; + value: string; + }; + GithubDeepDiscoveryConfigurationRequest: { + /** Format: int64 */ + expectedVersion: number; + enabled: boolean; + /** @enum {string} */ + budgetProfile: "ECONOMY" | "BALANCED" | "ACCELERATED"; + exclusions: components["schemas"]["ExclusionRequest"][]; + }; + ApiResponseGithubDeepDiscoveryConfigurationResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDeepDiscoveryConfigurationResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + Exclusion: { + /** @enum {string} */ + kind?: "OWNER" | "REPOSITORY"; + value?: string; + }; + GithubDeepDiscoveryConfigurationResponse: { + /** Format: int64 */ + version?: number; + enabled?: boolean; + /** @enum {string} */ + budgetProfile?: "ECONOMY" | "BALANCED" | "ACCELERATED"; + exclusions?: components["schemas"]["Exclusion"][]; + /** @description 尚未装载官方策略时为空；配置启用不表示正在执行 */ + strategyVersion?: string | null; + /** Format: date-time */ + updatedAt?: string; + updatedBy?: string | null; + }; + GithubCredentialUpsertRequest: { + label: string; + token: string; + }; + ApiResponseGithubCredentialResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubCredentialResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubCredentialQuota: { + resource?: string; + /** Format: int32 */ + remaining?: number; + /** Format: date-time */ + resetAt?: string; + /** Format: date-time */ + observedAt?: string; + }; + GithubCredentialResponse: { + /** Format: uuid */ + id?: string; + label?: string; + fingerprint?: string; + status?: string; + keyVersion?: string; + keyAvailable?: boolean; + /** Format: date-time */ + rateLimitedUntil?: string; + /** Format: date-time */ + lastUsedAt?: string; + /** Format: date-time */ + createdAt?: string; + /** Format: date-time */ + updatedAt?: string; + quotas?: components["schemas"]["GithubCredentialQuota"][]; + }; + GithubCredentialEnabledRequest: { + enabled: boolean; + }; + GithubSyncAdmissionPolicyUpdateRequest: { + /** Format: int32 */ + globalHigh: number; + /** Format: int32 */ + globalLow: number; + /** Format: int32 */ + namespaceHigh: number; + /** Format: int32 */ + namespaceLow: number; + /** Format: int32 */ + candidatePendingLimit: number; + /** Format: int32 */ + candidatePerLeaseLimit: number; + }; + ApiResponseGithubSyncAdmissionPolicyResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncAdmissionPolicyResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncAdmissionPolicyResponse: { + /** Format: int32 */ + globalHigh?: number; + /** Format: int32 */ + globalLow?: number; + /** Format: int32 */ + namespaceHigh?: number; + /** Format: int32 */ + namespaceLow?: number; + /** Format: int32 */ + candidatePendingLimit?: number; + /** Format: int32 */ + candidatePerLeaseLimit?: number; + globalPaused?: boolean; + /** Format: int32 */ + globalUnresolvedCount?: number; + /** Format: int32 */ + platformUnresolvedCount?: number; + /** Format: int32 */ + platformPendingCandidateCount?: number; + platformPaused?: boolean; + /** Format: date-time */ + updatedAt?: string; + }; SkillSuiteReasonRequest: { reason: string; }; @@ -6917,6 +8245,7 @@ export interface components { avatarRemoval?: boolean; previousVisibility?: string; requestedVisibility?: string; + originType?: string; }; ReviewTaskRequest: { /** Format: int64 */ @@ -7075,6 +8404,58 @@ export interface components { NamespaceLifecycleRequest: { reason?: string; }; + GithubSyncRetrySkillRequest: { + skillPath: string; + }; + ApiResponseGithubSourceRescanResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSourceRescanResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSourceRescanResponse: { + /** Format: int64 */ + sourceId?: number; + status?: string; + /** Format: uuid */ + taskId?: string; + taskStatus?: string; + /** Format: date-time */ + nextCheckAt?: string; + }; + GithubSyncManualBindingRequest: { + /** Format: int64 */ + skillId: number; + reason: string; + }; + ApiResponseGithubSyncManualBindingResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncManualBindingResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncManualBindingResponse: { + /** Format: uuid */ + ingestionId?: string; + /** Format: int64 */ + sourceId?: number; + /** Format: int64 */ + skillId?: number; + skillSlug?: string; + skillPath?: string; + status?: string; + }; + GithubTeamRepositoryRequest: { + ownerLogin: string; + repositoryName: string; + visibility?: string; + }; NamespaceInvitationTokenRequest: { token: string; }; @@ -7452,6 +8833,15 @@ export interface components { sortOrder?: number; translations: components["schemas"]["LabelTranslationItemRequest"][]; }; + GithubCuratedRepositoryRequest: { + ownerLogin: string; + repositoryName: string; + }; + GithubCuratedOwnerRequest: { + ownerLogin: string; + /** Format: int32 */ + targetHours?: number; + }; MergeVerifyRequest: { /** Format: int64 */ mergeRequestId: number; @@ -7760,6 +9150,8 @@ export interface components { status?: string; displayName?: string; summary?: string; + /** Format: int32 */ + syncRevision?: number; }; SkillSummaryResponse: { /** Format: int64 */ @@ -7956,6 +9348,8 @@ export interface components { /** Format: int64 */ id?: number; version?: string; + /** Format: int32 */ + syncRevision?: number; status?: string; changelog?: string | null; /** Format: int32 */ @@ -8050,6 +9444,21 @@ export interface components { timestamp?: string; requestId?: string; }; + GithubSkillSourceResponse: { + ownerLogin?: string; + repositoryName?: string; + skillPath?: string; + commitSha?: string; + /** Format: int32 */ + syncRevision?: number; + licenseStatus?: string; + licenseDocuments?: components["schemas"]["LicenseDocument"][]; + }; + LicenseDocument: { + scope?: string; + path?: string; + bundled?: boolean; + }; SkillDetailResponse: { /** Format: int64 */ id?: number; @@ -8085,6 +9494,7 @@ export interface components { memberOfSuites?: components["schemas"]["PageResponseSkillSuiteReferenceResponse"]; avatarUrl?: string; ownerReadOnly?: boolean; + githubSource?: components["schemas"]["GithubSkillSourceResponse"]; }; ApiResponseReviewSkillDetailResponse: { /** Format: int32 */ @@ -8131,6 +9541,94 @@ export interface components { timestamp?: string; requestId?: string; }; + ApiResponseGithubSyncReviewSourceResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncReviewSourceResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + EvidenceDocument: { + scope?: string; + path?: string; + gitSha?: string; + sha256?: string; + /** Format: int64 */ + size?: number; + bundled?: boolean; + text?: string; + }; + FileChange: { + path?: string; + changeType?: string; + /** Format: int64 */ + oldSize?: number; + /** Format: int64 */ + newSize?: number; + oldSha256?: string; + newSha256?: string; + }; + GithubSyncReviewSourceResponse: { + /** Format: uuid */ + ingestionId?: string; + /** Format: uuid */ + runId?: string; + /** Format: int64 */ + sourceId?: number; + /** Format: int64 */ + repositoryId?: number; + ownerLogin?: string; + repositoryName?: string; + commitSha?: string; + skillPath?: string; + contentSha256?: string; + /** Format: int32 */ + syncRevision?: number; + ingestionStatus?: string; + /** Format: date-time */ + receivedAt?: string; + evidence?: components["schemas"]["LicenseEvidence"]; + /** Format: int64 */ + baseVersionId?: number; + fileChanges?: components["schemas"]["FileChange"][]; + }; + LicenseEvidence: { + status?: string; + documents?: components["schemas"]["EvidenceDocument"][]; + }; + ApiResponsePageResponseGithubSyncReviewQueueItemResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncReviewQueueItemResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncReviewQueueItemResponse: { + review?: components["schemas"]["ReviewTaskResponse"]; + /** Format: uuid */ + ingestionId?: string; + /** Format: uuid */ + runId?: string; + /** Format: int64 */ + repositoryId?: number; + ownerLogin?: string; + repositoryName?: string; + skillPath?: string; + commitSha?: string; + }; + PageResponseGithubSyncReviewQueueItemResponse: { + items?: components["schemas"]["GithubSyncReviewQueueItemResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; ApiResponsePageResponseReviewTaskResponse: { /** Format: int32 */ code?: number; @@ -8433,6 +9931,165 @@ export interface components { InvitationStatus: { active?: boolean; }; + ApiResponsePageResponseGithubSyncRunSummaryResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncRunSummaryResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRunSummaryResponse: { + /** Format: uuid */ + id?: string; + kind?: string; + status?: string; + /** Format: date-time */ + createdAt?: string; + /** Format: date-time */ + finishedAt?: string; + /** Format: date-time */ + nextRetryAt?: string; + /** Format: int64 */ + taskCount?: number; + /** Format: int64 */ + receivedPackages?: number; + /** Format: int64 */ + processingPackages?: number; + /** Format: int64 */ + pendingReviewPackages?: number; + /** Format: int64 */ + blockedPackages?: number; + /** Format: int64 */ + failedPackages?: number; + lastErrorCode?: string; + target?: components["schemas"]["GithubSyncRunTarget"]; + /** Format: int64 */ + targetCount?: number; + }; + GithubSyncRunTarget: { + type?: string; + value?: string; + }; + PageResponseGithubSyncRunSummaryResponse: { + items?: components["schemas"]["GithubSyncRunSummaryResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; + ApiResponsePageResponseGithubSyncSkillPathResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncSkillPathResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncSkillPathResponse: { + skillPath?: string; + commitSha?: string; + acquisitionOutcome?: string; + acquisitionReason?: string; + /** Format: date-time */ + acquisitionRetryAt?: string; + /** Format: date-time */ + acquisitionUpdatedAt?: string; + /** Format: uuid */ + ingestionId?: string; + ingestionStatus?: string; + ingestionErrorCode?: string; + targetSkillSlug?: string; + /** Format: int64 */ + skillId?: number; + upstreamPresence?: string; + presenceCommitSha?: string; + /** Format: date-time */ + missingSince?: string; + /** Format: int64 */ + reviewId?: number; + reviewStatus?: string; + latestBindingIngestionStatus?: string; + retryRequested?: boolean; + retryInProgress?: boolean; + ignoredByCurrentPolicy?: boolean; + }; + PageResponseGithubSyncSkillPathResponse: { + items?: components["schemas"]["GithubSyncSkillPathResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; + ApiResponsePageResponseGithubSyncSourceSummaryResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncSourceSummaryResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncSourceSummaryResponse: { + /** Format: int64 */ + id?: number; + /** Format: int64 */ + repositoryId?: number; + ownerLogin?: string; + repositoryName?: string; + defaultBranch?: string; + visibility?: string; + status?: string; + lastObservedCommit?: string; + lastFullyProcessedCommit?: string; + /** Format: date-time */ + nextCheckAt?: string; + lastErrorCode?: string; + /** Format: uuid */ + latestTaskId?: string; + latestTaskStatus?: string; + latestTaskMode?: string; + latestTaskErrorCode?: string; + latestTaskErrorDetail?: string; + /** Format: date-time */ + latestTaskRetryAt?: string; + /** Format: int64 */ + receivedPackages?: number; + /** Format: int64 */ + processingPackages?: number; + /** Format: int64 */ + pendingReviewPackages?: number; + /** Format: int64 */ + blockedPackages?: number; + /** Format: int64 */ + failedPackages?: number; + /** Format: int64 */ + trackedSkills?: number; + }; + PageResponseGithubSyncSourceSummaryResponse: { + items?: components["schemas"]["GithubSyncSourceSummaryResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; + ApiResponseListGithubDiscoveryEntryResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDiscoveryEntryResponse"][]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; ApiResponsePageResponseNamespaceResponse: { /** Format: int32 */ code?: number; @@ -8559,7 +10216,7 @@ export interface components { status?: "ACTIVE" | "FROZEN" | "ARCHIVED"; description?: string; /** @enum {string} */ - type?: "GLOBAL" | "TEAM"; + type?: "GLOBAL" | "CATALOG" | "TEAM"; avatarUrl?: string; createdBy?: string; /** Format: date-time */ @@ -9299,6 +10956,498 @@ export interface components { timestamp?: string; requestId?: string; }; + ApiResponseGithubSyncWorkerStatusResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncWorkerStatusResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncWorkerStatusResponse: { + /** Format: date-time */ + lastSeenAt?: string; + online?: boolean; + }; + ApiResponseGithubSyncRunDetailResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncRunDetailResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRunDetailResponse: { + /** Format: uuid */ + id?: string; + kind?: string; + status?: string; + /** Format: date-time */ + createdAt?: string; + /** Format: date-time */ + finishedAt?: string; + /** Format: int64 */ + taskCount?: number; + /** Format: int64 */ + pathCount?: number; + /** Format: date-time */ + nextRetryAt?: string; + /** Format: int64 */ + receivedPackages?: number; + /** Format: int64 */ + processingPackages?: number; + /** Format: int64 */ + pendingReviewPackages?: number; + /** Format: int64 */ + blockedPackages?: number; + /** Format: int64 */ + failedPackages?: number; + lastErrorCode?: string; + }; + ApiResponsePageResponseGithubSyncRunTaskResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncRunTaskResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRunTaskResponse: { + /** Format: uuid */ + id?: string; + /** Format: int64 */ + sourceId?: number; + repository?: string; + kind?: string; + mode?: string; + status?: string; + fixedCommitSha?: string; + lastErrorCode?: string; + /** Format: date-time */ + retryAt?: string; + /** Format: date-time */ + createdAt?: string; + /** Format: date-time */ + completedAt?: string; + }; + PageResponseGithubSyncRunTaskResponse: { + items?: components["schemas"]["GithubSyncRunTaskResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; + ApiResponsePageResponseGithubSyncRunPathResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageResponseGithubSyncRunPathResponse"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRunPathResponse: { + /** Format: uuid */ + taskId?: string; + /** Format: int64 */ + sourceId?: number; + repository?: string; + skillPath?: string; + commitSha?: string; + outcome?: string; + reason?: string; + ingestionStatus?: string; + ingestionErrorCode?: string; + reviewStatus?: string; + /** Format: date-time */ + retryAt?: string; + /** Format: date-time */ + updatedAt?: string; + }; + PageResponseGithubSyncRunPathResponse: { + items?: components["schemas"]["GithubSyncRunPathResponse"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + }; + ApiResponsePageGithubSyncRepositorySummary: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageGithubSyncRepositorySummary"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRepositorySummary: { + /** Format: int64 */ + repositoryId?: number; + /** Format: int64 */ + sourceId?: number; + /** Format: int64 */ + candidateId?: number; + owner?: string; + name?: string; + /** Format: date-time */ + firstSeenAt?: string; + /** @enum {string} */ + state?: "REGISTERED" | "PENDING" | "VALIDATING" | "NO_VALID_SKILL" | "VALIDATION_FAILED" | "UNREACHABLE" | "BOUND_WITHOUT_SOURCE"; + /** @enum {string} */ + sourceStatus?: "ACTIVE" | "PAUSED" | "UNREACHABLE"; + candidateStatus?: string; + originTypes?: ("CURATED_REPOSITORY" | "CURATED_OWNER" | "SEARCH_RULE" | "OFFICIAL_DISCOVERY" | "TEAM_CONFIG")[]; + }; + PageGithubSyncRepositorySummary: { + items?: components["schemas"]["GithubSyncRepositorySummary"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + ApiResponseGithubSyncRepositoryDetail: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubSyncRepositoryDetail"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRepositoryDetail: { + repository?: components["schemas"]["GithubSyncRepositorySummary"]; + /** Format: uuid */ + latestRunId?: string; + /** Format: date-time */ + lastCheckedAt?: string; + /** Format: date-time */ + nextCheckAt?: string; + lastErrorCode?: string; + }; + ApiResponsePageGithubDeepDiscoveryPathResult: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageGithubDeepDiscoveryPathResult"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubDeepDiscoveryPathResult: { + skillPath?: string; + outcome?: string; + reasonCode?: string; + /** Format: int64 */ + skillId?: number; + /** Format: int64 */ + versionId?: number; + /** Format: int64 */ + reviewId?: number; + /** Format: uuid */ + runId?: string; + /** Format: date-time */ + observedAt?: string; + }; + PageGithubDeepDiscoveryPathResult: { + items?: components["schemas"]["GithubDeepDiscoveryPathResult"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + ApiResponsePageGithubSyncRepositoryOrigin: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["PageGithubSyncRepositoryOrigin"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubSyncRepositoryOrigin: { + /** @enum {string} */ + originType?: "CURATED_REPOSITORY" | "CURATED_OWNER" | "SEARCH_RULE" | "OFFICIAL_DISCOVERY" | "TEAM_CONFIG"; + originKey?: string; + /** Format: date-time */ + firstSeenAt?: string; + /** Format: date-time */ + lastSeenAt?: string; + /** Format: uuid */ + entryId?: string; + channel?: string; + /** Format: uuid */ + queryId?: string; + /** Format: uuid */ + runId?: string; + /** Format: int64 */ + seedRepositoryId?: number; + }; + PageGithubSyncRepositoryOrigin: { + items?: components["schemas"]["GithubSyncRepositoryOrigin"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + ApiResponseGithubDeepDiscoverySummary: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDeepDiscoverySummary"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubDeepDiscoveryCurrent: { + /** Format: int64 */ + pendingRepositoryChecks?: number; + /** Format: int64 */ + pendingReviewVersions?: number; + /** Format: date-time */ + observedAt?: string; + }; + GithubDeepDiscoveryRecent: { + /** Format: date-time */ + from?: string; + /** Format: date-time */ + until?: string; + /** Format: int64 */ + newCandidateRepositories?: number; + /** Format: int64 */ + firstValidRepositories?: number; + /** Format: int64 */ + newSkills?: number; + /** Format: int64 */ + updatedVersions?: number; + complete?: boolean; + /** Format: date-time */ + availableSince?: string; + }; + GithubDeepDiscoverySummary: { + /** Format: int64 */ + configurationVersion?: number; + strategyVersion?: string; + state?: string; + /** Format: date-time */ + observedAt?: string; + worker?: components["schemas"]["GithubOfficialWorkerAvailabilityResponse"]; + recent?: components["schemas"]["GithubDeepDiscoveryRecent"]; + current?: components["schemas"]["GithubDeepDiscoveryCurrent"]; + waits?: components["schemas"]["GithubDeepDiscoveryWait"][]; + recentRunIds?: string[]; + }; + GithubDeepDiscoveryWait: { + reason?: string; + affectedWork?: string[]; + continuingWork?: string[]; + /** Format: date-time */ + retryAt?: string; + recoveryCondition?: string; + action?: string; + }; + GithubOfficialWorkerAvailabilityResponse: { + state?: string; + /** Format: date-time */ + lastSeenAt?: string; + /** Format: int64 */ + compatibleInstanceCount?: number; + /** Format: date-time */ + observedAt?: string; + }; + ApiResponseDiscoveryPageGithubDeepDiscoveryRepositorySummary: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["DiscoveryPageGithubDeepDiscoveryRepositorySummary"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + DiscoveryPageGithubDeepDiscoveryRepositorySummary: { + items?: components["schemas"]["GithubDeepDiscoveryRepositorySummary"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + GithubDeepDiscoveryRepositorySummary: { + /** Format: int64 */ + repositoryId?: number; + owner?: string; + name?: string; + /** Format: date-time */ + firstDiscoveredAt?: string; + /** Format: date-time */ + lastDiscoveredAt?: string; + /** @enum {string} */ + status?: "PENDING" | "VALIDATING" | "VALID" | "NO_VALID_SKILL" | "VALIDATION_FAILED" | "UNREACHABLE"; + channels?: ("REPOSITORY_METADATA" | "README" | "OWNER_EXPANSION")[]; + /** Format: int64 */ + sourceId?: number; + paused?: boolean; + waitReason?: string; + /** Format: int64 */ + candidateId?: number; + }; + ApiResponseGithubDeepDiscoveryRepositoryDetail: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDeepDiscoveryRepositoryDetail"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubDeepDiscoveryRepositoryDetail: { + repository?: components["schemas"]["GithubDeepDiscoveryRepositorySummary"]; + /** Format: date-time */ + lastCheckedAt?: string; + /** Format: date-time */ + nextEligibleAt?: string; + notImportedReason?: string; + waits?: components["schemas"]["GithubDeepDiscoveryWait"][]; + /** Format: uuid */ + latestRunId?: string; + /** Format: date-time */ + historyAvailableSince?: string; + }; + ApiResponseDiscoveryPageGithubDeepDiscoveryProvenance: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["DiscoveryPageGithubDeepDiscoveryProvenance"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + DiscoveryPageGithubDeepDiscoveryProvenance: { + items?: components["schemas"]["GithubDeepDiscoveryProvenance"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + GithubDeepDiscoveryProvenance: { + /** Format: uuid */ + id?: string; + /** @enum {string} */ + channel?: "REPOSITORY_METADATA" | "README" | "OWNER_EXPANSION"; + strategyVersion?: string; + /** Format: uuid */ + queryId?: string; + /** Format: int64 */ + queryVersion?: number; + /** Format: uuid */ + shardId?: string; + /** Format: uuid */ + runId?: string; + /** Format: date-time */ + firstSeenAt?: string; + /** Format: date-time */ + lastSeenAt?: string; + /** Format: int64 */ + seedRepositoryId?: number; + }; + ApiResponseDiscoveryPageGithubDeepDiscoveryPathResult: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["DiscoveryPageGithubDeepDiscoveryPathResult"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + DiscoveryPageGithubDeepDiscoveryPathResult: { + items?: components["schemas"]["GithubDeepDiscoveryPathResult"][]; + /** Format: int64 */ + total?: number; + /** Format: int32 */ + page?: number; + /** Format: int32 */ + size?: number; + /** Format: date-time */ + asOf?: string; + }; + ApiResponseGithubDeepDiscoveryProgress: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubDeepDiscoveryProgress"]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; + GithubDeepDiscoveryChannelProgress: { + /** @enum {string} */ + channel?: "REPOSITORY_METADATA" | "README" | "OWNER_EXPANSION"; + state?: string; + /** Format: int64 */ + completedShards?: number; + /** Format: int64 */ + pendingShards?: number; + /** Format: int64 */ + gapShards?: number; + /** Format: date-time */ + lastProgressAt?: string; + waits?: components["schemas"]["GithubDeepDiscoveryWait"][]; + }; + GithubDeepDiscoveryProgress: { + strategyVersion?: string; + /** Format: date-time */ + observedAt?: string; + channels?: components["schemas"]["GithubDeepDiscoveryChannelProgress"][]; + quotas?: components["schemas"]["GithubDeepDiscoveryQuota"][]; + quotaObservationsTruncated?: boolean; + }; + GithubDeepDiscoveryQuota: { + resource?: string; + subjectKey?: string; + state?: string; + /** Format: int64 */ + remaining?: number; + /** Format: date-time */ + observedAt?: string; + /** Format: date-time */ + validUntil?: string; + /** Format: date-time */ + retryAt?: string; + }; + ApiResponseListGithubCredentialResponse: { + /** Format: int32 */ + code?: number; + msg?: string; + data?: components["schemas"]["GithubCredentialResponse"][]; + /** Format: date-time */ + timestamp?: string; + requestId?: string; + }; ApiResponsePageResponseAuditLogItemResponse: { /** Format: int32 */ code?: number; @@ -10688,6 +12837,214 @@ export interface operations { }; }; }; + setVisibility: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + entryId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubTeamSourceVisibilityRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + setVisibility_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + entryId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubTeamSourceVisibilityRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + setEnabled: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + entryId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubDiscoveryEntryEnabledRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + setEnabled_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + entryId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubDiscoveryEntryEnabledRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + ignorePatterns: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; + updateIgnorePatterns: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSourceIgnoreRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; + ignorePatterns_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; + updateIgnorePatterns_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSourceIgnoreRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; listDraftFiles: { parameters: { query?: never; @@ -11149,6 +13506,334 @@ export interface operations { }; }; }; + ignorePatterns_2: { + parameters: { + query?: never; + header?: never; + path: { + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; + updateIgnorePatterns_2: { + parameters: { + query?: never; + header?: never; + path: { + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSourceIgnoreRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceIgnoreResponse"]; + }; + }; + }; + }; + schedulePolicy: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncSchedulePolicyResponse"]; + }; + }; + }; + }; + updateSchedulePolicy: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncSchedulePolicyRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncSchedulePolicyResponse"]; + }; + }; + }; + }; + get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubPlatformControlState"]; + }; + }; + }; + }; + update_1: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubPlatformControlUpdateRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubPlatformControlState"]; + }; + }; + }; + }; + setEnabled_2: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubDiscoveryEntryEnabledRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + updateSearchRule: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSearchRuleRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + get_3: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDeepDiscoveryConfigurationResponse"]; + }; + }; + }; + }; + update_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubDeepDiscoveryConfigurationRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDeepDiscoveryConfigurationResponse"]; + }; + }; + }; + }; + replace: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubCredentialUpsertRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubCredentialResponse"]; + }; + }; + }; + }; + setEnabled_3: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubCredentialEnabledRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubCredentialResponse"]; + }; + }; + }; + }; + getAdmissionPolicy: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncAdmissionPolicyResponse"]; + }; + }; + }; + }; + updateAdmissionPolicy: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncAdmissionPolicyUpdateRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncAdmissionPolicyResponse"]; + }; + }; + }; + }; yankSkillSuiteVersion: { parameters: { query?: never; @@ -14171,6 +16856,256 @@ export interface operations { }; }; }; + retryRejectedSkill: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncRetrySkillRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + retryRejectedSkill_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncRetrySkillRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + rescan: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + rescan_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + bindExistingSkill: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + ingestionId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncManualBindingRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncManualBindingResponse"]; + }; + }; + }; + }; + bindExistingSkill_1: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + ingestionId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncManualBindingRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncManualBindingResponse"]; + }; + }; + }; + }; + list_6: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseListGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + create_2: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubTeamRepositoryRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + list_7: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseListGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + create_3: { + parameters: { + query?: never; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubTeamRepositoryRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; listNamespaces: { parameters: { query: { @@ -14623,7 +17558,7 @@ export interface operations { }; }; }; - list_4: { + list_8: { parameters: { query?: { page?: number; @@ -14646,7 +17581,7 @@ export interface operations { }; }; }; - create_2: { + create_4: { parameters: { query?: never; header?: never; @@ -15716,6 +18651,220 @@ export interface operations { }; }; }; + retryRejectedSkill_2: { + parameters: { + query?: never; + header?: never; + path: { + sourceId: number; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncRetrySkillRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + rescan_2: { + parameters: { + query?: never; + header?: never; + path: { + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSourceRescanResponse"]; + }; + }; + }; + }; + createSearchRule: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSearchRuleRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + previewSearchRule: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSearchRuleRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePreview"]; + }; + }; + }; + }; + createCuratedRepository: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubCuratedRepositoryRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + createCuratedOwner: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubCuratedOwnerRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + list_11: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseListGithubCredentialResponse"]; + }; + }; + }; + }; + create_5: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubCredentialUpsertRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubCredentialResponse"]; + }; + }; + }; + }; + bindExistingSkill_2: { + parameters: { + query?: never; + header?: never; + path: { + ingestionId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GithubSyncManualBindingRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncManualBindingResponse"]; + }; + }; + }; + }; verify: { parameters: { query?: never; @@ -17315,7 +20464,10 @@ export interface operations { }; getReviewSkillDetail: { parameters: { - query?: never; + query?: { + /** @description 是否包含 README.md/SKILL.md 正文；缺省 true，false 仅返回元数据 */ + includeDocumentation?: boolean; + }; header?: never; path: { id: number; @@ -17337,7 +20489,10 @@ export interface operations { }; getReviewSkillDetail_1: { parameters: { - query?: never; + query?: { + /** @description 是否包含 README.md/SKILL.md 正文；缺省 true，false 仅返回元数据 */ + includeDocumentation?: boolean; + }; header?: never; path: { id: number; @@ -17361,6 +20516,8 @@ export interface operations { parameters: { query: { path: string; + /** @description 是否按 256 KiB 上限读取预览；缺省 false 保留完整文件读取 */ + preview?: boolean; }; header?: never; path: { @@ -17385,6 +20542,8 @@ export interface operations { parameters: { query: { path: string; + /** @description 是否按 256 KiB 上限读取预览；缺省 false 保留完整文件读取 */ + preview?: boolean; }; header?: never; path: { @@ -17537,6 +20696,154 @@ export interface operations { }; }; }; + getSource: { + parameters: { + query?: never; + header?: never; + path: { + id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncReviewSourceResponse"]; + }; + }; + }; + }; + getSource_1: { + parameters: { + query?: never; + header?: never; + path: { + id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncReviewSourceResponse"]; + }; + }; + }; + }; + getBaseFile: { + parameters: { + query: { + path: string; + /** @description 是否按 256 KiB 上限读取预览；缺省 false 保留原读取行为 */ + preview?: boolean; + }; + header?: never; + path: { + id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": string; + }; + }; + }; + }; + getBaseFile_1: { + parameters: { + query: { + path: string; + /** @description 是否按 256 KiB 上限读取预览；缺省 false 保留原读取行为 */ + preview?: boolean; + }; + header?: never; + path: { + id: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": string; + }; + }; + }; + }; + list_2: { + parameters: { + query: { + status: string; + namespaceId?: number; + page?: number; + size?: number; + sortDirection?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncReviewQueueItemResponse"]; + }; + }; + }; + }; + list_3: { + parameters: { + query: { + status: string; + namespaceId?: number; + page?: number; + size?: number; + sortDirection?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncReviewQueueItemResponse"]; + }; + }; + }; + }; listPendingReviews: { parameters: { query: { @@ -18131,7 +21438,7 @@ export interface operations { }; }; }; - list_2: { + list_4: { parameters: { query?: { category?: string; @@ -18155,7 +21462,7 @@ export interface operations { }; }; }; - list_3: { + list_5: { parameters: { query?: { category?: string; @@ -18279,6 +21586,158 @@ export interface operations { }; }; }; + runs: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncRunSummaryResponse"]; + }; + }; + }; + }; + runs_1: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncRunSummaryResponse"]; + }; + }; + }; + }; + skillPaths: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSkillPathResponse"]; + }; + }; + }; + }; + skillPaths_1: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSkillPathResponse"]; + }; + }; + }; + }; + resolved: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSourceSummaryResponse"]; + }; + }; + }; + }; + resolved_1: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + namespaceSlug: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSourceSummaryResponse"]; + }; + }; + }; + }; listMySkillSuiteWorkspace: { parameters: { query?: { @@ -19596,7 +23055,7 @@ export interface operations { }; }; }; - list_5: { + list_9: { parameters: { query?: { status?: string; @@ -19694,6 +23153,460 @@ export interface operations { }; }; }; + workerStatus: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncWorkerStatusResponse"]; + }; + }; + }; + }; + sources: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSourceSummaryResponse"]; + }; + }; + }; + }; + skillPaths_2: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + sourceId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncSkillPathResponse"]; + }; + }; + }; + }; + runs_2: { + parameters: { + query?: { + page?: number; + size?: number; + kind?: "DISCOVERY" | "REPOSITORY_SYNC"; + status?: "QUEUED" | "LEASED" | "RETRY_WAIT" | "PARTIAL" | "ACQUISITION_COMPLETE" | "FAILED"; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncRunSummaryResponse"]; + }; + }; + }; + }; + runDetail: { + parameters: { + query?: never; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncRunDetailResponse"]; + }; + }; + }; + }; + runTasks: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncRunTaskResponse"]; + }; + }; + }; + }; + runPaths: { + parameters: { + query?: { + page?: number; + size?: number; + }; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageResponseGithubSyncRunPathResponse"]; + }; + }; + }; + }; + repositories: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + q?: string; + originType?: "CURATED_REPOSITORY" | "CURATED_OWNER" | "SEARCH_RULE" | "OFFICIAL_DISCOVERY" | "TEAM_CONFIG"; + state?: "REGISTERED" | "PENDING" | "VALIDATING" | "NO_VALID_SKILL" | "VALIDATION_FAILED" | "UNREACHABLE" | "BOUND_WITHOUT_SOURCE"; + sourceStatus?: "ACTIVE" | "PAUSED" | "UNREACHABLE"; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageGithubSyncRepositorySummary"]; + }; + }; + }; + }; + repository: { + parameters: { + query?: never; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubSyncRepositoryDetail"]; + }; + }; + }; + }; + paths: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + }; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageGithubDeepDiscoveryPathResult"]; + }; + }; + }; + }; + origins: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + }; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePageGithubSyncRepositoryOrigin"]; + }; + }; + }; + }; + list_10: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseListGithubDiscoveryEntryResponse"]; + }; + }; + }; + }; + searchRulePreview: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponsePreview"]; + }; + }; + }; + }; + summary_2: { + parameters: { + query: { + from: string; + until: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDeepDiscoverySummary"]; + }; + }; + }; + }; + repositories_1: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + q?: string; + status?: "PENDING" | "VALIDATING" | "VALID" | "NO_VALID_SKILL" | "VALIDATION_FAILED" | "UNREACHABLE"; + channel?: "REPOSITORY_METADATA" | "README" | "OWNER_EXPANSION"; + paused?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseDiscoveryPageGithubDeepDiscoveryRepositorySummary"]; + }; + }; + }; + }; + repository_1: { + parameters: { + query?: never; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDeepDiscoveryRepositoryDetail"]; + }; + }; + }; + }; + provenance: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + }; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseDiscoveryPageGithubDeepDiscoveryProvenance"]; + }; + }; + }; + }; + paths_1: { + parameters: { + query?: { + page?: number; + size?: number; + asOf?: string; + }; + header?: never; + path: { + repositoryId: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseDiscoveryPageGithubDeepDiscoveryPathResult"]; + }; + }; + }; + }; + progress: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "*/*": components["schemas"]["ApiResponseGithubDeepDiscoveryProgress"]; + }; + }; + }; + }; listAuditLogs: { parameters: { query?: { diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 7218968ce..7b95767b0 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -313,6 +313,7 @@ export interface SkillLifecycleVersion { id: number version: string status: string + syncRevision?: number | null } export interface ComplianceEvidence { @@ -351,6 +352,15 @@ export type SkillSuiteReference = Omit< } type GeneratedSkillDetail = components['schemas']['SkillDetailResponse'] +export interface GithubSkillSource { + ownerLogin: string + repositoryName: string + skillPath: string + commitSha: string + syncRevision: number + licenseStatus: string + licenseDocuments: Array<{ scope: string; path: string; bundled: boolean }> +} export type SkillDetail = Omit< RequiredGenerated< GeneratedSkillDetail, @@ -369,7 +379,7 @@ export type SkillDetail = Omit< | 'entryForSuites' | 'memberOfSuites' >, - 'avatarUrl' | 'labels' | 'headlineVersion' | 'publishedVersion' | 'ownerPreviewVersion' | 'entryForSuites' | 'memberOfSuites' + 'avatarUrl' | 'labels' | 'headlineVersion' | 'publishedVersion' | 'ownerPreviewVersion' | 'entryForSuites' | 'memberOfSuites' | 'githubSource' > & { avatarUrl?: string | null labels?: LabelItem[] @@ -378,6 +388,7 @@ export type SkillDetail = Omit< ownerPreviewVersion?: SkillLifecycleVersion entryForSuites?: SkillSuiteReference[] memberOfSuites?: PagedResponse + githubSource?: GithubSkillSource | null } export interface SubmitPromotionRequest { @@ -413,6 +424,7 @@ export interface PromotionRevocation { export interface SkillVersion { id: number version: string + syncRevision?: number | null status: string changelog?: string fileCount: number @@ -611,7 +623,7 @@ export interface ReviewTask { namespace: string skillSlug?: string | null version: string | null - status: 'PENDING' | 'APPROVED' | 'REJECTED' + status: 'PENDING' | 'APPROVED' | 'REJECTED' | 'SUPERSEDED' submittedBy: string submittedByName?: string reviewedBy?: string @@ -627,6 +639,55 @@ export interface ReviewTask { previousVisibility?: string requestedVisibility?: string avatarRemoval?: boolean + originType?: 'USER' | 'GITHUB_SYNC' +} + +export interface GithubSyncReviewSource { + ingestionId: string + runId: string + sourceId?: number | null + repositoryId: number + ownerLogin: string + repositoryName: string + commitSha: string + skillPath: string + contentSha256: string + syncRevision: number + ingestionStatus: string + receivedAt: string + baseVersionId?: number | null + fileChanges: Array<{ + path: string + changeType: 'ADDED' | 'MODIFIED' | 'REMOVED' + oldSize?: number | null + newSize?: number | null + oldSha256?: string | null + newSha256?: string | null + }> + evidence: { + status: string + documents: Array<{ + scope: string + path: string + gitSha?: string | null + sha256?: string | null + size: number + bundled: boolean + text?: string | null + }> + } +} + +export type GithubSyncReviewQueueItem = Omit & { + review: ReviewTask + ingestionId: string + runId: string + repositoryId: number + ownerLogin?: string | null + repositoryName?: string | null + skillPath: string + commitSha: string } export interface ReviewProgress extends Pick ( (!item.admin || governanceVisible) + && (!item.superAdmin || user?.platformRoles?.includes('SUPER_ADMIN')) && (!item.passwordCapability || user?.canChangePassword === true) )), })) @@ -87,6 +88,7 @@ export function Layout() { { label: t('nav.landing'), to: '/', exact: true }, { label: t('nav.publish'), to: '/dashboard/publish', auth: true }, { label: t('nav.marketplace'), to: '/search' }, + { label: t('nav.githubSkills'), to: '/github' }, { label: t('nav.suites'), to: '/suites' }, { label: t('nav.dashboard'), to: '/dashboard', auth: true }, ] diff --git a/web/src/app/router.tsx b/web/src/app/router.tsx index 12ff00754..d3b216493 100644 --- a/web/src/app/router.tsx +++ b/web/src/app/router.tsx @@ -1,7 +1,7 @@ import { lazy, Suspense, type ComponentType } from 'react' import { createRouter, createRoute, createRootRoute, redirect } from '@tanstack/react-router' import { Layout } from './layout' -import { getCurrentUser } from '@/api/client' +import { getCurrentUser, type GithubSyncRepositoryQuery, type GithubSyncRunQuery } from '@/api/client' import { RoleGuard } from '@/shared/components/role-guard' import { RouteError } from '@/shared/components/route-error' import { createRedirectAuthenticated, createRequireAuth, isSafeAuthReturnTo } from '@/shared/lib/auth-route' @@ -34,6 +34,10 @@ function SilentRouteFallback() { return
} +function allowedSearchValue(value: unknown, options: readonly T[]): T | undefined { + return typeof value === 'string' && options.some((option) => option === value) ? value as T : undefined +} + interface LazyRouteOptions { silentFallback?: boolean } @@ -93,6 +97,7 @@ const RegisterPage = createLazyRouteComponent(() => import('@/pages/register'), const ResetPasswordPage = createLazyRouteComponent(() => import('@/pages/reset-password'), 'ResetPasswordPage') const PrivacyPolicyPage = createLazyRouteComponent(() => import('@/pages/privacy'), 'PrivacyPolicyPage') const SearchPage = createLazyRouteComponent(() => import('@/pages/search'), 'SearchPage') +const GithubSkillsPage = createLazyRouteComponent(() => import('@/pages/github-skills'), 'GithubSkillsPage') const SuitesPage = createLazyRouteComponent(() => import('@/pages/suites'), 'SuitesPage') const SuiteDetailPage = createLazyRouteComponent(() => import('@/pages/suite-detail'), 'SuiteDetailPage') const TermsOfServicePage = createLazyRouteComponent(() => import('@/pages/terms'), 'TermsOfServicePage') @@ -153,12 +158,18 @@ const NamespaceReviewsPage = createLazyRouteComponent( 'NamespaceReviewsPage', dashboardRouteOptions, ) +const NamespaceGithubSourcesPage = createLazyRouteComponent( + () => import('@/pages/dashboard/namespace-github-sources'), 'NamespaceGithubSourcesPage', dashboardRouteOptions, +) const NamespaceReviewDetailPage = createLazyRouteComponent( () => import('@/pages/dashboard/review-detail'), 'NamespaceReviewDetailPage', dashboardRouteOptions, ) const GovernancePage = createLazyRouteComponent(() => import('@/pages/dashboard/governance'), 'GovernancePage', dashboardRouteOptions) +const GithubSyncPage = createRoleProtectedRouteComponent( + () => import('@/pages/dashboard/github-sync'), 'GithubSyncPage', ['SUPER_ADMIN'], dashboardRouteOptions, +) const ReviewsPage = createLazyRouteComponent(() => import('@/pages/dashboard/reviews'), 'ReviewsPage', dashboardRouteOptions) const ReviewProgressPage = createLazyRouteComponent( () => import('@/pages/dashboard/review-progress'), @@ -302,6 +313,17 @@ const searchRoute = createRoute({ }, }) +const githubSkillsRoute = createRoute({ + getParentRoute: () => rootRoute, + path: 'github', + component: GithubSkillsPage, + validateSearch: (search: Record): { q: string; sort: string; page: number } => ({ + q: normalizeSearchQuery(typeof search.q === 'string' ? search.q : ''), + sort: search.sort === 'downloads' ? 'downloads' : 'newest', + page: Math.max(0, Number(search.page) || 0), + }), +}) + const suitesRoute = createRoute({ getParentRoute: () => rootRoute, path: 'suites', @@ -508,9 +530,19 @@ const dashboardNamespaceReviewsRoute = createRoute({ getParentRoute: () => rootRoute, path: 'dashboard/namespaces/$slug/reviews', beforeLoad: requireAuth, + validateSearch: (search: Record): { queue?: 'user' | 'sync' } => ({ + queue: search.queue === 'sync' ? 'sync' : undefined, + }), component: NamespaceReviewsPage, }) +const dashboardNamespaceGithubSourcesRoute = createRoute({ + getParentRoute: () => rootRoute, + path: 'dashboard/namespaces/$slug/github-sources', + beforeLoad: requireAuth, + component: NamespaceGithubSourcesPage, +}) + const dashboardGovernanceRoute = createRoute({ getParentRoute: () => rootRoute, path: 'dashboard/governance', @@ -518,12 +550,54 @@ const dashboardGovernanceRoute = createRoute({ component: GovernancePage, }) +const dashboardGithubSyncRoute = createRoute({ + getParentRoute: () => rootRoute, + path: 'dashboard/github-sync', + beforeLoad: requireAuth, + validateSearch: (search: Record): { + runId?: string; sourceId?: number; repositoryId?: number; tab?: 'overview' | 'repositories' | 'runs'; + drawer?: 'collection' | 'settings'; page?: number; runPage?: number; q?: string; + originType?: GithubSyncRepositoryQuery['originType']; state?: GithubSyncRepositoryQuery['state']; + sourceStatus?: GithubSyncRepositoryQuery['sourceStatus']; runKind?: GithubSyncRunQuery['kind']; + runStatus?: GithubSyncRunQuery['status']; asOf?: string; + } => ({ + runId: typeof search.runId === 'string' && /^[0-9a-f-]{36}$/i.test(search.runId) ? search.runId : undefined, + sourceId: (typeof search.sourceId === 'number' || typeof search.sourceId === 'string') + && Number.isSafeInteger(Number(search.sourceId)) && Number(search.sourceId) > 0 + ? Number(search.sourceId) : undefined, + repositoryId: (typeof search.repositoryId === 'number' || typeof search.repositoryId === 'string') + && Number.isSafeInteger(Number(search.repositoryId)) && Number(search.repositoryId) > 0 + ? Number(search.repositoryId) : undefined, + tab: allowedSearchValue(search.tab, ['overview', 'repositories', 'runs'] as const), + drawer: allowedSearchValue(search.drawer, ['collection', 'settings'] as const), + page: Number.isSafeInteger(Number(search.page)) && Number(search.page) >= 0 ? Number(search.page) : undefined, + runPage: Number.isSafeInteger(Number(search.runPage)) && Number(search.runPage) >= 0 ? Number(search.runPage) : undefined, + q: typeof search.q === 'string' ? search.q.slice(0, 120) : undefined, + originType: allowedSearchValue(search.originType, ['CURATED_REPOSITORY', 'CURATED_OWNER', 'SEARCH_RULE', 'OFFICIAL_DISCOVERY', 'TEAM_CONFIG'] as const), + state: allowedSearchValue(search.state, ['REGISTERED', 'PENDING', 'VALIDATING', 'NO_VALID_SKILL', 'VALIDATION_FAILED', 'UNREACHABLE', 'BOUND_WITHOUT_SOURCE'] as const), + sourceStatus: allowedSearchValue(search.sourceStatus, ['ACTIVE', 'PAUSED', 'UNREACHABLE'] as const), + runKind: allowedSearchValue(search.runKind, ['DISCOVERY', 'REPOSITORY_SYNC'] as const), + runStatus: allowedSearchValue(search.runStatus, ['QUEUED', 'LEASED', 'RETRY_WAIT', 'PARTIAL', 'ACQUISITION_COMPLETE', 'FAILED'] as const), + asOf: typeof search.asOf === 'string' && !Number.isNaN(Date.parse(search.asOf)) ? search.asOf : undefined, + }), + component: GithubSyncPage, +}) + +const dashboardGithubDeepDiscoveryRoute = createRoute({ + getParentRoute: () => rootRoute, + path: 'dashboard/github-deep-discovery', + beforeLoad: async (ctx) => { + await requireAuth(ctx) + throw redirect({ to: '/dashboard/github-sync', search: { drawer: 'collection' } }) + }, +}) + const dashboardReviewsRoute = createRoute({ getParentRoute: () => rootRoute, path: 'dashboard/reviews', beforeLoad: requireAuth, - validateSearch: (search: Record): { type?: 'skill' | 'profile' } => ({ - type: search.type === 'skill' || search.type === 'profile' ? search.type : undefined, + validateSearch: (search: Record): { type?: 'skill' | 'profile' | 'sync' } => ({ + type: search.type === 'skill' || search.type === 'profile' || search.type === 'sync' ? search.type : undefined, }), component: ReviewsPage, }) @@ -706,6 +780,7 @@ const routeTree = rootRoute.addChildren([ resetPasswordRoute, privacyRoute, searchRoute, + githubSkillsRoute, suitesRoute, suiteDetailRoute, termsRoute, @@ -727,8 +802,11 @@ const routeTree = rootRoute.addChildren([ dashboardNamespaceSettingsRoute, dashboardNamespaceMembersRoute, dashboardNamespaceReviewsRoute, + dashboardNamespaceGithubSourcesRoute, dashboardNamespaceReviewDetailRoute, dashboardGovernanceRoute, + dashboardGithubSyncRoute, + dashboardGithubDeepDiscoveryRoute, dashboardReviewsRoute, dashboardReviewProgressRoute, dashboardReportsRoute, diff --git a/web/src/features/github-sync/github-admission-config.tsx b/web/src/features/github-sync/github-admission-config.tsx new file mode 100644 index 000000000..aa26616a1 --- /dev/null +++ b/web/src/features/github-sync/github-admission-config.tsx @@ -0,0 +1,75 @@ +import { useState, type FormEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { GithubSyncAdmissionPolicy, GithubSyncAdmissionPolicyInput } from '@/api/client' +import { useUpdateGithubSyncAdmissionPolicy } from './use-github-sync-admin' +import { toast } from '@/shared/lib/toast' +import { Card } from '@/shared/ui/card' +import { Input } from '@/shared/ui/input' +import { Button } from '@/shared/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/shared/ui/dialog' + +type Field = keyof GithubSyncAdmissionPolicyInput +const fields: Field[] = [ + 'globalHigh', 'globalLow', 'namespaceHigh', 'namespaceLow', + 'candidatePendingLimit', 'candidatePerLeaseLimit', +] + +export function GithubAdmissionConfig({ policy }: { policy: GithubSyncAdmissionPolicy }) { + const { t } = useTranslation() + const mutation = useUpdateGithubSyncAdmissionPolicy() + const [open, setOpen] = useState(false) + const [values, setValues] = useState>(() => Object.fromEntries( + fields.map((field) => [field, policy[field]?.toString() ?? '']), + ) as Record) + + const openEditor = () => { + setValues(Object.fromEntries(fields.map((field) => [field, policy[field]?.toString() ?? ''])) as Record) + setOpen(true) + } + + const save = async (event: FormEvent) => { + event.preventDefault() + const body = Object.fromEntries(fields.map((field) => [field, Number(values[field])])) as GithubSyncAdmissionPolicyInput + if (fields.some((field) => !Number.isInteger(body[field])) + || body.globalLow < 0 || body.globalLow >= body.globalHigh + || body.namespaceLow < 0 || body.namespaceLow >= body.namespaceHigh + || body.candidatePerLeaseLimit < 1 || body.candidatePerLeaseLimit > body.candidatePendingLimit) { + toast.error(t('githubSync.invalidWatermarks')) + return + } + try { + await mutation.mutateAsync(body) + setOpen(false) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + return +
+

{t('githubSync.capacityTitle')}

{t('githubSync.capacityHint')}

+ +
+
+
{t('githubSync.platformWatermark')}

{policy.globalLow ?? '—'} / {policy.globalHigh ?? '—'}

+
{t('githubSync.namespaceWatermark')}

{policy.namespaceLow ?? '—'} / {policy.namespaceHigh ?? '—'}

+
{t('githubSync.globalUnresolved')}

{policy.globalUnresolvedCount ?? '—'}

+
{t('githubSync.platformPendingCandidates')}

{policy.platformPendingCandidateCount ?? '—'} / {policy.candidatePendingLimit}

+
+ + + {t('githubSync.capacityTitle')}{t('githubSync.capacityConfigHelp')} +
+
+ {fields.map((field) => )} +
+
+
+ + + +} diff --git a/web/src/features/github-sync/github-credential-config.tsx b/web/src/features/github-sync/github-credential-config.tsx new file mode 100644 index 000000000..784602fbd --- /dev/null +++ b/web/src/features/github-sync/github-credential-config.tsx @@ -0,0 +1,129 @@ +import { useState, type FormEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { useGithubCredentialMutations, useGithubCredentials } from './use-github-sync-admin' +import { githubSyncStatusLabel } from './status-label' +import { formatLocalDateTime } from '@/shared/lib/date-time' +import { toast } from '@/shared/lib/toast' +import { Card } from '@/shared/ui/card' +import { Input } from '@/shared/ui/input' +import { Button } from '@/shared/ui/button' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/shared/ui/table' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/shared/ui/dialog' + +export function GithubCredentialConfig() { + const { t, i18n } = useTranslation() + const credentials = useGithubCredentials() + const mutations = useGithubCredentialMutations() + const [open, setOpen] = useState(false) + const [editingId, setEditingId] = useState(null) + const [label, setLabel] = useState('') + const [token, setToken] = useState('') + + const reset = () => { + setEditingId(null) + setLabel('') + setToken('') + } + + const save = async (event: FormEvent) => { + event.preventDefault() + try { + if (editingId) { + await mutations.replace.mutateAsync({ id: editingId, label: label.trim(), token }) + } else { + await mutations.create.mutateAsync({ label: label.trim(), token }) + } + reset() + setOpen(false) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + const setEnabled = async (id: string, enabled: boolean) => { + try { + await mutations.setEnabled.mutateAsync({ id, enabled }) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + const openCreate = () => { + reset() + setOpen(true) + } + + const openReplace = (id: string, currentLabel: string) => { + setEditingId(id) + setLabel(currentLabel) + setToken('') + setOpen(true) + } + + return ( +
+
+

{t('githubSync.credentials')}

{t('githubSync.credentialHelp')}

+ +
+ { setOpen(nextOpen); if (!nextOpen) reset() }}> + + {editingId ? t('githubSync.replaceCredential') : t('githubSync.addCredential')}{t('githubSync.credentialHelp')} +
+ + +
+
+ + + + {credentials.isLoading ?

{t('githubSync.loading')}

: null} + {credentials.error ?

{t('githubSync.loadFailed')}: {credentials.error.message}

: null} + {credentials.data?.length === 0 ?

{t('githubSync.noCredentials')}

: null} + {credentials.data?.length ? ( + + + {t('githubSync.credentialLabel')} + {t('githubSync.fingerprint')} + {t('githubSync.sourceState')} + {t('githubSync.keyVersion')} + {t('githubSync.rateLimitedUntil')} + {t('githubSync.observedQuota')} + {t('githubSync.lastUsed')} + {t('githubSync.details')} + + {credentials.data.map((credential) => ( + + {credential.label} + {credential.fingerprint ?? '—'} + {githubSyncStatusLabel(t, credential.status)} + {credential.keyVersion ?? '—'}{credential.keyAvailable === false ? {t('githubSync.keyUnavailable')} : null} + {formatLocalDateTime(credential.rateLimitedUntil, i18n.language)} + {credential.quotas?.length ? credential.quotas.map((quota) => ( +
{quota.resource}: {quota.remaining ?? '—'} · {formatLocalDateTime(quota.resetAt, i18n.language)}
+ )) : '—'} + {formatLocalDateTime(credential.lastUsedAt, i18n.language)} + + + + + + ))} +
+ ) : null} + +
+ ) +} diff --git a/web/src/features/github-sync/github-deep-discovery-config.tsx b/web/src/features/github-sync/github-deep-discovery-config.tsx new file mode 100644 index 000000000..57eaa4616 --- /dev/null +++ b/web/src/features/github-sync/github-deep-discovery-config.tsx @@ -0,0 +1,114 @@ +import { useState, type FormEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { useQueryClient } from '@tanstack/react-query' +import { Plus, Trash2 } from 'lucide-react' +import type { GithubDeepDiscoveryConfiguration, GithubDeepDiscoveryConfigurationInput } from '@/api/client' +import { ApiError } from '@/shared/lib/api-error' +import { toast } from '@/shared/lib/toast' +import { Button } from '@/shared/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/shared/ui/dialog' +import { Input } from '@/shared/ui/input' +import { DEEP_DISCOVERY_QUERY_KEY, useUpdateGithubDeepDiscoveryConfiguration } from './use-github-deep-discovery' + +type Exclusion = GithubDeepDiscoveryConfigurationInput['exclusions'][number] +type Budget = GithubDeepDiscoveryConfigurationInput['budgetProfile'] + +export function GithubDeepDiscoveryConfig({ configuration, onClose, embedded = false }: { + configuration: GithubDeepDiscoveryConfiguration + onClose: () => void + embedded?: boolean +}) { + const { t } = useTranslation() + const queryClient = useQueryClient() + const update = useUpdateGithubDeepDiscoveryConfiguration() + const [enabled, setEnabled] = useState(configuration.enabled ?? false) + const [budgetProfile, setBudgetProfile] = useState(configuration.budgetProfile ?? 'BALANCED') + const [exclusions, setExclusions] = useState(() => + (configuration.exclusions ?? []).map((item) => ({ kind: item.kind ?? 'REPOSITORY', value: item.value ?? '' }))) + const [kind, setKind] = useState('REPOSITORY') + const [value, setValue] = useState('') + const [conflict, setConflict] = useState(false) + + const addExclusion = (event: FormEvent) => { + event.preventDefault() + const normalized = value.trim().toLowerCase() + if (!normalized || exclusions.some((item) => item.kind === kind && item.value.toLowerCase() === normalized)) return + setExclusions((current) => [...current, { kind, value: normalized }]) + setValue('') + } + + const save = async () => { + setConflict(false) + try { + await update.mutateAsync({ + expectedVersion: configuration.version!, enabled, budgetProfile, exclusions, + }) + toast.success(t('deepDiscovery.configSaved')) + onClose() + } catch (error) { + if (error instanceof ApiError && error.status === 409) { + setConflict(true) + } else { + toast.error(t('deepDiscovery.configSaveFailed'), error instanceof Error ? error.message : '') + } + } + } + + const content = <> + + {t('deepDiscovery.configure')} + {t('deepDiscovery.configDescription')} + +
+ +
+ {t('deepDiscovery.budget')} +
+ {(['ECONOMY', 'BALANCED', 'ACCELERATED'] as const).map((profile) => )} +
+

{t('deepDiscovery.budgetScope')}

+
+
+

{t('deepDiscovery.exclusions')}

+

{t('deepDiscovery.exclusionsHelp')}

+
+ + setValue(event.target.value)} /> + +
+
+ {exclusions.map((item, index) =>
+ {t(item.kind === 'OWNER' ? 'deepDiscovery.exclusionOwner' : 'deepDiscovery.exclusionRepository')}: {item.value} + +
)} + {exclusions.length === 0 ?

{t('deepDiscovery.noExclusions')}

: null} +
+
+
+ {conflict ?
{t('deepDiscovery.configConflict')}
: null} + + + + + + + if (embedded) return
{content}
+ return { if (!open) onClose() }}> + {content} + +} diff --git a/web/src/features/github-sync/github-deep-discovery-pagination.test.tsx b/web/src/features/github-sync/github-deep-discovery-pagination.test.tsx new file mode 100644 index 000000000..d30365d35 --- /dev/null +++ b/web/src/features/github-sync/github-deep-discovery-pagination.test.tsx @@ -0,0 +1,26 @@ +import { createElement } from 'react' +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it, vi } from 'vitest' + +vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string, options?: { count?: number }) => `${key}:${options?.count ?? ''}` }) })) +vi.mock('@/shared/components/pagination', () => ({ + Pagination: ({ totalPages }: { totalPages: number }) => createElement('span', { 'data-pages': totalPages }), +})) + +import { deepDiscoveryPageCount, GithubDeepDiscoveryPagination } from './github-deep-discovery-pagination' + +describe('deep discovery pagination boundary', () => { + it('never offers a page rejected by page <= 1000 or offset <= 100000', () => { + expect(deepDiscoveryPageCount(20_021, 20)).toBe(1001) + expect(deepDiscoveryPageCount(200_000, 100)).toBe(1001) + expect(deepDiscoveryPageCount(100, 20)).toBe(5) + }) + + it('keeps the true result count out of the page limit and explains the boundary', () => { + const markup = renderToStaticMarkup(createElement(GithubDeepDiscoveryPagination, { + page: 0, total: 20_021, size: 20, onPageChange: () => {}, + })) + expect(markup).toContain('data-pages="1001"') + expect(markup).toContain('deepDiscovery.paginationLimit:20020') + }) +}) diff --git a/web/src/features/github-sync/github-deep-discovery-pagination.tsx b/web/src/features/github-sync/github-deep-discovery-pagination.tsx new file mode 100644 index 000000000..2edfa0f1a --- /dev/null +++ b/web/src/features/github-sync/github-deep-discovery-pagination.tsx @@ -0,0 +1,28 @@ +import { useTranslation } from 'react-i18next' +import { Pagination } from '@/shared/components/pagination' + +const MAX_PAGE = 1000 +const MAX_OFFSET = 100_000 + +export function deepDiscoveryPageCount(total: number, size: number) { + return Math.min(Math.ceil(total / size), MAX_PAGE + 1, Math.floor(MAX_OFFSET / size) + 1) +} + +export function GithubDeepDiscoveryPagination({ page, total, size, onPageChange, detail = false }: { + page: number + total: number + size: number + onPageChange: (page: number) => void + detail?: boolean +}) { + const { t } = useTranslation() + const pages = deepDiscoveryPageCount(total, size) + const limited = total > pages * size + + return
+ {pages > 1 ? : null} + {limited ?

+ {t(detail ? 'deepDiscovery.detailPaginationLimit' : 'deepDiscovery.paginationLimit', { count: pages * size })} +

: null} +
+} diff --git a/web/src/features/github-sync/github-deep-discovery-repository-drawer.tsx b/web/src/features/github-sync/github-deep-discovery-repository-drawer.tsx new file mode 100644 index 000000000..704ea5c53 --- /dev/null +++ b/web/src/features/github-sync/github-deep-discovery-repository-drawer.tsx @@ -0,0 +1,99 @@ +import { useState } from 'react' +import { Link } from '@tanstack/react-router' +import { useTranslation } from 'react-i18next' +import { ExternalLink } from 'lucide-react' +import { GithubDeepDiscoveryPagination } from './github-deep-discovery-pagination' +import { formatLocalDateTime } from '@/shared/lib/date-time' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/shared/ui/dialog' +import { useGithubDeepDiscoveryPaths, useGithubDeepDiscoveryProvenance, useGithubDeepDiscoveryRepository } from './use-github-deep-discovery' + +const PAGE_SIZE = 20 + +export function GithubDeepDiscoveryRepositoryDrawer({ repositoryId, onClose, visible }: { + repositoryId: number + onClose: () => void + visible: boolean +}) { + const { t, i18n } = useTranslation() + const [provenancePage, setProvenancePage] = useState(0) + const [provenanceAsOf, setProvenanceAsOf] = useState() + const [pathPage, setPathPage] = useState(0) + const [pathAsOf, setPathAsOf] = useState() + const detail = useGithubDeepDiscoveryRepository(repositoryId, visible) + const provenance = useGithubDeepDiscoveryProvenance(repositoryId, provenancePage, provenanceAsOf, visible) + const paths = useGithubDeepDiscoveryPaths(repositoryId, pathPage, pathAsOf, visible) + const repository = detail.data?.repository + const date = (value?: string | null) => formatLocalDateTime(value, i18n.language) + const separator = i18n.language.startsWith('zh') ? '、' : ', ' + + return { if (!open) onClose() }}> + + + {t('deepDiscovery.repositoryDetail')} + {t('deepDiscovery.detailHelp')} + +
+ {detail.isLoading ?

{t('deepDiscovery.loading')}

: null} + {detail.error ?

{t('deepDiscovery.loadFailed')}: {detail.error.message}

: null} + {repository ? <> +
+ {repository.owner}/{repository.name} +
+ {t(`deepDiscovery.status.${repository.status}`, { defaultValue: repository.status ?? '—' })} + {repository.paused ? {t('deepDiscovery.paused')} : null} + {repository.sourceId ? {t('deepDiscovery.registeredSource')} : null} +
+

{t('deepDiscovery.firstDiscovered')}: {date(repository.firstDiscoveredAt)} · {t('deepDiscovery.lastDiscovered')}: {date(repository.lastDiscoveredAt)}

+
+
+

{t('deepDiscovery.provenance')}

+ {provenance.isLoading ?

{t('deepDiscovery.loading')}

: null} + {provenance.error ?

{t('deepDiscovery.loadFailed')}: {provenance.error.message}

: null} + {provenance.data?.items?.length ?
{provenance.data.items.map((hit) =>
+ {t(`deepDiscovery.channel.${hit.channel}`, { defaultValue: hit.channel ?? '—' })} + {date(hit.lastSeenAt)} + {hit.seedRepositoryId ? {t('deepDiscovery.seedRepository')}: #{hit.seedRepositoryId} : null} +
)}
: !provenance.isLoading && !provenance.error ?

{t('deepDiscovery.noProvenance')}

: null} + {provenance.data ? { setProvenanceAsOf(provenanceAsOf ?? provenance.data?.asOf); setProvenancePage(next) }} /> : null} +
+
+

{t('deepDiscovery.checkResult')}

+
+
{t('deepDiscovery.lastChecked')}
{date(detail.data?.lastCheckedAt)}
+
{t('deepDiscovery.nextEligible')}
{date(detail.data?.nextEligibleAt)}
+
{t('deepDiscovery.processingStatus')}
{t(`deepDiscovery.reason.${detail.data?.notImportedReason}`, { defaultValue: detail.data?.notImportedReason ?? '—' })}
+
+ {detail.data?.waits?.map((wait, index) =>
+

{t(`deepDiscovery.reason.${wait.reason}`, { defaultValue: wait.reason ?? '—' })}

+

{t('deepDiscovery.affectedWork')}: {(wait.affectedWork ?? []).map((work) => t(`deepDiscovery.work.${work}`, { defaultValue: work })).join(separator)}

+ {wait.retryAt ?

{t('deepDiscovery.retryAt')}: {date(wait.retryAt)}

: null} +

{i18n.language.startsWith('zh') && wait.recoveryCondition ? wait.recoveryCondition : t(`deepDiscovery.recovery.${wait.reason}`, { defaultValue: t('deepDiscovery.autoResume') })}

+
)} +
+
+

{t('deepDiscovery.pathResults')} {paths.data ? `(${paths.data.total ?? 0})` : ''}

+ {paths.isLoading ?

{t('deepDiscovery.loading')}

: null} + {paths.error ?

{t('deepDiscovery.loadFailed')}: {paths.error.message}

: null} + {paths.data?.items?.length ?
{paths.data.items.map((path, index) =>
+

{path.skillPath}

+

{t(`deepDiscovery.pathOutcome.${path.outcome}`, { defaultValue: path.outcome ?? '—' })}{path.reasonCode ? ` · ${t(`githubSync.codes.${path.reasonCode}`, { defaultValue: path.reasonCode })}` : ''}

+ {path.reviewId ? {t('deepDiscovery.viewReview')} : null} +
)}
: !paths.isLoading && !paths.error ?

{t('deepDiscovery.noPaths')}

: null} + {paths.data ? { setPathAsOf(pathAsOf ?? paths.data?.asOf); setPathPage(next) }} /> : null} +
+
+ {repository.sourceId ? {t('deepDiscovery.viewSource')} : null} + {detail.data?.latestRunId ? {t('deepDiscovery.viewRun')} : null} + {t('deepDiscovery.viewReviewQueue')} +
+ : null} +
+ + +} diff --git a/web/src/features/github-sync/github-discovery-config.tsx b/web/src/features/github-sync/github-discovery-config.tsx new file mode 100644 index 000000000..aa67daca4 --- /dev/null +++ b/web/src/features/github-sync/github-discovery-config.tsx @@ -0,0 +1,305 @@ +import { useState, type FormEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { GithubDiscoveryEntry, GithubSearchRuleInput } from '@/api/client' +import { useGithubDiscoveryEntries, useGithubDiscoveryMutations, useGithubSearchPreview, useGithubSyncSchedulePolicy, useGithubSyncWorkerStatus } from './use-github-sync-admin' +import { githubSyncAdminApi } from '@/api/client' +import { toast } from '@/shared/lib/toast' +import { formatLocalDateTime } from '@/shared/lib/date-time' +import { Card } from '@/shared/ui/card' +import { Input } from '@/shared/ui/input' +import { Button } from '@/shared/ui/button' +import { githubSyncStatusLabel } from './status-label' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/shared/ui/table' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/shared/ui/dialog' + +type Selector = { kind: 'KEYWORD' | 'TOPIC'; value: string; priority: 'CORE' | 'EXTENSION' } +const initialSelector: Selector = { kind: 'KEYWORD', value: '', priority: 'CORE' } +type DiscoveryMode = 'repository' | 'owner' | 'rule' + +export function GithubDiscoveryConfig({ layout = 'default' }: { layout?: 'default' | 'other' }) { + const { t, i18n } = useTranslation() + const entries = useGithubDiscoveryEntries() + const schedule = useGithubSyncSchedulePolicy() + const workerStatus = useGithubSyncWorkerStatus() + const mutations = useGithubDiscoveryMutations() + const [repository, setRepository] = useState('') + const [mode, setMode] = useState(null) + const [owner, setOwner] = useState('') + const [ownerHours, setOwnerHours] = useState('') + const [ruleTitle, setRuleTitle] = useState('') + const [editingRuleId, setEditingRuleId] = useState(null) + const [ruleHours, setRuleHours] = useState('') + const [selectors, setSelectors] = useState([initialSelector]) + const [previewId, setPreviewId] = useState(null) + const [previewSnapshot, setPreviewSnapshot] = useState(null) + const [previewPending, setPreviewPending] = useState(false) + const shownEntries = layout === 'other' ? entries.data?.filter((entry) => entry.entryType !== 'CURATED_REPOSITORY') : entries.data + + const ruleBody = (): GithubSearchRuleInput => ({ + title: ruleTitle.trim(), targetHours: ruleHours === '' ? undefined : ruleHours, + selectors: selectors.map(({ kind, value, priority }) => ({ kind, value: value.trim(), priority })), + }) + const previewMatchesForm = previewId !== null && previewSnapshot === JSON.stringify(ruleBody()) + const preview = useGithubSearchPreview(previewMatchesForm ? previewId : null) + + const previewRule = async () => { + if (workerStatus.data?.online !== true) { + toast.error(t('githubSync.previewWorkerOffline')) + return + } + const body = ruleBody() + setPreviewPending(true) + setPreviewId(null) + try { + const queued = await githubSyncAdminApi.previewSearchRule(body) + setPreviewSnapshot(JSON.stringify(body)) + setPreviewId(queued.id) + } catch (error) { + toast.error(t('githubSync.previewFailed'), error instanceof Error ? error.message : '') + } finally { + setPreviewPending(false) + } + } + + const addRepository = async (event: FormEvent) => { + event.preventDefault() + const parts = repository.trim().replace(/^https:\/\/github\.com\//i, '').replace(/\.git\/?$/i, '').replace(/\/$/, '').split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + toast.error(t('githubSync.repositoryFormat')) + return + } + try { + await mutations.addRepository.mutateAsync({ ownerLogin: parts[0], repositoryName: parts[1] }) + setRepository('') + setMode(null) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + const addOwner = async (event: FormEvent) => { + event.preventDefault() + try { + await mutations.addOwner.mutateAsync({ ownerLogin: owner.trim(), + targetHours: ownerHours === '' ? undefined : ownerHours }) + setOwner('') + setOwnerHours('') + setMode(null) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + const clearRuleForm = () => { + setPreviewId(null) + setPreviewSnapshot(null) + setEditingRuleId(null) + setRuleTitle('') + setRuleHours('') + setSelectors([initialSelector]) + } + + const openForm = (nextMode: DiscoveryMode) => { + setRepository('') + setOwner('') + setOwnerHours('') + clearRuleForm() + setMode(nextMode) + } + + const closeForm = () => { + setMode(null) + setRepository('') + setOwner('') + setOwnerHours('') + clearRuleForm() + } + + const editRule = (entry: GithubDiscoveryEntry) => { + if (!entry.id || entry.entryType !== 'SEARCH_RULE') return + setEditingRuleId(entry.id) + setPreviewId(null) + setPreviewSnapshot(null) + setRuleTitle(entry.title ?? '') + setRuleHours(entry.targetHours ?? '') + setSelectors(entry.selectors?.map((selector) => ({ + kind: selector.kind as Selector['kind'], + value: selector.value ?? '', + priority: selector.priority as Selector['priority'], + })) ?? [initialSelector]) + setMode('rule') + } + + const saveRule = async (event: FormEvent) => { + event.preventDefault() + const body = ruleBody() + try { + if (editingRuleId) { + await mutations.updateSearchRule.mutateAsync({ id: editingRuleId, body }) + toast.success(t('githubSync.saved')) + } else { + await mutations.addSearchRule.mutateAsync(body) + toast.success(t('githubSync.ruleSavedDisabled')) + } + clearRuleForm() + setMode(null) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + const setEnabled = async (id: string, enabled: boolean) => { + try { + await mutations.setEnabled.mutateAsync({ id, enabled }) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + return ( +
+
+
+

{t(layout === 'other' ? 'githubSync.otherAcquisition' : 'githubSync.discoveryTitle')}

+

{t(layout === 'other' ? 'githubSync.otherAcquisitionHelp' : 'githubSync.discoveryIntro')}

+
+
+ {layout === 'default' ? : null} + + +
+
+ { if (!open) closeForm() }}> + + + {t(mode === 'repository' ? 'githubSync.addRepository' : mode === 'owner' ? 'githubSync.addOwner' : editingRuleId ? 'githubSync.editRule' : 'githubSync.addRule')} + {t(mode === 'repository' ? 'githubSync.repositoryHelp' : mode === 'owner' ? 'githubSync.ownerHelp' : 'githubSync.ruleHelp')} + + {mode === 'repository' ? ( +
+
+ setRepository(event.target.value)} required /> + +
+
+ ) : null} + {mode === 'owner' ? ( +
+
+ setOwner(event.target.value)} required /> + + +
+
+ ) : null} + {mode === 'rule' ? ( +
+
+
+ setRuleTitle(event.target.value)} required /> + + {selectors.map((selector, index) => ( +
+
+ + +
+
+ + {selectors.length > 1 ? : null} +
+
+ ))} + {selectors.length < 20 ? : null} +
+ + + +
+
+
+

{t('githubSync.rulePreviewTitle')}

+

{t('githubSync.previewCaveat')}

+ {previewId && !previewMatchesForm ?

{t('githubSync.previewStale')}

: null} + {workerStatus.isLoading ?

{t('githubSync.previewWorkerChecking')}

: null} + {workerStatus.error ?

{t('githubSync.previewWorkerUnknown')}

: null} + {!workerStatus.isLoading && !workerStatus.error && workerStatus.data?.online === false ?

{t('githubSync.previewWorkerOffline')}

: null} + {previewMatchesForm ?
+ {preview.isLoading ?

{t('githubSync.loading')}

: null} + {preview.error ?

{t('githubSync.previewFailed')}: {preview.error.message}

: null} + {preview.data?.status === 'QUEUED' && workerStatus.data?.online === true ?

{t('githubSync.previewQueued')}

: null} + {preview.data?.status === 'LEASED' ?

{t('githubSync.previewWaiting')}

: null} + {preview.data?.status === 'RETRY_WAIT' ?

{t('githubSync.previewDeferred')}{preview.data.retryAt ? ` · ${formatLocalDateTime(preview.data.retryAt, i18n.language)}` : ''}{preview.data.errorCode ? ` · ${preview.data.errorCode}` : ''}

: null} + {preview.data?.status === 'FAILED' ?

{t('githubSync.previewFailed')}{preview.data.errorCode ? ` · ${preview.data.errorCode}` : ''}

: null} + {preview.data?.status === 'ACQUISITION_COMPLETE' && !preview.data.result ?

{t('githubSync.previewMissingResult')}

: null} + {preview.data?.result?.selectors.map((selector, index) =>
+

{selector.kind}: {selector.value} · {t('githubSync.previewEstimate')}: {selector.totalCount}

+ {selector.limitReached || selector.incompleteResults ?

{t('githubSync.previewLimited')}

: null} +

{selector.samples.map((sample) => `${sample.owner}/${sample.name}`).join(' · ')}

+
)} +
: null} + {!previewId && workerStatus.data?.online === true ?

{t('githubSync.previewPrompt')}

: null} +
+
+
+ ) : null} + + + + +
+

{t('githubSync.existingEntries')}

+

{t('githubSync.existingEntriesHint')}

+
+ {entries.isLoading ?

{t('githubSync.loading')}

: null} + {entries.error ?

{t('githubSync.loadFailed')}: {entries.error.message}

: null} + {shownEntries?.length === 0 ?

{t('githubSync.empty')}

: null} + {shownEntries?.length ? ( + + + {t('githubSync.entryType')}{t('githubSync.entryName')} + {t('githubSync.ruleVersion')}{t('githubSync.nextCheck')} + {t('githubSync.sourceState')}{t('githubSync.details')} + + {shownEntries.map((entry) => ( + + {githubSyncStatusLabel(t, entry.entryType)} + {entry.title || [entry.ownerLogin, entry.repositoryName].filter(Boolean).join('/')} + {entry.ruleVersion ?? '—'} + {formatLocalDateTime(entry.nextDueAt, i18n.language)} + {entry.enabled ? t('githubSync.active') : t('githubSync.paused')} + + {entry.entryType === 'SEARCH_RULE' ? : null} + + + + ))} +
+ ) : null} + +
+ ) +} diff --git a/web/src/features/github-sync/github-retry-rejected-skill-button.tsx b/web/src/features/github-sync/github-retry-rejected-skill-button.tsx new file mode 100644 index 000000000..7dee16abf --- /dev/null +++ b/web/src/features/github-sync/github-retry-rejected-skill-button.tsx @@ -0,0 +1,36 @@ +import { useState } from 'react' +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { useTranslation } from 'react-i18next' +import { githubSyncAdminApi, githubTeamSourcesApi } from '@/api/client' +import type { GithubSyncSkillPath } from '@/api/client' +import { Button } from '@/shared/ui/button' + +export function GithubRetryRejectedSkillButton({ sourceId, path, namespaceSlug }: { + sourceId: number + path: GithubSyncSkillPath + namespaceSlug?: string +}) { + const { t } = useTranslation() + const queryClient = useQueryClient() + const [feedback, setFeedback] = useState('') + const skillPath = path.skillPath ?? '' + const retry = useMutation({ + mutationFn: () => namespaceSlug + ? githubTeamSourcesApi.retryRejectedSkill(namespaceSlug, sourceId, skillPath) + : githubSyncAdminApi.retryRejectedSkill(sourceId, skillPath), + onSuccess: () => { + setFeedback(t('githubSync.retryRejectedScheduled')) + void queryClient.invalidateQueries({ queryKey: ['github-sync'] }) + }, + onError: (error) => setFeedback(`${t('githubSync.retryRejectedFailed')}: ${error instanceof Error ? error.message : ''}`), + }) + + if (!skillPath || (path.latestBindingIngestionStatus !== 'REJECTED' && !path.retryRequested)) return null + return
+ + {feedback ?

{feedback}

: null} +
+} diff --git a/web/src/features/github-sync/github-schedule-config.tsx b/web/src/features/github-sync/github-schedule-config.tsx new file mode 100644 index 000000000..da3e8a85b --- /dev/null +++ b/web/src/features/github-sync/github-schedule-config.tsx @@ -0,0 +1,79 @@ +import { useState, type FormEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { GithubSyncSchedulePolicy, GithubSyncSchedulePolicyInput } from '@/api/client' +import { useUpdateGithubSyncSchedulePolicy } from './use-github-sync-admin' +import { toast } from '@/shared/lib/toast' +import { Card } from '@/shared/ui/card' +import { Input } from '@/shared/ui/input' +import { Button } from '@/shared/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/shared/ui/dialog' + +const fields = ['repositoryHours', 'curatedOwnerHours', 'searchRuleHours'] as const +type Field = typeof fields[number] + +export function GithubScheduleConfig({ policy }: { policy: GithubSyncSchedulePolicy }) { + const { t } = useTranslation() + const mutation = useUpdateGithubSyncSchedulePolicy() + const [open, setOpen] = useState(false) + const [hours, setHours] = useState>({ + repositoryHours: policy.repositoryHours?.toString() ?? '', + curatedOwnerHours: policy.curatedOwnerHours?.toString() ?? '24', + searchRuleHours: policy.searchRuleHours?.toString() ?? '72', + }) + + const openEditor = () => { + setHours({ + repositoryHours: policy.repositoryHours?.toString() ?? '', + curatedOwnerHours: policy.curatedOwnerHours?.toString() ?? '24', + searchRuleHours: policy.searchRuleHours?.toString() ?? '72', + }) + setOpen(true) + } + + const save = async (event: FormEvent) => { + event.preventDefault() + const body = { + repositoryHours: Number(hours.repositoryHours), + curatedOwnerHours: Number(hours.curatedOwnerHours), + searchRuleHours: Number(hours.searchRuleHours), + } satisfies GithubSyncSchedulePolicyInput + if (Object.values(body).some((value) => !Number.isInteger(value) || value < 1 || value > 720)) { + toast.error(t('githubSync.scheduleInvalid')) + return + } + try { + await mutation.mutateAsync(body) + setOpen(false) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + } + + return +
+

{t('githubSync.scheduleTitle')}

{t('githubSync.scheduleHelp')}

+ +
+
+ {fields.map((field) =>
{t(`githubSync.${field}`)}

{policy[field] == null ? '—' : `${policy[field]} ${t('githubSync.hoursUnit')}`}

)} +
+ {policy.repositoryHours == null ?

{t('githubSync.scheduleLegacy', { interval: policy.effectiveRepositoryInterval })}

: null} + + + {t('githubSync.scheduleTitle')}{t('githubSync.scheduleHelp')} +
+
+ {fields.map((field) => )} +
+
+
+ + + +} diff --git a/web/src/features/github-sync/github-source-ignore-editor.tsx b/web/src/features/github-sync/github-source-ignore-editor.tsx new file mode 100644 index 000000000..717a9fbf2 --- /dev/null +++ b/web/src/features/github-sync/github-source-ignore-editor.tsx @@ -0,0 +1,68 @@ +import { useState, type FormEvent } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useTranslation } from 'react-i18next' +import { githubSyncAdminApi, githubTeamSourcesApi } from '@/api/client' +import { toast } from '@/shared/lib/toast' +import { Button } from '@/shared/ui/button' +import { Textarea } from '@/shared/ui/textarea' + +export function GithubSourceIgnoreEditor({ sourceId, namespaceSlug }: { + sourceId: number + namespaceSlug?: string +}) { + const { t } = useTranslation() + const queryClient = useQueryClient() + const queryKey = ['github-sync', namespaceSlug ?? 'platform', 'source-ignore', sourceId] + const patterns = useQuery({ + queryKey, + queryFn: () => namespaceSlug + ? githubTeamSourcesApi.ignorePatterns(namespaceSlug, sourceId) + : githubSyncAdminApi.ignorePatterns(sourceId), + }) + const save = useMutation({ + mutationFn: (ignorePatterns: string[]) => namespaceSlug + ? githubTeamSourcesApi.updateIgnorePatterns(namespaceSlug, sourceId, ignorePatterns) + : githubSyncAdminApi.updateIgnorePatterns(sourceId, ignorePatterns), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey }) + queryClient.invalidateQueries({ queryKey: namespaceSlug + ? ['github-sync', 'team', namespaceSlug, 'resolved'] : ['github-sync', 'sources'] }) + }, + }) + + if (patterns.isLoading) return

{t('githubSync.loading')}

+ if (patterns.error) return

{t('githubSync.loadFailed')}: {patterns.error.message}

+ if (!patterns.data) return null + return { + try { + await save.mutateAsync(next) + toast.success(t('githubSync.saved')) + } catch (error) { + toast.error(t('githubSync.saveFailed'), error instanceof Error ? error.message : '') + } + }} /> +} + +function IgnoreForm({ initial, saving, onSave }: { + initial: string[] + saving: boolean + onSave: (patterns: string[]) => Promise +}) { + const { t } = useTranslation() + const [value, setValue] = useState(initial.join('\n')) + const submit = (event: FormEvent) => { + event.preventDefault() + void onSave(value.split('\n').map((line) => line.trim()).filter(Boolean)) + } + return ( +
+ +