mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-14 09:01:16 +00:00
feat(authoring): runtime bindings, execution adapters and docker isolation
Layer (b1) of the authoring platform (#934), the first half of the split runtime layer: - **Runtime bindings**: agent type + config + tool allowlist per draft, persisted as JSONB and structurally validated at save time, managed via GET/PUT /drafts/{id}/runtime. MCP server declarations are validated structurally (name/transport/endpoint format); the SSRF policy, stdio gate and envRef allowlist that guard them arrive with the MCP+security layer. - **CONFIG and BEHAVIOR validation layers**: validation.yaml spec parsing (defensive YAML), assertion syntax checks; script tasks execute through the local-script adapter — inline (cleared environment, isolated workspace, interpreter allowlist, path confinement) or docker (locked container: no network, dropped capabilities, read-only root, resource caps, tmpfs; workspace permissions and uid:gid mapping so container files stay readable and deletable on hardened hosts). - The orchestrator keeps every piece of the upstream lifecycle hardening: one FutureTask per run, wall-clock watchdog, settle-first cancellation. Verified: spec-parser and assertion-evaluator suites (20), docker isolation against real containers (4, auto-skipped without a daemon), the upstream orchestrator lifecycle suite against the extended constructor (5), and the execution flow integration test (script-only three-layer validation with inline execution). Signed-off-by: zjncs <18910855655@163.com>
This commit is contained in:
zjncs
parent
250336d3e9
commit
4aab1ec48d
40 changed files with 2900 additions and 28 deletions
|
|
@ -1,12 +1,12 @@
|
|||
package com.iflytek.skillhub.config;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* Feature configuration for the skill authoring and validation platform
|
||||
* ({@code skillhub.authoring.*}). This layer covers the draft authoring and
|
||||
* structure-validation core; runtime-binding configuration (script execution
|
||||
* backends, LLM endpoints, MCP servers) arrives with the runtime layer.
|
||||
* ({@code skillhub.authoring.*}).
|
||||
*/
|
||||
@ConfigurationProperties(prefix = "skillhub.authoring")
|
||||
public class AuthoringProperties {
|
||||
|
|
@ -23,6 +23,8 @@ public class AuthoringProperties {
|
|||
/** Active runs older than this are swept to TIMED_OUT by the maintenance task. */
|
||||
private int staleRunMinutes = 30;
|
||||
|
||||
private final LocalScript localScript = new LocalScript();
|
||||
|
||||
public String getWorkspaceRoot() {
|
||||
return workspaceRoot;
|
||||
}
|
||||
|
|
@ -54,4 +56,94 @@ public class AuthoringProperties {
|
|||
public void setStaleRunMinutes(int staleRunMinutes) {
|
||||
this.staleRunMinutes = staleRunMinutes;
|
||||
}
|
||||
|
||||
public LocalScript getLocalScript() {
|
||||
return localScript;
|
||||
}
|
||||
|
||||
/** Local script execution runtime. */
|
||||
public static class LocalScript {
|
||||
private boolean enabled = true;
|
||||
|
||||
/** Where scripts run: INLINE on the server host (dev), DOCKER in a locked-down container (production). */
|
||||
private ScriptExecutionMode executionMode = ScriptExecutionMode.INLINE;
|
||||
|
||||
private final Docker docker = new Docker();
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public ScriptExecutionMode getExecutionMode() {
|
||||
return executionMode;
|
||||
}
|
||||
|
||||
public void setExecutionMode(ScriptExecutionMode executionMode) {
|
||||
this.executionMode = executionMode;
|
||||
}
|
||||
|
||||
public Docker getDocker() {
|
||||
return docker;
|
||||
}
|
||||
|
||||
/** Container isolation profile for the docker execution mode. */
|
||||
public static class Docker {
|
||||
private String image = "alpine:3.20";
|
||||
private String memory = "256m";
|
||||
private String cpus = "1.0";
|
||||
private int pidsLimit = 128;
|
||||
private String tmpfsSize = "64m";
|
||||
|
||||
public String getImage() {
|
||||
return image;
|
||||
}
|
||||
|
||||
public void setImage(String image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
public String getMemory() {
|
||||
return memory;
|
||||
}
|
||||
|
||||
public void setMemory(String memory) {
|
||||
this.memory = memory;
|
||||
}
|
||||
|
||||
public String getCpus() {
|
||||
return cpus;
|
||||
}
|
||||
|
||||
public void setCpus(String cpus) {
|
||||
this.cpus = cpus;
|
||||
}
|
||||
|
||||
public int getPidsLimit() {
|
||||
return pidsLimit;
|
||||
}
|
||||
|
||||
public void setPidsLimit(int pidsLimit) {
|
||||
this.pidsLimit = pidsLimit;
|
||||
}
|
||||
|
||||
public String getTmpfsSize() {
|
||||
return tmpfsSize;
|
||||
}
|
||||
|
||||
public void setTmpfsSize(String tmpfsSize) {
|
||||
this.tmpfsSize = tmpfsSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Script execution backend selection. */
|
||||
public enum ScriptExecutionMode {
|
||||
INLINE,
|
||||
DOCKER
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package com.iflytek.skillhub.controller.authoring;
|
|||
|
||||
import com.iflytek.skillhub.controller.BaseApiController;
|
||||
import com.iflytek.skillhub.domain.authoring.SkillDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.service.RuntimeBindingService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.SkillDraftService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.ValidationRunService;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationRun;
|
||||
|
|
@ -10,6 +11,8 @@ import com.iflytek.skillhub.dto.CreateDraftRequest;
|
|||
import com.iflytek.skillhub.dto.DraftFileContentResponse;
|
||||
import com.iflytek.skillhub.dto.DraftFileResponse;
|
||||
import com.iflytek.skillhub.dto.DraftResponse;
|
||||
import com.iflytek.skillhub.dto.RuntimeBindingRequest;
|
||||
import com.iflytek.skillhub.dto.RuntimeBindingResponse;
|
||||
import com.iflytek.skillhub.dto.SaveDraftFileRequest;
|
||||
import com.iflytek.skillhub.dto.SaveDraftFileResponse;
|
||||
import com.iflytek.skillhub.dto.ValidationEventResponse;
|
||||
|
|
@ -21,6 +24,7 @@ import jakarta.validation.Valid;
|
|||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
|
|
@ -34,26 +38,30 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* REST endpoints for skill draft authoring and structure validation. Drafts
|
||||
* own a SKILL.md scaffold and arbitrary resource files; validation runs
|
||||
* execute the STRUCTURE layer and report findings with fix suggestions.
|
||||
* Runtime binding, behavior execution and the publish integration arrive with
|
||||
* the later layers.
|
||||
* REST endpoints for skill draft authoring and validation. Drafts own a
|
||||
* SKILL.md scaffold and arbitrary resource files; runtime bindings configure
|
||||
* the agent and tool allowlist; validation runs execute the structure, config
|
||||
* and behavior layers and report findings with fix suggestions. The MCP probe
|
||||
* and the binding security policy arrive with the MCP+security layer, and the
|
||||
* fix-apply loop and publish integration with the next layer.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping({"/api/v1/authoring", "/api/web/authoring"})
|
||||
public class SkillAuthoringController extends BaseApiController {
|
||||
|
||||
private final SkillDraftService draftService;
|
||||
private final RuntimeBindingService bindingService;
|
||||
private final ValidationRunService runService;
|
||||
private final ValidationRunOrchestrator orchestrator;
|
||||
|
||||
public SkillAuthoringController(com.iflytek.skillhub.dto.ApiResponseFactory responseFactory,
|
||||
SkillDraftService draftService,
|
||||
RuntimeBindingService bindingService,
|
||||
ValidationRunService runService,
|
||||
ValidationRunOrchestrator orchestrator) {
|
||||
super(responseFactory);
|
||||
this.draftService = draftService;
|
||||
this.bindingService = bindingService;
|
||||
this.runService = runService;
|
||||
this.orchestrator = orchestrator;
|
||||
}
|
||||
|
|
@ -155,6 +163,42 @@ public class SkillAuthoringController extends BaseApiController {
|
|||
return ok("response.success.deleted", null);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- runtime binding
|
||||
|
||||
@Operation(operationId = "getDraftRuntimeBinding", summary = "Get the draft's runtime binding")
|
||||
@GetMapping("/drafts/{draftId}/runtime")
|
||||
public ApiResponse<RuntimeBindingResponse> getRuntime(@PathVariable Long draftId,
|
||||
@RequestAttribute("userId") String userId,
|
||||
@RequestAttribute(value = "platformRoles", required = false)
|
||||
Set<String> platformRoles) {
|
||||
draftService.getOwnedDraft(draftId, userId, platformRoles);
|
||||
return ok("response.success", bindingService.findBinding(draftId)
|
||||
.map(this::toRuntimeResponse)
|
||||
.orElseGet(() -> new RuntimeBindingResponse(null, Map.of(), List.of(), List.of(), null)));
|
||||
}
|
||||
|
||||
@Operation(operationId = "saveDraftRuntimeBinding", summary = "Configure the agent runtime, tools, and MCP servers")
|
||||
@PutMapping("/drafts/{draftId}/runtime")
|
||||
public ApiResponse<RuntimeBindingResponse> saveRuntime(@PathVariable Long draftId,
|
||||
@Valid @RequestBody RuntimeBindingRequest request,
|
||||
@RequestAttribute("userId") String userId,
|
||||
@RequestAttribute(value = "platformRoles", required = false)
|
||||
Set<String> platformRoles) {
|
||||
return ok("response.success.updated", toRuntimeResponse(bindingService.saveBinding(
|
||||
draftId, userId, request.agentType(), request.config(),
|
||||
request.toolAllowlist(), request.mcpServers(), platformRoles)));
|
||||
}
|
||||
|
||||
private RuntimeBindingResponse toRuntimeResponse(
|
||||
com.iflytek.skillhub.domain.authoring.RuntimeBinding binding) {
|
||||
return new RuntimeBindingResponse(
|
||||
binding.getAgentType().identifier(),
|
||||
binding.getConfig() == null ? Map.of() : binding.getConfig(),
|
||||
binding.getToolAllowlist() == null ? List.of() : binding.getToolAllowlist(),
|
||||
binding.getMcpServers() == null ? List.of() : binding.getMcpServers(),
|
||||
binding.getUpdatedAt());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- validation runs
|
||||
|
||||
@Operation(operationId = "startValidationRun", summary = "Start a validation run for the draft's current revision")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
package com.iflytek.skillhub.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Request body for the per-draft runtime binding: which agent runtime executes
|
||||
* behavior tasks, its adapter configuration, the tool allowlist, and MCP server
|
||||
* declarations. Credentials must be referenced by name, never inlined.
|
||||
*/
|
||||
public record RuntimeBindingRequest(
|
||||
@NotBlank(message = "{error.badRequest}") String agentType,
|
||||
Map<String, Object> config,
|
||||
List<String> toolAllowlist,
|
||||
List<Map<String, Object>> mcpServers
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package com.iflytek.skillhub.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** Runtime binding as seen by the authoring UI. */
|
||||
public record RuntimeBindingResponse(
|
||||
String agentType,
|
||||
Map<String, Object> config,
|
||||
List<String> toolAllowlist,
|
||||
List<Map<String, Object>> mcpServers,
|
||||
Instant updatedAt
|
||||
) {
|
||||
}
|
||||
|
|
@ -1,9 +1,22 @@
|
|||
package com.iflytek.skillhub.service.authoring;
|
||||
|
||||
import com.iflytek.skillhub.config.AuthoringProperties;
|
||||
import com.iflytek.skillhub.domain.authoring.RuntimeBinding;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.AssertionEvaluator;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.AssertionFailure;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.RuntimeEventSink;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.RuntimeExecutionContext;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.SkillRuntimeAdapter;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.TaskResult;
|
||||
import com.iflytek.skillhub.domain.authoring.service.DraftStructureValidator;
|
||||
import com.iflytek.skillhub.domain.authoring.service.RuntimeBindingService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.RuntimeBindingValidator;
|
||||
import com.iflytek.skillhub.domain.authoring.service.SkillDraftService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.ValidationRunService;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.TaskType;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.ValidationSpec;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.ValidationSpecParser;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.ValidationTaskSpec;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.FindingDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.FindingSeverity;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationEvent;
|
||||
|
|
@ -13,62 +26,88 @@ import com.iflytek.skillhub.domain.authoring.validation.ValidationLayer;
|
|||
import com.iflytek.skillhub.domain.authoring.validation.ValidationRun;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationRunStatus;
|
||||
import com.iflytek.skillhub.domain.shared.exception.DomainConflictException;
|
||||
import com.iflytek.skillhub.domain.skill.validation.PackageEntry;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Executes validation runs for skill drafts. This layer runs the STRUCTURE
|
||||
* phase: package layout, SKILL.md frontmatter and file limits, reported as
|
||||
* structured findings with fix suggestions. The CONFIG and BEHAVIOR phases
|
||||
* (runtime binding, MCP probing, script/LLM task execution) arrive with the
|
||||
* runtime layer.
|
||||
* Executes validation runs for skill drafts. The STRUCTURE layer validates package
|
||||
* layout and SKILL.md frontmatter; the CONFIG layer parses validation.yaml, checks
|
||||
* the runtime binding and probes declared MCP servers; the BEHAVIOR layer executes
|
||||
* the spec's tasks through the bound runtime adapter and evaluates assertions.
|
||||
*
|
||||
* <p>The MCP probe of declared servers and the OpenAI-compatible prompt
|
||||
* adapter arrive with the MCP+security layer; this layer executes script
|
||||
* tasks through the local-script (inline or docker) adapter.
|
||||
*
|
||||
* <p>Run lifecycle hardening follows the merged upstream review: one FutureTask per
|
||||
* run (duplicate submits are ignored), a hard wall-clock watchdog on the timer,
|
||||
* cancellation settles first and then interrupts the worker, and the terminal
|
||||
* event is appended exactly once.
|
||||
*/
|
||||
@Service
|
||||
public class ValidationRunOrchestrator {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ValidationRunOrchestrator.class);
|
||||
private static final String SPEC_FILE = ValidationSpec.FILE_PATH;
|
||||
private static final int MAX_EVENT_TEXT = 4_000;
|
||||
|
||||
private final ValidationRunService runService;
|
||||
private final SkillDraftService draftService;
|
||||
private final RuntimeBindingService bindingService;
|
||||
private final DraftStructureValidator structureValidator;
|
||||
private final RuntimeBindingValidator bindingValidator;
|
||||
private final ValidationSpecParser specParser = new ValidationSpecParser();
|
||||
private final AssertionEvaluator assertionEvaluator = new AssertionEvaluator();
|
||||
private final List<SkillRuntimeAdapter> adapters;
|
||||
private final ValidationEventBroadcaster broadcaster;
|
||||
private final AuthoringProperties properties;
|
||||
private final Executor executor;
|
||||
|
||||
private final ScheduledExecutorService timer;
|
||||
private final AuthoringProperties properties;
|
||||
private final Map<Long, FutureTask<Void>> tasks = new ConcurrentHashMap<>();
|
||||
|
||||
public ValidationRunOrchestrator(ValidationRunService runService,
|
||||
SkillDraftService draftService,
|
||||
RuntimeBindingService bindingService,
|
||||
DraftStructureValidator structureValidator,
|
||||
RuntimeBindingValidator bindingValidator,
|
||||
List<SkillRuntimeAdapter> adapters,
|
||||
ValidationEventBroadcaster broadcaster,
|
||||
AuthoringProperties properties,
|
||||
@Qualifier("authoringValidationExecutor") Executor executor,
|
||||
@Qualifier("authoringValidationTimer") ScheduledExecutorService timer,
|
||||
AuthoringProperties properties) {
|
||||
@Qualifier("authoringValidationTimer") ScheduledExecutorService timer) {
|
||||
this.runService = runService;
|
||||
this.draftService = draftService;
|
||||
this.bindingService = bindingService;
|
||||
this.structureValidator = structureValidator;
|
||||
this.bindingValidator = bindingValidator;
|
||||
this.adapters = List.copyOf(adapters);
|
||||
this.broadcaster = broadcaster;
|
||||
this.properties = properties;
|
||||
this.executor = executor;
|
||||
this.timer = timer;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/** Enqueues one QUEUED run for execution. */
|
||||
/** Enqueues one QUEUED run for execution; duplicate submits for the same run are ignored. */
|
||||
public void submit(Long runId) {
|
||||
FutureTask<Void> task = new FutureTask<>(() -> { executeRun(runId); return null; }) {
|
||||
@Override protected void done() { tasks.remove(runId, this); }
|
||||
|
|
@ -168,6 +207,7 @@ public class ValidationRunOrchestrator {
|
|||
"draftName", snapshot.name(),
|
||||
"draftRevision", run.getDraftRevision()));
|
||||
|
||||
Path workspace = workspaceFor(runId);
|
||||
try {
|
||||
// ---- STRUCTURE layer
|
||||
emit(runId, ValidationEventType.PHASE_STARTED, "STRUCTURE", Map.of());
|
||||
|
|
@ -179,20 +219,191 @@ public class ValidationRunOrchestrator {
|
|||
"errors", context.errorCount(ValidationLayer.STRUCTURE),
|
||||
"warnings", context.warningCount(ValidationLayer.STRUCTURE)));
|
||||
|
||||
// ---- CONFIG layer
|
||||
emit(runId, ValidationEventType.PHASE_STARTED, "CONFIG", Map.of());
|
||||
ValidationSpec spec = runConfigLayer(context, run, snapshot.entries());
|
||||
emit(runId, ValidationEventType.PHASE_FINISHED, "CONFIG", Map.of(
|
||||
"errors", context.errorCount(ValidationLayer.CONFIG),
|
||||
"warnings", context.warningCount(ValidationLayer.CONFIG)));
|
||||
|
||||
// ---- BEHAVIOR layer
|
||||
emit(runId, ValidationEventType.PHASE_STARTED, "BEHAVIOR", Map.of());
|
||||
boolean blocked = context.errorCount(ValidationLayer.STRUCTURE) > 0
|
||||
|| context.errorCount(ValidationLayer.CONFIG) > 0;
|
||||
if (blocked) {
|
||||
emit(runId, ValidationEventType.PHASE_FINISHED, "BEHAVIOR",
|
||||
Map.of("skipped", true, "reason", "structure or configuration errors"));
|
||||
} else {
|
||||
materializeWorkspace(workspace, snapshot.entries());
|
||||
BehaviorOutcome outcome = runBehaviorLayer(context, run, spec, workspace);
|
||||
if (outcome.interrupted()) {
|
||||
return; // already settled (CANCELLED / TIMED_OUT) with terminal event
|
||||
}
|
||||
emit(runId, ValidationEventType.PHASE_FINISHED, "BEHAVIOR", Map.of(
|
||||
"skipped", false,
|
||||
"tasksRun", context.tasksRun,
|
||||
"tasksFailed", context.tasksFailed));
|
||||
}
|
||||
|
||||
settle(runId,
|
||||
context.errorCount == 0 ? ValidationRunStatus.SUCCEEDED : ValidationRunStatus.FAILED,
|
||||
context.errorCount, context.warningCount, summary(context));
|
||||
context.errorCount, context.warningCount, summary(context, blocked));
|
||||
} catch (DomainConflictException exception) {
|
||||
throw exception;
|
||||
} catch (Exception exception) {
|
||||
log.error("Validation pipeline for run {} failed", runId, exception);
|
||||
settle(runId, ValidationRunStatus.FAILED, context.errorCount + 1, context.warningCount,
|
||||
Map.of("internalError", truncate(String.valueOf(exception.getMessage()), 500)));
|
||||
} finally {
|
||||
deleteRecursively(workspace);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- events
|
||||
// ---------------------------------------------------------------- config layer
|
||||
|
||||
private ValidationSpec runConfigLayer(RunContext context, ValidationRun run,
|
||||
List<PackageEntry> entries) {
|
||||
String specContent = readEntryText(entries, SPEC_FILE);
|
||||
ValidationSpec spec = ValidationSpec.empty();
|
||||
if (specContent == null) {
|
||||
context.recordFinding(FindingDraft.warning(ValidationLayer.CONFIG, "SPEC_MISSING",
|
||||
SPEC_FILE, "validation.yaml is absent; behavior layer runs zero tasks"));
|
||||
} else {
|
||||
ValidationSpecParser.ParseOutcome outcome = specParser.parse(specContent);
|
||||
outcome.errors().forEach(error -> context.recordFinding(FindingDraft.error(
|
||||
ValidationLayer.CONFIG, error.ruleCode(), SPEC_FILE, error.message(), null)));
|
||||
if (outcome.isValid()) {
|
||||
spec = outcome.spec();
|
||||
}
|
||||
}
|
||||
|
||||
// The binding is checked (and its MCP servers probed) even when
|
||||
// validation.yaml is absent: a binding that cannot work must fail the
|
||||
// run here, not surface for the first time at submit.
|
||||
RuntimeBinding binding = bindingService.findBinding(run.getDraftId()).orElse(null);
|
||||
if (binding == null) {
|
||||
if (!spec.safeTasks().isEmpty()) {
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.CONFIG, "BINDING_MISSING",
|
||||
"No runtime binding configured, but validation.yaml declares "
|
||||
+ spec.safeTasks().size() + " task(s)"));
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
bindingValidator.validate(
|
||||
binding.getAgentType().identifier(),
|
||||
binding.getConfig(),
|
||||
binding.getToolAllowlist(),
|
||||
binding.getMcpServers())
|
||||
.forEach(context::recordFinding);
|
||||
|
||||
SkillRuntimeAdapter adapter = findAdapter(binding.getAgentType().identifier());
|
||||
if (adapter == null) {
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.CONFIG, "ADAPTER_NOT_FOUND",
|
||||
"No runtime adapter is registered for agent type '"
|
||||
+ binding.getAgentType().identifier() + "'"));
|
||||
} else if (!adapter.enabled()) {
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.CONFIG, "RUNTIME_DISABLED",
|
||||
"Runtime '" + binding.getAgentType().identifier()
|
||||
+ "' is disabled in server configuration"));
|
||||
}
|
||||
|
||||
for (ValidationTaskSpec task : spec.safeTasks()) {
|
||||
assertionEvaluator.validateSyntax(task.safeAssertions(), task.type() == TaskType.PROMPT)
|
||||
.forEach(problem -> context.recordFinding(FindingDraft.error(
|
||||
ValidationLayer.CONFIG, "ASSERTION_INVALID", SPEC_FILE,
|
||||
"task '" + task.name() + "': " + problem, null)));
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- behavior layer
|
||||
|
||||
private BehaviorOutcome runBehaviorLayer(RunContext context, ValidationRun run,
|
||||
ValidationSpec spec, Path workspace) {
|
||||
RuntimeBinding binding = bindingService.findBinding(run.getDraftId()).orElse(null);
|
||||
if (binding == null || spec.safeTasks().isEmpty()) {
|
||||
return BehaviorOutcome.COMPLETED;
|
||||
}
|
||||
SkillRuntimeAdapter adapter = findAdapter(binding.getAgentType().identifier());
|
||||
if (adapter == null || !adapter.enabled()) {
|
||||
return BehaviorOutcome.COMPLETED; // already reported by the config layer
|
||||
}
|
||||
|
||||
long deadline = System.currentTimeMillis() + properties.getRunTimeoutMs();
|
||||
Map<String, Object> adapterConfig = binding.getConfig() == null
|
||||
? Map.of() : binding.getConfig();
|
||||
List<String> toolAllowlist = binding.getToolAllowlist() == null
|
||||
? List.of() : binding.getToolAllowlist();
|
||||
RuntimeExecutionContext executionContext = new RuntimeExecutionContext(
|
||||
run.getId(), workspace, adapterConfig, toolAllowlist,
|
||||
binding.getMcpServers() == null ? List.of() : binding.getMcpServers(),
|
||||
// cancellation settles first and interrupts the worker, so the
|
||||
// cooperative signal is the interrupt status of this thread
|
||||
() -> Thread.currentThread().isInterrupted());
|
||||
|
||||
for (ValidationTaskSpec task : spec.safeTasks()) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
return BehaviorOutcome.interrupted("cancelled");
|
||||
}
|
||||
if (System.currentTimeMillis() > deadline) {
|
||||
settle(run.getId(), ValidationRunStatus.TIMED_OUT,
|
||||
context.errorCount, context.warningCount, summary(context, false));
|
||||
return BehaviorOutcome.interrupted("timeout");
|
||||
}
|
||||
executeTask(context, adapter, executionContext, task, workspace);
|
||||
}
|
||||
return BehaviorOutcome.COMPLETED;
|
||||
}
|
||||
|
||||
private void executeTask(RunContext context, SkillRuntimeAdapter adapter,
|
||||
RuntimeExecutionContext executionContext, ValidationTaskSpec task,
|
||||
Path workspace) {
|
||||
if (!adapter.supports(task.type())) {
|
||||
context.tasksFailed++;
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.BEHAVIOR, "ADAPTER_TASK_UNSUPPORTED",
|
||||
SPEC_FILE, "runtime '" + adapter.agentType() + "' cannot execute "
|
||||
+ task.type() + " task '" + task.name() + "'", null));
|
||||
return;
|
||||
}
|
||||
if (task.type() == TaskType.SCRIPT && !scriptExists(workspace, task.script())) {
|
||||
context.tasksFailed++;
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.BEHAVIOR, "TASK_SCRIPT_MISSING",
|
||||
task.script(), "script file not found in the skill: " + task.script(), null));
|
||||
return;
|
||||
}
|
||||
|
||||
context.tasksRun++;
|
||||
TaskResult result;
|
||||
try {
|
||||
result = adapter.execute(executionContext, task, context.sink());
|
||||
} catch (Exception exception) {
|
||||
context.tasksFailed++;
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.BEHAVIOR, "TASK_FAILED",
|
||||
SPEC_FILE, "task '" + task.name() + "' failed: "
|
||||
+ truncate(String.valueOf(exception.getMessage()), 500), null));
|
||||
return;
|
||||
}
|
||||
|
||||
List<AssertionFailure> failures =
|
||||
assertionEvaluator.evaluate(task.safeAssertions(), result);
|
||||
if (failures.isEmpty()) {
|
||||
emit(context.runId, ValidationEventType.LOG, "BEHAVIOR", Map.of(
|
||||
"task", task.name(), "stream", "summary",
|
||||
"line", "task '" + task.name() + "' passed "
|
||||
+ task.safeAssertions().size() + " assertion(s)"));
|
||||
return;
|
||||
}
|
||||
context.tasksFailed++;
|
||||
for (AssertionFailure failure : failures) {
|
||||
context.recordFinding(FindingDraft.error(ValidationLayer.BEHAVIOR, "ASSERTION_FAILED",
|
||||
SPEC_FILE, "task '" + task.name() + "': " + failure.describe(), null));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- events & terminal handling
|
||||
|
||||
/** Emits one event, persists it, and pushes it to live SSE subscribers. */
|
||||
private void emit(Long runId, ValidationEventType type, String phase, Map<String, Object> payload) {
|
||||
ValidationEvent event = runService.appendEvent(runId, type, phase, payload);
|
||||
broadcaster.publish(runId, event);
|
||||
|
|
@ -208,15 +419,75 @@ public class ValidationRunOrchestrator {
|
|||
|
||||
// ---------------------------------------------------------------- small helpers
|
||||
|
||||
private Map<String, Object> summary(RunContext context) {
|
||||
private boolean scriptExists(Path workspace, String script) {
|
||||
try {
|
||||
Path resolved = workspace.resolve(script).normalize();
|
||||
return resolved.startsWith(workspace.normalize()) && Files.isRegularFile(resolved);
|
||||
} catch (Exception exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private SkillRuntimeAdapter findAdapter(String agentType) {
|
||||
return adapters.stream()
|
||||
.filter(adapter -> adapter.agentType().equals(agentType))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private Map<String, Object> summary(RunContext context, boolean behaviorSkipped) {
|
||||
Map<String, Object> summary = new java.util.LinkedHashMap<>();
|
||||
summary.put("structureErrors", context.errorCount(ValidationLayer.STRUCTURE));
|
||||
summary.put("structureWarnings", context.warningCount(ValidationLayer.STRUCTURE));
|
||||
summary.put("configErrors", context.errorCount(ValidationLayer.CONFIG));
|
||||
summary.put("configWarnings", context.warningCount(ValidationLayer.CONFIG));
|
||||
summary.put("behaviorTasksRun", context.tasksRun);
|
||||
summary.put("behaviorTasksFailed", context.tasksFailed);
|
||||
summary.put("behaviorSkipped", behaviorSkipped);
|
||||
summary.put("errorCount", context.errorCount);
|
||||
summary.put("warningCount", context.warningCount);
|
||||
return summary;
|
||||
}
|
||||
|
||||
private Path workspaceFor(Long runId) {
|
||||
return Path.of(properties.getWorkspaceRoot(), "run-" + runId);
|
||||
}
|
||||
|
||||
private void materializeWorkspace(Path workspace, List<PackageEntry> entries)
|
||||
throws IOException {
|
||||
Files.createDirectories(workspace);
|
||||
for (PackageEntry entry : entries) {
|
||||
Path target = workspace.resolve(entry.path()).normalize();
|
||||
if (!target.startsWith(workspace.normalize())) {
|
||||
throw new IOException("package entry escapes the workspace: " + entry.path());
|
||||
}
|
||||
Files.createDirectories(target.getParent());
|
||||
Files.write(target, entry.content());
|
||||
}
|
||||
}
|
||||
|
||||
private String readEntryText(List<PackageEntry> entries, String path) {
|
||||
return entries.stream()
|
||||
.filter(entry -> entry.path().equals(path))
|
||||
.findFirst()
|
||||
.map(entry -> new String(entry.content(), StandardCharsets.UTF_8))
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private void deleteRecursively(Path root) {
|
||||
try (var stream = Files.walk(root)) {
|
||||
stream.sorted(Comparator.reverseOrder()).forEach(p -> {
|
||||
try {
|
||||
Files.deleteIfExists(p);
|
||||
} catch (IOException exception) {
|
||||
log.warn("Could not delete workspace file {}", p, exception);
|
||||
}
|
||||
});
|
||||
} catch (IOException exception) {
|
||||
log.warn("Could not clean up workspace {}", root, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static String truncate(String value, int limit) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
|
|
@ -224,6 +495,15 @@ public class ValidationRunOrchestrator {
|
|||
return value.length() <= limit ? value : value.substring(0, limit) + "…";
|
||||
}
|
||||
|
||||
private record BehaviorOutcome(boolean interrupted, String reason) {
|
||||
|
||||
static BehaviorOutcome COMPLETED = new BehaviorOutcome(false, null);
|
||||
|
||||
static BehaviorOutcome interrupted(String reason) {
|
||||
return new BehaviorOutcome(true, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-run finding bookkeeping; each finding is persisted and emitted as an event. */
|
||||
private final class RunContext {
|
||||
|
||||
|
|
@ -231,6 +511,8 @@ public class ValidationRunOrchestrator {
|
|||
private final List<FindingDraft> findings = new ArrayList<>();
|
||||
private int errorCount;
|
||||
private int warningCount;
|
||||
private int tasksRun;
|
||||
private int tasksFailed;
|
||||
|
||||
RunContext(Long runId) {
|
||||
this.runId = runId;
|
||||
|
|
@ -267,5 +549,37 @@ public class ValidationRunOrchestrator {
|
|||
"message", truncate(draft.message(), MAX_EVENT_TEXT),
|
||||
"hasSuggestion", draft.suggestion() != null));
|
||||
}
|
||||
|
||||
/** Serialized sink so per-run event sequence numbers cannot race. */
|
||||
RuntimeEventSink sink() {
|
||||
return new RuntimeEventSink() {
|
||||
@Override
|
||||
public synchronized void agentMessage(String taskName, String content) {
|
||||
emit(runId, ValidationEventType.AGENT_MESSAGE, "BEHAVIOR", Map.of(
|
||||
"task", taskName, "content", truncate(content, MAX_EVENT_TEXT)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void toolCall(String taskName, String tool, String argumentsJson) {
|
||||
emit(runId, ValidationEventType.TOOL_CALL, "BEHAVIOR", Map.of(
|
||||
"task", taskName, "tool", tool,
|
||||
"arguments", truncate(argumentsJson, MAX_EVENT_TEXT)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void toolResult(String taskName, String tool, String summaryJson) {
|
||||
emit(runId, ValidationEventType.TOOL_RESULT, "BEHAVIOR", Map.of(
|
||||
"task", taskName, "tool", tool,
|
||||
"summary", truncate(summaryJson, MAX_EVENT_TEXT)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void log(String taskName, String stream, String line) {
|
||||
emit(runId, ValidationEventType.LOG, "BEHAVIOR", Map.of(
|
||||
"task", taskName, "stream", stream,
|
||||
"line", truncate(line, 2_000)));
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,230 @@
|
|||
package com.iflytek.skillhub.service.authoring.adapter;
|
||||
|
||||
import com.iflytek.skillhub.config.AuthoringProperties;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Production backend: the script runs inside a throwaway container with a hard
|
||||
* isolation profile — no network, memory/CPU/pids caps, read-only root filesystem,
|
||||
* all capabilities dropped, no-new-privileges, no log persistence. Only the
|
||||
* validation workspace is mounted (read-write) so scripts can produce artifacts;
|
||||
* every other path the container sees is immutable or ephemeral tmpfs.
|
||||
*
|
||||
* <p>Network isolation is deliberately not configurable: a validation run must
|
||||
* never be able to reach other services. The image must provide the configured
|
||||
* interpreter (the default alpine ships {@code sh}; python/node skills need a
|
||||
* matching image).
|
||||
*/
|
||||
@Component
|
||||
public class DockerScriptCommandBuilder implements ScriptCommandBuilder {
|
||||
|
||||
/** Mount point of the validation workspace inside the container. */
|
||||
public static final String CONTAINER_WORKSPACE = "/workspace";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DockerScriptCommandBuilder.class);
|
||||
private static final Duration PROBE_TIMEOUT = Duration.ofSeconds(5);
|
||||
private static final Duration CLEANUP_TIMEOUT = Duration.ofSeconds(5);
|
||||
|
||||
private final AuthoringProperties properties;
|
||||
private volatile Boolean daemonReachable;
|
||||
|
||||
public DockerScriptCommandBuilder(AuthoringProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String backend() {
|
||||
return "docker";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean available() {
|
||||
Boolean cached = daemonReachable;
|
||||
if (cached == null) {
|
||||
cached = probeDaemon();
|
||||
daemonReachable = cached;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** Re-probes the daemon on the next {@link #available()} call. */
|
||||
void resetProbe() {
|
||||
daemonReachable = null;
|
||||
}
|
||||
|
||||
private boolean probeDaemon() {
|
||||
try {
|
||||
Process process = new ProcessBuilder("docker", "info", "--format", "{{.ServerVersion}}")
|
||||
.start();
|
||||
if (!process.waitFor(PROBE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
process.destroyForcibly();
|
||||
return false;
|
||||
}
|
||||
return process.exitValue() == 0;
|
||||
} catch (IOException | InterruptedException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScriptExecutionPlan plan(Path workspace, String interpreter, Path scriptPath,
|
||||
List<String> args, Map<String, String> scrubbedEnvironment,
|
||||
String runTag) {
|
||||
AuthoringProperties.LocalScript.Docker docker = properties.getLocalScript().getDocker();
|
||||
String hostWorkspace = realPath(workspace);
|
||||
ensureContainerReadable(workspace);
|
||||
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add("docker");
|
||||
command.add("run");
|
||||
command.add("--rm");
|
||||
command.add("--name");
|
||||
command.add(containerName(runTag));
|
||||
hostUser().ifPresent(user -> {
|
||||
command.add("--user");
|
||||
command.add(user);
|
||||
});
|
||||
command.add("--network");
|
||||
command.add("none");
|
||||
command.add("--memory");
|
||||
command.add(docker.getMemory());
|
||||
command.add("--cpus");
|
||||
command.add(docker.getCpus());
|
||||
command.add("--pids-limit");
|
||||
command.add(String.valueOf(docker.getPidsLimit()));
|
||||
command.add("--read-only");
|
||||
command.add("--tmpfs");
|
||||
command.add("/tmp:rw,size=" + docker.getTmpfsSize());
|
||||
command.add("--cap-drop");
|
||||
command.add("ALL");
|
||||
command.add("--security-opt");
|
||||
command.add("no-new-privileges");
|
||||
command.add("--log-driver");
|
||||
command.add("none");
|
||||
for (Map.Entry<String, String> entry : scrubbedEnvironment.entrySet()) {
|
||||
command.add("-e");
|
||||
command.add(entry.getKey() + "=" + containerEnvValue(entry.getKey(), entry.getValue()));
|
||||
}
|
||||
command.add("-v");
|
||||
command.add(hostWorkspace + ":" + CONTAINER_WORKSPACE);
|
||||
command.add("-w");
|
||||
command.add(CONTAINER_WORKSPACE);
|
||||
command.add(docker.getImage());
|
||||
command.add(interpreter);
|
||||
command.add(containerScriptPath(workspace, scriptPath));
|
||||
command.addAll(args);
|
||||
// the docker CLI needs its own environment to reach the daemon; the script
|
||||
// only sees the -e variables above, inside the container
|
||||
return ScriptExecutionPlan.inherited(List.copyOf(command));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cleanup(String runTag) {
|
||||
try {
|
||||
Process process = new ProcessBuilder("docker", "rm", "-f", containerName(runTag)).start();
|
||||
if (!process.waitFor(CLEANUP_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
process.destroyForcibly();
|
||||
}
|
||||
} catch (IOException | InterruptedException exception) {
|
||||
log.warn("Could not force-remove validation container {}: {}",
|
||||
containerName(runTag), exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Cached "uid:gid" of the host JVM user, so container-created files stay deletable. */
|
||||
private static final java.util.Optional<String> HOST_USER = resolveHostUser();
|
||||
|
||||
private static java.util.Optional<String> resolveHostUser() {
|
||||
try {
|
||||
String uid = readCommandOutput("id", "-u").trim();
|
||||
String gid = readCommandOutput("id", "-g").trim();
|
||||
if (uid.matches("\\d+") && gid.matches("\\d+")) {
|
||||
return java.util.Optional.of(uid + ":" + gid);
|
||||
}
|
||||
} catch (Exception exception) {
|
||||
// non-POSIX host (e.g. Windows without id): run as the image default
|
||||
}
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
|
||||
private static String readCommandOutput(String... command) throws Exception {
|
||||
Process process = new ProcessBuilder(command).start();
|
||||
String output;
|
||||
try (var reader = process.inputReader()) {
|
||||
output = reader.lines().findFirst().orElse("");
|
||||
}
|
||||
process.waitFor();
|
||||
return output;
|
||||
}
|
||||
|
||||
private static java.util.Optional<String> hostUser() {
|
||||
return HOST_USER;
|
||||
}
|
||||
|
||||
/**
|
||||
* The container drops every capability (including DAC_OVERRIDE), so the
|
||||
* workspace files must be world-readable (and directories traversable)
|
||||
* on the host or the in-container interpreter cannot open them. Fresh JVM
|
||||
* temp directories are owner-only on Linux, which would break script
|
||||
* execution there; macOS Docker Desktop masks this by remapping
|
||||
* ownership inside the VM.
|
||||
*/
|
||||
private static void ensureContainerReadable(Path root) {
|
||||
try (var stream = Files.walk(root)) {
|
||||
stream.forEach(path -> {
|
||||
try {
|
||||
if (Files.isDirectory(path)) {
|
||||
// traversable and writable so the script can create
|
||||
// artifacts inside the mounted workspace
|
||||
path.toFile().setExecutable(true, false);
|
||||
path.toFile().setWritable(true, false);
|
||||
}
|
||||
path.toFile().setReadable(true, false);
|
||||
} catch (Exception exception) {
|
||||
// best effort: a locked file surfaces as a task failure
|
||||
// with the interpreter's own error message
|
||||
}
|
||||
});
|
||||
} catch (IOException exception) {
|
||||
// the mount itself will fail visibly if the tree is unreadable
|
||||
}
|
||||
}
|
||||
|
||||
/** Container names allow [a-zA-Z0-9][a-zA-Z0-9_.-]+. */
|
||||
static String containerName(String runTag) {
|
||||
String sanitized = runTag == null ? "unknown" : runTag.replaceAll("[^a-zA-Z0-9_.-]", "-");
|
||||
if (sanitized.isEmpty() || !Character.isLetterOrDigit(sanitized.charAt(0))) {
|
||||
sanitized = "r" + sanitized;
|
||||
}
|
||||
return "skillhub-validation-" + sanitized;
|
||||
}
|
||||
|
||||
private String containerScriptPath(Path workspace, Path scriptPath) {
|
||||
return CONTAINER_WORKSPACE + "/" + workspace.relativize(scriptPath);
|
||||
}
|
||||
|
||||
/** HOME points at the workspace; inside the container that is the mount point. */
|
||||
private String containerEnvValue(String key, String value) {
|
||||
return "HOME".equals(key) ? CONTAINER_WORKSPACE : value;
|
||||
}
|
||||
|
||||
/** Resolves symlinks (e.g. macOS /var/folders → /private/var/folders) for bind mounts. */
|
||||
private String realPath(Path workspace) {
|
||||
try {
|
||||
return workspace.toRealPath().toString();
|
||||
} catch (IOException exception) {
|
||||
return workspace.toAbsolutePath().toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package com.iflytek.skillhub.service.authoring.adapter;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Default backend: the script runs as a plain subprocess of the server process
|
||||
* with a scrubbed environment. Suitable for local development and CI where the
|
||||
* drafts are the developer's own; production deployments should switch to the
|
||||
* docker backend.
|
||||
*/
|
||||
@Component
|
||||
public class InlineScriptCommandBuilder implements ScriptCommandBuilder {
|
||||
|
||||
@Override
|
||||
public String backend() {
|
||||
return "inline";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean available() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScriptExecutionPlan plan(Path workspace, String interpreter, Path scriptPath,
|
||||
List<String> args, Map<String, String> scrubbedEnvironment,
|
||||
String runTag) {
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add(interpreter);
|
||||
command.add(scriptPath.toString());
|
||||
command.addAll(args);
|
||||
return new ScriptExecutionPlan(List.copyOf(command), Map.copyOf(scrubbedEnvironment));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
package com.iflytek.skillhub.service.authoring.adapter;
|
||||
|
||||
import com.iflytek.skillhub.config.AuthoringProperties;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.RuntimeEventSink;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.RuntimeExecutionContext;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.SkillRuntimeAdapter;
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.TaskResult;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.TaskType;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.ValidationTaskSpec;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Behavior-layer adapter that executes the skill's own scripts in an isolated working
|
||||
* directory with a scrubbed environment. Only whitelisted interpreters may run and only
|
||||
* environment variables named in the binding's {@code envAllowlist} pass through.
|
||||
*
|
||||
* <p>Where the process actually runs is delegated to a {@link ScriptCommandBuilder}:
|
||||
* the inline backend spawns the interpreter on the server host, the docker backend
|
||||
* wraps it in a locked-down container. Everything else (streaming, buffering, timeout,
|
||||
* cancel) is backend-independent.
|
||||
*/
|
||||
@Component
|
||||
public class LocalScriptRuntimeAdapter implements SkillRuntimeAdapter {
|
||||
|
||||
private static final int MAX_STREAM_BUFFER = 256 * 1024;
|
||||
private static final long POLL_SLICE_MS = 200;
|
||||
|
||||
private final AuthoringProperties properties;
|
||||
private final Map<String, ScriptCommandBuilder> commandBuilders;
|
||||
|
||||
public LocalScriptRuntimeAdapter(AuthoringProperties properties,
|
||||
List<ScriptCommandBuilder> commandBuilders) {
|
||||
this.properties = properties;
|
||||
this.commandBuilders = commandBuilders.stream()
|
||||
.collect(Collectors.toUnmodifiableMap(ScriptCommandBuilder::backend,
|
||||
Function.identity()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String agentType() {
|
||||
return "local-script";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(TaskType taskType) {
|
||||
return taskType == TaskType.SCRIPT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean enabled() {
|
||||
return properties.getLocalScript().isEnabled() && commandBuilder().available();
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskResult execute(RuntimeExecutionContext context, ValidationTaskSpec task,
|
||||
RuntimeEventSink sink) throws Exception {
|
||||
ScriptCommandBuilder builder = commandBuilder();
|
||||
if (!builder.available()) {
|
||||
throw new IllegalStateException(
|
||||
"script execution backend '" + builder.backend() + "' is not available"
|
||||
+ " (is the docker daemon reachable?)");
|
||||
}
|
||||
String interpreter = resolveInterpreter(context);
|
||||
Path scriptPath = resolveInsideWorkingDirectory(context.workingDirectory(), task.script());
|
||||
String runTag = context.runId() + "-" + task.name();
|
||||
Map<String, String> scrubbedEnvironment = scrubEnvironment(context);
|
||||
|
||||
ScriptCommandBuilder.ScriptExecutionPlan plan = builder.plan(
|
||||
context.workingDirectory(), interpreter, scriptPath,
|
||||
task.safeArgs(), scrubbedEnvironment, runTag);
|
||||
|
||||
sink.toolCall(task.name(), interpreter + " [" + builder.backend() + "]",
|
||||
String.join(" ", task.safeArgs()));
|
||||
|
||||
ProcessBuilder processBuilder = new ProcessBuilder(plan.command());
|
||||
processBuilder.directory(context.workingDirectory().toFile());
|
||||
processBuilder.redirectErrorStream(false);
|
||||
if (plan.environment() != null) {
|
||||
processBuilder.environment().clear();
|
||||
processBuilder.environment().putAll(plan.environment());
|
||||
}
|
||||
|
||||
Process process = processBuilder.start();
|
||||
BoundedBuffer stdout = new BoundedBuffer(MAX_STREAM_BUFFER);
|
||||
BoundedBuffer stderr = new BoundedBuffer(MAX_STREAM_BUFFER);
|
||||
Thread stdoutReader = streamReader(process.getInputStream(), "stdout", task.name(), stdout, sink);
|
||||
Thread stderrReader = streamReader(process.getErrorStream(), "stderr", task.name(), stderr, sink);
|
||||
stdoutReader.start();
|
||||
stderrReader.start();
|
||||
|
||||
Integer exitCode = await(process, task, context, builder, runTag);
|
||||
stdoutReader.join(TimeUnit.SECONDS.toMillis(5));
|
||||
stderrReader.join(TimeUnit.SECONDS.toMillis(5));
|
||||
|
||||
sink.toolResult(task.name(), interpreter,
|
||||
"{\"backend\":\"" + builder.backend() + "\",\"exitCode\":" + exitCode
|
||||
+ ",\"stdoutBytes\":" + stdout.size()
|
||||
+ ",\"stderrBytes\":" + stderr.size() + "}");
|
||||
|
||||
return TaskResult.ofScript(exitCode, stdout.content(), stderr.content(), 1,
|
||||
new WorkingDirectoryArtifacts(context.workingDirectory()));
|
||||
}
|
||||
|
||||
private ScriptCommandBuilder commandBuilder() {
|
||||
String mode = properties.getLocalScript().getExecutionMode()
|
||||
== AuthoringProperties.ScriptExecutionMode.DOCKER ? "docker" : "inline";
|
||||
ScriptCommandBuilder builder = commandBuilders.get(mode);
|
||||
if (builder == null) {
|
||||
throw new IllegalStateException(
|
||||
"no script command builder registered for execution mode: " + mode);
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
|
||||
private String resolveInterpreter(RuntimeExecutionContext context) {
|
||||
String configured = context.configString("interpreter");
|
||||
if (configured != null && !configured.isBlank()) {
|
||||
return configured;
|
||||
}
|
||||
return "sh";
|
||||
}
|
||||
|
||||
private Map<String, String> scrubEnvironment(RuntimeExecutionContext context) {
|
||||
Map<String, String> environment = new java.util.LinkedHashMap<>();
|
||||
environment.put("PATH", System.getenv().getOrDefault("PATH", "/usr/bin:/bin"));
|
||||
environment.put("HOME", context.workingDirectory().toString());
|
||||
environment.put("LANG", "C.UTF-8");
|
||||
Object allowlist = context.safeAdapterConfig().get("envAllowlist");
|
||||
if (allowlist instanceof List<?> entries) {
|
||||
for (Object entry : entries) {
|
||||
if (entry != null) {
|
||||
String value = System.getenv(entry.toString());
|
||||
if (value != null) {
|
||||
environment.put(entry.toString(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for process exit while honoring the task timeout and the cooperative cancel
|
||||
* signal. Returns the exit code, or {@code null} when the process had to be killed;
|
||||
* abnormal exits also trigger the builder's cleanup (the docker backend must
|
||||
* force-remove the container because killing the CLI client does not stop it).
|
||||
*/
|
||||
private Integer await(Process process, ValidationTaskSpec task, RuntimeExecutionContext context,
|
||||
ScriptCommandBuilder builder, String runTag) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + Duration.ofMillis(task.timeoutMs()).toNanos();
|
||||
while (true) {
|
||||
if (process.waitFor(POLL_SLICE_MS, TimeUnit.MILLISECONDS)) {
|
||||
return process.exitValue();
|
||||
}
|
||||
if (context.cancellation() != null && context.cancellation().isCancelRequested()) {
|
||||
process.destroyForcibly();
|
||||
process.waitFor(5, TimeUnit.SECONDS);
|
||||
builder.cleanup(runTag);
|
||||
return null;
|
||||
}
|
||||
if (System.nanoTime() > deadline) {
|
||||
process.destroyForcibly();
|
||||
process.waitFor(5, TimeUnit.SECONDS);
|
||||
builder.cleanup(runTag);
|
||||
throw new IllegalStateException(
|
||||
"task '" + task.name() + "' exceeded its timeout of " + task.timeoutMs() + " ms");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Thread streamReader(java.io.InputStream stream, String streamName, String taskName,
|
||||
BoundedBuffer buffer, RuntimeEventSink sink) {
|
||||
Thread thread = new Thread(() -> {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(stream, StandardCharsets.UTF_8))) {
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
buffer.append(line);
|
||||
sink.log(taskName, streamName, line);
|
||||
}
|
||||
} catch (IOException ignored) {
|
||||
// stream closed when the process dies; buffered content is kept
|
||||
}
|
||||
}, "validation-stream-" + streamName);
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}
|
||||
|
||||
static Path resolveInsideWorkingDirectory(Path workingDirectory, String relativePath) {
|
||||
Path resolved = workingDirectory.resolve(relativePath).normalize();
|
||||
if (!resolved.startsWith(workingDirectory.normalize())) {
|
||||
throw new IllegalArgumentException(
|
||||
"path escapes the working directory: " + relativePath);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Cap-aware single-stream buffer: keeps the head of the output for findings. */
|
||||
private static final class BoundedBuffer {
|
||||
private final StringBuilder builder = new StringBuilder();
|
||||
private final int limit;
|
||||
|
||||
BoundedBuffer(int limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
synchronized void append(String line) {
|
||||
if (builder.length() >= limit) {
|
||||
return;
|
||||
}
|
||||
if (builder.length() > 0) {
|
||||
builder.append('\n');
|
||||
}
|
||||
builder.append(line, 0, Math.min(line.length(), limit - builder.length()));
|
||||
}
|
||||
|
||||
synchronized String content() {
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
synchronized int size() {
|
||||
return builder.length();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package com.iflytek.skillhub.service.authoring.adapter;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Builds the process command for one behavior-task script execution. Two backends
|
||||
* ship: {@code inline} runs the script as a subprocess on the server host (local
|
||||
* development only); {@code docker} runs it inside a locked-down container (no
|
||||
* network, memory/CPU/pids caps, read-only root filesystem) for production.
|
||||
*/
|
||||
public interface ScriptCommandBuilder {
|
||||
|
||||
/** Backend identifier matching {@code skillhub.authoring.local-script.execution-mode}. */
|
||||
String backend();
|
||||
|
||||
/** Whether the backend can execute right now (for docker: daemon reachable). */
|
||||
boolean available();
|
||||
|
||||
/**
|
||||
* Builds the execution plan for one task. {@code scrubbedEnvironment} carries
|
||||
* exactly the variables the script may see; backends decide how to deliver
|
||||
* them (inline: process environment; docker: {@code -e} flags, with the
|
||||
* workspace path remapped to its mount point inside the container).
|
||||
*
|
||||
* @param runTag stable per-task tag (run id + task name), used for container
|
||||
* naming so abnormal exits can be force-removed
|
||||
*/
|
||||
ScriptExecutionPlan plan(Path workspace, String interpreter, Path scriptPath,
|
||||
List<String> args, Map<String, String> scrubbedEnvironment,
|
||||
String runTag);
|
||||
|
||||
/**
|
||||
* Best-effort cleanup after the wrapping process was killed (timeout or
|
||||
* cancel). Killing the {@code docker run} client does not stop the container,
|
||||
* so the docker backend force-removes it here.
|
||||
*/
|
||||
default void cleanup(String runTag) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Command line plus environment. A null environment means the spawned client
|
||||
* (e.g. the docker CLI) needs the parent's environment intact; the isolation
|
||||
* then lives in the container flags instead.
|
||||
*/
|
||||
record ScriptExecutionPlan(List<String> command, Map<String, String> environment) {
|
||||
|
||||
public static ScriptExecutionPlan inherited(List<String> command) {
|
||||
return new ScriptExecutionPlan(command, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package com.iflytek.skillhub.service.authoring.adapter;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.runtime.TaskResult;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Reads artifact files from a run's isolated working directory, rejecting paths that
|
||||
* would escape it. Shared by all runtime adapters.
|
||||
*/
|
||||
record WorkingDirectoryArtifacts(Path workingDirectory) implements TaskResult.ArtifactResolver {
|
||||
|
||||
@Override
|
||||
public Optional<byte[]> read(String relativePath) {
|
||||
try {
|
||||
Path resolved = workingDirectory.resolve(relativePath).normalize();
|
||||
if (!resolved.startsWith(workingDirectory.normalize())) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (!Files.isRegularFile(resolved)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(Files.readAllBytes(resolved));
|
||||
} catch (IllegalArgumentException | IOException exception) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -142,6 +142,21 @@ skillhub:
|
|||
run-timeout-ms: ${SKILLHUB_AUTHORING_RUN_TIMEOUT_MS:900000}
|
||||
# Active runs older than this are swept to TIMED_OUT (crash recovery)
|
||||
stale-run-minutes: ${SKILLHUB_AUTHORING_STALE_RUN_MINUTES:30}
|
||||
maintenance-interval-ms: ${SKILLHUB_AUTHORING_MAINTENANCE_INTERVAL_MS:30000}
|
||||
local-script:
|
||||
enabled: ${SKILLHUB_AUTHORING_LOCAL_SCRIPT_ENABLED:true}
|
||||
# docker is the production default: scripts run in a locked-down container —
|
||||
# no network, memory/cpu/pids caps, read-only rootfs, all capabilities dropped.
|
||||
# inline runs subprocesses on the server host and is only allowed with a
|
||||
# local/dev/test profile (the startup guard fails fast otherwise).
|
||||
execution-mode: ${SKILLHUB_AUTHORING_LOCAL_SCRIPT_MODE:docker}
|
||||
docker:
|
||||
# must provide the interpreters your skills use (alpine ships sh only)
|
||||
image: ${SKILLHUB_AUTHORING_DOCKER_IMAGE:alpine:3.20}
|
||||
memory: ${SKILLHUB_AUTHORING_DOCKER_MEMORY:256m}
|
||||
cpus: ${SKILLHUB_AUTHORING_DOCKER_CPUS:1.0}
|
||||
pids-limit: ${SKILLHUB_AUTHORING_DOCKER_PIDS_LIMIT:128}
|
||||
tmpfs-size: ${SKILLHUB_AUTHORING_DOCKER_TMPFS_SIZE:64m}
|
||||
github-sync:
|
||||
discovery-scheduler-interval-ms: ${SKILLHUB_GITHUB_SYNC_DISCOVERY_SCHEDULER_INTERVAL_MS:60000}
|
||||
candidate-scheduler-interval-ms: ${SKILLHUB_GITHUB_SYNC_CANDIDATE_SCHEDULER_INTERVAL_MS:60000}
|
||||
|
|
|
|||
|
|
@ -392,3 +392,8 @@ error.githubSync.deepDiscovery.exclusionInvalid=Exclusions must be valid GitHub
|
|||
error.githubSync.deepDiscovery.configurationConflict=Official discovery configuration has changed. Reload the latest configuration before saving
|
||||
|
||||
error.githubSync.deepDiscovery.queryInvalid=Invalid deep discovery query parameters.
|
||||
# authoring runtime bindings
|
||||
error.authoring.binding.notFound=No runtime binding configured for draft {0}
|
||||
error.authoring.binding.agentType.unknown=Unknown agent runtime type: {0}
|
||||
error.authoring.binding.invalid=Invalid runtime binding: {0}
|
||||
error.authoring.binding.unserializable=Runtime binding could not be serialized
|
||||
|
|
|
|||
|
|
@ -335,3 +335,8 @@ error.githubSync.deepDiscovery.exclusionInvalid=Исключения должн
|
|||
error.githubSync.deepDiscovery.configurationConflict=Конфигурация официального поиска изменена. Загрузите актуальную конфигурацию перед сохранением
|
||||
|
||||
error.githubSync.deepDiscovery.queryInvalid=Недопустимые параметры запроса глубокого обнаружения.
|
||||
# authoring runtime bindings
|
||||
error.authoring.binding.notFound=Для черновика {0} не настроена привязка среды выполнения
|
||||
error.authoring.binding.agentType.unknown=Неизвестный тип среды выполнения агента: {0}
|
||||
error.authoring.binding.invalid=Недопустимая привязка среды выполнения: {0}
|
||||
error.authoring.binding.unserializable=Привязка среды выполнения не может быть сериализована
|
||||
|
|
|
|||
|
|
@ -394,3 +394,8 @@ error.githubSync.deepDiscovery.exclusionInvalid=排除项须为有效的GitHub
|
|||
error.githubSync.deepDiscovery.configurationConflict=官方探索配置已被修改，请重新读取最新配置后再保存
|
||||
|
||||
error.githubSync.deepDiscovery.queryInvalid=深度探索查询参数无效。
|
||||
# authoring runtime bindings
|
||||
error.authoring.binding.notFound=草稿 {0} 尚未配置运行时绑定
|
||||
error.authoring.binding.agentType.unknown=未知的 Agent 运行时类型：{0}
|
||||
error.authoring.binding.invalid=运行时绑定不合法：{0}
|
||||
error.authoring.binding.unserializable=运行时绑定无法序列化
|
||||
|
|
|
|||
|
|
@ -213,9 +213,13 @@ class AuthoringDraftFlowIntegrationTest {
|
|||
|
||||
assertThat(failed.getStatus()).isEqualTo(ValidationRunStatus.FAILED);
|
||||
List<ValidationFinding> findings = runService.listFindings(broken.getId());
|
||||
assertThat(findings).hasSize(1);
|
||||
ValidationFinding finding = findings.get(0);
|
||||
assertThat(finding.getRuleCode()).isEqualTo("FRONTMATTER_FIELD_MISSING");
|
||||
// the config layer (added with the execution layer) also warns that
|
||||
// validation.yaml is absent — the structure ERROR is what fails the run
|
||||
ValidationFinding finding = findings.stream()
|
||||
.filter(f -> "FRONTMATTER_FIELD_MISSING".equals(f.getRuleCode()))
|
||||
.findFirst().orElseThrow();
|
||||
assertThat(finding.getSeverity()).isEqualTo(
|
||||
com.iflytek.skillhub.domain.authoring.validation.FindingSeverity.ERROR);
|
||||
assertThat(finding.getSuggestion()).isNotNull();
|
||||
assertThat(finding.getSuggestion().safePatches()).hasSize(1);
|
||||
assertThat(runService.listEvents(broken.getId(), null))
|
||||
|
|
@ -233,7 +237,10 @@ class AuthoringDraftFlowIntegrationTest {
|
|||
|
||||
assertThat(succeeded.getStatus()).isEqualTo(ValidationRunStatus.SUCCEEDED);
|
||||
assertThat(succeeded.getErrorCount()).isZero();
|
||||
assertThat(runService.listFindings(repaired.getId())).isEmpty();
|
||||
// warnings (SPEC_MISSING for the absent validation.yaml) do not block
|
||||
assertThat(runService.listFindings(repaired.getId()))
|
||||
.extracting(ValidationFinding::getSeverity)
|
||||
.doesNotContain(com.iflytek.skillhub.domain.authoring.validation.FindingSeverity.ERROR);
|
||||
assertThat(draftService.getDraft(draft.getId()).getValidatedRevision())
|
||||
.isEqualTo(succeeded.getDraftRevision());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
package com.iflytek.skillhub.authoring;
|
||||
|
||||
import com.iflytek.skillhub.TestRedisConfig;
|
||||
import com.iflytek.skillhub.domain.authoring.SkillDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.service.RuntimeBindingService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.SkillDraftService;
|
||||
import com.iflytek.skillhub.domain.authoring.service.ValidationRunService;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationRun;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationRunStatus;
|
||||
import com.iflytek.skillhub.domain.namespace.Namespace;
|
||||
import com.iflytek.skillhub.domain.namespace.NamespaceMember;
|
||||
import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository;
|
||||
import com.iflytek.skillhub.domain.namespace.NamespaceRepository;
|
||||
import com.iflytek.skillhub.domain.namespace.NamespaceRole;
|
||||
import com.iflytek.skillhub.domain.user.UserAccount;
|
||||
import com.iflytek.skillhub.domain.user.UserAccountRepository;
|
||||
import com.iflytek.skillhub.service.authoring.ValidationRunOrchestrator;
|
||||
import com.iflytek.skillhub.storage.ObjectStorageService;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
/**
|
||||
* Execution layer of the authoring platform: three-layer validation with a
|
||||
* local-script binding (structure + config + behavior, the script subprocess
|
||||
* executed inline under the test profile). The MCP probe and the binding
|
||||
* security policy arrive with the MCP+security layer.
|
||||
*/
|
||||
@SpringBootTest
|
||||
@ActiveProfiles("test")
|
||||
@Import({TestRedisConfig.class, AuthoringDraftFlowIntegrationTest.InMemoryStorageConfig.class})
|
||||
@Testcontainers
|
||||
class AuthoringExecutionFlowIntegrationTest {
|
||||
|
||||
@Container
|
||||
private static final PostgreSQLContainer<?> POSTGRES =
|
||||
new PostgreSQLContainer<>("postgres:16-alpine");
|
||||
|
||||
@DynamicPropertySource
|
||||
static void configurePostgres(DynamicPropertyRegistry registry) {
|
||||
registry.add("spring.datasource.url", POSTGRES::getJdbcUrl);
|
||||
registry.add("spring.datasource.username", POSTGRES::getUsername);
|
||||
registry.add("spring.datasource.password", POSTGRES::getPassword);
|
||||
registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver");
|
||||
registry.add("spring.jpa.database-platform", () -> "org.hibernate.dialect.PostgreSQLDialect");
|
||||
registry.add("spring.flyway.enabled", () -> true);
|
||||
registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate");
|
||||
}
|
||||
|
||||
private static final String USER = "author-user-2";
|
||||
private static final String NS_SLUG = "authoring-test-ns";
|
||||
|
||||
@Autowired private SkillDraftService draftService;
|
||||
@Autowired private RuntimeBindingService bindingService;
|
||||
@Autowired private ValidationRunService runService;
|
||||
@Autowired private ValidationRunOrchestrator orchestrator;
|
||||
@Autowired private NamespaceRepository namespaceRepository;
|
||||
@Autowired private NamespaceMemberRepository namespaceMemberRepository;
|
||||
@Autowired private UserAccountRepository userAccountRepository;
|
||||
|
||||
@BeforeEach
|
||||
void setUpNamespace() {
|
||||
if (userAccountRepository.findById(USER).isEmpty()) {
|
||||
userAccountRepository.save(new UserAccount(USER, "Author User", "author@example.com", null));
|
||||
}
|
||||
if (namespaceRepository.findBySlug(NS_SLUG).isEmpty()) {
|
||||
Namespace namespace = new Namespace(NS_SLUG, "Authoring Test", USER);
|
||||
namespace.setStatus(com.iflytek.skillhub.domain.namespace.NamespaceStatus.ACTIVE);
|
||||
Namespace saved = namespaceRepository.save(namespace);
|
||||
if (namespaceMemberRepository.findByNamespaceIdAndUserId(saved.getId(), USER).isEmpty()) {
|
||||
namespaceMemberRepository.save(new NamespaceMember(saved.getId(), USER, NamespaceRole.MEMBER));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void threeLayerValidationExecutesScriptTasks() {
|
||||
SkillDraft draft = draftService.createDraft(NS_SLUG, USER, "runtime-flow",
|
||||
"summarizes documents", null);
|
||||
draftService.saveFile(draft.getId(), USER, "SKILL.md", validSkillMd().getBytes(StandardCharsets.UTF_8),
|
||||
"text/markdown", null, null);
|
||||
draftService.saveFile(draft.getId(), USER, "scripts/check.sh",
|
||||
"echo 'smoke OK'\n".getBytes(StandardCharsets.UTF_8), null, null, null);
|
||||
draftService.saveFile(draft.getId(), USER, "validation.yaml",
|
||||
validationYaml().getBytes(StandardCharsets.UTF_8), null, null, null);
|
||||
|
||||
// ---- bind the local-script runtime
|
||||
bindingService.saveBinding(draft.getId(), USER, "local-script",
|
||||
Map.of("interpreter", "sh"), null, null, null);
|
||||
|
||||
// ---- validate: all three layers run, the script task executes inline
|
||||
ValidationRun run = runService.startRun(draft.getId(), USER, null);
|
||||
orchestrator.submit(run.getId());
|
||||
ValidationRun finished = awaitTerminal(run.getId());
|
||||
|
||||
assertThat(finished.getStatus()).isEqualTo(ValidationRunStatus.SUCCEEDED);
|
||||
assertThat(finished.getErrorCount()).isZero();
|
||||
assertThat(runService.listFindings(run.getId())).isEmpty();
|
||||
assertThat(runService.listEvents(run.getId(), null))
|
||||
.extracting(event -> event.getEventType().name())
|
||||
.contains("RUN_STARTED", "TOOL_CALL", "LOG", "RUN_FINISHED");
|
||||
assertThat(runService.listEvents(run.getId(), null).stream()
|
||||
.map(event -> event.getPayload() == null ? "" : event.getPayload().toString())
|
||||
.filter(payload -> payload.contains("smoke OK"))
|
||||
.count()).isPositive();
|
||||
|
||||
SkillDraft validated = draftService.getDraft(draft.getId());
|
||||
assertThat(validated.isCurrentRevisionValidated()).isTrue();
|
||||
}
|
||||
|
||||
private String validSkillMd() {
|
||||
return "---\nname: runtime-flow\ndescription: summarizes documents\n---\n\n# runtime-flow\n";
|
||||
}
|
||||
|
||||
private String validationYaml() {
|
||||
return """
|
||||
version: 1
|
||||
tasks:
|
||||
- name: smoke
|
||||
description: script prints the greeting
|
||||
type: script
|
||||
script: scripts/check.sh
|
||||
args: []
|
||||
timeoutMs: 10000
|
||||
assertions:
|
||||
- type: exit_code
|
||||
equals: 0
|
||||
- type: stdout_contains
|
||||
value: smoke OK
|
||||
""";
|
||||
}
|
||||
|
||||
private static void assertBindingRejected(Runnable save, String reasonFragment) {
|
||||
try {
|
||||
save.run();
|
||||
fail("binding save should have been rejected: " + reasonFragment);
|
||||
} catch (com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException exception) {
|
||||
assertThat(String.valueOf(exception.messageArgs()[0])).contains(reasonFragment);
|
||||
}
|
||||
}
|
||||
|
||||
private ValidationRun awaitTerminal(Long runId) {
|
||||
long deadline = System.currentTimeMillis() + 60_000;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (runService.getRun(runId).getStatus().isTerminal()) {
|
||||
return runService.getRun(runId);
|
||||
}
|
||||
try {
|
||||
Thread.sleep(200);
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
fail("interrupted while waiting for the validation run to finish");
|
||||
}
|
||||
}
|
||||
fail("validation run did not reach a terminal state in time");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -30,6 +30,10 @@ class ValidationRunOrchestratorTest {
|
|||
private final SkillDraftService drafts = mock(SkillDraftService.class);
|
||||
private final DraftStructureValidator validator = mock(DraftStructureValidator.class);
|
||||
private final ValidationEventBroadcaster broadcaster = mock(ValidationEventBroadcaster.class);
|
||||
private final com.iflytek.skillhub.domain.authoring.service.RuntimeBindingService bindings =
|
||||
mock(com.iflytek.skillhub.domain.authoring.service.RuntimeBindingService.class);
|
||||
private final com.iflytek.skillhub.domain.authoring.service.RuntimeBindingValidator bindingValidator =
|
||||
mock(com.iflytek.skillhub.domain.authoring.service.RuntimeBindingValidator.class);
|
||||
|
||||
@Test
|
||||
void duplicateClaimCannotEmitAnActiveRunAsFinished() {
|
||||
|
|
@ -135,6 +139,7 @@ class ValidationRunOrchestratorTest {
|
|||
ScheduledExecutorService timer, long timeout) {
|
||||
AuthoringProperties properties = new AuthoringProperties();
|
||||
properties.setRunTimeoutMs(timeout);
|
||||
return new ValidationRunOrchestrator(runs, drafts, validator, broadcaster, executor, timer, properties);
|
||||
return new ValidationRunOrchestrator(runs, drafts, bindings, validator, bindingValidator,
|
||||
java.util.List.of(), broadcaster, properties, executor, timer);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,8 +49,11 @@ skillhub:
|
|||
security:
|
||||
scanner:
|
||||
enabled: false
|
||||
authoring:
|
||||
local-script:
|
||||
# tests exercise the inline script runtime directly; no docker needed
|
||||
execution-mode: inline
|
||||
|
||||
logging:
|
||||
level:
|
||||
root: ERROR
|
||||
com.iflytek.skillhub: ERROR
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
package com.iflytek.skillhub.domain.authoring;
|
||||
|
||||
/**
|
||||
* Runtime types a draft can bind for behavior validation. The registry of supported
|
||||
* adapters lives in the application layer; the domain only knows the stable identifiers.
|
||||
*/
|
||||
public enum AgentRuntimeType {
|
||||
|
||||
/** Executes the skill's own scripts in an isolated working directory. */
|
||||
LOCAL_SCRIPT("local-script"),
|
||||
|
||||
/** Sends prompts to an OpenAI-compatible chat completion endpoint with SKILL.md as context. */
|
||||
OPENAI_COMPATIBLE("openai-compatible");
|
||||
|
||||
private final String identifier;
|
||||
|
||||
AgentRuntimeType(String identifier) {
|
||||
this.identifier = identifier;
|
||||
}
|
||||
|
||||
public String identifier() {
|
||||
return identifier;
|
||||
}
|
||||
|
||||
public static AgentRuntimeType fromIdentifier(String identifier) {
|
||||
for (AgentRuntimeType type : values()) {
|
||||
if (type.identifier.equals(identifier)) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown agent runtime type: " + identifier);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
package com.iflytek.skillhub.domain.authoring;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.PreUpdate;
|
||||
import jakarta.persistence.Table;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.hibernate.annotations.JdbcTypeCode;
|
||||
import org.hibernate.type.SqlTypes;
|
||||
|
||||
/**
|
||||
* Per-draft runtime configuration used by behavior validation. Holds the agent type,
|
||||
* adapter-specific configuration, tool allowlist, and MCP server declarations.
|
||||
*
|
||||
* <p>Plain secrets must never be stored here: credentials are referenced by name and
|
||||
* resolved from server-side configuration at validation time. The JSON columns are
|
||||
* mapped as structured collections (not Strings) so the stored documents stay real
|
||||
* JSON objects and Hibernate's read path is symmetric with its write path.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "runtime_binding",
|
||||
uniqueConstraints = @jakarta.persistence.UniqueConstraint(
|
||||
name = "uq_runtime_binding_draft", columnNames = "draft_id"))
|
||||
public class RuntimeBinding {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(name = "draft_id", nullable = false)
|
||||
private Long draftId;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "agent_type", nullable = false, length = 32)
|
||||
private AgentRuntimeType agentType;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "config")
|
||||
private Map<String, Object> config;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "tool_allowlist")
|
||||
private List<String> toolAllowlist;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "mcp_servers")
|
||||
private List<Map<String, Object>> mcpServers;
|
||||
|
||||
@Column(name = "updated_by", length = 128)
|
||||
private String updatedBy;
|
||||
|
||||
@Column(name = "updated_at", nullable = false)
|
||||
private Instant updatedAt;
|
||||
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private Instant createdAt;
|
||||
|
||||
protected RuntimeBinding() {
|
||||
}
|
||||
|
||||
public RuntimeBinding(Long draftId, AgentRuntimeType agentType) {
|
||||
this.draftId = draftId;
|
||||
this.agentType = agentType;
|
||||
}
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
Instant now = Instant.now(Clock.systemUTC());
|
||||
createdAt = now;
|
||||
updatedAt = now;
|
||||
}
|
||||
|
||||
@PreUpdate
|
||||
protected void onUpdate() {
|
||||
updatedAt = Instant.now(Clock.systemUTC());
|
||||
}
|
||||
|
||||
public void update(AgentRuntimeType agentType, Map<String, Object> config,
|
||||
List<String> toolAllowlist, List<Map<String, Object>> mcpServers,
|
||||
String updatedBy) {
|
||||
this.agentType = agentType;
|
||||
this.config = config;
|
||||
this.toolAllowlist = toolAllowlist;
|
||||
this.mcpServers = mcpServers;
|
||||
this.updatedBy = updatedBy;
|
||||
}
|
||||
|
||||
// Getters
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public Long getDraftId() {
|
||||
return draftId;
|
||||
}
|
||||
|
||||
public AgentRuntimeType getAgentType() {
|
||||
return agentType;
|
||||
}
|
||||
|
||||
public Map<String, Object> getConfig() {
|
||||
return config;
|
||||
}
|
||||
|
||||
public List<String> getToolAllowlist() {
|
||||
return toolAllowlist;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> getMcpServers() {
|
||||
return mcpServers;
|
||||
}
|
||||
|
||||
public String getUpdatedBy() {
|
||||
return updatedBy;
|
||||
}
|
||||
|
||||
public Instant getUpdatedAt() {
|
||||
return updatedAt;
|
||||
}
|
||||
|
||||
public Instant getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package com.iflytek.skillhub.domain.authoring;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Domain repository contract for per-draft runtime bindings.
|
||||
*/
|
||||
public interface RuntimeBindingRepository {
|
||||
|
||||
RuntimeBinding save(RuntimeBinding binding);
|
||||
|
||||
Optional<RuntimeBinding> findByDraftId(Long draftId);
|
||||
|
||||
void deleteByDraftId(Long draftId);
|
||||
}
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.AssertionSpec;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Evaluates rule-based assertions against a task result. Deterministic checks only:
|
||||
* exit codes, regex/substring matching, JSON pointer equality, artifact presence and
|
||||
* hashes, and tool call counts. Semantic quality checks are intentionally not
|
||||
* approximated here — they belong to model-assisted review, which stays outside the
|
||||
* minimal closed loop.
|
||||
*/
|
||||
public class AssertionEvaluator {
|
||||
|
||||
/** Assertion types valid for SCRIPT tasks. */
|
||||
public static final List<String> SCRIPT_ASSERTION_TYPES = List.of(
|
||||
"exit_code", "stdout_matches", "stdout_contains", "stdout_json",
|
||||
"artifact_exists", "tool_call_count");
|
||||
|
||||
/** Assertion types valid for PROMPT tasks. */
|
||||
public static final List<String> PROMPT_ASSERTION_TYPES = List.of(
|
||||
"response_matches", "response_contains", "response_json", "tool_call_count");
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
/**
|
||||
* Validates assertion types and required parameters without a result. Used by the
|
||||
* configuration layer so malformed assertions fail fast, before any execution.
|
||||
*/
|
||||
public List<String> validateSyntax(List<AssertionSpec> assertions, boolean promptTask) {
|
||||
List<String> problems = new ArrayList<>();
|
||||
List<String> allowed = promptTask ? PROMPT_ASSERTION_TYPES : SCRIPT_ASSERTION_TYPES;
|
||||
for (AssertionSpec assertion : assertions) {
|
||||
String type = assertion.type();
|
||||
if (type == null || type.isBlank()) {
|
||||
problems.add("assertion without a type");
|
||||
continue;
|
||||
}
|
||||
if (!allowed.contains(type)) {
|
||||
problems.add("assertion type '" + type + "' is not valid for "
|
||||
+ (promptTask ? "prompt" : "script") + " tasks (allowed: "
|
||||
+ String.join(", ", allowed) + ")");
|
||||
continue;
|
||||
}
|
||||
switch (type) {
|
||||
case "exit_code" -> {
|
||||
if (assertion.paramInt("equals") == null) {
|
||||
problems.add("exit_code assertion requires an integer 'equals'");
|
||||
}
|
||||
}
|
||||
case "stdout_matches", "response_matches" -> {
|
||||
String pattern = assertion.paramString("pattern");
|
||||
if (pattern == null || pattern.isBlank()) {
|
||||
problems.add(type + " assertion requires a 'pattern'");
|
||||
} else if (!compiles(pattern)) {
|
||||
problems.add(type + " assertion has an invalid regex: " + pattern);
|
||||
}
|
||||
}
|
||||
case "stdout_contains", "response_contains" -> {
|
||||
if (assertion.paramString("value") == null) {
|
||||
problems.add(type + " assertion requires a 'value'");
|
||||
}
|
||||
}
|
||||
case "stdout_json", "response_json" -> {
|
||||
if (assertion.paramString("pointer") == null) {
|
||||
problems.add(type + " assertion requires a 'pointer'");
|
||||
}
|
||||
if (assertion.paramString("equals") == null && assertion.paramInt("equals") == null) {
|
||||
problems.add(type + " assertion requires an 'equals' value");
|
||||
}
|
||||
}
|
||||
case "artifact_exists" -> {
|
||||
if (assertion.paramString("path") == null || assertion.paramString("path").isBlank()) {
|
||||
problems.add("artifact_exists assertion requires a 'path'");
|
||||
}
|
||||
}
|
||||
case "tool_call_count" -> {
|
||||
Integer min = assertion.paramInt("min");
|
||||
Integer max = assertion.paramInt("max");
|
||||
if (min == null && max == null) {
|
||||
problems.add("tool_call_count assertion requires 'min' and/or 'max'");
|
||||
}
|
||||
}
|
||||
default -> problems.add("unknown assertion type: " + type);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Evaluates all assertions against the task result; empty list means all passed. */
|
||||
public List<AssertionFailure> evaluate(List<AssertionSpec> assertions, TaskResult result) {
|
||||
List<AssertionFailure> failures = new ArrayList<>();
|
||||
for (AssertionSpec assertion : assertions) {
|
||||
String reason = evaluateOne(assertion, result);
|
||||
if (reason != null) {
|
||||
failures.add(new AssertionFailure(assertion, reason));
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
private String evaluateOne(AssertionSpec assertion, TaskResult result) {
|
||||
String type = assertion.type();
|
||||
return switch (type == null ? "" : type) {
|
||||
case "exit_code" -> {
|
||||
Integer expected = assertion.paramInt("equals");
|
||||
if (result.exitCode() == null) {
|
||||
yield "task did not run to completion, no exit code";
|
||||
}
|
||||
if (expected == null || result.exitCode() != expected) {
|
||||
yield "expected exit code " + expected + " but got " + result.exitCode();
|
||||
}
|
||||
yield null;
|
||||
}
|
||||
case "stdout_matches" -> matchPattern(assertion.paramString("pattern"), result.stdout(), "stdout");
|
||||
case "stdout_contains" -> containsValue(assertion.paramString("value"), result.stdout(), "stdout");
|
||||
case "response_matches" -> matchPattern(assertion.paramString("pattern"), result.response(), "response");
|
||||
case "response_contains" -> containsValue(assertion.paramString("value"), result.response(), "response");
|
||||
case "stdout_json" -> jsonPointer(assertion, result.stdout(), "stdout");
|
||||
case "response_json" -> jsonPointer(assertion, result.response(), "response");
|
||||
case "artifact_exists" -> artifactExists(assertion, result);
|
||||
case "tool_call_count" -> toolCallCount(assertion, result);
|
||||
default -> "unknown assertion type: " + type;
|
||||
};
|
||||
}
|
||||
|
||||
private String matchPattern(String patternText, String actual, String field) {
|
||||
if (patternText == null) {
|
||||
return "pattern is missing";
|
||||
}
|
||||
if (actual == null) {
|
||||
return field + " is empty";
|
||||
}
|
||||
if (!Pattern.compile(patternText).matcher(actual).find()) {
|
||||
return field + " does not match pattern: " + patternText;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String containsValue(String value, String actual, String field) {
|
||||
if (value == null) {
|
||||
return "value is missing";
|
||||
}
|
||||
if (actual == null || !actual.contains(value)) {
|
||||
return field + " does not contain: " + value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String jsonPointer(AssertionSpec assertion, String actual, String field) {
|
||||
String pointer = assertion.paramString("pointer");
|
||||
String expected = assertion.paramString("equals");
|
||||
if (pointer == null || expected == null) {
|
||||
return "pointer or equals is missing";
|
||||
}
|
||||
if (actual == null || actual.isBlank()) {
|
||||
return field + " is empty";
|
||||
}
|
||||
JsonNode root;
|
||||
try {
|
||||
root = OBJECT_MAPPER.readTree(actual);
|
||||
} catch (Exception exception) {
|
||||
return field + " is not valid JSON";
|
||||
}
|
||||
JsonNode node = root.at(pointer);
|
||||
if (node.isMissingNode()) {
|
||||
return field + " has no value at pointer " + pointer;
|
||||
}
|
||||
String actualText = node.isTextual() ? node.asText() : node.toString();
|
||||
if (!expected.equals(actualText)) {
|
||||
return "value at " + pointer + " is " + actualText + ", expected " + expected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String artifactExists(AssertionSpec assertion, TaskResult result) {
|
||||
String path = assertion.paramString("path");
|
||||
if (path == null) {
|
||||
return "path is missing";
|
||||
}
|
||||
byte[] content = result.artifacts().read(path).orElse(null);
|
||||
if (content == null) {
|
||||
return "artifact not found: " + path;
|
||||
}
|
||||
String expectedSha = assertion.paramString("sha256");
|
||||
if (expectedSha != null && !expectedSha.isBlank()) {
|
||||
String actualSha = TaskResult.sha256Hex(content);
|
||||
if (!expectedSha.toLowerCase().equals(actualSha)) {
|
||||
return "artifact " + path + " sha256 is " + actualSha + ", expected " + expectedSha;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String toolCallCount(AssertionSpec assertion, TaskResult result) {
|
||||
Integer min = assertion.paramInt("min");
|
||||
Integer max = assertion.paramInt("max");
|
||||
int actual = result.toolCallCount();
|
||||
if (min != null && actual < min) {
|
||||
return "tool call count " + actual + " is below minimum " + min;
|
||||
}
|
||||
if (max != null && actual > max) {
|
||||
return "tool call count " + actual + " exceeds maximum " + max;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean compiles(String pattern) {
|
||||
try {
|
||||
Pattern.compile(pattern);
|
||||
return true;
|
||||
} catch (PatternSyntaxException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.spec.AssertionSpec;
|
||||
|
||||
/**
|
||||
* One assertion that did not hold, with a human-readable reason.
|
||||
*/
|
||||
public record AssertionFailure(AssertionSpec assertion, String reason) {
|
||||
|
||||
public String describe() {
|
||||
return "assertion '" + assertion.type() + "' failed: " + reason;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
/**
|
||||
* Callbacks a runtime adapter uses to stream execution progress into the persisted
|
||||
* run event log. The orchestrator assigns sequence numbers and broadcasts to SSE
|
||||
* subscribers; adapters only report what happened.
|
||||
*/
|
||||
public interface RuntimeEventSink {
|
||||
|
||||
/** An agent-level message (model response, task narration). */
|
||||
void agentMessage(String taskName, String content);
|
||||
|
||||
/** A tool invocation initiated by the runtime (script execution, HTTP tool call). */
|
||||
void toolCall(String taskName, String tool, String argumentsJson);
|
||||
|
||||
/** The outcome of a tool invocation. */
|
||||
void toolResult(String taskName, String tool, String summaryJson);
|
||||
|
||||
/** A raw log line from the runtime (stdout/stderr of a script, transport log). */
|
||||
void log(String taskName, String stream, String line);
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Everything a runtime adapter needs to execute one validation run: the isolated
|
||||
* working directory containing the draft files, the adapter-specific configuration,
|
||||
* the tool allowlist, the declared MCP servers, and the cancellation signal.
|
||||
*
|
||||
* @param runId validation run identifier, for logging
|
||||
* @param workingDirectory isolated directory holding the draft's files (read-write)
|
||||
* @param adapterConfig adapter-specific configuration map from the runtime binding
|
||||
* @param toolAllowlist allowed tool identifiers; empty means unrestricted
|
||||
* @param mcpServers MCP server declarations from the runtime binding (may be empty)
|
||||
* @param cancellation cooperative cancel signal
|
||||
*/
|
||||
public record RuntimeExecutionContext(
|
||||
Long runId,
|
||||
Path workingDirectory,
|
||||
Map<String, Object> adapterConfig,
|
||||
List<String> toolAllowlist,
|
||||
List<Map<String, Object>> mcpServers,
|
||||
TaskCancellation cancellation
|
||||
) {
|
||||
|
||||
/** Convenience constructor for runtimes that do not consume MCP servers. */
|
||||
public RuntimeExecutionContext(Long runId, Path workingDirectory, Map<String, Object> adapterConfig,
|
||||
List<String> toolAllowlist, TaskCancellation cancellation) {
|
||||
this(runId, workingDirectory, adapterConfig, toolAllowlist, List.of(), cancellation);
|
||||
}
|
||||
|
||||
public Map<String, Object> safeAdapterConfig() {
|
||||
return adapterConfig == null ? Map.of() : adapterConfig;
|
||||
}
|
||||
|
||||
public List<String> safeToolAllowlist() {
|
||||
return toolAllowlist == null ? List.of() : List.copyOf(toolAllowlist);
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> safeMcpServers() {
|
||||
return mcpServers == null ? List.of() : List.copyOf(mcpServers);
|
||||
}
|
||||
|
||||
public String configString(String key) {
|
||||
Object value = safeAdapterConfig().get(key);
|
||||
return value == null ? null : value.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.spec.TaskType;
|
||||
import com.iflytek.skillhub.domain.authoring.spec.ValidationTaskSpec;
|
||||
|
||||
/**
|
||||
* SPI implemented by every agent runtime the authoring platform can bind to a draft.
|
||||
*
|
||||
* <p>Implementations live in the application layer (process execution, HTTP clients)
|
||||
* and are selected by the runtime binding's agent type. The contract is deliberately
|
||||
* minimal: execute one task, stream progress through the sink, return or throw.
|
||||
* Assertion evaluation is handled by the orchestrator via {@link TaskResult}, not by
|
||||
* adapters.
|
||||
*/
|
||||
public interface SkillRuntimeAdapter {
|
||||
|
||||
/** The agent type identifier this adapter serves (see {@code AgentRuntimeType}). */
|
||||
String agentType();
|
||||
|
||||
/** Whether the adapter can execute the given task kind. */
|
||||
boolean supports(TaskType taskType);
|
||||
|
||||
/**
|
||||
* Whether the adapter is currently usable. Disabled adapters (missing server-side
|
||||
* configuration, feature flags) are reported as RUNTIME_DISABLED findings instead of
|
||||
* failed tasks when the orchestrator selects them.
|
||||
*/
|
||||
default boolean enabled() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes one validation task. Implementations must:
|
||||
* <ul>
|
||||
* <li>honor {@code context.cancellation()} while running;</li>
|
||||
* <li>enforce the task timeout themselves;</li>
|
||||
* <li>report progress through {@code sink};</li>
|
||||
* <li>never modify draft state — the working directory is a disposable copy.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @return the raw task outcome for assertion evaluation
|
||||
* @throws InterruptedException when cancelled mid-execution
|
||||
*/
|
||||
TaskResult execute(RuntimeExecutionContext context, ValidationTaskSpec task,
|
||||
RuntimeEventSink sink) throws Exception;
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
/**
|
||||
* Cooperative cancellation signal handed to runtime adapters. Adapters should poll
|
||||
* {@link #isCancelRequested()} while executing (at least between output chunks) and
|
||||
* abort as quickly as practical when it flips.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface TaskCancellation {
|
||||
|
||||
boolean isCancelRequested();
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* The raw outcome of one behavior validation task, consumed by the assertion
|
||||
* evaluator. Script tasks populate exit code and streams; prompt tasks populate
|
||||
* the agent response. The artifact resolver reads files produced inside the
|
||||
* working directory (used by artifact_exists assertions).
|
||||
*/
|
||||
public record TaskResult(
|
||||
Integer exitCode,
|
||||
String stdout,
|
||||
String stderr,
|
||||
String response,
|
||||
int toolCallCount,
|
||||
ArtifactResolver artifacts
|
||||
) {
|
||||
|
||||
/** Reads a file produced by the task, relative to the working directory. */
|
||||
public interface ArtifactResolver {
|
||||
Optional<byte[]> read(String relativePath);
|
||||
}
|
||||
|
||||
public static TaskResult ofScript(int exitCode, String stdout, String stderr,
|
||||
int toolCallCount, ArtifactResolver artifacts) {
|
||||
return new TaskResult(exitCode, stdout, stderr, null, toolCallCount, artifacts);
|
||||
}
|
||||
|
||||
public static TaskResult ofPrompt(String response, int toolCallCount, ArtifactResolver artifacts) {
|
||||
return new TaskResult(null, null, null, response, toolCallCount, artifacts);
|
||||
}
|
||||
|
||||
public String sha256OfArtifact(String relativePath) {
|
||||
return artifacts.read(relativePath)
|
||||
.map(TaskResult::sha256Hex)
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
public static String sha256Hex(byte[] content) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
return HexFormat.of().formatHex(digest.digest(content));
|
||||
} catch (NoSuchAlgorithmException exception) {
|
||||
throw new IllegalStateException("SHA-256 unavailable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
public static String sha256Hex(String content) {
|
||||
return sha256Hex(content.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
/**
|
||||
* Runtime adapter SPI and behavior-validation support types shared by the domain and
|
||||
* application layers.
|
||||
*/
|
||||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package com.iflytek.skillhub.domain.authoring.service;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.AgentRuntimeType;
|
||||
import com.iflytek.skillhub.domain.authoring.RuntimeBinding;
|
||||
import com.iflytek.skillhub.domain.authoring.RuntimeBindingRepository;
|
||||
import com.iflytek.skillhub.domain.authoring.SkillDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.SkillDraftRepository;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.FindingDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.FindingSeverity;
|
||||
import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException;
|
||||
import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException;
|
||||
import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* Domain service for a draft's runtime binding: persists the agent type, adapter
|
||||
* configuration, tool allowlist, and MCP server declarations. Validation findings are
|
||||
* the responsibility of {@link RuntimeBindingValidator}; this service only rejects
|
||||
* structurally invalid input at save time.
|
||||
*/
|
||||
@Service
|
||||
public class RuntimeBindingService {
|
||||
|
||||
private static final String SUPER_ADMIN = "SUPER_ADMIN";
|
||||
|
||||
private final RuntimeBindingRepository bindingRepository;
|
||||
private final SkillDraftRepository draftRepository;
|
||||
private final RuntimeBindingValidator bindingValidator;
|
||||
|
||||
public RuntimeBindingService(RuntimeBindingRepository bindingRepository,
|
||||
SkillDraftRepository draftRepository,
|
||||
RuntimeBindingValidator bindingValidator) {
|
||||
this.bindingRepository = bindingRepository;
|
||||
this.draftRepository = draftRepository;
|
||||
this.bindingValidator = bindingValidator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the runtime binding. Structurally invalid input (unknown agent type,
|
||||
* embedded credentials) is rejected immediately; the full validation report is
|
||||
* produced again as findings whenever a run starts.
|
||||
*/
|
||||
@Transactional
|
||||
public RuntimeBinding saveBinding(Long draftId, String userId, String agentType,
|
||||
Map<String, Object> adapterConfig,
|
||||
List<String> toolAllowlist,
|
||||
List<Map<String, Object>> mcpServers,
|
||||
Set<String> platformRoles) {
|
||||
SkillDraft draft = draftRepository.findById(draftId)
|
||||
.orElseThrow(() -> new DomainNotFoundException("error.authoring.draft.notFound", draftId));
|
||||
assertOwner(draft, userId, platformRoles);
|
||||
|
||||
AgentRuntimeType type;
|
||||
try {
|
||||
type = AgentRuntimeType.fromIdentifier(agentType == null ? "" : agentType);
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new DomainBadRequestException("error.authoring.binding.agentType.unknown", agentType);
|
||||
}
|
||||
|
||||
List<FindingDraft> findings = bindingValidator.validate(
|
||||
type.identifier(), adapterConfig, toolAllowlist, mcpServers);
|
||||
boolean hasErrors = findings.stream()
|
||||
.anyMatch(finding -> finding.severity() == FindingSeverity.ERROR);
|
||||
if (hasErrors) {
|
||||
String first = findings.stream()
|
||||
.filter(finding -> finding.severity() == FindingSeverity.ERROR)
|
||||
.findFirst().map(FindingDraft::message).orElse("invalid binding");
|
||||
throw new DomainBadRequestException("error.authoring.binding.invalid", first);
|
||||
}
|
||||
|
||||
RuntimeBinding binding = bindingRepository.findByDraftId(draftId)
|
||||
.orElseGet(() -> new RuntimeBinding(draftId, type));
|
||||
binding.update(type, adapterConfig, toolAllowlist, mcpServers, userId);
|
||||
return bindingRepository.save(binding);
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public RuntimeBinding getBinding(Long draftId) {
|
||||
return bindingRepository.findByDraftId(draftId)
|
||||
.orElseThrow(() -> new DomainNotFoundException(
|
||||
"error.authoring.binding.notFound", draftId));
|
||||
}
|
||||
|
||||
/** Optional accessor for callers that treat "not configured yet" as normal state. */
|
||||
@Transactional(readOnly = true)
|
||||
public java.util.Optional<RuntimeBinding> findBinding(Long draftId) {
|
||||
return bindingRepository.findByDraftId(draftId);
|
||||
}
|
||||
|
||||
private void assertOwner(SkillDraft draft, String userId, Set<String> platformRoles) {
|
||||
if (draft.getOwnerId().equals(userId)
|
||||
|| (platformRoles != null && platformRoles.contains(SUPER_ADMIN))) {
|
||||
return;
|
||||
}
|
||||
throw new DomainForbiddenException("error.authoring.draft.forbidden", draft.getId());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
package com.iflytek.skillhub.domain.authoring.service;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.validation.FindingDraft;
|
||||
import com.iflytek.skillhub.domain.authoring.validation.ValidationLayer;
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* The CONFIG validation layer: checks the runtime binding (agent type, adapter
|
||||
* configuration, tool allowlist, MCP server declarations) plus the syntactic validity
|
||||
* of validation task assertions. Rejects embedded plaintext credentials — bindings may
|
||||
* only reference server-side credential names. Outbound endpoints, the stdio
|
||||
* transport, and environment references are additionally gated by the
|
||||
* Structural checks only in this layer; the SSRF/stdio/envRef policy that
|
||||
* guards external endpoints arrives with the MCP+security layer.
|
||||
*/
|
||||
@Service
|
||||
public class RuntimeBindingValidator {
|
||||
|
||||
private static final Pattern TOOL_NAME = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$");
|
||||
private static final Pattern MCP_NAME = Pattern.compile("^[a-z0-9][a-z0-9-]{0,63}$");
|
||||
private static final Pattern CREDENTIAL_KEY = Pattern.compile(
|
||||
"(?i).*(secret|password|passwd|api[_-]?key|token|credential).*");
|
||||
private static final Pattern ENV_REF_NAME = Pattern.compile("^[A-Z][A-Z0-9_]{0,63}$");
|
||||
private static final List<String> TRANSPORTS = List.of("http", "sse", "stdio");
|
||||
private static final List<String> INTERPRETERS = List.of("sh", "bash", "python3", "node");
|
||||
|
||||
public RuntimeBindingValidator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param agentType agent type identifier (may be invalid — reported as finding)
|
||||
* @param adapterConfig adapter-specific configuration map
|
||||
* @param toolAllowlist tool identifiers (may be null = unrestricted)
|
||||
* @param mcpServers MCP server declarations (may be null)
|
||||
*/
|
||||
public List<FindingDraft> validate(String agentType, Map<String, Object> adapterConfig,
|
||||
List<String> toolAllowlist, List<Map<String, Object>> mcpServers) {
|
||||
List<FindingDraft> findings = new ArrayList<>();
|
||||
String resolvedAgentType = agentType == null ? "" : agentType;
|
||||
|
||||
switch (resolvedAgentType) {
|
||||
case "local-script" -> validateLocalScriptConfig(adapterConfig, findings);
|
||||
case "openai-compatible" -> validateOpenAiCompatibleConfig(adapterConfig, findings);
|
||||
default -> findings.add(FindingDraft.error(ValidationLayer.CONFIG, "AGENT_TYPE_UNKNOWN",
|
||||
"Unknown agent runtime type: " + resolvedAgentType
|
||||
+ " (supported: local-script, openai-compatible)"));
|
||||
}
|
||||
|
||||
if (toolAllowlist != null) {
|
||||
for (String tool : toolAllowlist) {
|
||||
if (tool == null || !TOOL_NAME.matcher(tool).matches()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "TOOL_ALLOWLIST_INVALID",
|
||||
"Invalid tool allowlist entry: " + tool));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mcpServers != null) {
|
||||
for (Map<String, Object> server : mcpServers) {
|
||||
validateMcpServer(server, findings);
|
||||
}
|
||||
}
|
||||
|
||||
scanForEmbeddedSecrets(agentType, adapterConfig, toolAllowlist, mcpServers, findings);
|
||||
return findings;
|
||||
}
|
||||
|
||||
private void validateLocalScriptConfig(Map<String, Object> config, List<FindingDraft> findings) {
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
Object interpreter = config.get("interpreter");
|
||||
if (interpreter != null && !INTERPRETERS.contains(interpreter.toString())) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "INTERPRETER_UNSUPPORTED",
|
||||
"Unsupported interpreter: " + interpreter + " (supported: "
|
||||
+ String.join(", ", INTERPRETERS) + ")"));
|
||||
}
|
||||
Object envAllowlist = config.get("envAllowlist");
|
||||
if (envAllowlist instanceof List<?> entries) {
|
||||
for (Object entry : entries) {
|
||||
if (entry == null || !ENV_REF_NAME.matcher(entry.toString()).matches()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "ENV_ALLOWLIST_INVALID",
|
||||
"Invalid environment variable name: " + entry));
|
||||
}
|
||||
}
|
||||
} else if (envAllowlist != null) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "ENV_ALLOWLIST_INVALID",
|
||||
"envAllowlist must be a list of variable names"));
|
||||
}
|
||||
}
|
||||
|
||||
private void validateOpenAiCompatibleConfig(Map<String, Object> config, List<FindingDraft> findings) {
|
||||
if (config == null) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "LLM_CONFIG_MISSING",
|
||||
"openai-compatible runtime requires a config object with endpoint and model"));
|
||||
return;
|
||||
}
|
||||
Object endpoint = config.get("endpoint");
|
||||
if (endpoint == null || endpoint.toString().isBlank()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "LLM_ENDPOINT_MISSING",
|
||||
"openai-compatible runtime requires an 'endpoint' URL"));
|
||||
} else if (!isHttpUrl(endpoint.toString())) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "LLM_ENDPOINT_INVALID",
|
||||
"endpoint must be an http(s) URL: " + endpoint));
|
||||
}
|
||||
Object model = config.get("model");
|
||||
if (model == null || model.toString().isBlank()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "LLM_MODEL_MISSING",
|
||||
"openai-compatible runtime requires a 'model' name"));
|
||||
}
|
||||
Object temperature = config.get("temperature");
|
||||
if (temperature instanceof Number number
|
||||
&& (number.doubleValue() < 0 || number.doubleValue() > 2)) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "LLM_TEMPERATURE_INVALID",
|
||||
"temperature must be between 0 and 2"));
|
||||
}
|
||||
// the API key is resolved from server configuration, never from the binding
|
||||
if (config.containsKey("apiKey") || config.containsKey("api_key")) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "CREDENTIAL_EMBEDDED",
|
||||
"apiKey must not be stored in the binding; configure it server-side"));
|
||||
}
|
||||
}
|
||||
|
||||
private void validateMcpServer(Map<String, Object> server, List<FindingDraft> findings) {
|
||||
Object name = server.get("name");
|
||||
if (name == null || !MCP_NAME.matcher(name.toString()).matches()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_NAME_INVALID",
|
||||
"MCP server name must be lowercase alphanumerics/hyphens: " + name));
|
||||
}
|
||||
Object transport = server.get("transport");
|
||||
if (transport == null || !TRANSPORTS.contains(transport.toString())) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_TRANSPORT_INVALID",
|
||||
"MCP server " + name + " has unknown transport: " + transport
|
||||
+ " (supported: " + String.join(", ", TRANSPORTS) + ")"));
|
||||
} else if ("http".equals(transport.toString()) || "sse".equals(transport.toString())) {
|
||||
Object endpoint = server.get("endpoint");
|
||||
if (endpoint == null || !isHttpUrl(endpoint.toString())) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_ENDPOINT_INVALID",
|
||||
"MCP server " + name + " requires an http(s) endpoint"));
|
||||
}
|
||||
} else {
|
||||
// stdio servers spawn host-side processes; the enable switch and the
|
||||
// docker isolation that gate them arrive with the security layer
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_STDIO_DISABLED",
|
||||
"MCP server " + name + " uses the stdio transport, which is not"
|
||||
+ " available yet; use http/sse instead"));
|
||||
Object command = server.get("command");
|
||||
if (command == null || command.toString().isBlank()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_COMMAND_MISSING",
|
||||
"MCP server " + name + " (stdio) requires a command"));
|
||||
}
|
||||
}
|
||||
Object toolFilters = server.get("toolFilters");
|
||||
if (toolFilters instanceof List<?> filters) {
|
||||
for (Object filter : filters) {
|
||||
if (filter == null || !TOOL_NAME.matcher(filter.toString()).matches()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_TOOL_FILTER_INVALID",
|
||||
"MCP server " + name + " has an invalid tool filter: " + filter));
|
||||
}
|
||||
}
|
||||
}
|
||||
Object envRefs = server.get("envRefs");
|
||||
if (envRefs instanceof List<?> refs) {
|
||||
for (Object ref : refs) {
|
||||
if (ref == null || !ENV_REF_NAME.matcher(ref.toString()).matches()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "MCP_ENV_REF_INVALID",
|
||||
"MCP server " + name + " has an invalid env reference: " + ref));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects secret-looking keys anywhere in the binding payload. */
|
||||
private void scanForEmbeddedSecrets(String agentType, Map<String, Object> adapterConfig,
|
||||
List<String> toolAllowlist,
|
||||
List<Map<String, Object>> mcpServers,
|
||||
List<FindingDraft> findings) {
|
||||
if (adapterConfig != null) {
|
||||
scanMap(adapterConfig, findings);
|
||||
}
|
||||
if (mcpServers != null) {
|
||||
for (Map<String, Object> server : mcpServers) {
|
||||
scanMap(server, findings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void scanMap(Map<String, Object> map, List<FindingDraft> findings) {
|
||||
for (Map.Entry<String, Object> entry : map.entrySet()) {
|
||||
if (CREDENTIAL_KEY.matcher(entry.getKey()).matches()
|
||||
&& entry.getValue() != null && !entry.getValue().toString().isBlank()) {
|
||||
findings.add(FindingDraft.error(ValidationLayer.CONFIG, "CREDENTIAL_EMBEDDED",
|
||||
"Binding must not embed credentials in field '" + entry.getKey()
|
||||
+ "'; reference server-side credentials instead"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isHttpUrl(String value) {
|
||||
try {
|
||||
URI uri = URI.create(value);
|
||||
String scheme = uri.getScheme();
|
||||
return ("http".equalsIgnoreCase(scheme) || "https".equalsIgnoreCase(scheme))
|
||||
&& uri.getHost() != null;
|
||||
} catch (IllegalArgumentException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* One assertion inside a validation task. Assertions are deliberately loosely typed:
|
||||
* {@code type} selects the check and {@code params} carries its typed parameters,
|
||||
* which the evaluator validates and coerces. Unknown types or missing parameters are
|
||||
* reported as configuration-layer findings before behavior execution starts.
|
||||
*/
|
||||
public record AssertionSpec(String type, Map<String, Object> params) {
|
||||
|
||||
public Map<String, Object> safeParams() {
|
||||
return params == null ? Map.of() : params;
|
||||
}
|
||||
|
||||
public String paramString(String key) {
|
||||
Object value = safeParams().get(key);
|
||||
return value == null ? null : value.toString();
|
||||
}
|
||||
|
||||
public Integer paramInt(String key) {
|
||||
Object value = safeParams().get(key);
|
||||
if (value instanceof Number number) {
|
||||
return number.intValue();
|
||||
}
|
||||
if (value instanceof String text && !text.isBlank()) {
|
||||
try {
|
||||
return Integer.parseInt(text.trim());
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
/**
|
||||
* Task kinds supported by behavior validation.
|
||||
*/
|
||||
public enum TaskType {
|
||||
|
||||
/** Executes a script shipped with the skill in the isolated working directory. */
|
||||
SCRIPT,
|
||||
|
||||
/** Sends a prompt to the bound agent runtime with the skill as context. */
|
||||
PROMPT
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Parsed content of a draft's {@code validation.yaml}: the ordered list of behavior
|
||||
* validation tasks executed by the BEHAVIOR layer. A draft without the file simply
|
||||
* runs zero behavior tasks.
|
||||
*/
|
||||
public record ValidationSpec(int version, List<ValidationTaskSpec> tasks) {
|
||||
|
||||
public static final String FILE_PATH = "validation.yaml";
|
||||
public static final int CURRENT_VERSION = 1;
|
||||
public static final int DEFAULT_TASK_TIMEOUT_MS = 60_000;
|
||||
|
||||
public static ValidationSpec empty() {
|
||||
return new ValidationSpec(CURRENT_VERSION, List.of());
|
||||
}
|
||||
|
||||
public List<ValidationTaskSpec> safeTasks() {
|
||||
return tasks == null ? List.of() : List.copyOf(tasks);
|
||||
}
|
||||
|
||||
public Optional<ValidationTaskSpec> findTask(String name) {
|
||||
return safeTasks().stream().filter(task -> task.name().equals(name)).findFirst();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.yaml.snakeyaml.LoaderOptions;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
import org.yaml.snakeyaml.constructor.SafeConstructor;
|
||||
|
||||
/**
|
||||
* Parses a draft's {@code validation.yaml} into {@link ValidationSpec}.
|
||||
*
|
||||
* <p>Follows the same defensive YAML loading rules as {@code SkillMetadataParser}:
|
||||
* SnakeYAML safe constructor, bounded aliases, nesting depth, and code points.
|
||||
* Parse problems are reported as structured spec errors (rule code + message) instead
|
||||
* of exceptions so the configuration validation layer can surface them as findings.
|
||||
*/
|
||||
public class ValidationSpecParser {
|
||||
|
||||
private static final int MAX_YAML_ALIASES = 20;
|
||||
private static final int MAX_YAML_NESTING_DEPTH = 20;
|
||||
private static final int MAX_YAML_CODE_POINTS = 100_000;
|
||||
private static final int MAX_TASKS = 50;
|
||||
private static final int MAX_TASK_TIMEOUT_MS = 600_000;
|
||||
|
||||
/** A structured parse or semantic error detected while reading the spec. */
|
||||
public record SpecError(String ruleCode, String message) {}
|
||||
|
||||
public record ParseOutcome(ValidationSpec spec, List<SpecError> errors) {
|
||||
|
||||
public boolean isValid() {
|
||||
return errors.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
public ParseOutcome parse(String content) {
|
||||
if (content == null || content.isBlank()) {
|
||||
return failure("SPEC_EMPTY", "validation.yaml is empty");
|
||||
}
|
||||
Object parsed;
|
||||
try {
|
||||
LoaderOptions loaderOptions = new LoaderOptions();
|
||||
loaderOptions.setAllowDuplicateKeys(false);
|
||||
loaderOptions.setMaxAliasesForCollections(MAX_YAML_ALIASES);
|
||||
loaderOptions.setNestingDepthLimit(MAX_YAML_NESTING_DEPTH);
|
||||
loaderOptions.setCodePointLimit(MAX_YAML_CODE_POINTS);
|
||||
Yaml yaml = new Yaml(new SafeConstructor(loaderOptions));
|
||||
parsed = yaml.load(content);
|
||||
} catch (Exception exception) {
|
||||
return failure("SPEC_YAML_INVALID", "validation.yaml is not valid YAML: "
|
||||
+ exception.getMessage());
|
||||
}
|
||||
if (!(parsed instanceof Map<?, ?> root)) {
|
||||
return failure("SPEC_NOT_MAP", "validation.yaml must be a YAML object");
|
||||
}
|
||||
|
||||
List<SpecError> errors = new ArrayList<>();
|
||||
Object versionValue = root.get("version");
|
||||
int version = 1;
|
||||
if (versionValue instanceof Number number) {
|
||||
version = number.intValue();
|
||||
} else if (versionValue != null) {
|
||||
try {
|
||||
version = Integer.parseInt(versionValue.toString().trim());
|
||||
} catch (NumberFormatException e) {
|
||||
errors.add(new SpecError("SPEC_VERSION_INVALID",
|
||||
"version must be an integer, got: " + versionValue));
|
||||
}
|
||||
}
|
||||
if (version != ValidationSpec.CURRENT_VERSION) {
|
||||
errors.add(new SpecError("SPEC_VERSION_INVALID",
|
||||
"unsupported spec version: " + version + " (expected "
|
||||
+ ValidationSpec.CURRENT_VERSION + ")"));
|
||||
}
|
||||
|
||||
List<ValidationTaskSpec> tasks = new ArrayList<>();
|
||||
java.util.Set<String> seenNames = new java.util.HashSet<>();
|
||||
Object tasksValue = root.get("tasks");
|
||||
if (tasksValue == null) {
|
||||
errors.add(new SpecError("SPEC_TASKS_MISSING", "validation.yaml must define a tasks list"));
|
||||
} else if (!(tasksValue instanceof List<?> rawTasks)) {
|
||||
errors.add(new SpecError("SPEC_TASKS_INVALID", "tasks must be a list"));
|
||||
} else {
|
||||
if (rawTasks.size() > MAX_TASKS) {
|
||||
errors.add(new SpecError("SPEC_TASKS_TOO_MANY",
|
||||
"too many tasks: " + rawTasks.size() + " (max " + MAX_TASKS + ")"));
|
||||
}
|
||||
for (int i = 0; i < rawTasks.size(); i++) {
|
||||
ValidationTaskSpec task = parseTask(rawTasks.get(i), i, errors, seenNames);
|
||||
if (task != null) {
|
||||
tasks.add(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!errors.isEmpty()) {
|
||||
return new ParseOutcome(ValidationSpec.empty(), List.copyOf(errors));
|
||||
}
|
||||
return new ParseOutcome(new ValidationSpec(version, List.copyOf(tasks)), List.of());
|
||||
}
|
||||
|
||||
private ValidationTaskSpec parseTask(Object rawTask, int index, List<SpecError> errors,
|
||||
java.util.Set<String> seenNames) {
|
||||
String where = "tasks[" + index + "]";
|
||||
if (!(rawTask instanceof Map<?, ?> taskMap)) {
|
||||
errors.add(new SpecError("TASK_NOT_MAP", where + " must be a YAML object"));
|
||||
return null;
|
||||
}
|
||||
|
||||
String name = stringField(taskMap, "name");
|
||||
if (name == null || name.isBlank()) {
|
||||
errors.add(new SpecError("TASK_NAME_MISSING", where + " is missing a name"));
|
||||
} else if (!name.matches("[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}")) {
|
||||
errors.add(new SpecError("TASK_NAME_INVALID",
|
||||
where + " has an invalid name: " + name));
|
||||
} else if (!seenNames.add(name)) {
|
||||
errors.add(new SpecError("TASK_NAME_DUPLICATE", where + " duplicates task name: " + name));
|
||||
}
|
||||
|
||||
String typeRaw = stringField(taskMap, "type");
|
||||
TaskType type;
|
||||
if (typeRaw == null) {
|
||||
errors.add(new SpecError("TASK_TYPE_MISSING", where + " is missing a type"));
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
type = TaskType.valueOf(typeRaw.trim().toUpperCase());
|
||||
} catch (IllegalArgumentException e) {
|
||||
errors.add(new SpecError("TASK_TYPE_INVALID",
|
||||
where + " has unknown type: " + typeRaw + " (expected script or prompt)"));
|
||||
return null;
|
||||
}
|
||||
|
||||
String script = stringField(taskMap, "script");
|
||||
String prompt = stringField(taskMap, "prompt");
|
||||
if (type == TaskType.SCRIPT && (script == null || script.isBlank())) {
|
||||
errors.add(new SpecError("TASK_SCRIPT_MISSING",
|
||||
where + " (script task) must define script"));
|
||||
}
|
||||
if (type == TaskType.PROMPT && (prompt == null || prompt.isBlank())) {
|
||||
errors.add(new SpecError("TASK_PROMPT_MISSING",
|
||||
where + " (prompt task) must define prompt"));
|
||||
}
|
||||
|
||||
List<String> args = new ArrayList<>();
|
||||
Object argsValue = taskMap.get("args");
|
||||
if (argsValue instanceof List<?> rawArgs) {
|
||||
for (Object arg : rawArgs) {
|
||||
if (arg != null) {
|
||||
args.add(arg.toString());
|
||||
}
|
||||
}
|
||||
} else if (argsValue != null) {
|
||||
errors.add(new SpecError("TASK_ARGS_INVALID", where + " args must be a list"));
|
||||
}
|
||||
|
||||
int timeoutMs = ValidationSpec.DEFAULT_TASK_TIMEOUT_MS;
|
||||
Object timeoutValue = taskMap.get("timeoutMs");
|
||||
if (timeoutValue instanceof Number number) {
|
||||
timeoutMs = number.intValue();
|
||||
} else if (timeoutValue != null) {
|
||||
errors.add(new SpecError("TASK_TIMEOUT_INVALID",
|
||||
where + " timeoutMs must be an integer"));
|
||||
}
|
||||
if (timeoutMs <= 0 || timeoutMs > MAX_TASK_TIMEOUT_MS) {
|
||||
errors.add(new SpecError("TASK_TIMEOUT_INVALID",
|
||||
where + " timeoutMs must be between 1 and " + MAX_TASK_TIMEOUT_MS));
|
||||
}
|
||||
|
||||
List<AssertionSpec> assertions = new ArrayList<>();
|
||||
Object assertionsValue = taskMap.get("assertions");
|
||||
if (assertionsValue instanceof List<?> rawAssertions) {
|
||||
for (Object rawAssertion : rawAssertions) {
|
||||
if (rawAssertion instanceof Map<?, ?> assertionMap) {
|
||||
Object assertionType = assertionMap.get("type");
|
||||
Map<String, Object> params = new LinkedHashMap<>();
|
||||
for (Map.Entry<?, ?> entry : assertionMap.entrySet()) {
|
||||
if (!"type".equals(entry.getKey()) && entry.getKey() != null) {
|
||||
params.put(entry.getKey().toString(), entry.getValue());
|
||||
}
|
||||
}
|
||||
if (assertionType == null || assertionType.toString().isBlank()) {
|
||||
errors.add(new SpecError("ASSERTION_TYPE_MISSING",
|
||||
where + " has an assertion without a type"));
|
||||
} else {
|
||||
assertions.add(new AssertionSpec(assertionType.toString(), params));
|
||||
}
|
||||
} else {
|
||||
errors.add(new SpecError("ASSERTION_NOT_MAP",
|
||||
where + " assertions must be objects"));
|
||||
}
|
||||
}
|
||||
} else if (assertionsValue != null) {
|
||||
errors.add(new SpecError("ASSERTIONS_INVALID", where + " assertions must be a list"));
|
||||
}
|
||||
|
||||
String description = stringField(taskMap, "description");
|
||||
return new ValidationTaskSpec(name == null ? where : name, description, type, script,
|
||||
List.copyOf(args), prompt, timeoutMs, List.copyOf(assertions));
|
||||
}
|
||||
|
||||
private String stringField(Map<?, ?> map, String key) {
|
||||
Object value = map.get(key);
|
||||
return value == null ? null : value.toString();
|
||||
}
|
||||
|
||||
private ParseOutcome failure(String ruleCode, String message) {
|
||||
return new ParseOutcome(ValidationSpec.empty(), List.of(new SpecError(ruleCode, message)));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* One behavior validation task parsed from {@code validation.yaml}.
|
||||
*/
|
||||
public record ValidationTaskSpec(
|
||||
String name,
|
||||
String description,
|
||||
TaskType type,
|
||||
String script,
|
||||
List<String> args,
|
||||
String prompt,
|
||||
int timeoutMs,
|
||||
List<AssertionSpec> assertions
|
||||
) {
|
||||
|
||||
public List<String> safeArgs() {
|
||||
return args == null ? List.of() : List.copyOf(args);
|
||||
}
|
||||
|
||||
public List<AssertionSpec> safeAssertions() {
|
||||
return assertions == null ? List.of() : List.copyOf(assertions);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
/**
|
||||
* Behavior validation task specifications parsed from a draft's {@code validation.yaml}.
|
||||
*/
|
||||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
package com.iflytek.skillhub.domain.authoring.runtime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.spec.AssertionSpec;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class AssertionEvaluatorTest {
|
||||
|
||||
private final AssertionEvaluator evaluator = new AssertionEvaluator();
|
||||
|
||||
private static TaskResult scriptResult(int exitCode, String stdout, String stderr) {
|
||||
return TaskResult.ofScript(exitCode, stdout, stderr, 1, path -> Optional.empty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void exitCodeAssertionPassesAndFails() {
|
||||
AssertionSpec assertion = new AssertionSpec("exit_code", Map.of("equals", 0));
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "", ""))).isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(1, "", "")))
|
||||
.singleElement()
|
||||
.satisfies(failure -> assertThat(failure.describe()).contains("exit code"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stdoutContainsChecksValue() {
|
||||
AssertionSpec assertion = new AssertionSpec("stdout_contains", Map.of("value", "OK"));
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "everything OK here", "")))
|
||||
.isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "failed", "")))
|
||||
.hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void stdoutMatchesUsesRegexFindSemantics() {
|
||||
AssertionSpec assertion =
|
||||
new AssertionSpec("stdout_matches", Map.of("pattern", "duration: \\d+ms"));
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "run finished, duration: 42ms", "")))
|
||||
.isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "no timing info", "")))
|
||||
.hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void jsonPointerAssertionComparesValue() {
|
||||
AssertionSpec assertion = new AssertionSpec("stdout_json",
|
||||
Map.of("pointer", "/status", "equals", "healthy"));
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(assertion),
|
||||
scriptResult(0, "{\"status\":\"healthy\"}", ""))).isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(assertion),
|
||||
scriptResult(0, "{\"status\":\"unhealthy\"}", ""))).hasSize(1);
|
||||
assertThat(evaluator.evaluate(List.of(assertion), scriptResult(0, "not json", "")))
|
||||
.singleElement()
|
||||
.satisfies(failure -> assertThat(failure.describe()).contains("not valid JSON"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void artifactExistsChecksContentHash() {
|
||||
byte[] content = "artifact-bytes".getBytes();
|
||||
String sha = TaskResult.sha256Hex(content);
|
||||
TaskResult result = TaskResult.ofScript(0, "", "", 0, path ->
|
||||
"out/report.txt".equals(path) ? Optional.of(content) : Optional.empty());
|
||||
|
||||
AssertionSpec exists = new AssertionSpec("artifact_exists", Map.of("path", "out/report.txt"));
|
||||
AssertionSpec missing = new AssertionSpec("artifact_exists", Map.of("path", "out/none.txt"));
|
||||
AssertionSpec wrongHash = new AssertionSpec("artifact_exists",
|
||||
Map.of("path", "out/report.txt", "sha256", "deadbeef"));
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(exists), result)).isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(missing), result)).hasSize(1);
|
||||
assertThat(evaluator.evaluate(List.of(wrongHash), result))
|
||||
.singleElement()
|
||||
.satisfies(failure -> assertThat(failure.describe()).contains("sha256"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void toolCallCountHonorsMinAndMax() {
|
||||
TaskResult result = TaskResult.ofScript(0, "", "", 2, path -> Optional.empty());
|
||||
|
||||
assertThat(evaluator.evaluate(
|
||||
List.of(new AssertionSpec("tool_call_count", Map.of("min", 1, "max", 3))), result))
|
||||
.isEmpty();
|
||||
assertThat(evaluator.evaluate(
|
||||
List.of(new AssertionSpec("tool_call_count", Map.of("min", 3))), result))
|
||||
.hasSize(1);
|
||||
assertThat(evaluator.evaluate(
|
||||
List.of(new AssertionSpec("tool_call_count", Map.of("max", 1))), result))
|
||||
.hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void responseAssertionsApplyToPromptResults() {
|
||||
TaskResult result = TaskResult.ofPrompt("the skill summarizes documents", 0,
|
||||
path -> Optional.empty());
|
||||
|
||||
assertThat(evaluator.evaluate(List.of(
|
||||
new AssertionSpec("response_contains", Map.of("value", "summarizes"))), result))
|
||||
.isEmpty();
|
||||
assertThat(evaluator.evaluate(List.of(
|
||||
new AssertionSpec("response_contains", Map.of("value", "translates"))), result))
|
||||
.hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownAssertionTypeFailsEvaluation() {
|
||||
List<AssertionSpec> assertions = List.of(new AssertionSpec("mind_read", Map.of()));
|
||||
|
||||
assertThat(evaluator.evaluate(assertions, scriptResult(0, "", "")))
|
||||
.singleElement()
|
||||
.satisfies(failure -> assertThat(failure.describe()).contains("unknown assertion type"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateSyntaxAcceptsKnownTypesPerTaskKind() {
|
||||
List<AssertionSpec> scriptAssertions = List.of(
|
||||
new AssertionSpec("exit_code", Map.of("equals", 0)),
|
||||
new AssertionSpec("stdout_contains", Map.of("value", "OK")));
|
||||
List<AssertionSpec> promptAssertions = List.of(
|
||||
new AssertionSpec("response_contains", Map.of("value", "OK")),
|
||||
new AssertionSpec("tool_call_count", Map.of("min", 0)));
|
||||
|
||||
assertThat(evaluator.validateSyntax(scriptAssertions, false)).isEmpty();
|
||||
assertThat(evaluator.validateSyntax(promptAssertions, true)).isEmpty();
|
||||
|
||||
List<AssertionSpec> crossKind = List.of(
|
||||
new AssertionSpec("exit_code", Map.of("equals", 0)),
|
||||
new AssertionSpec("response_contains", Map.of("value", "OK")));
|
||||
assertThat(evaluator.validateSyntax(crossKind, true)).hasSize(1);
|
||||
assertThat(evaluator.validateSyntax(crossKind, false)).hasSize(1);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
package com.iflytek.skillhub.domain.authoring.spec;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ValidationSpecParserTest {
|
||||
|
||||
private final ValidationSpecParser parser = new ValidationSpecParser();
|
||||
|
||||
@Test
|
||||
void parsesValidScriptTask() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: smoke
|
||||
description: basic check
|
||||
type: script
|
||||
script: scripts/check.sh
|
||||
args: ["--quiet"]
|
||||
timeoutMs: 5000
|
||||
assertions:
|
||||
- type: exit_code
|
||||
equals: 0
|
||||
- type: stdout_contains
|
||||
value: OK
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isTrue();
|
||||
ValidationSpec spec = outcome.spec();
|
||||
assertThat(spec.version()).isEqualTo(1);
|
||||
assertThat(spec.safeTasks()).hasSize(1);
|
||||
ValidationTaskSpec task = spec.safeTasks().get(0);
|
||||
assertThat(task.name()).isEqualTo("smoke");
|
||||
assertThat(task.type()).isEqualTo(TaskType.SCRIPT);
|
||||
assertThat(task.script()).isEqualTo("scripts/check.sh");
|
||||
assertThat(task.safeArgs()).containsExactly("--quiet");
|
||||
assertThat(task.timeoutMs()).isEqualTo(5000);
|
||||
assertThat(task.safeAssertions()).hasSize(2);
|
||||
assertThat(task.safeAssertions().get(0).type()).isEqualTo("exit_code");
|
||||
assertThat(task.safeAssertions().get(0).paramInt("equals")).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesPromptTaskWithoutOptionalFields() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: ask
|
||||
type: prompt
|
||||
prompt: What is this skill about?
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isTrue();
|
||||
ValidationTaskSpec task = outcome.spec().safeTasks().get(0);
|
||||
assertThat(task.type()).isEqualTo(TaskType.PROMPT);
|
||||
assertThat(task.timeoutMs()).isEqualTo(ValidationSpec.DEFAULT_TASK_TIMEOUT_MS);
|
||||
assertThat(task.safeAssertions()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyContentIsASpecError() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse(" \n");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("SPEC_EMPTY"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidYamlIsASpecErrorNotAnException() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("version: [unbalanced");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("SPEC_YAML_INVALID"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedVersionIsRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 2
|
||||
tasks: []
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("SPEC_VERSION_INVALID"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingTasksListIsRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("version: 1\n");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("SPEC_TASKS_MISSING"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateTaskNamesAreRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: smoke
|
||||
type: script
|
||||
script: a.sh
|
||||
- name: smoke
|
||||
type: script
|
||||
script: b.sh
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("TASK_NAME_DUPLICATE"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void scriptTaskRequiresScriptAndPromptTaskRequiresPrompt() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: no-script
|
||||
type: script
|
||||
- name: no-prompt
|
||||
type: prompt
|
||||
""");
|
||||
|
||||
List<String> codes = outcome.errors().stream().map(ValidationSpecParser.SpecError::ruleCode).toList();
|
||||
assertThat(codes).containsExactly("TASK_SCRIPT_MISSING", "TASK_PROMPT_MISSING");
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownTypeAndInvalidTimeoutAreRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: weird
|
||||
type: telepathy
|
||||
- name: slow
|
||||
type: script
|
||||
script: a.sh
|
||||
timeoutMs: 999999999
|
||||
""");
|
||||
|
||||
List<String> codes = outcome.errors().stream().map(ValidationSpecParser.SpecError::ruleCode).toList();
|
||||
assertThat(codes).contains("TASK_TYPE_INVALID", "TASK_TIMEOUT_INVALID");
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidTaskNameIsRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: "has space"
|
||||
type: script
|
||||
script: a.sh
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("TASK_NAME_INVALID"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void assertionWithoutTypeIsRejected() {
|
||||
ValidationSpecParser.ParseOutcome outcome = parser.parse("""
|
||||
version: 1
|
||||
tasks:
|
||||
- name: smoke
|
||||
type: script
|
||||
script: a.sh
|
||||
assertions:
|
||||
- equals: 0
|
||||
""");
|
||||
|
||||
assertThat(outcome.isValid()).isFalse();
|
||||
assertThat(outcome.errors()).singleElement()
|
||||
.satisfies(error -> assertThat(error.ruleCode()).isEqualTo("ASSERTION_TYPE_MISSING"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.iflytek.skillhub.infra.jpa;
|
||||
|
||||
import com.iflytek.skillhub.domain.authoring.RuntimeBinding;
|
||||
import com.iflytek.skillhub.domain.authoring.RuntimeBindingRepository;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
/**
|
||||
* JPA-backed repository for per-draft runtime bindings.
|
||||
*/
|
||||
@Repository
|
||||
public interface RuntimeBindingJpaRepository extends JpaRepository<RuntimeBinding, Long>,
|
||||
RuntimeBindingRepository {
|
||||
|
||||
Optional<RuntimeBinding> findByDraftId(Long draftId);
|
||||
|
||||
void deleteByDraftId(Long draftId);
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue