diff --git a/.env.release.example b/.env.release.example index e7bde2024..cc3c3e225 100644 --- a/.env.release.example +++ b/.env.release.example @@ -80,6 +80,8 @@ SESSION_COOKIE_SECURE=false # Observability defaults require no Collector or tracing backend. # Use json in container deployments when stdout is collected centrally. +# Opt in to protected metrics with health,info,metrics,prometheus; never use '*'. +SKILLHUB_ACTUATOR_EXPOSURE_INCLUDE=health,info SKILLHUB_TRACING_MODE=none SKILLHUB_LOG_FORMAT=json SKILLHUB_LOG_ASYNC_QUEUE_SIZE=1024 @@ -94,6 +96,10 @@ SKILLHUB_OTLP_COMPRESSION=gzip # Independent, lossy business analytics. Default off; no receiver required. # Full catalog, Secret injection and Collector query example: docs/business-event-logs.md SKILLHUB_BUSINESS_LOGS_ENABLED=false +# Managed by runtime.sh; unique names per installation on a shared Docker host. +SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED=false +SKILLHUB_BUSINESS_LOGS_NETWORK=skillhub-business-logs +SKILLHUB_BUSINESS_LOGS_PROJECT=skillhub-business-logs SKILLHUB_BUSINESS_LOGS_ENDPOINT= # URL-encoded name=value pairs, comma-separated; inject from deployment Secrets, never commit real values. SKILLHUB_BUSINESS_LOGS_HEADERS= diff --git a/business-logs-collector.yaml b/business-logs-collector.yaml new file mode 100644 index 000000000..479dacb65 --- /dev/null +++ b/business-logs-collector.yaml @@ -0,0 +1,21 @@ +# Production forwarding template; configure a durable downstream outside this repo. +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 64 + spike_limit_mib: 16 +exporters: + otlphttp: + endpoint: ${env:SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT} + headers: ${env:SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS} +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter] + exporters: [otlphttp] diff --git a/compose.business-logs-collector.yml b/compose.business-logs-collector.yml new file mode 100644 index 000000000..1d2617642 --- /dev/null +++ b/compose.business-logs-collector.yml @@ -0,0 +1,20 @@ +# Optional production forwarder. No host port and no local file storage. +name: ${SKILLHUB_BUSINESS_LOGS_PROJECT:-skillhub-business-logs} +services: + business-logs-collector: + image: ${SKILLHUB_BUSINESS_LOGS_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.111.0} + command: ["--config=/etc/otelcol-contrib/config.yaml"] + restart: unless-stopped + mem_limit: 128m + cpus: 0.5 + environment: + SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT: ${SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT:?configure business-logs/.env downstream endpoint first} + SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS: ${SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS:-{}} + volumes: + - ./business-logs-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + networks: + - business-logs +networks: + business-logs: + external: true + name: ${SKILLHUB_BUSINESS_LOGS_NETWORK:-skillhub-business-logs} diff --git a/compose.release.business-logs.yml b/compose.release.business-logs.yml new file mode 100644 index 000000000..2b46e5209 --- /dev/null +++ b/compose.release.business-logs.yml @@ -0,0 +1,10 @@ +# Optional attachment: retain the backend's ordinary dependency network. +services: + server: + networks: + - default + - business-logs +networks: + business-logs: + external: true + name: ${SKILLHUB_BUSINESS_LOGS_NETWORK:-skillhub-business-logs} diff --git a/scripts/runtime.sh b/scripts/runtime.sh index e2afdfad3..28e0c9f58 100755 --- a/scripts/runtime.sh +++ b/scripts/runtime.sh @@ -28,6 +28,8 @@ WEBSITE_ID="" ANALYTICS_UPSTREAM="" ANALYTICS_ADAPTER="umami" ANALYTICS_APP_ID="" +BUSINESS_LOGS_ENDPOINT="" +BUSINESS_LOGS_NETWORK="" while [ "$#" -gt 0 ]; do case "$1" in @@ -115,6 +117,16 @@ while [ "$#" -gt 0 ]; do ANALYTICS_APP_ID="$2" shift 2 ;; + --logs-endpoint) + [ "$#" -ge 2 ] || { echo "Missing value for --logs-endpoint" >&2; exit 1; } + BUSINESS_LOGS_ENDPOINT="$2" + shift 2 + ;; + --logs-network) + [ "$#" -ge 2 ] || { echo "Missing value for --logs-network" >&2; exit 1; } + BUSINESS_LOGS_NETWORK="$2" + shift 2 + ;; --help|-h) cat < umami (default) or datafinder (with --app-id) --app-id Existing DataFinder application ID +Business Logs (default off): + business-logs-up Prepare/start optional forwarder and enable server collection + First run creates business-logs/.env; configure downstream and retry + business-logs-enable Apply .env.release to server; --logs-endpoint overrides receiver + business-logs-disable Disable collection and recreate server; keep receiver and data + business-logs-down Stop only the optional Collector; keep files and network + business-logs-ps/logs Inspect only the optional Collector + --logs-endpoint Full OTLP/HTTP Logs URL (no credentials/query/fragment) + --logs-network Attach server to this external private network + Options: --version Use a specific image tag, for example v0.1.0 --aliyun Use the configured Aliyun mirror registry @@ -151,6 +173,13 @@ EOF esac done +if [ "$EXPLICIT_REF" = "false" ] && [ -n "$SKILLHUB_VERSION_VALUE" ]; then + case "$SKILLHUB_VERSION_VALUE" in + latest|edge) ;; # Floating image aliases are not Git refs; pin --ref explicitly. + *) SKILLHUB_REF="$SKILLHUB_VERSION_VALUE" ;; + esac +fi + if [ "$USE_ALIYUN" = "true" ]; then SKILLHUB_RAW_BASE="${SKILLHUB_RAW_BASE:-https://imageless.oss-cn-beijing.aliyuncs.com}" RUNTIME_SCRIPT_URL="$SKILLHUB_RAW_BASE/runtime.sh" @@ -167,6 +196,10 @@ ANALYTICS_COMPOSE_FILE="$ANALYTICS_HOME/compose.analytics-umami.yml" ANALYTICS_ENV_FILE="$ANALYTICS_HOME/.env" ENV_EXAMPLE_FILE="$SKILLHUB_HOME/.env.release.example" ENV_FILE="$SKILLHUB_HOME/.env.release" +BUSINESS_LOGS_HOME="$SKILLHUB_HOME/business-logs" +BUSINESS_LOGS_ENV_FILE="$BUSINESS_LOGS_HOME/.env" +BUSINESS_LOGS_COMPOSE_FILE="$BUSINESS_LOGS_HOME/compose.business-logs-collector.yml" +BUSINESS_LOGS_OVERLAY_FILE="$SKILLHUB_HOME/compose.release.business-logs.yml" find_compose() { if docker compose version >/dev/null 2>&1; then @@ -362,6 +395,10 @@ prepare_runtime_files() { umask "$old_umask" fi secure_env_file + if [ "$COMMAND" = "business-logs-enable" ] || [ "$COMMAND" = "business-logs-up" ] || + [ "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false)" = "true" ]; then + download_file "$SKILLHUB_RAW_BASE/compose.release.business-logs.yml" "$BUSINESS_LOGS_OVERLAY_FILE" + fi if [ -n "$SKILLHUB_MIRROR_REGISTRY_VALUE" ]; then mirror_registry="${SKILLHUB_MIRROR_REGISTRY_VALUE%/}" @@ -425,10 +462,78 @@ run_compose() { # Preserve the normal default network and attach only Web to analytics. set -- -f "$OVERLAY_FILE" "$@" fi + if [ "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false)" = "true" ]; then + ensure_business_logs_network + set -- -f "$BUSINESS_LOGS_OVERLAY_FILE" "$@" + fi # shellcheck disable=SC2086 $compose_cmd --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@" } +ensure_business_logs_network() { + SKILLHUB_BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs)" + export SKILLHUB_BUSINESS_LOGS_NETWORK + if ! docker network inspect "$SKILLHUB_BUSINESS_LOGS_NETWORK" >/dev/null 2>&1; then + docker network create "$SKILLHUB_BUSINESS_LOGS_NETWORK" >/dev/null + fi +} + +prepare_business_logs_files() { + mkdir -p "$BUSINESS_LOGS_HOME" + # Root-level names work unchanged on both versioned GitHub raw and flat OSS. + download_file "$SKILLHUB_RAW_BASE/compose.business-logs-collector.yml" "$BUSINESS_LOGS_COMPOSE_FILE" + download_file "$SKILLHUB_RAW_BASE/business-logs-collector.yaml" "$BUSINESS_LOGS_HOME/business-logs-collector.yaml" + if [ ! -f "$BUSINESS_LOGS_ENV_FILE" ]; then + old_umask="$(umask)" + umask 077 + { + printf 'SKILLHUB_BUSINESS_LOGS_PROJECT=%s\n' "$(get_env_value SKILLHUB_BUSINESS_LOGS_PROJECT skillhub-business-logs)" + printf 'SKILLHUB_BUSINESS_LOGS_NETWORK=%s\n' "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs)" + printf 'SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT=\n' + printf "SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS='{}'\n" + } >"$BUSINESS_LOGS_ENV_FILE" + umask "$old_umask" + fi + chmod 600 "$BUSINESS_LOGS_ENV_FILE" +} + +run_business_logs_compose() { + # The independent file owns its identity. Inherited Compose variables cannot redirect it. + SKILLHUB_BUSINESS_LOGS_PROJECT="$(get_env_value SKILLHUB_BUSINESS_LOGS_PROJECT skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")" + SKILLHUB_BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")" + export SKILLHUB_BUSINESS_LOGS_NETWORK + compose_cmd="$(find_compose)" + # shellcheck disable=SC2086 + $compose_cmd -p "$SKILLHUB_BUSINESS_LOGS_PROJECT" --env-file "$BUSINESS_LOGS_ENV_FILE" -f "$BUSINESS_LOGS_COMPOSE_FILE" "$@" +} + +validate_business_logs_endpoint() { + case "$1" in + http://*/*|https://*/*) ;; + *) echo "Business Logs requires a full HTTP(S) Logs URL" >&2; exit 1 ;; + esac + case "$1" in + *'@'*|*'?'*|*'#'*|*[[:space:]]*|*'\\'*) + echo "Business Logs URL must not contain credentials, query, fragment or whitespace" >&2; exit 1 ;; + esac +} + +enable_business_logs() { + receiver_endpoint="${BUSINESS_LOGS_ENDPOINT:-$(get_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT)}" + validate_business_logs_endpoint "$receiver_endpoint" + if [ -n "$BUSINESS_LOGS_NETWORK" ]; then + case "$BUSINESS_LOGS_NETWORK" in + *[!A-Za-z0-9_.-]*|[-.]* ) echo "Invalid Business Logs network name" >&2; exit 1 ;; + esac + set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK "$BUSINESS_LOGS_NETWORK" + set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED true + fi + set_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT "$receiver_endpoint" + set_env_value SKILLHUB_BUSINESS_LOGS_ENABLED true + run_compose up -d --no-deps --force-recreate server + echo "Business Logs enabled; produce a new event and query the receiver." +} + ensure_analytics_identity() { # Once initialized, the independent directory owns both identifiers. Neither # a recreated release file nor inherited Compose variables may redirect it. @@ -535,7 +640,7 @@ build_runtime_lifecycle_args() { if [ "$USE_ALIYUN" = "true" ]; then append_runtime_flag "--aliyun" - elif [ "$EXPLICIT_REF" = "true" ] || [ "$SKILLHUB_REF" != "main" ]; then + elif [ "$EXPLICIT_REF" = "true" ] || { [ "$SKILLHUB_REF" != "main" ] && [ "$SKILLHUB_REF" != "$SKILLHUB_VERSION_VALUE" ]; }; then append_runtime_option "--ref" "$SKILLHUB_REF" fi @@ -625,7 +730,11 @@ EOF ;; clean) run_compose down - if [ -d "$ANALYTICS_HOME" ]; then + if [ -d "$BUSINESS_LOGS_HOME" ] || [ -n "$(get_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT)" ]; then + # Retain operator credentials, Collector config and identities for safe restart. + rm -f "$COMPOSE_FILE" "$BUSINESS_LOGS_OVERLAY_FILE" "$ENV_EXAMPLE_FILE" + echo "Downloaded release files removed; operator environment and business-logs files retained." + elif [ -d "$ANALYTICS_HOME" ]; then # Keep independent platform credentials and files; never strand its database. rm -f "$COMPOSE_FILE" "$OVERLAY_FILE" "$ENV_EXAMPLE_FILE" "$ENV_FILE" echo "SkillHub files removed; independent analytics files retained in $ANALYTICS_HOME" @@ -642,6 +751,40 @@ EOF pull) run_compose pull ;; + business-logs-up) + prepare_business_logs_files + downstream_endpoint="$(get_env_value SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT "" "$BUSINESS_LOGS_ENV_FILE")" + if [ -z "$downstream_endpoint" ]; then + echo "Configure SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT (OTLP base URL) and optional JSON headers in $BUSINESS_LOGS_ENV_FILE, then retry." >&2 + exit 1 + fi + validate_business_logs_endpoint "${downstream_endpoint%/}/v1/logs" + # Read the independent project's stored network; do not overwrite operator secrets. + BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")" + set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK "$BUSINESS_LOGS_NETWORK" + ensure_business_logs_network + run_business_logs_compose config --quiet + run_business_logs_compose up -d --force-recreate + BUSINESS_LOGS_ENDPOINT=http://business-logs-collector:4318/v1/logs + enable_business_logs + ;; + business-logs-enable) + enable_business_logs + ;; + business-logs-disable) + set_env_value SKILLHUB_BUSINESS_LOGS_ENABLED false + set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false + run_compose up -d --no-deps --force-recreate server + echo "Business Logs disabled; receiver data and network retained." + ;; + business-logs-down|business-logs-ps|business-logs-logs) + [ -f "$BUSINESS_LOGS_ENV_FILE" ] || { echo "No optional Collector; run business-logs-up first" >&2; exit 1; } + case "$COMMAND" in + business-logs-down) run_business_logs_compose stop ;; + business-logs-ps) run_business_logs_compose ps ;; + business-logs-logs) run_business_logs_compose logs -f ;; + esac + ;; analytics-up) prepare_analytics_files ensure_analytics_network