mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-14 09:01:16 +00:00
feat: add one-click business logs receiver and Collector lifecycle
Signed-off-by: 千乘妍 (Xiaoyaner) <xiaoyaner0201@users.noreply.github.com>
This commit is contained in:
千乘妍 (Xiaoyaner)
parent
07d3c31482
commit
d525787076
5 changed files with 202 additions and 2 deletions
|
|
@ -80,6 +80,8 @@ SESSION_COOKIE_SECURE=false
|
|||
|
||||
# Observability defaults require no Collector or tracing backend.
|
||||
# Use json in container deployments when stdout is collected centrally.
|
||||
# Opt in to protected metrics with health,info,metrics,prometheus; never use '*'.
|
||||
SKILLHUB_ACTUATOR_EXPOSURE_INCLUDE=health,info
|
||||
SKILLHUB_TRACING_MODE=none
|
||||
SKILLHUB_LOG_FORMAT=json
|
||||
SKILLHUB_LOG_ASYNC_QUEUE_SIZE=1024
|
||||
|
|
@ -94,6 +96,10 @@ SKILLHUB_OTLP_COMPRESSION=gzip
|
|||
# Independent, lossy business analytics. Default off; no receiver required.
|
||||
# Full catalog, Secret injection and Collector query example: docs/business-event-logs.md
|
||||
SKILLHUB_BUSINESS_LOGS_ENABLED=false
|
||||
# Managed by runtime.sh; unique names per installation on a shared Docker host.
|
||||
SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED=false
|
||||
SKILLHUB_BUSINESS_LOGS_NETWORK=skillhub-business-logs
|
||||
SKILLHUB_BUSINESS_LOGS_PROJECT=skillhub-business-logs
|
||||
SKILLHUB_BUSINESS_LOGS_ENDPOINT=
|
||||
# URL-encoded name=value pairs, comma-separated; inject from deployment Secrets, never commit real values.
|
||||
SKILLHUB_BUSINESS_LOGS_HEADERS=
|
||||
|
|
|
|||
21
business-logs-collector.yaml
Normal file
21
business-logs-collector.yaml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# Production forwarding template; configure a durable downstream outside this repo.
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 64
|
||||
spike_limit_mib: 16
|
||||
exporters:
|
||||
otlphttp:
|
||||
endpoint: ${env:SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT}
|
||||
headers: ${env:SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS}
|
||||
service:
|
||||
pipelines:
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter]
|
||||
exporters: [otlphttp]
|
||||
20
compose.business-logs-collector.yml
Normal file
20
compose.business-logs-collector.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
# Optional production forwarder. No host port and no local file storage.
|
||||
name: ${SKILLHUB_BUSINESS_LOGS_PROJECT:-skillhub-business-logs}
|
||||
services:
|
||||
business-logs-collector:
|
||||
image: ${SKILLHUB_BUSINESS_LOGS_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.111.0}
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
environment:
|
||||
SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT: ${SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT:?configure business-logs/.env downstream endpoint first}
|
||||
SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS: ${SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS:-{}}
|
||||
volumes:
|
||||
- ./business-logs-collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
networks:
|
||||
- business-logs
|
||||
networks:
|
||||
business-logs:
|
||||
external: true
|
||||
name: ${SKILLHUB_BUSINESS_LOGS_NETWORK:-skillhub-business-logs}
|
||||
10
compose.release.business-logs.yml
Normal file
10
compose.release.business-logs.yml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Optional attachment: retain the backend's ordinary dependency network.
|
||||
services:
|
||||
server:
|
||||
networks:
|
||||
- default
|
||||
- business-logs
|
||||
networks:
|
||||
business-logs:
|
||||
external: true
|
||||
name: ${SKILLHUB_BUSINESS_LOGS_NETWORK:-skillhub-business-logs}
|
||||
|
|
@ -28,6 +28,8 @@ WEBSITE_ID=""
|
|||
ANALYTICS_UPSTREAM=""
|
||||
ANALYTICS_ADAPTER="umami"
|
||||
ANALYTICS_APP_ID=""
|
||||
BUSINESS_LOGS_ENDPOINT=""
|
||||
BUSINESS_LOGS_NETWORK=""
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
|
|
@ -115,6 +117,16 @@ while [ "$#" -gt 0 ]; do
|
|||
ANALYTICS_APP_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
--logs-endpoint)
|
||||
[ "$#" -ge 2 ] || { echo "Missing value for --logs-endpoint" >&2; exit 1; }
|
||||
BUSINESS_LOGS_ENDPOINT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--logs-network)
|
||||
[ "$#" -ge 2 ] || { echo "Missing value for --logs-network" >&2; exit 1; }
|
||||
BUSINESS_LOGS_NETWORK="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help|-h)
|
||||
cat <<EOF
|
||||
Usage: sh runtime.sh [up|down|clean|ps|logs|pull|analytics-up|analytics-enable|analytics-disable|analytics-down|analytics-ps|analytics-logs|analytics-pull] [options]
|
||||
|
|
@ -128,6 +140,16 @@ Analytics:
|
|||
--adapter <name> umami (default) or datafinder (with --app-id)
|
||||
--app-id <integer> Existing DataFinder application ID
|
||||
|
||||
Business Logs (default off):
|
||||
business-logs-up Prepare/start optional forwarder and enable server collection
|
||||
First run creates business-logs/.env; configure downstream and retry
|
||||
business-logs-enable Apply .env.release to server; --logs-endpoint overrides receiver
|
||||
business-logs-disable Disable collection and recreate server; keep receiver and data
|
||||
business-logs-down Stop only the optional Collector; keep files and network
|
||||
business-logs-ps/logs Inspect only the optional Collector
|
||||
--logs-endpoint <url> Full OTLP/HTTP Logs URL (no credentials/query/fragment)
|
||||
--logs-network <name> Attach server to this external private network
|
||||
|
||||
Options:
|
||||
--version <tag> Use a specific image tag, for example v0.1.0
|
||||
--aliyun Use the configured Aliyun mirror registry
|
||||
|
|
@ -151,6 +173,13 @@ EOF
|
|||
esac
|
||||
done
|
||||
|
||||
if [ "$EXPLICIT_REF" = "false" ] && [ -n "$SKILLHUB_VERSION_VALUE" ]; then
|
||||
case "$SKILLHUB_VERSION_VALUE" in
|
||||
latest|edge) ;; # Floating image aliases are not Git refs; pin --ref explicitly.
|
||||
*) SKILLHUB_REF="$SKILLHUB_VERSION_VALUE" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if [ "$USE_ALIYUN" = "true" ]; then
|
||||
SKILLHUB_RAW_BASE="${SKILLHUB_RAW_BASE:-https://imageless.oss-cn-beijing.aliyuncs.com}"
|
||||
RUNTIME_SCRIPT_URL="$SKILLHUB_RAW_BASE/runtime.sh"
|
||||
|
|
@ -167,6 +196,10 @@ ANALYTICS_COMPOSE_FILE="$ANALYTICS_HOME/compose.analytics-umami.yml"
|
|||
ANALYTICS_ENV_FILE="$ANALYTICS_HOME/.env"
|
||||
ENV_EXAMPLE_FILE="$SKILLHUB_HOME/.env.release.example"
|
||||
ENV_FILE="$SKILLHUB_HOME/.env.release"
|
||||
BUSINESS_LOGS_HOME="$SKILLHUB_HOME/business-logs"
|
||||
BUSINESS_LOGS_ENV_FILE="$BUSINESS_LOGS_HOME/.env"
|
||||
BUSINESS_LOGS_COMPOSE_FILE="$BUSINESS_LOGS_HOME/compose.business-logs-collector.yml"
|
||||
BUSINESS_LOGS_OVERLAY_FILE="$SKILLHUB_HOME/compose.release.business-logs.yml"
|
||||
|
||||
find_compose() {
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
|
|
@ -362,6 +395,10 @@ prepare_runtime_files() {
|
|||
umask "$old_umask"
|
||||
fi
|
||||
secure_env_file
|
||||
if [ "$COMMAND" = "business-logs-enable" ] || [ "$COMMAND" = "business-logs-up" ] ||
|
||||
[ "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false)" = "true" ]; then
|
||||
download_file "$SKILLHUB_RAW_BASE/compose.release.business-logs.yml" "$BUSINESS_LOGS_OVERLAY_FILE"
|
||||
fi
|
||||
|
||||
if [ -n "$SKILLHUB_MIRROR_REGISTRY_VALUE" ]; then
|
||||
mirror_registry="${SKILLHUB_MIRROR_REGISTRY_VALUE%/}"
|
||||
|
|
@ -425,10 +462,78 @@ run_compose() {
|
|||
# Preserve the normal default network and attach only Web to analytics.
|
||||
set -- -f "$OVERLAY_FILE" "$@"
|
||||
fi
|
||||
if [ "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false)" = "true" ]; then
|
||||
ensure_business_logs_network
|
||||
set -- -f "$BUSINESS_LOGS_OVERLAY_FILE" "$@"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
$compose_cmd --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
ensure_business_logs_network() {
|
||||
SKILLHUB_BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs)"
|
||||
export SKILLHUB_BUSINESS_LOGS_NETWORK
|
||||
if ! docker network inspect "$SKILLHUB_BUSINESS_LOGS_NETWORK" >/dev/null 2>&1; then
|
||||
docker network create "$SKILLHUB_BUSINESS_LOGS_NETWORK" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_business_logs_files() {
|
||||
mkdir -p "$BUSINESS_LOGS_HOME"
|
||||
# Root-level names work unchanged on both versioned GitHub raw and flat OSS.
|
||||
download_file "$SKILLHUB_RAW_BASE/compose.business-logs-collector.yml" "$BUSINESS_LOGS_COMPOSE_FILE"
|
||||
download_file "$SKILLHUB_RAW_BASE/business-logs-collector.yaml" "$BUSINESS_LOGS_HOME/business-logs-collector.yaml"
|
||||
if [ ! -f "$BUSINESS_LOGS_ENV_FILE" ]; then
|
||||
old_umask="$(umask)"
|
||||
umask 077
|
||||
{
|
||||
printf 'SKILLHUB_BUSINESS_LOGS_PROJECT=%s\n' "$(get_env_value SKILLHUB_BUSINESS_LOGS_PROJECT skillhub-business-logs)"
|
||||
printf 'SKILLHUB_BUSINESS_LOGS_NETWORK=%s\n' "$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs)"
|
||||
printf 'SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT=\n'
|
||||
printf "SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_HEADERS='{}'\n"
|
||||
} >"$BUSINESS_LOGS_ENV_FILE"
|
||||
umask "$old_umask"
|
||||
fi
|
||||
chmod 600 "$BUSINESS_LOGS_ENV_FILE"
|
||||
}
|
||||
|
||||
run_business_logs_compose() {
|
||||
# The independent file owns its identity. Inherited Compose variables cannot redirect it.
|
||||
SKILLHUB_BUSINESS_LOGS_PROJECT="$(get_env_value SKILLHUB_BUSINESS_LOGS_PROJECT skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")"
|
||||
SKILLHUB_BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")"
|
||||
export SKILLHUB_BUSINESS_LOGS_NETWORK
|
||||
compose_cmd="$(find_compose)"
|
||||
# shellcheck disable=SC2086
|
||||
$compose_cmd -p "$SKILLHUB_BUSINESS_LOGS_PROJECT" --env-file "$BUSINESS_LOGS_ENV_FILE" -f "$BUSINESS_LOGS_COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
validate_business_logs_endpoint() {
|
||||
case "$1" in
|
||||
http://*/*|https://*/*) ;;
|
||||
*) echo "Business Logs requires a full HTTP(S) Logs URL" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$1" in
|
||||
*'@'*|*'?'*|*'#'*|*[[:space:]]*|*'\\'*)
|
||||
echo "Business Logs URL must not contain credentials, query, fragment or whitespace" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
enable_business_logs() {
|
||||
receiver_endpoint="${BUSINESS_LOGS_ENDPOINT:-$(get_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT)}"
|
||||
validate_business_logs_endpoint "$receiver_endpoint"
|
||||
if [ -n "$BUSINESS_LOGS_NETWORK" ]; then
|
||||
case "$BUSINESS_LOGS_NETWORK" in
|
||||
*[!A-Za-z0-9_.-]*|[-.]* ) echo "Invalid Business Logs network name" >&2; exit 1 ;;
|
||||
esac
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK "$BUSINESS_LOGS_NETWORK"
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED true
|
||||
fi
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT "$receiver_endpoint"
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_ENABLED true
|
||||
run_compose up -d --no-deps --force-recreate server
|
||||
echo "Business Logs enabled; produce a new event and query the receiver."
|
||||
}
|
||||
|
||||
ensure_analytics_identity() {
|
||||
# Once initialized, the independent directory owns both identifiers. Neither
|
||||
# a recreated release file nor inherited Compose variables may redirect it.
|
||||
|
|
@ -535,7 +640,7 @@ build_runtime_lifecycle_args() {
|
|||
|
||||
if [ "$USE_ALIYUN" = "true" ]; then
|
||||
append_runtime_flag "--aliyun"
|
||||
elif [ "$EXPLICIT_REF" = "true" ] || [ "$SKILLHUB_REF" != "main" ]; then
|
||||
elif [ "$EXPLICIT_REF" = "true" ] || { [ "$SKILLHUB_REF" != "main" ] && [ "$SKILLHUB_REF" != "$SKILLHUB_VERSION_VALUE" ]; }; then
|
||||
append_runtime_option "--ref" "$SKILLHUB_REF"
|
||||
fi
|
||||
|
||||
|
|
@ -625,7 +730,11 @@ EOF
|
|||
;;
|
||||
clean)
|
||||
run_compose down
|
||||
if [ -d "$ANALYTICS_HOME" ]; then
|
||||
if [ -d "$BUSINESS_LOGS_HOME" ] || [ -n "$(get_env_value SKILLHUB_BUSINESS_LOGS_ENDPOINT)" ]; then
|
||||
# Retain operator credentials, Collector config and identities for safe restart.
|
||||
rm -f "$COMPOSE_FILE" "$BUSINESS_LOGS_OVERLAY_FILE" "$ENV_EXAMPLE_FILE"
|
||||
echo "Downloaded release files removed; operator environment and business-logs files retained."
|
||||
elif [ -d "$ANALYTICS_HOME" ]; then
|
||||
# Keep independent platform credentials and files; never strand its database.
|
||||
rm -f "$COMPOSE_FILE" "$OVERLAY_FILE" "$ENV_EXAMPLE_FILE" "$ENV_FILE"
|
||||
echo "SkillHub files removed; independent analytics files retained in $ANALYTICS_HOME"
|
||||
|
|
@ -642,6 +751,40 @@ EOF
|
|||
pull)
|
||||
run_compose pull
|
||||
;;
|
||||
business-logs-up)
|
||||
prepare_business_logs_files
|
||||
downstream_endpoint="$(get_env_value SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT "" "$BUSINESS_LOGS_ENV_FILE")"
|
||||
if [ -z "$downstream_endpoint" ]; then
|
||||
echo "Configure SKILLHUB_BUSINESS_LOGS_DOWNSTREAM_ENDPOINT (OTLP base URL) and optional JSON headers in $BUSINESS_LOGS_ENV_FILE, then retry." >&2
|
||||
exit 1
|
||||
fi
|
||||
validate_business_logs_endpoint "${downstream_endpoint%/}/v1/logs"
|
||||
# Read the independent project's stored network; do not overwrite operator secrets.
|
||||
BUSINESS_LOGS_NETWORK="$(get_env_value SKILLHUB_BUSINESS_LOGS_NETWORK skillhub-business-logs "$BUSINESS_LOGS_ENV_FILE")"
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK "$BUSINESS_LOGS_NETWORK"
|
||||
ensure_business_logs_network
|
||||
run_business_logs_compose config --quiet
|
||||
run_business_logs_compose up -d --force-recreate
|
||||
BUSINESS_LOGS_ENDPOINT=http://business-logs-collector:4318/v1/logs
|
||||
enable_business_logs
|
||||
;;
|
||||
business-logs-enable)
|
||||
enable_business_logs
|
||||
;;
|
||||
business-logs-disable)
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_ENABLED false
|
||||
set_env_value SKILLHUB_BUSINESS_LOGS_NETWORK_ENABLED false
|
||||
run_compose up -d --no-deps --force-recreate server
|
||||
echo "Business Logs disabled; receiver data and network retained."
|
||||
;;
|
||||
business-logs-down|business-logs-ps|business-logs-logs)
|
||||
[ -f "$BUSINESS_LOGS_ENV_FILE" ] || { echo "No optional Collector; run business-logs-up first" >&2; exit 1; }
|
||||
case "$COMMAND" in
|
||||
business-logs-down) run_business_logs_compose stop ;;
|
||||
business-logs-ps) run_business_logs_compose ps ;;
|
||||
business-logs-logs) run_business_logs_compose logs -f ;;
|
||||
esac
|
||||
;;
|
||||
analytics-up)
|
||||
prepare_analytics_files
|
||||
ensure_analytics_network
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue