Ty
15f6a6c713
Add operator health checks (make doctor) wrapping platform CLIs, and the fixtures-only daily board skill library with unit tests (make verify).
356 lines
12 KiB
Bash
Executable File
356 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/doctor.sh — S6: Lumina product health checks
|
|
#
|
|
# Composes platform-layer health checks:
|
|
# Docker · nemohermes · openshell · sandbox · policy · skills · inference
|
|
#
|
|
# Platform-first: wraps nemohermes / openshell / Docker. No parallel control API.
|
|
#
|
|
# Usage:
|
|
# ./scripts/doctor.sh # full check
|
|
# ./scripts/doctor.sh --json # machine-readable summary
|
|
# ./scripts/doctor.sh --help
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# Source shared helpers
|
|
# shellcheck source=lib/common.sh
|
|
source "$SCRIPT_DIR/lib/common.sh"
|
|
# shellcheck source=lib/env.sh
|
|
source "$SCRIPT_DIR/lib/env.sh"
|
|
|
|
# ── Defaults ───────────────────────────────────────────────────────────────
|
|
JSON_OUTPUT=0
|
|
|
|
# ── Parse args ─────────────────────────────────────────────────────────────
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--help|-h)
|
|
cat <<EOF
|
|
Usage: $(basename "$0") [OPTIONS]
|
|
|
|
S6: Lumina product health checks.
|
|
|
|
Checks:
|
|
Docker daemon, nemohermes CLI, openshell CLI, sandbox status,
|
|
policy overlays, skills directory, inference endpoint.
|
|
|
|
Options:
|
|
--json Machine-readable JSON summary
|
|
--help Show this help
|
|
|
|
Exit codes:
|
|
0 All critical checks passed (warnings are non-fatal)
|
|
1 One or more critical checks failed
|
|
EOF
|
|
exit 0
|
|
;;
|
|
--json)
|
|
JSON_OUTPUT=1
|
|
shift
|
|
;;
|
|
*)
|
|
log_error "Unknown argument: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# ── State tracking ─────────────────────────────────────────────────────────
|
|
CRITICAL_FAIL=0
|
|
WARN_COUNT=0
|
|
declare -a CHECK_RESULTS=()
|
|
|
|
# Strip ANSI escape codes from a string
|
|
strip_ansi() {
|
|
sed 's/\x1b\[[0-9;]*m//g' <<< "$1"
|
|
}
|
|
|
|
# Record a check result: "group|label|status|detail"
|
|
record_check() {
|
|
local group="$1" label="$2" status="$3" detail="$4"
|
|
# Strip any ANSI codes that may have leaked from CLI output
|
|
detail="$(strip_ansi "$detail")"
|
|
CHECK_RESULTS+=("${group}|${label}|${status}|${detail}")
|
|
if [[ "$status" == "FAIL" ]]; then
|
|
CRITICAL_FAIL=1
|
|
elif [[ "$status" == "WARN" ]]; then
|
|
WARN_COUNT=$((WARN_COUNT + 1))
|
|
fi
|
|
}
|
|
|
|
# ── Load .env (best-effort; warn if missing) ───────────────────────────────
|
|
if [[ $JSON_OUTPUT -eq 1 ]]; then
|
|
load_env >/dev/null 2>&1 || true
|
|
else
|
|
load_env 2>/dev/null || true
|
|
fi
|
|
|
|
SANDBOX_NAME="$(get_sandbox_name)"
|
|
SKILLS_DIR="$REPO_ROOT/skills"
|
|
POLICY_DIR="$REPO_ROOT/policy/openshell/overlays"
|
|
|
|
# ── Check: Docker ──────────────────────────────────────────────────────────
|
|
check_docker() {
|
|
if ! cmd_exists docker; then
|
|
record_check "Docker" "CLI" "FAIL" "docker command not found"
|
|
return
|
|
fi
|
|
if ! docker info &>/dev/null; then
|
|
record_check "Docker" "Daemon" "FAIL" "docker daemon not running or not accessible"
|
|
return
|
|
fi
|
|
local version
|
|
version="$(docker --version 2>/dev/null | sed 's/^Docker version //' | cut -d',' -f1 | tr -d ' ')"
|
|
record_check "Docker" "Daemon" "OK" "running ($version)"
|
|
}
|
|
|
|
# ── Check: nemohermes CLI ──────────────────────────────────────────────────
|
|
check_nemohermes() {
|
|
if ! cmd_exists nemohermes; then
|
|
record_check "CLI" "nemohermes" "FAIL" "nemohermes not found — install NemoClaw platform"
|
|
return
|
|
fi
|
|
local version
|
|
version="$(nemohermes --version 2>/dev/null | head -1 | grep -oP 'v[\d.]+' || echo 'unknown')"
|
|
record_check "CLI" "nemohermes" "OK" "$version"
|
|
}
|
|
|
|
# ── Check: openshell CLI ───────────────────────────────────────────────────
|
|
check_openshell() {
|
|
if ! cmd_exists openshell; then
|
|
record_check "CLI" "openshell" "FAIL" "openshell not found — install OpenShell"
|
|
return
|
|
fi
|
|
local version
|
|
version="$(openshell --version 2>/dev/null | head -1 | grep -oP '[\d.]+' || echo 'unknown')"
|
|
record_check "CLI" "openshell" "OK" "$version"
|
|
}
|
|
|
|
# ── Check: Sandbox status ──────────────────────────────────────────────────
|
|
check_sandbox() {
|
|
# Skip if nemohermes is missing (already flagged)
|
|
if ! cmd_exists nemohermes; then
|
|
record_check "Sandbox" "Status" "FAIL" "skipped — nemohermes not available"
|
|
return
|
|
fi
|
|
|
|
# Run nemohermes doctor for the sandbox — this is the authoritative platform check
|
|
local doctor_output
|
|
doctor_output="$(nemohermes "$SANDBOX_NAME" doctor 2>&1)" || true
|
|
|
|
# Check for summary line
|
|
if echo "$doctor_output" | grep -qi "Summary: healthy"; then
|
|
record_check "Sandbox" "Doctor" "OK" "$SANDBOX_NAME healthy"
|
|
elif echo "$doctor_output" | grep -qi "Summary:.*warning"; then
|
|
record_check "Sandbox" "Doctor" "WARN" "$SANDBOX_NAME has warnings"
|
|
elif echo "$doctor_output" | grep -qi "Summary:.*unhealthy\|Summary:.*critical"; then
|
|
record_check "Sandbox" "Doctor" "FAIL" "$SANDBOX_NAME unhealthy"
|
|
else
|
|
# Fallback: check status command
|
|
if nemohermes "$SANDBOX_NAME" status &>/dev/null; then
|
|
record_check "Sandbox" "Status" "OK" "$SANDBOX_NAME reachable"
|
|
else
|
|
record_check "Sandbox" "Status" "FAIL" "$SANDBOX_NAME not found or not reachable"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ── Check: Policy overlays ─────────────────────────────────────────────────
|
|
check_policy() {
|
|
# Skip if nemohermes is missing
|
|
if ! cmd_exists nemohermes; then
|
|
record_check "Policy" "Overlays" "FAIL" "skipped — nemohermes not available"
|
|
return
|
|
fi
|
|
|
|
# Check that policy overlay files exist in the repo
|
|
local inference_policy="$POLICY_DIR/inference.yaml"
|
|
if [[ ! -f "$inference_policy" ]]; then
|
|
record_check "Policy" "Overlay files" "WARN" "inference.yaml not found in $POLICY_DIR"
|
|
fi
|
|
|
|
# Check that lumina-inference preset is applied in the sandbox
|
|
local policy_list
|
|
policy_list="$(nemohermes "$SANDBOX_NAME" policy-list 2>&1)" || {
|
|
record_check "Policy" "policy-list" "FAIL" "could not list policy presets"
|
|
return
|
|
}
|
|
|
|
if echo "$policy_list" | grep -q "lumina-inference"; then
|
|
record_check "Policy" "lumina-inference" "OK" "preset applied"
|
|
else
|
|
record_check "Policy" "lumina-inference" "WARN" "preset not found in sandbox — run S5 or: nemohermes $SANDBOX_NAME policy-add --from-file $inference_policy --yes"
|
|
fi
|
|
}
|
|
|
|
# ── Check: Skills ──────────────────────────────────────────────────────────
|
|
check_skills() {
|
|
if [[ ! -d "$SKILLS_DIR" ]]; then
|
|
record_check "Skills" "Directory" "FAIL" "skills directory not found at $SKILLS_DIR"
|
|
return
|
|
fi
|
|
|
|
# Count skill directories (exclude _lib and hidden)
|
|
local skill_count=0
|
|
local skill_with_md=0
|
|
for skill_dir in "$SKILLS_DIR"/*/; do
|
|
[[ -d "$skill_dir" ]] || continue
|
|
local name
|
|
name="$(basename "$skill_dir")"
|
|
[[ "$name" == "_lib" ]] && continue
|
|
[[ "$name" == "README.md" ]] && continue
|
|
skill_count=$((skill_count + 1))
|
|
if [[ -f "$skill_dir/SKILL.md" ]]; then
|
|
skill_with_md=$((skill_with_md + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $skill_count -eq 0 ]]; then
|
|
record_check "Skills" "Pack" "WARN" "no skill directories found in $SKILLS_DIR"
|
|
elif [[ $skill_with_md -lt $skill_count ]]; then
|
|
record_check "Skills" "Pack" "WARN" "$skill_with_md/$skill_count skills have SKILL.md"
|
|
else
|
|
record_check "Skills" "Pack" "OK" "$skill_count skills with SKILL.md"
|
|
fi
|
|
}
|
|
|
|
# ── Check: Inference endpoint ──────────────────────────────────────────────
|
|
check_inference() {
|
|
# Check .env has the required keys
|
|
local env_file="${REPO_ROOT}/.env"
|
|
if [[ ! -f "$env_file" ]]; then
|
|
record_check "Inference" "Config" "FAIL" ".env not found — run S1 first"
|
|
return
|
|
fi
|
|
|
|
# Source .env to get variables (already done by load_env, but verify)
|
|
local base_url="${LUMINA_INFERENCE_BASE_URL:-}"
|
|
local model="${LUMINA_INFERENCE_MODEL:-}"
|
|
|
|
if [[ -z "$base_url" ]]; then
|
|
record_check "Inference" "Endpoint URL" "FAIL" "LUMINA_INFERENCE_BASE_URL not set in .env"
|
|
return
|
|
fi
|
|
|
|
if [[ -z "$model" ]]; then
|
|
record_check "Inference" "Model" "FAIL" "LUMINA_INFERENCE_MODEL not set in .env"
|
|
return
|
|
fi
|
|
|
|
# Check endpoint reachability
|
|
local url="$base_url"
|
|
# Ensure URL ends with /v1 for the models endpoint
|
|
if [[ "$url" != */v1 && "$url" != */v1/* ]]; then
|
|
url="${url%/}/v1"
|
|
fi
|
|
|
|
if curl -sf --max-time 15 "${url}/models" &>/dev/null; then
|
|
record_check "Inference" "Endpoint" "OK" "reachable ($base_url)"
|
|
else
|
|
record_check "Inference" "Endpoint" "FAIL" "unreachable at $base_url"
|
|
fi
|
|
|
|
# Check openshell inference config (optional — gateway may not be connected)
|
|
if cmd_exists openshell; then
|
|
local inf_output
|
|
inf_output="$(openshell inference get 2>&1)" || true
|
|
if echo "$inf_output" | grep -q "Provider:"; then
|
|
local provider
|
|
# Strip ANSI escape codes and whitespace
|
|
provider="$(echo "$inf_output" | grep "Provider:" | head -1 | sed 's/.*Provider: *//' | sed 's/\x1b\[[0-9;]*m//g' | tr -d '[:space:]')"
|
|
record_check "Inference" "Gateway route" "OK" "configured ($provider)"
|
|
else
|
|
record_check "Inference" "Gateway route" "WARN" "not configured via openshell"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ── Run all checks ─────────────────────────────────────────────────────────
|
|
if [[ $JSON_OUTPUT -eq 0 ]]; then
|
|
log_section "Lumina Doctor (S6)"
|
|
fi
|
|
|
|
check_docker
|
|
check_nemohermes
|
|
check_openshell
|
|
check_sandbox
|
|
check_policy
|
|
check_skills
|
|
check_inference
|
|
|
|
# ── Output results ─────────────────────────────────────────────────────────
|
|
if [[ $JSON_OUTPUT -eq 1 ]]; then
|
|
# Machine-readable JSON summary
|
|
checks_json="["
|
|
first=1
|
|
for entry in "${CHECK_RESULTS[@]}"; do
|
|
IFS='|' read -r group label status detail <<< "$entry"
|
|
if [[ $first -eq 1 ]]; then
|
|
first=0
|
|
else
|
|
checks_json+=","
|
|
fi
|
|
# Escape backslashes first, then double quotes (JSON-safe)
|
|
detail="${detail//\\/\\\\}"
|
|
detail="${detail//\"/\\\"}"
|
|
checks_json+="{\"group\":\"$group\",\"label\":\"$label\",\"status\":\"$status\",\"detail\":\"$detail\"}"
|
|
done
|
|
checks_json+="]"
|
|
|
|
overall="healthy"
|
|
[[ $CRITICAL_FAIL -eq 1 ]] && overall="unhealthy"
|
|
|
|
cat <<EOF
|
|
{
|
|
"product": "Lumina",
|
|
"stage": "S6",
|
|
"sandbox": "$SANDBOX_NAME",
|
|
"overall": "$overall",
|
|
"critical_failures": $CRITICAL_FAIL,
|
|
"warnings": $WARN_COUNT,
|
|
"checks": $checks_json
|
|
}
|
|
EOF
|
|
else
|
|
# Human-readable summary
|
|
log_section "Results"
|
|
|
|
ok_count=0
|
|
warn_count=0
|
|
fail_count=0
|
|
|
|
for entry in "${CHECK_RESULTS[@]}"; do
|
|
IFS='|' read -r group label status detail <<< "$entry"
|
|
case "$status" in
|
|
OK) printf " \033[0;32m[OK]\033[0m %-12s %s — %s\n" "$group" "$label" "$detail" ;;
|
|
WARN) printf " \033[1;33m[WARN]\033[0m %-12s %s — %s\n" "$group" "$label" "$detail" ;;
|
|
FAIL) printf " \033[0;31m[FAIL]\033[0m %-12s %s — %s\n" "$group" "$label" "$detail" ;;
|
|
esac
|
|
case "$status" in
|
|
OK) ok_count=$((ok_count + 1)) ;;
|
|
WARN) warn_count=$((warn_count + 1)) ;;
|
|
FAIL) fail_count=$((fail_count + 1)) ;;
|
|
esac
|
|
done
|
|
|
|
log_section "Summary"
|
|
printf " Checks: %d OK, %d WARN, %d FAIL\n" "$ok_count" "$warn_count" "$fail_count"
|
|
|
|
if [[ $CRITICAL_FAIL -eq 1 ]]; then
|
|
printf " Overall: \033[0;31mUNHEALTHY\033[0m\n"
|
|
elif [[ $WARN_COUNT -gt 0 ]]; then
|
|
printf " Overall: \033[1;33mHEALTHY (with warnings)\033[0m\n"
|
|
else
|
|
printf " Overall: \033[0;32mHEALTHY\033[0m\n"
|
|
fi
|
|
fi
|
|
|
|
# ── Exit code ──────────────────────────────────────────────────────────────
|
|
if [[ $CRITICAL_FAIL -eq 1 ]]; then
|
|
exit 1
|
|
fi
|
|
exit 0
|