Implement S6 doctor and A1 daily-board with fixtures.

Add operator health checks (make doctor) wrapping platform CLIs, and the fixtures-only daily board skill library with unit tests (make verify).
2026-07-27 12:41:40 -07:00
parent 0198ab6881
commit 15f6a6c713
24 changed files with 2011 additions and 40 deletions
@@ -1,6 +1,6 @@
 # Host scripts

-**Status:** S0b–S5 implemented. S6–S7 pending.
+**Status:** S0b–S6 implemented. S7 pending.

 All scripts wrap **`nemohermes` / `openshell` / Docker**. No parallel control API.

@@ -14,8 +14,8 @@ All scripts wrap **`nemohermes` / `openshell` / Docker**. No parallel control AP
 | `install/s2-models.sh` | S2: model + vision config + smoke | ✅ S2 |
 | `install/s4-sandbox.sh` | S4: sandbox verify (attach) or onboard | ✅ S4 |
 | `install/s5-policy-skills.sh` | S5: policy overlays + skills sync | ✅ S5 |
+| `doctor.sh` | Health checks (Docker, CLIs, sandbox, policy, skills, inference) | ✅ S6 |
 | `upgrade.sh` | Snapshot, pull pins, migrate, re-apply policy, doctor | ⏳ Pending |
-| `doctor.sh` | Health checks | ⏳ Pending |
 | `connect/*.sh` | Operator connect helpers (Square, QBO, Vagaro, channels) | ⏳ Pending |

 ## Shared library
@@ -42,6 +42,10 @@ All scripts wrap **`nemohermes` / `openshell` / Docker**. No parallel control AP
 ./scripts/install.sh --stage s5   # policy + skills
 ./scripts/install.sh --stage s3-s5 # S3 through S5

+# Health checks (S6)
+./scripts/doctor.sh               # human-readable
+./scripts/doctor.sh --json        # machine-readable
+
 # Or via Make
 make bootstrap
 make install
@@ -49,6 +53,7 @@ make install-s1
 make install-s2
 make install-s3-s5
 make install-s5
+make doctor
 ```

 ## Design reference
@@ -0,0 +1,355 @@
+#!/usr/bin/env bash
+# scripts/doctor.sh — S6: Lumina product health checks
+#
+# Composes platform-layer health checks:
+#   Docker · nemohermes · openshell · sandbox · policy · skills · inference
+#
+# Platform-first: wraps nemohermes / openshell / Docker. No parallel control API.
+#
+# Usage:
+#   ./scripts/doctor.sh              # full check
+#   ./scripts/doctor.sh --json       # machine-readable summary
+#   ./scripts/doctor.sh --help
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Source shared helpers
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR/lib/common.sh"
+# shellcheck source=lib/env.sh
+source "$SCRIPT_DIR/lib/env.sh"
+
+# ── Defaults ───────────────────────────────────────────────────────────────
+JSON_OUTPUT=0
+
+# ── Parse args ─────────────────────────────────────────────────────────────
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --help|-h)
+      cat <<EOF
+Usage: $(basename "$0") [OPTIONS]
+
+S6: Lumina product health checks.
+
+Checks:
+  Docker daemon, nemohermes CLI, openshell CLI, sandbox status,
+  policy overlays, skills directory, inference endpoint.
+
+Options:
+  --json    Machine-readable JSON summary
+  --help    Show this help
+
+Exit codes:
+  0  All critical checks passed (warnings are non-fatal)
+  1  One or more critical checks failed
+EOF
+      exit 0
+      ;;
+    --json)
+      JSON_OUTPUT=1
+      shift
+      ;;
+    *)
+      log_error "Unknown argument: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# ── State tracking ─────────────────────────────────────────────────────────
+CRITICAL_FAIL=0
+WARN_COUNT=0
+declare -a CHECK_RESULTS=()
+
+# Strip ANSI escape codes from a string
+strip_ansi() {
+  sed 's/\x1b\[[0-9;]*m//g' <<< "$1"
+}
+
+# Record a check result: "group|label|status|detail"
+record_check() {
+  local group="$1" label="$2" status="$3" detail="$4"
+  # Strip any ANSI codes that may have leaked from CLI output
+  detail="$(strip_ansi "$detail")"
+  CHECK_RESULTS+=("${group}|${label}|${status}|${detail}")
+  if [[ "$status" == "FAIL" ]]; then
+    CRITICAL_FAIL=1
+  elif [[ "$status" == "WARN" ]]; then
+    WARN_COUNT=$((WARN_COUNT + 1))
+  fi
+}
+
+# ── Load .env (best-effort; warn if missing) ───────────────────────────────
+if [[ $JSON_OUTPUT -eq 1 ]]; then
+  load_env >/dev/null 2>&1 || true
+else
+  load_env 2>/dev/null || true
+fi
+
+SANDBOX_NAME="$(get_sandbox_name)"
+SKILLS_DIR="$REPO_ROOT/skills"
+POLICY_DIR="$REPO_ROOT/policy/openshell/overlays"
+
+# ── Check: Docker ──────────────────────────────────────────────────────────
+check_docker() {
+  if ! cmd_exists docker; then
+    record_check "Docker" "CLI" "FAIL" "docker command not found"
+    return
+  fi
+  if ! docker info &>/dev/null; then
+    record_check "Docker" "Daemon" "FAIL" "docker daemon not running or not accessible"
+    return
+  fi
+  local version
+  version="$(docker --version 2>/dev/null | sed 's/^Docker version //' | cut -d',' -f1 | tr -d ' ')"
+  record_check "Docker" "Daemon" "OK" "running ($version)"
+}
+
+# ── Check: nemohermes CLI ──────────────────────────────────────────────────
+check_nemohermes() {
+  if ! cmd_exists nemohermes; then
+    record_check "CLI" "nemohermes" "FAIL" "nemohermes not found — install NemoClaw platform"
+    return
+  fi
+  local version
+  version="$(nemohermes --version 2>/dev/null | head -1 | grep -oP 'v[\d.]+' || echo 'unknown')"
+  record_check "CLI" "nemohermes" "OK" "$version"
+}
+
+# ── Check: openshell CLI ───────────────────────────────────────────────────
+check_openshell() {
+  if ! cmd_exists openshell; then
+    record_check "CLI" "openshell" "FAIL" "openshell not found — install OpenShell"
+    return
+  fi
+  local version
+  version="$(openshell --version 2>/dev/null | head -1 | grep -oP '[\d.]+' || echo 'unknown')"
+  record_check "CLI" "openshell" "OK" "$version"
+}
+
+# ── Check: Sandbox status ──────────────────────────────────────────────────
+check_sandbox() {
+  # Skip if nemohermes is missing (already flagged)
+  if ! cmd_exists nemohermes; then
+    record_check "Sandbox" "Status" "FAIL" "skipped — nemohermes not available"
+    return
+  fi
+
+  # Run nemohermes doctor for the sandbox — this is the authoritative platform check
+  local doctor_output
+  doctor_output="$(nemohermes "$SANDBOX_NAME" doctor 2>&1)" || true
+
+  # Check for summary line
+  if echo "$doctor_output" | grep -qi "Summary: healthy"; then
+    record_check "Sandbox" "Doctor" "OK" "$SANDBOX_NAME healthy"
+  elif echo "$doctor_output" | grep -qi "Summary:.*warning"; then
+    record_check "Sandbox" "Doctor" "WARN" "$SANDBOX_NAME has warnings"
+  elif echo "$doctor_output" | grep -qi "Summary:.*unhealthy\|Summary:.*critical"; then
+    record_check "Sandbox" "Doctor" "FAIL" "$SANDBOX_NAME unhealthy"
+  else
+    # Fallback: check status command
+    if nemohermes "$SANDBOX_NAME" status &>/dev/null; then
+      record_check "Sandbox" "Status" "OK" "$SANDBOX_NAME reachable"
+    else
+      record_check "Sandbox" "Status" "FAIL" "$SANDBOX_NAME not found or not reachable"
+    fi
+  fi
+}
+
+# ── Check: Policy overlays ─────────────────────────────────────────────────
+check_policy() {
+  # Skip if nemohermes is missing
+  if ! cmd_exists nemohermes; then
+    record_check "Policy" "Overlays" "FAIL" "skipped — nemohermes not available"
+    return
+  fi
+
+  # Check that policy overlay files exist in the repo
+  local inference_policy="$POLICY_DIR/inference.yaml"
+  if [[ ! -f "$inference_policy" ]]; then
+    record_check "Policy" "Overlay files" "WARN" "inference.yaml not found in $POLICY_DIR"
+  fi
+
+  # Check that lumina-inference preset is applied in the sandbox
+  local policy_list
+  policy_list="$(nemohermes "$SANDBOX_NAME" policy-list 2>&1)" || {
+    record_check "Policy" "policy-list" "FAIL" "could not list policy presets"
+    return
+  }
+
+  if echo "$policy_list" | grep -q "lumina-inference"; then
+    record_check "Policy" "lumina-inference" "OK" "preset applied"
+  else
+    record_check "Policy" "lumina-inference" "WARN" "preset not found in sandbox — run S5 or: nemohermes $SANDBOX_NAME policy-add --from-file $inference_policy --yes"
+  fi
+}
+
+# ── Check: Skills ──────────────────────────────────────────────────────────
+check_skills() {
+  if [[ ! -d "$SKILLS_DIR" ]]; then
+    record_check "Skills" "Directory" "FAIL" "skills directory not found at $SKILLS_DIR"
+    return
+  fi
+
+  # Count skill directories (exclude _lib and hidden)
+  local skill_count=0
+  local skill_with_md=0
+  for skill_dir in "$SKILLS_DIR"/*/; do
+    [[ -d "$skill_dir" ]] || continue
+    local name
+    name="$(basename "$skill_dir")"
+    [[ "$name" == "_lib" ]] && continue
+    [[ "$name" == "README.md" ]] && continue
+    skill_count=$((skill_count + 1))
+    if [[ -f "$skill_dir/SKILL.md" ]]; then
+      skill_with_md=$((skill_with_md + 1))
+    fi
+  done
+
+  if [[ $skill_count -eq 0 ]]; then
+    record_check "Skills" "Pack" "WARN" "no skill directories found in $SKILLS_DIR"
+  elif [[ $skill_with_md -lt $skill_count ]]; then
+    record_check "Skills" "Pack" "WARN" "$skill_with_md/$skill_count skills have SKILL.md"
+  else
+    record_check "Skills" "Pack" "OK" "$skill_count skills with SKILL.md"
+  fi
+}
+
+# ── Check: Inference endpoint ──────────────────────────────────────────────
+check_inference() {
+  # Check .env has the required keys
+  local env_file="${REPO_ROOT}/.env"
+  if [[ ! -f "$env_file" ]]; then
+    record_check "Inference" "Config" "FAIL" ".env not found — run S1 first"
+    return
+  fi
+
+  # Source .env to get variables (already done by load_env, but verify)
+  local base_url="${LUMINA_INFERENCE_BASE_URL:-}"
+  local model="${LUMINA_INFERENCE_MODEL:-}"
+
+  if [[ -z "$base_url" ]]; then
+    record_check "Inference" "Endpoint URL" "FAIL" "LUMINA_INFERENCE_BASE_URL not set in .env"
+    return
+  fi
+
+  if [[ -z "$model" ]]; then
+    record_check "Inference" "Model" "FAIL" "LUMINA_INFERENCE_MODEL not set in .env"
+    return
+  fi
+
+  # Check endpoint reachability
+  local url="$base_url"
+  # Ensure URL ends with /v1 for the models endpoint
+  if [[ "$url" != */v1 && "$url" != */v1/* ]]; then
+    url="${url%/}/v1"
+  fi
+
+  if curl -sf --max-time 15 "${url}/models" &>/dev/null; then
+    record_check "Inference" "Endpoint" "OK" "reachable ($base_url)"
+  else
+    record_check "Inference" "Endpoint" "FAIL" "unreachable at $base_url"
+  fi
+
+  # Check openshell inference config (optional — gateway may not be connected)
+  if cmd_exists openshell; then
+    local inf_output
+    inf_output="$(openshell inference get 2>&1)" || true
+    if echo "$inf_output" | grep -q "Provider:"; then
+      local provider
+      # Strip ANSI escape codes and whitespace
+      provider="$(echo "$inf_output" | grep "Provider:" | head -1 | sed 's/.*Provider: *//' | sed 's/\x1b\[[0-9;]*m//g' | tr -d '[:space:]')"
+      record_check "Inference" "Gateway route" "OK" "configured ($provider)"
+    else
+      record_check "Inference" "Gateway route" "WARN" "not configured via openshell"
+    fi
+  fi
+}
+
+# ── Run all checks ─────────────────────────────────────────────────────────
+if [[ $JSON_OUTPUT -eq 0 ]]; then
+  log_section "Lumina Doctor (S6)"
+fi
+
+check_docker
+check_nemohermes
+check_openshell
+check_sandbox
+check_policy
+check_skills
+check_inference
+
+# ── Output results ─────────────────────────────────────────────────────────
+if [[ $JSON_OUTPUT -eq 1 ]]; then
+  # Machine-readable JSON summary
+  checks_json="["
+  first=1
+  for entry in "${CHECK_RESULTS[@]}"; do
+    IFS='|' read -r group label status detail <<< "$entry"
+    if [[ $first -eq 1 ]]; then
+      first=0
+    else
+      checks_json+=","
+    fi
+    # Escape backslashes first, then double quotes (JSON-safe)
+    detail="${detail//\\/\\\\}"
+    detail="${detail//\"/\\\"}"
+    checks_json+="{\"group\":\"$group\",\"label\":\"$label\",\"status\":\"$status\",\"detail\":\"$detail\"}"
+  done
+  checks_json+="]"
+
+  overall="healthy"
+  [[ $CRITICAL_FAIL -eq 1 ]] && overall="unhealthy"
+
+  cat <<EOF
+{
+  "product": "Lumina",
+  "stage": "S6",
+  "sandbox": "$SANDBOX_NAME",
+  "overall": "$overall",
+  "critical_failures": $CRITICAL_FAIL,
+  "warnings": $WARN_COUNT,
+  "checks": $checks_json
+}
+EOF
+else
+  # Human-readable summary
+  log_section "Results"
+
+  ok_count=0
+  warn_count=0
+  fail_count=0
+
+  for entry in "${CHECK_RESULTS[@]}"; do
+    IFS='|' read -r group label status detail <<< "$entry"
+    case "$status" in
+      OK)   printf "  \033[0;32m[OK]\033[0m   %-12s %s — %s\n" "$group" "$label" "$detail" ;;
+      WARN) printf "  \033[1;33m[WARN]\033[0m %-12s %s — %s\n" "$group" "$label" "$detail" ;;
+      FAIL) printf "  \033[0;31m[FAIL]\033[0m %-12s %s — %s\n" "$group" "$label" "$detail" ;;
+    esac
+    case "$status" in
+      OK)   ok_count=$((ok_count + 1)) ;;
+      WARN) warn_count=$((warn_count + 1)) ;;
+      FAIL) fail_count=$((fail_count + 1)) ;;
+    esac
+  done
+
+  log_section "Summary"
+  printf "  Checks: %d OK, %d WARN, %d FAIL\n" "$ok_count" "$warn_count" "$fail_count"
+
+  if [[ $CRITICAL_FAIL -eq 1 ]]; then
+    printf "  Overall: \033[0;31mUNHEALTHY\033[0m\n"
+  elif [[ $WARN_COUNT -gt 0 ]]; then
+    printf "  Overall: \033[1;33mHEALTHY (with warnings)\033[0m\n"
+  else
+    printf "  Overall: \033[0;32mHEALTHY\033[0m\n"
+  fi
+fi
+
+# ── Exit code ──────────────────────────────────────────────────────────────
+if [[ $CRITICAL_FAIL -eq 1 ]]; then
+  exit 1
+fi
+exit 0