From 60bc2fb9d07ac1fc842f5b1fd6ca7eba199f3df6 Mon Sep 17 00:00:00 2001 From: Ty Balascio Date: Sat, 19 Sep 2026 16:02:04 -0700 Subject: [PATCH] feat: add schema drift handling and rebuild --- .planning/milestones/v0.5/M3/roadmap.md | 9 +-- .../milestones/v0.5/M3/task_breakdown.md | 76 ++++++++++++++----- backend/db/__init__.py | 25 ++++++ backend/db/schema.sql | 2 + backend/tests/test_schema_drift.py | 37 +++++++++ 5 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 backend/tests/test_schema_drift.py diff --git a/.planning/milestones/v0.5/M3/roadmap.md b/.planning/milestones/v0.5/M3/roadmap.md index e5bef18..fe4ded3 100644 --- a/.planning/milestones/v0.5/M3/roadmap.md +++ b/.planning/milestones/v0.5/M3/roadmap.md @@ -11,7 +11,8 @@ matching the reviewed ready set exists in the store. Create works with inference - Woo connection (WordPress application password) and taxonomy sync. - Review to the full ready set, with per-slab pricing and override. -- Inference assist (Call 1 / Call 2), optional and never blocking create. +- Inference assist (Call 1 / Call 2) is optional; create is independent of inference + (`METADATA-75`). - Publish: create the Woo product; `SLAB-UAT-*` always a draft. ## Sources of truth @@ -23,8 +24,4 @@ matching the reviewed ready set exists in the store. Create works with inference ## Exit -Per `docs/MILESTONES.md` and `docs/capabilities/CREATE.md` (QUALITY-110). - -## Note - -M3 implementation has not started. Decisions already ruled live in the contracts, not here. +Per `docs/capabilities/CREATE.md` (`QUALITY-110`). diff --git a/.planning/milestones/v0.5/M3/task_breakdown.md b/.planning/milestones/v0.5/M3/task_breakdown.md index 6a9fd8c..b048e5d 100644 --- a/.planning/milestones/v0.5/M3/task_breakdown.md +++ b/.planning/milestones/v0.5/M3/task_breakdown.md @@ -6,35 +6,74 @@ Objectives: `roadmap.md` in this folder. Behavior and acceptance: `docs/capabili Scope and IDs: `docs/MILESTONES.md`. Decisions live in `docs/contracts/`, not here. TDD every slice: the failing test precedes its implementation; never parallelize RED and GREEN of -the same scenario. +the same scenario. Execute slices **serially** (one local GPU; see `AGENTS.md`). Frontend ships +**with** the slice that first needs the screen, not as a trailing blob. -| Slice | Delivers | M3 IDs | +## Hard dependencies + +``` +S0 full schema.sql + factory + health + └─ S1 crypto + Settings + ├─ S2 slabs CRUD (needs S0 tables; tests seed fixture taxonomy rows) + │ └─ S3 photos (needs slab_id) + └─ S4 Woo + taxonomy (needs S1 credentials only — not S2/S3) + │ + S5 pricing + title/short-desc + ready ← S2 AND S4 + ├─ S6 publish ← S3 + S4 + S5 + └─ S7 inference last ← S2 + S3 + S4 + S5; not S6 +``` + +Default serial order for Atlas: **S0 → S1 → S2 → S3 → S4 → S5 → S6 → S7 → Exit**. + +S4 does not wait on S2/S3. If S2 is blocked after S1, start S4. S7 does not wait on S6 +(`METADATA-75`). + +## Slices + +| Slice | Delivers | M3 IDs | Hard deps | +|---|---|---|---| +| S0 | factory; **full** `schema.sql` from `docs/contracts/DATA-MODEL.md` (including empty taxonomy tables); config/startup guards; health/meta; JSON logs; Dockerfile | APP-30, APP-45, APP-55, APP-65, SETTINGS-10, SETTINGS-25, OPS-25 | — | +| S1 | AES-GCM crypto + Settings API/UI (masked, write-only secrets). Missing key refuses startup (S0); this slice encrypts/decrypts. Decrypt failure → owner re-enters credentials. | SETTINGS-5/15/20/25/30/75/85/90, QUALITY-125 | S0 | +| S2 | slabs CRUD, optimistic concurrency, soft delete. Fixture `woo_categories` rows for FK tests. Frontend: API client + slab list. | APP-5/15/20/35/40, REVIEW-45 | S0 | +| S3 | photos (multipart → BLOBs, original/processed, 410 after purge). Frontend: photo attach on the draft. | CAPTURE-95/105/130, PUBLISH-40 | S2 | +| S4 | Woo client (application-password Basic) + taxonomy sync + test-woo. Fills the S0 taxonomy tables. Owns `taxonomy_anchor` / `taxonomy_last_sync_at`. Frontend: Settings test-woo. | METADATA-5/10/15/20/30, SETTINGS-40/50/51/55/60/80, QUALITY-35 | S1 | +| S5 | species `$/bdft` rates; deterministic title + short_description; server-derived `ready`. Inference off: owner types `description` by hand (`METADATA-70`). Frontend: Review screen. | REVIEW-5/10/15/20/25/30/35/40/50, METADATA-70/80/85/90, SETTINGS-35, QUALITY-90/95 | S2, S4 | +| S6 | publish sequence, UAT draft, duplicate/stale/partial failure; upload processed PNGs; publish-time taxonomy refresh when last sync is stale (`SETTINGS-60`). Frontend: Publish screen. | PUBLISH-5/10/15/20/25/30/31/35/36/37/38/40/45/50/55/60, SETTINGS-60 | S3, S4, S5 | +| S7 | inference last: Call 1, Call 2 (one static seed; calculated title/short_description injected into the Call 2 prompt), prompt revision REST. Never blocks create. Frontend: Generate text control. | METADATA-35/40/45/50/55/60/65/66/75/95/100/105/106/110/115, SETTINGS-45/70 | S2, S3, S4, S5 | +| Exit | inference-off suite, Woo-mock integration, Compose UI E2E, real `SLAB-UAT-*` draft | QUALITY-20/35/50/105/110/120 | S0–S7 | + +IDs split on purpose (do not “move” them to one slice): + +| ID | Slices | Split | |---|---|---| -| S0 | factory, config/startup guards, health/meta, JSON logs, Dockerfile | APP-30, APP-45, APP-55, APP-65, SETTINGS-10, SETTINGS-25, OPS-25 | -| S1 | AES-GCM crypto + Settings (masked, write-only secrets) | SETTINGS-5/15/20/25/30/35/75/85/90, QUALITY-125 | -| S2 | slabs CRUD, optimistic concurrency, soft delete | APP-5/15/20/35/40, REVIEW-45 | -| S3 | photos (multipart → BLOBs, original/processed, 410 after purge) | CAPTURE-95/105/130, PUBLISH-40 | -| S4 | Woo client (application-password Basic) + taxonomy sync + test-woo | METADATA-5/10/15/20/30, SETTINGS-40/50/51/55/60/80, QUALITY-35 | -| S5 | pricing, deterministic templates, server-derived ready | REVIEW-5/10/15/20/25/30/35/40/50, METADATA-70/80/85/90, QUALITY-90/95 | -| S6 | publish (sequence, UAT draft, duplicate/stale/partial failure) | PUBLISH-5/10/15/20/25/30/31/35/36/37/38/40/45/50/55/60 | -| S7 | inference last (Call 1 / Call 2, prompt revisions, template fallback) | METADATA-35/40/45/50/55/60/65/66/75/95/100/105/106/110/115, SETTINGS-45/70 | -| Cross | frontend API client; Review / Settings / Publish screens | UI for the slices above | -| Exit | inference-off suite, Woo-mock integration, Compose UI E2E, real UAT draft | QUALITY-20/35/50/105/110/120 | +| SETTINGS-25 | S0 + S1 | S0 refuses startup without the key; S1 implements AES-GCM | +| SETTINGS-60 | S4 + S6 | S4 owns anchor + cache rebuild; S6 refreshes on publish when stale | +| PUBLISH-40 | S3 + S6 | S3 stores PNG BLOBs; S6 uploads them to Woo | +| QUALITY-35 | S4 + Exit | S4 writes the Woo-mock tests; Exit re-runs them | ## Parked work items -1. The seven per-leaf long-description prose templates (the specialized Call 2 prompts) are - authored when the Call 2 slice is built (`METADATA-85`): Knife blanks, Dimensional wood, Live - edge wood, Bowl blanks, logs and caps, Pen blanks, Pepper mill blanks. -2. `Specialty cuts` is out of M3: no title pattern, no prose template, no Call 2 prompt. -3. S7 ships the prompt revision REST interface (`METADATA-106`) so UAT prompt tuning can be done - against the running server. There is no prompt editing UI; tuning is an API call. +1. The seven per-leaf Call 2 prose templates are **not M3**. S7 ships the single static `call2` + seed in `docs/contracts/PROMPTS.md` §6. The model keys off the already-calculated title, + short_description, fact block, photos, and metadata. Per-leaf templates are later. +2. `Specialty cuts` is out of M3: no title pattern, no prose template, no Call 2 prompt. Owner will + not inventory that leaf in M3. +3. S7 ships the prompt revision REST interface (`METADATA-106`) so UAT prompt tuning is an API call + on the trusted LAN. There is no prompt editing UI. 4. Authentication for the prompt revision interface is `later` (`METADATA-107`), not M3. ## Frontend layout New code stays outside the Capture union: `frontend/src/api/*`, `frontend/src/create/*`. +| With slice | Screen | +|---|---| +| S1 | Settings (store creds, inference flags, secret masking) | +| S2 | Slab list / draft resume | +| S5 | Review (ready set, primary wood category, price, title/short_description) | +| S6 | Publish | +| S7 | Generate text | + ## Verification Slice bar: `docs/contracts/TEST-SPEC.md`. Docs gate: `python tools/check_requirements.py`. @@ -44,3 +83,4 @@ Exit bar: `docs/MILESTONES.md` and `docs/capabilities/CREATE.md`. - Long-horizon multi-step agentic loops (more than one or two tool round-trips). - M3 implementation has not started. + diff --git a/backend/db/__init__.py b/backend/db/__init__.py index a45b8a4..1e34e18 100644 --- a/backend/db/__init__.py +++ b/backend/db/__init__.py @@ -4,6 +4,7 @@ from __future__ import annotations import os import sqlite3 import threading +import logging from pathlib import Path from typing import Any @@ -42,6 +43,21 @@ def get_connection(db_path: str | None = None) -> sqlite3.Connection: _current_path = db_path return _conn +def _has_schema_version(conn: sqlite3.Connection) -> bool: + try: + conn.execute("SELECT 1 FROM schema_version").fetchone() + return True + except sqlite3.OperationalError: + return False + +def _has_settings_value_column(conn: sqlite3.Connection) -> bool: + try: + info = conn.execute("PRAGMA table_info(settings)").fetchall() + columns = {row[1]: row[2] for row in info} + return 'value' in columns + except sqlite3.OperationalError: + return False + def init_db(db_path: str | None = None) -> sqlite3.Connection: conn = get_connection(db_path) @@ -53,6 +69,15 @@ def init_db(db_path: str | None = None) -> sqlite3.Connection: with _lock: conn.executescript(schema) conn.commit() + if not _has_schema_version(conn): + if not _has_settings_value_column(conn): + logger = logging.getLogger("slab.db") + logger.info("Detected pre-M3 DB, rebuilding schema") + conn.executescript("DROP TABLE IF EXISTS settings") + conn.executescript(schema) +conn.commit() + conn.execute("INSERT OR REPLACE INTO schema_version (version) VALUES (1)") + return conn diff --git a/backend/db/schema.sql b/backend/db/schema.sql index 1037bf3..7a63c9e 100644 --- a/backend/db/schema.sql +++ b/backend/db/schema.sql @@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS settings ( value_enc TEXT ); +CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); + -- 7. prompts + prompt_revisions (append-only revisions) CREATE TABLE IF NOT EXISTS prompts ( id INTEGER PRIMARY KEY, diff --git a/backend/tests/test_schema_drift.py b/backend/tests/test_schema_drift.py new file mode 100644 index 0000000..b602694 --- /dev/null +++ b/backend/tests/test_schema_drift.py @@ -0,0 +1,37 @@ +import os +import tempfile +import sqlite3 + +import backend.db as db_mod + + + +def _setup_temp_db_with_m2_schema(tmp_dir: str) -> str: + db_path = os.path.join(tmp_dir, "test.db") + conn = sqlite3.connect(db_path) + # create old settings table + conn.execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT, value_enc TEXT)") + conn.commit() + conn.close() + return db_path + + +def test_drift_detection_and_rebuild(monkeypatch): + with tempfile.TemporaryDirectory() as tmp: + db_path = _setup_temp_db_with_m2_schema(tmp) + monkeypatch.setenv("SLAB_DB_PATH", db_path) + # init_db should rebuild + conn = db_mod.init_db() + # check schema_version exists + cur = conn.execute("SELECT version FROM schema_version").fetchone() + assert cur is not None + assert cur[0] == 1 + # check settings has columns value and value_enc + info = conn.execute("PRAGMA table_info(settings)").fetchall() + cols = {row[1] for row in info} + assert "value" in cols + assert "value_enc" in cols + # also ensure old settings table dropped and recreated + # no other data + rows = conn.execute("SELECT key FROM settings").fetchall() + assert rows == []