fix(inference): #86 tighten Call 1 prompt and closed lists
Ship locked expert Call 1 seed: order cats before edge, species=best confidence, mill-count edge rules, Wild for cookies/burl, multi figure, feature_tag_ids only (no free-text bags, no grade). LISTS keys match the seed; feat- prefix stripped for the model. Gate and review UI apply feature_tag_ids. Seed auto-activates on deploy when still seed-authored.
This commit is contained in:
@@ -58,6 +58,101 @@ def test_prompt_seed_and_revision(env):
|
||||
assert client.get("/api/v1/prompts/call1").json()["active_content"] is not None
|
||||
|
||||
|
||||
def test_call1_seed_is_locked_expert_prompt(env):
|
||||
"""#86: shipped Call 1 seed is the locked expert prompt (no free-text bags, no grade)."""
|
||||
client, _conn = env
|
||||
from backend.inference import CALL1_SEED
|
||||
|
||||
body = client.get("/api/v1/prompts/call1").json()
|
||||
assert body["active_content"] == CALL1_SEED
|
||||
assert "wood identification expert" in CALL1_SEED
|
||||
assert "feature_tag_ids" in CALL1_SEED
|
||||
assert "Decide wood categories before edge type" in CALL1_SEED
|
||||
assert "highest-confidence" in CALL1_SEED
|
||||
assert "Wild edge" in CALL1_SEED
|
||||
assert "cookies and burl" in CALL1_SEED.lower() or "cookies and burl" in CALL1_SEED
|
||||
assert "grade_term_id" not in CALL1_SEED
|
||||
assert '"character"' not in CALL1_SEED
|
||||
assert '"inclusions"' not in CALL1_SEED
|
||||
assert "character\":" not in CALL1_SEED.replace(" ", "")
|
||||
|
||||
|
||||
def test_taxonomy_lists_call1_closed_shape():
|
||||
"""#86: Call 1 lists are closed Woo picks only — no grade/thickness/moisture; feat names stripped."""
|
||||
import tempfile, os, shutil
|
||||
tmp = tempfile.mkdtemp(prefix="tax_")
|
||||
os.environ["SLAB_DB_PATH"] = os.path.join(tmp, "t.db")
|
||||
os.environ["SLAB_IMAGES_DIR"] = tmp
|
||||
os.environ.setdefault("WOO_BASE_URL", "https://x.com")
|
||||
os.environ.setdefault(
|
||||
"SLAB_AES_KEY",
|
||||
base64.b64encode(b"0123456789abcdef0123456789abcdef").decode(),
|
||||
)
|
||||
db_mod._reset_for_tests()
|
||||
conn = db_mod.init_db()
|
||||
conn.execute("INSERT INTO woo_categories(id,name,slug,parent_id,woo_id) VALUES (10,'Species','species',NULL,10)")
|
||||
conn.execute("INSERT INTO woo_categories(id,name,slug,parent_id,woo_id) VALUES (11,'Walnut','walnut',10,11)")
|
||||
conn.execute("INSERT INTO woo_categories(id,name,slug,parent_id,woo_id) VALUES (20,'Wcat','wcat',NULL,20)")
|
||||
conn.execute("INSERT INTO woo_categories(id,name,slug,parent_id,woo_id) VALUES (21,'Live edge wood','liveedge',20,21)")
|
||||
conn.execute("INSERT INTO woo_attributes(id,name,slug,woo_id) VALUES (6,'Edge type','pa_etype',6)")
|
||||
conn.execute("INSERT INTO woo_attributes(id,name,slug,woo_id) VALUES (8,'Figure','pa_fig',8)")
|
||||
conn.execute("INSERT INTO woo_attributes(id,name,slug,woo_id) VALUES (9,'Grade','pa_gr',9)")
|
||||
conn.execute("INSERT INTO woo_attributes(id,name,slug,woo_id) VALUES (10,'Thickness','pa_thk',10)")
|
||||
conn.execute("INSERT INTO woo_attributes(id,name,slug,woo_id) VALUES (7,'Moisture','pa_mst',7)")
|
||||
conn.execute("INSERT INTO woo_attribute_terms(id,attribute_id,name,slug,woo_id) VALUES (321,6,'Square edge','sqe',321)")
|
||||
conn.execute("INSERT INTO woo_attribute_terms(id,attribute_id,name,slug,woo_id) VALUES (328,8,'Plain','pl',328)")
|
||||
conn.execute("INSERT INTO woo_attribute_terms(id,attribute_id,name,slug,woo_id) VALUES (342,9,'Fancy','fnc',342)")
|
||||
conn.execute("INSERT INTO woo_attribute_terms(id,attribute_id,name,slug,woo_id) VALUES (350,10,'1\"- 1 1/2\"','4-6qtr',350)")
|
||||
conn.execute("INSERT INTO woo_attribute_terms(id,attribute_id,name,slug,woo_id) VALUES (326,7,'Kiln dried','kd',326)")
|
||||
conn.execute("INSERT INTO woo_tags(id,name,slug,woo_id) VALUES (1,'feat-knot','feat-knot',1)")
|
||||
conn.execute("INSERT INTO woo_tags(id,name,slug,woo_id) VALUES (2,'fig-curl','fig-curl',2)")
|
||||
conn.commit()
|
||||
from backend.inference import _taxonomy_lists
|
||||
|
||||
tax = _taxonomy_lists(conn)
|
||||
assert set(tax.keys()) == {"species", "wood_categories", "edge_terms", "figure_terms", "feature_tags"}
|
||||
assert tax["species"] == [{"id": 11, "name": "Walnut"}]
|
||||
assert tax["wood_categories"] == [{"id": 21, "name": "Live edge wood"}]
|
||||
assert tax["edge_terms"] == [{"id": 321, "name": "Square edge"}]
|
||||
assert tax["figure_terms"] == [{"id": 328, "name": "Plain"}]
|
||||
# feat- prefix stripped for the model; fig- tags are not feature_tags
|
||||
assert tax["feature_tags"] == [{"id": 1, "name": "knot"}]
|
||||
assert "grade_terms" not in tax
|
||||
assert "thickness_terms" not in tax
|
||||
assert "moisture_terms" not in tax
|
||||
db_mod._reset_for_tests()
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def test_gate_feature_tag_ids_not_free_text():
|
||||
"""#86: gate keeps feature_tag_ids at threshold; drops free-text bags and grade."""
|
||||
from backend.inference import _gate
|
||||
|
||||
parsed = {
|
||||
"species_id": 11, "species_confidence": 0.9,
|
||||
"wood_category_ids": [21], "wood_category_confidence": 0.8,
|
||||
"edge_type_term_id": 321, "edge_confidence": 0.85,
|
||||
"figure_term_ids": [328, 335], "figure_confidence": 0.75,
|
||||
"feature_tag_ids": [1], "feature_tag_confidence": 0.8,
|
||||
"grade_term_id": 342, "grade_confidence": 0.99,
|
||||
"character": ["rustic"], "inclusions": ["bark"], "voids": [], "checks": ["end"],
|
||||
}
|
||||
out = _gate(parsed)
|
||||
assert out["species_id"] == 11
|
||||
assert out["wood_category_ids"] == [21]
|
||||
assert out["edge_type_term_id"] == 321
|
||||
assert out["figure_term_ids"] == [328, 335]
|
||||
assert out["feature_tag_ids"] == [1]
|
||||
assert "grade_term_id" not in out
|
||||
assert "character" not in out
|
||||
assert "inclusions" not in out
|
||||
assert "voids" not in out
|
||||
assert "checks" not in out
|
||||
# below threshold feature tags dropped
|
||||
low = {**parsed, "feature_tag_confidence": 0.2}
|
||||
assert "feature_tag_ids" not in _gate(low)
|
||||
|
||||
|
||||
def test_call1_confidence_gate(env, monkeypatch):
|
||||
client, conn = env
|
||||
# enable inference
|
||||
@@ -69,11 +164,18 @@ def test_call1_confidence_gate(env, monkeypatch):
|
||||
from backend.inference import InferenceClient
|
||||
|
||||
def fake_chat(self, messages):
|
||||
# Capture list header for #86 shape
|
||||
text = messages[0]["content"][-1]["text"]
|
||||
assert "LISTS:" in text
|
||||
assert "grade_terms" not in text
|
||||
assert "species_leaves" not in text
|
||||
return json.dumps({
|
||||
"species_id": 1, "species_confidence": 0.9,
|
||||
"wood_category_ids": [1], "wood_category_confidence": 0.5, # below threshold
|
||||
"grade_term_id": 7, "grade_confidence": 0.3, # below
|
||||
"grade_term_id": 7, "grade_confidence": 0.99, # never suggested
|
||||
"figure_term_ids": [2], "figure_confidence": 0.8, # at/above
|
||||
"feature_tag_ids": [9], "feature_tag_confidence": 0.9,
|
||||
"character": ["x"], "inclusions": ["y"],
|
||||
})
|
||||
monkeypatch.setattr(InferenceClient, "chat", fake_chat)
|
||||
r = client.post("/api/v1/slabs/s1/inference/call1")
|
||||
@@ -81,8 +183,10 @@ def test_call1_confidence_gate(env, monkeypatch):
|
||||
s = r.json()["suggestions"]
|
||||
assert s.get("species_id") == 1
|
||||
assert s.get("figure_term_ids") == [2]
|
||||
assert s.get("feature_tag_ids") == [9]
|
||||
assert "wood_category_ids" not in s # gated out
|
||||
assert "grade_term_id" not in s # gated out
|
||||
assert "grade_term_id" not in s # never
|
||||
assert "character" not in s
|
||||
# inference_log written with call=1
|
||||
assert conn.execute("SELECT call FROM inference_log").fetchone()["call"] == 1
|
||||
|
||||
|
||||
@@ -206,14 +206,18 @@ def test_gate_reads_flat_confidence_keys(client, db):
|
||||
# etc., NOT f'{field}_confidence' (field 'species_id' would give the wrong key). A
|
||||
# high-confidence response must surface its picks; low-confidence ones must not.
|
||||
from backend.inference import _gate, _parse_json
|
||||
good = '```json\n{"species_id": 284, "species_confidence": 0.85, "wood_category_ids": [251], "wood_category_confidence": 1.0, "edge_type_term_id": 323, "edge_confidence": 0.95, "figure_term_ids": [332], "figure_confidence": 0.9, "character": [], "inclusions": ["bark"], "voids": [], "checks": []}\n```'
|
||||
good = '```json\n{"species_id": 284, "species_confidence": 0.85, "wood_category_ids": [251], "wood_category_confidence": 1.0, "edge_type_term_id": 323, "edge_confidence": 0.95, "figure_term_ids": [332], "figure_confidence": 0.9, "feature_tag_ids": [1], "feature_tag_confidence": 0.8, "character": [], "inclusions": ["bark"], "voids": [], "checks": []}\n```'
|
||||
out = _gate(_parse_json(good))
|
||||
assert out["species_id"] == 284
|
||||
assert out["wood_category_ids"] == [251]
|
||||
assert out["edge_type_term_id"] == 323
|
||||
assert out["figure_term_ids"] == [332]
|
||||
assert out["feature_tag_ids"] == [1]
|
||||
# grade must never be gated in (METADATA-40)
|
||||
assert "grade_term_id" not in out
|
||||
# free-text bags dropped (#86)
|
||||
assert "character" not in out
|
||||
assert "inclusions" not in out
|
||||
# low-confidence fields are dropped (METADATA-45)
|
||||
low = '```json\n{"species_id": 284, "species_confidence": 0.4, "wood_category_ids": [251], "wood_category_confidence": 0.8, "edge_type_term_id": 323, "edge_confidence": 0.2, "figure_term_ids": [332], "figure_confidence": 0.9}\n```'
|
||||
out2 = _gate(_parse_json(low))
|
||||
|
||||
Reference in New Issue
Block a user