diff --git a/backend/inference.py b/backend/inference.py index af52c83..b6282ad 100644 --- a/backend/inference.py +++ b/backend/inference.py @@ -311,10 +311,18 @@ async def call1(slab_id: str) -> dict: prompt_text = act[1] if act else CALL1_SEED # Build vision message with processed photos (METADATA-35: long edge <= 1024) + taxonomy. photos = conn.execute( - "SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", + "SELECT id, seq, processed_blob FROM slab_photos " + "WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", (slab_id,), ).fetchall() - image_parts = [_image_part_from_processed(p[0]) for p in photos] + image_parts = [] + photo_meta: list[dict] = [] + for p in photos: + blob = p["processed_blob"] if hasattr(p, "keys") else p[2] + pid = p["id"] if hasattr(p, "keys") else p[0] + seq = p["seq"] if hasattr(p, "keys") else p[1] + image_parts.append(_image_part_from_processed(blob)) + photo_meta.append(_photo_meta_row(pid, seq, blob)) tax = _taxonomy_lists(conn) text = (f"{prompt_text}\n\nLISTS:\n{json.dumps(tax, ensure_ascii=False)}\n\n" f"SKU={row['sku']} length_in={row['length_in']} thickness_in={row['thickness_in']}") @@ -328,7 +336,7 @@ async def call1(slab_id: str) -> dict: return JSONResponse(status_code=502, content=_error("inference_failed", str(e), {"recoverable": True})) latency = int((time.time() - t0) * 1000) parsed = _parse_json(raw) - _log_call(conn, slab_id, 1, act[0] if act else None, model, messages, raw, latency) + _log_call(conn, slab_id, 1, act[0] if act else None, model, messages, raw, latency, photo_meta) # Apply confidence gate (METADATA-45). suggestions = _gate(parsed) return {"ok": True, "suggestions": suggestions} @@ -471,10 +479,18 @@ async def call2(slab_id: str) -> dict: act = _active_prompt(conn, "call2") prompt_text = act[1] if act else CALL2_SEED photos = conn.execute( - "SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", + "SELECT id, seq, processed_blob FROM slab_photos " + "WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", (slab_id,), ).fetchall() - image_parts = [_image_part_from_processed(p[0]) for p in photos] + image_parts = [] + photo_meta: list[dict] = [] + for p in photos: + blob = p["processed_blob"] if hasattr(p, "keys") else p[2] + pid = p["id"] if hasattr(p, "keys") else p[0] + seq = p["seq"] if hasattr(p, "keys") else p[1] + image_parts.append(_image_part_from_processed(blob)) + photo_meta.append(_photo_meta_row(pid, seq, blob)) brand = conn.execute("SELECT value FROM settings WHERE key='brand_voice'").fetchone() geo = conn.execute("SELECT value FROM settings WHERE key='geo_context'").fetchone() text = (f"{prompt_text}\n\nDIMENSION FACT BLOCK (exact, invent no numbers):\n" @@ -490,7 +506,7 @@ async def call2(slab_id: str) -> dict: except InferenceError as e: return JSONResponse(status_code=502, content=_error("inference_failed", str(e), {"recoverable": True})) latency = int((time.time() - t0) * 1000) - _log_call(conn, slab_id, 2, act[0] if act else None, model, messages, raw, latency) + _log_call(conn, slab_id, 2, act[0] if act else None, model, messages, raw, latency, photo_meta) # Store as description, mark content_source=llm. conn.execute("UPDATE slabs SET description=?, content_source='llm', updated_at=? WHERE id=?", (raw.strip(), _now(), slab_id)) @@ -508,12 +524,106 @@ def server_description(slab: dict) -> str: ) +def _photo_meta_row(photo_id: str, seq: int, blob: bytes) -> dict[str, Any]: + """Metadata for one photo slot in the inference log (#87).""" + edge = None + try: + im = Image.open(io.BytesIO(blob)) + im.load() + edge = max(im.size) + except Exception: + edge = None + return { + "photo_id": photo_id, + "seq": seq, + "bytes": len(blob) if blob is not None else 0, + "infer_long_edge_px": edge, + "label": f"photo seq={seq} id={photo_id}", + } + + +def _redact_request_for_log(messages: list, photo_meta: list[dict] | None = None) -> list: + """Replace image data-URLs with photo refs for inference_log (#87 / METADATA-66). + + Wire messages still carry base64 for the model; only the logged copy is redacted. + Text parts are kept in full. Image parts become type=image_ref using photo_meta by + image-part index (same order as built from slab_photos). + """ + meta = list(photo_meta or []) + out: list = [] + img_i = 0 + for msg in messages: + if not isinstance(msg, dict): + out.append(msg) + continue + content = msg.get("content") + if not isinstance(content, list): + out.append(msg) + continue + new_parts: list = [] + for part in content: + if not isinstance(part, dict): + new_parts.append(part) + continue + ptype = part.get("type") + if ptype == "text": + new_parts.append({"type": "text", "text": part.get("text", "")}) + continue + if ptype == "image_url": + url = "" + iu = part.get("image_url") + if isinstance(iu, dict): + url = str(iu.get("url") or "") + if url.startswith("data:image") or "base64," in url: + ref = { + "type": "image_ref", + "photo_id": None, + "seq": None, + "bytes": None, + "infer_long_edge_px": None, + "label": f"photo slot={img_i}", + } + if img_i < len(meta): + m = meta[img_i] + ref.update({ + "photo_id": m.get("photo_id"), + "seq": m.get("seq"), + "bytes": m.get("bytes"), + "infer_long_edge_px": m.get("infer_long_edge_px"), + "label": m.get("label") or ref["label"], + }) + new_parts.append(ref) + img_i += 1 + else: + # Non-data URL (rare): keep URL string only, never invent bytes + new_parts.append({ + "type": "image_ref", + "url": url[:200], + "label": f"external image slot={img_i}", + }) + img_i += 1 + continue + new_parts.append(part) + out.append({**msg, "content": new_parts}) + return out + + +_LOG_PAYLOAD_CAP = 200_000 + + def _log_call(conn, slab_id: str, call: int, rev_id: int | None, model: str, - request: list, response: str, latency_ms: int) -> None: + request: list, response: str, latency_ms: int, + photo_meta: list[dict] | None = None) -> None: + """Append inference_log with redacted request (no image bytes) — #87 / METADATA-66.""" + redacted = _redact_request_for_log(request, photo_meta) + payload = json.dumps(redacted, ensure_ascii=False) + if len(payload) > _LOG_PAYLOAD_CAP: + # Safety net only; after redaction normal calls are far under the cap. + payload = payload[: _LOG_PAYLOAD_CAP - 20] + "\n/* truncated */" conn.execute( "INSERT INTO inference_log(slab_id, call, prompt_revision_id, model, request_payload, response_payload, latency_ms, created_at) " "VALUES (?,?,?,?,?,?,?,?)", - (slab_id, call, rev_id, model, json.dumps(request)[:200000], response, latency_ms, _now()), + (slab_id, call, rev_id, model, payload, response, latency_ms, _now()), ) conn.commit() diff --git a/backend/tests/test_inference.py b/backend/tests/test_inference.py index 8641fd4..7997880 100644 --- a/backend/tests/test_inference.py +++ b/backend/tests/test_inference.py @@ -274,6 +274,112 @@ def test_fit_long_edge_png_leaves_small_unchanged_dims(): assert im.size == (400, 300) +def test_redact_request_for_log_strips_data_urls_keeps_text(): + """#87: log sanitizer replaces data-URL images with photo refs; keeps text.""" + from backend.inference import _redact_request_for_log + + b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + messages = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64 * 20}"}}, + {"type": "text", "text": "LISTS:\n{\"species\":[]}\n\nSKU=X length_in=1 thickness_in=2"}, + ], + }] + meta = [ + {"photo_id": "p1", "seq": 1, "bytes": 100, "infer_long_edge_px": 64}, + {"photo_id": "p2", "seq": 2, "bytes": 200, "infer_long_edge_px": 128}, + ] + out = _redact_request_for_log(messages, meta) + blob = json.dumps(out) + assert "data:image" not in blob + assert "base64" not in blob.lower() + assert "LISTS:" in blob + assert "SKU=X" in blob + parts = out[0]["content"] + assert parts[0]["type"] == "image_ref" + assert parts[0]["photo_id"] == "p1" + assert parts[0]["seq"] == 1 + assert parts[1]["photo_id"] == "p2" + assert parts[2]["type"] == "text" + assert "LISTS:" in parts[2]["text"] + + +def test_call1_log_has_text_and_image_refs_no_base64(env, monkeypatch): + """#87 METADATA-66: inference_log stores text+refs, not image bytes.""" + client, conn = env + conn.execute("INSERT OR REPLACE INTO settings(key,value,value_enc) VALUES ('inference_enabled','true',NULL)") + conn.execute( + "INSERT INTO slabs(id,sku,length_in,thickness_in,status,server_rev,client_rev,created_at,updated_at) " + "VALUES ('s1','SLAB-01',100,2,'calibrated',1,1,'t','t')" + ) + png = _rgba_png(400, 300) + conn.execute( + "INSERT INTO slab_photos(id,slab_id,seq,processed_blob,created_at) VALUES ('photo-abc','s1',1,?, 't')", + (png,), + ) + conn.commit() + from backend.inference import InferenceClient + + def fake_chat(self, messages): + # Wire path still has data-URL for the model + content = messages[0]["content"] + assert any( + isinstance(p, dict) and p.get("type") == "image_url" + and str(p.get("image_url", {}).get("url", "")).startswith("data:image") + for p in content + ) + return json.dumps({"species_id": 1, "species_confidence": 0.9}) + + monkeypatch.setattr(InferenceClient, "chat", fake_chat) + r = client.post("/api/v1/slabs/s1/inference/call1") + assert r.status_code == 200, r.text + row = conn.execute( + "SELECT request_payload, length(request_payload) AS n FROM inference_log ORDER BY id DESC LIMIT 1" + ).fetchone() + payload = row["request_payload"] + assert row["n"] < 50000 + assert "data:image" not in payload + assert "LISTS:" in payload + assert "image_ref" in payload + assert "photo-abc" in payload + assert '"seq": 1' in payload or '"seq":1' in payload + logged = json.loads(payload) + refs = [p for p in logged[0]["content"] if p.get("type") == "image_ref"] + assert len(refs) == 1 + assert refs[0]["photo_id"] == "photo-abc" + assert refs[0]["bytes"] == len(png) + + +def test_call2_log_redacts_images(env, monkeypatch): + """#87: Call 2 log also redacts image data-URLs.""" + client, conn = env + conn.execute("INSERT OR REPLACE INTO settings(key,value,value_enc) VALUES ('content_llm_enabled','true',NULL)") + conn.execute( + "INSERT INTO slabs(id,sku,length_in,thickness_in,width_avg_in,bdft,title,status,server_rev,client_rev,created_at,updated_at) " + "VALUES ('s1','SLAB-01',100,2,20,18,'Walnut','ready',1,1,'t','t')" + ) + png = _rgba_png(200, 200) + conn.execute( + "INSERT INTO slab_photos(id,slab_id,seq,processed_blob,created_at) VALUES ('p2','s1',1,?, 't')", + (png,), + ) + conn.commit() + from backend.inference import InferenceClient + + monkeypatch.setattr(InferenceClient, "chat", lambda self, messages: "

ok

") + r = client.post("/api/v1/slabs/s1/inference/call2") + assert r.status_code == 200, r.text + payload = conn.execute( + "SELECT request_payload FROM inference_log WHERE call=2 ORDER BY id DESC LIMIT 1" + ).fetchone()["request_payload"] + assert "data:image" not in payload + assert "image_ref" in payload + assert "p2" in payload + assert "DIMENSION FACT BLOCK" in payload + + def test_call1_sends_long_edge_at_most_1024(env, monkeypatch): """METADATA-35 end-to-end: messages to the model use <=1024 long edge.""" client, conn = env diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 7687a05..d2ff6a1 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -75,7 +75,7 @@ The listing is built in two inference calls, both optional. Call 1 (vision) read | METADATA-65 | The model never invents or alters a dimension; every number it prints comes from the fact block. | M3 | — | | METADATA-95 | The dimension-overlay image is never sent to the model. | M3 | — | | METADATA-70 | With inference off, the server produces the long description. | M3 | — | -| METADATA-66 | Every content generation, from a deterministic pattern or from the model, is logged with its inputs, the model request and response when a model was used, and the final stored content. | M3 | — | +| METADATA-66 | Every content generation, from a deterministic pattern or from the model, is logged with its inputs, the model request and response when a model was used, and the final stored content. The logged request keeps full text and photo refs only — never base64 image payloads. | M3 | backend/tests/test_inference.py | | METADATA-75 | Inference never blocks create and never auto-publishes. | M3 | — | | METADATA-105 | Inference prompts are stored in the database with all revisions; each call uses the active revision, the revision history is retained, and the revision used is recorded with the call. | M3 | — | | METADATA-106 | The prompt revision interface is a REST API: append a revision, activate a revision, and read the active prompt with its revision list. There is no prompt editing UI. | M3 | — | diff --git a/docs/contracts/DATA-MODEL.md b/docs/contracts/DATA-MODEL.md index 45277b3..fcca98b 100644 --- a/docs/contracts/DATA-MODEL.md +++ b/docs/contracts/DATA-MODEL.md @@ -160,6 +160,13 @@ Append-only; never deleted on publish. Columns: `id`, `slab_id`, `woo_product_id Append-only audit. Columns: `id`, `slab_id`, `call` (1 | 2), `prompt_revision_id` FK, `model`, `request_payload`, `response_payload`, `latency_ms`, `created_at`. +`request_payload` is auditable JSON of the model request **without media bytes** (`METADATA-66`, +#87). Text parts (active prompt, LISTS / fact block, SKU/dims) are stored in full. Each vision +attachment is an `image_ref` (`photo_id`, `seq`, `bytes`, optional `infer_long_edge_px`, `label`) +pointing at `slab_photos` — never a `data:image/...;base64,...` URL. The wire request to the +inference API still includes images; only the log is redacted. A byte cap remains as a safety net +after redaction; normal Call 1/2 rows are far under it. + ## 10. Status transitions Cites APP-5.