fix(inference): #87 redact image bytes from inference_log
Log full text + image_ref (photo_id/seq/bytes) only; wire path still sends data-URLs. Call 1/2 SELECT id+seq for refs. Cap is safety net after redaction (METADATA-66).
This commit is contained in:
+118
-8
@@ -311,10 +311,18 @@ async def call1(slab_id: str) -> dict:
|
||||
prompt_text = act[1] if act else CALL1_SEED
|
||||
# Build vision message with processed photos (METADATA-35: long edge <= 1024) + taxonomy.
|
||||
photos = conn.execute(
|
||||
"SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8",
|
||||
"SELECT id, seq, processed_blob FROM slab_photos "
|
||||
"WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8",
|
||||
(slab_id,),
|
||||
).fetchall()
|
||||
image_parts = [_image_part_from_processed(p[0]) for p in photos]
|
||||
image_parts = []
|
||||
photo_meta: list[dict] = []
|
||||
for p in photos:
|
||||
blob = p["processed_blob"] if hasattr(p, "keys") else p[2]
|
||||
pid = p["id"] if hasattr(p, "keys") else p[0]
|
||||
seq = p["seq"] if hasattr(p, "keys") else p[1]
|
||||
image_parts.append(_image_part_from_processed(blob))
|
||||
photo_meta.append(_photo_meta_row(pid, seq, blob))
|
||||
tax = _taxonomy_lists(conn)
|
||||
text = (f"{prompt_text}\n\nLISTS:\n{json.dumps(tax, ensure_ascii=False)}\n\n"
|
||||
f"SKU={row['sku']} length_in={row['length_in']} thickness_in={row['thickness_in']}")
|
||||
@@ -328,7 +336,7 @@ async def call1(slab_id: str) -> dict:
|
||||
return JSONResponse(status_code=502, content=_error("inference_failed", str(e), {"recoverable": True}))
|
||||
latency = int((time.time() - t0) * 1000)
|
||||
parsed = _parse_json(raw)
|
||||
_log_call(conn, slab_id, 1, act[0] if act else None, model, messages, raw, latency)
|
||||
_log_call(conn, slab_id, 1, act[0] if act else None, model, messages, raw, latency, photo_meta)
|
||||
# Apply confidence gate (METADATA-45).
|
||||
suggestions = _gate(parsed)
|
||||
return {"ok": True, "suggestions": suggestions}
|
||||
@@ -471,10 +479,18 @@ async def call2(slab_id: str) -> dict:
|
||||
act = _active_prompt(conn, "call2")
|
||||
prompt_text = act[1] if act else CALL2_SEED
|
||||
photos = conn.execute(
|
||||
"SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8",
|
||||
"SELECT id, seq, processed_blob FROM slab_photos "
|
||||
"WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8",
|
||||
(slab_id,),
|
||||
).fetchall()
|
||||
image_parts = [_image_part_from_processed(p[0]) for p in photos]
|
||||
image_parts = []
|
||||
photo_meta: list[dict] = []
|
||||
for p in photos:
|
||||
blob = p["processed_blob"] if hasattr(p, "keys") else p[2]
|
||||
pid = p["id"] if hasattr(p, "keys") else p[0]
|
||||
seq = p["seq"] if hasattr(p, "keys") else p[1]
|
||||
image_parts.append(_image_part_from_processed(blob))
|
||||
photo_meta.append(_photo_meta_row(pid, seq, blob))
|
||||
brand = conn.execute("SELECT value FROM settings WHERE key='brand_voice'").fetchone()
|
||||
geo = conn.execute("SELECT value FROM settings WHERE key='geo_context'").fetchone()
|
||||
text = (f"{prompt_text}\n\nDIMENSION FACT BLOCK (exact, invent no numbers):\n"
|
||||
@@ -490,7 +506,7 @@ async def call2(slab_id: str) -> dict:
|
||||
except InferenceError as e:
|
||||
return JSONResponse(status_code=502, content=_error("inference_failed", str(e), {"recoverable": True}))
|
||||
latency = int((time.time() - t0) * 1000)
|
||||
_log_call(conn, slab_id, 2, act[0] if act else None, model, messages, raw, latency)
|
||||
_log_call(conn, slab_id, 2, act[0] if act else None, model, messages, raw, latency, photo_meta)
|
||||
# Store as description, mark content_source=llm.
|
||||
conn.execute("UPDATE slabs SET description=?, content_source='llm', updated_at=? WHERE id=?",
|
||||
(raw.strip(), _now(), slab_id))
|
||||
@@ -508,12 +524,106 @@ def server_description(slab: dict) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _photo_meta_row(photo_id: str, seq: int, blob: bytes) -> dict[str, Any]:
|
||||
"""Metadata for one photo slot in the inference log (#87)."""
|
||||
edge = None
|
||||
try:
|
||||
im = Image.open(io.BytesIO(blob))
|
||||
im.load()
|
||||
edge = max(im.size)
|
||||
except Exception:
|
||||
edge = None
|
||||
return {
|
||||
"photo_id": photo_id,
|
||||
"seq": seq,
|
||||
"bytes": len(blob) if blob is not None else 0,
|
||||
"infer_long_edge_px": edge,
|
||||
"label": f"photo seq={seq} id={photo_id}",
|
||||
}
|
||||
|
||||
|
||||
def _redact_request_for_log(messages: list, photo_meta: list[dict] | None = None) -> list:
|
||||
"""Replace image data-URLs with photo refs for inference_log (#87 / METADATA-66).
|
||||
|
||||
Wire messages still carry base64 for the model; only the logged copy is redacted.
|
||||
Text parts are kept in full. Image parts become type=image_ref using photo_meta by
|
||||
image-part index (same order as built from slab_photos).
|
||||
"""
|
||||
meta = list(photo_meta or [])
|
||||
out: list = []
|
||||
img_i = 0
|
||||
for msg in messages:
|
||||
if not isinstance(msg, dict):
|
||||
out.append(msg)
|
||||
continue
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
out.append(msg)
|
||||
continue
|
||||
new_parts: list = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
new_parts.append(part)
|
||||
continue
|
||||
ptype = part.get("type")
|
||||
if ptype == "text":
|
||||
new_parts.append({"type": "text", "text": part.get("text", "")})
|
||||
continue
|
||||
if ptype == "image_url":
|
||||
url = ""
|
||||
iu = part.get("image_url")
|
||||
if isinstance(iu, dict):
|
||||
url = str(iu.get("url") or "")
|
||||
if url.startswith("data:image") or "base64," in url:
|
||||
ref = {
|
||||
"type": "image_ref",
|
||||
"photo_id": None,
|
||||
"seq": None,
|
||||
"bytes": None,
|
||||
"infer_long_edge_px": None,
|
||||
"label": f"photo slot={img_i}",
|
||||
}
|
||||
if img_i < len(meta):
|
||||
m = meta[img_i]
|
||||
ref.update({
|
||||
"photo_id": m.get("photo_id"),
|
||||
"seq": m.get("seq"),
|
||||
"bytes": m.get("bytes"),
|
||||
"infer_long_edge_px": m.get("infer_long_edge_px"),
|
||||
"label": m.get("label") or ref["label"],
|
||||
})
|
||||
new_parts.append(ref)
|
||||
img_i += 1
|
||||
else:
|
||||
# Non-data URL (rare): keep URL string only, never invent bytes
|
||||
new_parts.append({
|
||||
"type": "image_ref",
|
||||
"url": url[:200],
|
||||
"label": f"external image slot={img_i}",
|
||||
})
|
||||
img_i += 1
|
||||
continue
|
||||
new_parts.append(part)
|
||||
out.append({**msg, "content": new_parts})
|
||||
return out
|
||||
|
||||
|
||||
_LOG_PAYLOAD_CAP = 200_000
|
||||
|
||||
|
||||
def _log_call(conn, slab_id: str, call: int, rev_id: int | None, model: str,
|
||||
request: list, response: str, latency_ms: int) -> None:
|
||||
request: list, response: str, latency_ms: int,
|
||||
photo_meta: list[dict] | None = None) -> None:
|
||||
"""Append inference_log with redacted request (no image bytes) — #87 / METADATA-66."""
|
||||
redacted = _redact_request_for_log(request, photo_meta)
|
||||
payload = json.dumps(redacted, ensure_ascii=False)
|
||||
if len(payload) > _LOG_PAYLOAD_CAP:
|
||||
# Safety net only; after redaction normal calls are far under the cap.
|
||||
payload = payload[: _LOG_PAYLOAD_CAP - 20] + "\n/* truncated */"
|
||||
conn.execute(
|
||||
"INSERT INTO inference_log(slab_id, call, prompt_revision_id, model, request_payload, response_payload, latency_ms, created_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?)",
|
||||
(slab_id, call, rev_id, model, json.dumps(request)[:200000], response, latency_ms, _now()),
|
||||
(slab_id, call, rev_id, model, payload, response, latency_ms, _now()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user