diff --git a/backend/inference.py b/backend/inference.py index 99de817..8f93b20 100644 --- a/backend/inference.py +++ b/backend/inference.py @@ -5,6 +5,7 @@ Cites METADATA-35..115, SETTINGS-30/45/90. import base64 import hashlib +import io import json import time import urllib.error @@ -13,12 +14,53 @@ from typing import Any from fastapi import APIRouter from fastapi.responses import JSONResponse +from PIL import Image from .db import get_connection from .crypto import decrypt router = APIRouter(prefix="/api/v1") +# METADATA-35: Call 1 (and Call 2) send clean processed photos at 1024 px long edge or less. +_INFER_MAX_LONG_EDGE = 1024 + + +def fit_long_edge_png(png_bytes: bytes, max_long_edge: int = _INFER_MAX_LONG_EDGE) -> bytes: + """Downscale a PNG so max(width, height) <= max_long_edge. Never upscale. Keep alpha. + + Used only for the inference request payload; stored processed_blob / Woo media stay full size. + Unreadable or non-image bytes are returned unchanged so a bad blob cannot crash Call 1. + """ + if not png_bytes or max_long_edge < 1: + return png_bytes + try: + im = Image.open(io.BytesIO(png_bytes)) + im.load() + except Exception: + return png_bytes + w, h = im.size + long_edge = max(w, h) + if long_edge <= max_long_edge: + # Already within budget; re-encode only if we need a stable PNG (caller always base64s). + return png_bytes + scale = max_long_edge / float(long_edge) + nw = max(1, int(round(w * scale))) + nh = max(1, int(round(h * scale))) + # Preserve alpha: convert palette/etc. to RGBA when needed, LANCZOS downscale. + if im.mode not in ("RGB", "RGBA", "L", "LA"): + im = im.convert("RGBA") + out = im.resize((nw, nh), Image.Resampling.LANCZOS) + buf = io.BytesIO() + out.save(buf, format="PNG", optimize=True) + return buf.getvalue() + + +def _image_part_from_processed(png_bytes: bytes) -> dict: + """Build one OpenAI-style image_url part from a processed PNG (METADATA-35 sized).""" + fitted = fit_long_edge_png(png_bytes) + b64 = base64.b64encode(fitted).decode() + return {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}} + CALL1_SEED = ( "Classify the slab's species, wood categories, edge type, and figure from the supplied " "photos, choosing ONLY from the supplied synced taxonomy lists (invent no category, term, " @@ -203,17 +245,12 @@ async def call1(slab_id: str) -> dict: client = InferenceClient(base, key, model) act = _active_prompt(conn, "call1") prompt_text = act[1] if act else CALL1_SEED - # Build vision message with processed photos (1024px) + taxonomy lists. + # Build vision message with processed photos (METADATA-35: long edge <= 1024) + taxonomy. photos = conn.execute( "SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", (slab_id,), ).fetchall() - image_parts = [] - for p in photos: - image_parts.append({ - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{base64.b64encode(p[0]).decode()}"}, - }) + image_parts = [_image_part_from_processed(p[0]) for p in photos] tax = _taxonomy_lists(conn) text = (f"{prompt_text}\n\nSYNCED TAXONOMY (only legal choices):\n{json.dumps(tax)}\n\n" f"SKU={row['sku']} length_in={row['length_in']} thickness_in={row['thickness_in']}") @@ -359,10 +396,7 @@ async def call2(slab_id: str) -> dict: "SELECT processed_blob FROM slab_photos WHERE slab_id=? AND processed_blob IS NOT NULL ORDER BY seq LIMIT 8", (slab_id,), ).fetchall() - image_parts = [ - {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64.b64encode(p[0]).decode()}"}} - for p in photos - ] + image_parts = [_image_part_from_processed(p[0]) for p in photos] brand = conn.execute("SELECT value FROM settings WHERE key='brand_voice'").fetchone() geo = conn.execute("SELECT value FROM settings WHERE key='geo_context'").fetchone() text = (f"{prompt_text}\n\nDIMENSION FACT BLOCK (exact, invent no numbers):\n" diff --git a/backend/requirements.txt b/backend/requirements.txt index e1b760c..b10e221 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,4 +5,5 @@ python-multipart==0.0.9 pytest==8.3.3 httpx==0.27.2 cryptography==43.0.1 +Pillow==11.3.0 # stdlib only for sqlite3, logging, json, contextvars, threading diff --git a/backend/tests/test_inference.py b/backend/tests/test_inference.py index 9e2b2ce..f72c9ce 100644 --- a/backend/tests/test_inference.py +++ b/backend/tests/test_inference.py @@ -126,3 +126,82 @@ def test_test_inference_unconfigured(env): r = client.post("/api/v1/admin/test-inference") assert r.status_code == 200 assert r.json()["ok"] is False and r.json()["mode"] == "fail" + + +def _rgba_png(w: int, h: int) -> bytes: + """RGBA PNG: opaque body + transparent top-left block (capture-like alpha).""" + from PIL import Image + import io + im = Image.new("RGBA", (w, h), (10, 20, 30, 255)) + for y in range(min(80, h)): + for x in range(min(80, w)): + im.putpixel((x, y), (0, 0, 0, 0)) + buf = io.BytesIO() + im.save(buf, format="PNG") + return buf.getvalue() + + +def test_fit_long_edge_png_downscales_over_1024(): + """METADATA-35: Call 1 images long edge <= 1024; never upscale; keep alpha.""" + from backend.inference import fit_long_edge_png + from PIL import Image + import io + + src = _rgba_png(800, 1067) # inventory 3:4, long edge 1067 + out = fit_long_edge_png(src, max_long_edge=1024) + im = Image.open(io.BytesIO(out)).convert("RGBA") + assert max(im.size) == 1024 + assert min(im.size) < 1024 + w, h = im.size + assert abs((w / h) - (800 / 1067)) < 0.02 + # Transparent block still transparent near origin after downscale + assert im.getpixel((0, 0))[3] == 0 + assert im.getpixel((5, 5))[3] == 0 + + +def test_fit_long_edge_png_leaves_small_unchanged_dims(): + from backend.inference import fit_long_edge_png + from PIL import Image + import io + + src = _rgba_png(400, 300) + out = fit_long_edge_png(src, max_long_edge=1024) + im = Image.open(io.BytesIO(out)) + assert im.size == (400, 300) + + +def test_call1_sends_long_edge_at_most_1024(env, monkeypatch): + """METADATA-35 end-to-end: messages to the model use <=1024 long edge.""" + client, conn = env + conn.execute("INSERT OR REPLACE INTO settings(key,value,value_enc) VALUES ('inference_enabled','true',NULL)") + conn.execute( + "INSERT INTO slabs(id,sku,length_in,thickness_in,status,server_rev,client_rev,created_at,updated_at) " + "VALUES ('s1','SLAB-01',100,2,'calibrated',1,1,'t','t')" + ) + big = _rgba_png(800, 1067) + conn.execute( + "INSERT INTO slab_photos(id,slab_id,seq,processed_blob,created_at) VALUES ('p1','s1',1,?, 't')", + (big,), + ) + conn.commit() + from backend.inference import InferenceClient + from PIL import Image + import io + + captured = {} + + def fake_chat(self, messages): + captured["messages"] = messages + return json.dumps({"species_id": 1, "species_confidence": 0.9}) + + monkeypatch.setattr(InferenceClient, "chat", fake_chat) + r = client.post("/api/v1/slabs/s1/inference/call1") + assert r.status_code == 200, r.text + content = captured["messages"][0]["content"] + imgs = [p for p in content if isinstance(p, dict) and p.get("type") == "image_url"] + assert len(imgs) == 1 + url = imgs[0]["image_url"]["url"] + assert url.startswith("data:image/png;base64,") + raw = base64.b64decode(url.split(",", 1)[1]) + im = Image.open(io.BytesIO(raw)) + assert max(im.size) <= 1024 diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 2d21a89..fd3e6ea 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -63,7 +63,7 @@ The listing is built in two inference calls, both optional. Call 1 (vision) read | METADATA-25 | Species, character, length, and free-text thickness are not product attributes. | M3 | — | | METADATA-30 | Product tags come from the synced Woo set and are all optional; no hardcoded prefixes, and the bare fig tag is kept. | M3 | — | | METADATA-100 | Call 1 free-text observations (character, inclusions, voids, checks) are mapped to existing synced feat-* tags only; WooCommerce is the authority for feat-* definitions. | M3 | — | -| METADATA-35 | Call 1 runs only when inference is enabled, automatically after a calibrated draft uploads or on demand, and sends the clean processed photos at 1024 px long edge or less plus user metadata and the synced taxonomy. | M3 | — | +| METADATA-35 | Call 1 runs only when inference is enabled, automatically after a calibrated draft uploads or on demand, and sends the clean processed photos at 1024 px long edge or less plus user metadata and the synced taxonomy. | M3 | backend/tests/test_inference.py | | METADATA-40 | Call 1 returns species, wood categories, edge, figure, and feature-tag suggestions constrained to the synced lists, each with confidence. Grade is not suggested by Call 1 (owner judgment). | M3 | — | | METADATA-45 | Call 1 confidence at or above the threshold (default 0.7) pre-fills review fields; below it the field stays empty for the owner to fill by hand or re-run Call 1; suggestions are always editable. | M3 | — | | METADATA-50 | Typed values always win over inferred values. | M3 | — |