From 037a4b870ae97cbce8ed958ef8e23e30ec015953 Mon Sep 17 00:00:00 2001 From: grokbot_ty Date: Wed, 16 Sep 2026 05:08:13 +0000 Subject: [PATCH] fix(photos): shared EXIF-oriented decode for axis/overlay space (#72) PhotoStage used new Image() while mask worker and calculate used bare createImageBitmap; Orientation-8 matched-set JPEGs can disagree across those paths so the approved pink axis is not the overlay (ux,uy). One decodeOrientedBitmap helper always requests imageOrientation from-image (with bare fallback), wired into PhotoStage, mask.worker, and session calculate/siblings. --- frontend/src/capture/PhotoStage.svelte | 155 +++++++++--------- .../src/capture/decodeOrientedBitmap.test.ts | 149 +++++++++++++++++ frontend/src/capture/session.ts | 5 +- frontend/src/decodeOrientedBitmap.ts | 29 ++++ frontend/src/png/overlay.test.ts | 37 +++++ frontend/src/worker/mask.worker.ts | 3 +- 6 files changed, 299 insertions(+), 79 deletions(-) create mode 100644 frontend/src/capture/decodeOrientedBitmap.test.ts create mode 100644 frontend/src/decodeOrientedBitmap.ts diff --git a/frontend/src/capture/PhotoStage.svelte b/frontend/src/capture/PhotoStage.svelte index 70e2cb3..7b003c2 100644 --- a/frontend/src/capture/PhotoStage.svelte +++ b/frontend/src/capture/PhotoStage.svelte @@ -2,9 +2,11 @@ import { onDestroy, onMount, tick } from 'svelte'; import type { BBox } from './capture.types'; import { fitDisplay } from './fitDisplay'; + import { decodeOrientedBitmap } from '../decodeOrientedBitmap'; // Display-only: renders the photo fitted to device width (height capped) plus // one overlay layer. All math lives on source pixels, not this canvas. + // Geometry must use the same EXIF-oriented bitmap space as the mask worker (#72). export let blob: Blob; export let maxH = 420; export let bbox: BBox | null = null; @@ -12,21 +14,12 @@ export let maskW = 0; export let maskH = 0; - let url: string | null = null; let canvas: HTMLCanvasElement | undefined; let stage: HTMLDivElement | undefined; - let imgW = 0, imgH = 0; // source px, learned once per blob + let imgW = 0, imgH = 0; // source px (oriented), learned once per blob let parentW = 0; - - const setBlobUrl = (b: Blob) => { - if (url) URL.revokeObjectURL(url); - url = URL.createObjectURL(b); - }; - - $: if (blob) { - setBlobUrl(blob); - imgW = 0; - } + let decodeGen = 0; + let decoding = false; const applyFit = () => { if (!canvas || !imgW) return; @@ -37,73 +30,83 @@ canvas.style.height = `${fit.fitH}px`; }; - const redraw = () => { - if (!url || !canvas) return; - const ctx = canvas.getContext('2d')!; - ctx.clearRect(0, 0, canvas.width, canvas.height); - const img = new Image(); - img.onload = () => { - if (!canvas) return; - imgW = img.width; - imgH = img.height; + const paintOverlays = (ctx: CanvasRenderingContext2D) => { + if (!canvas) return; + if (maskBitset && maskW > 0 && maskH > 0) { + const off = document.createElement('canvas'); + off.width = maskW; + off.height = maskH; + const octx = off.getContext('2d')!; + const d = octx.createImageData(maskW, maskH); + const tile = 8; + for (let p = 0; p < maskW * maskH; p++) { + if (!maskBitset[p]) { + const x = p % maskW; + const y = (p / maskW) | 0; + const light = ((x / tile) | 0) % 2 === ((y / tile) | 0) % 2; + d.data[p * 4] = light ? 220 : 180; + d.data[p * 4 + 1] = light ? 220 : 180; + d.data[p * 4 + 2] = light ? 220 : 180; + d.data[p * 4 + 3] = 220; + } + } + octx.putImageData(d, 0, 0); + ctx.drawImage(off, 0, 0, canvas.width, canvas.height); + } + if (bbox && bbox.corners.length === 4) { + const c = bbox.corners; + ctx.strokeStyle = '#00e5ff'; + ctx.lineWidth = Math.max(2, canvas.width / 400); + ctx.beginPath(); + ctx.moveTo(c[0][0], c[0][1]); + for (let i = 1; i < 4; i++) ctx.lineTo(c[i][0], c[i][1]); + ctx.closePath(); + ctx.stroke(); + // The axis reads corners[3] → corners[0], the long side. + const a = c[3]; + const b = c[0]; + ctx.strokeStyle = '#ff3d71'; + ctx.lineWidth = Math.max(3, canvas.width / 300); + ctx.beginPath(); + ctx.moveTo(a[0], a[1]); + ctx.lineTo(b[0], b[1]); + ctx.stroke(); + ctx.fillStyle = '#ff3d71'; + ctx.beginPath(); + const ang = Math.atan2(b[1] - a[1], b[0] - a[0]); + const r = Math.max(10, canvas.width / 60); + ctx.moveTo(b[0], b[1]); + ctx.lineTo(b[0] - r * Math.cos(ang - 0.4), b[1] - r * Math.sin(ang - 0.4)); + ctx.lineTo(b[0] - r * Math.cos(ang + 0.4), b[1] - r * Math.sin(ang + 0.4)); + ctx.closePath(); + ctx.fill(); + } + }; + + const redraw = async () => { + if (!blob || !canvas) return; + const gen = ++decodeGen; + decoding = true; + let bmp: ImageBitmap | null = null; + try { + bmp = await decodeOrientedBitmap(blob); + if (gen !== decodeGen || !canvas) return; + imgW = bmp.width; + imgH = bmp.height; canvas.width = imgW; canvas.height = imgH; applyFit(); - ctx.drawImage(img, 0, 0); - if (maskBitset && maskW > 0 && maskH > 0) { - const off = document.createElement('canvas'); - off.width = maskW; - off.height = maskH; - const octx = off.getContext('2d')!; - const d = octx.createImageData(maskW, maskH); - const tile = 8; - for (let p = 0; p < maskW * maskH; p++) { - if (!maskBitset[p]) { - const x = p % maskW; - const y = (p / maskW) | 0; - const light = ((x / tile) | 0) % 2 === ((y / tile) | 0) % 2; - d.data[p * 4] = light ? 220 : 180; - d.data[p * 4 + 1] = light ? 220 : 180; - d.data[p * 4 + 2] = light ? 220 : 180; - d.data[p * 4 + 3] = 220; - } - } - octx.putImageData(d, 0, 0); - ctx.drawImage(off, 0, 0, canvas.width, canvas.height); - } - if (bbox && bbox.corners.length === 4) { - const c = bbox.corners; - ctx.strokeStyle = '#00e5ff'; - ctx.lineWidth = Math.max(2, canvas.width / 400); - ctx.beginPath(); - ctx.moveTo(c[0][0], c[0][1]); - for (let i = 1; i < 4; i++) ctx.lineTo(c[i][0], c[i][1]); - ctx.closePath(); - ctx.stroke(); - // The axis reads corners[3] → corners[0], the long side. - const a = c[3]; - const b = c[0]; - ctx.strokeStyle = '#ff3d71'; - ctx.lineWidth = Math.max(3, canvas.width / 300); - ctx.beginPath(); - ctx.moveTo(a[0], a[1]); - ctx.lineTo(b[0], b[1]); - ctx.stroke(); - ctx.fillStyle = '#ff3d71'; - ctx.beginPath(); - const ang = Math.atan2(b[1] - a[1], b[0] - a[0]); - const r = Math.max(10, canvas.width / 60); - ctx.moveTo(b[0], b[1]); - ctx.lineTo(b[0] - r * Math.cos(ang - 0.4), b[1] - r * Math.sin(ang - 0.4)); - ctx.lineTo(b[0] - r * Math.cos(ang + 0.4), b[1] - r * Math.sin(ang + 0.4)); - ctx.closePath(); - ctx.fill(); - } - }; - img.src = url; + const ctx = canvas.getContext('2d')!; + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.drawImage(bmp, 0, 0); + paintOverlays(ctx); + } finally { + if (bmp) bmp.close(); + if (gen === decodeGen) decoding = false; + } }; - $: if (url || bbox || maskBitset) void tick().then(redraw); + $: if (blob || bbox || maskBitset) void tick().then(redraw); onMount(() => { if (stage) parentW = stage.clientWidth; @@ -119,12 +122,12 @@ return () => ro.disconnect(); }); onDestroy(() => { - if (url) URL.revokeObjectURL(url); + decodeGen += 1; });
- {#if url} + {#if imgW > 0 || decoding || blob} {:else}
Decoding photo…
diff --git a/frontend/src/capture/decodeOrientedBitmap.test.ts b/frontend/src/capture/decodeOrientedBitmap.test.ts new file mode 100644 index 0000000..df757ae --- /dev/null +++ b/frontend/src/capture/decodeOrientedBitmap.test.ts @@ -0,0 +1,149 @@ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { + decodeOrientedBitmap, + ORIENTED_BITMAP_OPTIONS, +} from '../decodeOrientedBitmap.ts'; + +const here = dirname(fileURLToPath(import.meta.url)); +const fixturePath = join(here, '../../tests/fixtures/set1-48x2/face-a.jpeg'); + +const RealCreateImageBitmap = globalThis.createImageBitmap; + +afterEach(() => { + if (RealCreateImageBitmap === undefined) { + delete (globalThis as unknown as { createImageBitmap?: unknown }).createImageBitmap; + } else { + globalThis.createImageBitmap = RealCreateImageBitmap; + } +}); + +/** Minimal EXIF Orientation + largest SOF size for the orient-8 fixture. */ +function readJpegOrientAndSof(buf: Buffer): { orientation: number; sofW: number; sofH: number } { + let orientation = 1; + const exif = buf.indexOf(Buffer.from('Exif\0\0')); + if (exif >= 0) { + const tiff = exif + 6; + const le = buf[tiff] === 0x49; + const u16 = (o: number) => (le ? buf.readUInt16LE(o) : buf.readUInt16BE(o)); + const u32 = (o: number) => (le ? buf.readUInt32LE(o) : buf.readUInt32BE(o)); + const ifd = tiff + u32(tiff + 4); + const n = u16(ifd); + for (let k = 0; k < n; k++) { + const e = ifd + 2 + k * 12; + const tag = u16(e); + const typ = u16(e + 2); + if (tag === 0x0112) { + orientation = typ === 3 ? u16(e + 8) : u32(e + 8); + } + } + } + let sofW = 0; + let sofH = 0; + let i = 0; + while (i < buf.length - 9) { + if (buf[i] === 0xff && buf[i + 1] >= 0xc0 && buf[i + 1] <= 0xc3) { + const h = buf.readUInt16BE(i + 5); + const w = buf.readUInt16BE(i + 7); + if (w * h > sofW * sofH) { + sofW = w; + sofH = h; + } + const segLen = buf.readUInt16BE(i + 2); + i += 2 + segLen; + } else { + i += 1; + } + } + return { orientation, sofW, sofH }; +} + +/** Orientation 5–8 swap width/height for display. */ +function orientedSize(sofW: number, sofH: number, orientation: number): { w: number; h: number } { + if (orientation >= 5 && orientation <= 8) return { w: sofH, h: sofW }; + return { w: sofW, h: sofH }; +} + +test('fixture face-a.jpeg is EXIF Orientation 8 (stored landscape → oriented portrait) (#72)', () => { + const buf = readFileSync(fixturePath); + const { orientation, sofW, sofH } = readJpegOrientAndSof(buf); + assert.equal(orientation, 8, 'matched-set JPEGs carry Orientation 8'); + assert.equal(sofW, 4000); + assert.equal(sofH, 2256); + const { w, h } = orientedSize(sofW, sofH, orientation); + assert.equal(w, 2256); + assert.equal(h, 4000); +}); + +test('decodeOrientedBitmap always requests imageOrientation from-image (#72)', async () => { + let seen: ImageBitmapOptions | undefined; + (globalThis as unknown as { createImageBitmap: typeof createImageBitmap }).createImageBitmap = + async (_src: ImageBitmapSource, opts?: ImageBitmapOptions) => { + seen = opts; + return { width: 2256, height: 4000, close() {} } as ImageBitmap; + }; + + const blob = new Blob([readFileSync(fixturePath)], { type: 'image/jpeg' }); + const bmp = await decodeOrientedBitmap(blob); + assert.deepEqual(seen, ORIENTED_BITMAP_OPTIONS); + assert.equal(ORIENTED_BITMAP_OPTIONS.imageOrientation, 'from-image'); + assert.equal(bmp.width, 2256); + assert.equal(bmp.height, 4000); +}); + +test('decodeOrientedBitmap falls back to bare createImageBitmap if option rejected (#72)', async () => { + let calls = 0; + (globalThis as unknown as { createImageBitmap: typeof createImageBitmap }).createImageBitmap = + async (_src: ImageBitmapSource, opts?: ImageBitmapOptions) => { + calls += 1; + if (opts && 'imageOrientation' in opts) { + throw new TypeError('imageOrientation unsupported'); + } + return { width: 90, height: 160, close() {} } as ImageBitmap; + }; + + const bmp = await decodeOrientedBitmap(new Blob([new Uint8Array([1, 2, 3])], { type: 'image/jpeg' })); + assert.equal(calls, 2, 'retry bare after option rejection'); + assert.equal(bmp.width, 90); + assert.equal(bmp.height, 160); +}); + +test('decodeOrientedBitmap: Chromium orients fixture to 2256×4000 (#72)', async (t) => { + let chromium: typeof import('@playwright/test').chromium; + try { + ({ chromium } = await import('@playwright/test')); + } catch { + t.skip('playwright not available'); + return; + } + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + const buf = readFileSync(fixturePath); + const b64 = buf.toString('base64'); + // Inline the same contract as decodeOrientedBitmap (worker-safe, no bundler). + const dims = await page.evaluate(async (b64jpeg: string) => { + const bin = atob(b64jpeg); + const arr = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i); + const blob = new Blob([arr], { type: 'image/jpeg' }); + const opts = { imageOrientation: 'from-image' as const }; + let bmp: ImageBitmap; + try { + bmp = await createImageBitmap(blob, opts); + } catch { + bmp = await createImageBitmap(blob); + } + const out = { w: bmp.width, h: bmp.height }; + bmp.close(); + return out; + }, b64); + assert.equal(dims.w, 2256, `oriented width (got ${dims.w})`); + assert.equal(dims.h, 4000, `oriented height (got ${dims.h})`); + } finally { + await browser.close(); + } +}); diff --git a/frontend/src/capture/session.ts b/frontend/src/capture/session.ts index 528d4ad..59e73fa 100644 --- a/frontend/src/capture/session.ts +++ b/frontend/src/capture/session.ts @@ -10,6 +10,7 @@ import { measure } from '../measure/measure'; import { orientedRect } from '../geometry/rect'; import { encodeInventoryPng, type InventoryInput } from '../png/inventory'; import { encodeOverlayPng, type EncodeOverlayInput } from '../png/overlay'; +import { decodeOrientedBitmap } from '../decodeOrientedBitmap'; const DEBOUNCE_MS = 2000; const BUSY_MS = 300; @@ -512,7 +513,7 @@ export function createCaptureSession(opts: SessionOpts = {}): CaptureSession { if (state.k !== 'measure') return; const { source, approvedMask, axis, lengthEighths, thicknessEighths } = state; if (lengthEighths === 0 || thicknessEighths === 0) return; - const bitmap = await createImageBitmap(source.blob); + const bitmap = await decodeOrientedBitmap(source.blob); const base = measure( approvedMask.bitset, source.widthPx, source.heightPx, axis.bbox, lengthEighths, thicknessEighths, approvedMask.slabPx, @@ -587,7 +588,7 @@ export function createCaptureSession(opts: SessionOpts = {}): CaptureSession { widthPx: derived.widthPx, heightPx: derived.heightPx, }; - const siblingBitmap = await createImageBitmap(photo.blob); + const siblingBitmap = await decodeOrientedBitmap(photo.blob); let encoded: { blob: Blob; sourceBelowFloor: boolean }; try { encoded = await encodeInventory({ bitmap: siblingBitmap, mask: derived.mask, source: siblingSource }); diff --git a/frontend/src/decodeOrientedBitmap.ts b/frontend/src/decodeOrientedBitmap.ts new file mode 100644 index 0000000..07e8928 --- /dev/null +++ b/frontend/src/decodeOrientedBitmap.ts @@ -0,0 +1,29 @@ +/** + * Shared decode for capture photos so PhotoStage, mask worker, inventory, and + * calculate all share one EXIF-oriented pixel space. + * + * Matched-set / DJI JPEGs often carry EXIF Orientation 8 (stored landscape, + * display portrait). `new Image()` and bare `createImageBitmap(blob)` can + * disagree across browsers → pink axis arrow space ≠ mask/overlay bitset space. + * + * Contract: always request `imageOrientation: 'from-image'`. If the runtime + * rejects that option (older engines), fall back to bare `createImageBitmap`; + * engines that already apply EXIF by default still match Image/canvas size. + * Engines that never apply EXIF may still diverge after fallback — document + * that path rather than inventing a software rotate. + */ +export const ORIENTED_BITMAP_OPTIONS: ImageBitmapOptions = { + imageOrientation: 'from-image', +}; + +export async function decodeOrientedBitmap(blob: Blob): Promise { + try { + return await createImageBitmap(blob, ORIENTED_BITMAP_OPTIONS); + } catch (first) { + try { + return await createImageBitmap(blob); + } catch { + throw first; + } + } +} diff --git a/frontend/src/png/overlay.test.ts b/frontend/src/png/overlay.test.ts index b9c53f3..37f8656 100644 --- a/frontend/src/png/overlay.test.ts +++ b/frontend/src/png/overlay.test.ts @@ -545,6 +545,43 @@ function tiltedAxis25Fixture() { return { W, H, bitset, bbox, ux0, uy0, rx0, ry0, halfL, cx, cy }; } + +test('buildOverlayPlan: angleDeg=90 vertical long side → width left→right within 2° of horizontal (#72)', () => { + // Upright slab: approved length axis is vertical (angleDeg 90). Width marks must + // run left→right ≈ horizontal (atan2 ≈ 0 or ±π), not slanted with the photo. + const W = 200, H = 400; + const bitset = new Uint8Array(W * H); + for (let y = 40; y <= 360; y++) { + for (let x = 60; x <= 140; x++) bitset[y * W + x] = 1; + } + const prev = minAreaRect(bitset, W, H)!; + const bbox = orientedRect(prev, 90); + assert.equal(bbox.angleDeg, 90); + const scale = 10; // 320px span ≈ 32" + const plan = buildOverlayPlan({ + bitset, + widthPx: W, + heightPx: H, + bbox, + scale, + lengthIn: 32, + lengthLabel: '32"', + mapToInventory: (x, y) => [x, y], + }); + assert.ok(plan.widths.length > 0, 'expected width stations'); + for (const w of plan.widths) { + const ang = Math.atan2(w.right[1] - w.left[1], w.right[0] - w.left[0]); + // Fold to [-90, 90] so left→right and right→left both count as horizontal. + let deg = (ang * 180) / Math.PI; + if (deg > 90) deg -= 180; + if (deg < -90) deg += 180; + assert.ok( + Math.abs(deg) <= 2, + `width station ${w.stationIn} left→right must be within 2° of horizontal (atan2 deg=${deg.toFixed(2)}; left=${w.left} right=${w.right})`, + ); + } +}); + test('buildOverlayPlan: length a→b || approved (ux,uy); width left→right || (rx,ry) (#70)', () => { const { W, H, bitset, bbox, cx, cy } = tiltedAxis25Fixture(); const scale = 10; diff --git a/frontend/src/worker/mask.worker.ts b/frontend/src/worker/mask.worker.ts index d7caa6c..8844901 100644 --- a/frontend/src/worker/mask.worker.ts +++ b/frontend/src/worker/mask.worker.ts @@ -1,5 +1,6 @@ import { minAreaRect, slabBounds } from '../geometry/rect'; import type { MaskParams, WorkerIn, WorkerOut } from '../capture/capture.types'; +import { decodeOrientedBitmap } from '../decodeOrientedBitmap'; let bitmap: ImageBitmap | null = null; let data: Uint8ClampedArray | null = null; @@ -69,7 +70,7 @@ self.onmessage = (e: MessageEvent) => { } data = null; w = h = 0; - createImageBitmap(msg.blob) + decodeOrientedBitmap(msg.blob) .then((bmp) => { w = bmp.width; h = bmp.height;