fix(photos): shared EXIF-oriented decode for axis/overlay space (#72)
PhotoStage used new Image() while mask worker and calculate used bare createImageBitmap; Orientation-8 matched-set JPEGs can disagree across those paths so the approved pink axis is not the overlay (ux,uy). One decodeOrientedBitmap helper always requests imageOrientation from-image (with bare fallback), wired into PhotoStage, mask.worker, and session calculate/siblings.
This commit is contained in:
@@ -2,9 +2,11 @@
|
||||
import { onDestroy, onMount, tick } from 'svelte';
|
||||
import type { BBox } from './capture.types';
|
||||
import { fitDisplay } from './fitDisplay';
|
||||
import { decodeOrientedBitmap } from '../decodeOrientedBitmap';
|
||||
|
||||
// Display-only: renders the photo fitted to device width (height capped) plus
|
||||
// one overlay layer. All math lives on source pixels, not this canvas.
|
||||
// Geometry must use the same EXIF-oriented bitmap space as the mask worker (#72).
|
||||
export let blob: Blob;
|
||||
export let maxH = 420;
|
||||
export let bbox: BBox | null = null;
|
||||
@@ -12,21 +14,12 @@
|
||||
export let maskW = 0;
|
||||
export let maskH = 0;
|
||||
|
||||
let url: string | null = null;
|
||||
let canvas: HTMLCanvasElement | undefined;
|
||||
let stage: HTMLDivElement | undefined;
|
||||
let imgW = 0, imgH = 0; // source px, learned once per blob
|
||||
let imgW = 0, imgH = 0; // source px (oriented), learned once per blob
|
||||
let parentW = 0;
|
||||
|
||||
const setBlobUrl = (b: Blob) => {
|
||||
if (url) URL.revokeObjectURL(url);
|
||||
url = URL.createObjectURL(b);
|
||||
};
|
||||
|
||||
$: if (blob) {
|
||||
setBlobUrl(blob);
|
||||
imgW = 0;
|
||||
}
|
||||
let decodeGen = 0;
|
||||
let decoding = false;
|
||||
|
||||
const applyFit = () => {
|
||||
if (!canvas || !imgW) return;
|
||||
@@ -37,73 +30,83 @@
|
||||
canvas.style.height = `${fit.fitH}px`;
|
||||
};
|
||||
|
||||
const redraw = () => {
|
||||
if (!url || !canvas) return;
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
if (!canvas) return;
|
||||
imgW = img.width;
|
||||
imgH = img.height;
|
||||
const paintOverlays = (ctx: CanvasRenderingContext2D) => {
|
||||
if (!canvas) return;
|
||||
if (maskBitset && maskW > 0 && maskH > 0) {
|
||||
const off = document.createElement('canvas');
|
||||
off.width = maskW;
|
||||
off.height = maskH;
|
||||
const octx = off.getContext('2d')!;
|
||||
const d = octx.createImageData(maskW, maskH);
|
||||
const tile = 8;
|
||||
for (let p = 0; p < maskW * maskH; p++) {
|
||||
if (!maskBitset[p]) {
|
||||
const x = p % maskW;
|
||||
const y = (p / maskW) | 0;
|
||||
const light = ((x / tile) | 0) % 2 === ((y / tile) | 0) % 2;
|
||||
d.data[p * 4] = light ? 220 : 180;
|
||||
d.data[p * 4 + 1] = light ? 220 : 180;
|
||||
d.data[p * 4 + 2] = light ? 220 : 180;
|
||||
d.data[p * 4 + 3] = 220;
|
||||
}
|
||||
}
|
||||
octx.putImageData(d, 0, 0);
|
||||
ctx.drawImage(off, 0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
if (bbox && bbox.corners.length === 4) {
|
||||
const c = bbox.corners;
|
||||
ctx.strokeStyle = '#00e5ff';
|
||||
ctx.lineWidth = Math.max(2, canvas.width / 400);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(c[0][0], c[0][1]);
|
||||
for (let i = 1; i < 4; i++) ctx.lineTo(c[i][0], c[i][1]);
|
||||
ctx.closePath();
|
||||
ctx.stroke();
|
||||
// The axis reads corners[3] → corners[0], the long side.
|
||||
const a = c[3];
|
||||
const b = c[0];
|
||||
ctx.strokeStyle = '#ff3d71';
|
||||
ctx.lineWidth = Math.max(3, canvas.width / 300);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(a[0], a[1]);
|
||||
ctx.lineTo(b[0], b[1]);
|
||||
ctx.stroke();
|
||||
ctx.fillStyle = '#ff3d71';
|
||||
ctx.beginPath();
|
||||
const ang = Math.atan2(b[1] - a[1], b[0] - a[0]);
|
||||
const r = Math.max(10, canvas.width / 60);
|
||||
ctx.moveTo(b[0], b[1]);
|
||||
ctx.lineTo(b[0] - r * Math.cos(ang - 0.4), b[1] - r * Math.sin(ang - 0.4));
|
||||
ctx.lineTo(b[0] - r * Math.cos(ang + 0.4), b[1] - r * Math.sin(ang + 0.4));
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
}
|
||||
};
|
||||
|
||||
const redraw = async () => {
|
||||
if (!blob || !canvas) return;
|
||||
const gen = ++decodeGen;
|
||||
decoding = true;
|
||||
let bmp: ImageBitmap | null = null;
|
||||
try {
|
||||
bmp = await decodeOrientedBitmap(blob);
|
||||
if (gen !== decodeGen || !canvas) return;
|
||||
imgW = bmp.width;
|
||||
imgH = bmp.height;
|
||||
canvas.width = imgW;
|
||||
canvas.height = imgH;
|
||||
applyFit();
|
||||
ctx.drawImage(img, 0, 0);
|
||||
if (maskBitset && maskW > 0 && maskH > 0) {
|
||||
const off = document.createElement('canvas');
|
||||
off.width = maskW;
|
||||
off.height = maskH;
|
||||
const octx = off.getContext('2d')!;
|
||||
const d = octx.createImageData(maskW, maskH);
|
||||
const tile = 8;
|
||||
for (let p = 0; p < maskW * maskH; p++) {
|
||||
if (!maskBitset[p]) {
|
||||
const x = p % maskW;
|
||||
const y = (p / maskW) | 0;
|
||||
const light = ((x / tile) | 0) % 2 === ((y / tile) | 0) % 2;
|
||||
d.data[p * 4] = light ? 220 : 180;
|
||||
d.data[p * 4 + 1] = light ? 220 : 180;
|
||||
d.data[p * 4 + 2] = light ? 220 : 180;
|
||||
d.data[p * 4 + 3] = 220;
|
||||
}
|
||||
}
|
||||
octx.putImageData(d, 0, 0);
|
||||
ctx.drawImage(off, 0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
if (bbox && bbox.corners.length === 4) {
|
||||
const c = bbox.corners;
|
||||
ctx.strokeStyle = '#00e5ff';
|
||||
ctx.lineWidth = Math.max(2, canvas.width / 400);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(c[0][0], c[0][1]);
|
||||
for (let i = 1; i < 4; i++) ctx.lineTo(c[i][0], c[i][1]);
|
||||
ctx.closePath();
|
||||
ctx.stroke();
|
||||
// The axis reads corners[3] → corners[0], the long side.
|
||||
const a = c[3];
|
||||
const b = c[0];
|
||||
ctx.strokeStyle = '#ff3d71';
|
||||
ctx.lineWidth = Math.max(3, canvas.width / 300);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(a[0], a[1]);
|
||||
ctx.lineTo(b[0], b[1]);
|
||||
ctx.stroke();
|
||||
ctx.fillStyle = '#ff3d71';
|
||||
ctx.beginPath();
|
||||
const ang = Math.atan2(b[1] - a[1], b[0] - a[0]);
|
||||
const r = Math.max(10, canvas.width / 60);
|
||||
ctx.moveTo(b[0], b[1]);
|
||||
ctx.lineTo(b[0] - r * Math.cos(ang - 0.4), b[1] - r * Math.sin(ang - 0.4));
|
||||
ctx.lineTo(b[0] - r * Math.cos(ang + 0.4), b[1] - r * Math.sin(ang + 0.4));
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
}
|
||||
};
|
||||
img.src = url;
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
ctx.drawImage(bmp, 0, 0);
|
||||
paintOverlays(ctx);
|
||||
} finally {
|
||||
if (bmp) bmp.close();
|
||||
if (gen === decodeGen) decoding = false;
|
||||
}
|
||||
};
|
||||
|
||||
$: if (url || bbox || maskBitset) void tick().then(redraw);
|
||||
$: if (blob || bbox || maskBitset) void tick().then(redraw);
|
||||
|
||||
onMount(() => {
|
||||
if (stage) parentW = stage.clientWidth;
|
||||
@@ -119,12 +122,12 @@
|
||||
return () => ro.disconnect();
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (url) URL.revokeObjectURL(url);
|
||||
decodeGen += 1;
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class="stage" bind:this={stage}>
|
||||
{#if url}
|
||||
{#if imgW > 0 || decoding || blob}
|
||||
<canvas bind:this={canvas}></canvas>
|
||||
{:else}
|
||||
<div class="loading">Decoding photo…</div>
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { test, afterEach } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import {
|
||||
decodeOrientedBitmap,
|
||||
ORIENTED_BITMAP_OPTIONS,
|
||||
} from '../decodeOrientedBitmap.ts';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const fixturePath = join(here, '../../tests/fixtures/set1-48x2/face-a.jpeg');
|
||||
|
||||
const RealCreateImageBitmap = globalThis.createImageBitmap;
|
||||
|
||||
afterEach(() => {
|
||||
if (RealCreateImageBitmap === undefined) {
|
||||
delete (globalThis as unknown as { createImageBitmap?: unknown }).createImageBitmap;
|
||||
} else {
|
||||
globalThis.createImageBitmap = RealCreateImageBitmap;
|
||||
}
|
||||
});
|
||||
|
||||
/** Minimal EXIF Orientation + largest SOF size for the orient-8 fixture. */
|
||||
function readJpegOrientAndSof(buf: Buffer): { orientation: number; sofW: number; sofH: number } {
|
||||
let orientation = 1;
|
||||
const exif = buf.indexOf(Buffer.from('Exif\0\0'));
|
||||
if (exif >= 0) {
|
||||
const tiff = exif + 6;
|
||||
const le = buf[tiff] === 0x49;
|
||||
const u16 = (o: number) => (le ? buf.readUInt16LE(o) : buf.readUInt16BE(o));
|
||||
const u32 = (o: number) => (le ? buf.readUInt32LE(o) : buf.readUInt32BE(o));
|
||||
const ifd = tiff + u32(tiff + 4);
|
||||
const n = u16(ifd);
|
||||
for (let k = 0; k < n; k++) {
|
||||
const e = ifd + 2 + k * 12;
|
||||
const tag = u16(e);
|
||||
const typ = u16(e + 2);
|
||||
if (tag === 0x0112) {
|
||||
orientation = typ === 3 ? u16(e + 8) : u32(e + 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
let sofW = 0;
|
||||
let sofH = 0;
|
||||
let i = 0;
|
||||
while (i < buf.length - 9) {
|
||||
if (buf[i] === 0xff && buf[i + 1] >= 0xc0 && buf[i + 1] <= 0xc3) {
|
||||
const h = buf.readUInt16BE(i + 5);
|
||||
const w = buf.readUInt16BE(i + 7);
|
||||
if (w * h > sofW * sofH) {
|
||||
sofW = w;
|
||||
sofH = h;
|
||||
}
|
||||
const segLen = buf.readUInt16BE(i + 2);
|
||||
i += 2 + segLen;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return { orientation, sofW, sofH };
|
||||
}
|
||||
|
||||
/** Orientation 5–8 swap width/height for display. */
|
||||
function orientedSize(sofW: number, sofH: number, orientation: number): { w: number; h: number } {
|
||||
if (orientation >= 5 && orientation <= 8) return { w: sofH, h: sofW };
|
||||
return { w: sofW, h: sofH };
|
||||
}
|
||||
|
||||
test('fixture face-a.jpeg is EXIF Orientation 8 (stored landscape → oriented portrait) (#72)', () => {
|
||||
const buf = readFileSync(fixturePath);
|
||||
const { orientation, sofW, sofH } = readJpegOrientAndSof(buf);
|
||||
assert.equal(orientation, 8, 'matched-set JPEGs carry Orientation 8');
|
||||
assert.equal(sofW, 4000);
|
||||
assert.equal(sofH, 2256);
|
||||
const { w, h } = orientedSize(sofW, sofH, orientation);
|
||||
assert.equal(w, 2256);
|
||||
assert.equal(h, 4000);
|
||||
});
|
||||
|
||||
test('decodeOrientedBitmap always requests imageOrientation from-image (#72)', async () => {
|
||||
let seen: ImageBitmapOptions | undefined;
|
||||
(globalThis as unknown as { createImageBitmap: typeof createImageBitmap }).createImageBitmap =
|
||||
async (_src: ImageBitmapSource, opts?: ImageBitmapOptions) => {
|
||||
seen = opts;
|
||||
return { width: 2256, height: 4000, close() {} } as ImageBitmap;
|
||||
};
|
||||
|
||||
const blob = new Blob([readFileSync(fixturePath)], { type: 'image/jpeg' });
|
||||
const bmp = await decodeOrientedBitmap(blob);
|
||||
assert.deepEqual(seen, ORIENTED_BITMAP_OPTIONS);
|
||||
assert.equal(ORIENTED_BITMAP_OPTIONS.imageOrientation, 'from-image');
|
||||
assert.equal(bmp.width, 2256);
|
||||
assert.equal(bmp.height, 4000);
|
||||
});
|
||||
|
||||
test('decodeOrientedBitmap falls back to bare createImageBitmap if option rejected (#72)', async () => {
|
||||
let calls = 0;
|
||||
(globalThis as unknown as { createImageBitmap: typeof createImageBitmap }).createImageBitmap =
|
||||
async (_src: ImageBitmapSource, opts?: ImageBitmapOptions) => {
|
||||
calls += 1;
|
||||
if (opts && 'imageOrientation' in opts) {
|
||||
throw new TypeError('imageOrientation unsupported');
|
||||
}
|
||||
return { width: 90, height: 160, close() {} } as ImageBitmap;
|
||||
};
|
||||
|
||||
const bmp = await decodeOrientedBitmap(new Blob([new Uint8Array([1, 2, 3])], { type: 'image/jpeg' }));
|
||||
assert.equal(calls, 2, 'retry bare after option rejection');
|
||||
assert.equal(bmp.width, 90);
|
||||
assert.equal(bmp.height, 160);
|
||||
});
|
||||
|
||||
test('decodeOrientedBitmap: Chromium orients fixture to 2256×4000 (#72)', async (t) => {
|
||||
let chromium: typeof import('@playwright/test').chromium;
|
||||
try {
|
||||
({ chromium } = await import('@playwright/test'));
|
||||
} catch {
|
||||
t.skip('playwright not available');
|
||||
return;
|
||||
}
|
||||
const browser = await chromium.launch();
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
const buf = readFileSync(fixturePath);
|
||||
const b64 = buf.toString('base64');
|
||||
// Inline the same contract as decodeOrientedBitmap (worker-safe, no bundler).
|
||||
const dims = await page.evaluate(async (b64jpeg: string) => {
|
||||
const bin = atob(b64jpeg);
|
||||
const arr = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i);
|
||||
const blob = new Blob([arr], { type: 'image/jpeg' });
|
||||
const opts = { imageOrientation: 'from-image' as const };
|
||||
let bmp: ImageBitmap;
|
||||
try {
|
||||
bmp = await createImageBitmap(blob, opts);
|
||||
} catch {
|
||||
bmp = await createImageBitmap(blob);
|
||||
}
|
||||
const out = { w: bmp.width, h: bmp.height };
|
||||
bmp.close();
|
||||
return out;
|
||||
}, b64);
|
||||
assert.equal(dims.w, 2256, `oriented width (got ${dims.w})`);
|
||||
assert.equal(dims.h, 4000, `oriented height (got ${dims.h})`);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import { measure } from '../measure/measure';
|
||||
import { orientedRect } from '../geometry/rect';
|
||||
import { encodeInventoryPng, type InventoryInput } from '../png/inventory';
|
||||
import { encodeOverlayPng, type EncodeOverlayInput } from '../png/overlay';
|
||||
import { decodeOrientedBitmap } from '../decodeOrientedBitmap';
|
||||
|
||||
const DEBOUNCE_MS = 2000;
|
||||
const BUSY_MS = 300;
|
||||
@@ -512,7 +513,7 @@ export function createCaptureSession(opts: SessionOpts = {}): CaptureSession {
|
||||
if (state.k !== 'measure') return;
|
||||
const { source, approvedMask, axis, lengthEighths, thicknessEighths } = state;
|
||||
if (lengthEighths === 0 || thicknessEighths === 0) return;
|
||||
const bitmap = await createImageBitmap(source.blob);
|
||||
const bitmap = await decodeOrientedBitmap(source.blob);
|
||||
const base = measure(
|
||||
approvedMask.bitset, source.widthPx, source.heightPx,
|
||||
axis.bbox, lengthEighths, thicknessEighths, approvedMask.slabPx,
|
||||
@@ -587,7 +588,7 @@ export function createCaptureSession(opts: SessionOpts = {}): CaptureSession {
|
||||
widthPx: derived.widthPx,
|
||||
heightPx: derived.heightPx,
|
||||
};
|
||||
const siblingBitmap = await createImageBitmap(photo.blob);
|
||||
const siblingBitmap = await decodeOrientedBitmap(photo.blob);
|
||||
let encoded: { blob: Blob; sourceBelowFloor: boolean };
|
||||
try {
|
||||
encoded = await encodeInventory({ bitmap: siblingBitmap, mask: derived.mask, source: siblingSource });
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Shared decode for capture photos so PhotoStage, mask worker, inventory, and
|
||||
* calculate all share one EXIF-oriented pixel space.
|
||||
*
|
||||
* Matched-set / DJI JPEGs often carry EXIF Orientation 8 (stored landscape,
|
||||
* display portrait). `new Image()` and bare `createImageBitmap(blob)` can
|
||||
* disagree across browsers → pink axis arrow space ≠ mask/overlay bitset space.
|
||||
*
|
||||
* Contract: always request `imageOrientation: 'from-image'`. If the runtime
|
||||
* rejects that option (older engines), fall back to bare `createImageBitmap`;
|
||||
* engines that already apply EXIF by default still match Image/canvas size.
|
||||
* Engines that never apply EXIF may still diverge after fallback — document
|
||||
* that path rather than inventing a software rotate.
|
||||
*/
|
||||
export const ORIENTED_BITMAP_OPTIONS: ImageBitmapOptions = {
|
||||
imageOrientation: 'from-image',
|
||||
};
|
||||
|
||||
export async function decodeOrientedBitmap(blob: Blob): Promise<ImageBitmap> {
|
||||
try {
|
||||
return await createImageBitmap(blob, ORIENTED_BITMAP_OPTIONS);
|
||||
} catch (first) {
|
||||
try {
|
||||
return await createImageBitmap(blob);
|
||||
} catch {
|
||||
throw first;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -545,6 +545,43 @@ function tiltedAxis25Fixture() {
|
||||
return { W, H, bitset, bbox, ux0, uy0, rx0, ry0, halfL, cx, cy };
|
||||
}
|
||||
|
||||
|
||||
test('buildOverlayPlan: angleDeg=90 vertical long side → width left→right within 2° of horizontal (#72)', () => {
|
||||
// Upright slab: approved length axis is vertical (angleDeg 90). Width marks must
|
||||
// run left→right ≈ horizontal (atan2 ≈ 0 or ±π), not slanted with the photo.
|
||||
const W = 200, H = 400;
|
||||
const bitset = new Uint8Array(W * H);
|
||||
for (let y = 40; y <= 360; y++) {
|
||||
for (let x = 60; x <= 140; x++) bitset[y * W + x] = 1;
|
||||
}
|
||||
const prev = minAreaRect(bitset, W, H)!;
|
||||
const bbox = orientedRect(prev, 90);
|
||||
assert.equal(bbox.angleDeg, 90);
|
||||
const scale = 10; // 320px span ≈ 32"
|
||||
const plan = buildOverlayPlan({
|
||||
bitset,
|
||||
widthPx: W,
|
||||
heightPx: H,
|
||||
bbox,
|
||||
scale,
|
||||
lengthIn: 32,
|
||||
lengthLabel: '32"',
|
||||
mapToInventory: (x, y) => [x, y],
|
||||
});
|
||||
assert.ok(plan.widths.length > 0, 'expected width stations');
|
||||
for (const w of plan.widths) {
|
||||
const ang = Math.atan2(w.right[1] - w.left[1], w.right[0] - w.left[0]);
|
||||
// Fold to [-90, 90] so left→right and right→left both count as horizontal.
|
||||
let deg = (ang * 180) / Math.PI;
|
||||
if (deg > 90) deg -= 180;
|
||||
if (deg < -90) deg += 180;
|
||||
assert.ok(
|
||||
Math.abs(deg) <= 2,
|
||||
`width station ${w.stationIn} left→right must be within 2° of horizontal (atan2 deg=${deg.toFixed(2)}; left=${w.left} right=${w.right})`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('buildOverlayPlan: length a→b || approved (ux,uy); width left→right || (rx,ry) (#70)', () => {
|
||||
const { W, H, bitset, bbox, cx, cy } = tiltedAxis25Fixture();
|
||||
const scale = 10;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { minAreaRect, slabBounds } from '../geometry/rect';
|
||||
import type { MaskParams, WorkerIn, WorkerOut } from '../capture/capture.types';
|
||||
import { decodeOrientedBitmap } from '../decodeOrientedBitmap';
|
||||
|
||||
let bitmap: ImageBitmap | null = null;
|
||||
let data: Uint8ClampedArray | null = null;
|
||||
@@ -69,7 +70,7 @@ self.onmessage = (e: MessageEvent<WorkerIn>) => {
|
||||
}
|
||||
data = null;
|
||||
w = h = 0;
|
||||
createImageBitmap(msg.blob)
|
||||
decodeOrientedBitmap(msg.blob)
|
||||
.then((bmp) => {
|
||||
w = bmp.width;
|
||||
h = bmp.height;
|
||||
|
||||
Reference in New Issue
Block a user