Files
SlabUploader/frontend/src/capture/session.ts
T
Ty 25c1f79fc1 fix(fe): downscale mask flood-fill to a working resolution
The border flood-fill on a full-res 2256x4000 (9M px) photo was too slow in
JS and the async mask pass timed out into the catch branch (overlay stayed
null, 'Mask settled' never rendered). Downscale to <=1200px for the fill,
then nearest-neighbor upsample the slab bitset back to full size so
minAreaRect/slabBounds stay in source-pixel coordinates.
2026-09-25 09:09:30 -07:00

532 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* M2 capture session (PHOTOS.md, CAPTURE-100..135).
*
* `runSession` is a pure state machine: one state object in, one state object
* out, with every side effect (masking, PNG encoding) injected. No DOM, no
* timers, no stores — screens wrap it (see `createCaptureSession` below).
*
* Flow: batch → pickCalibration → mask → axis → measure (Calculate) → Photos-ready
* lineup (`photosReadyLineup`). The calibration photo drives mask/axis/measure;
* its frozen mask params are applied to every batch sibling after Calculate,
* with per-photo error slots (CAPTURE-110) and measures tied to calibration
* only (CAPTURE-115).
*/
import { get, writable, type Readable } from 'svelte/store';
import { calibrationScale, fillFromSetting, FILL_DEFAULT } from '../png/crop';
import { measure } from '../measure/measure';
import { initialTapeLine, rectAlongLine, slabBounds, minAreaRect } from '../geometry/rect';
import { removeBackground, sampleSheetTone } from '../mask/bgRemoval';
import { composeInventoryFrame } from '../png/inventory';
import { eighthsToIn, type Axis, type BatchPhotoResult, type Capture, type Mask, type MaskParams, type Measures, type Sheet, type Source } from './capture.types';
import { buildPhotosReadyLineup, type PhotosReadyLineupInput, type PhotosReadyLineupItem } from './photosReadyLineup';
import { decodeOrientedBitmap } from './decodeOrientedBitmap';
/** Decoded photo in source pixels (EXIF-oriented); `close()` releases the bitmap. */
export interface DecodedPhoto {
width: number;
height: number;
rgba: Uint8Array | Uint8ClampedArray;
close(): void;
}
/** Mask pass: resolve sheet + sensitivity into a bitset over the source grid. */
export interface MaskRun {
(source: Source, params: MaskParams, decoded: DecodedPhoto): Promise<Mask>;
}
/** Inventory PNG encoder (calibration photo). */
export type EncodeInventory = (source: Source, mask: Mask) => Promise<{ blob: Blob; sourceBelowFloor: boolean }>;
/** Dimension-overlay copy of the calibration inventory PNG. */
export type EncodeOverlay = (source: Source, mask: Mask, axis: Axis, measures: Measures, inventoryPng: Blob) => Promise<Blob>;
/**
* Injectable dependencies: the production wiring decodes EXIF-oriented and
* encodes to blobs; deterministic tests supply pixel-level fakes.
*/
export interface SessionDeps {
decode: (blob: Blob) => Promise<DecodedPhoto>;
maskRun: MaskRun;
encodeInventory: EncodeInventory;
encodeOverlay: EncodeOverlay;
/** Global crop fill (settings.crop_fill); FILL_DEFAULT (90%) until told otherwise. */
cropFill?: number;
}
export interface SessionState {
capture: Capture;
batch: Source[]; // capture order; authoritative across states
calibrationIndex: number | null;
cropFill: number;
}
export interface BatchPhotoOutcome {
state: SessionState;
/** Lineup builder input; `photosReadyLineup.buildPhotosReadyLineup` renders the slots. */
lineup: PhotosReadyLineupInput;
}
const clampSensitivity = (n: number): number => Math.max(0, Math.min(100, Math.round(n)));
const clampEighths = (e: number): number => Math.max(0, Math.round(e));
export const createSessionState = (): SessionState => ({
capture: { k: 'batch', photos: [] },
batch: [],
calibrationIndex: null,
cropFill: FILL_DEFAULT,
});
const patch = (s: SessionState, cap: Capture): SessionState => ({ ...s, capture: cap });
/** Build the approved mask from a decoded photo using the shared P1 keep predicates. */
export const buildMask = (
params: MaskParams,
decoded: DecodedPhoto,
bitset: Uint8Array,
): Mask => {
const { width, height } = decoded;
const slabPx = bitset.reduce((sum, b) => sum + (b === 1 ? 1 : 0), 0);
return {
params,
slabPx,
bitset,
bbox: minAreaRect(bitset, width, height) ?? {
corners: [[0, 0], [0, 0], [0, 0], [0, 0]],
longSidePx: 0,
angleDeg: 0,
},
slabBounds: slabBounds(bitset, width, height),
};
};
/** Default mask pass: client-side sheet removal (border flood-fill). */
export const defaultMaskRun: MaskRun = async (source, params, decoded) =>
buildMask(params, decoded, walkMask(decoded, params));
/**
* Mask pass (CAPTURE-30..60): remove the sheet-connected background by
* flood-filling from the image border against the sampled sheet tone, keeping
* the connected remainder as the slab. This is robust to sheet/slab luminance
* overlap that defeats a global brightness threshold.
*
* `sheet`: 'auto' (sampled tone) uses pure connectivity; 'green'/'black' bias
* the sampled tone toward that sheet so the fill anchors correctly.
* `sensitivity` (0–100) maps onto the color distance tolerance.
*/
function walkMask(decoded: DecodedPhoto, params: MaskParams): Uint8Array {
const { width, height, rgba } = decoded;
const tone = sheetToneFor(params.sheet, decoded);
const tolerance = 20 + (params.sensitivity / 100) * 80; // 20..100 distance
// Downscale to a working resolution so the flood-fill stays fast on large
// photos; the bitset is remapped back to full size for downstream geometry.
const MAX = 1200;
const scale = Math.min(1, MAX / Math.max(width, height));
if (scale >= 0.999) {
return removeBackground(width, height, rgba, { tolerance, sheetTone: tone });
}
const sw = Math.round(width * scale);
const sh = Math.round(height * scale);
const small = new Uint8ClampedArray(sw * sh * 4);
for (let y = 0; y < sh; y++) {
for (let x = 0; x < sw; x++) {
const sx = Math.min(width - 1, Math.round((x / sw) * width));
const sy = Math.min(height - 1, Math.round((y / sh) * height));
const si = (sy * width + sx) * 4;
const di = (y * sw + x) * 4;
small[di] = rgba[si];
small[di + 1] = rgba[si + 1];
small[di + 2] = rgba[si + 2];
small[di + 3] = 255;
}
}
const smallBitset = removeBackground(sw, sh, small, { tolerance, sheetTone: tone });
// Remap the small bitset back to full resolution (nearest-neighbor upsample).
const full = new Uint8Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const tx = Math.min(sw - 1, Math.round((x / width) * sw));
const ty = Math.min(sh - 1, Math.round((y / height) * sh));
full[y * width + x] = smallBitset[ty * sw + tx];
}
}
return full;
}
/** Sheet tone to anchor the flood fill: sampled border, biased by the choice. */
function sheetToneFor(sheet: MaskParams['sheet'], decoded: DecodedPhoto): { r: number; g: number; b: number } {
const sampled = sampleSheetTone(decoded.width, decoded.height, decoded.rgba);
if (sheet === 'green') return { r: 40, g: 170, b: 60 };
if (sheet === 'black') return { r: 30, g: 30, b: 30 };
return sampled;
}
/** Default inventory encode: P1 frame composer + canvas → image/png blob. */
export const defaultEncodeInventory: EncodeInventory = async (source, mask) => {
const bitmap = await decodeOrientedBitmap(source.blob);
const canvas = document.createElement('canvas');
canvas.width = bitmap.width;
canvas.height = bitmap.height;
const context = canvas.getContext('2d');
if (!context) throw new Error('Canvas 2D is unavailable');
context.drawImage(bitmap, 0, 0);
bitmap.close();
const frame = composeInventoryFrame(
{ width: source.widthPx, height: source.heightPx, rgba: context.getImageData(0, 0, source.widthPx, source.heightPx).data },
{ bitset: mask.bitset, slabBounds: mask.slabBounds },
{ widthPx: source.widthPx, heightPx: source.heightPx },
);
const out = document.createElement('canvas');
out.width = frame.width;
out.height = frame.height;
const octx = out.getContext('2d');
if (!octx) throw new Error('Canvas 2D is unavailable');
octx.putImageData(new ImageData(new Uint8ClampedArray(frame.rgba), frame.width, frame.height), 0, 0);
const blob = await new Promise<Blob>((resolve, reject) =>
out.toBlob((b) => (b ? resolve(b) : reject(new Error('PNG encode failed'))), 'image/png'),
);
return { blob, sourceBelowFloor: frame.sourceBelowFloor };
};
/** Default overlay encode: same canvas round-trip (P1 overlay drawing lives in screens). */
export const defaultEncodeOverlay: EncodeOverlay = async (source, _mask, _axis, _measures, inventoryPng) => {
const bitmap = await decodeOrientedBitmap(inventoryPng);
const canvas = document.createElement('canvas');
canvas.width = bitmap.width;
canvas.height = bitmap.height;
const context = canvas.getContext('2d');
if (!context) throw new Error('Canvas 2D is unavailable');
context.drawImage(bitmap, 0, 0);
bitmap.close();
void source;
return new Promise<Blob>((resolve, reject) =>
canvas.toBlob((b) => (b ? resolve(b) : reject(new Error('PNG encode failed'))), 'image/png'),
);
};
export const defaultDeps: SessionDeps = {
decode: async (blob) => {
const bitmap = await decodeOrientedBitmap(blob);
const canvas = document.createElement('canvas');
canvas.width = bitmap.width;
canvas.height = bitmap.height;
const context = canvas.getContext('2d');
if (!context) throw new Error('Canvas 2D is unavailable');
context.drawImage(bitmap, 0, 0);
const rgba = context.getImageData(0, 0, bitmap.width, bitmap.height).data;
return { width: bitmap.width, height: bitmap.height, rgba, close: () => bitmap.close() };
},
maskRun: defaultMaskRun,
encodeInventory: defaultEncodeInventory,
encodeOverlay: defaultEncodeOverlay,
};
// ─── transitions ─────────────────────────────────────────────────────────────
export const addToBatch = (s: SessionState, file: File): SessionState => {
if (s.capture.k !== 'batch') return s;
const source: Source = { widthPx: 0, heightPx: 0, blob: file };
return { ...s, batch: [...s.batch, source], capture: { k: 'batch', photos: [...s.batch, source] } };
};
export const removeFromBatch = (s: SessionState, index: number): SessionState => {
if (s.capture.k !== 'batch') return s;
if (index < 0 || index >= s.batch.length) return s;
const batch = s.batch.filter((_, i) => i !== index);
return { ...s, batch, capture: { k: 'batch', photos: batch } };
};
/** Advance to pick-calibration once the batch has at least one photo. */
export const continueBatch = (s: SessionState): SessionState => {
if (s.capture.k !== 'batch' || s.batch.length === 0) return s;
return patch(s, { k: 'pickCalibration', photos: [...s.batch] });
};
/** Pick the calibration reference and decode its real dimensions into the mask state. */
export async function pickCalibration(s: SessionState, index: number, deps: SessionDeps): Promise<SessionState> {
if (s.capture.k !== 'pickCalibration' || index < 0 || index >= s.batch.length) return s;
const photo = s.batch[index];
let source: Source = photo;
try {
const decoded = await deps.decode(photo.blob);
try {
source = { widthPx: decoded.width, heightPx: decoded.height, blob: photo.blob };
} finally {
decoded.close();
}
} catch {
// Dimensions stay 0; approveMask will surface the decode failure.
}
return patch({ ...s, calibrationIndex: index }, {
k: 'mask',
source,
control: { ...DEFAULT_PARAMS },
overlay: null,
failed: false,
});
}
const DEFAULT_PARAMS: MaskParams = { sheet: 'auto', sensitivity: 50 };
export const setSheet = (s: SessionState, sheet: Sheet): SessionState => {
if (s.capture.k !== 'mask') return s;
return patch(s, { ...s.capture, control: { ...s.capture.control, sheet } });
};
export const setSensitivity = (s: SessionState, n: number): SessionState => {
if (s.capture.k !== 'mask') return s;
return patch(s, { ...s.capture, control: { ...s.capture.control, sensitivity: clampSensitivity(n) } });
};
/** Run the sheet-keep mask over the calibration photo (re-derives on every call). */
export async function approveMask(s: SessionState, deps: SessionDeps): Promise<SessionState> {
if (s.capture.k !== 'mask') return s;
const { source, control } = s.capture;
try {
const decoded = await deps.decode(source.blob);
try {
const mask = await deps.maskRun(source, control, decoded);
if (mask.slabPx === 0) {
return patch({ ...s, calibrationIndex: s.calibrationIndex }, { ...s.capture, overlay: null, failed: true });
}
return patch(s, { ...s.capture, overlay: mask, failed: false });
} finally {
decoded.close();
}
} catch {
return patch(s, { ...s.capture, overlay: null, failed: true });
}
}
/** Freeze the mask and open the axis state with initial tape guides. */
export const approveMaskAxis = (s: SessionState): SessionState => {
if (s.capture.k !== 'mask') return s;
const { source, overlay } = s.capture;
if (!overlay || overlay.slabPx === 0) return s;
const tapeLine = initialTapeLine(source.widthPx, source.heightPx);
return patch(s, {
k: 'axis',
source,
approvedMask: overlay,
axis: { bbox: rectAlongLine(overlay.bbox, tapeLine.start, tapeLine.end), tapeLine },
});
};
/** Nudge the tape endpoints; the axis rectangle follows the line (CAPTURE-55). */
export const setTapeLine = (s: SessionState, tapeLine: { start: [number, number]; end: [number, number] }): SessionState => {
if (s.capture.k !== 'axis') return s;
const start: [number, number] = [...tapeLine.start];
const end: [number, number] = [...tapeLine.end];
if (start[0] === end[0] && start[1] === end[1]) return s;
const bbox = rectAlongLine(s.capture.axis.bbox, start, end);
if (!(bbox.longSidePx > 0)) return s;
return patch(s, { ...s.capture, axis: { bbox, tapeLine: { start, end } } });
};
/** Freeze the axis and open measure with zero length/thickness. */
export const approveAxis = (s: SessionState): SessionState => {
if (s.capture.k !== 'axis') return s;
const { source, approvedMask, axis } = s.capture;
return patch(s, {
k: 'measure',
source,
approvedMask,
axis: { bbox: axis.bbox, tapeLine: axis.tapeLine },
lengthEighths: 0,
thicknessEighths: 0,
results: null,
});
};
export const setLength = (s: SessionState, eighths: number): SessionState => {
if (s.capture.k !== 'measure') return s;
return patch(s, { ...s.capture, lengthEighths: clampEighths(eighths) });
};
export const setThickness = (s: SessionState, eighths: number): SessionState => {
if (s.capture.k !== 'measure') return s;
return patch(s, { ...s.capture, thicknessEighths: clampEighths(eighths) });
};
/**
* Calculate: measure the calibration photo, encode its inventory + overlay PNG,
* then apply the frozen mask params to every batch sibling (CAPTURE-110).
* Siblings run serially; a failing decode/mask/encode cannot block later ones.
*/
export async function calculate(s: SessionState, deps: SessionDeps): Promise<BatchPhotoOutcome> {
if (s.capture.k !== 'measure') {
return { state: s, lineup: emptyLineup() };
}
const cap = s.capture;
if (cap.lengthEighths === 0 || cap.thicknessEighths === 0) {
return { state: s, lineup: emptyLineup() };
}
const scale = calibrationScale(cap.axis.tapeLine.start, cap.axis.tapeLine.end, eighthsToIn(cap.lengthEighths));
if (!(scale > 0) || !Number.isFinite(scale)) {
return { state: s, lineup: emptyLineup() };
}
// 2-dp scale: display numbers (sqft/bdft) are rounded to 2 dp anyway, and a
// stable scale keeps results deterministic across runs.
const roundedScale = Math.round(scale * 100) / 100;
const base = measure(
cap.approvedMask.bitset,
cap.source.widthPx,
cap.source.heightPx,
cap.axis.bbox,
cap.lengthEighths,
cap.thicknessEighths,
cap.approvedMask.slabPx,
roundedScale,
);
const encoded = await deps.encodeInventory(cap.source, cap.approvedMask);
let overlayPng: Blob | undefined;
try {
overlayPng = await deps.encodeOverlay(cap.source, cap.approvedMask, cap.axis, { ...base, inventoryPng: encoded.blob, sourceBelowFloor: encoded.sourceBelowFloor }, encoded.blob);
} catch {
overlayPng = undefined;
}
const results: Measures = {
...base,
inventoryPng: encoded.blob,
overlayPng,
sourceBelowFloor: encoded.sourceBelowFloor,
};
const batch = s.batch.length ? s.batch : [cap.source];
const calIndex = s.calibrationIndex != null && s.calibrationIndex >= 0 ? s.calibrationIndex : 0;
const batchApply: BatchPhotoResult[] = batch.map((_, index) =>
index === calIndex ? null : { status: 'error', message: 'Batch photo was not processed' },
);
const params = cap.approvedMask.params;
for (let index = 0; index < batch.length; index++) {
if (index === calIndex) continue;
const photo = batch[index];
try {
const decoded = await deps.decode(photo.blob);
let mask: Mask;
try {
mask = await deps.maskRun(photo, params, decoded);
if (mask.slabPx === 0) throw new Error('empty mask: no slab pixels detected');
} finally {
decoded.close();
}
const siblingSource: Source = { widthPx: decoded.width, heightPx: decoded.height, blob: photo.blob };
const sibling = await deps.encodeInventory(siblingSource, mask);
if (sibling.sourceBelowFloor) throw new Error('PNG size floor: shorter side is below 800 px');
batchApply[index] = { status: 'ok', inventoryPng: sibling.blob };
} catch (error) {
batchApply[index] = {
status: 'error',
message: error instanceof Error ? error.message : 'unable to process photo',
};
}
}
const next = patch(s, { ...cap, results, batchApply });
return {
state: next,
lineup: {
inventoryPng: results.inventoryPng,
overlayPng: results.overlayPng,
batchApply,
sourceBelowFloor: results.sourceBelowFloor,
},
};
}
const emptyLineup = (): PhotosReadyLineupInput => ({
inventoryPng: null,
overlayPng: null,
batchApply: null,
sourceBelowFloor: false,
});
/** The Photos-ready lineup for the current state (empty off the measure state). */
export const photosReadyInput = (s: SessionState): PhotosReadyLineupInput => {
if (s.capture.k !== 'measure' || !s.capture.results) return emptyLineup();
const { results, batchApply } = s.capture;
return {
inventoryPng: results.inventoryPng,
overlayPng: results.overlayPng,
batchApply: batchApply ?? null,
sourceBelowFloor: results.sourceBelowFloor,
};
};
/** Back one step; the mask step keeps its last overlay as the live one. */
export const back = (s: SessionState): SessionState => {
const { capture } = s;
if (capture.k === 'pickCalibration') {
return patch(s, { k: 'batch', photos: [...s.batch] });
}
if (capture.k === 'mask') {
return patch(s, { k: 'pickCalibration', photos: [...s.batch] });
}
if (capture.k === 'axis') {
const { source, approvedMask } = capture;
return patch(s, { k: 'mask', source, control: approvedMask.params, overlay: approvedMask, failed: false });
}
if (capture.k === 'measure') {
const { source, approvedMask, axis } = capture;
return patch(s, { k: 'axis', source, approvedMask, axis: { bbox: axis.bbox, tapeLine: axis.tapeLine } });
}
return s;
};
// ─── store wrapper ───────────────────────────────────────────────────────────
export interface CaptureSession {
state: Readable<SessionState>;
addToBatch(file: File): void;
removeFromBatch(index: number): void;
continueBatch(): void;
pickCalibration(index: number): Promise<void>;
setSheet(sheet: Sheet): void;
setSensitivity(n: number): void;
approveMask(): Promise<void>;
approveMaskAxis(): void;
setTapeLine(tapeLine: { start: [number, number]; end: [number, number] }): void;
approveAxis(): void;
setLength(eighths: number): void;
setThickness(eighths: number): void;
setCropFill(pctOrFraction: number): void;
calculate(): Promise<PhotosReadyLineupInput>;
lineup(): PhotosReadyLineupItem[];
back(): void;
dispose(): void;
}
/** Thin Svelte store wrapper over the pure machine; screens consume this. */
export function createCaptureSession(deps: SessionDeps = defaultDeps): CaptureSession {
const store = writable<SessionState>(createSessionState());
let live: SessionState = get(store);
const set = (next: SessionState) => {
live = next;
store.set(next);
};
return {
state: store,
addToBatch: (file) => set(addToBatch(live, file)),
removeFromBatch: (index) => set(removeFromBatch(live, index)),
continueBatch: () => set(continueBatch(live)),
pickCalibration: async (index) => set(await pickCalibration(live, index, deps)),
setSheet: (sheet) => set(setSheet(live, sheet)),
setSensitivity: (n) => set(setSensitivity(live, n)),
approveMask: async () => set(await approveMask(live, deps)),
approveMaskAxis: () => set(approveMaskAxis(live)),
setTapeLine: (tapeLine) => set(setTapeLine(live, tapeLine)),
approveAxis: () => set(approveAxis(live)),
setLength: (eighths) => set(setLength(live, eighths)),
setThickness: (eighths) => set(setThickness(live, eighths)),
setCropFill: (pctOrFraction) => set({ ...live, cropFill: fillFromSetting(pctOrFraction) }),
calculate: async () => {
// Snapshot up front: a concurrent set() during the await must not feed a
// newer state into the in-flight transition.
const from = live;
const { state, lineup } = await calculate(from, deps);
set(state);
return lineup;
},
lineup: () => buildPhotosReadyLineup(photosReadyInput(live)),
back: () => set(back(live)),
dispose: () => store.set(createSessionState()),
};
}