From 1c58f12c5d07a580ae5a369fac243afed28cd976 Mon Sep 17 00:00:00 2001 From: Ty Balascio Date: Fri, 25 Sep 2026 09:07:39 -0700 Subject: [PATCH] feat(fe): replace threshold mask with client-side sheet BG removal (flood-fill) The green/black-sheet luminance threshold could not separate the slab from the 'black' sheet, which in real photos reads at lum ~50 and overlaps the slab's bark/shadow tones (measured on the user's samples: 0% green, 60-80% dark, no brightness gap) - it removed 100% of the image. Replace the mask pass with connectivity-based background removal: sample the sheet tone from the border, flood-fill from the border marking sheet-connected pixels as background, keep the connected remainder as the slab. Robust to tonal overlap. Adds bgRemoval.ts (removeBackground + sampleSheetTone) with unit tests proving a sheet-toned slab pixel survives via enclosure. --- frontend/src/capture/session.ts | 41 ++++----- frontend/src/mask/bgRemoval.test.ts | 70 +++++++++++++++ frontend/src/mask/bgRemoval.ts | 128 ++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+), 20 deletions(-) create mode 100644 frontend/src/mask/bgRemoval.test.ts create mode 100644 frontend/src/mask/bgRemoval.ts diff --git a/frontend/src/capture/session.ts b/frontend/src/capture/session.ts index 6d8f1ca..6616354 100644 --- a/frontend/src/capture/session.ts +++ b/frontend/src/capture/session.ts @@ -15,7 +15,7 @@ import { get, writable, type Readable } from 'svelte/store'; import { calibrationScale, fillFromSetting, FILL_DEFAULT } from '../png/crop'; import { measure } from '../measure/measure'; import { initialTapeLine, rectAlongLine, slabBounds, minAreaRect } from '../geometry/rect'; -import { keepForSheet } from '../mask/sheetKeep'; +import { removeBackground, sampleSheetTone } from '../mask/bgRemoval'; import { composeInventoryFrame } from '../png/inventory'; import { eighthsToIn, type Axis, type BatchPhotoResult, type Capture, type Mask, type MaskParams, type Measures, type Sheet, type Source } from './capture.types'; import { buildPhotosReadyLineup, type PhotosReadyLineupInput, type PhotosReadyLineupItem } from './photosReadyLineup'; @@ -99,32 +99,33 @@ export const buildMask = ( }; }; -/** Default mask pass: keep predicates from `mask/sheetKeep` over the decoded grid. */ +/** Default mask pass: client-side sheet removal (border flood-fill). */ export const defaultMaskRun: MaskRun = async (source, params, decoded) => buildMask(params, decoded, walkMask(decoded, params)); -/** Walk the decoded RGBA grid through the sheet-keep predicate (CAPTURE-30..60). */ +/** + * Mask pass (CAPTURE-30..60): remove the sheet-connected background by + * flood-filling from the image border against the sampled sheet tone, keeping + * the connected remainder as the slab. This is robust to sheet/slab luminance + * overlap that defeats a global brightness threshold. + * + * `sheet`: 'auto' (sampled tone) uses pure connectivity; 'green'/'black' bias + * the sampled tone toward that sheet so the fill anchors correctly. + * `sensitivity` (0–100) maps onto the color distance tolerance. + */ function walkMask(decoded: DecodedPhoto, params: MaskParams): Uint8Array { - const keep = keepForSheet(params.sheet, params.sensitivity, samplePixels(decoded)); const { width, height, rgba } = decoded; - const bitset = new Uint8Array(width * height); - for (let i = 0; i < width * height; i++) { - bitset[i] = keep(rgba[i * 4], rgba[i * 4 + 1], rgba[i * 4 + 2]) ? 1 : 0; - } - return bitset; + const tone = sheetToneFor(params.sheet, decoded); + const tolerance = 20 + (params.sensitivity / 100) * 80; // 20..100 distance + return removeBackground(width, height, rgba, { tolerance, sheetTone: tone }); } -/** Stride-16 edge-pixel sample for `autoPickSheet` (matches the P1 mask worker). */ -function samplePixels(decoded: DecodedPhoto): { r: number; g: number; b: number }[] { - const { width, height, rgba } = decoded; - const samples: { r: number; g: number; b: number }[] = []; - for (let y = 0; y < height; y += 16) { - for (let x = 0; x < width; x += 16) { - const i = (y * width + x) * 4; - samples.push({ r: rgba[i], g: rgba[i + 1], b: rgba[i + 2] }); - } - } - return samples; +/** Sheet tone to anchor the flood fill: sampled border, biased by the choice. */ +function sheetToneFor(sheet: MaskParams['sheet'], decoded: DecodedPhoto): { r: number; g: number; b: number } { + const sampled = sampleSheetTone(decoded.width, decoded.height, decoded.rgba); + if (sheet === 'green') return { r: 40, g: 170, b: 60 }; + if (sheet === 'black') return { r: 30, g: 30, b: 30 }; + return sampled; } /** Default inventory encode: P1 frame composer + canvas → image/png blob. */ diff --git a/frontend/src/mask/bgRemoval.test.ts b/frontend/src/mask/bgRemoval.test.ts new file mode 100644 index 0000000..e6cbbd8 --- /dev/null +++ b/frontend/src/mask/bgRemoval.test.ts @@ -0,0 +1,70 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { removeBackground, sampleSheetTone } from './bgRemoval'; + +function grid(w: number, h: number, fill: (x: number, y: number) => [number, number, number]): { + rgba: Uint8ClampedArray; + width: number; + height: number; +} { + const rgba = new Uint8ClampedArray(w * h * 4); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const [r, g, b] = fill(x, y); + const i = (y * w + x) * 4; + rgba[i] = r; + rgba[i + 1] = g; + rgba[i + 2] = b; + rgba[i + 3] = 255; + } + } + return { rgba, width: w, height: h }; +} + +test('removeBackground: slab on a border sheet is kept; sheet removed', () => { + // 10x10; border = dark sheet (30,30,30); inner 4..5 = bright slab (180,140,90) + const { rgba, width, height } = grid(10, 10, (x, y) => + x >= 3 && x <= 6 && y >= 3 && y <= 6 ? [180, 140, 90] : [30, 30, 30]); + const bitset = removeBackground(width, height, rgba, { tolerance: 40 }); + const slabPx = bitset.reduce((s, b) => s + b, 0); + // Inner 4x4 slab kept + assert.equal(slabPx, 16); + // Border pixel is background + assert.equal(bitset[0], 0); + // Center slab pixel kept + assert.equal(bitset[(5 * 10 + 5)], 1); +}); + +test('removeBackground: slab overlapping sheet luminance is kept via connectivity', () => { + // The failure mode of a global threshold: a dark slab pixel the SAME color as + // the sheet survives ONLY because it is enclosed by bright slab, never + // connected to the border. (A dark pixel touching the border is sheet.) + const { rgba, width, height } = grid(9, 9, (x, y) => { + // slab occupies x,y in 1..7 (a 7x7 region), sheet is the outer ring + if (x >= 1 && x <= 7 && y >= 1 && y <= 7) { + // an inner dark column at x=4, only in interior rows y=2..6 so it is never + // adjacent to the border sheet + if (x === 4 && y >= 2 && y <= 6) return [30, 30, 30]; + return [150, 120, 80]; + } + return [30, 30, 30]; // sheet (border ring) + }); + const bitset = removeBackground(width, height, rgba, { tolerance: 25 }); + // The enclosed dark slab column (x=4, y=2..6) must be kept — surrounded by + // bright slab, never connected to the border. + for (let y = 2; y <= 6; y++) { + assert.equal(bitset[y * 9 + 4], 1, `enclosed dark slab px at (4,${y}) kept`); + } + // Border sheet removed + assert.equal(bitset[0], 0); + assert.equal(bitset[8], 0); + assert.equal(bitset[80], 0); +}); + +test('sampleSheetTone: returns the dominant border color', () => { + const { rgba, width, height } = grid(8, 8, () => [40, 170, 60]); + const tone = sampleSheetTone(width, height, rgba); + assert.equal(tone.r, 40); + assert.equal(tone.g, 170); + assert.equal(tone.b, 60); +}); diff --git a/frontend/src/mask/bgRemoval.ts b/frontend/src/mask/bgRemoval.ts new file mode 100644 index 0000000..b46848d --- /dev/null +++ b/frontend/src/mask/bgRemoval.ts @@ -0,0 +1,128 @@ +/** + * Client-side background (sheet) removal for the M1 capture mask (CAPTURE-30..60). + * + * The sheet and the slab frequently OVERLAP in luminance (a "black" sheet often + * reads at lum ~50 while the slab's bark/shadow edges dip into the same band), + * so a global brightness threshold cannot separate them. Instead this uses + * connectivity: the sheet is the region CONNECTED TO THE IMAGE BORDER whose + * pixels are close in color to the sampled sheet tone. Everything not reached + * by that border flood-fill is the slab. + * + * This is the standard, robust approach used by the prior prototypes and is + * stable across lighting because it anchors on the actual sheet color present + * in the frame, not an assumed black/green. + */ + +export interface BgRemovalOptions { + /** + * Tolerance for "sheet-like": a pixel joins the background flood-fill if its + * per-channel distance to the sampled sheet tone is within this value. + * Maps the UI "sensitivity" (0–100) onto a 0–~120 distance. Higher = more + * aggressive removal (wider tolerance). + */ + tolerance: number; + /** Optional fixed sheet tone; sampled from the border when omitted. */ + sheetTone?: { r: number; g: number; b: number }; +} + +/** Sample the sheet tone from the outer border ring (median of border pixels). */ +export function sampleSheetTone( + width: number, + height: number, + rgba: Uint8Array | Uint8ClampedArray, +): { r: number; g: number; b: number } { + const rs: number[] = []; + const gs: number[] = []; + const bs: number[] = []; + const step = Math.max(1, Math.floor(Math.max(width, height) / 200)); + const push = (x: number, y: number) => { + const i = (y * width + x) * 4; + rs.push(rgba[i]); + gs.push(rgba[i + 1]); + bs.push(rgba[i + 2]); + }; + for (let x = 0; x < width; x += step) { + push(x, 0); + push(x, height - 1); + } + for (let y = 0; y < height; y += step) { + push(0, y); + push(width - 1, y); + } + const median = (arr: number[]) => { + const s = [...arr].sort((a, b) => a - b); + return s[Math.floor(s.length / 2)] ?? 0; + }; + return { r: median(rs), g: median(gs), b: median(bs) }; +} + +/** + * Remove the sheet-connected background. Returns a slab bitset (1 = keep/slab). + * Flood-fills from every border pixel; a neighbor joins the fill only while it + * stays within `tolerance` of the sheet tone. The slab is the complement. + */ +export function removeBackground( + width: number, + height: number, + rgba: Uint8Array | Uint8ClampedArray, + opts: BgRemovalOptions, +): Uint8Array { + const tone = opts.sheetTone ?? sampleSheetTone(width, height, rgba); + const tol2 = opts.tolerance * opts.tolerance; + const isSheetLike = (i: number) => { + const dr = rgba[i] - tone.r; + const dg = rgba[i + 1] - tone.g; + const db = rgba[i + 2] - tone.b; + return dr * dr + dg * dg + db * db <= tol2; + }; + + const n = width * height; + const visited = new Uint8Array(n); // 1 = reached by flood fill (= background) + const stack: number[] = []; + + // Seed from all border pixels that look sheet-like. + const seed = (x: number, y: number) => { + const idx = y * width + x; + if (visited[idx]) return; + if (isSheetLike(idx * 4)) { + visited[idx] = 1; + stack.push(idx); + } + }; + for (let x = 0; x < width; x++) { + seed(x, 0); + seed(x, height - 1); + } + for (let y = 0; y < height; y++) { + seed(0, y); + seed(width - 1, y); + } + + // Iterative flood fill (4-neighbor). + while (stack.length > 0) { + const idx = stack.pop()!; + const x = idx % width; + const y = (idx / width) | 0; + // 4-neighbors + if (x > 0 && !visited[idx - 1] && isSheetLike((idx - 1) * 4)) { + visited[idx - 1] = 1; + stack.push(idx - 1); + } + if (x < width - 1 && !visited[idx + 1] && isSheetLike((idx + 1) * 4)) { + visited[idx + 1] = 1; + stack.push(idx + 1); + } + if (y > 0 && !visited[idx - width] && isSheetLike((idx - width) * 4)) { + visited[idx - width] = 1; + stack.push(idx - width); + } + if (y < height - 1 && !visited[idx + width] && isSheetLike((idx + width) * 4)) { + visited[idx + width] = 1; + stack.push(idx + width); + } + } + + const bitset = new Uint8Array(n); + for (let i = 0; i < n; i++) bitset[i] = visited[i] ? 0 : 1; + return bitset; +}