erikandClaude Opus 5.5
e70d56d957
- ScanSession: scan-mode context manager with bright-pixel auto-exposure and sensor colour gains; tools/scan.py scans on the scan button with live stitching - Read-only EEPROM access and calib.py parser (laser geometry, intrinsics, colour gains, black/white flat-field samples); flat-field correction in Stitcher - OV7740 sensor access via the MCU proxy; colour gains from EEPROM fix green cast - Stitcher: colour support, high-passed registration (robust on blank paper) - RE notes on the Mac app's colour, exposure, stitching and button handling Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
200 lines
9.1 KiB
Python
200 lines
9.1 KiB
Python
"""Assemble scan frames into a page image.
|
|
|
|
Poses come from dead reckoning on the two laser sensors embedded in each
|
|
frame's record (see frame.py), then each frame is refined against the canvas
|
|
built so far with ECC (Euclidean model) before it is blended in.
|
|
|
|
Laser calibration (from scan_text, 2026-10-02; fit against image registration):
|
|
v0/v1 = laser 1 cumulative (a, b) counts, v3/v4 = laser 2; 16-bit wrapping.
|
|
content shift between consecutive frames, in frame pixels:
|
|
dx = -0.022*a - 0.355*b, dy = 0.324*a + 0.002*b (a, b = mean of both lasers)
|
|
image rotation (rad) = ROT_GAIN * (db1 - db2)
|
|
"""
|
|
import numpy as np
|
|
import cv2
|
|
|
|
SHIFT = np.array([[-0.0222, -0.3551], # dx from (a, b)
|
|
[0.3239, 0.0020]]) # dy from (a, b)
|
|
ROT_GAIN = -4.4e-4
|
|
MIN_PHASE_RESP = 0.2 # phase-correlation peak needed to override the lasers
|
|
|
|
|
|
def _s16(v):
|
|
v = np.asarray(v, np.int64) & 0xFFFF
|
|
return np.where(v >= 0x8000, v - 0x10000, v)
|
|
|
|
|
|
def laser_deltas(v):
|
|
"""v: (N, 6) record values -> (N, 4) per-frame deltas (a1, b1, a2, b2); first row zero."""
|
|
p = _s16(np.asarray(v)[:, [0, 1, 3, 4]])
|
|
d = np.diff(p, axis=0, prepend=p[:1])
|
|
return (d + 0x8000) % 0x10000 - 0x8000
|
|
|
|
|
|
def _aff(theta, tx, ty):
|
|
c, s = np.cos(theta), np.sin(theta)
|
|
return np.array([[c, -s, tx], [s, c, ty], [0, 0, 1.0]])
|
|
|
|
|
|
def laser_shift(d):
|
|
"""Content shift (dx, dy) in frame pixels predicted from one frame's laser deltas."""
|
|
return SHIFT @ ((d[0] + d[2]) / 2, (d[1] + d[3]) / 2)
|
|
|
|
|
|
def relative_motion(d, center, rot_gain=ROT_GAIN, shift=None):
|
|
"""3x3 transform mapping current-frame pixels to previous-frame pixels.
|
|
shift overrides the laser-predicted content shift."""
|
|
dx, dy = laser_shift(d) if shift is None else shift
|
|
th = rot_gain * (d[1] - d[3])
|
|
cx, cy = center
|
|
# content moved by (dx, dy) and rotated by th about the center: p_prev = R(-th)(p - c - d) + c
|
|
return _aff(0, cx, cy) @ _aff(-th, 0, 0) @ _aff(0, -cx - dx, -cy - dy)
|
|
|
|
|
|
def _highpass(img, mask=None, sigma=12):
|
|
"""Remove illumination (vignetting, LED falloff) so registration sees only page texture.
|
|
With a mask, the local mean is taken over masked pixels only."""
|
|
if mask is None:
|
|
return img - cv2.GaussianBlur(img, (0, 0), sigma)
|
|
m = mask.astype(np.float32)
|
|
mean = cv2.GaussianBlur(img * m, (0, 0), sigma) / np.maximum(cv2.GaussianBlur(m, (0, 0), sigma), 1e-3)
|
|
return (img - mean) * m
|
|
|
|
|
|
def _feather(h, w):
|
|
y = np.minimum(np.arange(h), np.arange(h)[::-1]) + 1
|
|
x = np.minimum(np.arange(w), np.arange(w)[::-1]) + 1
|
|
return np.minimum.outer(y, x).astype(np.float32).clip(max=40) / 40
|
|
|
|
|
|
class Stitcher:
|
|
"""Feed frames (grayscale or BGR) with their raw record values; registration runs on
|
|
grayscale, blending keeps all channels. Dropped frames are fine: the laser
|
|
positions are cumulative, so deltas are taken against the last frame added."""
|
|
|
|
def __init__(self, frame_shape, margin=600, refine=True, rot_gain=ROT_GAIN, flat=None):
|
|
self.h, self.w = frame_shape[:2]
|
|
self.ch = frame_shape[2] if len(frame_shape) > 2 else 1
|
|
self.refine, self.rot_gain, self.flat = refine, rot_gain, flat
|
|
self.size = (self.w + 2 * margin, self.h + 2 * margin) # grows as needed
|
|
self.acc = np.zeros(self.size[::-1] + ((self.ch,) if self.ch > 1 else ()), np.float32)
|
|
self.wsum = np.zeros(self.size[::-1], np.float32)
|
|
self.pose = _aff(0, margin, margin) # frame -> canvas
|
|
self.feather = _feather(self.h, self.w)
|
|
self.window = cv2.createHanningWindow((self.w, self.h), cv2.CV_32F)
|
|
self.prev = None # previous frame, high-passed
|
|
self.prev_v = None # previous raw record values
|
|
self.n = 0
|
|
self.stats = {"refined": 0, "rejected": 0, "skipped": 0, "image_shift": 0}
|
|
|
|
def _corners(self, M):
|
|
c = np.array([[0, 0, 1], [self.w, 0, 1], [0, self.h, 1], [self.w, self.h, 1.0]]).T
|
|
return (M @ c)[:2]
|
|
|
|
def _grow(self, M):
|
|
q = self._corners(M)
|
|
pad = 400
|
|
left = int(max(0, pad - q[0].min())) if q[0].min() < 50 else 0
|
|
top = int(max(0, pad - q[1].min())) if q[1].min() < 50 else 0
|
|
right = int(max(0, q[0].max() + pad - self.size[0])) if q[0].max() > self.size[0] - 50 else 0
|
|
bottom = int(max(0, q[1].max() + pad - self.size[1])) if q[1].max() > self.size[1] - 50 else 0
|
|
if left or top or right or bottom:
|
|
self.acc = cv2.copyMakeBorder(self.acc, top, bottom, left, right, cv2.BORDER_CONSTANT, 0)
|
|
self.wsum = cv2.copyMakeBorder(self.wsum, top, bottom, left, right, cv2.BORDER_CONSTANT, 0)
|
|
self.size = self.wsum.shape[::-1]
|
|
shift = _aff(0, left, top)
|
|
self.pose = shift @ self.pose
|
|
return shift @ M
|
|
return M
|
|
|
|
def _roi(self, M):
|
|
"""Canvas rectangle (x0, y0, x1, y1) covered by a frame at pose M."""
|
|
q = self._corners(M)
|
|
x0, y0 = np.floor(q.min(1)).astype(int) - 1
|
|
x1, y1 = np.ceil(q.max(1)).astype(int) + 2
|
|
return max(x0, 0), max(y0, 0), min(x1, self.size[0]), min(y1, self.size[1])
|
|
|
|
def _gray_canvas(self, x0, y0, x1, y1):
|
|
acc = self.acc[y0:y1, x0:x1]
|
|
return acc.mean(axis=2) if self.ch > 1 else acc
|
|
|
|
def _refine(self, gray, M):
|
|
# Canvas content seen through the predicted pose, in frame coordinates.
|
|
x0, y0, x1, y1 = self._roi(M)
|
|
Mi = np.linalg.inv(_aff(0, -x0, -y0) @ M)[:2]
|
|
acc = cv2.warpAffine(self._gray_canvas(x0, y0, x1, y1), Mi, (self.w, self.h), flags=cv2.INTER_LINEAR)
|
|
ws = cv2.warpAffine(self.wsum[y0:y1, x0:x1], Mi, (self.w, self.h), flags=cv2.INTER_LINEAR)
|
|
mask = ws > 0.3
|
|
if mask.mean() < 0.3:
|
|
self.stats["skipped"] += 1
|
|
return M
|
|
patch = _highpass(np.where(mask, acc / np.maximum(ws, 1e-6), 0).astype(np.float32), mask)
|
|
W = np.eye(2, 3, dtype=np.float32)
|
|
try:
|
|
_, W = cv2.findTransformECC(patch, gray, W, cv2.MOTION_EUCLIDEAN,
|
|
(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 60, 1e-4),
|
|
mask.astype(np.uint8), 5)
|
|
except cv2.error:
|
|
self.stats["rejected"] += 1
|
|
return M
|
|
# patch(x) ~ frame(W x), patch pixel x sits at canvas M x, so frame pixel y sits at M W^-1 y.
|
|
W3 = np.vstack([W, [0, 0, 1]]).astype(np.float64)
|
|
if abs(np.arctan2(W3[1, 0], W3[0, 0])) > np.radians(3) or np.hypot(W3[0, 2], W3[1, 2]) > 25:
|
|
self.stats["rejected"] += 1
|
|
return M
|
|
self.stats["refined"] += 1
|
|
return M @ np.linalg.inv(W3)
|
|
|
|
def add(self, frame, v):
|
|
"""frame: HxW or HxWx3 uint8; v: the record's six raw values (cumulative laser counts)."""
|
|
frame = frame.astype(np.float32) if self.flat is None else self.flat(frame)
|
|
gray = frame.mean(axis=2) if frame.ndim == 3 else frame
|
|
gray = np.ascontiguousarray(gray, np.float32)
|
|
hp = _highpass(gray)
|
|
v = np.asarray(v)
|
|
if self.n:
|
|
deltas = laser_deltas(np.stack([self.prev_v, v]))[1]
|
|
# The lasers sometimes miscount at speed (one or both), so check their translation
|
|
# against frame-to-frame phase correlation. High-passed frames keep the fixed
|
|
# illumination pattern from reading as "no motion" on blank paper.
|
|
shift = None
|
|
(dx, dy), resp = cv2.phaseCorrelate(self.prev, hp, self.window)
|
|
if resp > MIN_PHASE_RESP and np.hypot(*(laser_shift(deltas) - (dx, dy))) > 2:
|
|
shift = (dx, dy)
|
|
self.stats["image_shift"] += 1
|
|
M = self.pose @ relative_motion(deltas, (self.w / 2, self.h / 2), self.rot_gain, shift)
|
|
M = self._grow(M)
|
|
if self.refine:
|
|
M = self._refine(hp, M)
|
|
else:
|
|
M = self.pose
|
|
self.pose, self.prev, self.prev_v = M, hp, v
|
|
x0, y0, x1, y1 = self._roi(M)
|
|
Ml = (_aff(0, -x0, -y0) @ M)[:2]
|
|
wf = self.feather[..., None] if frame.ndim == 3 else self.feather
|
|
self.acc[y0:y1, x0:x1] += cv2.warpAffine(frame * wf, Ml, (x1 - x0, y1 - y0), flags=cv2.INTER_LINEAR)
|
|
self.wsum[y0:y1, x0:x1] += cv2.warpAffine(self.feather, Ml, (x1 - x0, y1 - y0), flags=cv2.INTER_LINEAR)
|
|
self.n += 1
|
|
|
|
def image(self, white_balance=True, background=255):
|
|
covered = self.wsum > 0.05
|
|
ws = np.maximum(self.wsum, 1e-6)
|
|
out = self.acc / (ws[..., None] if self.ch > 1 else ws)
|
|
if white_balance:
|
|
out = white_patch(out, covered)
|
|
m = covered[..., None] if self.ch > 1 else covered
|
|
out = np.where(m, out, background).clip(0, 255).astype(np.uint8)
|
|
ys, xs = np.nonzero(covered)
|
|
if len(ys):
|
|
out = out[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
|
|
# Frames arrive transposed and mirrored relative to the page.
|
|
return cv2.flip(cv2.transpose(out), -1)
|
|
|
|
|
|
def white_patch(img, mask, pct=97, white=245):
|
|
"""Scale each channel so its bright percentile (the paper) maps to `white`."""
|
|
img = img.astype(np.float32)
|
|
px = img[mask]
|
|
ref = np.percentile(px, pct, axis=0)
|
|
return img * (white / np.maximum(ref, 1))
|