diff --git a/README.md b/README.md index 1acbda0..60f4d76 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ Linux driver tools for the Hama myScan (00052343) scanner mouse, a Dacuda design prior; another device can pass its own object with `predict()`), `register` (ECC, phase correlation, SIFT matching, overlap check), `canvas` (weighted canvases, page composition, parallel rendering), `posegraph` (robust pose-graph optimisation), `loops` (loop-closure candidates), and the two stitchers built from them: - `pagegraph` (`PageGraph`: assembles the page from all frames at once with loop closures between passes; + `pagegraph` (`PageGraph`: assembles the page from all frames at once with loop closures between passes, + dropping frames taken while the mouse lies still; used by `scan.py` and `stitch.py`) and `incremental` (`Stitcher`: frame by frame, `stitch.py --live`) - `tools/scan.py` — scan with the button: `python tools/scan.py` (press to start, press again to stop). `--view [PORT]` serves a live view at `http://:8000/`: the camera, the page as it is stitched diff --git a/myscan/live.py b/myscan/live.py index f182fd0..0152a06 100644 --- a/myscan/live.py +++ b/myscan/live.py @@ -108,6 +108,7 @@ class LiveSession: self.preview = LivePreview(graph) self.raw = self.rec = None # latest captured frame and its sensor record (on_capture) self.e6, self.times = 0, [] + self.frame_of = [] # graph frame showing each processed capture (still ones are dropped) self.dist, self.dist_done = 0.0, 0 viewer.set_status(sensors=None) self._solving = True @@ -124,7 +125,10 @@ class LiveSession: self.raw, self.rec, self.e6 = raw, rec, e6 def on_frame(self, corrected): - self.preview.add(corrected) + """Called with what PageGraph.add returned for each captured frame, in order.""" + self.frame_of.append(self.g.n - 1) + if corrected is not None: + self.preview.add(corrected) def sensors(self, window=0.3): """Live readings for the viewer, or None before the first registered frame: @@ -137,10 +141,11 @@ class LiveSession: only had the lasers, which happens when moving too fast or lifted); lasers: the two laser sensors' surface quality (record bytes 14, 15); exposure (ms), level: paper brightness (0-255), fps.""" - g, times, rec = self.g, self.times, self.rec - i = len(g.odo) # newest frame with an odometry edge - if i < 1 or i >= len(times) or rec is None: + g, times, rec, frame_of = self.g, self.times, self.rec, self.frame_of + k = len(frame_of) - 1 # newest processed capture + if k < 1 or rec is None: return None + i = frame_of[k] # and the graph frame that shows it c = np.array([g.w / 2, g.h / 2, 1.0]) def centre(k): @@ -152,10 +157,10 @@ class LiveSession: # The frame's x axis, (cos θ, sin θ) on the canvas, is (-sin θ, -cos θ) on the page: # it points up at θ = 0 and turns by -θ. heading = -np.degrees(np.arctan2(M[1, 0], M[0, 0])) - j = bisect.bisect_left(times, times[i] - window, 0, i) + j = bisect.bisect_left(times, times[k] - window, 0, k) p = centre(i) - dt = times[i] - times[j] - vel = (p - centre(j)) * self.mm / dt if j < i and dt > 0 else np.zeros(2) + dt = times[k] - times[j] + vel = (p - centre(frame_of[j])) * self.mm / dt if j < k and dt > 0 else np.zeros(2) for _, _, T, _ in g.odo[self.dist_done:i]: self.dist += float(np.hypot(*(T @ c - c)[:2])) * self.mm self.dist_done = i @@ -168,7 +173,7 @@ class LiveSession: "velocity": [float(v) for v in vel], "speed": float(np.hypot(*vel)), "distance": self.dist, "offset": [float(v) for v in (p - centre(0)) * self.mm], "area": self.preview.covered * (self.mm / self.preview.s) ** 2 / 100, - "tracking": sum(e[3] == SIGMA_ECC for e in recent) / len(recent), + "tracking": sum(e[3] == SIGMA_ECC for e in recent) / len(recent) if recent else 1.0, "lasers": [int(rec.b14), int(rec.b15)], "exposure": self.e6 / E6_PER_MS, "level": bright_level(raw) if raw is not None else 0.0, "fps": float(fps), diff --git a/myscan/scanner.py b/myscan/scanner.py index f4616af..0817ff3 100644 --- a/myscan/scanner.py +++ b/myscan/scanner.py @@ -112,7 +112,8 @@ def scan_page(x, a, frames_dir=None, corrector=None, gains=None, viewer=None, mm if viewer: viewer.set_status(t=t, frames=n + 1, backlog=work.qsize()) if t - last >= 1: - print(f"\r {t:5.1f}s {n + 1} frames ({g.n} registered) E6={s.e6}", end="", flush=True) + print(f"\r {t:5.1f}s {n + 1} frames ({g.n} stitched, {g.stats['still']} still) E6={s.e6}", + end="", flush=True) last = t print() if viewer: diff --git a/myscan/stitching/pagegraph.py b/myscan/stitching/pagegraph.py index b363ff1..d2938e4 100644 --- a/myscan/stitching/pagegraph.py +++ b/myscan/stitching/pagegraph.py @@ -17,6 +17,9 @@ wrong copy. Here every frame keeps its own pose and the poses are solved togethe pose; the rest (off the page, lifted, corrupt) only had the odometry and are left out, as are frames that disagree with the mosaic of all the others. +Frames that show nothing new (the mouse lying still) are dropped on arrival: they cost +registration, solving and rendering time and add nothing to the page. + A pose M (3x3) maps frame pixels to canvas pixels; an edge (i, j, T) says M_j ~ M_i @ T. """ import os @@ -37,6 +40,10 @@ SIGMA_ECC = 1.0 # px, edge registered by ECC SIGMA_ODO = 6.0 # px, odometry edge from the motion prior only SIGMA_LOOP = 1.5 # px, loop closure PRUNE = 5.0 # drop loop edges whose residual exceeds this many sigmas +STILL_PX = 1.0 # odometry motion (px at the frame corners) below which a frame may be still +STILL_DIFF = 0.01 # and thumbnail difference below which it is (sensor noise is ~0.002, + # slow motion 0.01-0.03; relative to the mean, so exposure changes don't count) +THUMB = (80, 37) # thumbnail size for that test class PageGraph: @@ -56,10 +63,12 @@ class PageGraph: self._hpf = {} self.odo, self.bridges, self.loops = [], [], [] # edges: (i, j, T, sigma) self.prev_v = None + self._thumb = None # of the last frame added self._feat, self._tried = {}, set() # kept across solves self._pool = None self.solution = None # (poses (n, 3), indices of frames to draw) - self.stats = {"frames": 0, "odo_ecc": 0, "odo_laser": 0, "bridges": 0, "loop_tries": 0, "pruned": 0} + self.stats = {"frames": 0, "odo_ecc": 0, "odo_laser": 0, "bridges": 0, "loop_tries": 0, "pruned": 0, + "still": 0} @property def n(self): @@ -76,13 +85,29 @@ class PageGraph: self._feat[i] = register.detect(self._hp(i)) return self._feat[i] + def _still(self, thumb, v): + """Whether a frame with this thumbnail and sensor values shows the same as the last + frame added: neither the odometry (without image shift) nor the image moved.""" + T, _ = self.odometry.predict(self.prev_v, v, self.center) + q = np.array([[0, self.w, 0, self.w], [0, 0, self.h, self.h], [1, 1, 1, 1]]) + if np.abs(T @ q - q).max() >= STILL_PX: + return False + return cv2.norm(thumb, self._thumb, cv2.NORM_L1) / thumb.size < STILL_DIFF + def add(self, frame, v): """frame: HxW or HxWx3 uint8 raw frame; v: its sensor values for the odometry (the - myScan record's six raw values). Returns the corrected float frame. + myScan record's six raw values). Returns the corrected float frame, or None if the + frame was dropped as still (see _still). Safe to call while solve() runs on another thread.""" + v = np.asarray(v) + thumb = cv2.resize(frame, THUMB, interpolation=cv2.INTER_AREA).astype(np.float32) + thumb = gray(thumb) / max(float(thumb.mean()), 1.0) + if self.n and self._still(thumb, v): + self.stats["still"] += 1 + return None + self._thumb = thumb f = self.correct(frame) hs = cv2.resize(highpass_fast(gray(f)), self.half, interpolation=cv2.INTER_AREA) - v = np.asarray(v) i = self.n self.frames.append(frame) self.hs.append(hs)