#!/usr/bin/env python3 """ pose_estimation.py ================== Estimate robot joint angles (x, y, z, a, b, c, e) from triangulated marker poses, using the kinematic model in robot.json (via robot_fk.py). Design ------ The estimator is parametrised over JOINT VARIABLES, not links. This handles the tricky cases of this robot family generically: * Links with zero own markers (Base/x, Hand/b, Palm/c) — their variable is observable only through descendant markers. * A variable shared by several links (FingerA & FingerB share 'e'). * Occluded middle links — global BA reconstructs them from the fingers. Four switchable methods (robot.json -> pose_estimation.method): sequential_vector : analytic per joint from marker-pair / normal vectors (fast) sequential_fk : block-wise least squares along the chain (robust, 1 marker ok) global_ba : all variables jointly, position + normal residuals, robust loss hybrid : sequential_fk init -> global_ba refine (default, most stable) Observation input: marker_observation = "corner_pose" -> aruco_marker_poses.json (pos + measured normal) marker_observation = "center_point" -> aruco_positions_*.json (pos only) Both the engine (estimate_pose) and a CLI (main) live here. """ from __future__ import annotations import argparse import json import math import os import sys import time from collections import defaultdict from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import numpy as np sys.path.insert(0, str(Path(__file__).parent)) from robot_fk import RobotFK, STATE_KEYS # noqa: E402 try: from scipy.optimize import least_squares HAVE_SCIPY = True except ImportError: HAVE_SCIPY = False # ================================================================== # Config # ================================================================== DEFAULT_CFG: Dict[str, Any] = { "method": "hybrid", "marker_observation": "corner_pose", "use_normals": True, "normal_weight": 100.0, "robust_loss": "huber", "huber_delta_mm": 8.0, "max_iterations": 200, "min_cameras_per_marker": 2, "finger_block_joints": ["b", "c", "e"], "per_link_method": {}, } def load_pose_cfg(robot_data: Dict[str, Any]) -> Dict[str, Any]: cfg = dict(DEFAULT_CFG) cfg.update(robot_data.get("pose_estimation", {}) or {}) return cfg # ================================================================== # Observations # ================================================================== def load_observations(path: str, use_normals: bool, min_cams: int = 2) -> Dict[int, Dict[str, Any]]: """ Load marker observations. Accepts aruco_marker_poses.json (with measured normal + num_cameras) or aruco_positions_*.json (position only). Returns: marker_id -> {pos_mm:(3,), normal:(3,)|None, link:str, n_cams:int} """ data = json.load(open(path, "r", encoding="utf-8")) out: Dict[int, Dict[str, Any]] = {} for m in data.get("markers", []): mid = int(m.get("marker_id", m.get("id", -1))) if mid < 0: continue n_cams = int(m.get("num_cameras", 99)) if n_cams < min_cams: continue if "position_mm" in m: pos = np.array(m["position_mm"], dtype=float) elif "position_m" in m: pos = np.array(m["position_m"], dtype=float) * 1000.0 else: continue nrm = None if use_normals and m.get("normal") is not None: nv = np.array(m["normal"], dtype=float) nn = np.linalg.norm(nv) if nn > 1e-9: nrm = nv / nn out[mid] = {"pos_mm": pos, "normal": nrm, "link": m.get("link", "?"), "n_cams": n_cams} return out # ================================================================== # Kinematic chain analysis # ================================================================== def analyze_chain(fk: RobotFK) -> Dict[str, Any]: """ Derive, generically from the FK topology: ordered_vars : movable joint variables, root->tip order, de-duplicated var_links : variable -> list of links it drives link_markers : link -> [model marker ids] blocks : sequential estimation blocks; each block groups the zero-marker ancestor variables with the next marker- bearing joint variable, estimated from that link's own markers (+ siblings sharing the same variable). """ links = fk.links topo = fk._topo link_markers: Dict[str, List[int]] = {} for ln, ld in links.items(): ids = [] for mk in ld.get("markers", []) or []: if "id" in mk and "position" in mk: ids.append(int(mk["id"])) link_markers[ln] = ids link_var: Dict[str, str] = {} for ln, ld in links.items(): j = ld.get("jointToParent", {}) or {} if str(j.get("type", "")).lower() in ("revolute", "linear"): v = str(j.get("variable", "")).lower() if v: link_var[ln] = v var_type: Dict[str, str] = {} var_links: Dict[str, List[str]] = defaultdict(list) for ln, v in link_var.items(): var_links[v].append(ln) var_type[v] = str(links[ln].get("jointToParent", {}).get("type", "")).lower() ordered_vars: List[str] = [] for ln in topo: if ln in link_var and link_var[ln] not in ordered_vars: ordered_vars.append(link_var[ln]) # ---- build blocks ---- blocks: List[Dict[str, Any]] = [] var_block: Dict[str, int] = {} pending: List[str] = [] for ln in topo: if ln not in link_var: continue v = link_var[ln] own = link_markers.get(ln, []) if v in var_block: # shared variable already in a block -> add this link's markers there if own: blocks[var_block[v]]["markers"].extend(own) continue if own: bvars = [] for x in pending + [v]: if x not in bvars and x not in var_block: bvars.append(x) blocks.append({"vars": bvars, "markers": list(own), "anchor": ln}) for x in bvars: var_block[x] = len(blocks) - 1 pending = [] else: if v not in pending: pending.append(v) if pending: blocks.append({"vars": pending, "markers": [], "anchor": None}) for x in pending: var_block[x] = len(blocks) - 1 return { "ordered_vars": ordered_vars, "var_type": var_type, "var_links": dict(var_links), "link_markers": link_markers, "blocks": blocks, } # ================================================================== # Residuals # ================================================================== def model_markers(fk: RobotFK, state: Dict[str, float]) -> Dict[int, Dict[str, np.ndarray]]: T = fk.compute(state) return fk.all_markers_world(T) # mid -> {world_mm, normal_world, link, local_mm} def residual_vector(state: Dict[str, float], fk: RobotFK, obs: Dict[int, Dict[str, Any]], marker_ids: List[int], cfg: Dict[str, Any]) -> np.ndarray: """Position (mm) + optional normal (scaled) residuals over the given markers.""" model = model_markers(fk, state) res: List[float] = [] w_n = float(cfg.get("normal_weight", 30.0)) use_n = bool(cfg.get("use_normals", True)) for mid in marker_ids: if mid not in model or mid not in obs: continue mm = model[mid] dp = np.asarray(mm["world_mm"], float) - obs[mid]["pos_mm"] res.extend(dp.tolist()) if use_n and obs[mid]["normal"] is not None and "normal_world" in mm: dn = (np.asarray(mm["normal_world"], float) - obs[mid]["normal"]) * w_n res.extend(dn.tolist()) return np.asarray(res, dtype=float) def _state_from_vec(var_names: List[str], vec: np.ndarray, base: Dict[str, float]) -> Dict[str, float]: s = dict(base) for name, val in zip(var_names, vec): s[name] = float(val) return s # ================================================================== # Method: global bundle adjustment # ================================================================== def estimate_global_ba(fk: RobotFK, obs: Dict[int, Dict[str, Any]], var_names: List[str], x0: Dict[str, float], cfg: Dict[str, Any]) -> Dict[str, float]: if not HAVE_SCIPY: print("[WARN] scipy missing — global_ba skipped, returning init") return dict(x0) marker_ids = list(obs.keys()) base = {k: 0.0 for k in STATE_KEYS} base.update(x0) vec0 = np.array([base.get(v, 0.0) for v in var_names], dtype=float) def fun(vec): st = _state_from_vec(var_names, vec, base) return residual_vector(st, fk, obs, marker_ids, cfg) loss = cfg.get("robust_loss", "huber") f_scale = float(cfg.get("huber_delta_mm", 8.0)) try: sol = least_squares(fun, vec0, loss=loss, f_scale=f_scale, max_nfev=int(cfg.get("max_iterations", 200)) * max(1, len(var_names))) return _state_from_vec(var_names, sol.x, base) except Exception as exc: print(f"[WARN] global_ba failed: {exc}") return dict(base) # ================================================================== # Method: sequential block-wise FK fit # ================================================================== def _multistart_values(vtype: str) -> List[float]: # revolute: scan the circle to escape local minima at large angles if vtype == "revolute": return [0.0, 60.0, 120.0, 180.0, 240.0, 300.0] return [0.0] def estimate_sequential_fk(fk: RobotFK, obs: Dict[int, Dict[str, Any]], chain: Dict[str, Any], cfg: Dict[str, Any]) -> Dict[str, float]: """Estimate block by block along the chain, freezing already-solved variables.""" state = {k: 0.0 for k in STATE_KEYS} var_type = chain["var_type"] for block in chain["blocks"]: bvars = block["vars"] bmarkers = [m for m in block["markers"] if m in obs] if not bvars: continue if not bmarkers: # unobservable block: leave at 0, flag later continue if not HAVE_SCIPY: continue base = dict(state) def fun(vec, _bvars=bvars, _bm=bmarkers, _base=base): st = _state_from_vec(_bvars, vec, _base) return residual_vector(st, fk, obs, _bm, cfg) # multi-start over the first revolute variable in the block starts = [[0.0] * len(bvars)] lead_type = var_type.get(bvars[0], "linear") if lead_type == "revolute": starts = [] for a0 in _multistart_values("revolute"): s = [0.0] * len(bvars) s[0] = a0 starts.append(s) best, best_cost = None, float("inf") for s0 in starts: try: sol = least_squares(fun, np.array(s0, dtype=float), loss=cfg.get("robust_loss", "huber"), f_scale=float(cfg.get("huber_delta_mm", 8.0)), max_nfev=200 * max(1, len(bvars))) if sol.cost < best_cost: best_cost, best = sol.cost, sol.x except Exception: continue if best is not None: for name, val in zip(bvars, best): state[name] = float(val) # wrap revolute angles to (-180, 180] for v, vt in var_type.items(): if vt == "revolute": state[v] = (state[v] + 180.0) % 360.0 - 180.0 return state # ================================================================== # Method: sequential analytic vector (per revolute joint) # ================================================================== def estimate_sequential_vector(fk: RobotFK, obs: Dict[int, Dict[str, Any]], chain: Dict[str, Any], cfg: Dict[str, Any]) -> Dict[str, float]: """ Analytic angle from marker geometry where possible. For revolute joints with >=2 markers on the link, use the perpendicular marker-pair vector. Falls back to the FK block solver for linear / zero-marker / single-marker cases, so it always returns a full state (still cheaper than full sequential_fk because well-populated joints are solved in closed form). """ state = {k: 0.0 for k in STATE_KEYS} var_type = chain["var_type"] link_markers = chain["link_markers"] var_links = chain["var_links"] for block in chain["blocks"]: bvars = block["vars"] if len(bvars) == 1 and var_type.get(bvars[0]) == "revolute": v = bvars[0] ln = var_links[v][0] mids = [m for m in link_markers.get(ln, []) if m in obs] if len(mids) >= 2: # model vectors must be expressed in the WORLD frame at angle=0 # (the link frame is already rotated by the parents y,z,...), so # use FK marker world positions with this joint set to 0. state_v0 = dict(state) state_v0[v] = 0.0 model_v0 = model_markers(fk, state_v0) axis_world = fk.joint_axis_world(ln, state_v0) ang = _angle_from_pairs_world(mids, model_v0, obs, axis_world) if ang is not None: state[v] = ang continue # fallback: block FK fit for this single block _fit_single_block(fk, obs, block, var_type, cfg, state) for v, vt in var_type.items(): if vt == "revolute": state[v] = (state[v] + 180.0) % 360.0 - 180.0 return state def _angle_from_pairs_world(mids: List[int], model_v0: Dict[int, Dict[str, np.ndarray]], obs: Dict[int, Dict[str, Any]], axis_world: np.ndarray) -> Optional[float]: from itertools import combinations a = np.asarray(axis_world, float) a = a / (np.linalg.norm(a) + 1e-12) angs, ws = [], [] for i, j in combinations(mids, 2): if i not in model_v0 or j not in model_v0: continue vm = np.asarray(model_v0[j]["world_mm"], float) - np.asarray(model_v0[i]["world_mm"], float) # world @ angle 0 vo = obs[j]["pos_mm"] - obs[i]["pos_mm"] # observed vector (world, mm) vm_p = vm - np.dot(vm, a) * a vo_p = vo - np.dot(vo, a) * a if np.linalg.norm(vm_p) < 5 or np.linalg.norm(vo_p) < 5: continue ang = math.atan2(float(np.dot(a, np.cross(vm_p, vo_p))), float(np.dot(vm_p, vo_p))) angs.append(ang) ws.append(np.linalg.norm(vm_p) * np.linalg.norm(vo_p)) if not angs: return None s = sum(w * math.sin(x) for w, x in zip(ws, angs)) c = sum(w * math.cos(x) for w, x in zip(ws, angs)) return math.degrees(math.atan2(s, c)) def _fit_single_block(fk, obs, block, var_type, cfg, state): if not HAVE_SCIPY: return bvars = block["vars"] bmarkers = [m for m in block["markers"] if m in obs] if not bvars or not bmarkers: return base = dict(state) def fun(vec): return residual_vector(_state_from_vec(bvars, vec, base), fk, obs, bmarkers, cfg) starts = [[0.0] * len(bvars)] if var_type.get(bvars[0]) == "revolute": starts = [[a0] + [0.0] * (len(bvars) - 1) for a0 in _multistart_values("revolute")] best, best_cost = None, float("inf") for s0 in starts: try: sol = least_squares(fun, np.array(s0, float), loss=cfg.get("robust_loss", "huber"), f_scale=float(cfg.get("huber_delta_mm", 8.0)), max_nfev=200 * max(1, len(bvars))) if sol.cost < best_cost: best_cost, best = sol.cost, sol.x except Exception: continue if best is not None: for name, val in zip(bvars, best): state[name] = float(val) # ================================================================== # Dispatch # ================================================================== def observability(chain: Dict[str, Any], obs: Dict[int, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: """ Per-variable confidence from how well its estimation block is determined. A block groups coupled variables (e.g. b,c,e on the fingers); confidence is driven by markers-per-variable in that block: high : >= 2 markers per variable (well over-determined) medium : >= 1 marker per variable low : fewer markers than variables (under-determined — distrust!) none : no markers at all (variable left at 0) """ info: Dict[str, Dict[str, Any]] = {} for block in chain["blocks"]: seen = [m for m in block["markers"] if m in obs] nvars = max(1, len(block["vars"])) ratio = len(seen) / nvars if len(seen) == 0: conf = "none" elif ratio >= 2.0: conf = "high" elif ratio >= 1.0: conf = "medium" else: conf = "low" for v in block["vars"]: info[v] = {"observable": len(seen) > 0, "n_markers": len(seen), "block_vars": len(block["vars"]), "confidence": conf, "block_anchor": block["anchor"]} return info def estimate_pose(fk: RobotFK, obs: Dict[int, Dict[str, Any]], cfg: Dict[str, Any]) -> Dict[str, Any]: chain = analyze_chain(fk) var_names = chain["ordered_vars"] method = str(cfg.get("method", "hybrid")).lower() obsv = observability(chain, obs) if method == "sequential_vector": state = estimate_sequential_vector(fk, obs, chain, cfg) elif method == "sequential_fk": state = estimate_sequential_fk(fk, obs, chain, cfg) elif method == "global_ba": init = estimate_sequential_fk(fk, obs, chain, cfg) # cheap robust init state = estimate_global_ba(fk, obs, var_names, init, cfg) else: # hybrid (default) init = estimate_sequential_fk(fk, obs, chain, cfg) state = estimate_global_ba(fk, obs, var_names, init, cfg) # final residual stats over all observed markers final_res = residual_vector(state, fk, obs, list(obs.keys()), cfg) rms = float(np.sqrt(np.mean(final_res ** 2))) if final_res.size else 0.0 return {"state": state, "method": method, "observability": obsv, "residual_rms": rms, "num_markers": len(obs)} # ================================================================== # CLI # ================================================================== def main() -> None: ap = argparse.ArgumentParser(description="Estimate robot joint angles from marker poses") ap.add_argument("markers", help="aruco_marker_poses.json (corner_pose) or aruco_positions_*.json (center)") ap.add_argument("-robot", "--robot", required=True) ap.add_argument("-out", "--out", default=None) ap.add_argument("--method", default=None, help="override robot.json method") args = ap.parse_args() robot_data = json.load(open(args.robot, "r", encoding="utf-8")) cfg = load_pose_cfg(robot_data) if args.method: cfg["method"] = args.method fk = RobotFK(robot_data) obs = load_observations(args.markers, cfg.get("use_normals", True), int(cfg.get("min_cameras_per_marker", 2))) print(f"[INFO] method={cfg['method']} | observed markers={len(obs)} | use_normals={cfg.get('use_normals')}") result = estimate_pose(fk, obs, cfg) st = result["state"] print("\nEstimated joint values:") for v in ["x", "y", "z", "a", "b", "c", "e"]: ob = result["observability"].get(v, {}) unit = "mm" if v in ("x", "e") else "deg" conf = ob.get("confidence", "?") tag = "" if ob.get("observable", False) else " [UNOBSERVABLE -> 0]" print(f" {v}: {st.get(v, 0.0):8.2f} {unit} (markers={ob.get('n_markers','?')}, conf={conf}){tag}") print(f"\n[INFO] residual RMS over {result['num_markers']} markers: {result['residual_rms']:.3f}") out = { "schema_version": "1.0", "created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "method": result["method"], "movements": {v: {"value": st.get(v, 0.0), "unit": "mm" if v in ("x", "e") else "deg", "observable": result["observability"].get(v, {}).get("observable", False), "confidence": result["observability"].get(v, {}).get("confidence", "none"), "n_markers": result["observability"].get(v, {}).get("n_markers", 0)} for v in ["x", "y", "z", "a", "b", "c", "e"]}, "residual_rms": result["residual_rms"], "num_markers": result["num_markers"], } out_path = args.out or os.path.join(os.path.dirname(args.markers), "robot_state.json") json.dump(out, open(out_path, "w", encoding="utf-8"), indent=2) print(f"[INFO] wrote {out_path}") if __name__ == "__main__": main()