"""Stateful firmware-update orchestration for the production web UI. Wraps the *validated* dpw_fw_tool engine (fw_client / staging / config_safety / protocol) and exposes a single rich status dict that drives all of the UI's graphical state (stepper, per-component bars, preflight, reboot watchdog). Pure stdlib. Thread-safe. No change to the tested engine modules — this is a fresh orchestration layer so the standalone "v1" webui keeps passing its tests. Merge target: NEW_Web_Configurator/src/firmware/fw_controller.py """ import json import os import shlex import shutil import socket import subprocess import sys import threading import time import zipfile from firmware import config_safety, fw_client, protocol, staging # firmware component identification by file extension _EXT_TO_ROLE = { ".rom": ("bootloader", "BTL"), ".img": ("kernel", "KRN"), ".ext4": ("rootfs", "RTF"), ".dtb": ("dtb", "DTB"), } _ALL_SIGS = ("BTL", "KRN", "RTF", "DTB") _ORDER = {s: i for i, s in enumerate(_ALL_SIGS)} _MB = 1024 * 1024 _SIDECAR = ".fw_build_dates.json" # name -> {"sha256":..., "build_date":...} (survives restart/recovery) # zip-bomb / disk-fill guards (largest real component, rootfs, is ~0.9 GB) _MAX_FILE = 1200 * _MB # per-component uncompressed cap _MAX_UNCOMPRESSED = 1536 * _MB # total recognized-component uncompressed cap def _zip_build_date(info): """Image build date from a ZIP entry's mtime. None if unset (epoch 1980).""" dt = getattr(info, "date_time", None) if not dt or dt[0] < 1981: return None return "%04d-%02d-%02d" % (dt[0], dt[1], dt[2]) # Lifecycle phases shown in the UI stepper (order matters). _PHASES = [ ("upload", "Upload"), ("verify", "Verify"), ("preflight", "Pre-flight"), ("backup", "Backup"), ("flash", "Flash"), ("commit", "Commit"), ("reboot", "Reboot"), ("config", "Config check"), ] def _now(): """Monotonic seconds — safe for elapsed math (no wall-clock dependency).""" return time.monotonic() class FirmwareController: def __init__(self, fw_host="127.0.0.1", fw_port=8990, staging_dir="/opt/fw_staging", backups_dir="/opt/config_backups", db_path="/home/root/db/dynamic_data.db", default_file="/var/www/html/config_device.json", restart_cmd="systemctl restart dpworldapp.service", run_update=None): # injectable flash runner (defaults to the real wire client) — lets the # flash state machine be unit-tested without a device. self._run_update = run_update or fw_client.run_update # normalise restart_cmd to an argv list so it is always run without a # shell (no injection surface), accepting either a string or a list. restart_argv = (list(restart_cmd) if isinstance(restart_cmd, (list, tuple)) else shlex.split(restart_cmd)) if not restart_argv or not all(isinstance(a, str) and a for a in restart_argv): raise ValueError("restart_cmd must yield a non-empty list of non-empty strings") self.cfg = dict(fw_host=fw_host, fw_port=fw_port, staging_dir=staging_dir, backups_dir=backups_dir, db_path=db_path, default_file=default_file, restart_cmd=restart_cmd, restart_argv=restart_argv) self._lock = threading.RLock() self._thread = None self._components = [] # list of component dicts (with sent/pct/state) self._phase_state = {} # key -> 'pending'|'active'|'done'|'error' self._phase_detail = {} # key -> str self._state = "idle" self._message = "" self._error = None self._backup = None self._slot = {"before": self._detect_slot(), "after": None} self._t0 = None # flash start (monotonic) self._reset_phases() # v1.5.2 B5 (M10/M15): defer _recover_staging to a background daemon thread so # sha256 hash of large staged files does not block server startup on TCC8030. threading.Thread(target=self._recover_staging, daemon=True).start() # ---- internal helpers ---------------------------------------------- def _recover_staging(self): """If staging_dir already holds recognized components (e.g. the service was restarted after an upload), rebuild the staged component list so the operator does not have to upload again. One-time sha256 cost on startup.""" sdir = self.cfg["staging_dir"] if not os.path.isdir(sdir): return try: with open(os.path.join(sdir, _SIDECAR)) as f: build_dates = json.load(f) except (OSError, ValueError): build_dates = {} found = [] for name in sorted(os.listdir(sdir)): path = os.path.join(sdir, name) if not os.path.isfile(path): continue ext = os.path.splitext(name)[1].lower() if ext not in _EXT_TO_ROLE: continue role, sig = _EXT_TO_ROLE[ext] try: actual_sha = staging.sha256_of(path) except OSError: continue # Fix 3: reject components whose on-disk sha256 does not match the # recorded metadata — this catches partial/corrupt files left behind # by a mid-extraction failure (disk-full, power-loss, etc.). A # component with no metadata entry at all is also rejected: we have # no baseline to verify against so we cannot trust it. meta = build_dates.get(name) # build_dates loaded from sidecar above if isinstance(meta, dict): # New incremental sidecar format: {"sha256": "...", "build_date": "..."} recorded_sha = meta.get("sha256") build_date = meta.get("build_date") else: # Legacy sidecar format stored build_date string directly (or None). # No sha256 was recorded — cannot verify; reject to be safe. recorded_sha = None build_date = meta # may be a date string or None if recorded_sha is None or actual_sha != recorded_sha: # Partial / unverifiable file — do not accept it as staged. # I-1: log so a rejection (legacy pre-v1.11.5 sidecar OR partial/corrupt # file) is NOT silent — otherwise recovery yields an empty set with no # explanation and the operator can't tell why a re-upload is needed. reason = ("no recorded sha256 (legacy/partial sidecar)" if recorded_sha is None else "sha256 mismatch (partial/corrupt)") sys.stderr.write( "[fw-recover] rejecting staged component %r: %s — re-upload required\n" % (name, reason)) # Remove it so it doesn't linger on disk. try: os.remove(path) except OSError: pass continue try: found.append({"role": role, "signature": sig, "name": name, "path": path, "size": os.path.getsize(path), "sha256": actual_sha, "build_date": build_date, "sent": 0, "pct": 0, "state": "pending"}) except OSError: continue if not found: return found.sort(key=lambda c: _ORDER.get(c["signature"], 9)) with self._lock: # v1.5.4.5 FW-1/CONC-2 fix: 신규 upload/flash 가 이미 진행 중이면 recovery # 결과는 stale — silent skip. 이전엔 background daemon thread 의 _recover_staging # 가 disk scan 동안 사용자가 신규 upload 를 시작해도 _components/_state 를 # stale recovery 값으로 덮어써 의도하지 않은 firmware 가 flash 될 path 존재. if self._state != "idle": return self._components = found self._state = "staged" self._set_phase("upload", "done", "recovered %d staged components" % len(found)) self._set_phase("verify", "done", "sha256 verified") self._message = "recovered %d staged components from disk" % len(found) def _reset_phases(self): self._phase_state = {k: "pending" for k, _ in _PHASES} self._phase_detail = {k: "" for k, _ in _PHASES} def _set_phase(self, key, state, detail=None): with self._lock: self._phase_state[key] = state if detail is not None: self._phase_detail[key] = detail @staticmethod def _tier_for(state): return {"done": "ok", "active": "warn", "error": "error"}.get(state, "na") def _detect_slot(self): """Best-effort active A/B slot from the rootfs device (p5=A, p6=B).""" try: with open("/proc/cmdline") as f: cmd = f.read() for tok in cmd.split(): if tok.startswith("root="): dev = tok.split("=", 1)[1] if dev.endswith("p5") or dev.endswith("5"): return "A" if dev.endswith("p6") or dev.endswith("6"): return "B" return dev.rsplit("/", 1)[-1] except OSError: pass return "?" # ---- status snapshot ------------------------------------------------ def status(self): with self._lock: sent = sum(c["sent"] for c in self._components) total = sum(c["size"] for c in self._components) pct = (sent * 100 // total) if total else (100 if self._state == "done" else 0) elapsed = (_now() - self._t0) if self._t0 else 0 active_sig = next((c["signature"] for c in self._components if c["state"] == "active"), None) phases = [{ "key": k, "label": label, "state": self._phase_state[k], "tier": self._tier_for(self._phase_state[k]), "detail": self._phase_detail[k], } for k, label in _PHASES] components = [{ "role": c["role"], "signature": c["signature"], "name": c["name"], "size": c["size"], "sha256": c["sha256"], "build_date": c.get("build_date"), "sent": c["sent"], "pct": c["pct"], "state": c["state"], } for c in self._components] return { "state": self._state, "phases": phases, "components": components, "overall": {"sent": sent, "total": total, "pct": pct, "elapsed_s": round(elapsed, 1)}, "active_signature": active_sig, "backup": self._backup, "slot": dict(self._slot), "message": self._message, "error": self._error, } # ---- staging -------------------------------------------------------- def stage_zip(self, zip_path): """Extract a firmware ZIP, auto-identify components by extension, stage them. Returns the ordered component list (with sha256). Resets state. v1.5.2 B1 (H4 TOCTOU fix): claim 'staging' state under lock before the long extraction begins so concurrent upload calls are blocked immediately. """ with self._lock: # #33 fix: also block during 'rebooting' and 'done' — an upload while # the device is rebooting (or just completed) can race the in-flight # flash/commit lifecycle and clobber staged state. if self._state in ("flashing", "staging", "rebooting", "done"): raise RuntimeError("a flash or stage is in progress — cannot stage new firmware") self._state = "staging" # claim under lock; concurrent callers blocked at this check try: sdir = self.cfg["staging_dir"] staging.clear(sdir) os.makedirs(sdir, exist_ok=True) found = [] build_dates = {} total_uncompressed = 0 seen = set() with zipfile.ZipFile(zip_path) as zf: for info in zf.infolist(): if info.is_dir(): continue name = os.path.basename(info.filename) ext = os.path.splitext(name)[1].lower() if ext not in _EXT_TO_ROLE: continue role, sig = _EXT_TO_ROLE[ext] # I-3 collision guard: two recognized entries sharing a basename # would cause the second to silently overwrite the first on disk # while both are appended to `found` with the first entry's now-stale # sha256 → wrong-image flash / brick risk. Reject the archive before # any colliding write occurs. if name in seen: raise ValueError( "duplicate component file name in archive: %r" % name) seen.add(name) # zip-bomb guard: reject by declared size, cap actual bytes written, # and confirm free space before extracting. if info.file_size > _MAX_FILE: raise ValueError("component '%s' too large (%d bytes)" % (name, info.file_size)) total_uncompressed += info.file_size if total_uncompressed > _MAX_UNCOMPRESSED: raise ValueError("archive uncompressed size exceeds %d bytes" % _MAX_UNCOMPRESSED) staging.check_free_space(sdir, info.file_size) dest = os.path.join(sdir, name) written = 0 try: with zf.open(info) as src, open(dest, "wb") as out: while True: chunk = src.read(_MB) if not chunk: break written += len(chunk) if written > _MAX_FILE: raise ValueError("component '%s' exceeds size cap during extraction" % name) out.write(chunk) except Exception: # Fix 1: delete the partial dest file before propagating so # a disk-full / write-error mid-extraction never leaves a # truncated component that _recover_staging could pick up and # flash as if it were valid. try: os.remove(dest) except OSError: pass raise bd = _zip_build_date(info) if bd: build_dates[name] = bd sha = staging.sha256_of(dest) found.append({"role": role, "signature": sig, "name": name, "path": dest, "size": os.path.getsize(dest), "sha256": sha, "build_date": bd, "sent": 0, "pct": 0, "state": "pending"}) # Fix 2: persist sidecar incrementally so each successfully # extracted component has its metadata (sha256 + build_date) # on disk immediately. A mid-loop failure therefore never # leaves already-extracted files with no sidecar entry; the # subsequent _recover_staging sha256 check will match them. _sidecar_entry = {"sha256": sha} if bd: _sidecar_entry["build_date"] = bd try: _sidecar_path = os.path.join(sdir, _SIDECAR) try: with open(_sidecar_path) as _sf: _cur = json.load(_sf) except (OSError, ValueError): _cur = {} _cur[name] = _sidecar_entry with open(_sidecar_path, "w") as _sf: json.dump(_cur, _sf) except OSError as _e: # #27 fix: don't swallow silently — a missing sidecar entry # makes _recover_staging reject this component on restart. sys.stderr.write( "[fw-stage] WARNING: sidecar write failed for %r: %s — " "this component may be rejected on restart\n" % (name, _e)) if not found: raise ValueError("no recognized firmware components in archive " "(.rom/.img/.ext4/.dtb)") found.sort(key=lambda c: _ORDER.get(c["signature"], 9)) missing = [s for s in _ALL_SIGS if s not in [c["signature"] for c in found]] with self._lock: self._components = found self._state = "staged" self._error = None self._backup = None self._t0 = None self._reset_phases() self._set_phase("upload", "done", "%d MB received" % ( sum(c["size"] for c in found) // _MB)) self._set_phase("verify", "done", "sha256 computed for %d components" % len(found)) self._message = "staged %d components%s" % ( len(found), "" if not missing else " (missing: %s)" % ",".join(missing)) return found except Exception: with self._lock: self._state = "idle" raise # ---- preflight ------------------------------------------------------ def preflight(self): """Return a gated checklist. `ok` is True only if all CRITICAL checks pass.""" checks = [] def add(key, label, ok, detail, critical=True): checks.append({"key": key, "label": label, "pass": bool(ok), "tier": "ok" if ok else ("error" if critical else "warn"), "critical": critical, "detail": detail}) with self._lock: comps = list(self._components) n = len(comps) add("staging", "Components staged", n > 0, ("%d components: %s" % (n, ", ".join(c["signature"] for c in comps))) if n else "no firmware uploaded yet") # free-space headroom. The components are ALREADY staged on this fs, so we # must NOT re-require their size (that double-counts — the bytes are # already on disk). dpworldapp buffers the transfer in /tmp (tmpfs), not # here, so only modest /opt headroom is needed for the pre-flash config # backup + incidental writes. headroom = 128 * _MB try: free = shutil.disk_usage(self.cfg["staging_dir"]).free staged = sum(c["size"] for c in comps) ok_space = free >= headroom detail = ("%d MB free (%d MB already staged)" % (free // _MB, staged // _MB) if comps else "%d MB free" % (free // _MB)) add("space", "Free space (headroom)", ok_space, detail) except OSError as e: add("space", "Free space (headroom)", False, str(e)) # dpworldapp FW port reachable ok_port, detail = self._probe_port(self.cfg["fw_host"], self.cfg["fw_port"]) add("fw_port", "dpworldapp FW port %d" % self.cfg["fw_port"], ok_port, detail) # config DB present (non-critical: flash works, restore-check needs it) db_ok = os.path.isfile(self.cfg["db_path"]) add("db", "Config DB present", db_ok, self.cfg["db_path"] if db_ok else "%s not found" % self.cfg["db_path"], critical=False) # active slot (informational) slot = self._detect_slot() add("slot", "Active slot", True, "system_%s" % slot.lower() if slot in ("A", "B") else slot, critical=False) ok = all(c["pass"] for c in checks if c["critical"]) with self._lock: self._set_phase("preflight", "done" if ok else "error", "all checks pass" if ok else "blocked") return {"ok": ok, "checks": checks} @staticmethod def _probe_port(host, port, timeout=2.0): try: with socket.create_connection((host, port), timeout=timeout): return True, "reachable" except OSError as e: return False, "unreachable (%s)" % e.__class__.__name__ # ---- flash ---------------------------------------------------------- def _flash_worker(self, components): try: with self._lock: self._t0 = _now() self._state = "flashing" self._message = "backing up config before flash" self._set_phase("backup", "active", "backing up config") backup = config_safety.backup_config(self.cfg["db_path"], [], self.cfg["backups_dir"]) with self._lock: self._backup = {"path": backup, "when": _readable_dirname(backup)} self._set_phase("backup", "done", "saved") steps = [fw_client.FirmwareStep(c["signature"], c["path"], protocol.TIMEOUTS_MS[c["signature"]]) for c in components] steps.append(fw_client.FirmwareStep( protocol.COMMIT_SIGNATURE, None, protocol.TIMEOUTS_MS["CPU"], is_message_only=True)) with self._lock: self._message = "flashing" self._set_phase("flash", "active", "transferring components") def progress(sig, sent, total): # send_message (CPU commit) never calls this — commit start is # detected when the LAST file component reaches 100%. with self._lock: comps = self._components for i, c in enumerate(comps): if c["signature"] != sig: continue c["sent"] = sent c["pct"] = (sent * 100 // total) if total else 100 c["state"] = "done" if c["pct"] >= 100 else "active" # defensively mark earlier components done for prev in comps[:i]: if prev["state"] != "done": prev["state"], prev["sent"], prev["pct"] = "done", prev["size"], 100 # last component fully sent → commit/write begins if i == len(comps) - 1 and c["pct"] >= 100: self._set_phase("flash", "done", "all components sent") self._set_phase("commit", "active", "device writing slot & committing") break outcome = self._run_update(self.cfg["fw_host"], self.cfg["fw_port"], steps, progress) with self._lock: for c in self._components: if outcome.ok: c["state"], c["sent"], c["pct"] = "done", c["size"], 100 if outcome.ok: self._set_phase("commit", "done", "slot committed") self._set_phase("reboot", "active", "device reboots in ~10s") # firmware is now committed to the slot → free the staging space freed = self._clear_staging() with self._lock: self._state = "rebooting" self._set_phase("flash", "done", "components sent · staging freed" if freed else "components sent (staging cleanup failed — free /opt manually)") self._slot["after"] = "B" if self._slot["before"] == "A" else ( "A" if self._slot["before"] == "B" else "?") self._message = ("flash OK — device reboots now. Reconnect after " "~1 min, then run Config check.") else: self._set_phase("flash", "error", "failed at %s" % outcome.failed_step) with self._lock: self._state = "failed" self._error = "flash FAILED at step %s" % outcome.failed_step self._message = self._error except Exception as e: # noqa: BLE001 — surface any failure to the UI with self._lock: self._state = "failed" self._error = "%s: %s" % (e.__class__.__name__, e) self._message = self._error # mark whichever phase was active as errored for k in ("backup", "flash", "commit"): if self._phase_state.get(k) == "active": self._phase_state[k] = "error" def start_flash(self): with self._lock: if self._state in ('flashing', 'staging', 'rebooting', 'done'): raise RuntimeError( "Cannot flash in state '%s'" % self._state) comps = list(self._components) if not comps: raise RuntimeError("no staged components — upload a firmware ZIP first") # #12 fix: reset phases so a retry after a prior failed attempt starts # clean — residual 'error' phases from the previous run must not linger. self._reset_phases() self._state = "flashing" self._error = None self._thread = threading.Thread(target=self._flash_worker, args=(comps,), daemon=True) self._thread.start() def _clear_staging(self): """Remove staged firmware from disk to reclaim space after a successful commit (the firmware is already written to the slot). Returns True on success. Config backups (a separate dir) are intentionally kept.""" try: staging.clear(self.cfg["staging_dir"]) return True except OSError: return False # ---- post-reboot config restore ------------------------------------ def restore_check(self, restart=True): self._set_phase("config", "active", "checking config") try: r = config_safety.detect_and_restore(self.cfg["db_path"], self.cfg["default_file"], self.cfg["backups_dir"]) except Exception as e: # noqa: BLE001 — missing/locked/empty DB must not 500 reason = "config DB not readable: %s" % e self._set_phase("config", "error", "DB unreadable") return {"restored": False, "reason": reason, "dpworldapp_restarted": False, "error": reason} result = {"restored": r.restored, "reason": r.reason, "dpworldapp_restarted": False} if r.restored and restart: try: # v1.5.4.5 STAB-2 fix: check=True so non-zero returncode raises # CalledProcessError → except branch sets restart_error. 이전 check=False # 패턴은 systemctl restart dpworldapp 실패 (returncode != 0) 시에도 except # 진입 안 함 → result["dpworldapp_restarted"]=True 로 silent lie 가능. # 이제 어떤 실패 (CalledProcessError / TimeoutExpired / OSError) 든 except # 통합 처리 후 restart_error 명시. subprocess.run(self.cfg["restart_argv"], timeout=30, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) result["dpworldapp_restarted"] = True except Exception as e: # noqa: BLE001 result["restart_error"] = str(e) # #6 fix: if the restart command FAILED, the phase detail + _message must # report the failure — never claim 'dpworldapp restarted'. The flash # itself succeeded so state may still go 'done'; only the restart sub-step # is reported failed (honesty: no green for an unverified restart). restart_failed = bool(result.get("restart_error")) with self._lock: if r.restored and restart and restart_failed: config_detail = ("config restored but dpworldapp restart FAILED: %s" % result["restart_error"]) else: config_detail = ("restored from backup" if r.restored else "config preserved") self._set_phase("config", "done", config_detail) if self._state == "rebooting": self._state = "done" if r.restored and restart and restart_failed: self._message = ("config restored but dpworldapp restart FAILED: %s" % result["restart_error"]) else: self._message = ("config restored & dpworldapp restarted" if r.restored else "update complete — config preserved") return result def _readable_dirname(path): """Backup paths look like /opt/config_backups/20260601-190437 — return the stamp.""" base = os.path.basename(path.rstrip("/")) if path else "" return base or path