You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
603 lines
30 KiB
603 lines
30 KiB
1 month ago
|
"""Stateful firmware-update orchestration for the production web UI.
|
||
|
|
|
||
|
|
Wraps the *validated* dpw_fw_tool engine (fw_client / staging / config_safety /
|
||
|
|
protocol) and exposes a single rich status dict that drives all of the UI's
|
||
|
|
graphical state (stepper, per-component bars, preflight, reboot watchdog).
|
||
|
|
|
||
|
|
Pure stdlib. Thread-safe. No change to the tested engine modules — this is a
|
||
|
|
fresh orchestration layer so the standalone "v1" webui keeps passing its tests.
|
||
|
|
|
||
|
|
Merge target: NEW_Web_Configurator/src/firmware/fw_controller.py
|
||
|
|
"""
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import shlex
|
||
|
|
import shutil
|
||
|
|
import socket
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
import zipfile
|
||
|
|
|
||
|
|
from firmware import config_safety, fw_client, protocol, staging
|
||
|
|
|
||
|
|
# firmware component identification by file extension
|
||
|
|
_EXT_TO_ROLE = {
|
||
|
|
".rom": ("bootloader", "BTL"),
|
||
|
|
".img": ("kernel", "KRN"),
|
||
|
|
".ext4": ("rootfs", "RTF"),
|
||
|
|
".dtb": ("dtb", "DTB"),
|
||
|
|
}
|
||
|
|
_ALL_SIGS = ("BTL", "KRN", "RTF", "DTB")
|
||
|
|
_ORDER = {s: i for i, s in enumerate(_ALL_SIGS)}
|
||
|
|
_MB = 1024 * 1024
|
||
|
|
_SIDECAR = ".fw_build_dates.json" # name -> {"sha256":..., "build_date":...} (survives restart/recovery)
|
||
|
|
# zip-bomb / disk-fill guards (largest real component, rootfs, is ~0.9 GB)
|
||
|
|
_MAX_FILE = 1200 * _MB # per-component uncompressed cap
|
||
|
|
_MAX_UNCOMPRESSED = 1536 * _MB # total recognized-component uncompressed cap
|
||
|
|
|
||
|
|
|
||
|
|
def _zip_build_date(info):
|
||
|
|
"""Image build date from a ZIP entry's mtime. None if unset (epoch 1980)."""
|
||
|
|
dt = getattr(info, "date_time", None)
|
||
|
|
if not dt or dt[0] < 1981:
|
||
|
|
return None
|
||
|
|
return "%04d-%02d-%02d" % (dt[0], dt[1], dt[2])
|
||
|
|
|
||
|
|
# Lifecycle phases shown in the UI stepper (order matters).
|
||
|
|
_PHASES = [
|
||
|
|
("upload", "Upload"),
|
||
|
|
("verify", "Verify"),
|
||
|
|
("preflight", "Pre-flight"),
|
||
|
|
("backup", "Backup"),
|
||
|
|
("flash", "Flash"),
|
||
|
|
("commit", "Commit"),
|
||
|
|
("reboot", "Reboot"),
|
||
|
|
("config", "Config check"),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def _now():
|
||
|
|
"""Monotonic seconds — safe for elapsed math (no wall-clock dependency)."""
|
||
|
|
return time.monotonic()
|
||
|
|
|
||
|
|
|
||
|
|
class FirmwareController:
|
||
|
|
def __init__(self, fw_host="127.0.0.1", fw_port=8990,
|
||
|
|
staging_dir="/opt/fw_staging", backups_dir="/opt/config_backups",
|
||
|
|
db_path="/home/root/db/dynamic_data.db",
|
||
|
|
default_file="/var/www/html/config_device.json",
|
||
|
|
restart_cmd="systemctl restart dpworldapp.service",
|
||
|
|
run_update=None):
|
||
|
|
# injectable flash runner (defaults to the real wire client) — lets the
|
||
|
|
# flash state machine be unit-tested without a device.
|
||
|
|
self._run_update = run_update or fw_client.run_update
|
||
|
|
# normalise restart_cmd to an argv list so it is always run without a
|
||
|
|
# shell (no injection surface), accepting either a string or a list.
|
||
|
|
restart_argv = (list(restart_cmd) if isinstance(restart_cmd, (list, tuple))
|
||
|
|
else shlex.split(restart_cmd))
|
||
|
|
if not restart_argv or not all(isinstance(a, str) and a for a in restart_argv):
|
||
|
|
raise ValueError("restart_cmd must yield a non-empty list of non-empty strings")
|
||
|
|
self.cfg = dict(fw_host=fw_host, fw_port=fw_port, staging_dir=staging_dir,
|
||
|
|
backups_dir=backups_dir, db_path=db_path,
|
||
|
|
default_file=default_file, restart_cmd=restart_cmd,
|
||
|
|
restart_argv=restart_argv)
|
||
|
|
self._lock = threading.RLock()
|
||
|
|
self._thread = None
|
||
|
|
self._components = [] # list of component dicts (with sent/pct/state)
|
||
|
|
self._phase_state = {} # key -> 'pending'|'active'|'done'|'error'
|
||
|
|
self._phase_detail = {} # key -> str
|
||
|
|
self._state = "idle"
|
||
|
|
self._message = ""
|
||
|
|
self._error = None
|
||
|
|
self._backup = None
|
||
|
|
self._slot = {"before": self._detect_slot(), "after": None}
|
||
|
|
self._t0 = None # flash start (monotonic)
|
||
|
|
self._reset_phases()
|
||
|
|
# v1.5.2 B5 (M10/M15): defer _recover_staging to a background daemon thread so
|
||
|
|
# sha256 hash of large staged files does not block server startup on TCC8030.
|
||
|
|
threading.Thread(target=self._recover_staging, daemon=True).start()
|
||
|
|
|
||
|
|
# ---- internal helpers ----------------------------------------------
|
||
|
|
def _recover_staging(self):
|
||
|
|
"""If staging_dir already holds recognized components (e.g. the service
|
||
|
|
was restarted after an upload), rebuild the staged component list so the
|
||
|
|
operator does not have to upload again. One-time sha256 cost on startup."""
|
||
|
|
sdir = self.cfg["staging_dir"]
|
||
|
|
if not os.path.isdir(sdir):
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
with open(os.path.join(sdir, _SIDECAR)) as f:
|
||
|
|
build_dates = json.load(f)
|
||
|
|
except (OSError, ValueError):
|
||
|
|
build_dates = {}
|
||
|
|
found = []
|
||
|
|
for name in sorted(os.listdir(sdir)):
|
||
|
|
path = os.path.join(sdir, name)
|
||
|
|
if not os.path.isfile(path):
|
||
|
|
continue
|
||
|
|
ext = os.path.splitext(name)[1].lower()
|
||
|
|
if ext not in _EXT_TO_ROLE:
|
||
|
|
continue
|
||
|
|
role, sig = _EXT_TO_ROLE[ext]
|
||
|
|
try:
|
||
|
|
actual_sha = staging.sha256_of(path)
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
# Fix 3: reject components whose on-disk sha256 does not match the
|
||
|
|
# recorded metadata — this catches partial/corrupt files left behind
|
||
|
|
# by a mid-extraction failure (disk-full, power-loss, etc.). A
|
||
|
|
# component with no metadata entry at all is also rejected: we have
|
||
|
|
# no baseline to verify against so we cannot trust it.
|
||
|
|
meta = build_dates.get(name) # build_dates loaded from sidecar above
|
||
|
|
if isinstance(meta, dict):
|
||
|
|
# New incremental sidecar format: {"sha256": "...", "build_date": "..."}
|
||
|
|
recorded_sha = meta.get("sha256")
|
||
|
|
build_date = meta.get("build_date")
|
||
|
|
else:
|
||
|
|
# Legacy sidecar format stored build_date string directly (or None).
|
||
|
|
# No sha256 was recorded — cannot verify; reject to be safe.
|
||
|
|
recorded_sha = None
|
||
|
|
build_date = meta # may be a date string or None
|
||
|
|
if recorded_sha is None or actual_sha != recorded_sha:
|
||
|
|
# Partial / unverifiable file — do not accept it as staged.
|
||
|
|
# I-1: log so a rejection (legacy pre-v1.11.5 sidecar OR partial/corrupt
|
||
|
|
# file) is NOT silent — otherwise recovery yields an empty set with no
|
||
|
|
# explanation and the operator can't tell why a re-upload is needed.
|
||
|
|
reason = ("no recorded sha256 (legacy/partial sidecar)"
|
||
|
|
if recorded_sha is None else "sha256 mismatch (partial/corrupt)")
|
||
|
|
sys.stderr.write(
|
||
|
|
"[fw-recover] rejecting staged component %r: %s — re-upload required\n"
|
||
|
|
% (name, reason))
|
||
|
|
# Remove it so it doesn't linger on disk.
|
||
|
|
try:
|
||
|
|
os.remove(path)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
found.append({"role": role, "signature": sig, "name": name,
|
||
|
|
"path": path, "size": os.path.getsize(path),
|
||
|
|
"sha256": actual_sha,
|
||
|
|
"build_date": build_date,
|
||
|
|
"sent": 0, "pct": 0, "state": "pending"})
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
if not found:
|
||
|
|
return
|
||
|
|
found.sort(key=lambda c: _ORDER.get(c["signature"], 9))
|
||
|
|
with self._lock:
|
||
|
|
# v1.5.4.5 FW-1/CONC-2 fix: 신규 upload/flash 가 이미 진행 중이면 recovery
|
||
|
|
# 결과는 stale — silent skip. 이전엔 background daemon thread 의 _recover_staging
|
||
|
|
# 가 disk scan 동안 사용자가 신규 upload 를 시작해도 _components/_state 를
|
||
|
|
# stale recovery 값으로 덮어써 의도하지 않은 firmware 가 flash 될 path 존재.
|
||
|
|
if self._state != "idle":
|
||
|
|
return
|
||
|
|
self._components = found
|
||
|
|
self._state = "staged"
|
||
|
|
self._set_phase("upload", "done", "recovered %d staged components" % len(found))
|
||
|
|
self._set_phase("verify", "done", "sha256 verified")
|
||
|
|
self._message = "recovered %d staged components from disk" % len(found)
|
||
|
|
|
||
|
|
def _reset_phases(self):
|
||
|
|
self._phase_state = {k: "pending" for k, _ in _PHASES}
|
||
|
|
self._phase_detail = {k: "" for k, _ in _PHASES}
|
||
|
|
|
||
|
|
def _set_phase(self, key, state, detail=None):
|
||
|
|
with self._lock:
|
||
|
|
self._phase_state[key] = state
|
||
|
|
if detail is not None:
|
||
|
|
self._phase_detail[key] = detail
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _tier_for(state):
|
||
|
|
return {"done": "ok", "active": "warn", "error": "error"}.get(state, "na")
|
||
|
|
|
||
|
|
def _detect_slot(self):
|
||
|
|
"""Best-effort active A/B slot from the rootfs device (p5=A, p6=B)."""
|
||
|
|
try:
|
||
|
|
with open("/proc/cmdline") as f:
|
||
|
|
cmd = f.read()
|
||
|
|
for tok in cmd.split():
|
||
|
|
if tok.startswith("root="):
|
||
|
|
dev = tok.split("=", 1)[1]
|
||
|
|
if dev.endswith("p5") or dev.endswith("5"):
|
||
|
|
return "A"
|
||
|
|
if dev.endswith("p6") or dev.endswith("6"):
|
||
|
|
return "B"
|
||
|
|
return dev.rsplit("/", 1)[-1]
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
return "?"
|
||
|
|
|
||
|
|
# ---- status snapshot ------------------------------------------------
|
||
|
|
def status(self):
|
||
|
|
with self._lock:
|
||
|
|
sent = sum(c["sent"] for c in self._components)
|
||
|
|
total = sum(c["size"] for c in self._components)
|
||
|
|
pct = (sent * 100 // total) if total else (100 if self._state == "done" else 0)
|
||
|
|
elapsed = (_now() - self._t0) if self._t0 else 0
|
||
|
|
active_sig = next((c["signature"] for c in self._components
|
||
|
|
if c["state"] == "active"), None)
|
||
|
|
phases = [{
|
||
|
|
"key": k, "label": label,
|
||
|
|
"state": self._phase_state[k],
|
||
|
|
"tier": self._tier_for(self._phase_state[k]),
|
||
|
|
"detail": self._phase_detail[k],
|
||
|
|
} for k, label in _PHASES]
|
||
|
|
components = [{
|
||
|
|
"role": c["role"], "signature": c["signature"], "name": c["name"],
|
||
|
|
"size": c["size"], "sha256": c["sha256"],
|
||
|
|
"build_date": c.get("build_date"),
|
||
|
|
"sent": c["sent"], "pct": c["pct"], "state": c["state"],
|
||
|
|
} for c in self._components]
|
||
|
|
return {
|
||
|
|
"state": self._state,
|
||
|
|
"phases": phases,
|
||
|
|
"components": components,
|
||
|
|
"overall": {"sent": sent, "total": total, "pct": pct,
|
||
|
|
"elapsed_s": round(elapsed, 1)},
|
||
|
|
"active_signature": active_sig,
|
||
|
|
"backup": self._backup,
|
||
|
|
"slot": dict(self._slot),
|
||
|
|
"message": self._message,
|
||
|
|
"error": self._error,
|
||
|
|
}
|
||
|
|
|
||
|
|
# ---- staging --------------------------------------------------------
|
||
|
|
def stage_zip(self, zip_path):
|
||
|
|
"""Extract a firmware ZIP, auto-identify components by extension, stage
|
||
|
|
them. Returns the ordered component list (with sha256). Resets state.
|
||
|
|
|
||
|
|
v1.5.2 B1 (H4 TOCTOU fix): claim 'staging' state under lock before the
|
||
|
|
long extraction begins so concurrent upload calls are blocked immediately.
|
||
|
|
"""
|
||
|
|
with self._lock:
|
||
|
|
# #33 fix: also block during 'rebooting' and 'done' — an upload while
|
||
|
|
# the device is rebooting (or just completed) can race the in-flight
|
||
|
|
# flash/commit lifecycle and clobber staged state.
|
||
|
|
if self._state in ("flashing", "staging", "rebooting", "done"):
|
||
|
|
raise RuntimeError("a flash or stage is in progress — cannot stage new firmware")
|
||
|
|
self._state = "staging" # claim under lock; concurrent callers blocked at this check
|
||
|
|
try:
|
||
|
|
sdir = self.cfg["staging_dir"]
|
||
|
|
staging.clear(sdir)
|
||
|
|
os.makedirs(sdir, exist_ok=True)
|
||
|
|
found = []
|
||
|
|
build_dates = {}
|
||
|
|
total_uncompressed = 0
|
||
|
|
seen = set()
|
||
|
|
with zipfile.ZipFile(zip_path) as zf:
|
||
|
|
for info in zf.infolist():
|
||
|
|
if info.is_dir():
|
||
|
|
continue
|
||
|
|
name = os.path.basename(info.filename)
|
||
|
|
ext = os.path.splitext(name)[1].lower()
|
||
|
|
if ext not in _EXT_TO_ROLE:
|
||
|
|
continue
|
||
|
|
role, sig = _EXT_TO_ROLE[ext]
|
||
|
|
# I-3 collision guard: two recognized entries sharing a basename
|
||
|
|
# would cause the second to silently overwrite the first on disk
|
||
|
|
# while both are appended to `found` with the first entry's now-stale
|
||
|
|
# sha256 → wrong-image flash / brick risk. Reject the archive before
|
||
|
|
# any colliding write occurs.
|
||
|
|
if name in seen:
|
||
|
|
raise ValueError(
|
||
|
|
"duplicate component file name in archive: %r" % name)
|
||
|
|
seen.add(name)
|
||
|
|
# zip-bomb guard: reject by declared size, cap actual bytes written,
|
||
|
|
# and confirm free space before extracting.
|
||
|
|
if info.file_size > _MAX_FILE:
|
||
|
|
raise ValueError("component '%s' too large (%d bytes)" % (name, info.file_size))
|
||
|
|
total_uncompressed += info.file_size
|
||
|
|
if total_uncompressed > _MAX_UNCOMPRESSED:
|
||
|
|
raise ValueError("archive uncompressed size exceeds %d bytes" % _MAX_UNCOMPRESSED)
|
||
|
|
staging.check_free_space(sdir, info.file_size)
|
||
|
|
dest = os.path.join(sdir, name)
|
||
|
|
written = 0
|
||
|
|
try:
|
||
|
|
with zf.open(info) as src, open(dest, "wb") as out:
|
||
|
|
while True:
|
||
|
|
chunk = src.read(_MB)
|
||
|
|
if not chunk:
|
||
|
|
break
|
||
|
|
written += len(chunk)
|
||
|
|
if written > _MAX_FILE:
|
||
|
|
raise ValueError("component '%s' exceeds size cap during extraction" % name)
|
||
|
|
out.write(chunk)
|
||
|
|
except Exception:
|
||
|
|
# Fix 1: delete the partial dest file before propagating so
|
||
|
|
# a disk-full / write-error mid-extraction never leaves a
|
||
|
|
# truncated component that _recover_staging could pick up and
|
||
|
|
# flash as if it were valid.
|
||
|
|
try:
|
||
|
|
os.remove(dest)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
raise
|
||
|
|
bd = _zip_build_date(info)
|
||
|
|
if bd:
|
||
|
|
build_dates[name] = bd
|
||
|
|
sha = staging.sha256_of(dest)
|
||
|
|
found.append({"role": role, "signature": sig, "name": name,
|
||
|
|
"path": dest, "size": os.path.getsize(dest),
|
||
|
|
"sha256": sha,
|
||
|
|
"build_date": bd,
|
||
|
|
"sent": 0, "pct": 0, "state": "pending"})
|
||
|
|
# Fix 2: persist sidecar incrementally so each successfully
|
||
|
|
# extracted component has its metadata (sha256 + build_date)
|
||
|
|
# on disk immediately. A mid-loop failure therefore never
|
||
|
|
# leaves already-extracted files with no sidecar entry; the
|
||
|
|
# subsequent _recover_staging sha256 check will match them.
|
||
|
|
_sidecar_entry = {"sha256": sha}
|
||
|
|
if bd:
|
||
|
|
_sidecar_entry["build_date"] = bd
|
||
|
|
try:
|
||
|
|
_sidecar_path = os.path.join(sdir, _SIDECAR)
|
||
|
|
try:
|
||
|
|
with open(_sidecar_path) as _sf:
|
||
|
|
_cur = json.load(_sf)
|
||
|
|
except (OSError, ValueError):
|
||
|
|
_cur = {}
|
||
|
|
_cur[name] = _sidecar_entry
|
||
|
|
with open(_sidecar_path, "w") as _sf:
|
||
|
|
json.dump(_cur, _sf)
|
||
|
|
except OSError as _e:
|
||
|
|
# #27 fix: don't swallow silently — a missing sidecar entry
|
||
|
|
# makes _recover_staging reject this component on restart.
|
||
|
|
sys.stderr.write(
|
||
|
|
"[fw-stage] WARNING: sidecar write failed for %r: %s — "
|
||
|
|
"this component may be rejected on restart\n" % (name, _e))
|
||
|
|
if not found:
|
||
|
|
raise ValueError("no recognized firmware components in archive "
|
||
|
|
"(.rom/.img/.ext4/.dtb)")
|
||
|
|
found.sort(key=lambda c: _ORDER.get(c["signature"], 9))
|
||
|
|
missing = [s for s in _ALL_SIGS if s not in [c["signature"] for c in found]]
|
||
|
|
with self._lock:
|
||
|
|
self._components = found
|
||
|
|
self._state = "staged"
|
||
|
|
self._error = None
|
||
|
|
self._backup = None
|
||
|
|
self._t0 = None
|
||
|
|
self._reset_phases()
|
||
|
|
self._set_phase("upload", "done", "%d MB received" % (
|
||
|
|
sum(c["size"] for c in found) // _MB))
|
||
|
|
self._set_phase("verify", "done", "sha256 computed for %d components" % len(found))
|
||
|
|
self._message = "staged %d components%s" % (
|
||
|
|
len(found),
|
||
|
|
"" if not missing else " (missing: %s)" % ",".join(missing))
|
||
|
|
return found
|
||
|
|
except Exception:
|
||
|
|
with self._lock:
|
||
|
|
self._state = "idle"
|
||
|
|
raise
|
||
|
|
|
||
|
|
# ---- preflight ------------------------------------------------------
|
||
|
|
def preflight(self):
|
||
|
|
"""Return a gated checklist. `ok` is True only if all CRITICAL checks pass."""
|
||
|
|
checks = []
|
||
|
|
|
||
|
|
def add(key, label, ok, detail, critical=True):
|
||
|
|
checks.append({"key": key, "label": label, "pass": bool(ok),
|
||
|
|
"tier": "ok" if ok else ("error" if critical else "warn"),
|
||
|
|
"critical": critical, "detail": detail})
|
||
|
|
|
||
|
|
with self._lock:
|
||
|
|
comps = list(self._components)
|
||
|
|
n = len(comps)
|
||
|
|
add("staging", "Components staged", n > 0,
|
||
|
|
("%d components: %s" % (n, ", ".join(c["signature"] for c in comps)))
|
||
|
|
if n else "no firmware uploaded yet")
|
||
|
|
|
||
|
|
# free-space headroom. The components are ALREADY staged on this fs, so we
|
||
|
|
# must NOT re-require their size (that double-counts — the bytes are
|
||
|
|
# already on disk). dpworldapp buffers the transfer in /tmp (tmpfs), not
|
||
|
|
# here, so only modest /opt headroom is needed for the pre-flash config
|
||
|
|
# backup + incidental writes.
|
||
|
|
headroom = 128 * _MB
|
||
|
|
try:
|
||
|
|
free = shutil.disk_usage(self.cfg["staging_dir"]).free
|
||
|
|
staged = sum(c["size"] for c in comps)
|
||
|
|
ok_space = free >= headroom
|
||
|
|
detail = ("%d MB free (%d MB already staged)" % (free // _MB, staged // _MB)
|
||
|
|
if comps else "%d MB free" % (free // _MB))
|
||
|
|
add("space", "Free space (headroom)", ok_space, detail)
|
||
|
|
except OSError as e:
|
||
|
|
add("space", "Free space (headroom)", False, str(e))
|
||
|
|
|
||
|
|
# dpworldapp FW port reachable
|
||
|
|
ok_port, detail = self._probe_port(self.cfg["fw_host"], self.cfg["fw_port"])
|
||
|
|
add("fw_port", "dpworldapp FW port %d" % self.cfg["fw_port"], ok_port, detail)
|
||
|
|
|
||
|
|
# config DB present (non-critical: flash works, restore-check needs it)
|
||
|
|
db_ok = os.path.isfile(self.cfg["db_path"])
|
||
|
|
add("db", "Config DB present", db_ok,
|
||
|
|
self.cfg["db_path"] if db_ok else "%s not found" % self.cfg["db_path"],
|
||
|
|
critical=False)
|
||
|
|
|
||
|
|
# active slot (informational)
|
||
|
|
slot = self._detect_slot()
|
||
|
|
add("slot", "Active slot", True, "system_%s" % slot.lower()
|
||
|
|
if slot in ("A", "B") else slot, critical=False)
|
||
|
|
|
||
|
|
ok = all(c["pass"] for c in checks if c["critical"])
|
||
|
|
with self._lock:
|
||
|
|
self._set_phase("preflight", "done" if ok else "error",
|
||
|
|
"all checks pass" if ok else "blocked")
|
||
|
|
return {"ok": ok, "checks": checks}
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _probe_port(host, port, timeout=2.0):
|
||
|
|
try:
|
||
|
|
with socket.create_connection((host, port), timeout=timeout):
|
||
|
|
return True, "reachable"
|
||
|
|
except OSError as e:
|
||
|
|
return False, "unreachable (%s)" % e.__class__.__name__
|
||
|
|
|
||
|
|
# ---- flash ----------------------------------------------------------
|
||
|
|
def _flash_worker(self, components):
|
||
|
|
try:
|
||
|
|
with self._lock:
|
||
|
|
self._t0 = _now()
|
||
|
|
self._state = "flashing"
|
||
|
|
self._message = "backing up config before flash"
|
||
|
|
self._set_phase("backup", "active", "backing up config")
|
||
|
|
backup = config_safety.backup_config(self.cfg["db_path"], [],
|
||
|
|
self.cfg["backups_dir"])
|
||
|
|
with self._lock:
|
||
|
|
self._backup = {"path": backup, "when": _readable_dirname(backup)}
|
||
|
|
self._set_phase("backup", "done", "saved")
|
||
|
|
|
||
|
|
steps = [fw_client.FirmwareStep(c["signature"], c["path"],
|
||
|
|
protocol.TIMEOUTS_MS[c["signature"]])
|
||
|
|
for c in components]
|
||
|
|
steps.append(fw_client.FirmwareStep(
|
||
|
|
protocol.COMMIT_SIGNATURE, None,
|
||
|
|
protocol.TIMEOUTS_MS["CPU"], is_message_only=True))
|
||
|
|
|
||
|
|
with self._lock:
|
||
|
|
self._message = "flashing"
|
||
|
|
self._set_phase("flash", "active", "transferring components")
|
||
|
|
|
||
|
|
def progress(sig, sent, total):
|
||
|
|
# send_message (CPU commit) never calls this — commit start is
|
||
|
|
# detected when the LAST file component reaches 100%.
|
||
|
|
with self._lock:
|
||
|
|
comps = self._components
|
||
|
|
for i, c in enumerate(comps):
|
||
|
|
if c["signature"] != sig:
|
||
|
|
continue
|
||
|
|
c["sent"] = sent
|
||
|
|
c["pct"] = (sent * 100 // total) if total else 100
|
||
|
|
c["state"] = "done" if c["pct"] >= 100 else "active"
|
||
|
|
# defensively mark earlier components done
|
||
|
|
for prev in comps[:i]:
|
||
|
|
if prev["state"] != "done":
|
||
|
|
prev["state"], prev["sent"], prev["pct"] = "done", prev["size"], 100
|
||
|
|
# last component fully sent → commit/write begins
|
||
|
|
if i == len(comps) - 1 and c["pct"] >= 100:
|
||
|
|
self._set_phase("flash", "done", "all components sent")
|
||
|
|
self._set_phase("commit", "active", "device writing slot & committing")
|
||
|
|
break
|
||
|
|
|
||
|
|
outcome = self._run_update(self.cfg["fw_host"], self.cfg["fw_port"],
|
||
|
|
steps, progress)
|
||
|
|
with self._lock:
|
||
|
|
for c in self._components:
|
||
|
|
if outcome.ok:
|
||
|
|
c["state"], c["sent"], c["pct"] = "done", c["size"], 100
|
||
|
|
if outcome.ok:
|
||
|
|
self._set_phase("commit", "done", "slot committed")
|
||
|
|
self._set_phase("reboot", "active", "device reboots in ~10s")
|
||
|
|
# firmware is now committed to the slot → free the staging space
|
||
|
|
freed = self._clear_staging()
|
||
|
|
with self._lock:
|
||
|
|
self._state = "rebooting"
|
||
|
|
self._set_phase("flash", "done",
|
||
|
|
"components sent · staging freed" if freed
|
||
|
|
else "components sent (staging cleanup failed — free /opt manually)")
|
||
|
|
self._slot["after"] = "B" if self._slot["before"] == "A" else (
|
||
|
|
"A" if self._slot["before"] == "B" else "?")
|
||
|
|
self._message = ("flash OK — device reboots now. Reconnect after "
|
||
|
|
"~1 min, then run Config check.")
|
||
|
|
else:
|
||
|
|
self._set_phase("flash", "error", "failed at %s" % outcome.failed_step)
|
||
|
|
with self._lock:
|
||
|
|
self._state = "failed"
|
||
|
|
self._error = "flash FAILED at step %s" % outcome.failed_step
|
||
|
|
self._message = self._error
|
||
|
|
except Exception as e: # noqa: BLE001 — surface any failure to the UI
|
||
|
|
with self._lock:
|
||
|
|
self._state = "failed"
|
||
|
|
self._error = "%s: %s" % (e.__class__.__name__, e)
|
||
|
|
self._message = self._error
|
||
|
|
# mark whichever phase was active as errored
|
||
|
|
for k in ("backup", "flash", "commit"):
|
||
|
|
if self._phase_state.get(k) == "active":
|
||
|
|
self._phase_state[k] = "error"
|
||
|
|
|
||
|
|
def start_flash(self):
|
||
|
|
with self._lock:
|
||
|
|
if self._state in ('flashing', 'staging', 'rebooting', 'done'):
|
||
|
|
raise RuntimeError(
|
||
|
|
"Cannot flash in state '%s'" % self._state)
|
||
|
|
comps = list(self._components)
|
||
|
|
if not comps:
|
||
|
|
raise RuntimeError("no staged components — upload a firmware ZIP first")
|
||
|
|
# #12 fix: reset phases so a retry after a prior failed attempt starts
|
||
|
|
# clean — residual 'error' phases from the previous run must not linger.
|
||
|
|
self._reset_phases()
|
||
|
|
self._state = "flashing"
|
||
|
|
self._error = None
|
||
|
|
self._thread = threading.Thread(target=self._flash_worker, args=(comps,),
|
||
|
|
daemon=True)
|
||
|
|
self._thread.start()
|
||
|
|
|
||
|
|
def _clear_staging(self):
|
||
|
|
"""Remove staged firmware from disk to reclaim space after a successful
|
||
|
|
commit (the firmware is already written to the slot). Returns True on
|
||
|
|
success. Config backups (a separate dir) are intentionally kept."""
|
||
|
|
try:
|
||
|
|
staging.clear(self.cfg["staging_dir"])
|
||
|
|
return True
|
||
|
|
except OSError:
|
||
|
|
return False
|
||
|
|
|
||
|
|
# ---- post-reboot config restore ------------------------------------
|
||
|
|
def restore_check(self, restart=True):
|
||
|
|
self._set_phase("config", "active", "checking config")
|
||
|
|
try:
|
||
|
|
r = config_safety.detect_and_restore(self.cfg["db_path"],
|
||
|
|
self.cfg["default_file"],
|
||
|
|
self.cfg["backups_dir"])
|
||
|
|
except Exception as e: # noqa: BLE001 — missing/locked/empty DB must not 500
|
||
|
|
reason = "config DB not readable: %s" % e
|
||
|
|
self._set_phase("config", "error", "DB unreadable")
|
||
|
|
return {"restored": False, "reason": reason,
|
||
|
|
"dpworldapp_restarted": False, "error": reason}
|
||
|
|
result = {"restored": r.restored, "reason": r.reason,
|
||
|
|
"dpworldapp_restarted": False}
|
||
|
|
if r.restored and restart:
|
||
|
|
try:
|
||
|
|
# v1.5.4.5 STAB-2 fix: check=True so non-zero returncode raises
|
||
|
|
# CalledProcessError → except branch sets restart_error. 이전 check=False
|
||
|
|
# 패턴은 systemctl restart dpworldapp 실패 (returncode != 0) 시에도 except
|
||
|
|
# 진입 안 함 → result["dpworldapp_restarted"]=True 로 silent lie 가능.
|
||
|
|
# 이제 어떤 실패 (CalledProcessError / TimeoutExpired / OSError) 든 except
|
||
|
|
# 통합 처리 후 restart_error 명시.
|
||
|
|
subprocess.run(self.cfg["restart_argv"], timeout=30,
|
||
|
|
check=True, stdout=subprocess.DEVNULL,
|
||
|
|
stderr=subprocess.DEVNULL)
|
||
|
|
result["dpworldapp_restarted"] = True
|
||
|
|
except Exception as e: # noqa: BLE001
|
||
|
|
result["restart_error"] = str(e)
|
||
|
|
# #6 fix: if the restart command FAILED, the phase detail + _message must
|
||
|
|
# report the failure — never claim 'dpworldapp restarted'. The flash
|
||
|
|
# itself succeeded so state may still go 'done'; only the restart sub-step
|
||
|
|
# is reported failed (honesty: no green for an unverified restart).
|
||
|
|
restart_failed = bool(result.get("restart_error"))
|
||
|
|
with self._lock:
|
||
|
|
if r.restored and restart and restart_failed:
|
||
|
|
config_detail = ("config restored but dpworldapp restart FAILED: %s"
|
||
|
|
% result["restart_error"])
|
||
|
|
else:
|
||
|
|
config_detail = ("restored from backup" if r.restored
|
||
|
|
else "config preserved")
|
||
|
|
self._set_phase("config", "done", config_detail)
|
||
|
|
if self._state == "rebooting":
|
||
|
|
self._state = "done"
|
||
|
|
if r.restored and restart and restart_failed:
|
||
|
|
self._message = ("config restored but dpworldapp restart FAILED: %s"
|
||
|
|
% result["restart_error"])
|
||
|
|
else:
|
||
|
|
self._message = ("config restored & dpworldapp restarted"
|
||
|
|
if r.restored else "update complete — config preserved")
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _readable_dirname(path):
|
||
|
|
"""Backup paths look like /opt/config_backups/20260601-190437 — return the stamp."""
|
||
|
|
base = os.path.basename(path.rstrip("/")) if path else ""
|
||
|
|
return base or path
|