teraWeb/backend/internal/networkconfig/os_controller.go

package networkconfig

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/netip"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"tera-clone/services/tera-api/internal/storage/atomicfile"
)

const (
	DefaultInterfacesPath = "/etc/network/interfaces"
	DefaultRoutingPath    = "/etc/network/if-up.d/20-tera-active-active"
	DefaultConfigPath     = "/var/lib/tera-network/config.json"
	DefaultPendingPath    = "/var/lib/tera-network/pending.json"
	DefaultHostnamePath   = "/etc/hostname"
)

type Runner interface {
	Run(context.Context, string, ...string) ([]byte, error)
}

type execRunner struct{}

func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
	return exec.CommandContext(ctx, name, args...).CombinedOutput()
}

type OSOptions struct {
	InterfacesPath      string
	RoutingPath         string
	ConfigPath          string
	PendingPath         string
	HostnamePath        string
	HostnameCommandPath string
	IfdownPath          string
	IfupPath            string
	IPPath              string
	PingPath            string
	ShellPath           string
	SysClassRoot        string
	Runner              Runner
	Now                 func() time.Time
}

type fileSnapshot struct {
	Path    string      `json:"path"`
	Exists  bool        `json:"exists"`
	Mode    os.FileMode `json:"mode"`
	Content []byte      `json:"content,omitempty"`
}

type pendingTransaction struct {
	TransactionID string         `json:"transaction_id"`
	Deadline      time.Time      `json:"deadline"`
	Before        []fileSnapshot `json:"before"`
	Config        Config         `json:"config"`
}

type OSController struct {
	options     OSOptions
	mu          sync.Mutex
	pending     *pendingTransaction
	timer       *time.Timer
	state       string
	current     Config
	oob         OOBRuntimeStatus
	oobFailures int
}

func NewOSController(options OSOptions) (*OSController, error) {
	options = normalizeOSOptions(options)
	for name, path := range map[string]string{
		"interfaces":       options.InterfacesPath,
		"routing":          options.RoutingPath,
		"config":           options.ConfigPath,
		"pending":          options.PendingPath,
		"hostname":         options.HostnamePath,
		"hostname_command": options.HostnameCommandPath,
		"ifdown":           options.IfdownPath,
		"ifup":             options.IfupPath,
		"ip":               options.IPPath,
		"ping":             options.PingPath,
		"shell":            options.ShellPath,
		"sys_class_root":   options.SysClassRoot,
	} {
		if path == "" || (!filepath.IsAbs(path) && !strings.HasPrefix(path, "/")) {
			return nil, fmt.Errorf("%s path must be absolute", name)
		}
	}
	for _, path := range []string{options.ConfigPath, options.PendingPath, options.InterfacesPath, options.RoutingPath, options.HostnamePath} {
		if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
			return nil, fmt.Errorf("prepare network configuration directory: %w", err)
		}
	}
	return &OSController{options: options, state: "idle"}, nil
}

func normalizeOSOptions(options OSOptions) OSOptions {
	if options.InterfacesPath == "" {
		options.InterfacesPath = DefaultInterfacesPath
	}
	if options.RoutingPath == "" {
		options.RoutingPath = DefaultRoutingPath
	}
	if options.ConfigPath == "" {
		options.ConfigPath = DefaultConfigPath
	}
	if options.PendingPath == "" {
		options.PendingPath = DefaultPendingPath
	}
	if options.HostnamePath == "" {
		options.HostnamePath = DefaultHostnamePath
	}
	if options.HostnameCommandPath == "" {
		options.HostnameCommandPath = "/bin/hostname"
	}
	if options.IfdownPath == "" {
		options.IfdownPath = "/sbin/ifdown"
	}
	if options.IfupPath == "" {
		options.IfupPath = "/sbin/ifup"
	}
	if options.IPPath == "" {
		options.IPPath = "/sbin/ip"
	}
	if options.PingPath == "" {
		options.PingPath = "/bin/ping"
	}
	if options.ShellPath == "" {
		options.ShellPath = "/bin/sh"
	}
	if options.SysClassRoot == "" {
		options.SysClassRoot = "/sys/class/net"
	}
	if options.Runner == nil {
		options.Runner = execRunner{}
	}
	if options.Now == nil {
		options.Now = time.Now
	}
	return options
}

// Recover restores the pre-change files when a daemon restart or device reboot
// occurs before the management client confirms a pending transaction.
func (c *OSController) Recover(ctx context.Context) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	data, err := os.ReadFile(c.options.PendingPath)
	if os.IsNotExist(err) {
		return c.loadCurrentConfigLocked()
	}
	if err != nil {
		return fmt.Errorf("read pending network transaction: %w", err)
	}
	var pending pendingTransaction
	decoder := json.NewDecoder(strings.NewReader(string(data)))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&pending); err != nil {
		return fmt.Errorf("decode pending network transaction: %w", err)
	}
	c.pending = &pending
	if err := c.rollbackLocked(ctx, "recovered_after_restart"); err != nil {
		return fmt.Errorf("recover pending network transaction: %w", err)
	}
	return c.loadCurrentConfigLocked()
}

func (c *OSController) Apply(ctx context.Context, config Config) (ApplyResult, error) {
	normalized, err := Normalize(config)
	if err != nil {
		return ApplyResult{}, err
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if err := ctx.Err(); err != nil {
		return ApplyResult{}, err
	}
	if c.pending != nil {
		return ApplyResult{}, errors.New("a network transaction is already pending confirmation")
	}

	before := make([]fileSnapshot, 0, 4)
	for _, path := range []string{c.options.InterfacesPath, c.options.RoutingPath, c.options.ConfigPath, c.options.HostnamePath} {
		snapshot, err := snapshotFile(path)
		if err != nil {
			return ApplyResult{}, err
		}
		before = append(before, snapshot)
	}
	transactionID, err := newTransactionID()
	if err != nil {
		return ApplyResult{}, err
	}
	deadline := c.options.Now().UTC().Add(time.Duration(normalized.RollbackTimeoutSecond) * time.Second)
	pending := &pendingTransaction{TransactionID: transactionID, Deadline: deadline, Before: before, Config: normalized}
	if err := c.persistPending(pending); err != nil {
		return ApplyResult{}, err
	}
	c.pending = pending
	changedDevices := changedNetworkDevices(c.current, normalized)
	if err := c.writeConfigFiles(normalized); err != nil {
		rollbackErr := c.rollbackLocked(context.Background(), "write_failed")
		return ApplyResult{}, combineApplyRollbackError("write network configuration", err, rollbackErr)
	}
	if err := c.reconfigure(ctx, normalized, changedDevices); err != nil {
		rollbackErr := c.rollbackLocked(context.Background(), "apply_failed")
		return ApplyResult{}, combineApplyRollbackError("apply network configuration", err, rollbackErr)
	}
	c.setCurrentConfigLocked(normalized)
	c.state = "pending_confirmation"
	c.timer = time.AfterFunc(time.Duration(normalized.RollbackTimeoutSecond)*time.Second, func() { c.rollbackExpired(transactionID) })
	return ApplyResult{TransactionID: transactionID, State: c.state, Deadline: deadline}, nil
}

func (c *OSController) Confirm(ctx context.Context, transactionID string) (Status, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if err := c.requirePending(ctx, transactionID); err != nil {
		return Status{}, err
	}
	if c.timer != nil {
		c.timer.Stop()
		c.timer = nil
	}
	if err := os.Remove(c.options.PendingPath); err != nil && !os.IsNotExist(err) {
		return Status{}, fmt.Errorf("commit network transaction: %w", err)
	}
	c.pending = nil
	c.state = "confirmed"
	return c.statusLocked(ctx), nil
}

func (c *OSController) Rollback(ctx context.Context, transactionID string) (Status, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if err := c.requirePending(ctx, transactionID); err != nil {
		return Status{}, err
	}
	if err := c.rollbackLocked(ctx, "operator_requested"); err != nil {
		return Status{}, err
	}
	return c.statusLocked(ctx), nil
}

func (c *OSController) Status(ctx context.Context) (Status, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if err := ctx.Err(); err != nil {
		return Status{}, err
	}
	return c.statusLocked(ctx), nil
}

func (c *OSController) requirePending(ctx context.Context, transactionID string) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	if c.pending == nil || transactionID == "" || c.pending.TransactionID != transactionID {
		return errors.New("network transaction not found")
	}
	return nil
}

func (c *OSController) rollbackExpired(transactionID string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.pending == nil || c.pending.TransactionID != transactionID {
		return
	}
	_ = c.rollbackLocked(context.Background(), "confirmation_timeout")
}

func (c *OSController) rollbackLocked(ctx context.Context, reason string) error {
	if c.pending == nil {
		return errors.New("no network transaction is pending")
	}
	if c.timer != nil {
		c.timer.Stop()
		c.timer = nil
	}
	var restoreErrors []error
	for _, snapshot := range c.pending.Before {
		if err := restoreFile(snapshot); err != nil {
			restoreErrors = append(restoreErrors, err)
		}
	}
	if len(restoreErrors) == 0 {
		if err := c.reconfigureRestored(ctx); err != nil {
			restoreErrors = append(restoreErrors, err)
		}
	}
	if len(restoreErrors) > 0 {
		c.state = "rollback_failed"
		return errors.Join(restoreErrors...)
	}
	if err := os.Remove(c.options.PendingPath); err != nil && !os.IsNotExist(err) {
		c.state = "rollback_failed"
		return fmt.Errorf("remove pending network transaction: %w", err)
	}
	c.pending = nil
	c.state = "rolled_back_" + reason
	if err := c.loadCurrentConfigLocked(); err != nil {
		return err
	}
	return nil
}

func (c *OSController) loadCurrentConfigLocked() error {
	payload, err := os.ReadFile(c.options.ConfigPath)
	if os.IsNotExist(err) {
		c.current = Config{}
		c.oob = OOBRuntimeStatus{}
		c.oobFailures = 0
		return nil
	}
	if err != nil {
		return fmt.Errorf("read active network configuration: %w", err)
	}
	decoder := json.NewDecoder(strings.NewReader(string(payload)))
	decoder.DisallowUnknownFields()
	var config Config
	if err := decoder.Decode(&config); err != nil {
		return fmt.Errorf("decode active network configuration: %w", err)
	}
	normalized, err := Normalize(config)
	if err != nil {
		return fmt.Errorf("validate active network configuration: %w", err)
	}
	c.setCurrentConfigLocked(normalized)
	return nil
}

func (c *OSController) setCurrentConfigLocked(config Config) {
	previousFallback := c.oob.FallbackActive
	previousAt := c.oob.LastFailoverAt
	c.current = config
	c.oobFailures = 0
	c.oob = OOBRuntimeStatus{
		PrimaryInterface: config.OOBPrimaryInterface, FallbackInterface: config.OOBFallbackInterface,
		PrimaryLink: "unknown", FallbackActive: previousFallback, FailoverReason: "none", LastFailoverAt: previousAt,
	}
	if config.OperatingMode != OperatingModeOOB {
		c.oob.FallbackActive = false
		c.oob.LastFailoverAt = nil
	}
}

// RunOOBMonitor continuously evaluates the configured primary path. It is
// intentionally owned by tera-netd so route changes stay inside the narrow
// privileged network boundary.
func (c *OSController) RunOOBMonitor(ctx context.Context) {
	ticker := time.NewTicker(time.Second)
	defer ticker.Stop()
	var next time.Time
	for {
		select {
		case <-ctx.Done():
			return
		case now := <-ticker.C:
			c.mu.Lock()
			interval := time.Duration(c.current.OOBHealthCheckSecond) * time.Second
			enabled := c.current.OperatingMode == OperatingModeOOB
			c.mu.Unlock()
			if !enabled || interval <= 0 || (!next.IsZero() && now.Before(next)) {
				continue
			}
			next = now.Add(interval)
			probeContext, cancel := context.WithTimeout(ctx, 5*time.Second)
			c.checkOOBHealth(probeContext)
			cancel()
		}
	}
}

func (c *OSController) checkOOBHealth(ctx context.Context) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.current.OperatingMode != OperatingModeOOB {
		return
	}
	primary, ok := configuredInterface(c.current, c.current.OOBPrimaryInterface)
	if !ok {
		c.oob.PrimaryLink = "unknown"
		c.oob.FailoverReason = "primary_not_configured"
		return
	}
	c.oob.PrimaryInterface = c.current.OOBPrimaryInterface
	c.oob.FallbackInterface = c.current.OOBFallbackInterface
	c.oob.PrimaryLink = c.interfaceLink(primary.Device)
	fallback, fallbackConfigured := configuredInterface(c.current, c.current.OOBFallbackInterface)
	c.oob.FallbackAvailable = fallbackConfigured && c.interfaceLink(fallback.Device) == "up"
	healthy := c.oob.PrimaryLink == "up"
	if healthy {
		_, err := c.options.Runner.Run(ctx, c.options.PingPath, "-I", primary.Device, "-c", "1", "-W", "2", c.current.OOBHealthCheckTarget)
		healthy = err == nil
	}
	if healthy {
		c.oobFailures = 0
		if c.oob.FallbackActive {
			if c.installPrimaryDefaultLocked(ctx, primary) == nil {
				c.oob.FallbackActive = false
				c.oob.FailoverReason = "primary_restored"
				now := c.options.Now().UTC()
				c.oob.LastFailoverAt = &now
			}
		} else {
			c.oob.FailoverReason = "none"
		}
		return
	}
	c.oobFailures++
	c.oob.FailoverReason = "primary_health_check_failed"
	if c.oobFailures < c.current.OOBFailureThreshold || c.oob.FallbackActive || !c.oob.FallbackAvailable {
		return
	}
	args := []string{"-4", "route", "replace", "default"}
	if fallback.Gateway != "" {
		args = append(args, "via", fallback.Gateway)
	}
	args = append(args, "dev", fallback.Device)
	if _, err := c.options.Runner.Run(ctx, c.options.IPPath, args...); err != nil {
		c.oob.FailoverReason = "fallback_route_failed"
		return
	}
	c.oob.FallbackActive = true
	c.oob.FailoverReason = "primary_failure_threshold_reached"
	now := c.options.Now().UTC()
	c.oob.LastFailoverAt = &now
}

func configuredInterface(config Config, name string) (Interface, bool) {
	for _, item := range config.Interfaces {
		if item.Name == name || item.Device == name {
			return item, true
		}
	}
	return Interface{}, false
}

func (c *OSController) interfaceLink(device string) string {
	device = strings.TrimSpace(device)
	if device == "" || strings.ContainsAny(device, `/\\`) {
		return "unknown"
	}
	if carrier, err := os.ReadFile(filepath.Join(c.options.SysClassRoot, device, "carrier")); err == nil {
		if strings.TrimSpace(string(carrier)) == "1" {
			return "up"
		}
		return "down"
	}
	if state, err := os.ReadFile(filepath.Join(c.options.SysClassRoot, device, "operstate")); err == nil {
		if strings.TrimSpace(string(state)) == "up" {
			return "up"
		}
		return "down"
	}
	if iface, err := net.InterfaceByName(device); err == nil && iface.Flags&net.FlagUp != 0 {
		return "up"
	}
	return "unknown"
}

func (c *OSController) installPrimaryDefaultLocked(ctx context.Context, primary Interface) error {
	args := []string{"-4", "route", "replace", "default"}
	if primary.Gateway != "" {
		args = append(args, "via", primary.Gateway)
	}
	args = append(args, "dev", primary.Device)
	_, err := c.options.Runner.Run(ctx, c.options.IPPath, args...)
	return err
}

func (c *OSController) persistPending(pending *pendingTransaction) error {
	payload, err := json.MarshalIndent(pending, "", "  ")
	if err != nil {
		return fmt.Errorf("encode pending network transaction: %w", err)
	}
	payload = append(payload, '\n')
	if err := atomicfile.Write(c.options.PendingPath, payload, 0o600); err != nil {
		return fmt.Errorf("persist pending network transaction: %w", err)
	}
	return nil
}

func (c *OSController) writeConfigFiles(config Config) error {
	interfaces, err := renderInterfaces(config, c.options.RoutingPath, c.options.IPPath)
	if err != nil {
		return err
	}
	routing, err := RenderRoutingScript(config, c.options.IPPath)
	if err != nil {
		return err
	}
	payload, err := json.MarshalIndent(config, "", "  ")
	if err != nil {
		return err
	}
	if err := atomicfile.Write(c.options.InterfacesPath, interfaces, 0o644); err != nil {
		return err
	}
	if err := atomicfile.Write(c.options.RoutingPath, routing, 0o755); err != nil {
		return err
	}
	if err := atomicfile.Write(c.options.ConfigPath, append(payload, '\n'), 0o600); err != nil {
		return err
	}
	if err := atomicfile.Write(c.options.HostnamePath, []byte(config.HostName+"\n"), 0o644); err != nil {
		return err
	}
	return nil
}

func (c *OSController) reconfigure(ctx context.Context, config Config, changedDevices map[string]bool) error {
	if output, err := c.options.Runner.Run(ctx, c.options.HostnameCommandPath, config.HostName); err != nil {
		return fmt.Errorf("set hostname: %s: %w", strings.TrimSpace(string(output)), err)
	}
	networkChanged := false
	for _, changed := range changedDevices {
		networkChanged = networkChanged || changed
	}
	if !networkChanged {
		return nil
	}
	// Tear down aggregate devices explicitly. The interfaces file has already
	// been replaced, so ifdown alone cannot reliably find the previous mode.
	for _, device := range []string{BondDevice, BridgeDevice} {
		_, _ = c.options.Runner.Run(ctx, c.options.IfdownPath, device)
		_, _ = c.options.Runner.Run(ctx, c.options.IPPath, "link", "delete", "dev", device)
	}
	if config.Advanced.Mode != AdvancedModeSingle {
		for _, device := range []string{Interface1Device, Interface2Device} {
			_, _ = c.options.Runner.Run(ctx, c.options.IfdownPath, device)
		}
		device := BondDevice
		if config.Advanced.Mode == AdvancedModeBridge {
			device = BridgeDevice
		}
		output, err := c.options.Runner.Run(ctx, c.options.IfupPath, device)
		if err != nil {
			return fmt.Errorf("ifup %s: %s: %w", device, strings.TrimSpace(string(output)), err)
		}
		if info, err := os.Stat(c.options.RoutingPath); err == nil && info.Mode().IsRegular() {
			_, _ = c.options.Runner.Run(ctx, c.options.ShellPath, c.options.RoutingPath)
		}
		return nil
	}
	for _, item := range config.Interfaces {
		if !changedDevices[item.Device] {
			continue
		}
		_, _ = c.options.Runner.Run(ctx, c.options.IfdownPath, item.Device)
		output, err := c.options.Runner.Run(ctx, c.options.IfupPath, item.Device)
		if err != nil {
			if item.Device == "eth0" {
				return fmt.Errorf("ifup %s: %s: %w", item.Device, strings.TrimSpace(string(output)), err)
			}
		}
	}
	if info, err := os.Stat(c.options.RoutingPath); err == nil && info.Mode().IsRegular() {
		_, _ = c.options.Runner.Run(ctx, c.options.ShellPath, c.options.RoutingPath)
	}
	return nil
}

func changedNetworkDevices(current, next Config) map[string]bool {
	changed := make(map[string]bool, len(next.Interfaces))
	currentByDevice := make(map[string]Interface, len(current.Interfaces))
	for _, item := range current.Interfaces {
		currentByDevice[item.Device] = item
	}
	advancedChanged := !reflect.DeepEqual(current.Advanced, next.Advanced)
	for _, item := range next.Interfaces {
		previous, exists := currentByDevice[item.Device]
		changed[item.Device] = !exists || advancedChanged || !reflect.DeepEqual(previous, item)
	}
	return changed
}

func (c *OSController) reconfigureRestored(ctx context.Context) error {
	if payload, err := os.ReadFile(c.options.ConfigPath); err == nil {
		var restored Config
		decoder := json.NewDecoder(strings.NewReader(string(payload)))
		decoder.DisallowUnknownFields()
		if decodeErr := decoder.Decode(&restored); decodeErr == nil {
			if normalized, normalizeErr := Normalize(restored); normalizeErr == nil {
				return c.reconfigure(ctx, normalized, map[string]bool{Interface1Device: true, Interface2Device: true})
			}
		}
	}
	if payload, err := os.ReadFile(c.options.HostnamePath); err == nil {
		hostname := strings.TrimSpace(string(payload))
		if hostname != "" {
			_, _ = c.options.Runner.Run(ctx, c.options.HostnameCommandPath, hostname)
		}
	}
	for _, device := range []string{BondDevice, BridgeDevice} {
		_, _ = c.options.Runner.Run(ctx, c.options.IfdownPath, device)
		_, _ = c.options.Runner.Run(ctx, c.options.IPPath, "link", "delete", "dev", device)
	}
	for _, device := range []string{Interface1Device, Interface2Device} {
		_, _ = c.options.Runner.Run(ctx, c.options.IfdownPath, device)
		_, _ = c.options.Runner.Run(ctx, c.options.IfupPath, device)
	}
	if info, err := os.Stat(c.options.RoutingPath); err == nil && info.Mode().IsRegular() {
		_, _ = c.options.Runner.Run(ctx, c.options.ShellPath, c.options.RoutingPath)
	}
	return nil
}

func (c *OSController) statusLocked(ctx context.Context) Status {
	state := c.state
	if state == "" {
		state = "idle"
	}
	status := Status{State: state, RoutingMode: RoutingModeActive}
	if c.current.OperatingMode != "" {
		current := c.current
		current.Interfaces = append([]Interface(nil), c.current.Interfaces...)
		status.Current = &current
		status.RoutingMode = c.current.OperatingMode
	}
	status.OOB = c.oob
	if c.pending != nil {
		deadline := c.pending.Deadline
		if c.state == "rollback_failed" {
			status.State = c.state
		} else {
			status.State = "pending_confirmation"
		}
		status.TransactionID = c.pending.TransactionID
		status.Deadline = &deadline
	}
	statusDevices := []string{Interface1Device, Interface2Device}
	if c.current.Advanced.Mode == AdvancedModeBonding {
		statusDevices = append(statusDevices, BondDevice)
	} else if c.current.Advanced.Mode == AdvancedModeBridge {
		statusDevices = append(statusDevices, BridgeDevice)
	}
	for _, device := range statusDevices {
		actual := ActualInterface{Device: device, Link: "unknown", Addresses: []string{}}
		if iface, err := net.InterfaceByName(device); err == nil {
			if iface.Flags&net.FlagUp != 0 {
				actual.Link = "up"
			} else {
				actual.Link = "down"
			}
			if addresses, err := iface.Addrs(); err == nil {
				for _, address := range addresses {
					if prefix, err := netip.ParsePrefix(address.String()); err == nil {
						actual.Addresses = append(actual.Addresses, prefix.String())
					}
				}
				sort.Strings(actual.Addresses)
			}
		}
		if carrier, err := os.ReadFile(filepath.Join(c.options.SysClassRoot, device, "carrier")); err == nil {
			switch strings.TrimSpace(string(carrier)) {
			case "1":
				actual.Link = "up"
			case "0":
				actual.Link = "down"
			}
		}
		if speed, err := os.ReadFile(filepath.Join(c.options.SysClassRoot, device, "speed")); err == nil {
			value := strings.TrimSpace(string(speed))
			if parsed, err := strconv.Atoi(value); err == nil && parsed > 0 {
				actual.Speed = fmt.Sprintf("%d Mbps", parsed)
			}
		}
		status.Interfaces = append(status.Interfaces, actual)
	}
	if output, err := c.options.Runner.Run(ctx, c.options.IPPath, "-4", "route", "show", "default"); err == nil {
		status.DefaultRoute = strings.TrimSpace(string(output))
	}
	return status
}

func snapshotFile(path string) (fileSnapshot, error) {
	info, err := os.Lstat(path)
	if os.IsNotExist(err) {
		return fileSnapshot{Path: path}, nil
	}
	if err != nil {
		return fileSnapshot{}, fmt.Errorf("inspect %s: %w", path, err)
	}
	if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
		return fileSnapshot{}, fmt.Errorf("refusing to replace non-regular network path %s", path)
	}
	content, err := os.ReadFile(path)
	if err != nil {
		return fileSnapshot{}, fmt.Errorf("read %s: %w", path, err)
	}
	return fileSnapshot{Path: path, Exists: true, Mode: info.Mode().Perm(), Content: content}, nil
}

func restoreFile(snapshot fileSnapshot) error {
	if snapshot.Exists {
		mode := snapshot.Mode
		if mode == 0 {
			mode = 0o600
		}
		return atomicfile.Write(snapshot.Path, snapshot.Content, mode)
	}
	if err := os.Remove(snapshot.Path); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove newly created network path %s: %w", snapshot.Path, err)
	}
	return nil
}

func newTransactionID() (string, error) {
	random := make([]byte, 12)
	if _, err := rand.Read(random); err != nil {
		return "", fmt.Errorf("create network transaction ID: %w", err)
	}
	return "network-" + hex.EncodeToString(random), nil
}

func combineApplyRollbackError(action string, applyErr, rollbackErr error) error {
	if rollbackErr != nil {
		return fmt.Errorf("%s: %v; automatic rollback failed: %w", action, applyErr, rollbackErr)
	}
	return fmt.Errorf("%s: %w; previous configuration restored", action, applyErr)
}

func RenderInterfaces(config Config, routingPath string) ([]byte, error) {
	return renderInterfaces(config, routingPath, "/sbin/ip")
}

func renderInterfaces(config Config, routingPath, ipPath string) ([]byte, error) {
	normalized, err := Normalize(config)
	if err != nil {
		return nil, err
	}
	if routingPath == "" || !isAbsoluteCommandPath(routingPath) || strings.ContainsAny(routingPath, "\r\n\t ") {
		return nil, errors.New("routing script path must be an absolute path without whitespace")
	}
	if ipPath == "" || !isAbsoluteCommandPath(ipPath) || strings.ContainsAny(ipPath, "\r\n\t ") {
		return nil, errors.New("ip command path must be an absolute path without whitespace")
	}
	var builder strings.Builder
	builder.WriteString("# Managed by tera-netd. Use the Web UI; manual edits are replaced.\n")
	builder.WriteString("auto lo\niface lo inet loopback\n\n")
	if normalized.Advanced.Mode == AdvancedModeSingle {
		for index, item := range normalized.Interfaces {
			writeLayer3Stanza(&builder, item, index == len(normalized.Interfaces)-1, routingPath, nil, nil)
		}
		return []byte(builder.String()), nil
	}

	builder.WriteString("# eth0 and eth1 are layer-2 members; Interface-1 owns the shared layer-3 configuration.\n")
	builder.WriteString("auto eth0\niface eth0 inet manual\n\nauto eth1\niface eth1 inet manual\n\n")
	effective := effectiveInterfaces(normalized)
	if len(effective) != 1 {
		return nil, errors.New("advanced network mode requires Interface-1")
	}
	item := effective[0]
	preUp := make([]string, 0, 8)
	postDown := make([]string, 0, 4)
	if normalized.Advanced.Mode == AdvancedModeBonding {
		preUp = append(preUp,
			fmt.Sprintf("%s link add name %s type bond 2>/dev/null || true", ipPath, BondDevice),
			fmt.Sprintf("%s link set dev %s type bond mode %s miimon %d", ipPath, BondDevice, normalized.Advanced.BondMode, normalized.Advanced.BondMii),
		)
	} else {
		stp := 0
		if normalized.Advanced.BridgeSTP {
			stp = 1
		}
		preUp = append(preUp,
			fmt.Sprintf("%s link add name %s type bridge 2>/dev/null || true", ipPath, BridgeDevice),
			fmt.Sprintf("%s link set dev %s type bridge stp_state %d", ipPath, BridgeDevice, stp),
		)
	}
	for _, device := range []string{Interface1Device, Interface2Device} {
		preUp = append(preUp,
			fmt.Sprintf("%s link set dev %s down", ipPath, device),
			fmt.Sprintf("%s address flush dev %s", ipPath, device),
			fmt.Sprintf("%s link set dev %s master %s", ipPath, device, item.Device),
			fmt.Sprintf("%s link set dev %s up", ipPath, device),
		)
		postDown = append(postDown, fmt.Sprintf("%s link set dev %s nomaster 2>/dev/null || true", ipPath, device))
	}
	postDown = append(postDown, fmt.Sprintf("%s link delete dev %s 2>/dev/null || true", ipPath, item.Device))
	writeLayer3Stanza(&builder, item, true, routingPath, preUp, postDown)
	return []byte(builder.String()), nil
}

func writeLayer3Stanza(builder *strings.Builder, item Interface, runRouting bool, routingPath string, preUp, postDown []string) {
	fmt.Fprintf(builder, "# %s role=%s weight=%d\nauto %s\n", item.Name, item.Role, item.Weight, item.Device)
	if item.AddressMode == AddressModeDHCP {
		fmt.Fprintf(builder, "iface %s inet dhcp\n", item.Device)
	} else {
		prefix := netip.MustParsePrefix(item.Address)
		mask := net.IP(net.CIDRMask(prefix.Bits(), 32)).String()
		fmt.Fprintf(builder, "iface %s inet static\n    address %s\n    netmask %s\n", item.Device, prefix.Addr(), mask)
	}
	for _, command := range preUp {
		fmt.Fprintf(builder, "    pre-up %s\n", command)
	}
	interfaceDNSServers := append([]string(nil), item.DNSServers...)
	// BusyBox ifupdown does not implement the Debian "inet6 auto" method.
	if item.IPv6Mode == IPv6ModeAuto {
		interfaceDNSServers = append(interfaceDNSServers, item.IPv6DNSServers...)
	}
	if len(interfaceDNSServers) > 0 {
		fmt.Fprintf(builder, "    dns-nameservers %s\n", strings.Join(interfaceDNSServers, " "))
	}
	if runRouting {
		fmt.Fprintf(builder, "    post-up %s\n", routingPath)
	}
	for _, command := range postDown {
		fmt.Fprintf(builder, "    post-down %s\n", command)
	}
	builder.WriteByte('\n')
	if item.IPv6Mode == IPv6ModeStatic {
		prefix := netip.MustParsePrefix(item.IPv6Address)
		fmt.Fprintf(builder, "iface %s inet6 static\n    address %s\n    netmask %d\n", item.Device, prefix.Addr(), prefix.Bits())
		if item.IPv6Gateway != "" {
			fmt.Fprintf(builder, "    gateway %s\n", item.IPv6Gateway)
		}
		if len(item.IPv6DNSServers) > 0 {
			fmt.Fprintf(builder, "    dns-nameservers %s\n", strings.Join(item.IPv6DNSServers, " "))
		}
		builder.WriteByte('\n')
	}
}

func RenderRoutingScript(config Config, ipPath string) ([]byte, error) {
	normalized, err := Normalize(config)
	if err != nil {
		return nil, err
	}
	if ipPath == "" || !isAbsoluteCommandPath(ipPath) || strings.ContainsAny(ipPath, "\r\n\t ") {
		return nil, errors.New("ip command path must be an absolute path without whitespace")
	}
	var builder strings.Builder
	interfaces := effectiveInterfaces(normalized)
	external := make([]Interface, 0, 2)
	dhcpExternal := false
	for _, item := range interfaces {
		if item.Role == RoleExternal && item.AddressMode == AddressModeDHCP {
			dhcpExternal = true
		}
		if item.Role == RoleExternal && item.AddressMode == AddressModeStatic {
			external = append(external, item)
		}
	}
	builder.WriteString("#!/bin/sh\nset -eu\nIP=")
	builder.WriteString(ipPath)
	builder.WriteString("\n\n# ifup invokes this script once per interface. Defer routes that depend on\n# an address which has not been installed yet.\nnetwork_ready() {\n  $IP -4 addr show dev \"$1\" 2>/dev/null | grep -Fq \"inet $2/\"\n}\n\nif $IP -4 rule show >/dev/null 2>&1; then\n")
	for index, item := range interfaces {
		if item.AddressMode != AddressModeStatic {
			continue
		}
		table := 100 + index
		priority := 10000 + index
		prefix := netip.MustParsePrefix(item.Address)
		fmt.Fprintf(&builder, "  while $IP -4 rule del priority %d 2>/dev/null; do :; done\n", priority)
		fmt.Fprintf(&builder, "  $IP -4 route flush table %d 2>/dev/null || true\n", table)
		fmt.Fprintf(&builder, "  if network_ready %s %s; then\n", item.Device, prefix.Addr())
		fmt.Fprintf(&builder, "    $IP -4 route add %s dev %s src %s table %d\n", prefix.Masked(), item.Device, prefix.Addr(), table)
		if item.Gateway != "" {
			fmt.Fprintf(&builder, "    $IP -4 route add default via %s dev %s table %d\n", item.Gateway, item.Device, table)
		}
		fmt.Fprintf(&builder, "    $IP -4 rule add priority %d from %s/32 lookup %d\n", priority, prefix.Addr(), table)
		fmt.Fprintf(&builder, "  else\n    echo 'deferring IPv4 policy route for %s until %s is configured' >&2\n  fi\n", item.Device, prefix.Addr())
	}
	builder.WriteString("else\n")
	if len(external) == 2 {
		builder.WriteString("  echo 'kernel lacks IPv4 policy routing required for dual-external active/active' >&2\n  exit 1\n")
	} else {
		builder.WriteString("  echo 'IPv4 policy routing unavailable; using connected routes plus one main default' >&2\n")
	}
	builder.WriteString("fi\n\n")
	if normalized.OperatingMode == OperatingModeOOB {
		primary, _ := configuredInterface(normalized, normalized.OOBPrimaryInterface)
		if primary.AddressMode == AddressModeDHCP {
			builder.WriteString("# DHCP owns the OOB primary default route until health monitoring activates fallback.\n")
		} else {
			builder.WriteString("while $IP -4 route del default 2>/dev/null; do :; done\n")
			primaryAddress := netip.MustParsePrefix(primary.Address).Addr()
			fmt.Fprintf(&builder, "if network_ready %s %s; then\n  $IP -4 route add default via %s dev %s\nelse\n  echo 'deferring OOB default route until %s is configured' >&2\nfi\n", primary.Device, primaryAddress, primary.Gateway, primary.Device, primary.Device)
		}
	} else if !dhcpExternal {
		builder.WriteString("while $IP -4 route del default 2>/dev/null; do :; done\n")
		if len(external) == 1 {
			address := netip.MustParsePrefix(external[0].Address).Addr()
			fmt.Fprintf(&builder, "if network_ready %s %s; then\n  $IP -4 route add default via %s dev %s\nelse\n  echo 'deferring main default route until %s is configured' >&2\nfi\n", external[0].Device, address, external[0].Gateway, external[0].Device, external[0].Device)
		} else if len(external) == 2 {
			firstAddress := netip.MustParsePrefix(external[0].Address).Addr()
			secondAddress := netip.MustParsePrefix(external[1].Address).Addr()
			fmt.Fprintf(&builder, "if network_ready %s %s && network_ready %s %s; then\n  $IP -4 route add default", external[0].Device, firstAddress, external[1].Device, secondAddress)
			for _, item := range external {
				fmt.Fprintf(&builder, " nexthop via %s dev %s weight %d", item.Gateway, item.Device, item.Weight)
			}
			fmt.Fprintf(&builder, "\nelif network_ready %s %s; then\n  $IP -4 route add default via %s dev %s\nelif network_ready %s %s; then\n  $IP -4 route add default via %s dev %s\nelse\n  echo 'deferring main default route until an external interface is configured' >&2\nfi\n", external[0].Device, firstAddress, external[0].Gateway, external[0].Device, external[1].Device, secondAddress, external[1].Gateway, external[1].Device)
		}
	} else {
		builder.WriteString("# DHCP owns the dynamic default route; do not replace it here.\n")
	}
	return []byte(builder.String()), nil
}

func isAbsoluteCommandPath(path string) bool {
	return filepath.IsAbs(path) || strings.HasPrefix(path, "/")
}

// FactoryReset replaces persistent network state without dropping the current
// management connection. Boot applies these files after the reset response.
func (c *OSController) FactoryReset(ctx context.Context, config Config) error {
	normalized, err := Normalize(config)
	if err != nil {
		return err
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if err := ctx.Err(); err != nil {
		return err
	}
	if c.timer != nil {
		c.timer.Stop()
		c.timer = nil
	}
	if err := c.writeConfigFiles(normalized); err != nil {
		return err
	}
	if err := os.Remove(c.options.PendingPath); err != nil && !os.IsNotExist(err) {
		return err
	}
	c.pending = nil
	c.setCurrentConfigLocked(normalized)
	c.state = "confirmed"
	return nil
}