teraWeb/backend/internal/serial/cr_probe.go

package serial

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"sync"
	"syscall"
	"time"
)

const (
	CRProbeStateDisabled   = "disabled"
	CRProbeStatePending    = "pending"
	CRProbeStateConfirming = "confirming"
	CRProbeStateResponded  = "responded"
	CRProbeStateNoResponse = "no_response"
	CRProbeStateBusy       = "busy"
	CRProbeStateSuppressed = "suppressed"
	CRProbeStateError      = "error"
)

// CRProbeState is a cached observation. A response proves that a remote serial
// endpoint answered; no_response does not prove that the cable is absent.
type CRProbeState struct {
	Channel              string     `json:"channel"`
	Enabled              bool       `json:"enabled"`
	State                string     `json:"state"`
	Method               string     `json:"method"`
	LastProbeAt          *time.Time `json:"last_probe_at,omitempty"`
	LastResponseAt       *time.Time `json:"last_response_at,omitempty"`
	LastConfirmedAt      *time.Time `json:"last_confirmed_at,omitempty"`
	ResponseBytes        int        `json:"response_bytes,omitempty"`
	ConsecutiveResponses int        `json:"consecutive_responses,omitempty"`
	ConsecutiveMisses    int        `json:"consecutive_misses,omitempty"`
	Up                   bool       `json:"up"`
	EverUp               bool       `json:"ever_up"`
	Message              string     `json:"message,omitempty"`
}

type CRProbeStateController interface {
	CRProbeStates(context.Context) ([]CRProbeState, error)
}

// CRProbeGroup contains channels backed by one PCI function/IRQ. Channels in
// the same group are always probed serially; separate groups may run in
// parallel without overlapping sweeps.
type CRProbeGroup struct {
	ID       string
	Channels []string
}

type CRProbeOptions struct {
	Enabled          bool
	ResponseTimeout  time.Duration
	SweepInterval    time.Duration
	BetweenChannels  time.Duration
	QuietPeriod      time.Duration
	StartupDelay     time.Duration
	Workers          int
	Groups           []CRProbeGroup
	ConfirmResponses int
	DownMisses       int
	Now              func() time.Time
}

// CRProbeManager sends one CR byte only while it owns the same exclusive lease
// used by normal sessions. Results are cached so API polling never touches UARTs.
type CRProbeManager struct {
	manager *StreamLeaseManager
	options CRProbeOptions

	mu     sync.Mutex
	states map[string]CRProbeState
	groups []CRProbeGroup
}

func NewCRProbeManager(manager *StreamLeaseManager, options CRProbeOptions) (*CRProbeManager, error) {
	if manager == nil {
		return nil, errors.New("serial stream manager is required")
	}
	if options.ResponseTimeout <= 0 {
		options.ResponseTimeout = 250 * time.Millisecond
	}
	if options.ResponseTimeout < 50*time.Millisecond {
		options.ResponseTimeout = 50 * time.Millisecond
	}
	if options.ResponseTimeout > 5*time.Second {
		options.ResponseTimeout = 5 * time.Second
	}
	if options.SweepInterval <= 0 {
		options.SweepInterval = 2 * time.Second
	}
	if options.BetweenChannels <= 0 {
		options.BetweenChannels = 25 * time.Millisecond
	}
	if options.QuietPeriod <= 0 {
		options.QuietPeriod = 5 * time.Second
	}
	if options.StartupDelay < 0 {
		options.StartupDelay = 0
	}
	if options.StartupDelay > 5*time.Minute {
		options.StartupDelay = 5 * time.Minute
	}
	if options.Workers <= 0 {
		options.Workers = 1
	}
	if options.Workers > 16 {
		options.Workers = 16
	}
	if options.ConfirmResponses <= 0 {
		options.ConfirmResponses = 1
	}
	if options.ConfirmResponses > 5 {
		options.ConfirmResponses = 5
	}
	if options.DownMisses <= 0 {
		options.DownMisses = 2
	}
	if options.DownMisses > 10 {
		options.DownMisses = 10
	}
	if options.Now == nil {
		options.Now = time.Now
	}
	groups, err := normalizeCRProbeGroups(options.Groups, options.Workers)
	if err != nil {
		return nil, err
	}
	probe := &CRProbeManager{
		manager: manager,
		options: options,
		states:  make(map[string]CRProbeState, ChannelCount),
		groups:  groups,
	}
	for index := 1; index <= ChannelCount; index++ {
		channel := fmt.Sprintf("CH%02d", index)
		state := CRProbeStatePending
		message := "CR response probe is waiting"
		if !options.Enabled {
			state = CRProbeStateDisabled
			message = "CR response probe is disabled"
		}
		probe.states[channel] = CRProbeState{
			Channel: channel, Enabled: options.Enabled, State: state, Method: "cr", Message: message,
		}
	}
	return probe, nil
}

func (m *CRProbeManager) Run(ctx context.Context) {
	if m == nil || !m.options.Enabled {
		return
	}
	if !waitForProbe(ctx, m.options.StartupDelay) {
		return
	}
	for {
		sweepStarted := time.Now()
		var groups sync.WaitGroup
		for _, configuredGroup := range m.groups {
			group := configuredGroup
			groups.Add(1)
			go func() {
				defer groups.Done()
				var previousCompletedAt time.Time
				for index, channel := range group.Channels {
					if ctx.Err() != nil {
						return
					}
					target := sweepStarted.Add(crProbeOffset(m.options.SweepInterval, len(group.Channels), index))
					if !previousCompletedAt.IsZero() {
						spacedTarget := previousCompletedAt.Add(m.options.BetweenChannels)
						if spacedTarget.After(target) {
							target = spacedTarget
						}
					}
					if !waitForProbe(ctx, time.Until(target)) {
						return
					}
					state, probeErr := m.ProbeOnce(ctx, channel)
					if probeErr == nil && shouldConfirmCRProbeMiss(state, m.options.DownMisses) {
						// A formerly confirmed link is not declared DOWN from one noisy read.
						// Confirm the miss immediately so the two-sample decision still fits
						// inside the four-second sweep for the normal single-link case.
						if !waitForProbe(ctx, m.options.BetweenChannels) {
							return
						}
						_, _ = m.ProbeOnce(ctx, channel)
					}
					previousCompletedAt = time.Now()
				}
			}()
		}
		groups.Wait()
		remaining := m.options.SweepInterval - time.Since(sweepStarted)
		if remaining < 0 {
			remaining = 0
		}
		if !waitForProbe(ctx, remaining) {
			return
		}
	}
}

func crProbeOffset(sweepInterval time.Duration, channelCount, index int) time.Duration {
	if sweepInterval <= 0 || channelCount <= 1 || index <= 0 {
		return 0
	}
	if index >= channelCount {
		index = channelCount - 1
	}
	return time.Duration(index) * sweepInterval / time.Duration(channelCount)
}

func shouldConfirmCRProbeMiss(state CRProbeState, downMisses int) bool {
	return downMisses > 1 && state.EverUp && state.Up && state.State == CRProbeStateNoResponse &&
		state.ConsecutiveMisses > 0 && state.ConsecutiveMisses < downMisses
}

func normalizeCRProbeGroups(configured []CRProbeGroup, workers int) ([]CRProbeGroup, error) {
	groups := configured
	if len(groups) == 0 {
		if workers < 1 {
			workers = 1
		}
		if workers > ChannelCount {
			workers = ChannelCount
		}
		groups = make([]CRProbeGroup, workers)
		for index := range groups {
			groups[index].ID = fmt.Sprintf("worker-%02d", index+1)
		}
		for index := 1; index <= ChannelCount; index++ {
			group := (index - 1) % workers
			groups[group].Channels = append(groups[group].Channels, fmt.Sprintf("CH%02d", index))
		}
	}

	result := make([]CRProbeGroup, 0, len(groups))
	seenGroups := make(map[string]bool, len(groups))
	seenChannels := make(map[string]bool, ChannelCount)
	for _, configuredGroup := range groups {
		id := strings.TrimSpace(configuredGroup.ID)
		if id == "" || seenGroups[id] {
			return nil, errors.New("CR probe groups require unique non-empty IDs")
		}
		if len(configuredGroup.Channels) == 0 {
			return nil, fmt.Errorf("CR probe group %s must contain channels", id)
		}
		seenGroups[id] = true
		group := CRProbeGroup{ID: id, Channels: make([]string, 0, len(configuredGroup.Channels))}
		for _, configuredChannel := range configuredGroup.Channels {
			channel, err := normalizeChannel(configuredChannel)
			if err != nil || channel != configuredChannel {
				return nil, fmt.Errorf("CR probe group %s contains an invalid channel", id)
			}
			if seenChannels[channel] {
				return nil, fmt.Errorf("CR probe groups contain duplicate %s", channel)
			}
			seenChannels[channel] = true
			group.Channels = append(group.Channels, channel)
		}
		result = append(result, group)
	}
	if len(seenChannels) != ChannelCount {
		return nil, fmt.Errorf("CR probe groups must contain exactly %d channels", ChannelCount)
	}
	for index := 1; index <= ChannelCount; index++ {
		channel := fmt.Sprintf("CH%02d", index)
		if !seenChannels[channel] {
			return nil, fmt.Errorf("CR probe groups are missing %s", channel)
		}
	}
	return result, nil
}

func (m *CRProbeManager) CRProbeStates(ctx context.Context) ([]CRProbeState, error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	states := make([]CRProbeState, 0, ChannelCount)
	for index := 1; index <= ChannelCount; index++ {
		states = append(states, m.states[fmt.Sprintf("CH%02d", index)])
	}
	return states, nil
}

func (m *CRProbeManager) ProbeOnce(ctx context.Context, channel string) (CRProbeState, error) {
	canonical, err := normalizeChannel(channel)
	if err != nil || canonical != channel {
		return CRProbeState{}, errors.New("CR probe channel must be canonical CH01 through CH48")
	}
	if !m.options.Enabled {
		return m.update(channel, CRProbeStateDisabled, 0, nil), nil
	}
	now := m.options.Now().UTC()
	if m.manager.RecentlyActive(channel, m.options.QuietPeriod, now) {
		return m.updateAt(channel, CRProbeStateSuppressed, 0, nil, now, "recent serial traffic; probe skipped"), nil
	}

	lease, err := m.manager.Acquire(ctx, channel, CRProbeClientID)
	if err != nil {
		if errors.Is(err, ErrStreamLeaseBusy) {
			return m.updateAt(channel, CRProbeStateBusy, 0, nil, now, "serial channel is in use; probe skipped"), nil
		}
		state := m.updateAt(channel, CRProbeStateError, 0, nil, now, err.Error())
		return state, err
	}
	defer lease.Close()
	if err := drainCRProbeInput(ctx, lease); err != nil {
		state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe input drain failed")
		return state, err
	}
	if _, err := lease.Write([]byte{'\r'}); err != nil {
		state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe write failed")
		return state, err
	}

	type readResult struct {
		bytes int
		err   error
	}
	resultChannel := make(chan readResult, 1)
	go func() {
		buffer := make([]byte, 256)
		responseBytes := 0
		for {
			readBuffer := buffer
			pending, queueSupported, queueErr := pendingSerialInput(lease.device)
			if queueErr != nil {
				resultChannel <- readResult{err: queueErr}
				return
			}
			if queueSupported {
				if pending <= 0 {
					select {
					case <-ctx.Done():
						resultChannel <- readResult{err: ctx.Err()}
						return
					case <-time.After(20 * time.Millisecond):
						continue
					}
				}
				if pending < len(readBuffer) {
					readBuffer = readBuffer[:pending]
				}
			}
			count, readErr := lease.Read(readBuffer)
			if count > 0 {
				responseBytes += count
				if meaningfulCRResponse(readBuffer[:count]) {
					resultChannel <- readResult{bytes: responseBytes}
					return
				}
				continue
			}
			if readErr == nil || errors.Is(readErr, syscall.EAGAIN) || errors.Is(readErr, syscall.EWOULDBLOCK) {
				select {
				case <-ctx.Done():
					resultChannel <- readResult{err: ctx.Err()}
					return
				case <-time.After(20 * time.Millisecond):
					continue
				}
			}
			resultChannel <- readResult{err: readErr}
			return
		}
	}()

	timer := time.NewTimer(m.options.ResponseTimeout)
	defer timer.Stop()
	select {
	case result := <-resultChannel:
		if result.bytes > 0 {
			responseAt := m.options.Now().UTC()
			return m.updateAt(channel, CRProbeStateResponded, result.bytes, &responseAt, now, "remote serial endpoint responded to CR"), nil
		}
		if result.err != nil && !errors.Is(result.err, context.Canceled) && !errors.Is(result.err, context.DeadlineExceeded) {
			state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe read failed")
			return state, result.err
		}
		return m.updateAt(channel, CRProbeStateNoResponse, 0, nil, now, "no CR response; cable state is indeterminate"), nil
	case <-timer.C:
		_ = lease.Close()
		return m.updateAt(channel, CRProbeStateNoResponse, 0, nil, now, "no CR response; cable state is indeterminate"), nil
	case <-ctx.Done():
		_ = lease.Close()
		return CRProbeState{}, ctx.Err()
	}
}

func (m *CRProbeManager) update(channel, state string, responseBytes int, responseAt *time.Time) CRProbeState {
	return m.updateAt(channel, state, responseBytes, responseAt, m.options.Now().UTC(), "")
}

func (m *CRProbeManager) updateAt(channel, state string, responseBytes int, responseAt *time.Time, probeAt time.Time, message string) CRProbeState {
	m.mu.Lock()
	defer m.mu.Unlock()
	current := m.states[channel]
	current.Channel = channel
	current.Enabled = m.options.Enabled
	current.State = state
	current.Method = "cr"
	current.LastProbeAt = &probeAt
	current.ResponseBytes = responseBytes
	current.Message = message
	if state == CRProbeStateResponded && responseAt != nil {
		current.LastResponseAt = responseAt
		current.ConsecutiveResponses++
		if current.ConsecutiveResponses > m.options.ConfirmResponses {
			current.ConsecutiveResponses = m.options.ConfirmResponses
		}
		current.ConsecutiveMisses = 0
		if current.ConsecutiveResponses >= m.options.ConfirmResponses {
			current.Up = true
			current.EverUp = true
			current.LastConfirmedAt = responseAt
			current.State = CRProbeStateResponded
			current.Message = "remote serial endpoint response confirmed"
		} else {
			current.State = CRProbeStateConfirming
			current.Message = "remote serial endpoint response is being confirmed"
		}
	} else if state == CRProbeStateNoResponse {
		current.ConsecutiveResponses = 0
		current.ConsecutiveMisses++
		if current.ConsecutiveMisses > m.options.DownMisses {
			current.ConsecutiveMisses = m.options.DownMisses
		}
		if current.ConsecutiveMisses >= m.options.DownMisses {
			current.Up = false
		}
	} else if state == CRProbeStateDisabled {
		current.Up = false
		current.ConsecutiveResponses = 0
		current.ConsecutiveMisses = 0
	}
	m.states[channel] = current
	return current
}

func drainCRProbeInput(ctx context.Context, lease *StreamLease) error {
	buffer := make([]byte, 256)
	for attempts := 0; attempts < 16; attempts++ {
		if err := ctx.Err(); err != nil {
			return err
		}
		pending, supported, err := pendingSerialInput(lease.device)
		if err != nil {
			return err
		}
		if !supported || pending <= 0 {
			return nil
		}
		readBuffer := buffer
		if pending < len(readBuffer) {
			readBuffer = readBuffer[:pending]
		}
		count, err := lease.Read(readBuffer)
		if count > 0 {
			continue
		}
		if err == nil || errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) {
			return nil
		}
		return err
	}
	return nil
}

func meaningfulCRResponse(payload []byte) bool {
	for _, value := range payload {
		switch value {
		case 0x00, '\r', '\n':
			continue
		default:
			return true
		}
	}
	return false
}

func waitForProbe(ctx context.Context, duration time.Duration) bool {
	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-timer.C:
		return true
	case <-ctx.Done():
		return false
	}
}