package serial import ( "context" "errors" "fmt" "strings" "sync" "syscall" "time" ) const ( CRProbeStateDisabled = "disabled" CRProbeStatePending = "pending" CRProbeStateConfirming = "confirming" CRProbeStateResponded = "responded" CRProbeStateNoResponse = "no_response" CRProbeStateBusy = "busy" CRProbeStateSuppressed = "suppressed" CRProbeStateError = "error" ) // CRProbeState is a cached observation. A response proves that a remote serial // endpoint answered; no_response does not prove that the cable is absent. type CRProbeState struct { Channel string `json:"channel"` Enabled bool `json:"enabled"` State string `json:"state"` Method string `json:"method"` LastProbeAt *time.Time `json:"last_probe_at,omitempty"` LastResponseAt *time.Time `json:"last_response_at,omitempty"` LastConfirmedAt *time.Time `json:"last_confirmed_at,omitempty"` ResponseBytes int `json:"response_bytes,omitempty"` ConsecutiveResponses int `json:"consecutive_responses,omitempty"` ConsecutiveMisses int `json:"consecutive_misses,omitempty"` Up bool `json:"up"` EverUp bool `json:"ever_up"` Message string `json:"message,omitempty"` } type CRProbeStateController interface { CRProbeStates(context.Context) ([]CRProbeState, error) } // CRProbeGroup contains channels backed by one PCI function/IRQ. Channels in // the same group are always probed serially; separate groups may run in // parallel without overlapping sweeps. type CRProbeGroup struct { ID string Channels []string } type CRProbeOptions struct { Enabled bool ResponseTimeout time.Duration SweepInterval time.Duration BetweenChannels time.Duration QuietPeriod time.Duration StartupDelay time.Duration Workers int Groups []CRProbeGroup ConfirmResponses int DownMisses int Now func() time.Time } // CRProbeManager sends one CR byte only while it owns the same exclusive lease // used by normal sessions. Results are cached so API polling never touches UARTs. type CRProbeManager struct { manager *StreamLeaseManager options CRProbeOptions mu sync.Mutex states map[string]CRProbeState groups []CRProbeGroup } func NewCRProbeManager(manager *StreamLeaseManager, options CRProbeOptions) (*CRProbeManager, error) { if manager == nil { return nil, errors.New("serial stream manager is required") } if options.ResponseTimeout <= 0 { options.ResponseTimeout = 250 * time.Millisecond } if options.ResponseTimeout < 50*time.Millisecond { options.ResponseTimeout = 50 * time.Millisecond } if options.ResponseTimeout > 5*time.Second { options.ResponseTimeout = 5 * time.Second } if options.SweepInterval <= 0 { options.SweepInterval = 2 * time.Second } if options.BetweenChannels <= 0 { options.BetweenChannels = 25 * time.Millisecond } if options.QuietPeriod <= 0 { options.QuietPeriod = 5 * time.Second } if options.StartupDelay < 0 { options.StartupDelay = 0 } if options.StartupDelay > 5*time.Minute { options.StartupDelay = 5 * time.Minute } if options.Workers <= 0 { options.Workers = 1 } if options.Workers > 16 { options.Workers = 16 } if options.ConfirmResponses <= 0 { options.ConfirmResponses = 1 } if options.ConfirmResponses > 5 { options.ConfirmResponses = 5 } if options.DownMisses <= 0 { options.DownMisses = 2 } if options.DownMisses > 10 { options.DownMisses = 10 } if options.Now == nil { options.Now = time.Now } groups, err := normalizeCRProbeGroups(options.Groups, options.Workers) if err != nil { return nil, err } probe := &CRProbeManager{ manager: manager, options: options, states: make(map[string]CRProbeState, ChannelCount), groups: groups, } for index := 1; index <= ChannelCount; index++ { channel := fmt.Sprintf("CH%02d", index) state := CRProbeStatePending message := "CR response probe is waiting" if !options.Enabled { state = CRProbeStateDisabled message = "CR response probe is disabled" } probe.states[channel] = CRProbeState{ Channel: channel, Enabled: options.Enabled, State: state, Method: "cr", Message: message, } } return probe, nil } func (m *CRProbeManager) Run(ctx context.Context) { if m == nil || !m.options.Enabled { return } if !waitForProbe(ctx, m.options.StartupDelay) { return } for { sweepStarted := time.Now() var groups sync.WaitGroup for _, configuredGroup := range m.groups { group := configuredGroup groups.Add(1) go func() { defer groups.Done() var previousCompletedAt time.Time for index, channel := range group.Channels { if ctx.Err() != nil { return } target := sweepStarted.Add(crProbeOffset(m.options.SweepInterval, len(group.Channels), index)) if !previousCompletedAt.IsZero() { spacedTarget := previousCompletedAt.Add(m.options.BetweenChannels) if spacedTarget.After(target) { target = spacedTarget } } if !waitForProbe(ctx, time.Until(target)) { return } state, probeErr := m.ProbeOnce(ctx, channel) if probeErr == nil && shouldConfirmCRProbeMiss(state, m.options.DownMisses) { // A formerly confirmed link is not declared DOWN from one noisy read. // Confirm the miss immediately so the two-sample decision still fits // inside the four-second sweep for the normal single-link case. if !waitForProbe(ctx, m.options.BetweenChannels) { return } _, _ = m.ProbeOnce(ctx, channel) } previousCompletedAt = time.Now() } }() } groups.Wait() remaining := m.options.SweepInterval - time.Since(sweepStarted) if remaining < 0 { remaining = 0 } if !waitForProbe(ctx, remaining) { return } } } func crProbeOffset(sweepInterval time.Duration, channelCount, index int) time.Duration { if sweepInterval <= 0 || channelCount <= 1 || index <= 0 { return 0 } if index >= channelCount { index = channelCount - 1 } return time.Duration(index) * sweepInterval / time.Duration(channelCount) } func shouldConfirmCRProbeMiss(state CRProbeState, downMisses int) bool { return downMisses > 1 && state.EverUp && state.Up && state.State == CRProbeStateNoResponse && state.ConsecutiveMisses > 0 && state.ConsecutiveMisses < downMisses } func normalizeCRProbeGroups(configured []CRProbeGroup, workers int) ([]CRProbeGroup, error) { groups := configured if len(groups) == 0 { if workers < 1 { workers = 1 } if workers > ChannelCount { workers = ChannelCount } groups = make([]CRProbeGroup, workers) for index := range groups { groups[index].ID = fmt.Sprintf("worker-%02d", index+1) } for index := 1; index <= ChannelCount; index++ { group := (index - 1) % workers groups[group].Channels = append(groups[group].Channels, fmt.Sprintf("CH%02d", index)) } } result := make([]CRProbeGroup, 0, len(groups)) seenGroups := make(map[string]bool, len(groups)) seenChannels := make(map[string]bool, ChannelCount) for _, configuredGroup := range groups { id := strings.TrimSpace(configuredGroup.ID) if id == "" || seenGroups[id] { return nil, errors.New("CR probe groups require unique non-empty IDs") } if len(configuredGroup.Channels) == 0 { return nil, fmt.Errorf("CR probe group %s must contain channels", id) } seenGroups[id] = true group := CRProbeGroup{ID: id, Channels: make([]string, 0, len(configuredGroup.Channels))} for _, configuredChannel := range configuredGroup.Channels { channel, err := normalizeChannel(configuredChannel) if err != nil || channel != configuredChannel { return nil, fmt.Errorf("CR probe group %s contains an invalid channel", id) } if seenChannels[channel] { return nil, fmt.Errorf("CR probe groups contain duplicate %s", channel) } seenChannels[channel] = true group.Channels = append(group.Channels, channel) } result = append(result, group) } if len(seenChannels) != ChannelCount { return nil, fmt.Errorf("CR probe groups must contain exactly %d channels", ChannelCount) } for index := 1; index <= ChannelCount; index++ { channel := fmt.Sprintf("CH%02d", index) if !seenChannels[channel] { return nil, fmt.Errorf("CR probe groups are missing %s", channel) } } return result, nil } func (m *CRProbeManager) CRProbeStates(ctx context.Context) ([]CRProbeState, error) { if err := ctx.Err(); err != nil { return nil, err } m.mu.Lock() defer m.mu.Unlock() states := make([]CRProbeState, 0, ChannelCount) for index := 1; index <= ChannelCount; index++ { states = append(states, m.states[fmt.Sprintf("CH%02d", index)]) } return states, nil } func (m *CRProbeManager) ProbeOnce(ctx context.Context, channel string) (CRProbeState, error) { canonical, err := normalizeChannel(channel) if err != nil || canonical != channel { return CRProbeState{}, errors.New("CR probe channel must be canonical CH01 through CH48") } if !m.options.Enabled { return m.update(channel, CRProbeStateDisabled, 0, nil), nil } now := m.options.Now().UTC() if m.manager.RecentlyActive(channel, m.options.QuietPeriod, now) { return m.updateAt(channel, CRProbeStateSuppressed, 0, nil, now, "recent serial traffic; probe skipped"), nil } lease, err := m.manager.Acquire(ctx, channel, CRProbeClientID) if err != nil { if errors.Is(err, ErrStreamLeaseBusy) { return m.updateAt(channel, CRProbeStateBusy, 0, nil, now, "serial channel is in use; probe skipped"), nil } state := m.updateAt(channel, CRProbeStateError, 0, nil, now, err.Error()) return state, err } defer lease.Close() if err := drainCRProbeInput(ctx, lease); err != nil { state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe input drain failed") return state, err } if _, err := lease.Write([]byte{'\r'}); err != nil { state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe write failed") return state, err } type readResult struct { bytes int err error } resultChannel := make(chan readResult, 1) go func() { buffer := make([]byte, 256) responseBytes := 0 for { readBuffer := buffer pending, queueSupported, queueErr := pendingSerialInput(lease.device) if queueErr != nil { resultChannel <- readResult{err: queueErr} return } if queueSupported { if pending <= 0 { select { case <-ctx.Done(): resultChannel <- readResult{err: ctx.Err()} return case <-time.After(20 * time.Millisecond): continue } } if pending < len(readBuffer) { readBuffer = readBuffer[:pending] } } count, readErr := lease.Read(readBuffer) if count > 0 { responseBytes += count if meaningfulCRResponse(readBuffer[:count]) { resultChannel <- readResult{bytes: responseBytes} return } continue } if readErr == nil || errors.Is(readErr, syscall.EAGAIN) || errors.Is(readErr, syscall.EWOULDBLOCK) { select { case <-ctx.Done(): resultChannel <- readResult{err: ctx.Err()} return case <-time.After(20 * time.Millisecond): continue } } resultChannel <- readResult{err: readErr} return } }() timer := time.NewTimer(m.options.ResponseTimeout) defer timer.Stop() select { case result := <-resultChannel: if result.bytes > 0 { responseAt := m.options.Now().UTC() return m.updateAt(channel, CRProbeStateResponded, result.bytes, &responseAt, now, "remote serial endpoint responded to CR"), nil } if result.err != nil && !errors.Is(result.err, context.Canceled) && !errors.Is(result.err, context.DeadlineExceeded) { state := m.updateAt(channel, CRProbeStateError, 0, nil, now, "CR probe read failed") return state, result.err } return m.updateAt(channel, CRProbeStateNoResponse, 0, nil, now, "no CR response; cable state is indeterminate"), nil case <-timer.C: _ = lease.Close() return m.updateAt(channel, CRProbeStateNoResponse, 0, nil, now, "no CR response; cable state is indeterminate"), nil case <-ctx.Done(): _ = lease.Close() return CRProbeState{}, ctx.Err() } } func (m *CRProbeManager) update(channel, state string, responseBytes int, responseAt *time.Time) CRProbeState { return m.updateAt(channel, state, responseBytes, responseAt, m.options.Now().UTC(), "") } func (m *CRProbeManager) updateAt(channel, state string, responseBytes int, responseAt *time.Time, probeAt time.Time, message string) CRProbeState { m.mu.Lock() defer m.mu.Unlock() current := m.states[channel] current.Channel = channel current.Enabled = m.options.Enabled current.State = state current.Method = "cr" current.LastProbeAt = &probeAt current.ResponseBytes = responseBytes current.Message = message if state == CRProbeStateResponded && responseAt != nil { current.LastResponseAt = responseAt current.ConsecutiveResponses++ if current.ConsecutiveResponses > m.options.ConfirmResponses { current.ConsecutiveResponses = m.options.ConfirmResponses } current.ConsecutiveMisses = 0 if current.ConsecutiveResponses >= m.options.ConfirmResponses { current.Up = true current.EverUp = true current.LastConfirmedAt = responseAt current.State = CRProbeStateResponded current.Message = "remote serial endpoint response confirmed" } else { current.State = CRProbeStateConfirming current.Message = "remote serial endpoint response is being confirmed" } } else if state == CRProbeStateNoResponse { current.ConsecutiveResponses = 0 current.ConsecutiveMisses++ if current.ConsecutiveMisses > m.options.DownMisses { current.ConsecutiveMisses = m.options.DownMisses } if current.ConsecutiveMisses >= m.options.DownMisses { current.Up = false } } else if state == CRProbeStateDisabled { current.Up = false current.ConsecutiveResponses = 0 current.ConsecutiveMisses = 0 } m.states[channel] = current return current } func drainCRProbeInput(ctx context.Context, lease *StreamLease) error { buffer := make([]byte, 256) for attempts := 0; attempts < 16; attempts++ { if err := ctx.Err(); err != nil { return err } pending, supported, err := pendingSerialInput(lease.device) if err != nil { return err } if !supported || pending <= 0 { return nil } readBuffer := buffer if pending < len(readBuffer) { readBuffer = readBuffer[:pending] } count, err := lease.Read(readBuffer) if count > 0 { continue } if err == nil || errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { return nil } return err } return nil } func meaningfulCRResponse(payload []byte) bool { for _, value := range payload { switch value { case 0x00, '\r', '\n': continue default: return true } } return false } func waitForProbe(ctx context.Context, duration time.Duration) bool { timer := time.NewTimer(duration) defer timer.Stop() select { case <-timer.C: return true case <-ctx.Done(): return false } }