amqp091-go PR #365: skip-and-continue topology recovery livelock repro

README.md

PR #365 repro: skip-and-continue topology recovery livelock

Standalone program used to verify a concern raised in review of rabbitmq/amqp091-go#365.

Setup

1. Start RabbitMQ: docker run --rm -d --name rabbitmq-verify -p 5672:5672 rabbitmq:4-management
2. Check out the PR branch somewhere, e.g.: git clone https://github.com/rabbitmq/amqp091-go.git /tmp/pr365 && cd /tmp/pr365 && git fetch origin pull/365/head:pr-365 && git checkout pr-365
3. In this directory, point the replace directive in go.mod at that checkout (edit the path if different from /tmp/pr365).
4. go run .

What it does

• Dials with Recovery enabled and OnTopologyEntityError returning true (skip-and-continue).
• Declares two durable queues (q1, q2) on one channel.
• From a second connection, redeclares both queues out-of-band with a conflicting x-max-length — a permanent, realistic topology conflict.
• Forces one soft error on the channel to kick off recovery, then watches how many times TopologyRecovery.RecoverTopology is invoked over ~30s via a counting wrapper (also tracks max concurrent invocations per channel, to check for a race between watchChannel's listener and the explicit reopenChannelIfClosed reopen path).

Result (see repro-run.log)

• Max concurrent RecoverTopology calls per channel: 1 — no literal race; go run -race reported nothing.
• But RecoverTopology keeps getting invoked again every ~5 seconds, indefinitely (7 calls observed in 30s, still climbing when the process was stopped), each time rediscovering and re-skipping the same two permanently conflicting queues. The 5s period matches notifyTimeout in channel.go (const notifyTimeout = 5 * time.Second, used in Channel.shutdown's full-notify-buffer fallback path).

go.mod

module verify365

go 1.21

require github.com/rabbitmq/amqp091-go v0.0.0

replace github.com/rabbitmq/amqp091-go => /tmp/pr365

main.go

package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"

	amqp "github.com/rabbitmq/amqp091-go"
)

// stdLogger adapts the standard library logger to amqp091.Logging with
// microsecond timestamps so we can see interleaving/overlap of log lines.
type stdLogger struct{ l *log.Logger }

func (s stdLogger) Printf(format string, v ...any) { s.l.Printf(format, v...) }

// countingTopologyRecovery wraps DefaultTopologyRecovery and tracks, per
// channel id, how many RecoverTopology calls are concurrently in flight.
// If maxConcurrent[chID] ever exceeds 1, that proves two independent
// recovery passes ran over the same channel's topology at the same time.
type countingTopologyRecovery struct {
	mu            sync.Mutex
	inFlight      map[*amqp.Channel]int
	maxConcurrent map[*amqp.Channel]int
	totalCalls    int32
}

func newCountingTopologyRecovery() *countingTopologyRecovery {
	return &countingTopologyRecovery{
		inFlight:      make(map[*amqp.Channel]int),
		maxConcurrent: make(map[*amqp.Channel]int),
	}
}

func (c *countingTopologyRecovery) RecoverTopology(conn *amqp.Connection, channels []*amqp.Channel) ([]amqp.TopologyRecoveryEntity, error) {
	call := atomic.AddInt32(&c.totalCalls, 1)
	c.mu.Lock()
	snapshot := make([]int, len(channels))
	for i, ch := range channels {
		c.inFlight[ch]++
		if c.inFlight[ch] > c.maxConcurrent[ch] {
			c.maxConcurrent[ch] = c.inFlight[ch]
		}
		snapshot[i] = c.inFlight[ch]
	}
	c.mu.Unlock()
	log.Printf(">>> RecoverTopology call #%d START channels=%v inFlightNow=%v", call, channels, snapshot)

	defer func() {
		c.mu.Lock()
		for _, ch := range channels {
			c.inFlight[ch]--
		}
		c.mu.Unlock()
		log.Printf("<<< RecoverTopology call #%d END channels=%v", call, channels)
	}()

	return (&amqp.DefaultTopologyRecovery{}).RecoverTopology(conn, channels)
}

func (c *countingTopologyRecovery) report() {
	c.mu.Lock()
	defer c.mu.Unlock()
	fmt.Println("=== RecoverTopology concurrency report ===")
	fmt.Printf("total calls: %d\n", c.totalCalls)
	for ch, max := range c.maxConcurrent {
		fmt.Printf("channel %p: max concurrent RecoverTopology calls = %d\n", ch, max)
		if max > 1 {
			fmt.Printf("  ^^^ RE-ENTRANCY CONFIRMED: channel %p had %d overlapping topology-recovery passes\n", ch, max)
		}
	}
}

const url = "amqp://guest:guest@localhost:5672/"

func must(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %v", msg, err)
	}
}

func main() {
	log.SetFlags(log.Ldate | log.Lmicroseconds)
	amqp.SetLogger(stdLogger{l: log.New(os.Stdout, "[amqp] ", log.Lmicroseconds)})

	counter := newCountingTopologyRecovery()

	conn, err := amqp.DialConfig(url, amqp.Config{
		Recovery: &amqp.Recovery{
			ReconnectionConfig: &amqp.ReconnectionConfig{
				MaxRetryCount: 10,
				RetryInterval: 100 * time.Millisecond,
			},
			TopologyRecoveryMode: amqp.TopologyRecoveryAllEnabled,
			TopologyRecovery:     counter,
			OnTopologyEntityError: func(_ *amqp.Connection, e amqp.TopologyRecoveryEntity) bool {
				log.Printf("OnTopologyEntityError: skipping %s %q on channel %d: %v", e.EntityType, e.EntityName, e.ChannelID, e.Err)
				return true // skip-and-continue
			},
		},
	})
	must(err, "DialConfig")
	defer conn.Close()

	ch1, err := conn.Channel()
	must(err, "conn.Channel")
	defer ch1.Close()

	// Two queues on the SAME channel, both durable with an arg, so we can
	// corrupt both out-of-band and force two entity failures within a
	// single RecoverTopology pass for this one channel.
	q1, q2 := "verify365_q1", "verify365_q2"
	_, _ = ch1.QueueDelete(q1, false, false, false)
	_, _ = ch1.QueueDelete(q2, false, false, false)

	_, err = ch1.QueueDeclare(q1, true, false, false, false, amqp.Table{"x-max-length": int32(10)})
	must(err, "QueueDeclare q1")
	_, err = ch1.QueueDeclare(q2, true, false, false, false, amqp.Table{"x-max-length": int32(10)})
	must(err, "QueueDeclare q2")

	log.Printf("channel ptr = %p", ch1)

	// Corrupt q1 and q2 out-of-band via a second connection so that when
	// ch1's own recovery pass tries to redeclare them (durable=true), the
	// broker responds with PRECONDITION_FAILED for both.
	admin, err := amqp.Dial(url)
	must(err, "admin Dial")
	adminCh, err := admin.Channel()
	must(err, "admin Channel")

	for _, q := range []string{q1, q2} {
		_, err = adminCh.QueueDelete(q, false, false, false)
		must(err, "admin QueueDelete "+q)
		// Redeclare durable (required by broker for non-exclusive queues) but with
		// a different x-max-length arg -> PRECONDITION_FAILED on ch1's redeclare.
		_, err = adminCh.QueueDeclare(q, true, false, false, false, amqp.Table{"x-max-length": int32(99)})
		must(err, "admin QueueDeclare(conflicting) "+q)
	}

	// Now trigger a channel-level soft error UNRELATED to q1/q2 to force
	// ch1 to close and go through Channel.Reconnect -> RecoverTopology,
	// which will hit the q1/q2 conflicts we just planted.
	q3 := "verify365_q3_trigger"
	_, _ = adminCh.QueueDelete(q3, false, false, false)
	_, err = adminCh.QueueDeclare(q3, true, false, false, false, nil)
	must(err, "admin QueueDeclare q3")

	log.Println("--- triggering channel-level soft error on ch1 ---")
	_, err = ch1.QueueDeclare(q3, true, true /* mismatched auto-delete -> 406 */, false, false, nil)
	if err == nil {
		log.Println("WARNING: expected a PRECONDITION_FAILED error on ch1 but got none")
	} else {
		log.Printf("ch1.QueueDeclare(q3) failed as expected: %v", err)
	}

	// Give the async recovery machinery (watchChannel goroutine + the
	// explicit reopen-and-continue path inside RecoverTopology) time to run,
	// and watch whether the number of RecoverTopology calls keeps climbing
	// (a self-sustaining loop) or stabilizes.
	for i := 0; i < 6; i++ {
		time.Sleep(5 * time.Second)
		n := atomic.LoadInt32(&counter.totalCalls)
		log.Printf("=== checkpoint %d: totalCalls=%d ===", i, n)
	}

	counter.report()

	_, _ = adminCh.QueueDelete(q1, false, false, false)
	_, _ = adminCh.QueueDelete(q2, false, false, false)
	_, _ = adminCh.QueueDelete(q3, false, false, false)
	_ = adminCh.Close()
	_ = admin.Close()
}

repro-run.log

2026/07/15 13:36:20.438065 channel ptr = 0xc00016e000
2026/07/15 13:36:20.461377 --- triggering channel-level soft error on ch1 ---
[amqp] 13:36:20.462867 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'auto_delete' for queue 'verify365_q3_trigger' in vhost '/': received 'true' but current is 'false'"
[amqp] 13:36:20.463259 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'auto_delete' for queue 'verify365_q3_trigger' in vhost '/': received 'true' but current is 'false'"
[amqp] 13:36:20.463343 Initiating channel 1 recovery
[amqp] 13:36:20.463385 Channel 1 recovery attempt 1 of 10
2026/07/15 13:36:20.463684 ch1.QueueDeclare(q3) failed as expected: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'auto_delete' for queue 'verify365_q3_trigger' in vhost '/': received 'true' but current is 'false'"
2026/07/15 13:36:20.464473 >>> RecoverTopology call #1 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:20.465850 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:20.465977 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.466069 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
[amqp] 13:36:20.468590 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:20.468742 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.468852 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:20.469967 <<< RecoverTopology call #1 END channels=[0xc00016e000]
[amqp] 13:36:20.470066 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.470202 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.470320 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.470398 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.470489 Initiating channel 1 recovery
2026/07/15 13:36:20.470531 >>> RecoverTopology call #2 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:20.472306 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:20.472394 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:20.472473 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:25.467108 === checkpoint 0: totalCalls=2 ===
[amqp] 13:36:25.472958 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:25.473329 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.473435 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:25.488113 <<< RecoverTopology call #2 END channels=[0xc00016e000]
[amqp] 13:36:25.488363 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.488584 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.488881 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.489065 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.489143 Initiating channel 1 recovery
2026/07/15 13:36:25.489173 >>> RecoverTopology call #3 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:25.493289 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:25.493483 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:25.494393 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:30.471468 === checkpoint 1: totalCalls=3 ===
[amqp] 13:36:30.493709 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:30.493885 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.493962 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:30.496350 <<< RecoverTopology call #3 END channels=[0xc00016e000]
[amqp] 13:36:30.496454 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.496554 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.496622 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.496676 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.496723 Initiating channel 1 recovery
2026/07/15 13:36:30.496751 >>> RecoverTopology call #4 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:30.499441 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:30.499595 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:30.499655 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:35.472808 === checkpoint 2: totalCalls=4 ===
[amqp] 13:36:35.511672 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:35.514979 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.515276 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:35.518756 <<< RecoverTopology call #4 END channels=[0xc00016e000]
[amqp] 13:36:35.518849 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.518913 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.518967 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.518996 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.519027 Initiating channel 1 recovery
2026/07/15 13:36:35.519043 >>> RecoverTopology call #5 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:35.521034 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:35.521114 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:35.521165 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:40.474384 === checkpoint 3: totalCalls=5 ===
[amqp] 13:36:40.521700 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:40.522111 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.522250 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:40.528798 <<< RecoverTopology call #5 END channels=[0xc00016e000]
[amqp] 13:36:40.528952 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.529083 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.529195 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.529282 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.529339 Initiating channel 1 recovery
2026/07/15 13:36:40.529363 >>> RecoverTopology call #6 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:40.532030 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:40.532131 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:40.532208 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:45.475589 === checkpoint 4: totalCalls=6 ===
[amqp] 13:36:45.534140 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:45.534479 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.534593 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:45.537650 <<< RecoverTopology call #6 END channels=[0xc00016e000]
[amqp] 13:36:45.537803 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.537954 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.538024 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.538088 Channel 1 closed with error: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.538143 Initiating channel 1 recovery
2026/07/15 13:36:45.538180 >>> RecoverTopology call #7 START channels=[0xc00016e000] inFlightNow=[1]
[amqp] 13:36:45.540392 failed to recover queue verify365_q1 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:45.540559 OnTopologyEntityError: skipping queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:45.540626 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
2026/07/15 13:36:50.478772 === checkpoint 5: totalCalls=7 ===
=== RecoverTopology concurrency report ===
total calls: 7
channel 0xc00016e000: max concurrent RecoverTopology calls = 1
[amqp] 13:36:50.540708 failed to recover queue verify365_q2 on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
2026/07/15 13:36:50.540934 OnTopologyEntityError: skipping queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:50.541015 topology recovery: channel 1 closed by broker soft error; reopening for remaining entities
[amqp] 13:36:55.542910 topology recovery: failed to reopen channel 1: Exception (504) Reason: "channel/connection is not open"
2026/07/15 13:36:55.542994 <<< RecoverTopology call #7 END channels=[0xc00016e000]
[amqp] 13:36:55.543016 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q1" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q1' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:55.543060 Channel 1 topology recovery skipped entity: topology recovery: queue "verify365_q2" on channel 1: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"
[amqp] 13:36:55.543089 Channel 1 closed unexpectedly: Exception (406) Reason: "PRECONDITION_FAILED - inequivalent arg 'x-max-length' for queue 'verify365_q2' in vhost '/': received '10' but current is '99'"