[BUG] Failed to bind queue to exchange

[gaoran10]

Error logs

2022-07-11T19:14:28,459+0000 [pulsar-web-34-8] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:28 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:30,196+0000 [pulsar-ph-amqp-42-1] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [public/vhost3/persistent/__amqp_exchange__exchange-3] Attempted to use a fenced managed ledger
2022-07-11T19:14:30,196+0000 [pulsar-ph-amqp-42-1] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - Error adding entry
org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException: java.lang.Exception: Attempted to use a fenced managed ledger
Caused by: java.lang.Exception: Attempted to use a fenced managed ledger
	at org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException.<init>(ManagedLedgerException.java:80) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.checkFenced(ManagedLedgerImpl.java:3582) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.asyncOpenCursor(ManagedLedgerImpl.java:908) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.openCursor(ManagedLedgerImpl.java:863) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.openCursor(ManagedLedgerImpl.java:849) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange.lambda$createCursorIfNotExists$2(PersistentExchange.java:307) ~[?:?]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap$Section.put(ConcurrentOpenHashMap.java:404) ~[io.streamnative-pulsar-common-2.10.1.1.jar:2.10.1.1]
	at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap.computeIfAbsent(ConcurrentOpenHashMap.java:238) ~[io.streamnative-pulsar-common-2.10.1.1.jar:2.10.1.1]
	at io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange.createCursorIfNotExists(PersistentExchange.java:300) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange.addQueue(PersistentExchange.java:259) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.AbstractAmqpQueue.bindExchange(AbstractAmqpQueue.java:97) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.impl.PersistentQueue.bindExchange(PersistentQueue.java:127) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.QueueServiceImpl.lambda$bind$5(QueueServiceImpl.java:264) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) ~[?:?]
	at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.QueueServiceImpl.bind(QueueServiceImpl.java:253) ~[?:?]
	at io.streamnative.pulsar.handlers.amqp.QueueServiceImpl.lambda$queueBind$2(QueueServiceImpl.java:143) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) [?:?]
	at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) [?:?]
	at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) [?:?]
	at io.streamnative.pulsar.handlers.amqp.QueueServiceImpl.queueBind(QueueServiceImpl.java:132) [GUCkhqZQsAFQF1UMvm01tw/:?]
	at io.streamnative.pulsar.handlers.amqp.AmqpChannel.receiveQueueBind(AmqpChannel.java:182) [GUCkhqZQsAFQF1UMvm01tw/:?]
	at org.apache.qpid.server.protocol.v0_8.transport.QueueBindBody.process(QueueBindBody.java:176) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.ServerDecoder.processMethod(ServerDecoder.java:163) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.AMQDecoder.processFrame(AMQDecoder.java:203) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.AMQDecoder.processInput(AMQDecoder.java:185) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.AMQDecoder.processAMQPFrames(AMQDecoder.java:138) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.AMQDecoder.decode(AMQDecoder.java:118) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at org.apache.qpid.server.protocol.v0_8.ServerDecoder.decodeBuffer(ServerDecoder.java:44) [qpid-broker-plugins-amqp-0-8-protocol-8.0.0.jar:8.0.0]
	at io.streamnative.pulsar.handlers.amqp.AmqpBrokerDecoder.decodeBuffer(AmqpBrokerDecoder.java:65) [GUCkhqZQsAFQF1UMvm01tw/:?]
	at io.streamnative.pulsar.handlers.amqp.AmqpConnection.channelRead(AmqpConnection.java:165) [GUCkhqZQsAFQF1UMvm01tw/:?]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.handler.flush.FlushConsolidationHandler.channelRead(FlushConsolidationHandler.java:152) [io.netty-netty-handler-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [io.netty-netty-handler-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) [io.netty-netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:487) [io.netty-netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) [io.netty-netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) [io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
	at java.lang.Thread.run(Thread.java:829) [?:?]
2022-07-11T19:14:30,198+0000 [pulsar-ph-amqp-42-1] ERROR io.streamnative.pulsar.handlers.amqp.impl.PersistentExchange - Error new cursor for topic persistent://public/vhost3/__amqp_exchange__exchange-3 - org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException: java.lang.Exception: Attempted to use a fenced managed ledger. will cause fetch data error.
2022-07-11T19:14:30,205+0000 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [public/vhost3/persistent/__amqp_exchange__exchange-3] Update managedLedger's properties failed
org.apache.bookkeeper.mledger.ManagedLedgerException$BadVersionException: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_exchange__exchange-3
2022-07-11T19:14:30,205+0000 [bookkeeper-ml-scheduler-OrderedScheduler-2-0] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [public/vhost3/persistent/__amqp_queue__topic-1] Update managedLedger's properties failed
org.apache.bookkeeper.mledger.ManagedLedgerException$BadVersionException: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_queue__topic-1
2022-07-11T19:14:30,205+0000 [bookkeeper-ml-scheduler-OrderedScheduler-2-0] ERROR io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils - [topic-1] update properties failed message: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_queue__topic-1, properties:{ROUTERS=[{"exchangeName":"exchange-2","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}},{"exchangeName":"exchange-3","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}},{"exchangeName":"vhost3","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}}], QUEUE=topic-1}
2022-07-11T19:14:30,205+0000 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] ERROR io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils - [exchange-3] update properties failed message: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_exchange__exchange-3, properties:{QUEUES=["topic-1"], EXCHANGE=exchange-3, TYPE=Fanout}
2022-07-11T19:14:30,427+0000 [pulsar-ph-amqp-42-1] WARN  org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://public/vhost3/__amqp_queue__topic-1] Failed to create subscription for defaultSubscription: ManagedLedger public/vhost3/persistent/__amqp_queue__topic-1 has already been closed
2022-07-11T19:14:31,032+0000 [pulsar-web-34-3] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:31 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:37,765+0000 [pulsar-web-34-13] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:37 +0000] "GET /metrics HTTP/1.1" 302 0 "-" "vm_promscrape" 1
2022-07-11T19:14:37,768+0000 [prometheus-stats-35-1] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:37 +0000] "GET /metrics/ HTTP/1.1" 200 36537 "-" "vm_promscrape" 2
2022-07-11T19:14:38,459+0000 [pulsar-web-34-11] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:38 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:41,032+0000 [pulsar-web-34-1] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:41 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:43,081+0000 [BookKeeperClientWorker-OrderedExecutor-0-0] ERROR io.streamnative.pulsar.handlers.amqp.AmqpEntryWriter - [persistent://public/vhost3/__amqp_exchange__exchange-3] Failed to write entry.
org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException: java.lang.Exception: Attempted to use a fenced managed ledger
Caused by: java.lang.Exception: Attempted to use a fenced managed ledger
	at org.apache.bookkeeper.mledger.ManagedLedgerException$ManagedLedgerFencedException.<init>(ManagedLedgerException.java:80) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.internalAsyncAddEntry(ManagedLedgerImpl.java:751) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.lambda$asyncAddEntry$3(ManagedLedgerImpl.java:725) ~[io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.mledger.util.SafeRun$1.safeRun(SafeRun.java:32) [io.streamnative-managed-ledger-2.10.1.1.jar:2.10.1.1]
	at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36) [org.apache.bookkeeper-bookkeeper-common-4.14.5.jar:4.14.5]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
	at java.lang.Thread.run(Thread.java:829) [?:?]
2022-07-11T19:14:48,459+0000 [pulsar-web-34-9] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:48 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:51,032+0000 [pulsar-web-34-16] INFO  org.eclipse.jetty.server.RequestLog - 127.0.0.6 - - [11/Jul/2022:19:14:51 +0000] "GET /status.html HTTP/1.1" 200 2 "-" "kube-probe/1.20+" 0
2022-07-11T19:14:51,420+0000 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [public/vhost3/persistent/__amqp_exchange__exchange-3] Update managedLedger's properties failed
org.apache.bookkeeper.mledger.ManagedLedgerException$BadVersionException: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_exchange__exchange-3
2022-07-11T19:14:51,420+0000 [bookkeeper-ml-scheduler-OrderedScheduler-2-0] ERROR org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl - [public/vhost3/persistent/__amqp_queue__topic-1] Update managedLedger's properties failed
org.apache.bookkeeper.mledger.ManagedLedgerException$BadVersionException: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_queue__topic-1
2022-07-11T19:14:51,420+0000 [bookkeeper-ml-scheduler-OrderedScheduler-4-0] ERROR io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils - [exchange-3] update properties failed message: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_exchange__exchange-3, properties:{QUEUES=["topic-1"], EXCHANGE=exchange-3, TYPE=Fanout}
2022-07-11T19:14:51,420+0000 [bookkeeper-ml-scheduler-OrderedScheduler-2-0] ERROR io.streamnative.pulsar.handlers.amqp.utils.PulsarTopicMetadataUtils - [topic-1] update properties failed message: org.apache.pulsar.metadata.api.MetadataStoreException$BadVersionException: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /managed-ledgers/public/vhost3/persistent/__amqp_queue__topic-1, properties:{ROUTERS=[{"exchangeName":"exchange-2","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}},{"exchangeName":"exchange-3","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}},{"exchangeName":"vhost3","type":"Fanout","bindingKeys":["topic-1"],"arguments":{}}], QUEUE=topic-1}
2022-07-11T19:14:51,651+0000 [pulsar-ph-amqp-42-3] WARN  org.apache.pulsar.broker.service.persistent.PersistentTopic - [persistent://public/vhost3/__amqp_queue__topic-1] Failed to create subscription for defaultSubscription: ManagedLedger public/vhost3/persistent/__amqp_queue__topic-1 has already been closed