【Redis】redis集群的 MASTER 与 SLAVE

发布时间 2023-07-05 21:03:34作者: aaacarrot

这个问题,从一个坑说起:
部署了一个新环境,但发现服务一直在报错:

2023-07-05 20:42:54.186 ERROR  --- [   scheduling-1] o.s.s.s.TaskUtils$LoggingErrorHandler    : Unexpected error occurred in scheduled task

org.redisson.client.RedisException: Unexpected exception while processing command
        at org.redisson.command.CommandAsyncService.convertException(CommandAsyncService.java:276) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.command.CommandAsyncService.get(CommandAsyncService.java:115) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.RedissonObject.get(RedissonObject.java:82) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.RedissonLock.unlock(RedissonLock.java:306) ~[redisson-3.17.6.jar!/:3.17.6]
        ......
        at jdk.internal.reflect.GeneratedMethodAccessor275.invoke(Unknown Source) ~[?:?]
        at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
        at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
        at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) ~[spring-context-6.0.4.jar!/:6.0.4]
        at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-6.0.4.jar!/:6.0.4]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) ~[?:?]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
        at java.lang.Thread.run(Thread.java:833) ~[?:?]
Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: None of slaves were synced
        at org.redisson.RedissonBaseLock.lambda$evalWriteAsync$0(RedissonBaseLock.java:225) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.command.CommandBatchService.lambda$executeAsync$7(CommandBatchService.java:322) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.command.RedisCommonBatchExecutor.handleResult(RedisCommonBatchExecutor.java:163) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.command.RedisExecutor.checkAttemptPromise(RedisExecutor.java:524) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.command.RedisExecutor.lambda$execute$4(RedisExecutor.java:176) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.client.handler.CommandDecoder.decodeCommandBatch(CommandDecoder.java:318) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decodeCommand(CommandDecoder.java:210) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decode(CommandDecoder.java:137) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decode(CommandDecoder.java:113) ~[redisson-3.17.6.jar!/:3.17.6]
        at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.handler.codec.ReplayingDecoder.callDecode(ReplayingDecoder.java:366) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:682) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:617) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:534) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:906) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        ... 1 more
Caused by: java.lang.IllegalStateException: None of slaves were synced
        at org.redisson.RedissonBaseLock.lambda$evalWriteAsync$0(RedissonBaseLock.java:225) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.command.CommandBatchService.lambda$executeAsync$7(CommandBatchService.java:322) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.command.RedisCommonBatchExecutor.handleResult(RedisCommonBatchExecutor.java:163) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.command.RedisExecutor.checkAttemptPromise(RedisExecutor.java:524) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.command.RedisExecutor.lambda$execute$4(RedisExecutor.java:176) ~[redisson-3.17.6.jar!/:3.17.6]
        at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
        at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
        at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
        at org.redisson.client.handler.CommandDecoder.decodeCommandBatch(CommandDecoder.java:318) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decodeCommand(CommandDecoder.java:210) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decode(CommandDecoder.java:137) ~[redisson-3.17.6.jar!/:3.17.6]
        at org.redisson.client.handler.CommandDecoder.decode(CommandDecoder.java:113) ~[redisson-3.17.6.jar!/:3.17.6]
        at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.handler.codec.ReplayingDecoder.callDecode(ReplayingDecoder.java:366) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:682) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:617) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:534) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:906) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-all-4.1.36.Final.jar!/:4.1.36.Final]
        ... 1 more

并且在不断地刷,不断地刷。

根本不知道什么原因。
本人百度能力不行,看到的帖子感觉没有一个对得上的。


然后,只能从0开始想原因,
为什么偏偏这个环境有问题?(新部署的环境)
代码是同一套代码,按道理不会是代码层面的问题了。
那是环境的问题?
是redis 集群的问题吗?还是哪里的配置问题呢?

如果说是 redis 集群的问题,我拿不出证据,因为运维同学已经证明主从同步是没问题了的。
如果说是配置的问题,茫茫这么多配置,究竟是哪个配置引发的呢?


再换个思路:
如果换一个redis集群能不能好呢?
从一个正常的环境里面拉配置过来,用这边的服务,读取那边的redis集群,会不会找到原因呢?

emmm, 如果一开始就这样干,早就发现问题了。

通过一对比,发现了一个事情:
application-redis.yml 的配置项里面,
有一个配置项:

    readMode: "SLAVE"
    subscriptionMode: "SLAVE"

但另外的环境配置的可都是:

    readMode: "MASTER"
    subscriptionMode: "MASTER"

诶?
会不会是这个原因呢?

再结合团队的大神给过的一个方向: redis 主从集群,3主3从的话,在干活的其实只有主节点,其实从节点连读取数据都不会提供服务,意思就是从节点只有在 master挂掉之后,它才顶上,它并不支持读写分离什么的。如果要满足读写分离,需要:先发一个readOnly的指令过来,从节点才能读。呃。。。。
还有开启密钥。。。

终于找到原因,就是这么一个坑,真的搞了好几天,一开始根本没往这个方面去想。(主要是真的对这个玩意不熟悉。)

记录一下。。。。