查询报错远程调用接口异常，F12查看网络和标投和预览到是mgr的接口此错误
#根据报错提示：Error in execution; nested excepti on is io.lettuce.core.RedisCommandExecutionException: MISCONF Errors writing to the AOF file: No space left on device
#查看k8s的mgr的pod日志 [root@VM_1_74_centos conf]# kubectl logs -f --tail -n500 itsm-mgr-xxxxx ===2024-01-12 14:51:23.659 ERROR com.crc.crcloud.itsm.mgr.base.BaseController Line:61 http-nio-5020-exec-7 - Error in execution; nested exception is io.lettuce.core.RedisCommandExecutionException: MISCONF Errors writing to the AOF file: No space left on deviceorigin Exception（TraceId:001_1_905678887298781184）: org.springframework.data.redis.RedisSystemException: Error in execution; nested exception is io.lettuce.core.RedisCommandExecutionException: MISCONF Errors writing to the AOF file: No space left on device at org.springframework.data.redis.connection.lettuce.LettuceExceptionConverter.convert(LettuceExceptionConverter.java:54) at org.springframework.data.redis.connection.lettuce.LettuceExceptionConverter.convert(LettuceExceptionConverter.java:52) at org.springframework.data.redis.connection.lettuce.LettuceExceptionConverter.convert(LettuceExceptionConverter.java:41) at org.springframework.data.redis.PassThroughExceptionTranslationStrategy.translate(PassThroughExceptionTranslationStrategy.java:44) at org.springframework.data.redis.FallbackExceptionTranslationStrategy.translate(FallbackExceptionTranslationStrategy.java:42) at org.springframework.data.redis.connection.lettuce.LettuceConnection.convertLettuceAccessException(LettuceConnection.java:273) at org.springframework.data.redis.connection.lettuce.LettuceStringCommands.convertLettuceAccessException(LettuceStringCommands.java:799) at org.springframework.data.redis.connection.lettuce.LettuceStringCommands.set(LettuceStringCommands.java:180) at org.springframework.data.redis.connection.DefaultedRedisConnection.set(DefaultedRedisConnection.java:294) at org.springframework.data.redis.cache.DefaultRedisCacheWriter.lambda$put$0(DefaultRedisCacheWriter.java:90) at org.springframework.data.redis.cache.DefaultRedisCacheWriter.execute(DefaultRedisCacheWriter.java:242) at org.springframework.data.redis.cache.DefaultRedisCacheWriter.put(DefaultRedisCacheWriter.java:87) at org.springframework.data.redis.cache.RedisCache.put(RedisCache.java:150) at org.springframework.cache.transaction.TransactionAwareCacheDecorator.put(TransactionAwareCacheDecorator.java:105) at org.springframework.cache.interceptor.AbstractCacheInvoker.doPut(AbstractCacheInvoker.java:87) at org.springframework.cache.interceptor.CacheAspectSupport$CachePutRequest.apply(CacheAspectSupport.java:821) at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:430) at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:346) at org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:61) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691) at com.crc.crcloud.itsm.mgr.common.controller.CommonController$$EnhancerBySpringCGLIB$$7d8074f.findByDictTypeList(<generated>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105) at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878) at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792) at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1589) at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:748) Caused by: io.lettuce.core.RedisCommandExecutionException: MISCONF Errors writing to the AOF file: No space left on device #查看redis的连接 [root@VM_1_74_centos conf]# lsof -i:6379 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME java 16157 root 118u IPv6 499466215 0t0 TCP mysql.crcloud.com:51472->es.crcloud.com:6379 (ESTABLISHED) java 16157 root 121u IPv6 533277047 0t0 TCP mysql.crcloud.com:38704->redis.crcloud.com:6379 (ESTABLISHED) java 16188 root 173u IPv6 497879727 0t0 TCP mysql.crcloud.com:36704->es.crcloud.com:6379 (ESTABLISHED) java 16188 root 176u IPv6 497881098 0t0 TCP mysql.crcloud.com:47652->redis.crcloud.com:6379 (ESTABLISHED) java 16291 root 119u IPv6 497883199 0t0 TCP mysql.crcloud.com:36870->es.crcloud.com:6379 (ESTABLISHED) java 16291 root 121u IPv6 497881205 0t0 TCP mysql.crcloud.com:47828->redis.crcloud.com:6379 (ESTABLISHED) java 16515 root 62u IPv6 497868480 0t0 TCP mysql.crcloud.com:35968->es.crcloud.com:6379 (ESTABLISHED) java 16515 root 65u IPv6 497868486 0t0 TCP mysql.crcloud.com:46928->redis.crcloud.com:6379 (ESTABLISHED) java 16520 root 106u IPv6 497872237 0t0 TCP mysql.crcloud.com:47164->redis.crcloud.com:6379 (ESTABLISHED) java 18096 root 103u IPv6 498919212 0t0 TCP mysql.crcloud.com:47292->redis.crcloud.com:6379 (ESTABLISHED) java 18096 root 166u IPv6 505668048 0t0 TCP mysql.crcloud.com:39422->redis.crcloud.com:6379 (ESTABLISHED) #此时和redis的6379端口是通的 #ping redis的域名可以确定ip是10.169.0.17 [root@VM_1_74_centos conf]# ping redis.crcloud.com PING redis.crcloud.com (10.169.0.17) 56(84) bytes of data. #登录10.169.0.17的服务器查看redis [yxit@VM_0_17_centos ~]$ ps -ef|grep redis root 11390 1 0 2023 ? 08:30:30 /usr/local/bin/redis-server 0.0.0.0:6379 [cluster] root 11401 1 0 2023 ? 09:39:56 /usr/local/bin/redis-server 0.0.0.0:6389 [cluster] #查看磁盘发现磁盘满了 [yxit@VM_0_17_centos ~]$ df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 592K 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/vda1 50G 6.6G 41G 14% / /dev/mapper/vgdata-lvdata 150G 150G 20K 100% /data #存放数据的目录 tmpfs 783M 0 783M 0% /run/user/0 tmpfs 783M 0 783M 0% /run/user/1000 #进入data目录检查 [yxit@VM_0_17_centos 6379]$ cd /data/ [yxit@VM_0_17_centos data]$ du -sh * 142G mongodb 377M redis #查看mongo的日志已经到达138G [root@VM_0_17_centos ~]# cd /data/mongodb/mongos/log/ [root@VM_0_17_centos log]# du -sh mongos.log 138G mongos.log #清空日志 [root@VM_0_17_centos log]# >mongos.log #检查 [root@VM_0_17_centos log]# ls -lh /data/mongodb/mongos/log/ total 4.0K drwx------ 2 root root 4.0K Jan 5 20:34 mongos.diagnostic.data -rw------- 1 root root 0 Jan 12 15:11 mongos.log #df [root@VM_0_17_centos log]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 624K 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/vda1 50G 6.6G 41G 14% / /dev/mapper/vgdata-lvdata 150G 13G 138G 9% /data tmpfs 783M 0 783M 0% /run/user/0 tmpfs 783M 0 783M 0% /run/user/1000 #再次服务恢复正常