526互联

使用NLog通过Kafka实现日志收集

发布时间 2023-05-26 10:02:23作者: 广州大雄

使用NLog通过Kafka实现日志收集，最终在Kibana展示

 NuGet包引用

<PackageReference Include="NLog.Kafka" Version="0.2.1" />
<PackageReference Include="NLog.Web.AspNetCore" Version="4.14.0" />

 

Logstash配置

input {
    kafka {
       bootstrap_servers => "127.0.0.1:9092"
       group_id => "logstash"
       topics => "loges"
       codec => "json"
    }
}
output{
  elasticsearch {
        hosts => ["127.0.0.1:9002"]
        index => "log_{[appname]}_%{+YYYY.MM.dd}"
  }
}

项目NLog配置 nlog.config

<?xml version="1.0" encoding="utf-8" ?>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      autoReload="true"
      throwConfigExceptions="true"
      internalLogLevel="info"
      internalLogFile="App_Data/logs/internal-nlog.txt">
    <!--autoReload：修改后自动加载，可能会有延迟-->
    <!--throwConfigExceptions：NLog日志系统抛出异常-->
    <!--internalLogLevel：内部日志的级别-->
    <!--internalLogFile：内部日志保存路径，日志的内容大概就是NLog的版本信息，配置文件的地址等等-->
    <!--输出日志的配置，用于rules读取-->

    <!--扩展的程序集，官方文档：https://github.com/nlog/NLog/wiki/Extending%20NLog-->
    <extensions>
        <add assembly="NLog.Web.AspNetCore"/>
        <add assembly="WebApplication1"/>
        <add assembly="NLog.Kafka"/>
        <add assembly="Sentry.NLog"/>        
    </extensions>
    
    <!-- Layout布局
    ${var:basePath} basePath是前面自定义的变量
    ${longdate} 日期格式 2017-01-17 16:58:03.8667
    ${shortdate}日期格式 2017-01-17
    ${date:yyyyMMddHHmmssFFF} 日期 20170117165803866
    ${message} 输出内容
    ${guid} guid
    ${level}日志记录的等级
    ${logger} 配置的logger
    -->

    <!-- NLog记录等级
    Trace - 最常见的记录信息，一般用于普通输出
    Debug - 同样是记录信息，不过出现的频率要比Trace少一些，一般用来调试程序
    Info - 信息类型的消息
    Warn - 警告信息，一般用于比较重要的场合
    Error - 错误信息
    Fatal - 致命异常信息。一般来讲，发生致命异常之后程序将无法继续执行。
    自上而下，等级递增。
    -->
    
    <variable name="consoleLayout"  value="${longdate}|${event-properties:item=EventId_Id}|${uppercase:${level}}|${logger}|${message} ${exception:format=tostring}" />
    
    <variable name="fileLayout" value="${longdate}|${event-properties:item=EventId_Id}|${uppercase:${level}}|${logger}|${message} ${exception:format=tostring}|url: ${aspnet-request-url}|action: ${aspnet-mvc-action}" />

    <targets>
        <!--控制台日志-->
        <target name="console" xsi:type="ColoredConsole" layout="${consoleLayout}" />
        <!--文件日志，archive相关参数：文件拆分，archiveAboveSize="104857600"：每100M拆分一个新文件 -->
        <target name="logFile" xsi:type="File"  layout="${fileLayout}"  fileName="C:/logs/${shortdate}/${hostname}/${logName}${shortdate}.log"
                archiveEvery="Day"
                archiveFileName="C:/logs/${shortdate}/${hostname}/${logName}${shortdate}.{####}.log"
                archiveNumbering="DateAndSequence"
                archiveDateFormat="yyyy-MM-dd"
                archiveAboveSize="104857600"/>

        <target name="kafka" xsi:type="Kafka" topic="logs.mbridge.taskapi" bootstrapServers="192.168.100.100:9092">
            <property name="security.protocol" value="Plaintext" />
            <property name="sasl.mechanism" value="PLAIN" />
            <property name="acks" value="0" />

            <layout xsi:type="JsonLayout">
                <attribute name="Time" layout="${longdate}" />
                <attribute name="EventId" layout="${event-properties:item=EventId_Id}" />
                <attribute name="Level" layout="${uppercase:${level}}"/>
                <attribute name="Logger" layout="${logger}" />
                <attribute name="Hostname" layout="${hostname}" />
                <attribute name="LogName" layout="${logName}" />
                <attribute name="Message" layout="${message}" />
                <attribute name="Exception" layout="${exception:format=tostring}" />
            </layout>
        </target>
    </targets>
    
    <rules>
        <!--路由顺序会对日志打印产生影响。路由匹配逻辑为顺序匹配。-->
        
        <!--路由映射关系
        writeTo - 有writeTo才会写日志，没有此attr则忽略；writeTo的值对应<target>里面的 name="xxx"
        final - final="true"表示路由结束
        -->
        
        <!-- NLog等级使用
        指定特定等级 如：level="Warn"
        指定多个等级 如：levels=“Warn,Debug“ 以逗号隔开
        指定等级范围 如：minlevel="Warn" maxlevel="Error"
        -->

        <!--跳过microsoft的日志,不记录-->
        <logger name="Microsoft.AspNetCore.*" maxLevel="Warn" final="true" />
        <!--跳过EasyNetQ的日志,不记录-->
        <logger name="EasyNetQ.*" maxLevel="Warn" final="true" />

        <!--所有的日志, 包含以Microsoft开头的-->
        <logger name="*" maxlevel="Debug" writeTo="console" />
        <logger name="*" minlevel="Debug" writeTo="logFile" />
        <logger name="*" minlevel="Debug" writeTo="kafka" />
    </rules>
    
</nlog>

代码

 

    public class LogUtlExHelper
    {
        private static bool _kafka = false;
        private static bool _file = false;

        /// <summary>
        /// NLog扩展的Property
        /// </summary>
        public const string NLOG_PROPERTY_LOGNAME = "logName";
        /// <summary>
        /// NLog扩展的Property
        /// </summary>
        public const string NLOG_PROPERTY_HOSTNAME = "hostName";

        public const string NLOG_TARGET_NAME_LOGFILE = "logFile";
        public const string NLOG_TARGET_NAME_CONSOLE = "console";
        public const string NLOG_TARGET_NAME_KAFKA = "kafka";

        /// <summary>
        /// 文件日志的Logger，使用的name与NLog配置的target.name对应
        /// </summary>
        public static Logger fileLogger;
        /// <summary>
        /// 控制台日志的Logger，使用的name与NLog配置的target.name对应
        /// </summary>
        public static Logger consoleLogger;
        /// <summary>
        /// 控制台日志的Logger，使用的name与NLog配置的target.name对应
        /// </summary>
        public static Logger kafkaLogger;

        static LogUtlExHelper()
        {
            InitNLog();
        }

        public static void InitNLog()
        {
            fileLogger = LogManager.GetLogger(NLOG_TARGET_NAME_LOGFILE);
            consoleLogger = LogManager.GetLogger(NLOG_TARGET_NAME_CONSOLE);
            kafkaLogger = LogManager.GetLogger(NLOG_TARGET_NAME_KAFKA);

            SetNLogTargets();
        }

        public static void SetNLogTargets()
        {
            //Kafka
            var target = (NLog.Targets.Kafka.KafkaTarget)kafkaLogger.Factory.Configuration.FindTargetByName(NLOG_TARGET_NAME_KAFKA);
            if (target != null && !string.IsNullOrEmpty(target.BootstrapServers))
            {
                _kafka = true;
            }

            //File
            var targetFile = (NLog.Targets.FileTarget)kafkaLogger.Factory.Configuration.FindTargetByName(NLOG_TARGET_NAME_LOGFILE);
            if (targetFile != null && targetFile.FileName != null)
            {
                _file = true;
            }
        }

        #region File
        /// <summary>
        /// 添加日志
        /// </summary>
        /// <param name="fileName">日志文件名称</param>
        /// <param name="message">日志内容</param>
        /// <param name="folderName">日志文件夹名称</param>
        public static void Info(string fileName, string message, string folderName = "")
        {
            WriteLog(LogLevel.Info, message, fileName, folderName);
        }

        /// <summary>
        /// 添加警告日志
        /// </summary>
        /// <param name="fileName">日志文件名称</param>
        /// <param name="message">日志内容</param>
        /// <param name="folderName">日志文件夹名称</param>
        public static void Warn(string fileName, string message, string folderName = "")
        {
            WriteLog(LogLevel.Warn, message, fileName, folderName);
        }

        /// <summary>
        /// 添加异常日志
        /// </summary>
        /// <param name="exception"></param>
        /// <param name="message">日志内容</param>
        /// <param name="fileName">日志文件名称</param>
        /// <param name="folderName">日志文件夹名称</param>
        public static void Error(Exception exception, string message = "", string fileName = "", string folderName = "")
        {
            Error(message, fileName, folderName, exception);
        }

        /// <summary>
        /// 添加异常日志
        /// </summary>
        /// <param name="message">日志内容</param>
        /// <param name="fileName">日志文件名称</param>
        /// <param name="folderName">日志文件夹名称</param>
        /// <param name="exception"></param>
        public static void Error(string message, string fileName = "", string folderName = "", Exception exception = null)
        {
            if (string.IsNullOrEmpty(folderName))
            {
                folderName = "Error";
            }

            WriteLog(LogLevel.Error, message, fileName, folderName, exception);
        }

        /// <summary>
        /// 添加日志
        /// </summary>
        /// <param name="logLevel">日志等级</param>
        /// <param name="message">日志内容</param>
        /// <param name="fileName">日志文件名称</param>
        /// <param name="folderName">日志文件夹名称</param>
        /// <param name="exception"></param>
        public static void WriteLog(LogLevel logLevel, string message, string fileName = "", string folderName = "", Exception exception = null)
        {
            string logName = string.Empty;
            string loggerName = string.Empty;

            if (_kafka)
            {
                //Kafka
                loggerName = kafkaLogger.Name;
                if (!string.IsNullOrEmpty(folderName) && !string.IsNullOrEmpty(fileName))
                {
                    logName = $"{folderName}/{fileName}";
                }
                else if (!string.IsNullOrEmpty(folderName))
                {
                    logName = folderName ?? string.Empty;
                }
                else
                {
                    logName = fileName ?? string.Empty;
                }

                LogEventInfo theEvent = new LogEventInfo(logLevel, loggerName, message);
                theEvent.Properties[NLOG_PROPERTY_LOGNAME] = logName;
                theEvent.Properties[NLOG_PROPERTY_HOSTNAME] = Environment.MachineName;
                theEvent.Exception = exception;

                kafkaLogger.Log(theEvent);
            }

            if (_file)
            {
                //本地文件日志
                loggerName = fileLogger.Name;
                if (!string.IsNullOrEmpty(fileName))
                {
                    fileName = $"{fileName}_";
                }
                if (!string.IsNullOrEmpty(folderName))
                {
                    logName = $"{folderName}/{fileName}";
                }
                else
                {
                    logName = fileName ?? string.Empty;
                }

                LogEventInfo theEvent = new LogEventInfo(logLevel, loggerName, message);
                theEvent.Properties[NLOG_PROPERTY_LOGNAME] = logName;
                theEvent.Properties[NLOG_PROPERTY_HOSTNAME] = Environment.MachineName;
                theEvent.Exception = exception;

                fileLogger.Log(theEvent);
            }
        }
        #endregion

        #region Console
        /// <summary>
        /// 控制台日志
        /// </summary>
        /// <param name="message"></param>
        /// <param name="exception"></param>
        public static void ConsoleLog(string message, Exception exception = null)
        {
            ConsoleLog(LogLevel.Debug, message, exception);
        }
        /// <summary>
        /// 控制台日志
        /// </summary>
        /// <param name="logLevel"></param>
        /// <param name="message"></param>
        /// <param name="exception"></param>
        public static void ConsoleLog(LogLevel logLevel, string message, Exception exception = null)
        {
            LogEventInfo theEvent = new LogEventInfo(logLevel, consoleLogger.Name, message);
            //theEvent.Properties["EventId"] = new Microsoft.Extensions.Logging.EventId(2, "eventId2");
            theEvent.Exception = exception;
            consoleLogger.Log(theEvent);
        }
        #endregion
    }
/// <summary>
/// custom layout renderer
/// </summary>
[LayoutRenderer(LogUtlExHelper.NLOG_PROPERTY_LOGNAME)]
[ThreadSafe]
public class NLogLayoutRender : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (!logEvent.Properties.ContainsKey(LogUtlExHelper.NLOG_PROPERTY_LOGNAME))
        {
            return;
        }
        string logName = logEvent.Properties[LogUtlExHelper.NLOG_PROPERTY_LOGNAME] as string;
        if (!string.IsNullOrEmpty(logName))
        {
            builder.Append(logName);
        }
    }
}

/// <summary>
/// custom layout renderer
/// </summary>
[LayoutRenderer(LogUtlExHelper.NLOG_PROPERTY_HOSTNAME)]
[ThreadSafe]
public class NLogLayoutRender2 : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (!logEvent.Properties.ContainsKey(LogUtlExHelper.NLOG_PROPERTY_HOSTNAME))
        {
            return;
        }
        string hostName = logEvent.Properties[LogUtlExHelper.NLOG_PROPERTY_HOSTNAME] as string;
        if (!string.IsNullOrEmpty(hostName))
        {
            builder.Append(hostName);
        }
    }
}

 Kibana配置