实现nginx+jenkins全链路数据追踪

1. 部署Jenkins

IP：10.0.0.94

1.1 安装、配置jenkins

# 1.安装jdk11
apt update
apt install -y openjdk-11-jdk

# 2.下载tomcat
mdkir /apps & cd /apps
wget https://dlcdn.apache.org/tomcat/tomcat-8/v8.5.93/bin/apache-tomcat-8.5.93.tar.gz
tar xvf apache-tomcat-8.5.93.tar.gz
ln -s /apps/apache-tomcat-8.5.93 /apps/apache-tomcat

# 3.下载skywalking java agent
mdkir /data & cd /data
wget https://archive.apache.org/dist/skywalking/java-agent/9.0.0/apache-skywalking-java-agent-9.0.0.tgz
tar xvf apache-skywalking-java-agent-9.0.0.tgz

# 4.修改tomcat javaagent配置参数
vim /apps/apache-tomcat/bin/catalina.sh
...
# 最开始出添加CATALINA_OPTS
# -----------------------------------------------------------------------------
CATALINA_OPTS="$CATALINA_OPTS -javaagent:/data/skywalking-agent/skywalking-agent.jar";export CATALINA_OPTS

# 5.配置skywalking agent service参数
vim /data/skywalking-agent/config/agent.config
...
agent.service_name=${SW_AGENT_NAME:jenkins-service}	# skywalking网页显示的服务名称
...
agent.namespace=${SW_AGENT_NAMESPACE:jenkins}		# namespace
...
# 后端服务即skywalking-oap平台地址，11800为grpc端口，12800为http端口
collector.backend_service=${SW_AGENT_COLLECTOR_BACKEND_SERVICES:10.0.0.91:11800}

# 6.下载Jenkins
wget https://mirrors.tuna.tsinghua.edu.cn/jenkins/war-stable/2.375.1/jenkins.war
cp jenkins.war /apps/apache-tomcat/webapps/

启动服务

/apps/apache-tomcat/bin/catalina.sh start

1.2 验证网页

登录首页

登录jenkins

# 查看jenkins密码
#cat /root/.jenkins/secrets/initialAdminPassword 
430a2752ac5241b9883024f3cc9c20c7

skywalking验证

2. 部署nginx

IP：10.0.0.93

2.1 安装编译环境

apt install -y iproute2 ntpdate tcpdump telnet traceroute \
nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev \
libpcre3 libpcre3-dev zlib1g-dev gcc make openssh-server iotop unzip zip

2.2 安装lua环境

2.2.1. 编译安装luajit

下载并编译安装

源码地址：https://github.com/openresty/luajit2/tags

wget https://github.com/openresty/luajit2/archive/refs/tags/v2.1-20220411.tar.gz
mv v2.1-20220411.tar.gz luajit2-2.1-20220411.tar.gz
tar xvf luajit2-2.1-20220411.tar.gz
cd luajit2-2.1-20220411/
make install PREFIX=/usr/local/luajit2-2.1

配置系统环境变量

编译安装nginx的时候使用

vim /etc/profile
# 末尾添加环境变量
export LUAJIT_LIB=/usr/local/luajit2-2.1/lib
export LUAJIT_INC=/usr/local/luajit2-2.1/include/luajit-2.1


# 环境变量生效
source /etc/profile

加载luajit2模块

nginx启动需要调用libluajit-5.1.so.2模块

vim /etc/ld.so.conf.d/libc.conf
/usr/local/lib
# 添加如下路径
/usr/local/luajit2-2.1/lib/

# 执行更新动态库
ldconfig

2.2.2. 编译安装lua核心库

nginx需要加载lua库

lua库地址：

https://github.com/openresty/lua-resty-core/tags

https://github.com/openresty/lua-resty-lrucache/tags

https://github.com/openresty/lua-cjson/tags

lua-resty-core

wget https://github.com/openresty/lua-resty-core/archive/refs/tags/v0.1.23.tar.gz
mv v0.1.23.tar.gz lua-resty-core-0.1.23.tar.gz
tar xvf lua-resty-core-0.1.23.tar.gz
cd lua-resty-core-0.1.23/
make install PREFIX=/usr/local/luacore

lua-resty-lrucache

wget https://github.com/openresty/lua-resty-lrucache/archive/refs/tags/v0.13.tar.gz
mv v0.13.tar.gz lua-resty-lrucache-0.13.tar.gz
tar xvf lua-resty-lrucache-0.13.tar.gz
cd lua-resty-lrucache-0.13/
make install PREFIX=/usr/local/luacore

lua-cjson

wget https://github.com/openresty/lua-cjson/archive/refs/tags/2.1.0.10.tar.gz
mv 2.1.0.10.tar.gz lua-cjson-2.1.0.10.tar.gz
tar xvf lua-cjson-2.1.0.10.tar.gz
cd lua-cjson-2.1.0.10/

# 修改Makefile第21行，路径为luajit模块路径
vim Makefile
21 LUA_INCLUDE_DIR =  /usr/local/luajit2-2.1/include/luajit-2.1

# 修改lua_cjson.c文件第1421行，去掉static
vim lua_cjson.c
...
1412 /* ===== INITIALISATION ===== */
1413 
1414 #if !defined(LUA_VERSION_NUM) || LUA_VERSION_NUM < 502
1415 /* Compatibility for Lua 5.1 and older LuaJIT.
1416  *
1417  * compat_luaL_setfuncs() is used to create a module table where the functions
1418  * have json_config_t as their first upvalue. Code borrowed from Lua 5.2
1419  * source's luaL_setfuncs().
1420  */
1421  void compat_luaL_setfuncs(lua_State *l, const luaL_Reg *reg, int nup)

# 执行make有警告无影响
make
make install

2.3 编译安装nginx

源码地址：https://github.com/vision5/ngx_devel_kit/tags

https://github.com/openresty/lua-nginx-module/tags

https://nginx.org/en/download.html

准备ngx_devel_kit源码

wget https://github.com/vision5/ngx_devel_kit/archive/refs/tags/v0.3.1.tar.gz
mv v0.3.1.tar.gz ngx_devel_kit-0.3.1.tar.gz
tar xvf ngx_devel_kit-0.3.1.tar.gz

准备lua-nginx-module源码

wget https://github.com/openresty/lua-nginx-module/archive/refs/tags/v0.10.21.tar.gz
mv v0.10.21.tar.gz lua-nginx-module-0.10.21.tar.gz
tar xvf lua-nginx-module-0.10.21.tar.gz

编译安装nginx

wget https://nginx.org/download/nginx-1.20.2.tar.gz
tar xvf nginx-1.20.2.tar.gz
cd nginx-1.20.2/
./configure --prefix=/apps/nginx \
--with-http_ssl_module \
--with-http_v2_module \
--with-http_realip_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-pcre \
--with-stream \
--with-stream_ssl_module \
--with-stream_realip_module \
--add-module=../ngx_devel_kit-0.3.1/ \		# 添加ngx_devel_kit模块
--add-module=../lua-nginx-module-0.10.21/	# 添加lua-nginx-module模块

# 执行编译
make && make install

2.4 验证lua解析

2.4.1 修改nginx配置

vim /apps/nginx/conf/nginx.conf
...
http {
    include       mime.types;
    default_type  application/octet-stream;
    lua_package_path "/usr/local/luacore/lib/lua/?.lua;;"; # 加载lua环境

...
    server {
...
        location / {
            root   html;
            index  index.html index.htm;
        }

		# 添加lua测试
        location /hello {  
            default_type   text/html;
              content_by_lua_block {
              ngx.say("Hello Lua!")
            }   
        }

语法测试

#/apps/nginx/sbin/nginx -t
nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /apps/nginx/conf/nginx.conf test is successful


# 启动nginx
/apps/nginx/sbin/nginx

‍

2.4.2 访问测试

curl测试

#curl 10.0.0.93/hello
Hello Lua!

浏览器验证

2.5 部署skywalking nginx agent

源码地址：

https://skywalking.apache.org/downloads/#NginxLUAAgent

https://github.com/apache/skywalking-nginx-lua/releases

https://github.com/openresty/lua-tablepool/tags

2.5.1 下载skywalking nginx agent

mkdir /data
cd /data
wget https://github.com/apache/skywalking-nginx-lua/archive/refs/tags/v0.6.0.tar.gz
#wget https://dlcdn.apache.org/skywalking/nginx-lua/0.6.0/skywalking-nginx-lua-0.6.0-src.tgz
mv v0.6.0.tar.gz skywalking-nginx-lua-0.6.0.tar.gz
tar xvf skywalking-nginx-lua-0.6.0.tar.gz
ln -s /data/skywalking-nginx-lua-0.6.0 /data/skywalking-nginx-lua

2.5.2 下载lua-tablepool模块

wget https://github.com/openresty/lua-tablepool/archive/refs/tags/v0.02.tar.gz
mv v0.02.tar.gz lua-tablepool-0.02.tar.gz
tar xvf lua-tablepool-0.02.tar.gz
cp lua-tablepool-0.02/lib/tablepool.lua /data/skywalking-nginx-lua/lib/

2.5.3 nginx.conf配置

添加skywalking nginx agent模块，添加连接skywalking oap配置

[root@nginx conf]#cat /apps/nginx/conf/nginx.conf
worker_processes  auto;
#daemon off;
#error_log /dev/stdout debug;

events {
    worker_connections 1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
	# 增加skywalking nginx lua agent路径
    lua_package_path "/usr/local/luacore/lib/lua/?.lua;/data/skywalking-nginx-lua/lib/?.lua;;";
    lua_shared_dict tracing_buffer 100m;
    sendfile        on;
    keepalive_timeout  65;

	# 配置连接skywalking oap
    init_worker_by_lua_block {
        local metadata_buffer = ngx.shared.tracing_buffer
		-- skywalking页面显示服务名
        metadata_buffer:set('serviceName', 'myserver-nginx')
		-- 实例名
        metadata_buffer:set('serviceInstanceName', 'myserver-nginx-node1')
        metadata_buffer:set('includeHostInEntrySpan', false)

        require("skywalking.util").set_randomseed()
		-- skywalking oap地址
        require("skywalking.client"):startBackendTimer("http://10.0.0.91:12800")

        skywalking_tracer = require("skywalking.tracer")
    }
    server {
        listen       80;
        server_name  localhost;
        location / {
            root   html;
            index  index.html index.htm;
        }
        location /hello {
          default_type	text/html;
	      content_by_lua_block {
	      ngx.say("Hello Lua!")
	      }
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
    include /apps/nginx/conf/conf.d/*.conf;		# 进行配置分离
}

myserver.conf

监听8080，转发至后端jenkins

[root@nginx conf]#cat /apps/nginx/conf/conf.d/myserver.conf
server {
    	listen 8080;
    	server_name www.myserver.com;				# www.myserver.com
    	location /jenkins {							# 访问jenkins跳转到后端jenkins服务器
        	default_type text/html;

        	rewrite_by_lua_block {
					-- 域名
            		skywalking_tracer:start("www.myserver.com")
        	}

        	proxy_pass http://10.0.0.94:8080/jenkins;	# 跳转到后端jenkins服务器

        	body_filter_by_lua_block {
            		if ngx.arg[2] then
                		skywalking_tracer:finish()
            		end
        	}

        	log_by_lua_block {
            		skywalking_tracer:prepareForReport()
        	}
    	}

    location / {				# 访问本地
     	default_type text/html;
    	root html;				# /apps/nginx/html
     	rewrite_by_lua_block {
         		skywalking_tracer:start("backend service")
     	}

     	body_filter_by_lua_block {
         		if ngx.arg[2] then
             		skywalking_tracer:finish()
         		end
     	}

     	log_by_lua_block {
         		skywalking_tracer:prepareForReport()
     	}
    }
}