526互联

架构第三次作业-20230723

发布时间 2023-07-31 14:30:49作者: wuhaolam
1、安装containerd及CNI组件
2、使用基于nerdctl创建并管理容器
3、部署https的harbor服务器、
4、基于Alpine和Ubuntu作为基础镜像实现的业务镜像构建
5、基于docker-compose单机编排运行Nginx+Java APP+MySQL服务

扩展：
    1、对GPU服务器安装驱动、nvidia-container-toolkit
    2、创建容器并验证对GPU的资源限制
    3、基于nginx反代harbor并实现https

一、安装containerd及CNI组件

二进制安装

1、准备containerd二进制文件与service文件

# 下载文件
root@ubuntu20-server2-112:~# wget https://github.com/containerd/containerd/releases/download/v1.7.2/containerd-1.7.2-linux-amd64.tar.gz
root@ubuntu20-server2-112:~# ls
containerd-1.7.2-linux-amd64.tar.gz

# 解压文件
root@ubuntu20-server2-112:~# tar xvf containerd-1.7.2-linux-amd64.tar.gz -C /usr/local/src/
root@ubuntu20-server2-112:~# cd /usr/local/src/
root@ubuntu20-server2-112:/usr/local/src# cp bin/* /usr/local/bin/

# 验证结果
root@ubuntu20-server2-112:~# containerd -v
containerd github.com/containerd/containerd v1.7.2 0cae528dd6cb557f7201036e9f43420650207b58

# 创建service文件
root@ubuntu20-server2-112:~# vim /lib/systemd/system/containerd.service
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

2、编辑配置文件

root@ubuntu20-server2-112:~# mkdir /etc/containerd
root@ubuntu20-server2-112:~# containerd config default > /etc/containerd/config.toml
root@ubuntu20-server2-112:~# vim /etc/containerd/config.toml
# 修改成国内的基础镜像
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
# 配置镜像加速
168       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
169         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
170           endpoint = ["https://frncu3gx.mirror.aliyuncs.com"]
root@ubuntu20-server2-112:~# systemctl restart containerd.service 
root@ubuntu20-server2-112:~# systemctl enable containerd.service
root@ubuntu20-server2-112:~# systemctl status containerd.service 
● containerd.service - containerd container runtime
     Loaded: loaded (/lib/systemd/system/containerd.service; enabled; vendor preset: enabled)
     Active: active (running) since Tue 2023-07-25 14:32:10 UTC; 41s ago
       Docs: https://containerd.io
   Main PID: 2728 (containerd)
      Tasks: 8
     Memory: 15.0M
     CGroup: /system.slice/containerd.service
             └─2728 /usr/local/bin/containerd

3、部署runc

# 准备runc文件
# 下载地址: https://github.com/opencontainers/runc/releases/download/v1.1.8/runc.amd64
root@ubuntu20-server2-112:~# ls
containerd-1.7.2-linux-amd64.tar.gz  runc.amd64
root@ubuntu20-server2-112:~# chmod +x runc.amd64
root@ubuntu20-server2-112:~# mv runc.amd64 /usr/bin/runc
root@ubuntu20-server2-112:~# runc -v
runc version 1.1.8
commit: v1.1.8-0-g82f18fe0
spec: 1.0.2-dev
go: go1.20.3
libseccomp: 2.5.4

4、安装 CNI 组件

# 下载地址: https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
root@ubuntu20-server2-112:~# ls
cni-plugins-linux-amd64-v1.3.0.tgz

# 插件的存放路径
root@ubuntu20-server2-112:~# mkdir -p /opt/cni/bin 
root@ubuntu20-server2-112:~# tar xvf cni-plugins-linux-amd64-v1.3.0.tgz -C /opt/cni/bin/

二、使用基于nerdctl创建并管理容器

2.1 nerdctl 的安装配置

# 安装 nerdctl
# 下载地址: https://github.com/containerd/nerdctl/releases/download/v1.4.0/nerdctl-1.4.0-linux-amd64.tar.gz
root@ubuntu20-server2-112:~# ls
nerdctl-1.4.0-linux-amd64.tar.gz

# 配置文件
root@ubuntu20-server2-112:~# mkdir /etc/nerdctl/
root@ubuntu20-server2-112:~# vim /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
debug = false
debug_full = false
insecure_registry = true

2.2 nerdctl 的使用

# 基于之前containerd的安装开始nerdctl的使用

# 创建nginx的容器
root@ubuntu20-server2-112:~# nerdctl run -d -p 8080:80 --name=nginx-web1 --restart=always nginx
root@ubuntu20-server2-112:~# nerdctl ps
CONTAINER ID    IMAGE                             COMMAND                   CREATED           STATUS    PORTS                   NAMES
790bd2e21163    docker.io/library/nginx:latest    "/docker-entrypoint.…"    39 seconds ago    Up        0.0.0.0:8080->80/tcp    nginx-web1
root@ubuntu20-server1-111:~# curl -I 192.168.119.112:8080
HTTP/1.1 200 OK
Server: nginx/1.25.1
Date: Tue, 25 Jul 2023 15:37:03 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Tue, 13 Jun 2023 15:08:10 GMT
Connection: keep-alive
ETag: "6488865a-267"
Accept-Ranges: bytes

# 进入容器
root@ubuntu20-server2-112:~# nerdctl exec -it 790b bash
root@790bd2e21163:/# hostname -I
10.4.0.2
root@790bd2e21163:/# curl www.baidu.com -k -I
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: private, no-cache, no-store, proxy-revalidate, no-transform
Connection: keep-alive
Content-Length: 277
Content-Type: text/html
Date: Tue, 25 Jul 2023 15:39:11 GMT
Etag: "575e1f60-115"
Last-Modified: Mon, 13 Jun 2016 02:50:08 GMT
Pragma: no-cache
Server: bfe/1.0.8.18

# 删除容器
root@ubuntu20-server2-112:~# nerdctl rm -f 9ad03

# 查看镜像
# nerdctl 存储镜像会有两份，一份是nerdctl，一份是containerd
# 如果镜像的名称和tag完全一样，那么另外一份会显示<none>
root@ubuntu20-server2-112:~# nerdctl images
REPOSITORY    TAG       IMAGE ID        CREATED           PLATFORM       SIZE         BLOB SIZE
nginx         latest    08bc36ad5247    12 minutes ago    linux/amd64    192.1 MiB    67.3 MiB
<none>        <none>    08bc36ad5247    12 minutes ago    linux/amd64    192.1 MiB    67.3 MiB

三、部署https的harbor服务器

3.1 准备好阿里云证书文件

image
image

# 将下载完成的证书上传到harbor服务器
root@ubuntu20-server3-113:~# mkdir -p /data/{certfile,privatekey}
root@ubuntu20-server3-113:~# ls /data/{certfile,privatekey}
/data/certfile:
www.wuhaolam.top.pem

/data/privatekey:
www.wuhaolam.top.key

3.2 安装harbor

# 安装前准备好docker环境
root@ubuntu20-server3-113:~# docker --version
Docker version 24.0.4, build 3713ee1

# 准备harbor安装包
root@ubuntu20-server3-113:~# ls
harbor-offline-installer-v2.8.2.tgz

# 安装harbor
root@ubuntu20-server3-113:~# tar xvf harbor-offline-installer-v2.8.2.tgz -C /usr/local/src/
root@ubuntu20-server3-113:~# cd /usr/local/src/harbor/
root@ubuntu20-server3-113:/usr/local/src/harbor# cp harbor.yml.tmpl harbor.yml

# 编辑配置文件
root@ubuntu20-server3-113:/usr/local/src/harbor# vim harbor.yml
hostname: www.wuhaolam.top
https:
  # https port for harbor, default is 443
  port: 443
  # The path of cert and key files for nginx
  certificate: /data/certfile/
  private_key: /data/privatekey/
harbor_admin_password: 123456
data_volume: /data/harbor
## 开始安装，--with-trivy 是开启安全扫描
root@ubuntu20-server3-113:/usr/local/src/harbor# ./install.sh --with-trivy
···
···
[Step 5]: starting Harbor ...
[+] Running 11/11
 ✔ Network harbor_harbor        Created                                                           0.0s 
 ✔ Container harbor-log         Started                                                           0.4s 
 ✔ Container registryctl        Started                                                           0.7s 
 ✔ Container registry           Started                                                           1.0s 
 ✔ Container harbor-db          Started                                                           1.0s 
 ✔ Container harbor-portal      Started                                                           1.1s 
 ✔ Container redis              Started                                                           1.1s 
 ✔ Container harbor-core        Started                                                           1.5s 
 ✔ Container trivy-adapter      Started                                                           1.4s 
 ✔ Container harbor-jobservice  Started                                                           1.9s 
 ✔ Container nginx              Started                                                           1.9s 
✔ ----Harbor has been installed and started successfully.----


# 如果之前已经部署过
# ./prepare 清楚之前的配置
# docker-compose up -d      重新启动harbor
root@ubuntu20-server3-113:/usr/local/src/harbor# ./prepare
root@ubuntu20-server3-113:/usr/local/src/harbor# docker-compose up -d

3.3 编辑hosts文件实现域名解析

# 在windows和Ubuntu的客户端中都需要添加如下解析
root@ubuntu20-server2-112:~# vim /etc/hosts
192.168.119.113  www.wuhaolam.top

3.4 使用浏览器验证

image

3.5 使用nerdctl命令上传和下载镜像

# 镜像的上传
## 登录到镜像仓库
root@ubuntu20-server2-112:~# nerdctl login www.wuhaolam.top
Enter Username: admin
Enter Password: 
WARN[0005] skipping verifying HTTPS certs for "www.wuhaolam.top" 
WARNING: Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded
## 为本地镜像打 tag
root@ubuntu20-server2-112:~# nerdctl tag nginx:latest www.wuhaolam.top/myserver/nginx:latest
root@ubuntu20-server2-112:~# nerdctl images
REPOSITORY                         TAG       IMAGE ID        CREATED           PLATFORM       SIZE         BLOB SIZE
nginx                              latest    08bc36ad5247    22 hours ago      linux/amd64    192.1 MiB    67.3 MiB
<none>                             <none>    08bc36ad5247    22 hours ago      linux/amd64    192.1 MiB    67.3 MiB
www.wuhaolam.top/myserver/nginx    latest    08bc36ad5247    17 seconds ago    linux/amd64    192.1 MiB    67.3 MiB
## 上传镜像
root@ubuntu20-server2-112:~# nerdctl push www.wuhaolam.top/myserver/nginx:latest
## 结果如下

image

# 镜像的下载
## 下载myserver仓库中的alpine镜像
root@ubuntu20-server2-112:~# nerdctl pull www.wuhaolam.top/myserver/alpine:20230718

root@ubuntu20-server2-112:~# nerdctl images
REPOSITORY                          TAG         IMAGE ID        CREATED           PLATFORM       SIZE       
www.wuhaolam.top/myserver/alpine    20230718    e7d88de73db3    16 seconds ago    linux/amd64    5.9 MiB      2.7 MiB

四、基于Alpine和Ubuntu作为基础镜像实现的业务镜像构建

4.1 基于 alpine 构建 nginx 镜像

# 准备 alpine 镜像的国内加速地址
root@ubuntu20-server2-112:/data/ubuntu-nginx# cat repositories 
https://mirrors.ustc.edu.cn/alpine/v3.5/main
https://mirrors.ustc.edu.cn/alpine/v3.5/community

# 准备nginx源码包以及自定义的配置文件
root@ubuntu20-server2-112:/data/ubuntu-nginx# ls
index.html  nginx-1.18.0.tar.gz  nginx.conf
root@ubuntu20-server2-112:/data/ubuntu-nginx# cat index.html 
ubuntu nginx web page!

# 准备镜像构建和上传脚本
root@ubuntu20-server2-112:/data/ubuntu-nginx# cat image-build.sh 
#!/bin/bash
docker build -t www.wuhaolam.top/myserver/nginx-ubuntu:v1 .
docker push www.wuhaolam.top/myserver/nginx-ubuntu:v1

# 准备 Dockerfile 文件
root@ubuntu20-server2-112:/data/ubuntu-nginx# cat Dockerfile 
FROM ubuntu:22.04
LABEL author="xxx@gmail.com"


RUN apt update && apt -y install  iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev  gcc openssh-server iotop unzip zip make
RUN mkdir -p /apps/nginx && useradd -r -s /sbin/nologin nginx

ADD nginx-1.18.0.tar.gz /usr/local/src/

RUN cd /usr/local/src/nginx-1.18.0 && ./configure --prefix=/apps/nginx/ --user=nginx --group=nginx && make && make install && ln -s /apps/nginx/sbin/nginx /usr/sbin/nginx && chown -R nginx:nginx /apps/nginx/

RUN mkdir -p /apps/nginx/run/
ADD nginx.conf /apps/nginx/conf/
ADD index.html /apps/nginx/html/


CMD ["nginx","-g","daemon off;"]


# 开始构建镜像和上传
root@ubuntu20-server2-112:/data/ubuntu-nginx# bash image-build.sh

# 启动镜像并验证
root@ubuntu20-server2-112:/data/ubuntu-nginx# docker run -itd -p 80:80 www.wuhaolam.top/myserver/nginx:alpine-v1 
cbe18abb99a47eb9c7c906f18c39d9c356aaeb5c63aed5bd883a241d18535d8c

image

4.2 基于 Ubuntu 构建 nginx 镜像

# 准备nginx源码包以及自定义的配置文件
root@ubuntu20-server2-112:/data/alpine-nginx# ls
index.html  nginx-1.18.0.tar.gz  nginx.conf
root@ubuntu20-server2-112:/data/alpine-nginx# cat index.html 
alpine nginx web page!

# 准备镜像构建和上传脚本
root@ubuntu20-server2-112:/data/ubuntu-nginx# cat ../alpine-nginx/image-build.sh 
#!/bin/bash
docker build -t www.wuhaolam.top/myserver/nginx:alpine-v1 .
docker push www.wuhaolam.top/myserver/nginx:alpine-v1

# 准备 Dockerfile 文件
root@ubuntu20-server2-112:/data/alpine-nginx# cat Dockerfile 
FROM alpine:3.5

MAINTAINER xxx@gmail.com

COPY repositories /etc/apk/repositories
RUN apk update
RUN apk  update && apk add vim iotop  gcc libgcc libc-dev libcurl libc-utils gzip zlib zlib-dev libnfs make  pcre pcre2 pcre-dev zip unzip net-tools pstree wget libevent libevent-dev iproute2
RUN mkdir -p /apps/nginx && addgroup  -S nginx && adduser -S  -G nginx -s /sbin/nologin nginx

ADD nginx-1.18.0.tar.gz /usr/local/src/

RUN cd /usr/local/src/nginx-1.18.0 && ./configure --prefix=/apps/nginx/ --user=nginx --group=nginx && make && make install && ln -s /apps/nginx/sbin/nginx /usr/sbin/nginx

RUN chown  -R nginx.nginx /apps/nginx/
RUN mkdir -p /apps/nginx/run/
ADD nginx.conf /apps/nginx/conf/
ADD index.html /apps/nginx/html/


CMD ["nginx","-g","daemon off;"]


# 开始构建镜像和上传
root@ubuntu20-server2-112:/data/ubuntu-nginx# bash image-build.sh

# 启动镜像并验证
root@ubuntu20-server2-112:/data/ubuntu-nginx# docker run -itd -p 8080:80 www.wuhaolam.top/myserver/nginx-ubuntu:v1
4d56fd192b1ab27152428cf3eb7da5089b4e50ec0e2e8b065835d7d6fd16a5ef

image

五、基于docker-compose单机编排运行Nginx+Java APP+MySQL服务

1、安装docker-compose
# 下载地址 https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64
root@ubuntu20-server1-111:~# ls
docker-compose-Linux-x86_64
root@ubuntu20-server1-111:~# cp docker-compose-Linux-x86_64 /usr/bin/docker-compose
root@ubuntu20-server1-111:~# chmod +x /usr/bin/docker-compose
root@ubuntu20-server1-111:~# docker-compose version
docker-compose version 1.29.2, build 5becea4c
docker-py version: 5.0.0
CPython version: 3.7.10
OpenSSL version: OpenSSL 1.1.0l  10 Sep 2019

2、编辑docker-compose文件
root@ubuntu20-server1-111:/data/docker-compose# vim docker-compose.yml
version: '3.8'
services:
  nginx-server:
    image: nginx:1.22.0-alpine
    container_name: nginx-web1
    expose:
      - 80
      - 443
    ports:
      - "80:80"
      - "443:443"
    networks:
      - front
      - backend
    links:
      - tomcat-server

  tomcat-server:
    image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/tomcat-myapp:v1
    container_name: tomcat-app1
    networks:
      - backend
    links:
      - mysql-server

  mysql-server:
    image: mysql:5.6.48
    container_name: mysql-container
    volumes:
      - /data/mysql:/var/lib/mysql
    environment:
      - "MYSQL_ROOT_PASSWORD=12345678"
      - "TZ=Asia/Shanghai"
    expose:
      - 3306
    ports:
      - "3306:3306"
    networks:
      - backend

networks:
  front:
    driver: bridge
  backend:
    driver: bridge
  default:
    external:
      name: bridge

4、创建并启动容器
root@ubuntu20-server1-111:/data/docker-compose# docker-compose up -d
228a250ae8e9   nginx:1.22.0-alpine                                             "/docker-entrypoint.…"   About a minute ago   Up About a minute   0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp   nginx-web1
ea25e962d2e5   registry.cn-hangzhou.aliyuncs.com/zhangshijie/tomcat-myapp:v1   "/apps/tomcat/bin/do…"   About a minute ago   Up About a minute   8080/tcp, 8443/tcp                                                         tomcat-app1
8de758346403   mysql:5.6.48                                                    "docker-entrypoint.s…"   About a minute ago   Up About a minute   0.0.0.0:3306->3306/tcp, :::3306->3306/tcp                                  mysql-container

5、配置nginx代理至tomcat服务
# 查看tomcat数据文件目录
root@ubuntu20-server1-111:~# docker exec -it ea2 /bin/bash
[root@ea25e962d2e5 conf]# vim server.xml
....
<Host name="localhost"  appBase="/data/tomcat/webapps"  unpackWARs="false" autoDeploy="false">
....
[root@ea25e962d2e5 ~]# cd /data/tomcat/webapps/myapp/
[root@ea25e962d2e5 myapp]# cat index.jsp 
<%@page import="java.util.Enumeration"%>
<br />
host: <%try{out.println(""+java.net.InetAddress.getLocalHost().getHostName());}catch(Exception e){}%>
<br />
remoteAddr: <%=request.getRemoteAddr()%>
<br />
remoteHost: <%=request.getRemoteHost()%>
<br />
sessionId: <%=request.getSession().getId()%>
<br />
serverName:<%=request.getServerName()%>
<br />
scheme:<%=request.getScheme()%>
<br />
<%request.getSession().setAttribute("t1","t2");%>
<%
        Enumeration en = request.getHeaderNames();
        while(en.hasMoreElements()){
        String hd = en.nextElement().toString();
                out.println(hd+" : "+request.getHeader(hd));
                out.println("<br />");
        }
%>
# 配置nginx转发至tomcat服务器中的myapp中index.jsp文件
root@ubuntu20-server1-111:/data/docker-compose# docker exec -it 228 sh
/ # sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
/ # apk update
/ # apk add vim
/ # vim /etc/nginx/conf.d/default.conf
...
# 在server语句块中添加一下代码
location /myapp {
	proxy_pass http://tomcat-server:8080;
}
...
/ # nginx -t
/ # nginx -s reload

6、测试访问

image
image

7、关于网络结果
# nginx 中会自动创建两个IP地址
/ # hostname -i
172.17.0.4 172.18.0.2

# tomcat 中一个
[root@ea25e962d2e5 /]# hostname -I
172.17.0.3

# MySQL中IP地址
root@8de758346403:/# hostname -I
172.17.0.2