Kubernetes Jenkins

Kubernetes Jenkins

Posted by Vito on February 4, 2024

Jenkins Master

  • PVC
1
2
3
4
5
6
7
8
9
10
11
12
13

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins
  namespace: devops
spec:
  accessModes:
    - ReadWriteMany 
  volumeMode: Filesystem 
  resources:
    requests:
      storage: 2Gi
  #storageClassName: "managed-nfs-storage"
  • Install
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168

apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
  namespace: devops
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins
rules:
- apiGroups: ["apps"]
  resources: ["deployments", "statefulsets"]
  verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: ["batch"]
  resources: ["cronjobs"]
  verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]  # 安装 jenkins master 不需要 "services" 权限，这是后来 jenkins slave 用 ServiceAccount['jenkins'] 部署 service 时增加的
  resources: ["services"]
  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: ["networking.k8s.io"]
  resources: ["ingresses"]
  verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
  resources: ["pods/log", "events"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["secrets", "configmaps"]
  verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins
    namespace: devops
roleRef:
  kind: ClusterRole
  name: jenkins
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: devops
spec:
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccount: jenkins
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - jenkins
              topologyKey: kubernetes.io/hostname
      containers:
        - name: jenkins
          #image: jenkins/jenkins:lts-jdk17
          image: jenkins/jenkins:2.426.2-lts-jdk17
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true  # 拥有特权
            runAsUser: 0      # 设置以ROOT用户运行容器
          ports:
            - containerPort: 8080
              name: web
              protocol: TCP
            - containerPort: 50000
              name: agent
              protocol: TCP
          resources:
            limits:
              cpu: 1500m
              memory: 1024Mi
            requests:
              cpu: 100m
              memory: 512Mi
          env:
          - name: TZ
            value: 'Asia/Shanghai'
          - name: LIMITS_MEMORY
            valueFrom:
              resourceFieldRef:
                divisor: 1Mi
                resource: limits.memory
          - name: JAVA_OPTS
            value: >-
              -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm
              -Dhudson.slaves.NodeProvisioner.initialDelay=0
              -Dhudson.slaves.NodeProvisioner.MARGIN=50
              -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12
          volumeMounts:
            - name: jenkinshome
              mountPath: /var/jenkins_home
      volumes:
        - name: jenkinshome
          persistentVolumeClaim:
            claimName: jenkins
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: devops
  labels:
    app: jenkins
spec:
  selector:
    app: jenkins
  type: ClusterIP
  ports:
  - name: web
    port: 8080
    targetPort: web
  - name: agent
    port: 50000
    targetPort: agent
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jenkins
  namespace: devops
spec: 
  ingressClassName: nginx
  tls:
  - hosts:
    - jenkins.zhch.lan
    secretName: zhch.lan
  rules:
  - host: jenkins.zhch.lan
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: jenkins
            port:
              number: 8080

  • 访问

    • https://jenkins.zhch.lan

      1
2

      [root@k8s-master1 ~]# cat /root/nfs_data/rw/default/devops-jenkins-pvc-b3bbf3a2-e00a-4e8a-b51f-5f3cd9753cfa/secrets/initialAdminPassword 
70e04c70b96d4dba894916e82ea7714f

    • 暂先不安装任何插件

    • 修改 /var/lib/jenkins/updates/default.json 文件，替换插件地址

      1
2

      sed -i 's/https:\/\/updates.jenkins.io\/download\/plugins/https:\/\/mirrors.tuna.tsinghua.edu.cn\/jenkins\/plugins/g' default.json
sed -i 's/https:\/\/www.google.com/https:\/\/www.baidu.com/g' default.json

    • 修改插件更新地址为国内地址

      • Manage Jenkins -> Plugins -> Advanced settings -> Update Site
      • https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
      • 通过 https://jenkins.zhch.lan/restart 重启 jenkins

    • 按需安装插件

      • Localization: Chinese (Simplified)

      • Git

        • 正常情况下，想要通过 SSH 拉取代码，需要将 Gitlab host 添加到 Jenkins 的 ~/.ssh/known_hosts 中

        • 可通过命令 [root@Jenkins ~]# ssh -T git@git.zhch.lan 达成该目的

        • 但现在 Gitlab 和 Jenkins 均是部署在容器中，因此不便如此操作，故而取消 Git Host Key 验证

      • Gitlab

      • Git Parameter

        • 用于动态从 Github 或者 Gitlab 中检索项目分支信息，在 Jenkins Job 参数化构建中提供选择分支项，方便用户在执行构建时候执行选择的分支。

      • Pipeline

      • Pipeline: Stage View

      • Environment Injector

        • Item 范围的环境变量

      • Extended Choice Parameter

        • 复选框

      • Kubernetes

      • Config File Provider

      • Build Authorization Token Root

      • SonarQube Scanner

      • Node and Label parameter

自定义 Jenkins Slave 镜像

1
2
3
4
5
6
7
8
9

# 准备工作
# 1. 下载 Maven (略)

# 2. 下载 sonar-scanner-cli
wget https://repo1.maven.org/maven2/org/sonarsource/scanner/cli/sonar-scanner-cli/5.0.1.3006/sonar-scanner-cli-5.0.1.3006-linux.zip
# Dockerfile ADD 时不能自动解压 zip 文件
unzip sonar-scanner-cli-5.0.1.3006-linux.zip
[root@k8s-master1 jenkins-slave]# ls
apache-maven-3.9.6-bin.tar.gz  Dockerfile  sonar-scanner-5.0.1.3006-linux  sonar-scanner-cli-5.0.1.3006-linux.zip

Maven

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# Dockerfile
# https://github.com/jenkinsci/docker-inbound-agent
# https://hub.docker.com/r/jenkins/inbound-agent/
# 选择源镜像时，注意其使用的 Java 版本，要与 Jenkins Master 的镜像使用的 JDK 版本一致
# 若使用 Java 8 ，可使用 Tag 3046.v38db_38a_b_7a_86-1-jdk8
FROM jenkins/inbound-agent:3198.v03a_401881f3e-1-jdk17
MAINTAINER vito<hczhch@yeah.net>

# 切换到 root 账户进行操作
USER root

# 时区
ENV TZ Asia/Shanghai
# RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

# 安装 maven
ADD apache-maven-3.9.6-bin.tar.gz .
RUN mv apache-maven-3.9.6 /usr/local/maven
RUN ln -s /usr/local/maven/bin/mvn /usr/bin/mvn

# 安装 sonar-scanner-cli
COPY sonar-scanner-5.0.1.3006-linux /usr/local/sonar-scanner
RUN ln -s /usr/local/sonar-scanner/bin/sonar-scanner /usr/bin/sonar-scanner
# 解决使用 sonar-scanner 命令时因自签证书问题，无法连接 SonarQube 服务器问题
# 方法一：将 SonarQube 服务的证书添加到 sonar-scanner 使用的 jre 的信任证书库
# COPY zhch.lan.crt /home/jenkins/
# RUN echo 'yes' | keytool -importcert -trustcacerts -keystore '/usr/local/sonar-scanner/jre/lib/security/cacerts' -storepass changeit -alias zhch.lan -file /home/jenkins/zhch.lan.crt
# 方法二：在 Jenkins Master 中配置 SonarQube servers 时使用内部地址，例如：http://sonarqube.devops:9000

# 安装 sudo
RUN apt-get update
RUN apt-get install -y sudo
# 在镜像挂载 /var/run/docker.sock 和 /usr/bin/docker 后，让 jenkins 用户能够免密执行 sudo docker 命令
RUN echo 'jenkins    ALL=(ALL)    NOPASSWD: /usr/bin/docker' >> /etc/sudoers

USER jenkins

1
2
3
4
5

docker build -t harbor.zhch.lan/library/inbound-agent:3198.jdk17-maven3.9 .

docker login -u vito -p Harbor12345 harbor.zhch.lan

docker push harbor.zhch.lan/library/inbound-agent:3198.jdk17-maven3.9
  • Maven settings.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

apiVersion: v1
kind: ConfigMap
metadata:
  name: maven
  namespace: devops
data:
  settings.xml: |
    <?xml version="1.0" encoding="UTF-8"?>
    <settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
              xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 https://maven.apache.org/xsd/settings-1.2.0.xsd">
    <localRepository>/home/jenkins/repo</localRepository>
      <pluginGroups>
        <!-- <pluginGroup>org.sonarsource.scanner.maven</pluginGroup> -->
      </pluginGroups>
    
      <proxies>
      </proxies>
    
      <servers>
      </servers>
    
      <mirrors>
        <mirror>
          <id>nexus-aliyun</id>
          <mirrorOf>central</mirrorOf>
          <name>Nexus aliyun</name>
          <url>https://maven.aliyun.com/nexus/content/groups/public</url>
        </mirror>
      </mirrors>
    
      <profiles>
        <!-- <profile>
          <id>sonar</id>
          <activation>
            <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
            <sonar.host.url>http://sonarqube.devops:9000</sonar.host.url>
          </properties>
        </profile> -->
      </profiles>
    
      <activeProfiles>
      </activeProfiles>
    </settings>

NodeJS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# Dockerfile
# https://github.com/jenkinsci/docker-inbound-agent
# https://hub.docker.com/r/jenkins/inbound-agent/
# 选择源镜像时，注意其使用的 Java 版本，要与 Jenkins Master 的镜像使用的 JDK 版本一致
# 若使用 Java 8 ，可使用 Tag 3046.v38db_38a_b_7a_86-1-jdk8
FROM jenkins/inbound-agent:3198.v03a_401881f3e-1-jdk17
MAINTAINER vito<hczhch@yeah.net>

# 切换到 root 账户进行操作
USER root

# 时区
ENV TZ Asia/Shanghai
# RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

# 安装 NodeJS
# https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.gz
ADD node-v12.22.12-linux-x64.tar.gz .
RUN mv node-v12.22.12-linux-x64 /usr/local/nodejs
RUN ln -s /usr/local/nodejs/bin/node /usr/bin/node
RUN ln -s /usr/local/nodejs/bin/npm /usr/bin/npm
RUN ln -s /usr/local/nodejs/bin/npx /usr/bin/npx
# 安装 cnpm （需要主要 cnpm 与 node 的版本兼容问题）
RUN npm install cnpm@7.1.1 -g --registry=https://registry.npm.taobao.org
RUN ln -s /usr/local/nodejs/bin/cnpm /usr/bin/cnpm

# 安装 sonar-scanner-cli
COPY sonar-scanner-5.0.1.3006-linux /usr/local/sonar-scanner
RUN ln -s /usr/local/sonar-scanner/bin/sonar-scanner /usr/bin/sonar-scanner
# 解决使用 sonar-scanner 命令时因自签证书问题，无法连接 SonarQube 服务器问题
# 方法一：将 SonarQube 服务的证书添加到 sonar-scanner 使用的 jre 的信任证书库
# COPY zhch.lan.crt /home/jenkins/
# RUN echo 'yes' | keytool -importcert -trustcacerts -keystore '/usr/local/sonar-scanner/jre/lib/security/cacerts' -storepass changeit -alias zhch.lan -file /home/jenkins/zhch.lan.crt
# 方法二：在 Jenkins Master 中配置 SonarQube servers 时使用内部地址，例如：http://sonarqube.devops:9000

# 安装 sudo
RUN apt-get update
RUN apt-get install -y sudo
# 在镜像挂载 /var/run/docker.sock 和 /usr/bin/docker 后，让 jenkins 用户能够免密执行 sudo docker 命令
RUN echo 'jenkins    ALL=(ALL)    NOPASSWD: /usr/bin/docker' >> /etc/sudoers

USER jenkins

1
2
3
4
5

docker build -t harbor.zhch.lan/library/inbound-agent:3198.jdk17-node12 .

docker login -u vito -p Harbor12345 harbor.zhch.lan

docker push harbor.zhch.lan/library/inbound-agent:3198.jdk17-node12

Jenkins Slave NFS 权限问题

  • 观察 Jenkins Slave 镜像的构建过程，该镜像创建了组 ID 为 1000 的用户组 jenkins，并在该用户组中创建了用户 ID 为 1000 的用户 jenkins；并最终以 jenkins 身份运行。
  • 在 NFS 服务器中创建挂载目录，用户存储 maven 下载的依赖包，并修改该目录的权限 
    1

    [root@k8s-master1 ~]# chown 1000:1000 -R /root/nfs_data/rw/maven/repo
  • 把该目录挂载到 jenkins slave 中 
    1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

    // https://www.jenkins.io/doc/pipeline/steps/kubernetes/
podTemplate(label: 'jenkins-slave', cloud: 'k8s', containers: [
    containerTemplate(
      name: 'jnlp',
      image: "harbor.zhch.lan/library/inbound-agent:3198.jdk17-maven3.9"
    ),
    containerTemplate(
      name: 'docker',
      image: "docker:24.0.7",
      ttyEnabled: true,
      command: 'cat'
    )
  ],
  volumes: [
    hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    nfsVolume(mountPath: '/home/jenkins/repo', serverAddress: 'k8s-master1.zhch.lan' , serverPath: '/root/nfs_data/rw/maven/repo', readOnly: false),
    configMapVolume(mountPath: '/usr/local/maven/conf/settings.xml', subPath: 'settings.xml', configMapName: 'maven')
  ],
  //security(fsGroup: 2000)
)
{
    node("jenkins-slave") {
      
    }
}
CATALOG
    FEATURED TAGS
    Docker Kubernetes Jenkins GitLab Maven Harbor SonarQube Elasticsearch Git
    FRIENDS