K8s实现nginx、tomcat动静分离

K8s环境部署nginx、tomcat动静分离，部署elk+filebeat日志分析系统。通过nginx代理连接内外网。通过添加节点标签将nginx及tomcat相关pod定向调度至node1节点，elk相关pod定向调度至node2节点，filebeat的kind类型设置为DaemonSet。

节点：

master：172.28.9.90

Node1：172.28.9.92

Node2：172.28.9.91

Nginx负载：172.28.9.93

Nginx、Tomcat网页文件目录：

Node节点标签：

镜像下载：

docker pull docker.elastic.co/elasticsearch/elasticsearch:7.11.1

docker pull docker.elastic.co/logstash/logstash:7.11.1

docker pull docker.elastic.co/kibana/kibana:7.11.1

docker pull docker.elastic.co/beats/filebeat:7.11.1

docker pull docker.io/nginx:latest

docker pull docker.io/tomcat:latest

配置文件：

nginx.yaml、tomcat.yaml、filebeat.yaml、logstash.yaml、elasticsearch.yaml、kibana.yaml

service及pod运行状态：

Nginx及tomcat网页：

Elk网页及nginx日志：

Yaml文件内容：

nginx.yaml

—

apiVersion: v1

kind: PersistentVolume

metadata:

  name: nginx-log-pv

  labels:

    name: nginx-log-pv

spec:

  capacity:

    storage: 5Mi

  accessModes:

  – ReadWriteOnce

  persistentVolumeReclaimPolicy: Recycle

  hostPath:

    path: /var/log/nginx/

—

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: nginx-log-pvc

spec:

  accessModes:

  – ReadWriteOnce

  resources:

    requests:

      storage: 5Mi

  selector:

    matchLabels:

      name: nginx-log-pv

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-nginx-config

data:

  nginx.conf: |

    user  nginx;

    worker_processes  1;

    error_log  /var/log/nginx/error.log warn;

    pid        /var/run/nginx.pid;

    events {

        worker_connections  1024;

    }

    http {

        include       /etc/nginx/mime.types;

        default_type  application/octet-stream;

        log_format  main   $remote_addr – $remote_user [$time_local] “$request”

                           $status $body_bytes_sent “$http_referer”

                           “$http_user_agent” “$http_x_forwarded_for” ;

        access_log  /var/log/nginx/access.log  main;

        sendfile        on;

        keepalive_timeout  65;

        upstream tomcat_server1 {

           server tomcat-sts-0.tomcatserver.default:8080;

           server tomcat-sts-1.tomcatserver.default:8080;

        }

        upstream tomcat_server2 {

           server tomcat-sts-0.tomcatserver.default:8081;

           server tomcat-sts-1.tomcatserver.default:8081;

        }

          server {

            listen       80;

            server_name  test1;

            location / {

                root   /usr/share/nginx/html1;

                index  test1.html;

            }

            location /docs {

                alias /usr/share/nginx/docs1/;

                index test1-docs.html;

            }

            location ~ .*.jsp$ {

                proxy_pass http://tomcat_server1;

                proxy_set_header Host $host:$server_port;

                proxy_set_header REMOTE-HOST $remote_addr;

                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

                proxy_set_header X-Real-IP $remote_addr;

            }

          }

          server {

            listen 81;

            server_name test2;

            location / {

              root   /usr/share/nginx/html2;

              index  test2.html;

            }

            location /docs {

              alias   /usr/share/nginx/docs2/;

              index   test2-docs.html;

            }

            location ~ .*.jsp$ {

               proxy_pass http://tomcat_server2;

                proxy_set_header Host $host:$server_port;

                proxy_set_header REMOTE-HOST $remote_addr;

                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

                proxy_set_header X-Real-IP $remote_addr;

            }

          }

    }

—

apiVersion: apps/v1

kind: Deployment

metadata:

  name: nginx-deployment

spec:

  selector:

    matchLabels:

      app: nginx

  replicas: 2

  template:

    metadata:

      labels:

        app: nginx

    spec:

      containers:

      – name: nginx

        image: docker.io/nginx:latest

        imagePullPolicy: Never

        command: [ “nginx” ]

        args:

        – “-g daemon off;”

        – -c

        – /etc/nginx/nginx.conf

        ports:

        – containerPort: 80

          name: test1

        – containerPort: 81

          name: test2

        volumeMounts:

        – name: nginx-config

          mountPath: /etc/nginx/nginx.conf

          subPath: nginx.conf

        – name: nginx-html-docs

          mountPath: /usr/share/nginx/

        – name: nginx-log-pvc

          mountPath: /var/log/nginx/

      nodeSelector:

        type: web

      volumes:

      – name: nginx-config

        configMap:

          name: cm-nginx-config

      – name: nginx-log-pvc

        persistentVolumeClaim:

          claimName: nginx-log-pvc

      – name: nginx-html-docs

        hostPath:

          path: /usr/share/nginx/

          type: Directory

—

apiVersion: v1

kind: Service

metadata:

  name: nginx

spec:

  type: NodePort

  ports:

  – name: nginx1

    port: 80

    nodePort: 30001

  – name: nginx2

    port: 81

    nodePort: 30002

  selector:

    app: nginx

tomcat.yaml

—

apiVersion: v1

kind: PersistentVolume

metadata:

  name: tomcat-log-pv

  labels:

    name: tomcat-log-pv

spec:

  capacity:

    storage: 5Mi

  accessModes:

  – ReadWriteOnce

  persistentVolumeReclaimPolicy: Recycle

  hostPath:

    path: /var/log/tomcat/

—

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: tomcat-log-pvc

spec:

  accessModes:

  – ReadWriteOnce

  resources:

    requests:

      storage: 5Mi

  selector:

    matchLabels:

      name: tomcat-log-pv

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-tomcat-server-config

data:

  server.xml: |

    <?xml version= 1.0 encoding= utf-8 ?>

    <Server port=”8005″ shutdown=”SHUTDOWN”>

      <Listener className=”org.apache.catalina.core.AprLifecycleListener” SSLEngine=”on” />

      <Listener className=”org.apache.catalina.core.JreMemoryLeakPreventionListener” />

      <Listener className=”org.apache.catalina.mbeans.GlobalResourcesLifecycleListener” />

      <Listener className=”org.apache.catalina.core.ThreadLocalLeakPreventionListener” />

      <GlobalNamingResources>

        <Resource name=”UserDatabase” auth=”Container”

                  type=”org.apache.catalina.UserDatabase”

                  description=”User database that can be updated and saved”

                  factory=”org.apache.catalina.users.MemoryUserDatabaseFactory”

                  pathname=”conf/tomcat-users.xml” />

      </GlobalNamingResources>

      <Service name=”test1″>

        <Connector port=”8080″ protocol=”HTTP/1.1″

                   connectionTimeout=”20000″

                   redirectPort=”8443″ />

        <Engine name=”test1″ defaultHost=”test1″>

          <Realm className=”org.apache.catalina.realm.LockOutRealm”>

            <Realm className=”org.apache.catalina.realm.UserDatabaseRealm”

                   resourceName=”UserDatabase”/>

          </Realm>

          <Host name=”test1″  appBase=”/usr/share/tomcat/test1″

                unpackWARs=”true” autoDeploy=”true”>

            <Valve className=”org.apache.catalina.valves.AccessLogValve” directory=”/var/log/tomcat/”

                   prefix=”test1_access_log” suffix=”.txt”

                   pattern=”%{X-Forwarded-For}i %h %l %u %t %r %s %b %{Referer}i %{User-Agent}i” />

          </Host>

        </Engine>

      </Service>

      <Service name=”test2″>

        <Connector port=”8081″ protocol=”HTTP/1.1″

                   connectionTimeout=”20000″

                   redirectPort=”8443″ />

        <Engine name=”test2″ defaultHost=”test2″>

          <Realm className=”org.apache.catalina.realm.LockOutRealm”>

            <Realm className=”org.apache.catalina.realm.UserDatabaseRealm”

                   resourceName=”UserDatabase”/>

          </Realm>

          <Host name=”test2″  appBase=”/usr/share/tomcat/test2″

                unpackWARs=”true” autoDeploy=”true”>

            <Valve className=”org.apache.catalina.valves.AccessLogValve” directory=”/var/log/tomcat/”

                   prefix=”test2_access_log” suffix=”.txt”

                   pattern=”%{X-Forwarded-For}i %h %l %u %t %r %s %b %{Referer}i %{User-Agent}i” />

          </Host>

        </Engine>

      </Service>

    </Server>

—

apiVersion: apps/v1

kind: StatefulSet

metadata:

  name: tomcat-sts

spec:

  selector:

    matchLabels:

      app: tomcat

  replicas: 2

  serviceName: tomcatserver

  template:

    metadata:

      labels:

        app: tomcat

    spec:

      containers:

      – name: tomcat

        image: docker.io/tomcat:latest

        imagePullPolicy: Never

        command: [ “/bin/sh”, “-c” ]

        args:

        – “bin/catalina.sh run;”

        ports:

        – containerPort: 8080

          name: test1

        – containerPort: 8081

          name: test2

        volumeMounts:

        – name: tomcat-server-config

          mountPath: /usr/local/tomcat/conf/server.xml

          subPath: server.xml

        – name: tomcat-webapp

          mountPath: /usr/share/tomcat/

        – name: tomcat-log-pvc

          mountPath: /var/log/tomcat/

      nodeSelector:

        type: web

      volumes:

      – name: tomcat-server-config

        configMap:

          name: cm-tomcat-server-config

      – name: tomcat-log-pvc

        persistentVolumeClaim:

          claimName: tomcat-log-pvc

      – name: tomcat-webapp

        hostPath:

          path: /usr/share/tomcat/

          type: Directory

—

apiVersion: v1

kind: Service

metadata:

  name: tomcat

  labels:

    app: tomcat

spec:

  clusterIP: None

  ports:

  – port: 8080

    name: test1

  – port: 8081

    name: test2

  selector:

    app: tomcat

filebeat.yaml

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-filebeat-config

data:

  filebeat.yml: |

    filebeat.inputs:

    – type: log

      enabled: true

      paths:

      – /var/log/nginx/*.log

    filebeat.config.modules:

      path: ${path.config}/modules.d/*.yml

      reload.enabled: false

    output.logstash:

      hosts: [ “logstash:5044” ]

—

apiVersion: apps/v1

kind: DaemonSet

metadata:

  name: filebeat

spec:

  selector:

    matchLabels:

      app: filebeat

  template:

    metadata:

      labels:

        app: filebeat

    spec:

      containers:

      – name: filebeat

        image: docker.elastic.co/beats/filebeat:7.11.1

        imagePullPolicy: Never

        command: [ “./filebeat” ]

        args:

        – -e

        – -c

        – filebeat.yml

        volumeMounts:

        – name: filebeat-config

          mountPath: /usr/share/filebeat/filebeat.yml

          subPath: filebeat.yml

        – name: nginx-log-pvc

          mountPath: /var/log/nginx/

      volumes:

      – name: filebeat-config

        configMap:

          name: cm-filebeat-config

      – name: nginx-log-pvc

        persistentVolumeClaim:

          claimName: nginx-log-pvc

logstash.yaml

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-logstash-yml-config

data:

  logstash.yml: |

    http.host: “0.0.0.0”

    xpack.monitoring.elasticsearch.hosts: [“http://elasticsearch:9200”]

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-logstash-config

data:

  logstash.conf: |

    input {

      beats {

        port => 5044

      }

    }

    output {

      elasticsearch {

        hosts => “elasticsearch:9200”

        index => “%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}”

      }

    }

—

apiVersion: apps/v1

kind: Deployment

metadata:

  name: logstash

spec:

  replicas: 1

  selector:

    matchLabels:

      app: logstash

  template:

    metadata:

      labels:

        app: logstash

    spec:

      containers:

      – image: docker.elastic.co/logstash/logstash:7.11.1

        name: logstash

        imagePullPolicy: Never

        command: [ “bin/logstash” ]

        args:

        – -f

        – config/logstash.conf

        ports:

        – name: log-es

          containerPort: 5044

        volumeMounts:

        – name: logstash-yml-config

          mountPath: /usr/share/logstash/config/logstash.yml

          subPath: logstash.yml

        – name: logstash-config

          mountPath: /usr/share/logstash/config/logstash.conf

          subPath: logstash.conf

      nodeSelector:

        type: elk

      volumes:

      – name: logstash-yml-config

        configMap:

          name: cm-logstash-yml-config

      – name: logstash-config

        configMap:

          name: cm-logstash-config

—

apiVersion: v1

kind: Service

metadata:

  name: logstash

spec:

  type: ClusterIP

  ports:

  – name: log-es

    port: 5044

  selector:

    app: logstash

elasticsearch.yaml

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-elasticsearch-config

data:

  elasticsearch.yml: |

    network.host: 0.0.0.0

    http.port: 9200

    cluster.initial_master_nodes: ${POD_IP}

—

apiVersion: apps/v1

kind: Deployment

metadata:

  name: elasticsearch

spec:

  replicas: 1

  selector:

    matchLabels:

      app: elasticsearch

  template:

    metadata:

      labels:

        app: elasticsearch

    spec:

      containers:

      – image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1

        name: elasticsearch

        imagePullPolicy: Never

        env:

        – name: POD_IP

          valueFrom:

            fieldRef:

              fieldPath: status.podIP

        ports:

        – name: es-kibana

          containerPort: 9200

        volumeMounts:

        – name: elasticsearch-config

          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml

          subPath: elasticsearch.yml

      nodeSelector:

        type: elk

      volumes:

      – name: elasticsearch-config

        configMap:

          name: cm-elasticsearch-config

—

apiVersion: v1

kind: Service

metadata:

  name: elasticsearch

spec:

  type: ClusterIP

  ports:

  – name: log

    port: 9200

  selector:

    app: elasticsearch

kibana.yaml

—

apiVersion: v1

kind: ConfigMap

metadata:

  name: cm-kibana-config

data:

  kibana.yml: |

    server.port: 5601

    server.host: “0.0.0.0”

    elasticsearch.hosts: [“http://elasticsearch:9200”]

—

apiVersion: apps/v1

kind: Deployment

metadata:

  name: kibana

spec:

  replicas: 1

  selector:

    matchLabels:

      app: kibana

  template:

    metadata:

      labels:

        app: kibana

    spec:

      containers:

      – image: docker.elastic.co/kibana/kibana:7.11.1

        name: kibana

        imagePullPolicy: Never

        command: [ “bin/kibana” ]

        args:

        – –allow-root

        ports:

        – name: web

          containerPort: 5601

        volumeMounts:

        – name: kibana-config

          mountPath: /usr/share/kibana/config/kibana.yml

          subPath: kibana.yml

      nodeSelector:

        type: elk

      volumes:

      – name: kibana-config

        configMap:

          name: cm-kibana-config

—

apiVersion: v1

kind: Service

metadata:

  name: kibana

spec:

  type: NodePort

  ports:

  – name: web

    port: 5601

    nodePort: 30003

  selector:

    app: kibana