aws云部署k8s日志服务

weixin_43982893

1170人浏览 · 2025-03-19 16:28:53

weixin_43982893 · 2025-03-19 16:28:53 发布

1、利用operator部署生产级别的Elasticserach集群和kibana。

部署eck operator

kubectl create -f https://download.elastic.co/downloads/eck/2.10.0/crds.yaml
kubectl apply -f https://download.elastic.co/downloads/eck/2.10.0/operator.yaml
kubectl -n elastic-system get pod
kubectl -n elastic-system logs -f statefulset.apps/elastic-operator
kubectl create ns es

部署 elasticsearch 集群(我这边的持久化存储用的是aws的ebs)

cat es-ebs.yaml
apiVersion: /v1
kind: StorageClass
metadata: 
  name: ebs-sc
provisioner: 
volumeBindingMode: WaitForFirstConsumer


cat quickstart.yaml 
apiVersion: /v1
kind: Elasticsearch
metadata:
  name: quickstart
  namespace: es
spec:
  version: 8.11.3
  nodeSets:
  - name: default
    count: 1
    config:
      node.store.allow_mmap: false
    podTemplate:
      spec:
        containers:
        - name: elasticsearch
          env:
          - name: ES_JAVA_OPTS
            value: -Xms2g -Xmx2g
          resources:
            requests:
              memory: 4Gi
              cpu: 1
            limits:
              memory: 4Gi
              cpu: 1
        initContainers:
        - name: sysctl
          securityContext:
            privileged: true
          command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 500Gi
        storageClassName: ebs-sc
  http:
    tls:
      selfSignedCertificate:
        disabled: true

测试ES集群

PASSWORD=$(kubectl -n es get secret quickstart-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')
curl -u "elastic:$PASSWORD"  http://$(kubectl -n es get svc|grep es-http|awk '{print $3}'):9200
echo "elastic:$PASSWORD"

# 检查ES创建结果(可以通过edit来修改、用delete来删除)
kubectl -n es get es

NAME         HEALTH   NODES   VERSION   PHASE   AGE
quickstart   green    1       8.11.3    Ready   8m5s

部署 kibana

cat kibana.yaml 
apiVersion: /v1
kind: Kibana
metadata:
  name: quickstart
  namespace: es
spec:
  version: 8.11.3
  count: 1
  elasticsearchRef:
    name: quickstart
  http:
    tls:
      selfSignedCertificate:
        disabled: true
  config:  ##   删除这两行默认就是英文界面
    i18n.locale: "zh-CN" # 添加中文支持

生成tls证书

我这边是网站上申请的证书分别是这两个文件：xxx.net.crt xxx.net.key
生成tls文件：kubectl create secret tls xxx-net-city-tls --cert=xxx.net.crt --key=xxx.net.key --dry-run=client -o yaml > xxx-net-city-tls.yaml
然后部署证书：kubectl apply -f xxx-net-city-tls.yaml -n es

配置kibana的ingress

cat ingress.yaml 
apiVersion: /v1
kind: Ingress
metadata:
  name: kibana
  namespace: es
  annotations:
    /force-ssl-redirect: "true"
    kubernetes.io/ingress.class: nginx
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - aws-kibana.rchamet.net
    secretName: rchamet-net-city-tls
  rules:
  - host: aws-kibana.rchamet.net
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: quickstart-kb-http
            port:
              number: 5601

完成以后可以配置一下DNS解析，通过对应域名进行访问：aws-kibana.xxx.net

账号密码请看(测试ES集群)这步。

2、利用Log-pilot收集POD内业务日志文件。

整体的架构是

模拟业务服务   日志收集      日志存储     消费日志       消费目的服务
 nginx ---> log-pilot ---> kafka ---> filebeat ---> elasticsearch ---> kibana

首先准备一个kafka作为日志的存储。

cat docker-compose.yml 

version: '2'
services:
  zookeeper:
    image: zookeeper
    ports:
      - "2181:2181"
    restart: unless-stopped

  kafka:
    image: bitnami/kafka:2.8.1
    ports:
      - "9092:9092"
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://192.168.35.226:9092
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - /mnt/kafka:/kafka
    restart: unless-stopped
    
    
#启动 docker-compose up -d
#停止 docker-compose down
#查看 docker-compose ps

#run test-docker
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=192.168.35.226 -e ZK=192.168.35.226:2181 -i -t wurstmeister/kafka /bin/bash

#list topic
kafka-topics.sh --zookeeper 192.168.35.226:2181 --list

部署日志收集服务 log-pilot (适配容器运行时 Containerd)

cat log-pilot5.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: log-pilot5-configuration
  namespace: ns-elastic
data:
  logging_output: "kafka"
  kafka_brokers: "192.168.35.226:9092"
  kafka_version: "0.10.0"
  kafka_topics: "agent-nginx-test,svip-nginx-test"
  #kafka_username: "user"
  #kafka_password: "bogeit"

---
apiVersion: v1
data:
  filebeat.tpl: |+
    {{range .configList}}
    - type: log
      enabled: true
      paths:
          - {{ .HostDir }}/{{ .File }}
      scan_frequency: 5s
      fields_under_root: true
      {{if eq .Format "json"}}
      json.keys_under_root: true
      {{end}}
      fields:
          {{range $key, $value := .Tags}}
          {{ $key }}: {{ $value }}
          {{end}}
          {{range $key, $value := $.container}}
          {{ $key }}: {{ $value }}
          {{end}}
      tail_files: false
      close_inactive: 2h
      close_eof: false
      close_removed: true
      clean_removed: true
      close_renamed: false

    {{end}}

kind: ConfigMap
metadata:
  name: filebeat5-tpl
  namespace: ns-elastic

---
apiVersion: v1
data:
  config.filebeat: |+
    #!/bin/sh

    set -e

    FILEBEAT_CONFIG=/etc/filebeat/filebeat.yml
    if [ -f "$FILEBEAT_CONFIG" ]; then
        echo "$FILEBEAT_CONFIG has been existed"
        exit
    fi

    mkdir -p /etc/filebeat/prospectors.d

    assert_not_empty() {
        arg=$1
        shift
        if [ -z "$arg" ]; then
            echo "$@"
            exit 1
        fi
    }

    cd $(dirname $0)

    base() {
    cat >> $FILEBEAT_CONFIG << EOF
    path.config: /etc/filebeat
    path.logs: /var/log/filebeat
    path.data: /var/lib/filebeat/data
    filebeat.registry_file: /var/lib/filebeat/registry
    filebeat.shutdown_timeout: ${FILEBEAT_SHUTDOWN_TIMEOUT:-0}
    logging.level: ${FILEBEAT_LOG_LEVEL:-info}
    logging.metrics.enabled: ${FILEBEAT_METRICS_ENABLED:-false}
    logging.files.rotateeverybytes: ${FILEBEAT_LOG_MAX_SIZE:-104857600}
    logging.files.keepfiles: ${FILEBEAT_LOG_MAX_FILE:-10}
    logging.files.permissions: ${FILEBEAT_LOG_PERMISSION:-0600}
    ${FILEBEAT_MAX_PROCS:+max_procs: ${FILEBEAT_MAX_PROCS}}
    setup.template.name: "${FILEBEAT_INDEX:-filebeat}"
    setup.template.pattern: "${FILEBEAT_INDEX:-filebeat}-*"
    filebeat.config:
        prospectors:
            enabled: true
            path: \${path.config}/prospectors.d/*.yml
            reload.enabled: true
            reload.period: 10s
    EOF
    }

    es() {
    if [ -f "/run/secrets/es_credential" ]; then
        ELASTICSEARCH_USER=$(cat /run/secrets/es_credential | awk -F":" '{ print $1 }')
        ELASTICSEARCH_PASSWORD=$(cat /run/secrets/es_credential | awk -F":" '{ print $2 }')
    fi

    if [ -n "$ELASTICSEARCH_HOSTS" ]; then
        ELASTICSEARCH_HOSTS=$(echo $ELASTICSEARCH_HOSTS|awk -F, '{for(i=1;i<=NF;i++){printf "\"%s\",", $i}}')
        ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS%,}
    else
        assert_not_empty "$ELASTICSEARCH_HOST" "ELASTICSEARCH_HOST required"
        assert_not_empty "$ELASTICSEARCH_PORT" "ELASTICSEARCH_PORT required"
        ELASTICSEARCH_HOSTS="\"$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT\""
    fi

    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.elasticsearch:
        hosts: [$ELASTICSEARCH_HOSTS]
        index: ${ELASTICSEARCH_INDEX:-filebeat}-%{+yyyy.MM.dd}
        ${ELASTICSEARCH_SCHEME:+protocol: ${ELASTICSEARCH_SCHEME}}
        ${ELASTICSEARCH_USER:+username: ${ELASTICSEARCH_USER}}
        ${ELASTICSEARCH_PASSWORD:+password: ${ELASTICSEARCH_PASSWORD}}
        ${ELASTICSEARCH_WORKER:+worker: ${ELASTICSEARCH_WORKER}}
        ${ELASTICSEARCH_PATH:+path: ${ELASTICSEARCH_PATH}}
        ${ELASTICSEARCH_BULK_MAX_SIZE:+bulk_max_size: ${ELASTICSEARCH_BULK_MAX_SIZE}}
    EOF
    }

    default() {
    echo "use default output"
    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.console:
        pretty: ${CONSOLE_PRETTY:-false}
    EOF
    }

    file() {
    assert_not_empty "$FILE_PATH" "FILE_PATH required"

    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.file:
        path: $FILE_PATH
        ${FILE_NAME:+filename: ${FILE_NAME}}
        ${FILE_ROTATE_SIZE:+rotate_every_kb: ${FILE_ROTATE_SIZE}}
        ${FILE_NUMBER_OF_FILES:+number_of_files: ${FILE_NUMBER_OF_FILES}}
        ${FILE_PERMISSIONS:+permissions: ${FILE_PERMISSIONS}}
    EOF
    }

    logstash() {
    assert_not_empty "$LOGSTASH_HOST" "LOGSTASH_HOST required"
    assert_not_empty "$LOGSTASH_PORT" "LOGSTASH_PORT required"

    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.logstash:
        hosts: ["$LOGSTASH_HOST:$LOGSTASH_PORT"]
        index: ${FILEBEAT_INDEX:-filebeat}-%{+yyyy.MM.dd}
        ${LOGSTASH_WORKER:+worker: ${LOGSTASH_WORKER}}
        ${LOGSTASH_LOADBALANCE:+loadbalance: ${LOGSTASH_LOADBALANCE}}
        ${LOGSTASH_BULK_MAX_SIZE:+bulk_max_size: ${LOGSTASH_BULK_MAX_SIZE}}
        ${LOGSTASH_SLOW_START:+slow_start: ${LOGSTASH_SLOW_START}}
    EOF
    }

    redis() {
    assert_not_empty "$REDIS_HOST" "REDIS_HOST required"
    assert_not_empty "$REDIS_PORT" "REDIS_PORT required"

    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.redis:
        hosts: ["$REDIS_HOST:$REDIS_PORT"]
        key: "%{[fields.topic]:filebeat}"
        ${REDIS_WORKER:+worker: ${REDIS_WORKER}}
        ${REDIS_PASSWORD:+password: ${REDIS_PASSWORD}}
        ${REDIS_DATATYPE:+datatype: ${REDIS_DATATYPE}}
        ${REDIS_LOADBALANCE:+loadbalance: ${REDIS_LOADBALANCE}}
        ${REDIS_TIMEOUT:+timeout: ${REDIS_TIMEOUT}}
        ${REDIS_BULK_MAX_SIZE:+bulk_max_size: ${REDIS_BULK_MAX_SIZE}}
    EOF
    }

    kafka() {
    assert_not_empty "$KAFKA_BROKERS" "KAFKA_BROKERS required"
    KAFKA_BROKERS=$(echo $KAFKA_BROKERS|awk -F, '{for(i=1;i<=NF;i++){printf "\"%s\",", $i}}')
    KAFKA_BROKERS=${KAFKA_BROKERS%,}

    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.kafka:
        hosts: [$KAFKA_BROKERS]
        topic: '%{[topic]}'
        codec.format:
            string: '%{[message]}'
        ${KAFKA_VERSION:+version: ${KAFKA_VERSION}}
        ${KAFKA_USERNAME:+username: ${KAFKA_USERNAME}}
        ${KAFKA_PASSWORD:+password: ${KAFKA_PASSWORD}}
        ${KAFKA_WORKER:+worker: ${KAFKA_WORKER}}
        ${KAFKA_PARTITION_KEY:+key: ${KAFKA_PARTITION_KEY}}
        ${KAFKA_PARTITION:+partition: ${KAFKA_PARTITION}}
        ${KAFKA_CLIENT_ID:+client_id: ${KAFKA_CLIENT_ID}}
        ${KAFKA_METADATA:+metadata: ${KAFKA_METADATA}}
        ${KAFKA_BULK_MAX_SIZE:+bulk_max_size: ${KAFKA_BULK_MAX_SIZE}}
        ${KAFKA_BROKER_TIMEOUT:+broker_timeout: ${KAFKA_BROKER_TIMEOUT}}
        ${KAFKA_CHANNEL_BUFFER_SIZE:+channel_buffer_size: ${KAFKA_CHANNEL_BUFFER_SIZE}}
        ${KAFKA_KEEP_ALIVE:+keep_alive ${KAFKA_KEEP_ALIVE}}
        ${KAFKA_MAX_MESSAGE_BYTES:+max_message_bytes: ${KAFKA_MAX_MESSAGE_BYTES}}
        ${KAFKA_REQUIRE_ACKS:+required_acks: ${KAFKA_REQUIRE_ACKS}}
    EOF
    }

    count(){
    cat >> $FILEBEAT_CONFIG << EOF
    $(base)
    output.count:
    EOF
    }

    if [ -n "$FILEBEAT_OUTPUT" ]; then
        LOGGING_OUTPUT=$FILEBEAT_OUTPUT
    fi

    case "$LOGGING_OUTPUT" in
        elasticsearch)
            es;;
        logstash)
            logstash;;
        file)
            file;;
        redis)
            redis;;
        kafka)
            kafka;;
        count)
            count;;
        *)
            default
    esac



kind: ConfigMap
metadata:
  name: config5.filebeat
  namespace: ns-elastic

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: log-pilot5
  namespace: ns-elastic
  labels:
    k8s-app: log-pilot5
spec:
  updateStrategy:
    type: RollingUpdate
  selector:
    matchLabels:
      k8s-app: log-pilot5
  template:
    metadata:
      labels:
        k8s-app: log-pilot5
    spec:
      tolerations:
      - key: /master
        effect: NoSchedule
      - effect: NoSchedule
        key: ToBeDeletedByClusterAutoscaler
        operator: Exists
      containers:
      - name: log-pilot5
        image: williamguozi/log-pilot-filebeat:containerd
        imagePullPolicy: IfNotPresent
        env:
          - name: "LOGGING_OUTPUT"
            valueFrom:
              configMapKeyRef:
                name: log-pilot5-configuration
                key: logging_output
          - name: "KAFKA_BROKERS"
            valueFrom:
              configMapKeyRef:
                name: log-pilot5-configuration
                key: kafka_brokers
          - name: "KAFKA_VERSION"
            valueFrom:
              configMapKeyRef:
                name: log-pilot5-configuration
                key: kafka_version
          #- name: "KAFKA_USERNAME"
          #  valueFrom:
          #    configMapKeyRef:
          #      name: log-pilot5-configuration
          #      key: kafka_username
          #- name: "KAFKA_PASSWORD"
          #  valueFrom:
          #    configMapKeyRef:
          #      name: log-pilot5-configuration
          #      key: kafka_password
          - name: "NODE_NAME"
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
        volumeMounts:
        - name: sock
          mountPath: /var/run/containerd/containerd.sock
        - name: logs
          mountPath: /var/log/filebeat
        - name: state
          mountPath: /var/lib/filebeat
        - name: root
          mountPath: /host
          readOnly: true
        - name: localtime
          mountPath: /etc/localtime
        - name: config-volume
          mountPath: /etc/filebeat/config
        - name: filebeat-tpl
          mountPath: /pilot/filebeat.tpl
          subPath: filebeat.tpl
        - name: config-filebeat
          mountPath: /pilot/config.filebeat
          subPath: config.filebeat
        securityContext:
          capabilities:
            add:
            - SYS_ADMIN
      terminationGracePeriodSeconds: 30
      volumes:
      - name: sock
        hostPath:
          path: /var/run/containerd/containerd.sock
          type: Socket
      - name: logs
        hostPath:
          path: /var/log/filebeat
          type: DirectoryOrCreate
      - name: state
        hostPath:
          path: /var/lib/filebeat
          type: DirectoryOrCreate
      - name: root
        hostPath:
          path: /
          type: Directory
      - name: localtime
        hostPath:
          path: /etc/localtime
          type: File
      - name: config-volume
        configMap:
          name: log-pilot5-configuration
          items:
          - key: kafka_topics
            path: kafka_topics
      - name: filebeat-tpl
        configMap:
          name: filebeat5-tpl
      - name: config-filebeat
        configMap:
          name: config5.filebeat
          defaultMode: 0777

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.
155.
156.
157.
158.
159.
160.
161.
162.
163.
164.
165.
166.
167.
168.
169.
170.
171.
172.
173.
174.
175.
176.
177.
178.
179.
180.
181.
182.
183.
184.
185.
186.
187.
188.
189.
190.
191.
192.
193.
194.
195.
196.
197.
198.
199.
200.
201.
202.
203.
204.
205.
206.
207.
208.
209.
210.
211.
212.
213.
214.
215.
216.
217.
218.
219.
220.
221.
222.
223.
224.
225.
226.
227.
228.
229.
230.
231.
232.
233.
234.
235.
236.
237.
238.
239.
240.
241.
242.
243.
244.
245.
246.
247.
248.
249.
250.
251.
252.
253.
254.
255.
256.
257.
258.
259.
260.
261.
262.
263.
264.
265.
266.
267.
268.
269.
270.
271.
272.
273.
274.
275.
276.
277.
278.
279.
280.
281.
282.
283.
284.
285.
286.
287.
288.
289.
290.
291.
292.
293.
294.
295.
296.
297.
298.
299.
300.
301.
302.
303.
304.
305.
306.
307.
308.
309.
310.
311.
312.
313.
314.
315.
316.
317.
318.
319.
320.
321.
322.
323.
324.
325.
326.
327.
328.
329.
330.
331.
332.
333.
334.
335.
336.
337.
338.
339.
340.
341.
342.
343.
344.
345.
346.
347.
348.
349.
350.
351.
352.
353.
354.
355.
356.
357.
358.
359.
360.
361.
362.
363.
364.
365.
366.
367.
368.

部署 filebeat 服务

cat filebeat.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config-test
  namespace: es
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: kafka
      hosts: ["192.168.35.226:9092"]
      topics: ["agent-nginx-test", "svip-nginx-test"]
      group_id: "filebeat"
      ssl.enabled: false
    filebeat.shutdown_timeout: 30s

    output.elasticsearch:
      enabled: true
      hosts: [':9200']
      username: elastic
      password: lM0r0qG4OT2Yc5FD7jr6t879
      indices:
        - index: "agent-nginx-test-%{+yyyy.MM.dd}"
          when.equals:
            kafka.topic: "agent-nginx-test"
        - index: "svip-nginx-test-%{+yyyy.MM.dd}"
          when.equals:
            kafka.topic: "svip-nginx-test"	
    
    logging.level: debug

---

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app-name: filebeat
  name: filebeat-test
  namespace: es
spec:
  replicas: 1
  selector:
    matchLabels:
      app-name: filebeat
  template:
    metadata:
      labels:
        app-name: filebeat
    spec:
      containers:
      - command:
        - filebeat
        - -e
        - -c
        - /etc/filebeat.yml
        env:
        - name: TZ
          value: Asia/Shanghai
        image: /elastic/filebeat:8.11.3
        imagePullPolicy: IfNotPresent
        name: filebeat
        resources:
          #limits:
            #cpu: 100m
            #memory: 200M
          requests:
            cpu: 100m
            memory: 200M
        volumeMounts:
        - mountPath: /etc/filebeat.yml
          name: filebeat-config
          subPath: filebeat.yml
      volumes:
      - configMap:
          defaultMode: 420
          name: filebeat-config-test
        name: filebeat-config

部署测试服务nginx

cat svip-nginx-alb.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
  labels:
    app: svip-nginx1
  name: svip-nginx-deployment
  namespace: server-svip
spec:
  selector:
    matchLabels:
      app: svip-nginx1
  template:
    metadata:
      labels:
        app: svip-nginx1
    spec:
      containers:
        - env:
            - name: aliyun_logs_svip-nginx-test           #名字要这样aliyun_logs是log-pilot默认的
              value: /var/log/nginx/*.log                 #日志的文件路径
          image: /svip/test1/nginx:v0.0.1
          imagePullPolicy: IfNotPresent
          name: svip-nginx1
          ports:
            - containerPort: 80
              protocol: TCP
          volumeMounts:
            - mountPath: /var/log/nginx/                  #这个地方是把容器内部的路径共享到外边
              name: log-pilot
      volumes:
        - name: log-pilot
          emptyDir: {}                                    #意思是当Pod销毁时，EmptyDir中的数据也会被永久删除