背景

在大型分布式系统中，服务之间调用复杂，链路追踪可以帮助梳理请求流向，现代系统也需要实时监控来快速响应事件以及故障，让我们了解系统瓶颈和高负载路径，从而可以进行优化。
Ingress-nginx是在 Kubernetes 环境中使用的，专门用于管理进入 Kubernetes 集群的外部访问流量。它基于 Nginx，利用其作为反向代理和负载均衡器的能力，但专门配置和优化以适应 Kubernetes 的架构。Ingress Controller 的主要任务是根据预先定义的规则（通过 Kubernetes Ingress 资源设置）将外部请求路由到集群内的特定服务。
前提
ingress-nginx 版本 >= 1.10.0
应用服务已经接入Opentelemetry采集链路数据

部署示例服务

这里我们会部署一个spring boot的服务，A服务会调用B服务。本示例中java版本是17，Maven版本是3.9.10。
由于采集ingress-nginx的链路需要和后端链路打通，所以在部署业务镜像的时候需要讲otel探针一并打包到业务镜像。
以下是在服务Dockerfile中将Agnet打包到业务服务容器镜像的配置，为服务提供采集链路数据的基础能力

FROM curlimages/curl:latest AS agent-download
USER root
RUN curl -Lo /opentelemetry-javaagent.jar \
    https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar

FROM openjdk:17-jdk-slim

WORKDIR /app

COPY --from=agent-download /opentelemetry-javaagent.jar /app/opentelemetry-javaagent.jar

COPY target/serviceb-1.0-SNAPSHOT.jar /app/service-b.jar

ENV OTEL_SERVICE_NAME="service-b" \
    OTEL_EXPORTER_OTLP_ENDPOINT="http://datakit-endpoint:4317" \
    OTEL_TRACES_SAMPLER="parentbased_always_on" \
    OTEL_PROPAGATORS="tracecontext,baggage" \
    OTEL_METRICS_EXPORTER="none" \
    OTEL_LOGS_EXPORTER="none"

# 修改启动命令，添加 Java Agent
CMD ["java", "-javaagent:/app/opentelemetry-javaagent.jar", "-jar", "/app/service-b.jar"]

创建 k8s-java-app.yaml部署服务

apiVersion: apps/v1
kind: Deployment
metadata:
  name: service-a
spec:
  replicas: 1
  selector:
    matchLabels:
      app: service-a
  template:
    metadata:
      labels:
        app: service-a
    spec:
      containers:
        - name: service-a
          image: <your-repo>/service-a:otel-1.0
          ports:
            - containerPort: 9090
          env:
          - name: HOST_IP
            valueFrom:
              fieldRef:
                fieldPath: status.hostIP
          - name: SPRING_MAIN_ALLOW_CIRCULAR_REFERENCES
            value: "true"
          - name: OTEL_SERVICE_NAME
            value: "service-a"
          - name: OTEL_EXPORTER
            value: "otlp"
          - name: OTEL_EXPORTER_OTLP_PROTOCOL
            value: "grpc"
          - name: OTEL_EXPORTER_OTLP_ENDPOINT
            value: "http://$(HOST_IP):4317"
          - name: OTEL_PROPAGATORS
            value: "tracecontext,baggage"

apiVersion: v1
kind: Service
metadata:
  name: service-a
spec:
  ports:
    - port: 9090
      targetPort: 9090
  selector:
    app: service-a
apiVersion: apps/v1
kind: Deployment
metadata:
  name: service-b
spec:
  replicas: 1
  selector:
    matchLabels:
      app: service-b
  template:
    metadata:
      labels:
        app: service-b
    spec:
      containers:
        - name: service-b
          image: <your-repo>/service-b:otel-1.0
          ports:
            - containerPort: 8090
          env:
          - name: HOST_IP
            valueFrom:
              fieldRef:
                fieldPath: status.hostIP
          - name: OTEL_SERVICE_NAME
            value: "service-b"
          - name: OTEL_EXPORTER
            value: "otlp"
          - name: OTEL_EXPORTER_OTLP_PROTOCOL
            value: "grpc"
          - name: OTEL_EXPORTER_OTLP_ENDPOINT
            value: "http://$(HOST_IP):4317"
          - name: OTEL_PROPAGATORS
            value: "tracecontext,baggage"
apiVersion: v1
kind: Service
metadata:
  name: service-b
spec:
  ports:
    - port: 8090
      targetPort: 8090
  selector:
    app: service-b

2. 安装 Ingress Nginx

创建一个 Ingress-nginx.yaml 文件：

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
      - services
    verbs:
      - list
      - watch
      - get
  - apiGroups:
      - "discovery.k8s.io"
    resources:
      - endpointslices
    verbs:
      - list
      - watch
  - apiGroups:
      - "coordination.k8s.io"
    resources:
      - leases
    verbs:
      - get
      - watch
      - list
      - create
      - update
  - apiGroups:
      - "networking.k8s.io"
    resources:
      - ingresses
      - ingressclasses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "networking.k8s.io"
    resources:
      - ingresses/status
    verbs:
      - update
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ingress-nginx
subjects:
  - kind: ServiceAccount
    name: ingress-nginx
    namespace: ingress-nginx
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ingress-nginx
  template:
    metadata:
      labels:
        app: ingress-nginx
    spec:
      hostNetwork: true
      serviceAccountName: ingress-nginx
      containers:
      - name: controller
        image: k8s.gcr.io/ingress-nginx/controller:v1.10.0
        args:
        - /nginx-ingress-controller
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
        - --election-id=ingress-controller-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=ingress-nginx/ingress-nginx-controller
        env:
        - name: HOST_IP
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://$(HOST_IP):4317"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        ports:
        - name: http
          containerPort: 80
        - name: https
          containerPort: 443
---
apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    targetPort: 80
  - name: https
    port: 443
    targetPort: 443
  selector:
    app: ingress-nginx
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
data:
  enable-opentelemetry: "true"
  otel-sampler: AlwaysOn
  opentelemetry-operation-name: "HTTP $request_method $service_name $uri $opentelemetry_trace_id"
  opentelemetry-trust-incoming-span: "true"
  # Defaults
  # otel-service-name: "nginx"
  # otel-sampler-ratio: 0.01

应用该配置：

kubectl apply -f ingress-nginx.yaml

在这里插入图片描述

采集ingress-nginx链路配置

Datakit开启Otel采集器

datakit.yaml中采用CM挂载方式开启集群的otel采集器：
在volumeMounts添加：

        - mountPath: /usr/local/datakit/conf.d/opentelemetry/opentelemetry.conf
          name: datakit-conf
          subPath: opentelemetry.conf

在这里插入图片描述
在CM处添加采集器：

    opentelemetry.conf: |-
        [[inputs.opentelemetry]]
          [inputs.opentelemetry.http]
           enable = true
           http_status_ok = 200
           trace_api = "/otel/v1/traces"
          [inputs.opentelemetry.grpc]
           trace_enable = true
           metric_enable = true
           addr = "0.0.0.0:4317"

在这里插入图片描述
重启datakit

kubectl apply -f datakit.yaml

OTEL Agent采集链路数据

在服务Dockerfile中将Agnet打包到业务服务容器镜像，为服务提供采集链路数据的基础能力

FROM curlimages/curl:latest AS agent-download
USER root
RUN curl -Lo /opentelemetry-javaagent.jar \
    https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar

FROM openjdk:17-jdk-slim

WORKDIR /app

COPY --from=agent-download /opentelemetry-javaagent.jar /app/opentelemetry-javaagent.jar

COPY target/serviceb-1.0-SNAPSHOT.jar /app/service-b.jar

ENV OTEL_SERVICE_NAME="service-b" \
    OTEL_EXPORTER_OTLP_ENDPOINT="http://datakit-endpoint:4317" \
    OTEL_TRACES_SAMPLER="parentbased_always_on" \
    OTEL_PROPAGATORS="tracecontext,baggage" \
    OTEL_METRICS_EXPORTER="none" \
    OTEL_LOGS_EXPORTER="none"

# 修改启动命令，添加 Java Agent
CMD ["java", "-javaagent:/app/opentelemetry-javaagent.jar", "-jar", "/app/service-b.jar"]

在服务部署的yaml中配置环境变量

          - name: OTEL_EXPORTER
            value: "otlp"
          - name: OTEL_EXPORTER_OTLP_PROTOCOL
            value: "grpc"
          - name: OTEL_EXPORTER_OTLP_ENDPOINT
            value: "http://$(HOST_IP):4317"
          - name: OTEL_PROPAGATORS
            value: "tracecontext,baggage"

编辑ingress-controller CM 资源

如果ingress-controller服务有configmap则在CM中增加如下四行：

enable-opentelemetry: "true"
otel-sampler: AlwaysOn
opentelemetry-operation-name: "HTTP $request_method $service_name $uri $opentelemetry_trace_id"
opentelemetry-trust-incoming-span: "true"

在这里插入图片描述
Apply 相应的ingress的yaml，并重启Ingress-controller

增加ingress-controller环境变量

在部署ingress-controller配置文件ingress-nginx.yaml的deployment部分中添加otel配置，位置在spec.template.spec.containers.env下，注意端口开启

        - name: OTEL_EXPORTER
          value: "otlp"
        - name: OTEL_EXPORTER_OTLP_PROTOCOL
          value: "grpc"

        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://$(HOST_IP):4317"

        - name: OTEL_SERVICE_NAME
          value: "nginx"
        - name: OTEL_TRACES_SAMPLER
          value: "always_on"
        - name: OTEL_PROPAGATORS
          value: "tracecontext,baggage"

在这里插入图片描述
重新apply ingress-nginx.yaml，重启ingress-controller容器

观测云

再次访问ingress域名制造数据
在这里插入图片描述
到观测云控制台应用性能检测部分，可以看到ingress-nginx链路数据正常上报
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述

# 运维
Logo
开源鸿蒙跨平台开发者社区

开源鸿蒙跨平台开发社区汇聚开发者与厂商，共建“一次开发，多端部署”的开源生态，致力于降低跨端开发门槛，推动万物智联创新。

更多推荐

cover

刷机全流程SD-eMMC-单分区—【万物智能之开源鸿蒙OpenHarmony系统实战开发系列教程】

avatar 开源鸿蒙跨平台开发者社区

开启RN之旅——React基础

文件格式: 后缀名为.json的纯JSON格式文件，包含大括号包裹的多个字段定义核心作用: 管理React Native工程的所有配置和依赖关系name: 应用名称，如示例中的"demo"version: 应用版本号，格式为"主版本.次版本.修订号"，如"0.0.1"private: 布尔值，设为true可防止工程被意外发布到npm仓库注意: 当设置为true时，执行npm publish会报错s

avatar 开源鸿蒙跨平台开发者社区
cover

react navite调试方案：Flipper、远程调试

avatar 开源鸿蒙跨平台开发者社区