k8s 可观测性

# 查看 Pod 日志
kubectl logs nginx-deploy-7c5b9b6f6d-x9k2m

# 跟踪实时日志
kubectl logs -f nginx-deploy-7c5b9b6f6d-x9k2m

# 查看多容器 Pod 中指定容器
kubectl logs nginx-deploy-7c5b9b6f6d-x9k2m -c sidecar

# 查看最近 1 小时的日志
kubectl logs --since=1h nginx-deploy-7c5b9b6f6d-x9k2m

# 查看前一个容器的日志（崩溃后排查）
kubectl logs --previous nginx-deploy-7c5b9b6f6d-x9k2m

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: kube-system
  labels:
    k8s-app: fluent-bit
spec:
  selector:
    matchLabels:
      k8s-app: fluent-bit
  template:
    metadata:
      labels:
        k8s-app: fluent-bit
    spec:
      containers:
      - name: fluent-bit
        image: fluent/fluent-bit:2.2.0
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: config
          mountPath: /fluent-bit/etc/
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: config
        configMap:
          name: fluent-bit-config

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: kube-system
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush         5
        Log_Level     info

    [INPUT]
        Name              tail
        Path              /var/log/containers/*.log
        Parser            docker
        Tag               kube.*
        Mem_Buf_Limit     10MB
        Skip_Long_Lines   On

    [OUTPUT]
        Name            es
        Match           *
        Host            elasticsearch.logging.svc.cluster.local
        Port            9200
        Index           k8s-logs
        Type            _doc

# 添加 Helm 仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# 安装到 monitoring 命名空间
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace

# 查看 Pod 状态
kubectl get pods -n monitoring

kubectl patch svc kube-prometheus-stack-grafana -n monitoring \
  -p '{"spec":{"type":"NodePort"}}'

# 获取初始密码
kubectl get secret -n monitoring kube-prometheus-stack-grafana \
  -o jsonpath="{.data.admin-password}" | base64 -d ; echo

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: node-alerts
  namespace: monitoring
  labels:
    prometheus: k8s
spec:
  groups:
  - name: node.rules
    rules:
    - alert: NodeHighCPUUsage
      expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "节点 CPU 使用率过高"
        description: "节点 {{ $labels.instance }} CPU 使用率超过 80%，持续 5 分钟。"

    - alert: PodCrashLooping
      expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "容器频繁重启"
        description: "Pod {{ $labels.pod }} 在 1 小时内重启超过 5 次。"

apiVersion: v1
kind: Secret
metadata:
  name: alertmanager-config
  namespace: monitoring
type: Opaque
stringData:
  alertmanager.yaml: |
    global:
      resolve_timeout: 5m
    route:
      group_by: ['alertname']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      receiver: 'default'
    receivers:
    - name: 'default'
      webhook_configs:
      - url: 'https://hooks.slack.com/services/xxx'

# 查看某 Pod 的所有事件
kubectl get events --field-selector involvedObject.name=<pod-name>

# 按类型筛选 Warning 事件
kubectl get events --field-selector type=Warning

# 按时间排序查看命名空间事件
kubectl get events -n default --sort-by=.metadata.creationTimestamp

# 以 YAML 格式查看事件详情
kubectl get event <event-name> -o yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
  namespace: tracing
spec:
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      containers:
      - name: otel-collector
        image: otel/opentelemetry-collector-contrib:0.90.0
        ports:
        - containerPort: 4317
          name: otlp-grpc
        - containerPort: 4318
          name: otlp-http
        - containerPort: 16686
          name: jaeger-ui