Filebeat Kubernetes 配置 - 收集和发送日志到 Kafka

apiVersion: v1 kind: ConfigMap metadata: name: filebeat-config namespace: filebeat labels: k8s-app: filebeat data: filebeat.yml: | filebeat.autodiscover: providers: - type: kubernetes node: '${NODE_NAME}' hints.enabled: true templates: - config: - type: container paths: - /var/lib/docker/containers/*/${data.kubernetes.container.id}-json.log exclude_lines: ['^\s+[-`('.|_]'] close_timeout: 2h filebeat.inputs: - type: log paths: - /var/log/messages fields: kubernetes.pod.ip: '${POD_IP}' fields_under_root: true include_lines: ['(atomic-openshift-node|dockerd-current)'] processors: - add_cloud_metadata: - drop_event.when.not.or: - equals.input.type: 'log' - equals.kubernetes.namespace: 'default' - equals.kubernetes.namespace: 'kube-system' - equals.kubernetes.namespace: 'openshift-monitoring' - equals.kubernetes.namespace: 'openshift-sdn' - equals.kubernetes.namespace: 'openshift-node' - equals.kubernetes.namespace: 'ccnp-system' - equals.kubernetes.namespace: 'openshift-infra' - equals.kubernetes.namespace: 'openshift-metrics-server' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-node-exporter' cluster: 'cdgxsc-01' namespace: 'openshift-monitoring' when: and: - equals: kubernetes.namespace: 'openshift-monitoring' - contains: kubernetes.pod.name: 'node-exporter' - equals: kubernetes.container.name: 'node-exporter' - add_fields: fields: sysCode: 'l002x0' middleware: 'prometheus-k8s' cluster: 'cdgxsc-01' namespace: 'openshift-monitoring' when: and: - equals: kubernetes.namespace: 'openshift-monitoring' - contains: kubernetes.pod.name: 'prometheus-k8s' - equals: kubernetes.container.name: 'prometheus' - add_fields: fields: sysCode: 'l002x0' middleware: 'k8s-ccnp-cluster-controller' cluster: 'cdgxsc-01' namespace: 'ccnp-system' when: and: - contains: kubernetes.pod.name: 'ccnp-cluster-controller' - equals: kubernetes.namespace: 'ccnp-system' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-etcd' cluster: 'cdgxsc-01' namespace: 'kube-system' when: and: - contains: kubernetes.pod.name: 'master-etcd' - equals: kubernetes.namespace: 'kube-system' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-apiserver' cluster: 'cdgxsc-01' namespace: 'kube-system' when: and: - contains: kubernetes.pod.name: 'master-api' - equals: kubernetes.namespace: 'kube-system' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-controller' cluster: 'cdgxsc-01' namespace: 'kube-system' when: and: - contains: kubernetes.pod.name: 'master-controllers' - equals: kubernetes.namespace: 'kube-system' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-ovs' cluster: 'cdgxsc-01' namespace: 'openshift-sdn' when: and: - contains: kubernetes.pod.name: 'ovs' - equals: kubernetes.namespace: 'openshift-sdn' - add_fields: fields: sysCode: 'l002x0' middleware: 'ocp-sdn' cluster: 'cdgxsc-01' namespace: 'openshift-sdn' when: and: - contains: kubernetes.pod.name: 'sdn' - equals: kubernetes.namespace: 'openshift-sdn' - add_fields: fields: sysCode: 'l002x0' middleware: 'atomic-openshift-node' cluster: 'cdgxsc-01' namespace: '' when: and: - contains: message: 'atomic-openshift-node' - equals: log.file.path: '/var/log/messages' - add_fields: fields: sysCode: 'l002x0' middleware: 'docker-ocp' cluster: 'cdgxsc-01' namespace: '' when: and: - contains: message: 'dockerd-current' - equals: log.file.path: '/var/log/messages' - script: lang: javascript source: > function process(event) { event.Put('host.ip', [event.Get('kubernetes.pod.ip')]); } - drop_fields: fields: ['agent', 'input', 'ecs', 'host.os', 'host.architecture', 'host.id', 'host.mac', 'host.containerized', 'host.hostname', 'kubernetes', 'node', 'container']

output:
  kafka:
    enabled: true
    hosts: ['10.8.54.58:9092','10.8.54.59:9092','10.8.54.60:9092']
    topic: middleware-l002x0
    username: middleware-l002x0
    password: sYkc_0104
    sasl.mechanism: 'SCRAM-SHA-256'
    worker: 3
   # 最大重试次数
    max_retries: 3
    #保持连接时间
    keep_alive: 60
    partition.hash:
      reachable_only: false
    required_ack: 1