244 lines
7.4 KiB
YAML
244 lines
7.4 KiB
YAML
---
|
|
apiVersion: opentelemetry.io/v1beta1
|
|
kind: OpenTelemetryCollector
|
|
metadata:
|
|
name: otel-collector-logs
|
|
namespace: otel-lgtm-stack
|
|
spec:
|
|
config:
|
|
exporters:
|
|
debug:
|
|
verbosity: detailed
|
|
otlp:
|
|
endpoint: http://lgtm-stack.otel-lgtm-stack.svc.cluster.local:4317
|
|
tls:
|
|
insecure: true
|
|
otlphttp/ops-loki:
|
|
auth:
|
|
authenticator: basicauth/loki-tenant
|
|
endpoint: http://otel-collector-log-proxy.headscale.svc.cluster.local:80/otlp
|
|
headers:
|
|
X-Scope-OrgID: loki-tenant-cluster-name
|
|
tls:
|
|
insecure: true
|
|
extensions:
|
|
basicauth/loki-tenant:
|
|
client_auth:
|
|
password: loki-tenant-demo-password
|
|
username: loki_tenant_demo
|
|
processors:
|
|
attributes:
|
|
actions:
|
|
- action: insert
|
|
key: k8s_cluster_name
|
|
value: cluster-name
|
|
batch:
|
|
send_batch_size: 2000
|
|
timeout: 10s
|
|
k8sattributes:
|
|
auth_type: serviceAccount
|
|
extract:
|
|
labels:
|
|
- from: pod
|
|
key: app.kubernetes.io/name
|
|
tag_name: service.name
|
|
- from: pod
|
|
key: k8s-app
|
|
tag_name: service.name
|
|
- from: pod
|
|
key: app.kubernetes.io/instance
|
|
tag_name: k8s.app.instance
|
|
- from: pod
|
|
key: app.kubernetes.io/version
|
|
tag_name: service.version
|
|
- from: pod
|
|
key: app.kubernetes.io/component
|
|
tag_name: k8s.app.component
|
|
- from: pod
|
|
key: airm.silogen.ai/component-id
|
|
tag_name: component.id
|
|
- from: pod
|
|
key: airm.silogen.ai/workload-id
|
|
tag_name: workload.id
|
|
metadata:
|
|
- k8s.namespace.name
|
|
- k8s.pod.name
|
|
- k8s.pod.uid
|
|
- k8s.node.name
|
|
- k8s.pod.start_time
|
|
- k8s.deployment.name
|
|
- k8s.replicaset.name
|
|
- k8s.replicaset.uid
|
|
- k8s.daemonset.name
|
|
- k8s.daemonset.uid
|
|
- k8s.job.name
|
|
- k8s.job.uid
|
|
- k8s.container.name
|
|
- k8s.cronjob.name
|
|
- k8s.statefulset.name
|
|
- k8s.statefulset.uid
|
|
- container.image.tag
|
|
- container.image.name
|
|
- k8s.cluster.uid
|
|
passthrough: false
|
|
pod_association:
|
|
- sources:
|
|
- from: resource_attribute
|
|
name: k8s.pod.uid
|
|
- sources:
|
|
- from: resource_attribute
|
|
name: k8s.pod.name
|
|
- from: resource_attribute
|
|
name: k8s.namespace.name
|
|
- from: resource_attribute
|
|
name: k8s.node.name
|
|
- sources:
|
|
- from: resource_attribute
|
|
name: k8s.pod.ip
|
|
- sources:
|
|
- from: resource_attribute
|
|
name: k8s.pod.name
|
|
- from: resource_attribute
|
|
name: k8s.namespace.name
|
|
- sources:
|
|
- from: connection
|
|
resourcedetection:
|
|
detectors:
|
|
- env
|
|
override: true
|
|
timeout: 2s
|
|
transform:
|
|
error_mode: ignore
|
|
log_statements:
|
|
- context: log
|
|
statements:
|
|
- set(resource.attributes["k8s.cluster.name"], attributes["k8s.cluster.name"])
|
|
- set(resource.attributes["component.id"], attributes["component.id"])
|
|
- set(resource.attributes["workload.id"], attributes["workload.id"])
|
|
receivers:
|
|
filelog/std:
|
|
exclude:
|
|
- /var/log/pods/*/otel-collector/*.log
|
|
- /var/log/pods/*/otc-container/*.log
|
|
- /var/log/pods/*/openobserve-ingester/*.log
|
|
include:
|
|
- /var/log/pods/*/*/*.log
|
|
include_file_name: false
|
|
include_file_path: true
|
|
operators:
|
|
- id: get-format
|
|
routes:
|
|
- expr: body matches "^\\{"
|
|
output: parser-docker
|
|
- expr: body matches "^[^ Z]+ "
|
|
output: parser-crio
|
|
- expr: body matches "^[^ Z]+Z"
|
|
output: parser-containerd
|
|
type: router
|
|
- id: parser-crio
|
|
output: extract_metadata_from_filepath
|
|
regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
|
|
timestamp:
|
|
layout: 2006-01-02T15:04:05.999999999Z07:00
|
|
layout_type: gotime
|
|
parse_from: attributes.time
|
|
type: regex_parser
|
|
- id: parser-containerd
|
|
output: extract_metadata_from_filepath
|
|
regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
|
|
timestamp:
|
|
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
|
|
parse_from: attributes.time
|
|
type: regex_parser
|
|
- id: parser-docker
|
|
output: extract_metadata_from_filepath
|
|
timestamp:
|
|
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
|
|
parse_from: attributes.time
|
|
type: json_parser
|
|
- cache:
|
|
size: 128
|
|
id: extract_metadata_from_filepath
|
|
parse_from: attributes["log.file.path"]
|
|
regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
|
|
type: regex_parser
|
|
- from: attributes.log
|
|
to: body
|
|
type: move
|
|
- from: attributes.stream
|
|
to: attributes["log.iostream"]
|
|
type: move
|
|
- from: attributes.container_name
|
|
to: resource["k8s.container.name"]
|
|
type: move
|
|
- from: attributes.namespace
|
|
to: resource["k8s.namespace.name"]
|
|
type: move
|
|
- from: attributes.pod_name
|
|
to: resource["k8s.pod.name"]
|
|
type: move
|
|
- from: attributes.restart_count
|
|
to: resource["k8s.container.restart_count"]
|
|
type: move
|
|
- from: attributes.uid
|
|
to: resource["k8s.pod.uid"]
|
|
type: move
|
|
start_at: end
|
|
service:
|
|
extensions:
|
|
- basicauth/loki-tenant
|
|
pipelines:
|
|
logs:
|
|
exporters:
|
|
- otlp
|
|
- otlphttp/ops-loki
|
|
processors:
|
|
- batch
|
|
- k8sattributes
|
|
- attributes
|
|
- transform
|
|
receivers:
|
|
- filelog/std
|
|
env:
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.113.0
|
|
mode: daemonset
|
|
podAnnotations:
|
|
prometheus.io/port: "8888"
|
|
prometheus.io/scrape: "true"
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: 200m
|
|
memory: 400Mi
|
|
securityContext:
|
|
privileged: true
|
|
runAsGroup: 0
|
|
runAsUser: 0
|
|
serviceAccount: otel-collector
|
|
tolerations:
|
|
- effect: NoSchedule
|
|
key: exampleKey1
|
|
operator: Equal
|
|
value: "true"
|
|
volumeMounts:
|
|
- mountPath: /var/log
|
|
name: varlog
|
|
- mountPath: /var/lib/docker/containers
|
|
name: varlibdockercontainers
|
|
readOnly: true
|
|
volumes:
|
|
- hostPath:
|
|
path: /var/log
|
|
type: ""
|
|
name: varlog
|
|
- hostPath:
|
|
path: /var/lib/docker/containers
|
|
type: ""
|
|
name: varlibdockercontainers
|