Configura ingestão de logs pelo OTel Collector

Author: ogecece

Makefile

@@ -135,7 +135,7 @@ load-data:
 
 .PHONY: re-run
 re-run: setup-environment wait-opensearch wait-database
-	$(call run-command, opentelemetry-instrument --traces_exporter console --metrics_exporter console --logs_exporter console --service_name querido-diario-api python main)
+	$(call run-command, opentelemetry-instrument python main)
 
 .PHONY: runshell
 runshell:
@@ -193,7 +193,8 @@ start-otel-collector:
 	podman run -d --rm -ti \
 		--name $(OTEL_COLLECTOR_CONTAINER_NAME) \
 		--pod $(POD_NAME) \
-		docker.io/otel/opentelemetry-collector-contrib:0.97.0
+		--volume $(PWD)/config/otel-collector-config.yaml:/etc/otel-collector-config.yaml \
+		docker.io/otel/opentelemetry-collector-contrib:0.97.0 "--config=/etc/otel-collector-config.yaml"
 
 stop-otel-collector:
 	podman rm --force --ignore $(OTEL_COLLECTOR_CONTAINER_NAME)

api/api.py

@@ -16,7 +16,6 @@
 from themed_excerpts import ThemedExcerptAccessInterface, ThemedExcerptAccessInterface
 from themed_excerpts.themed_excerpt_access import ThemedExcerptRequest
 
-logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 config = load_configuration()
 

config/otel-collector-config.yaml

@@ -0,0 +1,16 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+exporters:
+  debug:
+    verbosity: detailed
+processors:
+  batch:
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+      processors: [batch]

config/sample.env

@@ -32,3 +32,17 @@ THEMED_EXCERPT_EMBEDDING_SCORE_FIELD=excerpt_embedding_score
 THEMED_EXCERPT_TFIDF_SCORE_FIELD=excerpt_tfidf_score
 THEMED_EXCERPT_FRAGMENT_SIZE=10000
 THEMED_EXCERPT_NUMBER_OF_FRAGMENTS=1
+
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true
+OTEL_PYTHON_LOG_CORRELATION=true
+OTEL_PYTHON_LOG_LEVEL=debug
+OTEL_PYTHON_LOG_FORMAT="%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s trace_sampled=%(otelTraceSampled)s] - %(message)s"
+OTEL_SERVICE_NAME=querido-diario-api
+OTEL_LOGS_EXPORTER=console,otlp
+OTEL_TRACES_EXPORTER=console,otlp
+OTEL_METRICS_EXPORTER=console,otlp
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:4317
+
+# valor alto para não atrapalhar tanto a leitura dos logs durante o desenvolvimento
+OTEL_METRIC_EXPORT_INTERVAL=60000

requirements.txt

@@ -8,4 +8,5 @@ psycopg2==2.8.5
 SQLAlchemy==1.3.19
 opensearch-py==2.3.2
 mailjet-rest==1.3.4
-opentelemetry-distro==0.44b0
+opentelemetry-distro==0.45b0
+opentelemetry-exporter-otlp==1.24.0