Configura ingestão de logs pelo OTel Collector (#70)

Author: ogecece

Dockerfile

@@ -15,6 +15,4 @@ RUN chmod 644 censo.csv
 ADD https://raw.githubusercontent.com/okfn-brasil/querido-diario-data-processing/main/config/themes_config.json themes_config.json
 RUN chmod 644 themes_config.json
 
-RUN opentelemetry-bootstrap -a install
-
 USER gazette

Makefile

@@ -135,7 +135,7 @@ load-data:
 
 .PHONY: re-run
 re-run: setup-environment wait-opensearch wait-database
-	$(call run-command, opentelemetry-instrument --traces_exporter console --metrics_exporter console --logs_exporter console --service_name querido-diario-api python main)
+	$(call run-command, opentelemetry-instrument python main)
 
 .PHONY: runshell
 runshell:
@@ -193,10 +193,15 @@ start-otel-collector:
 	podman run -d --rm -ti \
 		--name $(OTEL_COLLECTOR_CONTAINER_NAME) \
 		--pod $(POD_NAME) \
-		docker.io/otel/opentelemetry-collector-contrib:0.97.0
+		--volume $(PWD)/config/otel-collector-config.yaml:/etc/otel-collector-config.yaml \
+		docker.io/otel/opentelemetry-collector-contrib:0.97.0 "--config=/etc/otel-collector-config.yaml"
 
 stop-otel-collector:
 	podman rm --force --ignore $(OTEL_COLLECTOR_CONTAINER_NAME)
 
 wait-otel-collector:
 	$(call wait-for, localhost:4317)
+
+otel-auto-instrumentation-list:
+	@echo "These packages were detected and can be auto-instrumented (maybe add/update them in requirements.txt):"
+	@$(call run-command, opentelemetry-bootstrap -a requirements)

api/api.py

@@ -16,8 +16,10 @@
 from themed_excerpts import ThemedExcerptAccessInterface, ThemedExcerptAccessInterface
 from themed_excerpts.themed_excerpt_access import ThemedExcerptRequest
 
-logging.basicConfig(level=logging.DEBUG)
+
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 config = load_configuration()
 
 app = FastAPI(

config/otel-collector-config.yaml

@@ -0,0 +1,24 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+exporters:
+  debug:
+    verbosity: detailed
+processors:
+  batch:
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      exporters: [debug]
+      processors: [batch]
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+      processors: [batch]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug]
+      processors: [batch]

config/sample.env

@@ -32,3 +32,17 @@ THEMED_EXCERPT_EMBEDDING_SCORE_FIELD=excerpt_embedding_score
 THEMED_EXCERPT_TFIDF_SCORE_FIELD=excerpt_tfidf_score
 THEMED_EXCERPT_FRAGMENT_SIZE=10000
 THEMED_EXCERPT_NUMBER_OF_FRAGMENTS=1
+
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true
+OTEL_PYTHON_LOG_CORRELATION=true
+OTEL_PYTHON_LOG_LEVEL=debug
+OTEL_PYTHON_LOG_FORMAT="%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s trace_sampled=%(otelTraceSampled)s] - %(message)s"
+OTEL_SERVICE_NAME=querido-diario-api
+OTEL_LOGS_EXPORTER=console,otlp
+OTEL_TRACES_EXPORTER=console,otlp
+OTEL_METRICS_EXPORTER=console,otlp
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:4317
+
+# valor alto para não atrapalhar tanto a leitura dos logs durante o desenvolvimento
+OTEL_METRIC_EXPORT_INTERVAL=60000

requirements.txt

@@ -8,4 +8,22 @@ psycopg2==2.8.5
 SQLAlchemy==1.3.19
 opensearch-py==2.3.2
 mailjet-rest==1.3.4
-opentelemetry-distro==0.44b0
+opentelemetry-distro==0.45b0
+opentelemetry-exporter-otlp==1.24.0
+opentelemetry-instrumentation==0.45b0
+opentelemetry-instrumentation-asyncio==0.45b0
+opentelemetry-instrumentation-aws-lambda==0.45b0
+opentelemetry-instrumentation-dbapi==0.45b0
+opentelemetry-instrumentation-logging==0.45b0
+opentelemetry-instrumentation-sqlite3==0.45b0
+opentelemetry-instrumentation-urllib==0.45b0
+opentelemetry-instrumentation-wsgi==0.45b0
+opentelemetry-instrumentation-asgi==0.45b0
+opentelemetry-instrumentation-fastapi==0.45b0
+opentelemetry-instrumentation-grpc==0.45b0
+opentelemetry-instrumentation-psycopg2==0.45b0
+opentelemetry-instrumentation-requests==0.45b0
+opentelemetry-instrumentation-sqlalchemy==0.45b0
+opentelemetry-instrumentation-starlette==0.45b0
+opentelemetry-instrumentation-tortoiseorm==0.45b0
+opentelemetry-instrumentation-urllib3==0.45b0