Add Otel integration and Byte encode json attachments in feedback emails

apigateway/app.py

@@ -11,6 +11,19 @@
 from apigateway import exceptions, extensions, views
 from apigateway.models import OAuth2Client, OAuth2Token, User
 
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader, ConsoleMetricExporter
+
+
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
 
 def register_extensions(app: Flask):
     """Register extensions.
@@ -224,6 +237,37 @@ def create_app(**config):
         except ValueError:
             app.logger.warning("Most likely the SECRET_KEY is not in hex format")
 
+    # Service name is required for most backends,
+    # and although it's not necessary for console export,
+    # it's good to set service name anyways.
+    if app.config.get('ENABLE_OTEL', False):
+        resource = Resource(attributes={
+            SERVICE_NAME: app.config.get('OTEL_SERVICE_NAME')
+        })
+        if app.config.get('ENABLE_OTEL') == 'EXPORTER':
+            tracerProvider = TracerProvider(resource=resource)
+            processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=app.config.get('OTEL_TRACES_ENDPOINT','http://localhost:4318/v1/traces')))
+            tracerProvider.add_span_processor(processor)
+            trace.set_tracer_provider(tracerProvider)
+
+            if app.config.get('OTEL_ENABLE_METRICS'):
+                reader = PeriodicExportingMetricReader(
+                    OTLPMetricExporter(endpoint=app.config.get('OTEL_METRICS_ENDPOINT', 'http://localhost:9091/v1/metrics'))
+                )
+                meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
+                metrics.set_meter_provider(meterProvider)
+
+        elif app.config.get('ENABLE_OTEL') == 'CONSOLE':
+            tracerProvider = TracerProvider(resource=resource)
+            processor = BatchSpanProcessor(ConsoleSpanExporter())
+            tracerProvider.add_span_processor(processor)
+            trace.set_tracer_provider(tracerProvider)
+
+            if app.config.get('OTEL_ENABLE_METRICS'):
+                reader = PeriodicExportingMetricReader(ConsoleMetricExporter())
+                meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
+                metrics.set_meter_provider(meterProvider)
+
     flask_api = Api(app)
     register_verbose_exception_logging(app)
     register_extensions(app)

apigateway/config.py

@@ -116,3 +116,9 @@
 KAFKA_PRODUCER_SERVICE_BOOTSTRAP_SERVERS = ["localhost:9092"]
 KAFKA_PRODUCER_SERVICE_REQUEST_TOPIC = "gatewayRequests"
 KAFKA_PRODUCER_SERVICE_REQUEST_TIMEOUT_MS = 500
+
+OTEL_ENABLE_METRICS = False
+ENABLE_OTEL = 'CONSOLE'
+OTEL_SERVICE_NAME = "api-gateway"
+OTEL_TRACES_ENDPOINT = "http://localhost:4318/v1/traces"
+OTEL_METRICS_ENDPOINT = "http://localhost:4318/v1/metrics"

apigateway/utils.py

@@ -40,6 +40,8 @@ def send_email(
     verification_url: str = "",
     mail_server: str = None,
 ):
+    current_app.logger.info("From send_email: Host header: {}".format(request.headers.get("Host", "No Host Specified")))
+
     # Do not send emails if in debug mode
     if current_app.config.get("TESTING", False):
         current_app.logger.warning(
@@ -83,7 +85,9 @@ def send_feedback_email(
     mail_server: str = None,
 ):
     # Do not send emails if in debug mode
-    if current_app.config.get("TESTING", False):
+    current_app.logger.info("From send_feedback_email: Host header: {}".format(request.headers.get("Host", "No Host Specified")))
+
+    if current_app.config.get("TESTING", False) or request.headers.get("Host", "").endswith("shadow"):
         current_app.logger.warning(
             "Feedback email with subject {} was NOT sent due to TESTING flag = True".format(
                 subject
@@ -107,14 +111,17 @@ def send_feedback_email(
     if attachments:
         for attachment in attachments:
             message.add_attachment(
-                json.dumps(attachment[1]),
-                filename=attachment[0],
+                json.dumps(attachment[1]).encode('utf-8'),
+                filename=attachment[0].strip('.txt')+str('.json'),
                 maintype="application",
                 subtype="json",
             )
 
     with smtplib.SMTP(mail_server) as s:
-        s.send_message(message)
+        try:
+            s.send_message(message)
+        except Exception:
+            current_app.logger.exception("Failed to send message for Feedback email from: {}".format(submitter_email))
 
 
 def verify_recaptcha(request: Request, endpoint: str = None):
@@ -129,9 +136,9 @@ def verify_recaptcha(request: Request, endpoint: str = None):
     Returns:
         bool: True if reCAPTCHA verification is successful, False otherwise.
     """
-
+    current_app.logger.info("From verify recaptcha: Host header: {}".format(request.headers.get("Host", "No Host Specified")))
     # Skip reCAPTCHA verification if in debug mode
-    if current_app.config.get("TESTING", False):
+    if current_app.config.get("TESTING", False) or request.headers.get("Host", "").endswith("shadow"):
         current_app.logger.warning("reCAPTCHA is NOT verified during testing")
         return True
 

apigateway/views.py

@@ -567,6 +567,29 @@ def get(self, account_data):
                         client_id=client.id, user_id=session_data["user_id"]
                     ).first()
 
+                    if token:
+                        return self._translate(token, source="session:user_id")
+                    else:
+                        # Token not found in database
+                        return {"message": "Identifier not found [ERR 020]"}, 404
+                else:
+                    # Client ID not found in database
+                    return {"message": "Identifier not found [ERR 030]"}, 404
+            elif "_user_id" in session_data:
+                # There can be more than one token per user (generally one for
+                # BBB and one for API requests), when client id is not stored
+                # in the session (typically for authenticated users) we pick
+                # just the first in the database that corresponds to BBB since
+                # sessions are used by BBB and not API requests
+                client = OAuth2Client.query.filter_by(
+                    user_id=session_data["_user_id"], name="BB client"
+                ).first()
+
+                if client:
+                    token = OAuth2Token.query.filter_by(
+                        client_id=client.id, user_id=session_data["_user_id"]
+                    ).first()
+
                     if token:
                         return self._translate(token, source="session:user_id")
                     else:
@@ -705,6 +728,7 @@ def _send_email(self, params, submitter_email, subject, email_body, attachments)
             )
             return True
         except:  # noqa
+            current_app.logger.exception("Failed to send feedback email from user: {}".format(submitter_email))
             return False
 
     def _post_to_slack(self, slack_data):

docker-compose.yaml

@@ -3,6 +3,10 @@ services:
   apigateway:
     build: .
     command: bash -c "alembic upgrade head && python wsgi.py"
+    environment:
+      OTEL_SERVICE_NAME: api-gateway
+      ENABLE_OTEL: 'EXPORTER'
+
     ports:
       - "5000:8181"
     depends_on:

migrations/versions/f2f57cb2a5b0_add_index_on_access_token.py

@@ -0,0 +1,34 @@
+"""Add index on access token
+
+Revision ID: f2f57cb2a5b0
+Revises: cb232e6e9425
+Create Date: 2025-02-24 21:33:31.076672
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "f2f57cb2a5b0"
+down_revision: Union[str, None] = "fc5e2283de94"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint("oauth2token_access_token_key1", "oauth2token", type_="unique")
+    op.create_index(
+        op.f("ix_oauth2token_access_token"), "oauth2token", ["access_token"], unique=True
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_oauth2token_access_token"), table_name="oauth2token")
+    op.create_unique_constraint("oauth2token_access_token_key1", "oauth2token", ["access_token"])
+    # ### end Alembic commands ###

pyproject.toml

@@ -33,7 +33,13 @@ dependencies = [
     'flask-cors==4.0.0',
     'flask-talisman==1.1.0',
     'jsondiff==2.0.0',
-    'sqlalchemy-citext==1.8.0'
+    'sqlalchemy-citext==1.8.0',
+    'opentelemetry-instrumentation-flask==0.51b0',
+    'opentelemetry-sdk==1.30.0',
+    'opentelemetry-exporter-otlp==1.30.0',
+    'opentelemetry-instrumentation-sqlalchemy==0.51b0'
+
+
 ]
 
 [project.urls]

wsgi.py

@@ -1,10 +1,17 @@
 from werkzeug.middleware.dispatcher import DispatcherMiddleware
 from werkzeug.serving import run_simple
 from werkzeug.wrappers import Response
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 from apigateway.app import create_app
 
-application = DispatcherMiddleware(Response("Not Found", status=404), {"/v1": create_app()})
+app = create_app()
+FlaskInstrumentor.instrument_app(app)
+
+SQLAlchemyInstrumentor().instrument(engine=app.db.engine)
+
+application = DispatcherMiddleware(Response("Not Found", status=404), {"/v1": app})
 
 if __name__ == "__main__":
 
@@ -13,6 +20,6 @@
         8181,
         application,
         use_reloader=True,
-        use_debugger=True,
+        use_debugger=False,
         threaded=True,
     )