FMD-232: Run Download report as Scheduled Job (#93)

* add copilot job for download report

* add override IAM policy

* do soft lookup on query params

* update scheduler script arguments

* scope down policy permissions

* add email override for production

* add sentry integration

copilot/download-report/manifest.yml

@@ -0,0 +1,40 @@
+# The manifest for the "download-report" job.
+# Read the full specification for the "Scheduled Job" type at:
+#  https://aws.github.io/copilot-cli/docs/manifest/scheduled-job/
+
+# Your job name will be used in naming your resources like log groups, ECS Tasks, etc.
+name: download-report
+type: Scheduled Job
+
+# Trigger for your task.
+on:
+  # The scheduled trigger for your job. You can specify a Unix cron schedule or keyword (@weekly) or a rate (@every 1h30m)
+  # AWS Schedule Expressions are also accepted: https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html
+  schedule: "@monthly"
+#retries: 3        # Optional. The number of times to retry the job before failing.
+timeout: 1h    # Optional. The timeout after which to stop the job if it's still running. You can use the units (h, m, s).
+
+# Configuration for your container and task.
+image:
+  location: ghcr.io/communitiesuk/funding-service-design-post-award-data-frontend:latest
+entrypoint: launcher
+command: python3 ./scripts/extract_download_logs.py --environment=${COPILOT_ENVIRONMENT_NAME} -m=1 --email --disable-write-file
+cpu: 256       # Number of CPU units for the task.
+memory: 512    # Amount of memory in MiB used by the task.
+
+# Optional fields for more advanced use-cases.
+#
+variables:                    # Pass environment variables as key value pairs.
+ # Sentry DSN is OK to be public see: https://docs.sentry.io/product/sentry-basics/dsn-explainer/#dsn-utilization
+ SENTRY_DSN: https://[email protected]/4505358184415232
+ FLASK_ENV:  ${COPILOT_ENVIRONMENT_NAME}
+
+secrets:                      # Pass secrets from AWS Systems Manager (SSM) Parameter Store.
+  NOTIFY_API_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/NOTIFY_API_KEY
+
+# Set email for production
+environments:
+ production:
+   secrets:
+     NOTIFY_API_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/NOTIFY_API_KEY
+     NOTIFY_SEND_EMAIL: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/NOTIFY_SEND_EMAIL

copilot/download-report/overrides/cfn.patches.yml

@@ -0,0 +1,29 @@
+# Delete the task role resource
+# - op: remove
+#   path: /Resources/TaskRole
+
+# Add a service connect alias
+# - op: add
+#   path: /Resources/Service/Properties/ServiceConnectConfiguration/Services/0/ClientAliases/-
+#   value:
+#     Port: !Ref TargetPort
+#     DnsName: yamlpatchiscool
+
+# Replace the task role in the task definition
+# - op: replace
+#   path: /Resources/TaskDefinition/Properties/TaskRoleArn
+#   value: arn:aws:iam::123456789012:role/MyTaskRole
+
+# Add Cloudwatch logs required permissions
+- op: add
+  path: /Resources/TaskRole/Properties/Policies/-
+  value:
+    PolicyName: AllowCloudwatchLogsQuery
+    PolicyDocument:
+      Version: '2012-10-17'
+      Statement:
+        - Effect: 'Allow'
+          Action:
+            - 'logs:GetQueryResults'
+            - 'logs:StartQuery'
+          Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/copilot/${AppName}-${EnvName}-data-frontend:*'

scripts/extract_download_logs.py

@@ -18,6 +18,11 @@
 from notifications_python_client import prepare_upload
 from notifications_python_client.notifications import NotificationsAPIClient
 
+# Default FLASK_ENV here to allow import when running locally
+if not os.getenv("FLASK_ENV"):
+    os.environ["FLASK_ENV"] = "development"
+from fsd_utils import init_sentry
+
 
 def send_notify(
     from_date: datetime.datetime,
@@ -51,7 +56,7 @@ def parse_item(item: List[dict]) -> dict:
         message = json.loads([i for i in item if i["field"] == "@message"][0]["value"])
         user_id = message["user_id"]
         email = message.get("email")
-        query_params = message["query_params"]
+        query_params = message.get("query_params", {})
         timestamp = [i for i in item if i["field"] == "@timestamp"][0]["value"]
         return {
             "timestamp": timestamp,
@@ -131,6 +136,7 @@ def main(args):
 
 
 if __name__ == "__main__":
+    init_sentry()
     parser = argparse.ArgumentParser(
         description="Output a report of downloads (requires AWS authentication)",
     )