get info from bucket

api/report/management/commands/update_report.py

@@ -7,6 +7,8 @@
 from apscheduler.schedulers.blocking import BlockingScheduler
 from django.core.management.base import BaseCommand
 from django.utils.timezone import make_aware
+from django.conf import settings
+import boto3
 
 from api.report.models import *
 
@@ -17,11 +19,23 @@ def cron(*args, **options):
     if 6 <= datetime.now().hour <= 20:
 
         print(f"Cron job is running. The time is {datetime.now()}")
-        request = requests.get(url)
 
-        content = request.content.decode('utf8').replace('var database=', '')
+        s3resource = boto3.resource('s3',
+                                    aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+                                    aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
+
+        url = "%s%s"%(settings.BINO_URL, '/crawl/ministerio_saude_brasil')
+        response = requests.post(url = url, data = {})
+        file_name = json.loads(response.content.decode('utf-8'))['path']
+
+        bucket_name = settings.AWS_STORAGE_BUCKET_NAME
+        obj = s3resource.Object(bucket_name,"ministerio_saude_brasil/2020-03-20/16-47/rawData.json")
+
+        content = obj.get()['Body'].read().decode('utf-8')
         data = json.loads(content)
 
+        print('Object body: {}'.format(data['brazil']))
+
         for record in data['brazil']:
             date_time = datetime.strptime(f"{record['date']} {record['time']}",
                                           '%d/%m/%Y %H:%M')
@@ -57,6 +71,6 @@ def handle(self, *args, **options):
         print('Cron started! Wait the job starts!')
 
         scheduler = BlockingScheduler()
-        scheduler.add_job(cron, 'cron', minute=20, timezone='America/Maceio')
+        scheduler.add_job(cron, 'interval', minutes=1, timezone=settings.TIME_ZONE)
 
         scheduler.start()

api/settings.py

@@ -70,6 +70,12 @@
 
 STATIC_URL = '/static/'
 
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
+AWS_LOCATION = os.environ['AWS_LOCATION']
+BINO_URL = os.environ['BINO_URL']
+
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',

docker-compose.yaml

@@ -6,4 +6,11 @@ services:
       context: .
       target: release
     image: hummingbird
-
+    environment:
+      AWS_ACCESS_KEY_ID:
+      AWS_SECRET_ACCESS_KEY:
+      AWS_STORAGE_BUCKET_NAME:
+      AWS_LOCATION:
+      BINO_URL: 'http://localhost:8080/'
+    network_mode: host
+

requirements.txt

@@ -10,3 +10,4 @@ requests==2.23.0
 sqlparse==0.3.1
 urllib3==1.25.8
 apscheduler==3.6.3
+boto3==1.12.25