mongo_extract.py

from pymongo import MongoClient
import csv
import boto3
import datetime
from datetime import timedelta
import configparser


# load the mongo_config values
parser = configparser.ConfigParser()
parser.read('pipeline.conf')

hostname = parser.get('mongo_config', 'hostname')
username = parser.get('mongo_config', 'username')
password = parser.get('mongo_config', 'password')
database_name = parser.get('mongo_config', 'database')
collection_name = parser.get('mongo_config', 'collection')


# altered the parameter because of an error
uri = ("mongodb://"
       + username
       + ":" + password
       + "@" + hostname
       + "/"
       + database_name
       + "?retryWrites=true&"
       + "w=majority&ssl=false&"
       + "authSource=admin"
       )

mongo_client = MongoClient(uri)

# connect to the db where the collection resides
mongo_db = mongo_client[database_name]

# choose the collection to query documents from
mongo_collection = mongo_db[collection_name]

start_date = datetime.datetime.today() + timedelta(days=-1)
end_date = start_date + timedelta(days=1)

mongo_query = {"$and":
               [
                   {
                       "event_timestamp":
                       {
                           "$gte": start_date
                       }
                   },
                   {
                       "event_timestamp":
                       {
                           "$lt": end_date
                       }
                   }
               ]
               }

event_docs = mongo_collection.find(mongo_query, batch_size=3000)

# create a blank list to store the results
all_events = []

# iterate through the cursor
for doc in event_docs:
    # Includ default values
    event_id = str(doc.get("event_id", -1))
    event_timestamp = doc.get("event_timestamp", None)
    event_name = doc.get("event_name", None)

    # add all the event properties into a list
    current_event = []
    current_event.append(event_id)
    current_event.append(event_timestamp)
    current_event.append(event_name)

    # add the event to the final list of events
    all_events.append(current_event)

export_file = "export_file.csv"

with open(export_file, "w") as fp:
    csvw = csv.writer(fp, delimiter="|")
    csvw.writerows(all_events)

fp.close()
mongo_client.close()

acces_key = parser.get('aws_boto_credentials', 'access_key')
secret_key = parser.get('aws_boto_credentials', 'secret_key')
bucket_name = parser.get('aws_boto_credentials', 'bucket_name')

s3 = boto3.client('s3', aws_access_key_id=acces_key,
                  aws_secret_access_key=secret_key)
s3_file = export_file
s3.upload_file(export_file, bucket_name, s3_file)