From e451670ba5723c353c8519719e9a19bafd949669 Mon Sep 17 00:00:00 2001 From: auxlife Date: Tue, 28 Nov 2017 13:46:27 -0800 Subject: [PATCH 1/7] Fixed some errors Fixed get folder error for non-existent folders Removes temp files before exiting --- mongodb_backup.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mongodb_backup.py b/mongodb_backup.py index 241ebe1..2527956 100644 --- a/mongodb_backup.py +++ b/mongodb_backup.py @@ -15,6 +15,7 @@ from datetime import date import glob import tarfile +import shutil try: @@ -26,20 +27,23 @@ SCOPES = 'https://www.googleapis.com/auth/drive' CLIENT_SECRET_FILE = str(os.path.abspath(os.path.dirname(__file__))) + '/client_secret.json' APPLICATION_NAME = 'mongo-backup' -JSON_DIR = str(os.path.abspath(os.path.dirname(__file__)))+'/data' +JSON_DIR = str(os.path.abspath(os.path.dirname(__file__)))+'/mongobackuptmp' FOLDER = 'mongodb_backup' -FILE_MIMYTYPE = 'application/json' -DB_NAME = 'test' # input databasename crowi's +FILE_MIMETYPE = 'application/tar' +DB_NAME = 'DB NAME' +DB_USER = 'DB USERNAME' +DB_PASS = 'DB PASSWORD' EXTENSION = '.json' BACKUP_EXTENSION = '.tar.gz' def mongodb_backup(backup_db_dir): client = pymongo.MongoClient(host="127.0.0.1", port=27017) database = client[DB_NAME] - # Please write username and password of mongodb if mongodb requires authentication. - # authenticated = database.authenticate("username","passwd") - # assert authenticated, "Could not authenticate to database!" + #Comment out next line if mongo does not require authentication + authenticated = database.authenticate(DB_USER,DB_PASS) collections = database.collection_names() + if not os.path.exists(backup_db_dir): + os.makedirs(backup_db_dir) for i, collection_name in enumerate(collections): col = getattr(database, collections[i]) collection = col.find() @@ -47,7 +51,7 @@ def mongodb_backup(backup_db_dir): jsonpath = join(backup_db_dir, jsonpath) with open(jsonpath, 'w') as jsonfile: jsonfile.write(dumps(collection)) - tar = tarfile.open(os.path.join(backup_db_dir, 'daxiv_'+date.today().isoformat()+BACKUP_EXTENSION), 'w:gz') + tar = tarfile.open(os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION), 'w:gz') tar.add(backup_db_dir) tar.close() @@ -72,52 +76,52 @@ def get_credentials(): def create_folder(service): - print("Create folder: %s" % (FOLDER)) + print("Creating folder: %s" % (FOLDER)) folder_metadata = { 'name': FOLDER, 'mimeType': 'application/vnd.google-apps.folder' } folder = service.files().create(body=folder_metadata, fields='id').execute() - print("File ID: %s" % folder.get('id')) return folder.get('id') def get_folder(service): - response = service.files().list(q="name="+"'"+FOLDER+"'", spaces='drive').execute() - if not response: + response = service.files().list(q="mimeType='application/vnd.google-apps.folder' and name="+"'"+FOLDER+"'", spaces='drive', fields='files(id)').execute() + if not response.get('files',[]): return False folder = response.get('files', [])[0] return folder.get('id') - - def upload_file(service, file_name, folder_id): - media_body = MediaFileUpload(file_name, mimetype=FILE_MIMYTYPE, resumable=True) + media_body = MediaFileUpload(file_name, mimetype=FILE_MIMETYPE, resumable=True) body = { 'name': os.path.split(file_name)[-1], - 'mimeType': FILE_MIMYTYPE, + 'mimeType': FILE_MIMETYPE, 'parents': [folder_id], } service.files().create(body=body, media_body=media_body).execute() def main(): - mongodb_backup(JSON_DIR) credentials = get_credentials() http = credentials.authorize(httplib2.Http()) service = discovery.build('drive', 'v3', http=http) + folder_id = get_folder(service) + if not folder_id: + folder_id = create_folder(service) + + mongodb_backup(JSON_DIR) file_path = os.path.join(JSON_DIR, '*' + BACKUP_EXTENSION) files = glob.glob(file_path) if not files: print("No files to upload.") sys.exit() - folder_id = get_folder(service) - if not folder_id: - folder_id = create_folder(service) + for file_name in files: if os.path.split(file_name)[-1] == 'client_secret.json': continue print('upload: ' + file_name) upload_file(service, file_name, folder_id) - os.remove(file_name) + + shutil.rmtree(JSON_DIR) if __name__ == "__main__": main() From ffdbb8c6c1733cf30297b18a957d9aea74468108 Mon Sep 17 00:00:00 2001 From: auxlife Date: Tue, 28 Nov 2017 13:55:32 -0800 Subject: [PATCH 2/7] Updated directions Added steps to get turn on API. --- README.md | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ceafed3..de8739c 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,27 @@ To back up to Google Drive This tool is made in Python3 #Introduction -* pip install pymongo -* pip install bson -* pip install httplib2 -* pip install --upgrade google-api-python-client -* Write the name of the database on line 28 -* Rename the client_secret.json -* Placing them(mongodb_backup.py and client_secret.json) in the home directory - -##Options -Write username and password of mongodb on line 36-37 if mongodb requires authentication. +* pip3 install pymongo +* pip3 install bson +* pip3 install httplib2 +* pip3 install --upgrade google-api-python-client + +##Required: +line 33: DB Name + +Enable Google Drive API: +Use this wizard [https://console.developers.google.com/start/api?id=drive] to create or select a project in the Google Developers Console and automatically turn on the API. Click Continue, then Go to credentials. +On the Add credentials to your project page, click the Cancel button. +At the top of the page, select the OAuth consent screen tab. Select an Email address, enter a Product name if not already set, and click the Save button. +Select the Credentials tab, click the Create credentials button and select OAuth client ID. +Select the application type Other, enter the name "mongo-backup", and click the Create button. +Click OK to dismiss the resulting dialog. +Click the file_download (Download JSON) button to the right of the client ID. +Move this file to your directory and rename it client_secret.json. + +When you run the script for the first time it will give you a url to authorize the script to access your gdrive. + +##Options: +line 31: Folder name in GDrive +line 34/45: DB Username/Password +** Comment out line 43 if db does not require authentication From 231fe692f04e99c4a2bb808839794ecc5814acbc Mon Sep 17 00:00:00 2001 From: auxlife Date: Mon, 4 Dec 2017 09:11:26 -0800 Subject: [PATCH 3/7] Switched to 7z to add password protection --- mongodb_backup.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mongodb_backup.py b/mongodb_backup.py index 2527956..16b79c5 100644 --- a/mongodb_backup.py +++ b/mongodb_backup.py @@ -14,7 +14,6 @@ from googleapiclient.http import MediaFileUpload from datetime import date import glob -import tarfile import shutil @@ -34,13 +33,15 @@ DB_USER = 'DB USERNAME' DB_PASS = 'DB PASSWORD' EXTENSION = '.json' -BACKUP_EXTENSION = '.tar.gz' +BACKUP_EXTENSION = '.zip' +BACKUP_PASSWORD = 'somepassword' def mongodb_backup(backup_db_dir): client = pymongo.MongoClient(host="127.0.0.1", port=27017) database = client[DB_NAME] - #Comment out next line if mongo does not require authentication + # Please write username and password of mongodb if mongodb requires authentication. authenticated = database.authenticate(DB_USER,DB_PASS) + # assert authenticated, "Could not authenticate to database!" collections = database.collection_names() if not os.path.exists(backup_db_dir): os.makedirs(backup_db_dir) @@ -51,9 +52,16 @@ def mongodb_backup(backup_db_dir): jsonpath = join(backup_db_dir, jsonpath) with open(jsonpath, 'w') as jsonfile: jsonfile.write(dumps(collection)) - tar = tarfile.open(os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION), 'w:gz') - tar.add(backup_db_dir) - tar.close() + + command = "7z a -p" + BACKUP_PASSWORD + ' \"' + os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION) + "\" " + backup_db_dir + + print("Executing command: " + command) + callreturn = os.system(command); + if (callreturn == 0): + print('Backup successful') + else: + print('Something went wrong, 7zip returned: ' + callreturn) + def get_credentials(): home_dir = os.path.expanduser('~') From 451965c6c90303a2d1959701e435ea2043027149 Mon Sep 17 00:00:00 2001 From: auxlife Date: Mon, 4 Dec 2017 09:20:34 -0800 Subject: [PATCH 4/7] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index de8739c..4a902e8 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ # backup-mongodb -To back up to Google Drive -This tool is made in Python3 +Used to back up mongodb to Google Drive in an encrypted 7z file #Introduction * pip3 install pymongo * pip3 install bson * pip3 install httplib2 * pip3 install --upgrade google-api-python-client +* apt-get install 7zip-full ##Required: line 33: DB Name +line 37: zip password Enable Google Drive API: Use this wizard [https://console.developers.google.com/start/api?id=drive] to create or select a project in the Google Developers Console and automatically turn on the API. Click Continue, then Go to credentials. @@ -25,5 +26,5 @@ When you run the script for the first time it will give you a url to authorize t ##Options: line 31: Folder name in GDrive -line 34/45: DB Username/Password +line 34/35: DB Username/Password ** Comment out line 43 if db does not require authentication From 92f00d30a2997fd1a8a18efc48e5bb403e56b49d Mon Sep 17 00:00:00 2001 From: auxlife Date: Mon, 4 Dec 2017 09:22:22 -0800 Subject: [PATCH 5/7] Update mongodb_backup.py --- mongodb_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_backup.py b/mongodb_backup.py index 16b79c5..f028396 100644 --- a/mongodb_backup.py +++ b/mongodb_backup.py @@ -53,7 +53,7 @@ def mongodb_backup(backup_db_dir): with open(jsonpath, 'w') as jsonfile: jsonfile.write(dumps(collection)) - command = "7z a -p" + BACKUP_PASSWORD + ' \"' + os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION) + "\" " + backup_db_dir + command = "7z a -t7z -m0=lzma2 -mx=9 -ms=on -mhe=on -p" + BACKUP_PASSWORD + ' \"' + os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION) + "\" " + backup_db_dir print("Executing command: " + command) callreturn = os.system(command); From 4bb1ecd1c16bd51bdf5d7086da04b599f8673a9a Mon Sep 17 00:00:00 2001 From: auxlife Date: Fri, 8 Dec 2017 08:33:06 -0800 Subject: [PATCH 6/7] Switched from pymongo to mongodump Switched from pymongo to mongodump --- mongodb_backup.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/mongodb_backup.py b/mongodb_backup.py index f028396..7808716 100644 --- a/mongodb_backup.py +++ b/mongodb_backup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 from __future__ import print_function from os.path import join -from bson.json_util import dumps -import pymongo import httplib2 import os import sys @@ -37,21 +35,14 @@ BACKUP_PASSWORD = 'somepassword' def mongodb_backup(backup_db_dir): - client = pymongo.MongoClient(host="127.0.0.1", port=27017) - database = client[DB_NAME] - # Please write username and password of mongodb if mongodb requires authentication. - authenticated = database.authenticate(DB_USER,DB_PASS) - # assert authenticated, "Could not authenticate to database!" - collections = database.collection_names() - if not os.path.exists(backup_db_dir): - os.makedirs(backup_db_dir) - for i, collection_name in enumerate(collections): - col = getattr(database, collections[i]) - collection = col.find() - jsonpath = collection_name + EXTENSION - jsonpath = join(backup_db_dir, jsonpath) - with open(jsonpath, 'w') as jsonfile: - jsonfile.write(dumps(collection)) + command = "mongodump -u "+DB_USER+" -p "+DB_PASS+" --db "+DB_NAME+" --out "+backup_db_dir + + print("Executing command: " + command) + callreturn = os.system(command); + if (callreturn == 0): + print('Backup successful') + else: + print('Something went wrong, mongodump returned: ' + callreturn) command = "7z a -t7z -m0=lzma2 -mx=9 -ms=on -mhe=on -p" + BACKUP_PASSWORD + ' \"' + os.path.join(backup_db_dir, 'NSdb_'+date.today().isoformat()+BACKUP_EXTENSION) + "\" " + backup_db_dir From 98dca5aac954a202337d20ec7d7af07da6a3ff18 Mon Sep 17 00:00:00 2001 From: auxlife Date: Fri, 8 Dec 2017 08:35:25 -0800 Subject: [PATCH 7/7] Switched from pymongo to mongodump Switched from pymongo to mongodump --- README.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4a902e8..ef79db3 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,14 @@ Used to back up mongodb to Google Drive in an encrypted 7z file #Introduction -* pip3 install pymongo -* pip3 install bson * pip3 install httplib2 * pip3 install --upgrade google-api-python-client * apt-get install 7zip-full ##Required: -line 33: DB Name -line 37: zip password +line 30: DB Name +line 35: zip password +line 31/32: DB Username/Password Enable Google Drive API: Use this wizard [https://console.developers.google.com/start/api?id=drive] to create or select a project in the Google Developers Console and automatically turn on the API. Click Continue, then Go to credentials. @@ -25,6 +24,4 @@ Move this file to your directory and rename it client_secret.json. When you run the script for the first time it will give you a url to authorize the script to access your gdrive. ##Options: -line 31: Folder name in GDrive -line 34/35: DB Username/Password -** Comment out line 43 if db does not require authentication +line 28: Folder name in GDrive