Version 0.8 of vk-backup

* A number of speed optimizations
* Generated self appid with many permissions requirements
* Added known error return codes, like access restrictions
* Enabled json database sort keys
* Fixed copy_history processing
* Added request comments for photos, wall, videos
* Rewrited update of media
* Added user updated filed to know, if it already updated this session

Author: rabits

lib/Api.py

@@ -16,11 +16,14 @@
 
 c.log('debug', 'Init Api')
 
+# Session start time
+_START_TIME = long(time.time())
+
 # Vk application ID
-_CLIENT_ID = '2951857'
+_CLIENT_ID = '4603710'
 
 # Get token & user_id by login
-(_TOKEN, _USER_ID) = vk_auth.auth(c.cfg('user'), c.cfg('password'), _CLIENT_ID, "messages")
+(_TOKEN, _USER_ID) = vk_auth.auth(c.cfg('user'), c.cfg('password'), _CLIENT_ID, "messages,audio,docs,video,photos,wall,friends")
 
 # Last time api call to prevent service overloading
 _LAST_API_CALL = 0
@@ -39,7 +42,16 @@ def request(method, params):
             url = "https://api.vk.com/method/%s?%s" % (method, urlencode(params))
             data = json.loads(urllib2.urlopen(url, None, 30).read())
             if 'response' not in data:
-                raise Exception('no correct response while calling api method "%s", data: %s' % (method, data))
+                if 'error' in data:
+                    c.log('warning', 'Api responded error: %s' % data['error']['error_msg'])
+                    if data['error']['error_code'] in [7, 15, 212]:
+                        return
+                    elif data['error']['error_code'] in [10]:
+                        continue
+                    else:
+                        raise Exception('unknown error code %i, "%s", data: %s' % (data['error']['error_code'], method, data))
+                else:
+                    raise Exception('no correct response while calling api method "%s", data: %s' % (method, data))
             break
         except Exception as e:
             c.log('warning', 'Retry request %i (3): %s' % (retry, str(e)))
@@ -55,3 +67,7 @@ def getUserId():
     global _USER_ID
     return str(_USER_ID)
 
+def getStartTime():
+    global _START_TIME
+    return _START_TIME
+

lib/Database.py

@@ -30,7 +30,7 @@ def store(self):
             if not os.path.isdir(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
             with codecs.open(path, 'w', 'utf-8') as outfile:
-                json.dump(self.data[i], outfile, indent=1, ensure_ascii=False)
+                json.dump(self.data[i], outfile, indent=1, ensure_ascii=False, sort_keys=True)
 
     def load(self, subdir = None):
         path = self.path if subdir == None else os.path.join(self.path, subdir)

lib/Media.py

@@ -17,6 +17,8 @@
 
 from Database import Database
 
+import Api
+
 class Media(Database):
     class Downloader(threading.Thread):
         def __init__(self, queue, report):
@@ -83,7 +85,7 @@ def download(self):
             return self.success
 
     def stopDownloads(self):
-        c.log('debug', 'Stopping download threads')
+        c.log('debug', 'Stopping download threads (%i)' % len(self.threads))
         for i in self.threads:
             i.stop()
 
@@ -125,14 +127,15 @@ def loadAttachments(self, data):
         if 'attachment' in data:
             attachments.append(data['attachment'])
         if 'copy_history' in data:
-            self.loadAttachments(data['copy_history'])
+            for subdata in data['copy_history']:
+                self.loadAttachments(subdata)
         for attach in attachments:
             c.log('debug', 'Processing %s' % attach['type'])
             funcname = 'process' + attach['type'].title()
             if funcname in dir(self):
                 getattr(self, funcname)(attach[attach['type']])
             else:
-                c.log('error', '  unable to find attachment processing function "Media.%s"' % funcname)
+                c.log('error', '  media processing function "Media.%s" is not implemented' % funcname)
                 c.log('debug', str(attach))
 
     def addDownload(self, url, path = None):
@@ -165,15 +168,59 @@ def preprocess(self, data, data_type):
             path = os.path.join(data_type, str(mydata['id']))
 
         if path in self.data:
-            return None
+            return path
+
         self.data[path] = mydata
 
         return path
 
+    def requestComments(self, data, data_type, owner_id):
+        if str(owner_id) != Api.getUserId():
+            return
+
+        c.log('debug', 'Requesting comments for %s %i' % (data_type, data['id']))
+
+        if data_type == 'photo':
+            api_method = 'photos.getComments'
+            api_id_name = 'photo_id'
+        elif data_type == 'video':
+            api_method = 'video.getComments'
+            api_id_name = 'video_id'
+        elif data_type == 'wall':
+            api_method = 'wall.getComments'
+            api_id_name = 'post_id'
+        else:
+            c.log('warning', 'Unable to request comments for %s %i - not implemented' % (data_type, data['id']))
+            return
+
+        if 'comments' not in data:
+            data['comments'] = {}
+        if not isinstance(data['comments'], dict):
+            data['comments'] = {}
+
+        req_data = {'owner_id': int(owner_id), api_id_name: int(data['id']), 'count': 100, 'offset': 0}
+
+        while True:
+            subdata = Api.request(api_method, req_data)
+            if subdata == None:
+                return
+            count = subdata['count']
+            subdata = subdata['items']
+            for d in subdata:
+                data['comments'][str(d['date'])] = d
+                self.loadAttachments(data['comments'][str(d['date'])])
+
+            req_data['offset'] += 100
+            if req_data['offset'] >= count:
+                break
+
     def processPhoto(self, data):
+        c.log('debug', 'Processing photo media')
         path = self.preprocess(data, 'photo')
-        if path != None:
+        if 'localpath' not in self.data[path]:
             url = None
+            if 'url' in self.data[path]:
+                url = self.data[path]['url']
             size = 0
             for key in self.data[path].keys():
                 if key.startswith('photo_'):
@@ -188,46 +235,53 @@ def processPhoto(self, data):
 
             self.data[path]['url'] = url
             self.data[path]['localpath'] = self.addDownload(self.data[path]['url'])
+        self.requestComments(self.data[path], 'photo', self.data[path]['owner_id'])
 
     def processDoc(self, data):
+        c.log('debug', 'Processing doc media')
         path = self.preprocess(data, 'doc')
-        if path != None:
+        if 'localpath' not in self.data[path]:
             self.data[path]['localpath'] = self.addDownload(self.data[path]['url'])
 
     def processAudio(self, data):
+        c.log('debug', 'Processing audio media')
         path = self.preprocess(data, 'audio')
-        if path != None:
+        if 'localpath' not in self.data[path]:
             self.data[path]['localpath'] = self.addDownload(self.data[path]['url'])
 
     def processWall(self, data):
-        c.log('debug', 'Processing wall attachments')
+        c.log('debug', 'Processing wall media')
+        data['comments'].pop('count', None)
+        data['comments'].pop('can_post', None)
+        self.requestComments(data, 'wall', data['from_id'])
         self.loadAttachments(data)
 
     def processGeo(self, data):
         self.preprocess(data, 'geo')
-        c.log('debug', 'Skipping geo attachment - no data to download')
+        c.log('debug', 'Skipping geo media - no data to download')
 
     def processVideo(self, data):
-        self.preprocess(data, 'video')
-        c.log('debug', 'Skipping video attachment - size of the file is too big')
+        path = self.preprocess(data, 'video')
+        self.requestComments(self.data[path], 'video', self.data[path]['owner_id'])
+        c.log('debug', 'Skipping video media - size of the file is too big')
 
     def processSticker(self, data):
         self.preprocess(data, 'sticker')
-        c.log('debug', 'Skipping sticker attachment - idiotizm')
+        c.log('debug', 'Skipping sticker media - idiotizm')
 
     def processLink(self, data):
-        c.log('debug', 'Skipping link attachment - no data to download')
+        c.log('debug', 'Skipping link media - no data to download')
 
     def processPoll(self, data):
         self.preprocess(data, 'poll')
-        c.log('debug', 'Skipping poll attachment - no data to download')
+        c.log('debug', 'Skipping poll media - no data to download')
 
     def processNote(self, data):
         self.preprocess(data, 'note')
-        c.log('debug', 'Skipping note attachment - no data to download')
+        c.log('debug', 'Skipping note media - no data to download')
 
     def processPresent(self, data):
         self.preprocess(data, 'present')
-        c.log('debug', 'Skipping present attachment - stupid present')
+        c.log('debug', 'Skipping present media - stupid present')
 
 S = Media()

lib/Users.py

@@ -41,12 +41,15 @@ def addUser(self, newdata):
                 'data': { long(time.time()): newdata },
             }
         else:
+            if self.data[newdata_id]['updated'] > Api.getStartTime():
+                return
             user_data = self.data[newdata_id]['data']
             if len(set(user_data[max(user_data)].items()) & set(newdata.items())) != len(newdata):
                 c.log('debug', 'Adding new data for user %s' % newdata_id)
                 user_data[long(time.time())] = newdata
 
         self.requestProfilePhotos(newdata_id)
+        self.data[newdata_id]['updated'] = long(time.time())
 
     def requestProfilePhotos(self, user_id):
         c.log('debug', 'Requesting profile photos')
@@ -102,7 +105,7 @@ def requestBlog(self, user_id):
             data = data['items']
             for d in data:
                 self.data[user_id]['blog'][str(d['date'])] = d
-                Media.loadAttachments(self.data[user_id]['blog'][str(d['date'])])
+                Media.processWall(self.data[user_id]['blog'][str(d['date'])])
 
             req_data['offset'] += 100
             if req_data['offset'] >= count:

vk-backup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
-'''VK-Backup 0.7.0
+'''VK-Backup 0.8.0
 
 Author:      Rabit <[email protected]>
 License:     GPL v3
@@ -46,7 +46,7 @@ def store(self):
         Chats.store()
         Media.store()
 
-        with open(os.path.join(self.path, 'backup.json'), 'w') as outfile:
+        with open(os.path.join(self.path, 'backup.id'), 'w') as outfile:
             outfile.write(str(Api.getUserId()))
 
     def process(self):
@@ -69,10 +69,9 @@ def process(self):
 
             # Store data
             backup.store()
-        except Exception as e:
+        except (Exception, KeyboardInterrupt) as e:
             Media.stopDownloads()
             c.log('error', 'Exception: %s' % str(e))
-            raise e
 
 from lib import Api