Updated scraper for new Kaggle Api >=1.7

Author: beverleyy

.gitignore

@@ -4,3 +4,5 @@ Gemfile.lock
 *.gem
 .jekyll-cache
 .jekyll-cache
+*.sqlite
+kaggle_stats.json

kaggle_json.py

@@ -1,5 +1,5 @@
 from github import Github
-import github,requests_cache,re,os,kaggle,json,numpy as np
+import traceback,github,requests_cache,re,os,kaggle,json,numpy as np
 
 GITHUB_TOKEN = os.getenv("GIST_TOKEN")
 GIST_ID = "c9112c25c5acd400b90741efa81aa411"
@@ -81,20 +81,41 @@ def unformat_bytes(string):
 
     for dsn in dataset_names:
         print(f'Processing {dsn}...')
+        # Old Kaggle Api <1.7
         try:
             user = dsn.split("/")[0]
             dataset = vars(next((d for d in usernames[user] if vars(d)['ref'] == dsn)))
             downloads.append(int(dataset['downloadCount']))
             views.append(int(dataset['viewCount']))
             sizes.append(int(dataset['totalBytes']))
             print(f'{dsn} done.')
-            
-        except Exception as e:
-            print(f'{e} when reading {dsn}')
+
+	# New Kaggle Api >=1.7
+        except KeyError:
+            try:
+                user = dsn.split("/")[0]
+                dataset = next((d for d in usernames[user] if d.ref == dsn))
+                downloads.append(int(dataset.download_count))
+                views.append(int(dataset.view_count))
+                sizes.append(int(dataset.total_bytes))
+                print(f'{dsn} done.')
+
+            except Exception:
+                traceback.print_exc()
+                print(f'Error when reading {dsn}')
+                print(f'Continuing with 0 values...')
+                downloads.append(0)
+                views.append(0)
+                sizes.append(0)
+                
+        except Exception:
+            traceback.print_exc()
+            print(f'Error when reading {dsn}')
             print(f'Continuing with 0 values...')
             downloads.append(0)
             views.append(0)
             sizes.append(0)
+
 
     views = np.array(views)
     downloads = np.array(downloads)
@@ -121,7 +142,7 @@ def unformat_bytes(string):
             'size': ds_size,
             'views': ds_views,
             'downloads': ds_downs,
-    }
+        }
     json_dump[filename] = kaggle_stats
     total_bytes += int(np.sum(downloads*size_in_bytes))
     total_size += int(np.sum(size_in_bytes))

requirements.txt

@@ -1,4 +1,4 @@
-kaggle>=1.6,<1.7
+kaggle>=1.6
 numpy>=1.2
 requests-cache>=1.2.1
 pygithub>=2.4.0