Add Django command to fetch GCI data

Separates the fetch operation from the load operation,
allowing the build on forks to use a cached dataset.

Related to coala#3

.ci/build.sh

@@ -2,7 +2,9 @@
 
 set -e -x
 
-mkdir _site public
+mkdir private _site public
+
+python manage.py fetch_gci_task_data private
 
 python manage.py collectstatic --noinput
 python manage.py distill-local public --force

.gitignore

@@ -100,5 +100,6 @@ ENV/
 # mypy
 .mypy_cache/
 
+/private/
 _site/
 /public/

gci/config.py

@@ -1,7 +1,18 @@
+import ruamel.yaml
 import os
 
 API_KEY_FILE = '.%s_API_KEY'
 
+GCI_DATA_DIR = os.path.join(
+    os.path.dirname(__file__), '..',
+    'private',
+)
+
+
+class TokenMissing(KeyError):
+    """The requested token is not available.
+    """
+
 
 def get_api_key(name):
     env_val = os.environ.get('%s_TOKEN' % name)
@@ -16,5 +27,10 @@ def get_api_key(name):
             api_key = api_key_file.readline().strip()
             return api_key
     except IOError:
-        print('Please put your %s API key at %s.' % (name, filename))
-        exit(1)
+        raise TokenMissing('Please put your %s API key at %s.' %
+                           (name, filename))
+
+
+def load_cache(filename):
+    with open(os.path.join(GCI_DATA_DIR, filename), 'r') as f:
+        return ruamel.yaml.load(f, Loader=ruamel.yaml.Loader)

gci/management/commands/fetch_gci_task_data.py

@@ -0,0 +1,40 @@
+from collections import OrderedDict
+import os.path
+
+from ruamel.yaml import YAML
+
+from django.core.management.base import BaseCommand
+
+from gci.students import (
+    _get_instances,
+    _get_tasks,
+)
+
+
+class Command(BaseCommand):
+    help = 'Fetch GCI data'
+
+    def add_arguments(self, parser):
+        parser.add_argument('output_dir', nargs='?', type=str)
+
+    def handle(self, *args, **options):
+        output_dir = options.get('output_dir')
+
+        tasks = {}
+        for task in _get_tasks():
+            tasks[int(task['id'])] = task
+
+        instances = {}
+        for instance in _get_instances():
+            instances[int(instance['id'])] = instance
+
+        tasks = OrderedDict(sorted(tasks.items(), key=lambda t: t[0]))
+        instances = OrderedDict(sorted(instances.items(), key=lambda t: t[0]))
+
+        yaml = YAML()
+
+        with open(os.path.join(output_dir, 'tasks.yaml'), 'w') as f:
+            yaml.dump(tasks, f)
+
+        with open(os.path.join(output_dir, 'instances.yaml'), 'w') as f:
+            yaml.dump(instances, f)

gci/students.py

@@ -1,13 +1,14 @@
+import os
 import re
 
 from .client import GCIAPIClient
 
-from .config import get_api_key
+from .config import get_api_key, load_cache
 from .gitorg import get_issue
+from .task import get_task
 
 _client = None
-_org = {}
-_tasks = {}
+_instances = {}
 
 
 def get_client():
@@ -39,20 +40,7 @@ def _get_tasks():
                 page = int(result.group(1))
 
 
-def get_tasks():
-    global _tasks
-    if not _tasks:
-        for task in _get_tasks():
-            _tasks[task['id']] = task
-    return _tasks
-
-
-def get_task(task_id):
-    tasks = get_tasks()
-    return tasks[task_id]
-
-
-def get_instances():
+def _get_instances():
     client = get_client()
     page = 1
 
@@ -68,9 +56,17 @@ def get_instances():
                 page = int(result.group(1))
 
 
+def get_instances():
+    global _instances
+    if not _instances:
+        _instances = load_cache('instances.yaml')
+
+    return _instances
+
+
 def get_students():
     students = {}
-    for instance in get_instances():
+    for _, instance in get_instances().items():
         student_id = instance['student_id']
         if student_id not in students:
             student = {

gci/task.py

@@ -0,0 +1,15 @@
+from .config import load_cache
+
+_tasks = {}
+
+
+def get_tasks():
+    global _tasks
+    if not _tasks:
+        _tasks = load_cache('tasks.yaml')
+
+    return _tasks
+
+
+def get_task(task_id):
+    return get_tasks()[task_id]

gci/views.py

@@ -3,9 +3,9 @@
 from calendar import timegm
 import requests
 
-from gci.config import get_api_key
 from .students import get_students, get_linked_students
 from .gitorg import get_logo
+from .task import get_tasks
 
 STUDENT_URL = (
     'https://codein.withgoogle.com/dashboard/task-instances/?'
@@ -16,14 +16,11 @@
 
 def index(request):
     try:
-        client = get_api_key('GCI')
-    except BaseException:
-        client = None
-
-    if client:
-        s = gci_overview()
-    else:
+        tasks = get_tasks()
+    except FileNotFoundError:
         s = ['GCI data not available']
+    else:
+        s = gci_overview()
 
     return HttpResponse('\n'.join(s))
 

requirements.txt

@@ -5,3 +5,4 @@ IGitt
 requests
 python-dateutil
 pillow
+ruamel.yaml