[AIRFLOW-1519] Add server side paging in DAGs list

Airflow's main page previously did paging client-
side via a
jQuery plugin (DataTable) which was very slow at
loading all DAGs.
The browser would load all DAGs in the table.
The result was performance degradation when having
a number of
DAGs in the range of 1K.

This commit implements server-side paging using
the webserver page
size setting, sending to the browser only the
elements for the
specific page.

Closes apache#2531 from edgarRd/erod-ui-dags-paging

airflow/www/templates/airflow/dags.html

@@ -29,11 +29,23 @@
   <h2>DAGs</h2>
 
   <div id="main_content" style="display:none;">
+    <div class="row">
+      <div class="col-sm-2">
+      </div>
+      <div class="col-sm-10">
+        <form id="search_form" class="form-inline" style="width: 100%; text-align: right;">
+            <div id="dags_filter" class="form-group" style="width: 100%;">
+              <label for="dag_query" style="width:20%; text-align: right;">Search:</label>
+              <input id="dag_query" type="text" class="typeahead form-control" data-provide="typeahead" style="width:50%;" value="{{search_query}}">
+            </div>
+        </form>
+      </div>
+    </div>
     <table id="dags" class="table table-striped table-bordered">
         <thead>
             <tr>
                 <th></th>
-                <th width="12"><span id="pause_header"class="glyphicon glyphicon-info-sign" title="Use this toggle to pause a DAG. The scheduler won't schedule new tasks instances for a paused DAG. Tasks already running at pause time won't be affected."></span></th>
+                <th width="12"><span id="pause_header" class="glyphicon glyphicon-info-sign" title="Use this toggle to pause a DAG. The scheduler won't schedule new tasks instances for a paused DAG. Tasks already running at pause time won't be affected."></span></th>
                 <th>DAG</th>
                 <th>Schedule</th>
                 <th>Owner</th>
@@ -51,7 +63,7 @@ <h2>DAGs</h2>
             </tr>
         </thead>
         <tbody>
-        {% for dag_id in all_dag_ids %}
+        {% for dag_id in dag_ids_in_page %}
             {% set dag = webserver_dags[dag_id] if dag_id in webserver_dags else None %}
             <tr>
                 <!-- Column 1: Edit dag -->
@@ -174,6 +186,19 @@ <h2>DAGs</h2>
         {% endfor %}
         </tbody>
     </table>
+    <div class="row">
+      <div class="col-sm-12" style="text-align:right;">
+        <div class="dataTables_info" id="dags_info" role="status" aria-live="polite" style="padding-top: 0px;">Showing {{num_dag_from}} to {{num_dag_to}} of {{num_of_all_dags}} entries</div>
+      </div>
+    </div>
+    <div class="row">
+      <div class="col-sm-12" style="text-align:left;">
+        <div class="dataTables_info" id="dags_paginate">
+          {{paging}}
+        </div>
+      </div>
+
+    </div>
     {% if not hide_paused %}
     <a href="/admin/?showPaused=False">Hide Paused DAGs</a>
     {% else %}
@@ -187,8 +212,26 @@ <h2>DAGs</h2>
   <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
   <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script>
   <script src="{{ url_for('static', filename='bootstrap-toggle.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='bootstrap3-typeahead.min.js') }}"></script>
   <script>
 
+      const DAGS_INDEX = {{ url_for('admin.index') }}
+      const ENTER_KEY_CODE = 13
+
+      $('#dag_query').on('keypress', function (e) {
+        // check for key press on ENTER (key code 13) to trigger the search
+        if (e.which === ENTER_KEY_CODE) {
+          search_query = $('#dag_query').val();
+          window.location = DAGS_INDEX + "?search="+ encodeURI(search_query);
+          e.preventDefault();
+        }
+      });
+
+      $('#page_size').on('change', function() {
+        p_size = $(this).val();
+        window.location = DAGS_INDEX + "?page_size=" + p_size;
+      });
+
       function confirmTriggerDag(dag_id){
           return confirm("Are you sure you want to run '"+dag_id+"' now?");
       }
@@ -205,10 +248,37 @@ <h2>DAGs</h2>
           $.post(url);
         });
       });
+
+      var $input = $(".typeahead");
+      unique_options_search = new Set([
+          {% for token in auto_complete_data %}
+            "{{token}}",
+          {% endfor %}
+        ]);
+
+      $input.typeahead({
+        source: [...unique_options_search],
+        autoSelect: false,
+        afterSelect: function(value) {
+          search_query = value.trim()
+          if (search_query) {
+            window.location = DAGS_INDEX + "?search="+ encodeURI(search_query);
+          }
+        }
+      });
+
+      $input.change(function() {
+        var current = $input.typeahead("getActive");
+
+      });
+
       $('#dags').dataTable({
         "iDisplayLength": 500,
         "bSort": false,
-        "pageLength": 25,
+        "searching": false,
+        "ordering": false,
+        "paging": false,
+        "info": false
       });
       $("#main_content").show(250);
       diameter = 25;

airflow/www/utils.py

@@ -76,6 +76,127 @@ def is_accessible(self):
         )
 
 
+def generate_pages(current_page, num_of_pages,
+                   search=None, showPaused=None, window=7):
+    """
+    Generates the HTML for a paging component using a similar logic to the paging
+    auto-generated by Flask managed views. The paging component defines a number of
+    pages visible in the pager (window) and once the user goes to a page beyond the
+    largest visible, it would scroll to the right the page numbers and keeps the
+    current one in the middle of the pager component. When in the last pages,
+    the pages won't scroll and just keep moving until the last page. Pager also contains
+    <first, previous, ..., next, last> pages.
+    This component takes into account custom parameters such as search and showPaused,
+    which could be added to the pages link in order to maintain the state between
+    client and server. It also allows to make a bookmark on a specific paging state.
+    :param current_page:
+        the current page number, 0-indexed
+    :param num_of_pages:
+        the total number of pages
+    :param search:
+        the search query string, if any
+    :param showPaused:
+        false if paused dags will be hidden, otherwise true to show them
+    :param window:
+        the number of pages to be shown in the paging component (7 default)
+    :return:
+        the HTML string of the paging component
+    """
+
+    def get_params(**kwargs):
+        params = []
+        for k, v in kwargs.items():
+            if k == 'showPaused':
+                # True is default or None
+                if v or v is None:
+                    continue
+                params.append('{}={}'.format(k, v))
+            elif v:
+                params.append('{}={}'.format(k, v))
+        return '&'.join(params)
+
+    void_link = 'javascript:void(0)'
+    first_node = """<li class="paginate_button {disabled}" id="dags_first">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&laquo;</a>
+</li>"""
+
+    previous_node = """<li class="paginate_button previous {disabled}" id="dags_previous">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&lt;</a>
+</li>"""
+
+    next_node = """<li class="paginate_button next {disabled}" id="dags_next">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&gt;</a>
+</li>"""
+
+    last_node = """<li class="paginate_button {disabled}" id="dags_last">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&raquo;</a>
+</li>"""
+
+    page_node = """<li class="paginate_button {is_active}">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="2" tabindex="0">{page_num}</a>
+</li>"""
+
+    output = ['<ul class="pagination" style="margin-top:0px;">']
+
+    is_disabled = 'disabled' if current_page <= 0 else ''
+    output.append(first_node.format(href_link="?{}"
+                                    .format(get_params(page=0,
+                                                       search=search,
+                                                       showPaused=showPaused)),
+                                    disabled=is_disabled))
+
+    page_link = void_link
+    if current_page > 0:
+        page_link = '?{}'.format(get_params(page=(current_page - 1),
+                                            search=search,
+                                            showPaused=showPaused))
+
+    output.append(previous_node.format(href_link=page_link,
+                                       disabled=is_disabled))
+
+    mid = int(window / 2)
+    last_page = num_of_pages - 1
+
+    if current_page <= mid or num_of_pages < window:
+        pages = [i for i in range(0, min(num_of_pages, window))]
+    elif mid < current_page < last_page - mid:
+        pages = [i for i in range(current_page - mid, current_page + mid + 1)]
+    else:
+        pages = [i for i in range(num_of_pages - window, last_page + 1)]
+
+    def is_current(current, page):
+        return page == current
+
+    for page in pages:
+        vals = {
+            'is_active': 'active' if is_current(current_page, page) else '',
+            'href_link': void_link if is_current(current_page, page)
+                         else '?{}'.format(get_params(page=page,
+                                                      search=search,
+                                                      showPaused=showPaused)),
+            'page_num': page + 1
+        }
+        output.append(page_node.format(**vals))
+
+    is_disabled = 'disabled' if current_page >= num_of_pages - 1 else ''
+
+    page_link = (void_link if current_page >= num_of_pages - 1
+                 else '?{}'.format(get_params(page=current_page + 1,
+                                              search=search,
+                                              showPaused=showPaused)))
+
+    output.append(next_node.format(href_link=page_link, disabled=is_disabled))
+    output.append(last_node.format(href_link="?{}"
+                                   .format(get_params(page=last_page,
+                                                      search=search,
+                                                      showPaused=showPaused)),
+                                   disabled=is_disabled))
+
+    output.append('</ul>')
+
+    return wtforms.widgets.core.HTMLString('\n'.join(output))
+
+
 def limit_sql(sql, limit, conn_type):
     sql = sql.strip()
     sql = sql.rstrip(';')