Add peeks for idents

elixir/api.py

@@ -46,14 +46,15 @@ def on_get(self, req, resp, project, ident):
         if version == 'latest':
             version = query.query('latest')
 
-        symbol_definitions, symbol_references, symbol_doccomments = query.query('ident', version, ident, family)
+        symbol_definitions, symbol_references, symbol_doccomments, peeks = query.query('ident', version, ident, family)
 
         resp.status = falcon.HTTP_200
         resp.content_type = falcon.MEDIA_JSON
         resp.media = {
             'definitions': [sym.__dict__ for sym in symbol_definitions],
             'references': [sym.__dict__ for sym in symbol_references],
-            'documentations': [sym.__dict__ for sym in symbol_doccomments]
+            'documentations': [sym.__dict__ for sym in symbol_doccomments],
+            'peeks': peeks
         }
 
         query.close()

elixir/lib.py

@@ -26,15 +26,15 @@
 
 CURRENT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../')
 
-def script(*args, env=None):
+def script(*args, input=None, env=None):
     args = (os.path.join(CURRENT_DIR, 'script.sh'),) + args
     # subprocess.run was introduced in Python 3.5
     # fall back to subprocess.check_output if it's not available
     if hasattr(subprocess, 'run'):
-        p = subprocess.run(args, stdout=subprocess.PIPE, env=env)
+        p = subprocess.run(args, stdout=subprocess.PIPE, input=input, env=env)
         p = p.stdout
     else:
-        p = subprocess.check_output(args)
+        p = subprocess.check_output(args, input=input)
     return p
 
 def run_cmd(*args, env=None):

elixir/query.py

@@ -63,8 +63,8 @@ def __init__(self, data_dir, repo_dir):
         self.db = data.DB(data_dir, readonly=True, dtscomp=self.dts_comp_support)
         self.file_cache = {}
 
-    def script(self, *args):
-        return script(*args, env=self.getEnv())
+    def script(self, *args, input=None):
+        return script(*args, input=input, env=self.getEnv())
 
     def scriptLines(self, *args):
         return scriptLines(*args, env=self.getEnv())
@@ -257,9 +257,12 @@ def query(self, cmd, *args):
 
             # DT bindings compatible strings are handled differently
             if family == 'B':
-                return self.get_idents_comps(version, ident)
+                defs, refs, docs = self.get_idents_comps(version, ident)
             else:
-                return self.get_idents_defs(version, ident, family)
+                defs, refs, docs = self.get_idents_defs(version, ident, family)
+
+            peeks = self.get_peeks_of_syms(version, defs, refs)
+            return defs, refs, docs, peeks
 
         else:
             return 'Unknown subcommand: ' + cmd + '\n'
@@ -401,9 +404,55 @@ def get_idents_defs(self, version, ident, family):
 
         return symbol_definitions, symbol_references, symbol_doccomments
 
+    def get_files_and_zip(self, version, syms):
+        batch = b"\n".join([f"{version}:{sym.path}".encode() for sym in syms])
+        batch_res = self.script('get-files-batch', input=batch)
+
+        # See https://git-scm.com/docs/git-cat-file#_batch_output for the format:
+        #
+        # <oid> SP <type> SP <size> LF
+        # <contents> LF
+        # <oid> SP <type> SP <size> LF
+        # <contents> LF
+        # <oid> SP <type> SP <size> LF
+        # <contents> LF
+        # 
+        for sym in syms:
+            meta, batch_res = batch_res.split(b"\n", 1)
+            _, _, size = meta.split(b" ")
+            size = int(size) + 1 # newline after each file
+            content = batch_res[:size].split(b"\n")
+            batch_res = batch_res[size:]
+            yield sym, content
+
+    def get_peeks_of_syms(self, version, symbol_definitions, symbol_references):
+        peeks = {}
+
+        def request_peeks(syms):
+            if len(syms) > 100:
+                return
+
+            for sym, content in self.get_files_and_zip(version, syms):
+                if sym.path not in peeks:
+                    peeks[sym.path] = {}
+
+                if type(sym.line) is int:
+                    lines = (sym.line,)
+                else:
+                    lines = map(int, sym.line.split(','))
+
+                for num in lines:
+                    index = num - 1
+                    if index >= 0 and index < len(content):
+                        peeks[sym.path][num] = decode(content[index]).strip()
+
+        request_peeks(symbol_definitions)
+        request_peeks(symbol_references)
+        return peeks
+
 
 def cmd_ident(q, version, ident, family, **kwargs):
-    symbol_definitions, symbol_references, symbol_doccomments = q.query("ident", version, ident, family)
+    symbol_definitions, symbol_references, symbol_doccomments, peeks = q.query("ident", version, ident, family)
     print("Symbol Definitions:")
     for symbol_definition in symbol_definitions:
         print(symbol_definition)
@@ -416,10 +465,33 @@ def cmd_ident(q, version, ident, family, **kwargs):
     for symbol_doccomment in symbol_doccomments:
         print(symbol_doccomment)
 
+    print("\nSymbol peeks:")
+    for file, content in peeks.items():
+        for num, line in content.items():
+            print(f"{file}:{num}: {line}")
+
 def cmd_file(q, version, path, **kwargs):
     code = q.query("file", version, path)
     print(code)
 
+def profile(q, family, version, ident):
+    return q.query('ident', version, ident, family)
+
+def cmd_profile(q, family, version, ident, **kwargs):
+    import cProfile
+    cProfile.runctx('profile(q, family, version, ident)',
+                    globals={
+                        "profile": profile
+                    },
+                    locals={
+                        "q": q,
+                        "family": family,
+                        "version": version,
+                        "ident": ident,
+                    },
+                    sort='tottime'
+    )
+
 if __name__ == "__main__":
     import argparse
 
@@ -438,5 +510,10 @@ def cmd_file(q, version, path, **kwargs):
     file_subparser.add_argument('path', type=str, help="The path of the source file")
     file_subparser.set_defaults(func=cmd_file, q=query)
 
+    profile_subparser = subparsers.add_parser('perf', help="Get a source file")
+    profile_subparser.add_argument('ident', type=str, help="The name of the identifier")
+    profile_subparser.add_argument('family', type=str, help="The file family requested")
+    profile_subparser.set_defaults(func=cmd_profile, q=query)
+
     args = parser.parse_args()
     args.func(**vars(args))

elixir/web.py

@@ -27,7 +27,7 @@
 import datetime
 from collections import OrderedDict, namedtuple
 from re import search, sub
-from typing import Any, Callable, NamedTuple, Tuple
+from typing import Any, Callable, Dict, NamedTuple, Tuple
 from urllib import parse
 import falcon
 import jinja2
@@ -662,23 +662,26 @@ def generate_source_page(ctx: RequestContext, q: Query,
 # type : type of the symbol
 # path: path of the file that contains the symbol
 # line: list of LineWithURL
-SymbolEntry = namedtuple('SymbolEntry', 'type, path, lines')
+# peeks: map of code line previews for this path
+SymbolEntry = namedtuple('SymbolEntry', 'type, path, lines, peeks')
 
 # Converts SymbolInstance into SymbolEntry
 # path of SymbolInstance will be appended to base_url
-def symbol_instance_to_entry(base_url: str, symbol: SymbolInstance) -> SymbolEntry:
+def symbol_instance_to_entry(base_url: str, symbol: SymbolInstance, peeks: Dict[str, Dict[int, str]]) -> SymbolEntry:
     # TODO this should be a responsibility of Query
     if type(symbol.line) is str:
         line_numbers = symbol.line.split(',')
     else:
         line_numbers = [symbol.line]
 
     lines = [
-        LineWithURL(l, f'{ base_url }/{ symbol.path }#L{ l }')
+        LineWithURL(int(l), f'{ base_url }/{ symbol.path }#L{ l }')
         for l in line_numbers
     ]
 
-    return SymbolEntry(symbol.type, symbol.path, lines)
+    current_peeks = peeks.get(symbol.path, {})
+
+    return SymbolEntry(symbol.type, symbol.path, lines, current_peeks)
 
 # Generates response (status code and optionally HTML) of the `ident` route
 # basedir: path to data directory, ex: "/srv/elixir-data"
@@ -687,14 +690,17 @@ def generate_ident_page(ctx: RequestContext, q: Query,
 
     status = falcon.HTTP_OK
     source_base_url = get_source_base_url(project, version)
-    symbol_definitions, symbol_references, symbol_doccomments = q.query('ident', version, ident, family)
+
+    symbol_definitions, symbol_references, symbol_doccomments, peeks = q.query('ident', version, ident, family)
+
     symbol_sections = []
+    empty_peeks = {}
 
     if len(symbol_definitions) or len(symbol_references):
         if len(symbol_doccomments):
             symbol_sections.append({
                 'title': 'Documented',
-                'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym) for sym in symbol_doccomments]},
+                'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym, empty_peeks) for sym in symbol_doccomments]},
             })
 
         if len(symbol_definitions):
@@ -703,9 +709,9 @@ def generate_ident_page(ctx: RequestContext, q: Query,
             # TODO this should be a responsibility of Query
             for sym in symbol_definitions:
                 if sym.type not in defs_by_type:
-                    defs_by_type[sym.type] = [symbol_instance_to_entry(source_base_url, sym)]
+                    defs_by_type[sym.type] = [symbol_instance_to_entry(source_base_url, sym, peeks)]
                 else:
-                    defs_by_type[sym.type].append(symbol_instance_to_entry(source_base_url, sym))
+                    defs_by_type[sym.type].append(symbol_instance_to_entry(source_base_url, sym, peeks))
 
             symbol_sections.append({
                 'title': 'Defined',
@@ -719,7 +725,7 @@ def generate_ident_page(ctx: RequestContext, q: Query,
         if len(symbol_references):
             symbol_sections.append({
                 'title': 'Referenced',
-                'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym) for sym in symbol_references]},
+                'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym, peeks) for sym in symbol_references]},
             })
         else:
             symbol_sections.append({

script.sh

@@ -85,6 +85,11 @@ get_file()
     git cat-file blob "$v:`denormalize $opt2`" 2>/dev/null
 }
 
+get_files_batch()
+{
+    git cat-file --batch 2>/dev/null
+}
+
 get_dir()
 {
         v=`echo $opt1 | version_rev`
@@ -263,6 +268,10 @@ case $cmd in
         get_file
         ;;
 
+    get-files-batch)
+        get_files_batch
+        ;;
+
     get-dir)
         get_dir
         ;;

static/dynamic-references.js

@@ -28,7 +28,7 @@ function identUrl(project, ident, version, family) {
   ]
 */
 
-function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) {
+function generateSymbolDefinitionsHTML(symbolDefinitions, peeks, project, version) {
   let result = "";
   let typesCount = {};
   let previous_type = "";
@@ -55,7 +55,7 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) {
       previous_type = sd.type;
     }
     let ln = sd.line.toString().split(',');
-    if (ln.length == 1) {
+    if (ln.length == 1 && !peeks) {
       let n = ln[0];
       result += `<li><a href="/${project}/${version}/source/${sd.path}#L${n}"><strong>${sd.path}</strong>, line ${n} <em>(as a ${sd.type})</em></a>`;
     } else {
@@ -66,7 +66,14 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) {
         result += `<li><a href="/${project}/${version}/source/${sd.path}#L${ln[0]}"><strong>${sd.path}</strong> <em>(as a ${sd.type})</em></a>`;
         result += '<ul>';
         for(let n of ln) {
-          result += `<li><a href="/${project}/${version}/source/${sd.path}#L${n}">line ${n}</a></li>`;
+          result += `<li><a href="/${project}/${version}/source/${sd.path}#L${n}"><span>line ${n}</span>`;
+          let srcLine = peeks?.[sd.path]?.[n];
+          if(srcLine) {
+            let tag = document.createElement("pre");
+            tag.textContent = srcLine;
+            result += tag.outerHTML;
+          }
+          result += '</a></li>'
         }
         result += '</ul>';
       }
@@ -77,7 +84,7 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) {
   return result;
 }
 
-function generateSymbolReferencesHTML(symbolReferences, project, version) {
+function generateSymbolReferencesHTML(symbolReferences, peeks, project, version) {
   let result = "";
 
   if(symbolReferences.length == 0) {
@@ -88,7 +95,7 @@ function generateSymbolReferencesHTML(symbolReferences, project, version) {
   result += '<ul>';
   for (let sr of symbolReferences) {
     let ln = sr.line.split(',');
-    if (ln.length == 1) {
+    if (ln.length == 1 && !peeks) {
       let n = ln[0];
       result += `<li><a href="/${project}/${version}/source/${sr.path}#L${n}"><strong>${sr.path}</strong>, line ${n}</a>`;
     } else {
@@ -99,7 +106,14 @@ function generateSymbolReferencesHTML(symbolReferences, project, version) {
         result += `<li><a href="/${project}/${version}/source/${sr.path}#L${ln[0]}"><strong>${sr.path}</strong></a>`;
         result += '<ul>'
         for(let n of ln) {
-          result += `<li><a href="/${project}/${version}/source/${sr.path}#L${n}">line ${n}</a>`
+          result += `<li><a href="/${project}/${version}/source/${sr.path}#L${n}"><span>line ${n}</span>`
+          let srcLine = peeks?.[sr.path]?.[n];
+          if (srcLine) {
+            let tag = document.createElement("pre");
+            tag.textContent = srcLine;
+            result += tag.outerHTML;
+          }
+          result += '</a></li>'
         }
         result += '</ul>'
       }
@@ -143,10 +157,11 @@ function generateReferencesHTML(data, project, version) {
   let symbolDefinitions = data["definitions"];
   let symbolReferences = data["references"];
   let symbolDocumentations = data["documentations"];
+  let peeks = data["peeks"];
   return '<div class="lxrident">' +
     generateDocCommentsHTML(symbolDocumentations, project, version) +
-    generateSymbolDefinitionsHTML(symbolDefinitions, project, version) +
-    generateSymbolReferencesHTML(symbolReferences, project, version) +
+    generateSymbolDefinitionsHTML(symbolDefinitions, peeks, project, version) +
+    generateSymbolReferencesHTML(symbolReferences, peeks, project, version) +
     '</div>';
 }
 

static/style.css

@@ -718,6 +718,12 @@ h2 {
   font-weight: 400;
   color: #000;
 }
+
+.lxrident a span {
+  width: 8em;
+  display: inline-block;
+}
+
 .lxrident li li::before {
   color: #444;
 }
@@ -739,6 +745,11 @@ h2 {
   content: '└╴';
 }
 
+.lxrident li pre {
+    display: inline-block;
+    margin: auto;
+}
+
 
 /* tree */