Merge pull request #118 from dentonmwood/master

Getting end lines from code

Author: dentonmwood

cli/__init__.py

@@ -1 +1 @@
-"""Package containing implementation of the application's CLI (command line interface)."""
+"""Package containing implementation of the application's CLI (command start interface)."""

cli/app.py

@@ -13,7 +13,7 @@
 def main():
     """Entry point of the application."""
     try:
-        # Parse command line arguments
+        # Parse command start arguments
         repos, algorithm = handle_cli_args()
 
         time_snap("Arguments handled; Repositories parsed")
@@ -42,8 +42,13 @@ def main():
         time_snap("Analysis completed")
 
         # Save detection result to a JSON file.
-        json_filename = "clones_" + \
-            datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".json"
+
+        repo_0 = repos[0][repos[0].rfind('/') + 1:]
+        json_filename = "clones_" + algorithm + "_" + repo_0 + "_"
+        if repos[1]:
+            repo_1 = repos[1][repos[1].rfind('/') + 1:]
+            json_filename += repo_1 + "_"
+        json_filename += datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".json"
 
         with open(json_filename, "w") as f:
             f.write(result.json())

cli/args_handler.py

@@ -1,4 +1,4 @@
-"""Module containing functions for handling command-line arguments supplied by the user."""
+"""Module containing functions for handling command-start arguments supplied by the user."""
 
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from os.path import isdir
@@ -63,7 +63,7 @@ def repo_path_to_local_path(repo_path):
 
 def handle_cli_args():
     """
-    Parse the command line arguments and handle them.
+    Parse the command start arguments and handle them.
 
     If there is any problem, an error message will be printed
     and the script will exit with a non-zero exit code.

engine/algorithms/chlorine/chlorine.py

@@ -136,7 +136,7 @@ def chlorine_single_repo(modules):
 
     Clones must satisfy the settings at the top of this source file.
     Detected code clones are printed on STDOUT, including the common skeleton,
-    path to each clones (source file path, line number, column offset),
+    path to each clones (source file path, start number, column offset),
     size of each clone (number of nodes in its syntax tree) and their
     similarity percentage (number of matching nodes / total number of nodes).
 

engine/errors/user_input.py

@@ -3,10 +3,10 @@
 
 class UserInputError(Exception):
     """
-    Exception representing invalid user input such as command line arguments.
+    Exception representing invalid user input such as command start arguments.
 
     Alternatively, this can also represented a problem caused by
-    invalid user input further down the line.
+    invalid user input further down the start.
     Simply put, the problem can / must be fixed by modifying the user input.
 
     Attributes:

engine/nodes/nodeorigin.py

@@ -7,23 +7,21 @@ class NodeOrigin:
 
     Attributes:
         file {string} -- Source file from which the node originates.
-        line {int|None} -- Line number at which the node was found.
-        col_offset {int|None} -- Column offset within the line.
-                                 Number of characters on the same
-                                 line before the node's token.
+        start {int|None} -- starting line number at which the node was found.
+        end {int|None} -- ending line number at which the node was found.
 
     """
 
-    def __init__(self, file_path, line=None, col_offset=None):
+    def __init__(self, file_path, start=None, end=None):
         """
         Initialize a new node origin instance.
 
         Arguments:
             file_path {string} -- Path to the node's source file.
 
         Keyword Arguments:
-            line {int} -- Line number of node's origin. (default: {None})
-            col_offset {int} -- Column offset of node. (default: {None})
+            start {int} -- Starting line number of node's origin. (default: {None})
+            end {int} -- Ending line number of node's origin. (default: {None})
 
         Raises:
             ValueError -- When file path is None or when only one of the two
@@ -34,18 +32,18 @@ def __init__(self, file_path, line=None, col_offset=None):
             raise ValueError(
                 "File path must always be set to a non-None value")
 
-        if (line is None) != (col_offset is None):
+        if (start is None) != (end is None):
             raise ValueError(
-                "Either both line number and column offset must be set or neither")
+                "Either both the start and end lines must be set or neither")
 
         self.file = file_path
-        self.line = line
-        self.col_offset = col_offset
+        self.start = start
+        self.end = end
 
     def __str__(self):
         """Convert the node origin into a human-readable string representation."""
-        return self.file + (f" (L: {self.line} C: {self.col_offset})"
-                            if self.line and self.col_offset else "")
+        return self.file + (f" ({self.start}, {self.end})"
+                            if self.start and self.end else "")
 
     def __repr__(self):
         """Return a string representation of the node origin."""

engine/nodes/tree.py

@@ -1,6 +1,7 @@
 """Module containing the `TreeNode` class."""
 
 import ast
+from asttokens import LineNumbers
 from .nodeorigin import NodeOrigin
 
 _IGNORE_CLASSES = [ast.Load, ast.Store, ast.Del,
@@ -16,7 +17,7 @@ class TreeNode:
 
     Attributes:
         node {AST} -- Original AST node generated by Python's built-in parser.
-        origin {NodeOrigin} -- Origin of the node (file path, line and column).
+        origin {NodeOrigin} -- Origin of the node (file path, start and column).
         children {list[TreeNode]} -- List of direct children of this node.
         weight {int} -- Total number of nodes in this node's tree.
         names {list[string]} -- All names / symbols used in this node's tree.
@@ -27,25 +28,30 @@ class TreeNode:
 
     """
 
-    def __init__(self, node, origin_file):
+    def __init__(self, atok, lino, node, origin_file):
         """
         Initialize a new tree node instance.
 
         Arguments:
+            atok -- asttokens object containing the tokens for the AST
+            lino - asttokens LineNumber object containing the conversion for offsets to line numbers
             node -- Single raw node produced by the Python AST parser.
             origin_file {string} -- Relative path to the source file.
 
         """
         self.node = node
-        self.origin = NodeOrigin(origin_file, node.lineno, node.col_offset) \
-            if node._attributes else NodeOrigin(origin_file)
+
+        start, end = atok.get_text_range(node)
+        start_line, start_char = LineNumbers.offset_to_line(lino, start)
+        end_line, end_char = LineNumbers.offset_to_line(lino, end)
+        self.origin = NodeOrigin(origin_file, start_line, end_line)
 
         # Check if this type of node can have docstring.
         can_have_docstring = node.__class__ in [ast.ClassDef, ast.FunctionDef]
 
         # HACK: Ignore useless context-related children.
         # This should greatly reduce the total number of nodes.
-        self.children = [TreeNode(n, origin_file) for n in
+        self.children = [TreeNode(atok, lino, n, origin_file) for n in
                          ast.iter_child_nodes(node)
                          if n.__class__ not in _IGNORE_CLASSES and
                          # Ignore docstrings in class and function definitions.

engine/preprocessing/module_parser.py

@@ -1,6 +1,7 @@
 """Module containing code used for parsing of modules and nodes from Python code."""
 
 import ast
+from asttokens import asttokens, LineNumbers
 from os import listdir, path
 from os.path import isdir, isfile, relpath
 from ..nodes.tree import TreeNode
@@ -17,20 +18,6 @@ def _read_whole_file(file_path):
         return f.read()
 
 
-def _read_ast_from_file(file_path):
-    """
-    Parse a module AST from the specified file.
-
-    Arguments:
-        file_path {string} -- Path of file to parse the AST from.
-
-    Returns:
-        AST parsed from the specified file.
-
-    """
-    return ast.parse(_read_whole_file(file_path))
-
-
 def _get_tree_node_from_file(file_path, repo_path):
     """
     Parse a TreeNode representing the module in the specified file.
@@ -42,7 +29,10 @@ def _get_tree_node_from_file(file_path, repo_path):
         TreeNode -- TreeNode parsed from the specified file.
 
     """
-    return TreeNode(_read_ast_from_file(file_path),
+    atok = asttokens.ASTTokens(_read_whole_file(file_path), True)
+    root = atok.tree
+    lino = LineNumbers(atok.get_text(root))
+    return TreeNode(atok, lino, root,
                     relpath(file_path, repo_path))
 
 

requirements.txt

@@ -1,3 +1,4 @@
+asttokens
 # Interface for working with Git repositories
 gitpython
 # "Work list" binary table in Lee's algorithm (Iodine)

web/analyzer.py

@@ -42,7 +42,7 @@ def _extract_patterns(conn, commit_id, modules):
         pattern_id = _get_pattern_id(conn, n)
 
         conn.run("""INSERT INTO pattern_instances """ +
-                 """(pattern_id, commit_id, "file", "line", col_offset) """ +
+                 """(pattern_id, commit_id, "file", "start", end) """ +
                  """VALUES (%s, %s, %s, %s, %s);""",
                  pattern_id, commit_id,
                  n.origin.file, n.origin.line, n.origin.col_offset)
@@ -85,7 +85,7 @@ def analyze_repo(repo_info, repo_id, algorithm=OXYGEN):
                                       commit_id, c.value, c.match_weight)
 
                 for o, s in c.origins.items():
-                    conn.run("""INSERT INTO origins (cluster_id, file, line, col_offset, similarity) VALUES (%s, %s, %s, %s, %s);""",
+                    conn.run("""INSERT INTO origins (cluster_id, file, start, end, similarity) VALUES (%s, %s, %s, %s, %s);""",
                              cluster_id, o.file, o.line, o.col_offset, s)
 
             log.success(
@@ -121,7 +121,7 @@ def find_repo_results(conn, repo_id):
     for c in conn.iter_dict("""SELECT id, "value", weight FROM clusters WHERE commit_id = %s;""", commit_id):
         origins = {}
 
-        for o in conn.iter_dict("""SELECT file, line, col_offset, similarity FROM origins WHERE cluster_id = %s;""", c.id):
+        for o in conn.iter_dict("""SELECT file, start, end, similarity FROM origins WHERE cluster_id = %s;""", c.id):
             origins[NodeOrigin(o.file, o.line, o.col_offset)] = o.similarity
 
         clones.append(DetectedClone(c.value, c.weight, origins=origins))