Merge pull request #480 from github/pip-fixes

Use pip list to get packages for pip source

Author: jonabc

.licenses/bundler/bundler.dep.yml

@@ -1,6 +1,6 @@
 ---
 name: bundler
-version: 2.3.8
+version: 2.3.9
 type: bundler
 summary: The best way to manage your application's dependencies
 homepage: https://bundler.io

docs/sources/pip.md

@@ -2,11 +2,6 @@
 
 The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the `virtual_env_dir` specific directory before running `licensed`.
 
-Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command:
-```
-pip freeze > requirements.txt
-```
-
 A `virtualenv` directory is required before running `licensed`. You can setup a `virtualenv` by running the command:
 ```
 virtualenv <your_venv_dir>

lib/licensed/sources/pip.rb

@@ -7,17 +7,14 @@
 module Licensed
   module Sources
     class Pip < Source
-      VERSION_OPERATORS = %w(< > <= >= == !=).freeze
-      PACKAGE_REGEX = /^([\w\.-]+)(#{VERSION_OPERATORS.join("|")})?/
+      PACKAGE_INFO_SEPARATOR = "\n---\n"
 
       def enabled?
-        return unless virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip)
-        File.exist?(config.pwd.join("requirements.txt"))
+        virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip)
       end
 
       def enumerate_dependencies
-        Parallel.map(packages_from_requirements_txt, in_threads: Parallel.processor_count) do |package_name|
-          package = package_info(package_name)
+        packages.map do |package|
           location = File.join(package["Location"], package["Name"].gsub("-", "_") +  "-" + package["Version"] + ".dist-info")
           Dependency.new(
             name: package["Name"],
@@ -34,25 +31,45 @@ def enumerate_dependencies
 
       private
 
-      def packages_from_requirements_txt
-        File.read(config.pwd.join("requirements.txt"))
-            .lines
-            .reject { |line| line.include?("://") }
-            .map { |line| line.strip.match(PACKAGE_REGEX) { |match| match.captures.first } }
-            .compact
+      # Returns parsed information for all packages used by the project,
+      # using `pip list` to determine what packages are used and `pip show`
+      # to gather package information
+      def packages
+        all_packages = pip_show_command(package_names)
+        all_packages.split(PACKAGE_INFO_SEPARATOR).reduce([]) do |accum, val|
+          accum << parse_package_info(val)
+        end
+      end
+
+      # Returns the names of all of the packages used by the current project,
+      # as returned from `pip list`
+      def package_names
+        @package_names ||= begin
+          JSON.parse(pip_list_command).map { |package| package["name"] }
+        rescue JSON::ParserError => e
+          message = "Licensed was unable to parse the output from 'npm list'. JSON Error: #{e.message}"
+          raise Licensed::Sources::Source::Error, message
+        end
       end
 
-      def package_info(package_name)
-        p_info = pip_command(package_name).lines
-        p_info.each_with_object(Hash.new(0)) { |pkg, a|
+      # Returns a hash filled with package info parsed from the email-header formatted output
+      # returned by `pip show`
+      def parse_package_info(package_info)
+        package_info.lines.each_with_object(Hash.new(0)) { |pkg, a|
           k, v = pkg.split(":", 2)
           next if k.nil? || k.empty?
           a[k.strip] = v&.strip
         }
       end
 
-      def pip_command(*args)
-        Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "show", *args)
+      # Returns the output from `pip list --format=json`
+      def pip_list_command
+        Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "list", "--format=json")
+      end
+
+      # Returns the output from `pip show <package> <package> ...`
+      def pip_show_command(packages)
+        Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "show", *packages)
       end
 
       def virtual_env_pip

test/sources/pip_test.rb

@@ -25,109 +25,20 @@
     end
 
     describe "dependencies" do
-      it "detects dependencies without a version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "scapy" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with == version constraint" do
+      it "detects explicit dependencies" do
         Dir.chdir fixtures do
           dep = source.dependencies.detect { |d| d.name == "Jinja2" }
           assert dep
+          assert_equal "2.9.6", dep.version
           assert_equal "pip", dep.record["type"]
           assert dep.record["homepage"]
           assert dep.record["summary"]
         end
       end
 
-      it "detects dependencies with >= version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "requests" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with <= version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "tqdm" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with < version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "Pillow" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with > version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "Scrapy" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with != version constraint" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "numpy" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with whitespace between the package name and version operator" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "botocore" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with multiple version constraints" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "boto3" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with hyphens in package name" do
-        Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "lazy-object-proxy" }
-          assert dep
-          assert_equal "pip", dep.record["type"]
-          assert dep.record["homepage"]
-          assert dep.record["summary"]
-        end
-      end
-
-      it "detects dependencies with dots in package name" do
+      it "detects transitive dependencies" do
         Dir.chdir fixtures do
-          dep = source.dependencies.detect { |d| d.name == "backports.shutil-get-terminal-size" }
+          dep = source.dependencies.detect { |d| d.name == "MarkupSafe" }
           assert dep
           assert_equal "pip", dep.record["type"]
           assert dep.record["homepage"]