Rebuild enrichment branch on top of 0.6.0 changes

Author: andrew

Gemfile.lock

@@ -15,13 +15,16 @@ PATH
   specs:
     git-pkgs (0.6.1)
       ecosystems-bibliothecary (~> 15.0)
+      purl (~> 1.6)
       rugged (~> 1.0)
       sequel (>= 5.0)
       sqlite3 (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
     benchmark (0.5.0)
     bigdecimal (4.0.1)
     csv (3.3.5)
@@ -46,6 +49,9 @@ GEM
     psych (5.3.1)
       date
       stringio
+    public_suffix (7.0.2)
+    purl (1.7.0)
+      addressable (~> 2.8)
     racc (1.8.1)
     rake (13.3.1)
     rdoc (7.0.3)
@@ -100,6 +106,7 @@ DEPENDENCIES
   simplecov
 
 CHECKSUMS
+  addressable (2.8.8) sha256=7c13b8f9536cf6364c03b9d417c19986019e28f7c00ac8132da4eb0fe393b057
   benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
   bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
   csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
@@ -118,6 +125,8 @@ CHECKSUMS
   prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
   prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103
   psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974
+  public_suffix (7.0.2) sha256=9114090c8e4e7135c1fd0e7acfea33afaab38101884320c65aaa0ffb8e26a857
+  purl (1.7.0) sha256=e25a6b951975e94104a17d8d40e8529fa882a5a63717c68af2390e9b8d0ac3f2
   racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
   rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
   rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9

git-pkgs.gemspec

@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sequel", ">= 5.0"
   spec.add_dependency "sqlite3", ">= 2.0"
   spec.add_dependency "ecosystems-bibliothecary", "~> 15.0"
+  spec.add_dependency "purl", "~> 1.6"
 end

lib/git/pkgs.rb

@@ -9,12 +9,15 @@
 require_relative "pkgs/repository"
 require_relative "pkgs/analyzer"
 
+require_relative "pkgs/purl_helper"
 require_relative "pkgs/models/branch"
 require_relative "pkgs/models/branch_commit"
 require_relative "pkgs/models/commit"
 require_relative "pkgs/models/manifest"
 require_relative "pkgs/models/dependency_change"
 require_relative "pkgs/models/dependency_snapshot"
+require_relative "pkgs/models/package"
+require_relative "pkgs/models/version"
 
 require_relative "pkgs/commands/init"
 require_relative "pkgs/commands/update"

lib/git/pkgs/database.rb

@@ -15,7 +15,7 @@ module Git
   module Pkgs
     class Database
       DB_FILE = "pkgs.sqlite3"
-      SCHEMA_VERSION = 1
+      SCHEMA_VERSION = 2
 
       class << self
         attr_accessor :db
@@ -82,7 +82,9 @@ def self.refresh_models
           Git::Pkgs::Models::Commit,
           Git::Pkgs::Models::Manifest,
           Git::Pkgs::Models::DependencyChange,
-          Git::Pkgs::Models::DependencySnapshot
+          Git::Pkgs::Models::DependencySnapshot,
+          Git::Pkgs::Models::Package,
+          Git::Pkgs::Models::Version
         ].each do |model|
           model.dataset = @db[model.table_name]
           # Clear all cached association data that may reference old db
@@ -177,6 +179,36 @@ def self.create_schema(with_indexes: true)
           DateTime :updated_at
         end
 
+        @db.create_table?(:packages) do
+          primary_key :id
+          String :purl, null: false
+          String :latest_version
+          String :license
+          String :description, text: true
+          String :homepage
+          String :repository_url
+          String :source
+          DateTime :enriched_at
+          DateTime :created_at
+          DateTime :updated_at
+          index :purl, unique: true
+        end
+
+        @db.create_table?(:versions) do
+          primary_key :id
+          String :purl, null: false
+          String :package_purl, null: false
+          String :license
+          DateTime :published_at
+          String :integrity, text: true
+          String :source
+          DateTime :enriched_at
+          DateTime :created_at
+          DateTime :updated_at
+          index :purl, unique: true
+          index :package_purl
+        end
+
         set_version
         create_bulk_indexes if with_indexes
         refresh_models

lib/git/pkgs/models/dependency_change.rb

@@ -28,6 +28,14 @@ def for_platform(platform)
             where(ecosystem: platform)
           end
         end
+
+        def purl(with_version: true)
+          version = nil
+          if with_version && manifest&.kind == "lockfile"
+            version = requirement
+          end
+          PurlHelper.build_purl(ecosystem: ecosystem, name: name, version: version)
+        end
       end
     end
   end

lib/git/pkgs/models/dependency_snapshot.rb

@@ -29,6 +29,14 @@ def self.current_for_branch(branch)
 
           where(commit: commit)
         end
+
+        def purl(with_version: true)
+          version = nil
+          if with_version && manifest&.kind == "lockfile"
+            version = requirement
+          end
+          PurlHelper.build_purl(ecosystem: ecosystem, name: name, version: version)
+        end
       end
     end
   end

lib/git/pkgs/models/package.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Git
+  module Pkgs
+    module Models
+      class Package < Sequel::Model
+        one_to_many :versions, key: :package_purl, primary_key: :purl
+
+        def parsed_purl
+          @parsed_purl ||= Purl.parse(purl)
+        end
+
+        def registry_url
+          parsed_purl.registry_url
+        end
+
+        def enriched?
+          !enriched_at.nil?
+        end
+      end
+    end
+  end
+end

lib/git/pkgs/models/version.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Git
+  module Pkgs
+    module Models
+      class Version < Sequel::Model
+        many_to_one :package, key: :package_purl, primary_key: :purl
+
+        def parsed_purl
+          @parsed_purl ||= Purl.parse(purl)
+        end
+
+        def version_string
+          parsed_purl.version
+        end
+
+        def registry_url
+          parsed_purl.registry_url
+        end
+
+        def enriched?
+          !enriched_at.nil?
+        end
+      end
+    end
+  end
+end

lib/git/pkgs/purl_helper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "purl"
+
+module Git
+  module Pkgs
+    module PurlHelper
+      # Mapping from Bibliothecary/ecosyste.ms ecosystem names to PURL types
+      # Source: https://packages.ecosyste.ms/api/v1/registries/
+      ECOSYSTEM_TO_PURL_TYPE = {
+        "npm" => "npm",
+        "go" => "golang",
+        "docker" => "docker",
+        "pypi" => "pypi",
+        "nuget" => "nuget",
+        "maven" => "maven",
+        "packagist" => "composer",
+        "cargo" => "cargo",
+        "rubygems" => "gem",
+        "cocoapods" => "cocoapods",
+        "pub" => "pub",
+        "bower" => "bower",
+        "cpan" => "cpan",
+        "alpine" => "alpine",
+        "actions" => "githubactions",
+        "cran" => "cran",
+        "clojars" => "clojars",
+        "conda" => "conda",
+        "hex" => "hex",
+        "hackage" => "hackage",
+        "julia" => "julia",
+        "swiftpm" => "swift",
+        "openvsx" => "openvsx",
+        "spack" => "spack",
+        "homebrew" => "brew",
+        "puppet" => "puppet",
+        "deno" => "deno",
+        "elm" => "elm",
+        "vcpkg" => "vcpkg",
+        "racket" => "racket",
+        "bioconductor" => "bioconductor",
+        "carthage" => "carthage",
+        "elpa" => "melpa"
+      }.freeze
+
+      def self.purl_type_for(ecosystem)
+        ECOSYSTEM_TO_PURL_TYPE.fetch(ecosystem, ecosystem)
+      end
+
+      def self.build_purl(ecosystem:, name:, version: nil)
+        type = purl_type_for(ecosystem)
+        Purl::PackageURL.new(type: type, name: name, version: version)
+      end
+    end
+  end
+end