Make manifests authoritative for documentation

Author: thomashoneyman

docgen/README.md

@@ -32,6 +32,11 @@ artifact. Pursuit, search, and static rendering consume it; they do not define
 it. This is why `docgen` is a top-level workspace package rather than part of a
 Pursuit application or the public `registry-lib` package.
 
+Package metadata always comes from the canonical `purs.json` stored in the
+registry tarball. Historical Pursuit JSON is used during migration only for
+compiler-produced documentation and resolution data; it is not an alternative
+source for package identity, metadata, dependencies, or source files.
+
 The main modules are:
 
 - `Registry.Docgen.Docs`: the canonical in-memory documentation model.

docgen/src/Registry/Docgen/Convert.purs

@@ -15,43 +15,51 @@ import Data.Maybe (Maybe(..), fromMaybe)
 import Data.Newtype (unwrap)
 import Data.String as String
 import Data.Traversable (traverse)
-import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocConstraint(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocPackage(..), DocReexport(..), DocType(..), ForallBinding(..), FunDep(..), Ident(..), InfixAlias(..), IntLiteral(..), ModuleName(..), OperatorName(..), Qualified(..), RawRange, Readme, RowLabel(..), RowRep, SourceArtifact, SourceSpan(..), StringLiteral, TypeName(..), TypeVar(..), ValueName(..), isPrim, schemaVersion)
+import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocConstraint(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocPackage(..), DocReexport(..), DocType(..), ForallBinding(..), FunDep(..), Ident(..), InfixAlias(..), IntLiteral(..), ModuleName(..), OperatorName(..), Qualified(..), RawRange(..), Readme, RowLabel(..), RowRep, SourceArtifact, SourceSpan(..), StringLiteral, TypeName(..), TypeVar(..), ValueName(..), isPrim, schemaVersion)
 import Registry.Docgen.Legacy.Docs (InPackage(..))
 import Registry.Docgen.Legacy.Docs as L
-import Registry.Location (Location(..))
-import Registry.PackageName (PackageName)
+import Registry.LimitedString as LimitedString
+import Registry.Manifest (Manifest(..))
+import Registry.PackageName as PackageName
+import Registry.Range as Range
 import Registry.Version as Version
 import Safe.Coerce (coerce)
 
--- | Convert historical Pursuit documentation while taking declared dependency
--- | ranges from the authoritative registry manifest.
+-- | Convert historical Pursuit documentation while taking all package metadata
+-- | from the authoritative registry manifest.
 fromLegacyPackage
-  :: { dependencies :: Map PackageName RawRange
+  :: { manifest :: Manifest
      , readme :: Maybe Readme
      , sourceArtifact :: SourceArtifact
      , sourcePaths :: Map ModuleName String
      }
   -> L.DocPackage
   -> Either String DocPackage
-fromLegacyPackage input (L.DocPackage pkg@{ github: L.GithubData github, packageMeta: L.DocPackageMeta meta }) = do
-  compilerVersion <- Version.parse pkg.compilerVersion
-  modules <- traverse convertModule pkg.modules
-  pure $ DocPackage
-    { schemaVersion
-    , compilerVersion
-    , sourceArtifact: input.sourceArtifact
-    , dependencies: input.dependencies
-    , description: meta.description
-    , license: meta.license
-    , location
-    , locationRef: Just pkg.versionTag
-    , name: coerce meta.name
-    , modules
-    , readme: input.readme
-    , resolvedDependencies: pkg.resolvedDependencies
-    , resolvedModulePackages: pkg.moduleMap
-    , version: pkg.version
-    }
+fromLegacyPackage input (L.DocPackage pkg@{ packageMeta: L.DocPackageMeta meta }) = do
+  let Manifest manifest = input.manifest
+  if meta.name /= manifest.name then
+    Left $ "Legacy documentation package " <> PackageName.print meta.name <> " does not match manifest package " <> PackageName.print manifest.name
+  else if pkg.version /= manifest.version then
+    Left $ "Legacy documentation version " <> Version.print pkg.version <> " does not match manifest version " <> Version.print manifest.version
+  else do
+    compilerVersion <- Version.parse pkg.compilerVersion
+    modules <- traverse convertModule pkg.modules
+    pure $ DocPackage
+      { schemaVersion
+      , compilerVersion
+      , sourceArtifact: input.sourceArtifact
+      , dependencies: map (RawRange <<< Range.print) manifest.dependencies
+      , description: map LimitedString.print manifest.description
+      , license: manifest.license
+      , location: manifest.location
+      , locationRef: Just manifest.ref
+      , name: manifest.name
+      , modules
+      , readme: input.readme
+      , resolvedDependencies: pkg.resolvedDependencies
+      , resolvedModulePackages: pkg.moduleMap
+      , version: manifest.version
+      }
   where
   convertModule legacy@(L.DocModule { name })
     | isPrim name = Right $ fromLegacyModule sourcePaths legacy
@@ -61,13 +69,6 @@ fromLegacyPackage input (L.DocPackage pkg@{ github: L.GithubData github, package
 
   sourcePaths = input.sourcePaths
 
-  location :: Location
-  location = GitHub
-    { owner: github.user
-    , repo: github.repo
-    , subdir: Nothing
-    }
-
 fromLegacyModule :: Map ModuleName String -> L.DocModule -> DocModule
 fromLegacyModule sourcePaths (L.DocModule mod@{ name: ownerModule }) =
   DocModule

docgen/src/Registry/Docgen/Generate.purs

@@ -22,14 +22,15 @@ import Data.String as String
 import Data.Tuple (Tuple(..))
 import PureScript.CST.Types as CST
 import Registry.Docgen.Convert as Convert
-import Registry.Docgen.Docs (DocModule(..), DocPackage(..), ModuleName, RawRange, Readme, SourceArtifact, schemaVersion)
+import Registry.Docgen.Docs (DocModule(..), DocPackage(..), ModuleName, RawRange(..), Readme, SourceArtifact, schemaVersion)
 import Registry.Docgen.Legacy.Docs as Legacy
 import Registry.Docgen.Reexports (ReexportError)
 import Registry.Docgen.Reexports as Reexports
-import Registry.License (License)
-import Registry.Location (Location)
+import Registry.LimitedString as LimitedString
+import Registry.Manifest (Manifest(..))
 import Registry.PackageName (PackageName)
 import Registry.PackageName as PackageName
+import Registry.Range as Range
 import Registry.Version (Version)
 import Safe.Coerce (coerce)
 
@@ -42,17 +43,11 @@ type ModuleInput =
 
 type PackageInput =
   { compilerVersion :: Version
-  , dependencies :: Map PackageName RawRange
-  , description :: Maybe String
-  , license :: License
-  , location :: Location
-  , locationRef :: Maybe String
+  , manifest :: Manifest
   , modules :: Array ModuleInput
-  , name :: PackageName
   , readme :: Maybe Readme
   , resolvedDependencies :: Map PackageName Version
   , sourceArtifact :: SourceArtifact
-  , version :: Version
   }
 
 data GenerationError
@@ -79,12 +74,12 @@ printGenerationError = case _ of
     Reexports.printReexportError error
 
 generatePackage :: PackageInput -> Either GenerationError DocPackage
-generatePackage input = do
+generatePackage input@{ manifest: Manifest manifest } = do
   let sortedInputs = Array.sortBy (comparing docsModuleName) input.modules
   _ <- foldM validateModule Set.empty sortedInputs
-  let packageModules = Array.filter (_.package >>> eq input.name) sortedInputs
+  let packageModules = Array.filter (_.package >>> eq manifest.name) sortedInputs
   if Array.null packageModules then
-    Left $ NoPackageModules input.name
+    Left $ NoPackageModules manifest.name
   else do
     let sourcePaths = Map.fromFoldable $ map (\moduleInput -> Tuple (docsModuleName moduleInput) moduleInput.sourcePath) sortedInputs
     let converted = map (Convert.fromLegacyModule sourcePaths <<< _.docs) sortedInputs
@@ -94,17 +89,17 @@ generatePackage input = do
       { schemaVersion
       , compilerVersion: input.compilerVersion
       , sourceArtifact: input.sourceArtifact
-      , dependencies: input.dependencies
-      , description: input.description
-      , license: input.license
-      , location: input.location
-      , locationRef: input.locationRef
-      , modules: Array.filter (\(DocModule { name }) -> Map.lookup name modulePackages == Just input.name) resolved
-      , name: input.name
+      , dependencies: map (RawRange <<< Range.print) manifest.dependencies
+      , description: map LimitedString.print manifest.description
+      , license: manifest.license
+      , location: manifest.location
+      , locationRef: Just manifest.ref
+      , modules: Array.filter (\(DocModule { name }) -> Map.lookup name modulePackages == Just manifest.name) resolved
+      , name: manifest.name
       , readme: input.readme
       , resolvedDependencies: input.resolvedDependencies
       , resolvedModulePackages: modulePackages
-      , version: input.version
+      , version: manifest.version
       }
   where
   validateModule seen moduleInput = do
@@ -116,7 +111,7 @@ generatePackage input = do
       Left $ DuplicateModule docsName
     else if not (isPackageRelativePath moduleInput.sourcePath) then
       Left $ InvalidSourcePath docsName moduleInput.sourcePath
-    else if moduleInput.package /= input.name && not (Map.member moduleInput.package input.resolvedDependencies) then
+    else if moduleInput.package /= manifest.name && not (Map.member moduleInput.package input.resolvedDependencies) then
       Left $ UnknownModulePackage docsName moduleInput.package
     else
       Right $ Set.insert docsName seen

docgen/test/Test/Registry/Docgen.purs

@@ -31,9 +31,13 @@ import Registry.Docgen.Legacy.JSON as Legacy.JSON
 import Registry.Docgen.Package.Render (defaultPackageLinker, htmlCodeRenderer, renderDeclarationInfo, renderDocument, renderModule)
 import Registry.Docgen.Reexports (ReexportError(..), modulesWithReexports, printReexportError)
 import Registry.License as License
+import Registry.LimitedString as LimitedString
 import Registry.Location (Location(..))
+import Registry.Manifest (Manifest(..))
 import Registry.PackageName (PackageName)
 import Registry.PackageName as PackageName
+import Registry.Range (Range)
+import Registry.Range as Range
 import Registry.Sha256 as Sha256
 import Registry.Test.Assert as Assert
 import Registry.Test.Utils as Utils
@@ -82,8 +86,8 @@ main = runSpecAndExitProcess [ consoleReporter ] do
       let readme = Readme { content: "# Undefined", extension: Just "md" }
       let
         conversionInput =
-          { dependencies: Map.empty
-          , readme: Just readme
+          { readme: Just readme
+          , manifest: undefinedManifest
           , sourceArtifact: sourceArtifactFixture
           , sourcePaths: Map.singleton (ModuleName "Undefined") "custom/Undefined.purs"
           }
@@ -99,10 +103,33 @@ main = runSpecAndExitProcess [ consoleReporter ] do
           span.path `Assert.shouldEqual` "custom/Undefined.purs"
         _ ->
           Assert.fail "Expected one converted declaration with a source span"
-      let manifestDependencies = Map.singleton (packageName "prelude") (RawRange ">=4.0.0 <5.0.0")
-      case Convert.fromLegacyPackage (conversionInput { dependencies = manifestDependencies }) legacy of
-        Right (DocPackage { dependencies }) -> dependencies `Assert.shouldEqual` manifestDependencies
-        Left err -> Assert.fail $ "Failed to convert historical fixture with manifest dependencies: " <> err
+      let Manifest manifest = undefinedManifest
+      let canonicalDescription = Utils.fromRight "description" $ LimitedString.parse "Canonical package metadata"
+      let manifestDependencies = Map.singleton (packageName "prelude") (range ">=4.0.0 <5.0.0")
+      let
+        authoritativeManifest = Manifest $ manifest
+          { dependencies = manifestDependencies
+          , description = Just canonicalDescription
+          , license = Utils.fromRight "license" $ License.parse "BSD-3-Clause"
+          , location = Git { url: "https://example.com/undefined.git", subdir: Nothing }
+          , ref = "release-1.0.2"
+          }
+      case Convert.fromLegacyPackage (conversionInput { manifest = authoritativeManifest }) legacy of
+        Right (DocPackage convertedPackage) -> do
+          convertedPackage.dependencies `Assert.shouldEqual` Map.singleton (packageName "prelude") (RawRange ">=4.0.0 <5.0.0")
+          convertedPackage.description `Assert.shouldEqual` Just "Canonical package metadata"
+          convertedPackage.license `Assert.shouldEqual` (Utils.fromRight "license" $ License.parse "BSD-3-Clause")
+          convertedPackage.location `Assert.shouldEqual` Git { url: "https://example.com/undefined.git", subdir: Nothing }
+          convertedPackage.locationRef `Assert.shouldEqual` Just "release-1.0.2"
+        Left err -> Assert.fail $ "Failed to convert historical fixture with authoritative manifest: " <> err
+      let mismatchedManifest = Manifest $ manifest { name = packageName "other" }
+      case Convert.fromLegacyPackage (conversionInput { manifest = mismatchedManifest }) legacy of
+        Left err -> shouldContainString err "does not match manifest package other"
+        Right _ -> Assert.fail "Conversion unexpectedly accepted a mismatched manifest"
+      let mismatchedVersion = Manifest $ manifest { version = version "2.0.0" }
+      case Convert.fromLegacyPackage (conversionInput { manifest = mismatchedVersion }) legacy of
+        Left err -> shouldContainString err "does not match manifest version 2.0.0"
+        Right _ -> Assert.fail "Conversion unexpectedly accepted a mismatched manifest version"
       case Convert.fromLegacyPackage (conversionInput { sourcePaths = Map.empty }) legacy of
         Left err -> shouldContainString err "Missing package-relative source path for module Undefined"
         Right _ -> Assert.fail "Conversion unexpectedly accepted a missing module source path"
@@ -128,7 +155,13 @@ main = runSpecAndExitProcess [ consoleReporter ] do
         let reversed = Utils.fromRight "Reversed generation failed" $ Generate.generatePackage generationInput { modules = Array.reverse generationInput.modules }
         Codec.encode Docgen.Codec.docPackage generated `Assert.shouldEqual` Codec.encode Docgen.Codec.docPackage reversed
         case generated of
-          DocPackage { modules: [ DocModule { name: ModuleName "A", reexports: [ DocReexport { moduleName: ModuleName "B", declarations: [ DocDeclaration { sourceSpan: Just (SourceSpan { path: "src/B.purs" }) } ] } ] } ], resolvedModulePackages } -> do
+          DocPackage { dependencies, description, location, locationRef, modules: [ DocModule { name: ModuleName "A", reexports: [ DocReexport { moduleName: ModuleName "B", declarations: [ DocDeclaration { sourceSpan: Just (SourceSpan { path: "src/B.purs" }) } ] } ] } ], name, resolvedModulePackages, version: generatedVersion } -> do
+            dependencies `Assert.shouldEqual` Map.singleton dependency (RawRange ">=2.0.0 <3.0.0")
+            description `Assert.shouldEqual` Just "Example"
+            location `Assert.shouldEqual` Git { url: "https://example.com/repo.git", subdir: Just "packages/example" }
+            locationRef `Assert.shouldEqual` Just "v1.0.0"
+            name `Assert.shouldEqual` packageName "example"
+            generatedVersion `Assert.shouldEqual` version "1.0.0"
             Map.lookup (ModuleName "A") resolvedModulePackages `Assert.shouldEqual` Just (packageName "example")
             Map.lookup (ModuleName "B") resolvedModulePackages `Assert.shouldEqual` Just dependency
           _ -> Assert.fail "Generated package did not preserve deterministic ownership, reexports, and source paths"
@@ -138,7 +171,7 @@ main = runSpecAndExitProcess [ consoleReporter ] do
         case Generate.generatePackage duplicate of
           Left (DuplicateModule (ModuleName "B")) -> pure unit
           _ -> Assert.fail "Generation unexpectedly accepted a duplicate module"
-        let absolute = generationInput { modules = map (\moduleInput -> if moduleInput.package == generationInput.name then moduleInput { sourcePath = "/tmp/A.purs" } else moduleInput) generationInput.modules }
+        let absolute = generationInput { modules = map (\moduleInput -> if moduleInput.package == packageName "example" then moduleInput { sourcePath = "/tmp/A.purs" } else moduleInput) generationInput.modules }
         case Generate.generatePackage absolute of
           Left (InvalidSourcePath (ModuleName "A") "/tmp/A.purs") -> pure unit
           _ -> Assert.fail "Generation unexpectedly accepted an absolute source path"
@@ -438,11 +471,7 @@ legacyValue title signature span = L.Declaration
 generationInput :: Generate.PackageInput
 generationInput =
   { compilerVersion: version "0.15.15"
-  , dependencies: Map.singleton dependency (RawRange ">=2.0.0 <3.0.0")
-  , description: Just "Example"
-  , license: Utils.fromRight "license" $ License.parse "BSD-3-Clause"
-  , location: Git { url: "https://example.com/repo.git", subdir: Just "packages/example" }
-  , locationRef: Nothing
+  , manifest: generationManifest
   , modules:
       [ { docs: legacyModuleWithDeclaration "B" (legacyValue "value" (L.TypeVar "a") (Just sourceSpan))
         , package: dependency
@@ -455,13 +484,39 @@ generationInput =
         , sourcePath: "custom/A.purs"
         }
       ]
-  , name: packageName "example"
   , readme: Just $ Readme { content: "# Example", extension: Just "md" }
   , resolvedDependencies: Map.singleton dependency (version "2.3.4")
   , sourceArtifact: sourceArtifactFixture
+  }
+
+generationManifest :: Manifest
+generationManifest = Manifest
+  { dependencies: Map.singleton dependency (range ">=2.0.0 <3.0.0")
+  , description: Just $ Utils.fromRight "description" $ LimitedString.parse "Example"
+  , excludeFiles: Nothing
+  , includeFiles: Nothing
+  , license: Utils.fromRight "license" $ License.parse "BSD-3-Clause"
+  , location: Git { url: "https://example.com/repo.git", subdir: Just "packages/example" }
+  , name: packageName "example"
+  , owners: Nothing
+  , ref: "v1.0.0"
   , version: version "1.0.0"
   }
 
+undefinedManifest :: Manifest
+undefinedManifest = Manifest
+  { dependencies: Map.empty
+  , description: Just $ Utils.fromRight "description" $ LimitedString.parse "Package containing the undefined value."
+  , excludeFiles: Nothing
+  , includeFiles: Nothing
+  , license: Utils.fromRight "license" $ License.parse "MIT"
+  , location: GitHub { owner: "bklaric", repo: "purescript-undefined", subdir: Nothing }
+  , name: packageName "undefined"
+  , owners: Nothing
+  , ref: "v1.0.2"
+  , version: version "1.0.2"
+  }
+
 package :: DocPackage
 package = DocPackage
   { schemaVersion
@@ -489,6 +544,9 @@ packageName = Utils.fromRight "package name" <<< PackageName.parse
 version :: String -> Version
 version = Utils.fromRight "version" <<< Version.parse
 
+range :: String -> Range
+range = Utils.fromRight "range" <<< Range.parse
+
 sourceArtifactFixture :: SourceArtifact
 sourceArtifactFixture = SourceArtifact
   { bytes: 42.0