[FLORA-794] Replace the usage of datalog with Haskell for category normalisation (#822)

feat: replaced use of datalog with haskell for category normalisation

---------

Co-authored-by: Théophile Choutri <theophile@choutri.eu>

Author: mau5mat

CONTRIBUTING.md

@@ -18,7 +18,6 @@ The following Haskell command-line tools will have to be installed:
 
 (Some of the above packages have incompatible dependencies, so don't try to install them all at once with `cabal install`)
 
-* [Soufflé datalog engine v2.2](https://github.com/souffle-lang/souffle/releases/tag/2.2): The datalog engine for package classification
 * `libsodium-1.0.18`: The system library that powers most of the cryptography happening in flora
 * `yarn`: The tool that handles the JavaScript code bases
 * `esbuild`: The tool that handles asset bundling

Dockerfile

@@ -1,5 +1,4 @@
-# this is a pinned ubuntu:22.04 (newer versions have incomptible
-# library versions for souffle)
+# this is a pinned ubuntu:22.04
 FROM ubuntu@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21
 
 ARG GID=1000
@@ -19,7 +18,7 @@ ARG POSTGRESQL_MIGRATION_VERSION=0.2.1.8
 USER "root"
 ARG USER="local"
 RUN groupadd -g "$GID" -o "$USER" \
-    && useradd -r -u "$UID" -g "$GID" -m -s /bin/zsh "$USER"
+  && useradd -r -u "$UID" -g "$GID" -m -s /bin/zsh "$USER"
 
 # We create the folder explicitly so that we can give nonprivileged user the appropriate access
 RUN mkdir /flora-server
@@ -30,7 +29,20 @@ RUN chown -R $USER:$USER /home/$USER/.cabal
 WORKDIR /flora-server
 
 RUN apt update && \
-    apt install -y build-essential curl libffi-dev libffi8 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 git libsodium-dev pkg-config
+  apt install -y  \
+    build-essential \
+    curl \
+    libffi-dev \
+    libffi8 \
+    libgmp-dev \
+    libgmp10 \
+    libncurses-dev \
+    libncurses5 \
+    libtinfo5 \
+    git \
+    libsodium-dev \
+    pkg-config \
+    zlib1g-dev
 
 # install dependencies (pg_config, postgresql-client, yarn)
 ENV BOOTSTRAP_HASKELL_NONINTERACTIVE="YES"
@@ -48,18 +60,12 @@ RUN git config --global --add safe.directory "*"
 RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
 
 RUN ghcup install hls $HLS_VERSION \
-    && ghcup install ghc $GHC_VERSION \
-    && ghcup set ghc $GHC_VERSION \
-    && ghcup install cabal $CABAL_VERSION
+  && ghcup install ghc $GHC_VERSION \
+  && ghcup set ghc $GHC_VERSION \
+  && ghcup install cabal $CABAL_VERSION
 
 USER ${USER}
 
-# install soufflé
-USER "root"
-RUN wget --content-disposition https://github.com/souffle-lang/souffle/releases/download/2.2/x86_64-ubuntu-2004-souffle-2.2-Linux.deb
-RUN apt install -f -y ./x86_64-ubuntu-2004-souffle-2.2-Linux.deb
-USER ${USER}
-
 RUN echo $PATH
 
 # install Haskell tooling (note that for cabal, it's probably better
@@ -83,10 +89,8 @@ COPY --chown=${USER} scripts/.zshrc /home/$USER/.zshrc
 COPY --chown=${USER} cabal.project flora.cabal cabal.project.freeze ./
 RUN cabal build --only-dependencies -j
 
-# compile Souffle source files
+# copy makefile
 COPY --chown=${USER} Makefile ./
-COPY --chown=${USER} cbits ./cbits
-RUN make souffle
 
 # copy and build the assets
 COPY --chown=${USER} assets ./assets

Makefile

@@ -4,10 +4,10 @@ init: ## Set up git hooks properly - needs calling once when cloning the repo
 start: ## Start flora-server
 	@cabal run exe:flora-server
 
-build: soufflé ## Build the server
+build: ## Build the server
 	@cabal build
 
-build-release: soufflé ## Build the server for production
+build-release: ## Build the server for production
 	@cabal freeze --project-file cabal.project.release
 	@cabal build --project-file cabal.project.release
 
@@ -91,21 +91,21 @@ db-test-provision-packages: ## Load development data in the database
 import-from-hackage: ## Imports every cabal file from the ./index-01 directory
 	@cabal run -- flora-cli import-packages ./01-index
 
-repl: soufflé ## Start a cabal REPL
+repl: ## Start a cabal REPL
 	@cabal repl lib:flora
 
 ghci: repl ## Start a cabal REPL (alias for `make repl`)
 
-watch: soufflé ## Load the main library and reload on file change
+watch: ## Load the main library and reload on file change
 	@ghcid --target flora-server -l
 
-test: build soufflé ## Run the test suite
+test:  ## Run the test suite
 	./scripts/run-tests.sh
 
-watch-test: soufflé ## Load the tests in ghcid and reload them on file change
+watch-test: ## Load the tests in ghcid and reload them on file change
 	./scripts/run-tests.sh --watch
 
-watch-server: soufflé ## Start flora-server in ghcid
+watch-server: ## Start flora-server in ghcid
 	@ghcid --target=flora-server --restart="src" --test 'FloraWeb.Server.runFlora'
 
 lint-hs: ## Run the code linter (HLint)
@@ -145,9 +145,6 @@ docker-enter: ## Enter the docker environment
 start-tmux: ## Start a Tmux session with hot code reloading
 	./scripts/start-tmux.sh
 
-soufflé: ## Generate C++ files from the Soufflé Datalog definitions
-	cd cbits ; souffle -g categorise.{cpp,dl}
-
 tags: ## Generate ctags for the project with `ghc-tags`
 	@ghc-tags -c
 

cabal.project

@@ -15,7 +15,6 @@ allow-newer: hashable:filepath
            , qrcode-juicypixels:bytestring
            , qrcode-juicypixels:text
            , servant-lucid:text
-           , souffle-haskell:text
            , tasty-test-reporter:ansi-terminal
            , tasty-test-reporter:tasty
            , type-errors-pretty:base
@@ -88,11 +87,6 @@ source-repository-package
     location: https://github.com/saurabhnanda/odd-jobs
     tag: a159d7a17913725a0cb2a2251fa0b812acd3d160
 
-source-repository-package
-    type: git
-    location: https://github.com/luc-tielen/souffle-haskell
-    tag: 268a11283ca9293b5eacabf7a0b79d9368232478
-
 source-repository-package
   type: git
   location: https://github.com/goodlyrottenapple/tasty-test-reporter

cabal.project.freeze

@@ -395,7 +395,6 @@ constraints: any.Cabal ==3.10.3.0,
              any.some ==1.0.6,
              some +newtype-unsafe,
              any.sop-core ==0.5.0.2,
-             any.souffle-haskell ==4.0.0,
              any.split ==0.2.5,
              any.splitmix ==0.1.1,
              splitmix -optimised-mixer,

cbits/categorise.dl

@@ -0,0 +1,8 @@
+synopsis: Replace the usage of datalog with Haskell for category normalisation
+
+prs: #822
+issues: #794
+
+description: {
+  Replaces Souffle as much as possible for category normalisation
+}

changelog.d/794

@@ -67,9 +67,6 @@ common common-rts-options
 library
   import:          common-extensions
   import:          common-ghc-options
-  extra-libraries: stdc++
-  cxx-options:     -std=c++17 -Wall -D__EMBEDDED_SOUFFLE__
-  cxx-sources:     cbits/categorise.cpp
   hs-source-dirs:  ./src/core ./src/datatypes
 
   -- cabal-fmt: expand src/core
@@ -99,7 +96,6 @@ library
     Flora.Environment.Config
     Flora.Environment.Env
     Flora.Import.Categories
-    Flora.Import.Categories.Tuning
     Flora.Import.Package
     Flora.Import.Package.Bulk
     Flora.Import.Package.Types
@@ -148,6 +144,7 @@ library
     Flora.Model.User.Query
     Flora.Model.User.Update
     Flora.Monitoring
+    Flora.Normalise
     Flora.QRCode
     Flora.Tracing
     JSON
@@ -215,7 +212,6 @@ library
     , servant-prometheus
     , servant-server
     , slugify
-    , souffle-haskell        ==4.0.0
     , streamly
     , streamly-core
     , tar

flora.cabal

@@ -1,22 +1,22 @@
 module Flora.Import.Categories where
 
 import Control.Monad.IO.Class
+import Data.Text (Text)
 import Data.Text.IO qualified as T
 import Effectful
 import Effectful.PostgreSQL.Transact.Effect
 
-import Flora.Import.Categories.Tuning as Tuning
 import Flora.Model.Category.Types (Category, mkCategory, mkCategoryId)
 import Flora.Model.Category.Update (insertCategory)
+import Flora.Normalise
 
 importCategories :: (DB :> es, IOE :> es) => Eff es ()
 importCategories = do
-  liftIO $ T.putStrLn "Sourcing categories from Datalog"
-  canonicalCategories <- liftIO Tuning.sourceCategories
-  categories <- mapM fromCanonical canonicalCategories
+  liftIO $ T.putStrLn "Sourcing categories"
+  categories <- mapM fromCanonical floraCategories
   mapM_ insertCategory categories
 
-fromCanonical :: IOE :> es => CanonicalCategory -> Eff es Category
-fromCanonical (CanonicalCategory slug name synopsis) = do
+fromCanonical :: IOE :> es => (Text, Text, Text) -> Eff es Category
+fromCanonical (slug, name, synopsis) = do
   categoryId <- liftIO mkCategoryId
   pure $ mkCategory categoryId name (Just slug) synopsis

src/core/Flora/Import/Categories.hs

@@ -39,7 +39,7 @@ import Data.ByteString qualified as BS
 import Data.List.NonEmpty qualified as NE
 import Data.Map (Map)
 import Data.Map.Strict qualified as Map
-import Data.Maybe
+import Data.Maybe as Maybe
 import Data.Set (Set)
 import Data.Set qualified as Set
 import Data.Text (Text, pack)
@@ -77,7 +77,7 @@ import Distribution.Types.PackageDescription ()
 import Distribution.Types.TestSuite
 import Distribution.Types.Version (Version)
 import Distribution.Types.VersionRange (VersionRange, withinRange)
-import Distribution.Utils.ShortText qualified as Cabal
+import Distribution.Utils.ShortText (fromShortText)
 import Distribution.Version qualified as Version
 import Effectful
 import Effectful.Log (Log)
@@ -94,9 +94,10 @@ import System.FilePath qualified as FilePath
 
 import Effectful.Poolboy (Poolboy)
 import Effectful.Poolboy qualified as Poolboy
-import Flora.Import.Categories.Tuning qualified as Tuning
 import Flora.Import.Package.Types
 import Flora.Import.Types
+import Flora.Model.Category.Query as Query
+import Flora.Model.Category.Types
 import Flora.Model.Category.Update qualified as Update
 import Flora.Model.Component.Types as Component
 import Flora.Model.Package.Orphans ()
@@ -112,6 +113,7 @@ import Flora.Model.Requirement
   , deterministicRequirementId
   )
 import Flora.Model.User
+import Flora.Normalise
 
 coreLibraries :: Set PackageName
 coreLibraries =
@@ -300,7 +302,7 @@ persistImportOutput (ImportOutput package categories release components) = State
       [ "package_name" .= packageName
       , "version" .= display release.version
       ]
-  persistPackage
+  persistPackage package.packageId
   if Set.member (package.namespace, package.name, release.version) packageCache
     then do
       Log.logInfo "Release already present" $
@@ -317,12 +319,16 @@ persistImportOutput (ImportOutput package categories release components) = State
   where
     parallelRun :: Foldable t => (a -> Eff es ()) -> t a -> Eff es ()
     parallelRun f xs = forM_ xs (Poolboy.enqueue . f)
+
     packageName = display package.namespace <> "/" <> display package.name
-    persistPackage = do
-      let packageId = package.packageId
-      Update.upsertPackage package
-      forM_ categories (\case Tuning.NormalisedPackageCategory cat -> Update.addToCategoryByName packageId cat)
 
+    persistPackage :: PackageId -> Eff es ()
+    persistPackage packageId = do
+      Update.upsertPackage package
+      categoriesByName <- catMaybes <$> traverse Query.getCategoryByName categories
+      forM_
+        categoriesByName
+        (\c -> Update.addToCategoryByName packageId c.name)
     persistComponent :: (PackageComponent, List ImportDependency) -> Eff es ()
     persistComponent (packageComponent, deps) = do
       Log.logInfo
@@ -398,9 +404,9 @@ extractPackageDataFromCabal userId repository@(repositoryName, repositoryPackage
   let releaseId = deterministicReleaseId packageId packageVersion
   timestamp <- Time.currentTime
   let sourceRepos = getRepoURL packageName packageDesc.sourceRepos
-  let rawCategoryField = packageDesc ^. #category % to Cabal.fromShortText % to Text.pack
-  let categoryList = fmap (Tuning.UserPackageCategory . Text.stripStart . Text.stripEnd) (Text.splitOn "," rawCategoryField)
-  categories <- liftIO $ Tuning.normalisedCategories <$> Tuning.normalise categoryList
+  let rawCategoryField = packageDesc ^. #category % to fromShortText % to Text.pack
+  let categoryList = fmap (Text.stripStart . Text.stripEnd) (Text.splitOn "," rawCategoryField)
+  let categories = Maybe.mapMaybe normaliseCategory categoryList
   let package =
         Package
           { packageId
@@ -670,7 +676,6 @@ getVersions supportedCompilers =
   foldMap
     (\version -> Vector.foldMap (checkVersion version) supportedCompilers)
     versionList
-
 checkVersion :: Version -> VersionRange -> Vector Version
 checkVersion version versionRange =
   if version `withinRange` versionRange

src/core/Flora/Import/Categories/Tuning.hs

@@ -3,10 +3,10 @@ module Flora.Import.Package.Types where
 import Control.DeepSeq
 import Data.Aeson
 import Data.List.NonEmpty (NonEmpty)
+import Data.Text (Text)
 import GHC.Generics
 import GHC.List (List)
 
-import Flora.Import.Categories.Tuning qualified as Tuning
 import Flora.Model.Component.Types
 import Flora.Model.Package.Types
 import Flora.Model.Release.Types
@@ -28,7 +28,7 @@ type DependentName = (Namespace, PackageName)
 
 data ImportOutput = ImportOutput
   { package :: Package
-  , categories :: [Tuning.NormalisedPackageCategory]
+  , categories :: [Text]
   , release :: Release
   , components :: NonEmpty (PackageComponent, List ImportDependency)
   }

src/core/Flora/Import/Package.hs

@@ -13,7 +13,6 @@ import Database.PostgreSQL.Simple
 import Database.PostgreSQL.Simple.FromField (FromField)
 import Database.PostgreSQL.Simple.ToField (ToField)
 import GHC.Generics
-import Language.Souffle.Interpreted qualified as Souffle
 import Servant
 import Text.Slugify
 
@@ -27,7 +26,6 @@ newtype CategoryId = CategoryId {getCategoryId :: UUID}
 
 newtype CategoryName = CategoryName {getCategoryName :: Text}
   deriving stock (Generic, Show)
-  deriving anyclass (Souffle.Marshal)
   deriving
     (Eq, NFData, Ord)
     via Text

src/core/Flora/Import/Package/Types.hs

@@ -40,7 +40,6 @@ import Deriving.Aeson
 import Distribution.Pretty (Pretty (..))
 import Distribution.SPDX.License qualified as SPDX
 import Distribution.Types.Version (Version)
-import Language.Souffle.Interpreted qualified as Souffle
 import Lucid
 import Optics.Core hiding (element)
 import Servant (FromHttpApiData (..))
@@ -71,7 +70,6 @@ deterministicPackageId (Namespace ns) (PackageName name) =
 
 newtype PackageName = PackageName Text
   deriving stock (Generic, Show)
-  deriving anyclass (Souffle.Marshal)
   deriving
     (Eq, FromField, FromJSON, NFData, Ord, ToField, ToHtml, ToHttpApiData, ToJSON)
     via Text

src/core/Flora/Model/Category/Types.hs

@@ -0,0 +1,174 @@
+module Flora.Normalise where
+
+import Data.List qualified as List
+import Data.Maybe (isJust)
+import Data.Text (Text)
+
+normaliseCategory :: Text -> Maybe Text
+normaliseCategory string =
+  if isJust $ List.find (\(_, name, _) -> name == string) floraCategories
+    then Just string
+    else case string of
+      "Algorithm" -> Just "Algorithms"
+      "Crypto" -> Just "Cryptography"
+      "CLI" -> Just "CLI & TUI Development"
+      "TUI" -> Just "CLI & TUI Development"
+      "Command Line" -> Just "CLI & TUI Development"
+      "CommandLine" -> Just "CLI & TUI Development"
+      "Numeric" -> Just "Mathematics"
+      "Numerical" -> Just "Mathematics"
+      "Numerics" -> Just "Mathematics"
+      "Arithmetic" -> Just "Mathematics"
+      "Number Theory" -> Just "Mathematics"
+      "Math" -> Just "Mathematics"
+      "Mathematics" -> Just "Mathematics"
+      "mathematics" -> Just "Mathematics"
+      "Maths" -> Just "Mathematics"
+      "Algebra" -> Just "Mathematics"
+      "Graph" -> Just "Mathematics"
+      "Graphs" -> Just "Mathematics"
+      "Geometry" -> Just "Mathematics"
+      "Tropical Geometry" -> Just "Mathematics"
+      "Parser Builder" -> Just "Parsers"
+      "Parser Combinators" -> Just "Parsers"
+      "Parser" -> Just "Parsers"
+      "ParserCombinators" -> Just "Parsers"
+      "Parsers" -> Just "Parsers"
+      "Parsing" -> Just "Parsers"
+      "Parsing Text" -> Just "Parsers"
+      "Network" -> Just "Network Development"
+      "Data Network" -> Just "Network Development"
+      "Network APIs" -> Just "Network Development"
+      "Network Control" -> Just "Network Development"
+      "NetworkAPI" -> Just "Network Development"
+      "NetworkAPIs" -> Just "Network Development"
+      "Networking" -> Just "Network Development"
+      "Web" -> Just "Web Development"
+      "Yesod" -> Just "Web Development"
+      "Javascript" -> Just "Web Development"
+      "OpenAPI" -> Just "Web Development"
+      "Snap" -> Just "Web Development"
+      "Servant" -> Just "Web Development"
+      "Servant Web" -> Just "Web Development"
+      "Web development" -> Just "Web Development"
+      "Happstack" -> Just "Web Development"
+      "Semantic Web" -> Just "Web Development"
+      "Optics" -> Just "Lenses"
+      "Lens" -> Just "Lenses"
+      "Conduit" -> Just "Streaming"
+      "Streamly" -> Just "Streaming"
+      "Pipes" -> Just "Streaming"
+      "Monad" -> Just "Monads"
+      "MonadIO" -> Just "Monads"
+      "Transformers" -> Just "Monads"
+      "Monad Transformers" -> Just "Monads"
+      "Mtl" -> Just "Monads"
+      "User interfaces" -> Just "GUI"
+      "User interface" -> Just "GUI"
+      "UserInterface" -> Just "GUI"
+      "UI" -> Just "GUI"
+      "User Interfaces" -> Just "GUI"
+      "graphics" -> Just "Graphics"
+      "Code Generation" -> Just "FFI"
+      "Foreign binding" -> Just "FFI"
+      "Elm" -> Just "FFI"
+      "TypeScript" -> Just "FFI"
+      "Java" -> Just "FFI"
+      "JVM" -> Just "FFI"
+      "Jvm" -> Just "FFI"
+      "Erlang" -> Just "FFI"
+      "PHP" -> Just "FFI"
+      "Foreign" -> Just "FFI"
+      "Types" -> Just "Type System"
+      "Validity" -> Just "Testing"
+      "QuickCheck" -> Just "Testing"
+      "Test" -> Just "Testing"
+      "Sound" -> Just "Audio"
+      "Algorithmic Music Composition" -> Just "Audio"
+      "Automatic Music Generation" -> Just "Audio"
+      "Music" -> Just "Audio"
+      "Zip" -> Just "Compression"
+      "ZLib" -> Just "Compression"
+      "Tar" -> Just "Compression"
+      "Cloud" -> Just "Cloud Computing"
+      "Google" -> Just "Cloud Computing"
+      "AWS" -> Just "Cloud Computing"
+      "Compilers/Interpreters" -> Just "Compilers and Interpreters"
+      "Interpreters" -> Just "Compilers and Interpreters"
+      "Compiler" -> Just "Compilers and Interpreters"
+      "DSL" -> Just "Compilers and Interpreters"
+      "Database" -> Just "Databases"
+      "PostgreSQL" -> Just "Databases"
+      "NLP" -> Just "Natural Language Processing"
+      "Japanese Natural Language Processing" -> Just "Natural Language Processing"
+      "Natural Language" -> Just "Natural Language Processing"
+      "Natural Language Processing" -> Just "Natural Language Processing"
+      "Stemming" -> Just "Natural Language Processing"
+      "Natural-language-processing" -> Just "Natural Language Processing"
+      "Containers" -> Just "Data Structures"
+      "Game" -> Just "Game Development"
+      "Game Engine" -> Just "Game Development"
+      "Concurrent" -> Just "Concurrency"
+      "Parallel" -> Just "Parallelism"
+      "Distributed Computing" -> Just "Distributed Systems & Computation"
+      "Filesystem" -> Just "Systems Programming"
+      "system" -> Just "Systems Programming"
+      "System" -> Just "Systems Programming"
+      "SYstem" -> Just "Systems Programming"
+      "Embedded" -> Just "Systems Programming"
+      "Distribution" -> Just "Package Distribution"
+      "Trace" -> Just "Profiling"
+      "Debug" -> Just "Profiling"
+      "Debugging" -> Just "Profiling"
+      "OpenTelemetry" -> Just "Telemetry"
+      "Metrics" -> Just "Telemetry"
+      "Regex" -> Just "Text"
+      _ -> Nothing
+
+floraCategories :: [(Text, Text, Text)]
+floraCategories =
+  [ ("algorithms", "Algorithms", "Algorithms implemented in Haskell, like sorting, searching")
+  , ("audio", "Audio", "Process digital signal, make music")
+  , ("bioinformatics", "Bioinformatics", "Methods and software for the analysis of biological data")
+  , ("cloud", "Cloud Computing", "Bindings to Cloud Computing platforms")
+  , ("command-line", "CLI & TUI tooling", "Libraries to develop command-line interfaces")
+  , ("compilers-interpreters", "Compilers and Interpreters", "Tooling to create compilers and interpreters")
+  , ("compression", "Data compression", "Reducing the size of things")
+  , ("concurrency", "Concurrency", "Concurrent programming techniques and tools")
+  , ("cryptography", "Cryptography", "Algorithms for encrypting and hashing data")
+  , ("data-structures", "Data Structures", "Data structures, whether purely functional or mutable")
+  , ("databases", "Databases", "Database drivers and interfaces")
+  , ("development", "Development", "Development helpers, integration with other languages")
+  , ("distributed", "Distributed Systems & Computation", "Tooling and techniques for writing distributed systems")
+  , ("distribution", "Package Distribution", "Building, Packaging and Distributing software in Haskell")
+  , ("ffi", "FFI", "Working with other languages and generating bindings")
+  , ("frp", "FRP", "Functional Reactive Programming")
+  , ("game-dev", "Game Development", "Libraries used for game development")
+  , ("generics", "Generics", "Working with Haskell's Generics mechanism")
+  , ("graphics", "Graphics", "Programming the system's rendering APIs")
+  , ("gui", "GUI", "Creating graphical user interfaces")
+  , ("hardware", "Hardware", "Digital circuit description and hardware interfacing")
+  , ("json", "JSON", "Parsing, producing and manipulating JSON")
+  , ("language", "Language", "Interfacing with other programming languages from Haskell")
+  , ("lenses", "Lenses", "Functional references such as Lenses, Folds and Traversals")
+  , ("maths", "Mathematics", "Numerical and Mathematical packages")
+  , ("monads", "Monads", "Effectful sequential computations")
+  , ("network", "Network Development", "Connection pools, DNS, HTTP, API clients and network protocols")
+  , ("nlp", "Natural Language Processing", "Tooling to work with natural languages")
+  , ("parallelism", "Parallelism", "Parallel programming")
+  , ("parser-implementations", "Parser Implementations", "Parsing data formats")
+  , ("parsers", "Parsers", "Libraries to ingest and parse data")
+  , ("parsing", "Parsing", "Parser generators, combinators and tools to help with parsing")
+  , ("physics", "Physics", "The study of matter, its consituents, motion, and behaviour")
+  , ("prelude", "Prelude", "Libraries that provide default imports")
+  , ("profiling", "Profiling", "Measure the behaviour of your programs")
+  , ("streaming", "Streaming", "Data streaming for continuous processing")
+  , ("system", "Systems Programming", "Programming and communicating with the Operating System")
+  , ("telemetry", "Telemetry", "Systems Observability")
+  , ("template-haskell", "Template Haskell", "Metaprogramming with Template Haskell")
+  , ("testing", "Testing", "Test frameworks")
+  , ("text", "Text", "Working with textual data and algorithms")
+  , ("type-system", "Type System", "Enhancing the Haskell type system")
+  , ("web", "Web Development", "Programming for the web")
+  , ("xml", "XML", "Libraries to consume and produce XML documents")
+  ]

src/core/Flora/Model/Package/Types.hs

@@ -1,28 +1,36 @@
 module Flora.CategorySpec where
 
-import Control.Monad.IO.Class
+import Data.Set qualified as Set
 import Test.Tasty
 
-import Flora.Import.Categories.Tuning as Tuning
+import Flora.Normalise
 import Flora.TestUtils
 
 spec :: TestEff TestTree
 spec =
   testThese
-    "category tuning"
-    [ testThis "Test that the category unification algorithm works" testUnificationAlgorithm
+    "Category Normalisation"
+    [ testThis "Normalisation of Mathematics categories" testNormalisationOfMathematicsCategories
     ]
 
-testUnificationAlgorithm :: TestEff ()
-testUnificationAlgorithm = do
-  liftIO (Tuning.normalise [UserPackageCategory "Algorithm"])
-    >>= assertEqual (Results [NormalisedPackageCategory "Algorithms"] [])
-
-  liftIO (Tuning.normalise [UserPackageCategory "Crypto"])
-    >>= assertEqual (Results [NormalisedPackageCategory "Cryptography"] [])
-
-  liftIO (Tuning.normalise [UserPackageCategory "CLI", UserPackageCategory "TUI"])
-    >>= assertEqual (Results [NormalisedPackageCategory "CLI & TUI Development"] [])
-
-  liftIO (Tuning.normalise [UserPackageCategory "Numeric", UserPackageCategory "Parser Builder"])
-    >>= assertEqual (Results [NormalisedPackageCategory "Mathematics", NormalisedPackageCategory "Parsers"] [])
+testNormalisationOfMathematicsCategories :: TestEff ()
+testNormalisationOfMathematicsCategories = do
+  let mathematicsExceptions =
+        Set.fromList
+          [ "Numeric"
+          , "Numerical"
+          , "Numerics"
+          , "Arithmetic"
+          , "Number Theory"
+          , "Math"
+          , "Mathematics"
+          , "mathematics"
+          , "Maths"
+          , "Algebra"
+          , "Graph"
+          , "Graphs"
+          , "Geometry"
+          ]
+  assertEqual
+    (Set.singleton (Just "Mathematics"))
+    (Set.map normaliseCategory mathematicsExceptions)

src/core/Flora/Normalise.hs

@@ -17,7 +17,6 @@ import Optics.Core
 import Test.Tasty
 
 import Flora.Import.Package
-import Flora.Model.Category (Category (..))
 import Flora.Model.Category.Query qualified as Query
 import Flora.Model.Component.Types
 import Flora.Model.Package
@@ -35,8 +34,6 @@ spec =
     "package tests"
     [ testThis "Check Cabal dependencies" testCabalDeps
     , testThis "Insert containers and its dependencies" testInsertContainers
-    , testThis "@haskell/base belongs to the \"Prelude\" category" testThatBaseisInPreludeCategory
-    , testThis "@hackage/semigroups belongs to appropriate categories" testThatSemigroupsIsInMathematicsAndDataStructures
     , testThis "The \"haskell\" namespace has the correct number of packages" testCorrectNumberInHaskellNamespace
     , testThis "Packages are not shown as their own dependent" testNoSelfDependent
     , testThis "Searching for `text` returns expected results by namespace/package name" testSearchResultText
@@ -123,12 +120,6 @@ testThatBaseisInPreludeCategory = do
   result <- Query.getPackagesFromCategorySlug "prelude"
   assertBool $ Set.member (PackageName "base") (Set.fromList $ Vector.toList $ fmap (view #name) result)
 
-testThatSemigroupsIsInMathematicsAndDataStructures :: TestEff ()
-testThatSemigroupsIsInMathematicsAndDataStructures = do
-  semigroups <- fromJust <$> Query.getPackageByNamespaceAndName (Namespace "hackage") (PackageName "semigroups")
-  result <- Query.getPackageCategories semigroups.packageId
-  assertEqual (Set.fromList ["data-structures", "maths"]) (Set.fromList $ slug <$> Vector.toList result)
-
 testCorrectNumberInHaskellNamespace :: TestEff ()
 testCorrectNumberInHaskellNamespace = do
   results <- Query.getPackagesByNamespace (Namespace "haskell")

test/Flora/CategorySpec.hs

@@ -249,10 +249,10 @@ assertBool boolean = liftIO $ Test.assertBool "" boolean
 --  Usage:
 --
 --  >>> assertEqual expected actual
-assertEqual :: (Eq a, Show a) => a -> a -> TestEff ()
+assertEqual :: (Eq a, HasCallStack, Show a) => a -> a -> TestEff ()
 assertEqual expected actual = liftIO $ Test.assertEqual "" expected actual
 
-assertFailure :: MonadIO m => String -> m ()
+assertFailure :: (HasCallStack, MonadIO m) => String -> m ()
 assertFailure = liftIO . Test.assertFailure
 
 assertJust :: HasCallStack => Maybe a -> TestEff a

test/Flora/PackageSpec.hs

@@ -68,9 +68,6 @@ common common-rts-options
 library
   import:          common-extensions
   import:          common-ghc-options
-  extra-libraries: stdc++
-  cxx-options:     -std=c++17 -Wall -D__EMBEDDED_SOUFFLE__
-  cxx-sources:     cbits/categorise.cpp
   hs-source-dirs:  ./src/core ./src/orphans
 
   -- cabal-fmt: expand src/core
@@ -84,7 +81,6 @@ library
     Flora.Environment
     Flora.Environment.Config
     Flora.Import.Categories
-    Flora.Import.Categories.Tuning
     Flora.Import.Package
     Flora.Import.Package.Bulk
     Flora.Import.Package.Types
@@ -160,7 +156,6 @@ library
     , servant-lucid
     , servant-server
     , slugify
-    , souffle-haskell        ==3.5.1
     , streamly
     , streamly-core
     , tar