From 9ae512e1850f30237f24d3fd9976cc9ac05bc7cd Mon Sep 17 00:00:00 2001 From: Neil Mitchell Date: Sun, 20 Jun 2021 21:01:16 +0100 Subject: [PATCH 01/12] Run the update build twice --- misc/Upgrade.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/Upgrade.hs b/misc/Upgrade.hs index 3be0d212..b33808dc 100644 --- a/misc/Upgrade.hs +++ b/misc/Upgrade.hs @@ -41,6 +41,7 @@ main = do echo system_ "cabal v2-update" echo system_ "cabal v2-build -j1 --only-dependencies --upgrade-dependencies --force-reinstalls --ghc-options=\"+RTS -M1G\"" echo system_ "cabal v2-configure \"--ghc-options=-rtsopts -O2\"" + echo system_ "cabal v2-build -j1 --only-dependencies --upgrade-dependencies --force-reinstalls --ghc-options=\"+RTS -M1G\"" echo system_ "GHCRTS=-M1G cabal v2-install -j1 --installdir=bin" let exe = normalise "bin/hoogle" From d23196007965558ae911ba0c370b95efd9796ce3 Mon Sep 17 00:00:00 2001 From: Neil Mitchell Date: Sat, 3 Jul 2021 16:38:58 +0100 Subject: [PATCH 02/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 929f0372..daa1e54c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Hoogle [![Hackage version](https://img.shields.io/hackage/v/hoogle.svg?label=Hackage)](https://hackage.haskell.org/package/hoogle) [![Stackage version](https://www.stackage.org/package/hoogle/badge/nightly?label=Stackage)](https://www.stackage.org/package/hoogle) [![Build status](https://img.shields.io/github/workflow/status/ndmitchell/hoogle/ci.svg)](https://github.com/ndmitchell/hoogle/actions) +# Hoogle [![Hackage version](https://img.shields.io/hackage/v/hoogle.svg?label=Hackage)](https://hackage.haskell.org/package/hoogle) [![Stackage version](https://www.stackage.org/package/hoogle/badge/nightly?label=Stackage)](https://www.stackage.org/package/hoogle) [![Build status](https://img.shields.io/github/workflow/status/ndmitchell/hoogle/ci/master.svg)](https://github.com/ndmitchell/hoogle/actions) Hoogle is a Haskell API search engine, which allows you to search many standard Haskell libraries by either function name, or by approximate type signature. The online version can be found at https://hoogle.haskell.org/ and searches [Stackage](https://www.stackage.org/). From e83fb60749fd76ad4511c6008c5d1d9a85a485e2 Mon Sep 17 00:00:00 2001 From: Neil Mitchell Date: Sat, 3 Jul 2021 22:02:32 +0100 Subject: [PATCH 03/12] Delete .travis.yml --- .travis.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3eef49c2..00000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: minimal -env: -- GHCVER=8.2 -- GHCVER=8.4 -- GHCVER=8.6 -- GHCVER=8.8 -- GHCVER=8.10 -- GHCVER=head - -script: -- curl -sSL https://raw.github.com/ndmitchell/neil/master/travis.sh | sh From bd2919c22692b714535c1e0aae65f78dcd6080a5 Mon Sep 17 00:00:00 2001 From: Neil Mitchell Date: Sat, 3 Jul 2021 22:04:38 +0100 Subject: [PATCH 04/12] Delete appveyor.yml --- appveyor.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index d19dbdad..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,7 +0,0 @@ -build: off -cache: "c:\\sr -> appveyor.yml" - -test_script: -- ps: Invoke-Expression (Invoke-WebRequest 'https://raw.githubusercontent.com/ndmitchell/neil/master/appveyor.ps1') -- stack exec -- hoogle generate --insecure -- stack exec -- hoogle test From ba58af779fb2ee8a32043a1a6359ed12e4833d44 Mon Sep 17 00:00:00 2001 From: Peter Siska Date: Sun, 16 May 2021 20:28:29 +0200 Subject: [PATCH 05/12] Allow for JSONL output for search command Reason: in situation where one cannot use the shell postprocessing it is easier to output one entry per one line for which JSONL is suitable. --- src/Action/CmdLine.hs | 10 ++++++---- src/Action/Search.hs | 13 ++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Action/CmdLine.hs b/src/Action/CmdLine.hs index 948c775d..30266573 100644 --- a/src/Action/CmdLine.hs +++ b/src/Action/CmdLine.hs @@ -8,14 +8,14 @@ module Action.CmdLine( whenLoud, whenNormal ) where +import Data.List.Extra +import Data.Version +import General.Util +import Paths_hoogle (version) import System.Console.CmdArgs import System.Directory import System.Environment import System.FilePath -import Data.List.Extra -import Data.Version -import General.Util -import Paths_hoogle(version) data Language = Haskell | Frege deriving (Data,Typeable,Show,Eq,Enum,Bounded) @@ -23,6 +23,7 @@ data CmdLine = Search {color :: Maybe Bool ,json :: Bool + ,jsonl :: Bool ,link :: Bool ,numbers :: Bool ,info :: Bool @@ -111,6 +112,7 @@ cmdLineMode = cmdArgsMode $ modes [search_ &= auto,generate,server,replay,test] search_ = Search {color = def &= name "colour" &= help "Use colored output (requires ANSI terminal)" ,json = def &= name "json" &= help "Get result as JSON" + ,jsonl = def &= name "jsonl" &= help "Get result as JSONL (JSON Lines)" ,link = def &= help "Give URL's for each result" ,numbers = def &= help "Give counter for each result" ,info = def &= help "Give extended information about the first result" diff --git a/src/Action/Search.hs b/src/Action/Search.hs index 2f781205..5631220b 100644 --- a/src/Action/Search.hs +++ b/src/Action/Search.hs @@ -1,4 +1,5 @@ -{-# LANGUAGE LambdaCase, RecordWildCards, ScopedTypeVariables, TupleSections #-} +{-# LANGUAGE LambdaCase, MultiWayIf, RecordWildCards, ScopedTypeVariables, + TupleSections #-} module Action.Search (actionSearch, withSearch, search @@ -8,17 +9,17 @@ module Action.Search ) where import Control.DeepSeq -import Control.Monad.Extra import Control.Exception.Extra +import Control.Monad.Extra import qualified Data.Aeson as JSON +import qualified Data.ByteString.Lazy.Char8 as LBS import Data.Functor.Identity import Data.List.Extra -import Text.Blaze.Renderer.Utf8 -import qualified Data.ByteString.Lazy.Char8 as LBS import qualified Data.Map as Map import Data.Maybe import qualified Data.Set as Set import System.Directory +import Text.Blaze.Renderer.Utf8 import Action.CmdLine import General.Store @@ -49,7 +50,9 @@ actionSearch Search{..} = replicateM_ repeat_ $ -- deliberately reopen the datab putStr $ targetInfo $ head res else do let toShow = if numbers && not info then addCounter shown else shown - if json then LBS.putStrLn $ JSON.encode $ maybe id take count $ map unHTMLtargetItem res else putStr $ unlines toShow + if | json -> LBS.putStrLn $ JSON.encode $ maybe id take count $ map unHTMLtargetItem res + | jsonl -> mapM_ (LBS.putStrLn . JSON.encode) $ maybe id take count $ map unHTMLtargetItem res + | otherwise -> putStr $ unlines toShow when (hidden /= [] && not json) $ do whenNormal $ putStrLn $ "-- plus more results not shown, pass --count=" ++ show (count'+10) ++ " to see more" else do From 53a4c6755b4f159bcbc3042f8da855fd2169751e Mon Sep 17 00:00:00 2001 From: Neil Mitchell Date: Sun, 4 Jul 2021 20:56:52 +0100 Subject: [PATCH 06/12] Update CHANGES.txt --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 1ea802f4..5017d0bb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ Changelog for Hoogle (* = API change, @ = database format change) + #370, add --jsonl to produce JSON on separate lines 5.0.18.1, released 2020-12-27 #365, add flag to disable network tests 5.0.18, released 2020-07-19 From 37a201a114585c1f012312ab864b8a550fffcb9c Mon Sep 17 00:00:00 2001 From: Michael James Date: Sun, 11 Jul 2021 16:50:08 -0700 Subject: [PATCH 07/12] Specify a single .txt file --- src/Action/CmdLine.hs | 2 ++ src/Action/Generate.hs | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Action/CmdLine.hs b/src/Action/CmdLine.hs index 30266573..b57eeac0 100644 --- a/src/Action/CmdLine.hs +++ b/src/Action/CmdLine.hs @@ -37,6 +37,7 @@ data CmdLine | Generate {download :: Maybe Bool ,database :: FilePath + ,source :: Maybe FilePath ,insecure :: Bool ,include :: [String] ,count :: Maybe Int @@ -127,6 +128,7 @@ search_ = Search generate = Generate {download = def &= help "Download all files from the web" ,insecure = def &= help "Allow insecure HTTPS connections" + ,source = def &=help "Specify a textbase source" ,include = def &= args &= typ "PACKAGE" ,local_ = def &= opt "" &= help "Index local packages and link to local haddock docs" ,count = Nothing &= name "n" &= help "Maximum number of packages to index (defaults to all)" diff --git a/src/Action/Generate.hs b/src/Action/Generate.hs index 7d8c0e02..60a0660a 100644 --- a/src/Action/Generate.hs +++ b/src/Action/Generate.hs @@ -177,6 +177,18 @@ readHaskellGhcpkg timing settings = do in Map.map (\p -> p{packageTags = ts ++ packageTags p}) cbl pure (cbl, Map.keysSet cbl, source) +readHaskellHaddockFile :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) +readHaskellHaddockFile timing settings txt = do + let name = strPack $ take (length txt - (length ".txt")) txt + let source = + whenM (liftIO $ doesFileExist txt) $ do + src <- liftIO $ bstrReadFile txt + let url = "file://./" ++ txt + yield (name, url, lbstrFromChunks [src]) + pure (Map.empty, Set.singleton name, source) + + where docDir name Package{..} = name ++ "-" ++ strUnpack packageVersion + readHaskellHaddock :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellHaddock timing settings docBaseDir = do cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings @@ -204,7 +216,8 @@ actionGenerate g@Generate{..} = withTiming (if debug then Just $ replaceExtensio download <- pure $ downloadInput timing insecure download (takeDirectory database) settings <- loadSettings (cbl, want, source) <- case language of - Haskell | Just dir <- haddock -> readHaskellHaddock timing settings dir + Haskell | Just txt <- source -> readHaskellHaddockFile timing settings txt + | Just dir <- haddock -> readHaskellHaddock timing settings dir | [""] <- local_ -> readHaskellGhcpkg timing settings | [] <- local_ -> readHaskellOnline timing settings download | otherwise -> readHaskellDirs timing settings local_ From 420e6a5ce640c859640714bb99eec9f35c980816 Mon Sep 17 00:00:00 2001 From: Michael James Date: Fri, 16 Jul 2021 14:32:58 -0700 Subject: [PATCH 08/12] change name to convert like hoogle4 --- src/Action/CmdLine.hs | 4 ++-- src/Action/Generate.hs | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Action/CmdLine.hs b/src/Action/CmdLine.hs index b57eeac0..a4d2c0a2 100644 --- a/src/Action/CmdLine.hs +++ b/src/Action/CmdLine.hs @@ -37,7 +37,7 @@ data CmdLine | Generate {download :: Maybe Bool ,database :: FilePath - ,source :: Maybe FilePath + ,convert :: [FilePath] ,insecure :: Bool ,include :: [String] ,count :: Maybe Int @@ -128,7 +128,7 @@ search_ = Search generate = Generate {download = def &= help "Download all files from the web" ,insecure = def &= help "Allow insecure HTTPS connections" - ,source = def &=help "Specify a textbase source" + ,convert = def &=help "Convert a set of textbases to the hoo format." ,include = def &= args &= typ "PACKAGE" ,local_ = def &= opt "" &= help "Index local packages and link to local haddock docs" ,count = Nothing &= name "n" &= help "Maximum number of packages to index (defaults to all)" diff --git a/src/Action/Generate.hs b/src/Action/Generate.hs index 60a0660a..01a234f4 100644 --- a/src/Action/Generate.hs +++ b/src/Action/Generate.hs @@ -1,4 +1,5 @@ {-# LANGUAGE ViewPatterns, TupleSections, RecordWildCards, ScopedTypeVariables, PatternGuards #-} +{-# OPTIONS_GHC -Wno-incomplete-patterns #-} module Action.Generate(actionGenerate) where @@ -37,6 +38,8 @@ import Action.CmdLine import General.Conduit import Control.DeepSeq +import Debug.Trace + {- @@ -177,15 +180,17 @@ readHaskellGhcpkg timing settings = do in Map.map (\p -> p{packageTags = ts ++ packageTags p}) cbl pure (cbl, Map.keysSet cbl, source) -readHaskellHaddockFile :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) -readHaskellHaddockFile timing settings txt = do - let name = strPack $ take (length txt - (length ".txt")) txt +readHaskellHaddockFile :: Timing -> Settings -> [FilePath] -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) +readHaskellHaddockFile timing settings txtdbs = do let source = - whenM (liftIO $ doesFileExist txt) $ do - src <- liftIO $ bstrReadFile txt - let url = "file://./" ++ txt - yield (name, url, lbstrFromChunks [src]) - pure (Map.empty, Set.singleton name, source) + forM_ txtdbs $ \txt -> do + let name = strPack $ take (length txt - (length ".txt")) txt + whenM (liftIO $ doesFileExist txt) $ do + src <- liftIO $ bstrReadFile txt + let url = "file://./" ++ txt + yield (name, url, lbstrFromChunks [src]) + let pkgs = Set.fromList $ map (\txt -> strPack $ take (length txt - (length ".txt")) txt) txtdbs + pure (Map.empty, pkgs, source) where docDir name Package{..} = name ++ "-" ++ strUnpack packageVersion @@ -216,7 +221,7 @@ actionGenerate g@Generate{..} = withTiming (if debug then Just $ replaceExtensio download <- pure $ downloadInput timing insecure download (takeDirectory database) settings <- loadSettings (cbl, want, source) <- case language of - Haskell | Just txt <- source -> readHaskellHaddockFile timing settings txt + Haskell | txtdbs@(_:_) <- convert -> readHaskellHaddockFile timing settings txtdbs | Just dir <- haddock -> readHaskellHaddock timing settings dir | [""] <- local_ -> readHaskellGhcpkg timing settings | [] <- local_ -> readHaskellOnline timing settings download From 455888a4110d0c217efcefc3fc2889d1aaa8f4e5 Mon Sep 17 00:00:00 2001 From: Michael James Date: Thu, 22 Jul 2021 11:32:22 -0700 Subject: [PATCH 09/12] Added documentation --- src/Action/Generate.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Action/Generate.hs b/src/Action/Generate.hs index 01a234f4..65418488 100644 --- a/src/Action/Generate.hs +++ b/src/Action/Generate.hs @@ -124,6 +124,10 @@ readHaskellOnline timing settings download = do pure (cbl, want, source) +-- | readHaskellDirs will look for .txt files anywhere under @dirs@. +-- It uses these local files as sources for indexing. +-- Links to the results will default to Hackage, but if @url directives +-- are in the .txt files they can override the link destination. readHaskellDirs :: Timing -> Settings -> [FilePath] -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellDirs timing settings dirs = do files <- concatMapM listFilesRecursive dirs @@ -164,6 +168,8 @@ readFregeOnline timing download = do pure (Map.empty, Set.singleton $ strPack "frege", source) +-- | readHaskellGhcpkg will use every installed package (info via ghc-pkg) +-- as a source. It uses each package's documentation directory as source for indexing. readHaskellGhcpkg :: Timing -> Settings -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellGhcpkg timing settings = do cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings @@ -180,6 +186,8 @@ readHaskellGhcpkg timing settings = do in Map.map (\p -> p{packageTags = ts ++ packageTags p}) cbl pure (cbl, Map.keysSet cbl, source) +-- | readHaskellHaddockFile will read a list of text databases, foo.txt, and use +-- them as sources for indexing. readHaskellHaddockFile :: Timing -> Settings -> [FilePath] -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellHaddockFile timing settings txtdbs = do let source = @@ -194,6 +202,10 @@ readHaskellHaddockFile timing settings txtdbs = do where docDir name Package{..} = name ++ "-" ++ strUnpack packageVersion +-- | readHaskellHaddock takes a @baseDocDir@, which is the filepath prefix for +-- ghc-pkg's documentation directories. This function reads all installed +-- packages on the system (via ghc-pkg), and uses their listed documentation +-- directory as a source for indexing. readHaskellHaddock :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellHaddock timing settings docBaseDir = do cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings From 971e61b4b6bc8f3c311f931964142ad2b668d403 Mon Sep 17 00:00:00 2001 From: Michael James Date: Thu, 22 Jul 2021 13:33:51 -0700 Subject: [PATCH 10/12] Included extra documentation --- docs/Install.md | 9 ++++++--- src/Action/Generate.hs | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/Install.md b/docs/Install.md index 0563716f..f1eb802d 100644 --- a/docs/Install.md +++ b/docs/Install.md @@ -29,6 +29,9 @@ Run `hoogle generate --local` to query `ghc-pkg` and generate links for all pack Run `hoogle generate --local=mydir` to generate an index for the packages in `mydir`, which must contain `foo.txt` Hoogle input files. Links to the results will default to Hackage, but if `@url` directives are in the `.txt` files they can override the link destination. +### Index a text database +Run `hoogle generate --source=MyPackage.txt` to generate an index for one Hoogle-formated textual database. Do not expect links in the results to work, this mode is meant for CLI search. Such a database can be generated with `haddock --hoogle --package-name=MyPackage --odir= ` + ## Searching a Hoogle database You can run searches on the command line or by spawning a web server. @@ -45,7 +48,7 @@ If your database points at the local file system pass `--local` to reserve `file ### GHCi Integration -Ever feel like having access to hoogle whilst messing around in GHCi? It's relatively easy to integrate the two. +Ever feel like having access to hoogle whilst messing around in GHCi? It's relatively easy to integrate the two. The following will install hoogle as a shell command, and configure GHCi to have a command ":hoogle": @@ -62,9 +65,9 @@ Done! On Windows you should add the same line :def hoogle \x -> return $ ":!hoogle \"" ++ x ++ "\"" to file (XP/2003): - C:\Documents and Settings\[your windows account]\Application Data\ghc\ghci.conf + C:\Documents and Settings\[your windows account]\Application Data\ghc\ghci.conf or(Windows Vista/7): - C:\users\[your windows account]\Application Data\ghc\ghci.conf + C:\users\[your windows account]\Application Data\ghc\ghci.conf #### How it works diff --git a/src/Action/Generate.hs b/src/Action/Generate.hs index 65418488..f790ba36 100644 --- a/src/Action/Generate.hs +++ b/src/Action/Generate.hs @@ -204,8 +204,8 @@ readHaskellHaddockFile timing settings txtdbs = do -- | readHaskellHaddock takes a @baseDocDir@, which is the filepath prefix for -- ghc-pkg's documentation directories. This function reads all installed --- packages on the system (via ghc-pkg), and uses their listed documentation --- directory as a source for indexing. +-- packages on the system (via ghc-pkg), it tries to read all files matching +-- @baseDocDir@/@-@/@.txt@ readHaskellHaddock :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ()) readHaskellHaddock timing settings docBaseDir = do cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings From 84922c3690cad080b4a1e29d05256c925f2f922b Mon Sep 17 00:00:00 2001 From: Michael James Date: Thu, 22 Jul 2021 13:36:04 -0700 Subject: [PATCH 11/12] remove extra additions --- src/Action/Generate.hs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Action/Generate.hs b/src/Action/Generate.hs index f790ba36..67d3415b 100644 --- a/src/Action/Generate.hs +++ b/src/Action/Generate.hs @@ -1,5 +1,4 @@ {-# LANGUAGE ViewPatterns, TupleSections, RecordWildCards, ScopedTypeVariables, PatternGuards #-} -{-# OPTIONS_GHC -Wno-incomplete-patterns #-} module Action.Generate(actionGenerate) where @@ -38,8 +37,6 @@ import Action.CmdLine import General.Conduit import Control.DeepSeq -import Debug.Trace - {- From 69a6bdea41cbc8bc3d5817c04d182e1797970159 Mon Sep 17 00:00:00 2001 From: Michael James Date: Thu, 22 Jul 2021 13:41:14 -0700 Subject: [PATCH 12/12] the flag is convert, not source --- docs/Install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Install.md b/docs/Install.md index f1eb802d..ddbed86a 100644 --- a/docs/Install.md +++ b/docs/Install.md @@ -30,7 +30,7 @@ Run `hoogle generate --local` to query `ghc-pkg` and generate links for all pack Run `hoogle generate --local=mydir` to generate an index for the packages in `mydir`, which must contain `foo.txt` Hoogle input files. Links to the results will default to Hackage, but if `@url` directives are in the `.txt` files they can override the link destination. ### Index a text database -Run `hoogle generate --source=MyPackage.txt` to generate an index for one Hoogle-formated textual database. Do not expect links in the results to work, this mode is meant for CLI search. Such a database can be generated with `haddock --hoogle --package-name=MyPackage --odir= ` +Run `hoogle generate --convert=MyPackage.txt` to generate an index for one Hoogle-formated textual database. Do not expect links in the results to work, this mode is meant for CLI search. Such a database can be generated with `haddock --hoogle --package-name=MyPackage --odir= ` ## Searching a Hoogle database