Merge pull request #680 from IntersectMBO/wenkokke/full

feat(full): add full api

Author: jorisdral

README.md

@@ -24,8 +24,7 @@ It has support for:
 - Range lookups, which efficiently retrieve the values for all keys in a
   given range.
 
-- Monoidal upserts (or "mupserts") which combine the stored and new
-  values.
+- Monoidal upserts which combine the stored and new values.
 
 - BLOB storage which assocates a large auxiliary BLOB with a key.
 
@@ -43,9 +42,8 @@ This package exports two modules:
   This module exports a simplified API which picks sensible defaults for
   a number of configuration parameters.
 
-  It does not support mupserts or BLOBs, due to their unintuitive
-  interaction, see [Mupserts and
-  BLOBs](#mupsertsandblobs "#mupsertsandblobs").
+  It does not support upserts or BLOBs, due to their unintuitive
+  interaction, see [Upsert and BLOB](#upsertandblob "#upsertandblob").
 
   If you are looking at this package for the first time, it is strongly
   recommended that you start by reading this module.
@@ -54,13 +52,12 @@ This package exports two modules:
 
   This module exports the full API.
 
-### Mupserts and BLOBs <span id="mupsertsandblobs" class="anchor"></span>
+### Upsert and BLOB <span id="upsertandblob" class="anchor"></span>
 
-The interaction between mupserts and BLOBs is unintuitive. A mupsert
-updates the value associated with the key by combining the old and new
-value with a user-specified function. However, this does not apply to
-any BLOB value associated with the key, which is simply overwritten by
-the new BLOB value.
+The interaction between upserts and BLOBs is unintuitive. A upsert
+updates the value associated with the key by combining the new and old
+values with a user-specified function. However, any BLOB associated with
+the key is simply deleted.
 
 ### Portability <span id="portability" class="anchor"></span>
 
@@ -114,7 +111,8 @@ constants:
   describe the complexity of an operation that involves multiple tables,
   it refers to the sum of all table entries.
 
-- The variable *t* refers to the number of open tables in the session.
+- The variable *o* refers to the number of open tables and cursors in
+  the session.
 
 - The variable *s* refers to the number of snapshots in the session.
 
@@ -157,7 +155,7 @@ Otherwise, the merge policy is listed as N/A.
 <td></td>
 <td>Close</td>
 <td><code>MergePolicyLazyLevelling</code></td>
-<td><span class="math inline">$O(t \: T \: \log_T
+<td><span class="math inline">$O(o \: T \: \log_T
 \frac{n}{B})$</span></td>
 </tr>
 <tr>

bench/macro/lsm-tree-bench-wp8.hs

@@ -415,7 +415,7 @@ doSetup' gopts opts = do
     let name = LSM.toSnapshotName "bench"
 
     LSM.withSession (mkTracer gopts) hasFS hasBlockIO (FS.mkFsPath []) $ \session -> do
-        tbl <- LSM.new @IO @K @V @B session (mkTableConfigSetup gopts opts benchTableConfig)
+        tbl <- LSM.newTableWith @IO @K @V @B (mkTableConfigSetup gopts opts benchTableConfig) session
 
         forM_ (groupsOfN 256 [ 0 .. initialSize gopts ]) $ \batch -> do
             -- TODO: this procedure simply inserts all the keys into initial lsm tree
@@ -426,7 +426,7 @@ doSetup' gopts opts = do
                 | i <- NE.toList batch
                 ]
 
-        LSM.createSnapshot label name tbl
+        LSM.saveSnapshot name label tbl
 
 -------------------------------------------------------------------------------
 -- dry-run
@@ -583,8 +583,8 @@ doRun gopts opts = do
         -- reference version starts with empty (as it's not practical or
         -- necessary for testing to load the whole snapshot).
         tbl <- if check opts
-                then LSM.new  @IO @K @V @B session (mkTableConfigRun gopts benchTableConfig)
-                else LSM.openSnapshot @IO @K @V @B session (mkOverrideDiskCachePolicy gopts) label name
+                then LSM.newTableWith @IO @K @V @B (mkTableConfigRun gopts benchTableConfig) session
+                else LSM.openTableFromSnapshotWith @IO @K @V @B (mkOverrideDiskCachePolicy gopts) session name label
 
         -- In checking mode, compare each output against a pure reference.
         checkvar <- newIORef $ pureReference
@@ -765,7 +765,7 @@ pipelinedIteration h output !initialSize !batchSize
 
       -- At this point, after syncing, our peer is guaranteed to no longer be
       -- using tbl_n. They used it to generate tbl_n+1 (which they gave us).
-      LSM.close tbl_n
+      LSM.closeTable tbl_n
       output b $! applyUpdates delta (V.zip ls lrs)
       pure tbl_n1
 
@@ -868,8 +868,8 @@ updateToLookupResult :: LSM.Update v b -> LSM.LookupResult v ()
 updateToLookupResult (LSM.Insert v Nothing)  = LSM.Found v
 updateToLookupResult (LSM.Insert v (Just _)) = LSM.FoundWithBlob v ()
 updateToLookupResult  LSM.Delete             = LSM.NotFound
-updateToLookupResult (LSM.Mupsert _)         = error $
-                                               "Unexpected mupsert encountered"
+updateToLookupResult (LSM.Upsert _)         = error $
+                                               "Unexpected upsert encountered"
 
 -- | Return the adjacent batches where there is overlap between one batch's
 -- inserts and the next batch's lookups. Testing the pipelined version needs

bench/micro/Bench/Database/LSMTree.hs

@@ -14,14 +14,12 @@ import qualified Data.Vector as V
 import           Data.Void
 import           Data.Word
 import           Database.LSMTree hiding (withTable)
-import qualified Database.LSMTree.Common as Common
 import           Database.LSMTree.Extras
 import           Database.LSMTree.Extras.Orphans ()
 import           Database.LSMTree.Internal.Assertions (fromIntegralChecked)
 import qualified Database.LSMTree.Internal.RawBytes as RB
-import           Database.LSMTree.Internal.Serialise.Class
 import           GHC.Generics (Generic)
-import           Prelude hiding (getContents)
+import           Prelude hiding (getContents, take)
 import           System.Directory (removeDirectoryRecursive)
 import qualified System.FS.API as FS
 import qualified System.FS.BlockIO.API as FS
@@ -35,9 +33,9 @@ benchmarks = bgroup "Bench.Database.LSMTree" [
       benchLargeValueVsSmallValueBlob
     , benchCursorScanVsRangeLookupScan
     , benchInsertBatches
-    , benchInsertsVsMupserts
-    , benchLookupsInsertsVsMupserts
-    , benchLookupInsertsVsLookupMupserts
+    , benchInsertsVsUpserts
+    , benchLookupsInsertsVsUpserts
+    , benchLookupInsertsVsLookupUpserts
     ]
 
 {-------------------------------------------------------------------------------
@@ -75,17 +73,13 @@ newtype V3 = V3 Word64
 
 type B3 = Void
 
--- Simple addition
-resolve :: V3 -> V3 -> V3
-resolve = (+)
-
 instance ResolveValue V3 where
-  resolveValue = resolveDeserialised resolve
+  resolve = (+)
 
-benchConfig :: Common.TableConfig
-benchConfig = Common.defaultTableConfig {
-      Common.confWriteBufferAlloc  = Common.AllocNumEntries (Common.NumEntries 20000)
-    , Common.confFencePointerIndex = Common.CompactIndex
+benchConfig :: TableConfig
+benchConfig = defaultTableConfig
+    { confWriteBufferAlloc  = AllocNumEntries (NumEntries 20000)
+    , confFencePointerIndex = CompactIndex
     }
 
 {-------------------------------------------------------------------------------
@@ -142,12 +136,12 @@ benchLargeValueVsSmallValueBlob =
       initialise inss = do
           (tmpDir, hfs, hbio) <- mkFiles
           s <- openSession nullTracer hfs hbio (FS.mkFsPath [])
-          t <- new s benchConfig
+          t <- newTableWith benchConfig s
           V.mapM_ (inserts t) inss
           pure (tmpDir, hfs, hbio, s, t)
 
       cleanup (tmpDir, hfs, hbio, s, t) = do
-          close t
+          closeTable t
           closeSession s
           cleanupFiles (tmpDir, hfs, hbio)
 
@@ -172,11 +166,11 @@ benchCursorScanVsRangeLookupScan =
           bgroup "cursor-scan-vs-range-lookup-scan" [
               bench "cursor-scan-full" $ whnfIO $ do
                 withCursor t $ \c -> do
-                  readCursor initialSize c
+                  take initialSize c
             , bench "cursor-scan-chunked" $ whnfIO $ do
                 withCursor t $ \c -> do
                   forM_ ([1 .. numChunks] :: [Int]) $ \_ -> do
-                    readCursor readSize c
+                    take readSize c
             , bench "range-scan-full" $ whnfIO $ do
                 rangeLookup t (FromToIncluding (K minBound) (K maxBound))
             , bench "range-scan-chunked" $ whnfIO $ do
@@ -227,12 +221,12 @@ benchCursorScanVsRangeLookupScan =
       initialise inss = do
           (tmpDir, hfs, hbio) <- mkFiles
           s <- openSession nullTracer hfs hbio (FS.mkFsPath [])
-          t <- new s benchConfig
+          t <- newTableWith benchConfig s
           V.mapM_ (inserts t) inss
           pure (tmpDir, hfs, hbio, s, t)
 
       cleanup (tmpDir, hfs, hbio, s, t) = do
-          close t
+          closeTable t
           closeSession s
           cleanupFiles (tmpDir, hfs, hbio)
 
@@ -251,9 +245,9 @@ benchInsertBatches =
       !initialSize = 100_000
       !batchSize = 256
 
-      _benchConfig :: Common.TableConfig
+      _benchConfig :: TableConfig
       _benchConfig = benchConfig {
-          Common.confWriteBufferAlloc = Common.AllocNumEntries (Common.NumEntries 1000)
+          confWriteBufferAlloc = AllocNumEntries (NumEntries 1000)
         }
 
       randomInserts :: Int -> V.Vector (K, V2, Maybe Void)
@@ -272,31 +266,31 @@ benchInsertBatches =
       initialise = do
           (tmpDir, hfs, hbio) <- mkFiles
           s <- openSession nullTracer hfs hbio (FS.mkFsPath [])
-          t <- new s _benchConfig
+          t <- newTableWith _benchConfig s
           pure (tmpDir, hfs, hbio, s, t)
 
       cleanup (tmpDir, hfs, hbio, s, t) = do
-          close t
+          closeTable t
           closeSession s
           cleanupFiles (tmpDir, hfs, hbio)
 
 {-------------------------------------------------------------------------------
-  Inserts vs. Mupserts
+  Inserts vs. Upserts
 -------------------------------------------------------------------------------}
 
--- | Compare inserts and mupserts. The logical contents of the resulting
+-- | Compare inserts and upserts. The logical contents of the resulting
 -- database are the same.
-benchInsertsVsMupserts :: Benchmark
-benchInsertsVsMupserts =
+benchInsertsVsUpserts :: Benchmark
+benchInsertsVsUpserts =
     env (pure $ snd $ randomEntriesGrouped 800_000 250) $ \ess ->
       env (pure $ V.map mkInserts ess) $ \inss ->
-        bgroup "inserts-vs-mupserts" [
+        bgroup "inserts-vs-upserts" [
           bench "inserts" $
             withEmptyTable $ \(_, _, _, _, t) ->
               V.mapM_ (inserts t) inss
-        , bench "mupserts" $
+        , bench "upserts" $
             withEmptyTable $ \(_, _, _, _, t) ->
-              V.mapM_ (mupserts t) ess
+              V.mapM_ (upserts t) ess
         ]
     where
       withEmptyTable =
@@ -311,18 +305,18 @@ benchInsertsVsMupserts =
             )
 
 {-------------------------------------------------------------------------------
-  Lookups plus Inserts vs. Mupserts
+  Lookups plus Inserts vs. Upserts
 -------------------------------------------------------------------------------}
 
--- | Compare lookups+inserts to mupserts. The former costs 2 LSMT operations,
---  while Mupserts only cost 1 LSMT operation. The number of operations do not
+-- | Compare lookups+inserts to upserts. The former costs 2 LSMT operations,
+--  while Upserts only cost 1 LSMT operation. The number of operations do not
 --  directly translate to the number of I\/O operations, but one can assume that
---  lookup+insert is roughly twice as costly as mupsert.
-benchLookupsInsertsVsMupserts :: Benchmark
-benchLookupsInsertsVsMupserts =
+--  lookup+insert is roughly twice as costly as upsert.
+benchLookupsInsertsVsUpserts :: Benchmark
+benchLookupsInsertsVsUpserts =
     env (pure $ snd $ randomEntriesGrouped 800_000 250) $ \ess ->
       env (pure $ V.map mkInserts ess) $ \inss ->
-        bgroup "lookups-inserts-vs-mupserts" [
+        bgroup "lookups-inserts-vs-upserts" [
           bench "lookups-inserts" $
             withTable inss $ \(_, _, _, _, t) ->
               -- Insert the same keys again, but we sum the existing values in
@@ -333,12 +327,12 @@ benchLookupsInsertsVsMupserts =
                 lrs <- lookups t (V.map fst es)
                 let ins' = V.zipWith f es lrs
                 inserts t ins'
-        , bench "mupserts" $
+        , bench "upserts" $
             withTable inss $ \(_, _, _, _, t) ->
               -- Insert the same keys again, but we sum the existing values in
               -- the table with the values we are going to insert: submit
-              -- mupserts with the insert values.
-              V.forM_ ess $ \es -> mupserts t es
+              -- upserts with the insert values.
+              V.forM_ ess $ \es -> upserts t es
         ]
   where
     f (k, v) = \case
@@ -359,20 +353,20 @@ benchLookupsInsertsVsMupserts =
           )
 
 {-------------------------------------------------------------------------------
-  Lookup Inserts vs. Lookup Mupserts
+  Lookup Inserts vs. Lookup Upserts
 -------------------------------------------------------------------------------}
 
--- | Compare lookups after inserts against lookups after mupserts.
-benchLookupInsertsVsLookupMupserts :: Benchmark
-benchLookupInsertsVsLookupMupserts =
+-- | Compare lookups after inserts against lookups after upserts.
+benchLookupInsertsVsLookupUpserts :: Benchmark
+benchLookupInsertsVsLookupUpserts =
     env (pure $ snd $ randomEntriesGrouped 80_000 250) $ \ess ->
       env (pure $ V.map mkInserts ess) $ \inss ->
-        bgroup "lookup-inserts-vs-lookup-mupserts" [
+        bgroup "lookup-inserts-vs-lookup-upserts" [
           bench "lookup-inserts" $
             withInsertTable inss $ \(_, _, _, _, t) -> do
                 V.forM_ ess $ \es -> lookups t (V.map fst es)
-        , bench "lookup-mupserts" $
-            withMupsertTable ess $ \(_, _, _, _, t) -> do
+        , bench "lookup-upserts" $
+            withUpsertTable ess $ \(_, _, _, _, t) -> do
                 V.forM_ ess $ \es -> lookups t (V.map fst es)
         ]
   where
@@ -393,14 +387,14 @@ benchLookupInsertsVsLookupMupserts =
               cleanupFiles (tmpDir, hfs, hbio)
           )
 
-    withMupsertTable ess =
+    withUpsertTable ess =
         perRunEnvWithCleanup
-          -- Mupsert the same key 10 times. The results in a logical database
+          -- Upsert the same key 10 times. The results in a logical database
           -- containing the original keys with the original value *10.
           (do (tmpDir, hfs, hbio) <- mkFiles
               (s, t) <- mkTable hfs hbio benchConfig
               V.forM_ [1..10] $ \(_::Int) ->
-                V.mapM_ (mupserts t) ess
+                V.mapM_ (upserts t) ess
               pure (tmpDir, hfs, hbio, s, t)
           )
           (\(tmpDir, hfs, hbio, s, t) -> do
@@ -458,7 +452,7 @@ mkTable ::
         )
 mkTable hfs hbio conf = do
     sesh <- openSession nullTracer hfs hbio (FS.mkFsPath [])
-    t <- new sesh conf
+    t <- newTableWith conf sesh
     pure (sesh, t)
 
 cleanupTable ::
@@ -467,5 +461,5 @@ cleanupTable ::
      )
   -> IO ()
 cleanupTable (s, t) = do
-    close t
+    closeTable t
     closeSession s

doc/format-page.md

@@ -25,7 +25,7 @@ Logically, a page consists of a value of type
 type Page = [(Key, Operation, Maybe BlobRef)]
 ```
 where the operation can be insert (with a value), delete (with no value) or
-mupsert (with a value). The entries are sorted by key.
+upsert (with a value). The entries are sorted by key.
 
 The file is structured in a page-oriented way to support efficient I/O using
 random page-sized reads. By page-oriented we mean that the information is