From 13469706e66afdb21ff51cc3ab94ec742403373b Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 10 Jul 2025 14:29:32 +0200 Subject: [PATCH 1/6] chore(deps): update hasql to 1.8.1.4 --- nix/overlays/haskell-packages.nix | 9 ----- postgrest.cabal | 10 ++--- src/PostgREST/AppState.hs | 39 ++++++++++--------- src/PostgREST/Error.hs | 11 +++++- src/PostgREST/Metrics.hs | 2 +- src/PostgREST/Observation.hs | 6 ++- stack.yaml | 11 +++++- stack.yaml.lock | 65 +++++++++++++++++++++++++++---- 8 files changed, 107 insertions(+), 46 deletions(-) diff --git a/nix/overlays/haskell-packages.nix b/nix/overlays/haskell-packages.nix index 1e0feefd56..6d4b2343c1 100644 --- a/nix/overlays/haskell-packages.nix +++ b/nix/overlays/haskell-packages.nix @@ -49,15 +49,6 @@ let # Before upgrading fuzzyset to 0.3, check: https://github.com/PostgREST/postgrest/issues/3329 # jailbreak, because hspec limit for tests fuzzyset = prev.fuzzyset_0_2_4; - - # Downgrade hasql and related packages while we are still on GHC 9.4 for the static build. - hasql = lib.dontCheck (lib.doJailbreak prev.hasql_1_6_4_4); - hasql-dynamic-statements = lib.dontCheck prev.hasql-dynamic-statements_0_3_1_5; - hasql-implicits = lib.dontCheck prev.hasql-implicits_0_1_1_3; - hasql-notifications = lib.dontCheck prev.hasql-notifications_0_2_2_2; - hasql-pool = lib.dontCheck prev.hasql-pool_1_0_1; - hasql-transaction = lib.dontCheck prev.hasql-transaction_1_1_0_1; - postgresql-binary = lib.dontCheck (lib.doJailbreak prev.postgresql-binary_0_13_1_3); }; in { diff --git a/postgrest.cabal b/postgrest.cabal index fa28807b4e..74f946d683 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -111,10 +111,10 @@ library , either >= 4.4.1 && < 5.1 , extra >= 1.7.0 && < 2.0 , fuzzyset >= 0.2.4 && < 0.3 - , hasql >= 1.6.1.1 && < 1.7 + , hasql >= 1.7 && < 1.9 , hasql-dynamic-statements >= 0.3.1 && < 0.4 - , hasql-notifications >= 0.2.2.2 && < 0.2.3 - , hasql-pool >= 1.0.1 && < 1.1 + , hasql-notifications >= 0.2.2.0 && < 0.3 + , hasql-pool >= 1.1 && < 1.3 , hasql-transaction >= 1.0.1 && < 1.2 , heredoc >= 0.2 && < 0.3 , http-types >= 0.12.2 && < 0.13 @@ -129,8 +129,6 @@ library , network-uri >= 2.6.1 && < 2.8 , optparse-applicative >= 0.13 && < 0.19 , parsec >= 3.1.11 && < 3.2 - -- Technically unused, can be removed after updating to hasql >= 1.7 - , postgresql-libpq >= 0.10 , prometheus-client >= 1.1.1 && < 1.2.0 , protolude >= 0.3.1 && < 0.4 , regex-tdfa >= 1.2.2 && < 1.4 @@ -266,7 +264,7 @@ test-suite spec , bytestring >= 0.10.8 && < 0.13 , case-insensitive >= 1.2 && < 1.3 , containers >= 0.5.7 && < 0.7 - , hasql-pool >= 1.0.1 && < 1.1 + , hasql-pool >= 1.0.1 && < 1.3 , hasql-transaction >= 1.0.1 && < 1.2 , heredoc >= 0.2 && < 0.3 , hspec >= 2.3 && < 2.12 diff --git a/src/PostgREST/AppState.hs b/src/PostgREST/AppState.hs index 9a65871cc6..8ee2e963ac 100644 --- a/src/PostgREST/AppState.hs +++ b/src/PostgREST/AppState.hs @@ -214,21 +214,30 @@ initPool AppConfig{..} observer = do -- | Run an action with a database connection. usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a) usePool AppState{stateObserver=observer, stateMainThreadId=mainThreadId, ..} sess = do - observer PoolRequest + observer PoolRequest - res <- SQL.use statePool sess + res <- SQL.use statePool sess - observer PoolRequestFullfilled + observer PoolRequestFullfilled - whenLeft res (\case - SQL.AcquisitionTimeoutUsageError -> - observer $ PoolAcqTimeoutObs SQL.AcquisitionTimeoutUsageError - err@(SQL.ConnectionUsageError e) -> - let failureMessage = BS.unpack $ fromMaybe mempty e in - when (("FATAL: password authentication failed" `isInfixOf` failureMessage) || ("no password supplied" `isInfixOf` failureMessage)) $ do - observer $ ExitDBFatalError ServerAuthError err - killThread mainThreadId - err@(SQL.SessionUsageError (SQL.QueryError tpl _ (SQL.ResultError resultErr))) -> do + whenLeft res (\case + SQL.AcquisitionTimeoutUsageError -> + observer $ PoolAcqTimeoutObs SQL.AcquisitionTimeoutUsageError + err@(SQL.ConnectionUsageError e) -> + let failureMessage = BS.unpack $ fromMaybe mempty e in + when (("FATAL: password authentication failed" `isInfixOf` failureMessage) || ("no password supplied" `isInfixOf` failureMessage)) $ do + observer $ ExitDBFatalError ServerAuthError err + killThread mainThreadId + err@(SQL.SessionUsageError (SQL.QueryError tpl _ (SQL.ResultError resultErr))) -> handleResultError err tpl resultErr + -- Passing the empty template will not work for schema cache queries, see TODO further below. + err@(SQL.SessionUsageError (SQL.PipelineError (SQL.ResultError resultErr))) -> handleResultError err mempty resultErr + err@(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ClientError _))) -> observer $ QueryErrorCodeHighObs err + SQL.SessionUsageError (SQL.PipelineError (SQL.ClientError _)) -> pure () + ) + + return res + where + handleResultError err tpl resultErr = do case resultErr of SQL.UnexpectedResult{} -> do observer $ ExitDBFatalError ServerPgrstBug err @@ -261,12 +270,6 @@ usePool AppState{stateObserver=observer, stateMainThreadId=mainThreadId, ..} ses SQL.ServerError{} -> when (Error.status (Error.PgError False err) >= HTTP.status500) $ observer $ QueryErrorCodeHighObs err - err@(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ClientError _))) -> - -- An error on the client-side, usually indicates problems wth connection - observer $ QueryErrorCodeHighObs err - ) - - return res -- | Flush the connection pool so that any future use of the pool will -- use connections freshly established after this call. diff --git a/src/PostgREST/Error.hs b/src/PostgREST/Error.hs index 39217610cf..4649d094e1 100644 --- a/src/PostgREST/Error.hs +++ b/src/PostgREST/Error.hs @@ -526,18 +526,22 @@ instance JSON.ToJSON SQL.UsageError where instance ErrorBody SQL.UsageError where code (SQL.ConnectionUsageError _) = "PGRST000" + code (SQL.SessionUsageError (SQL.PipelineError e)) = code e code (SQL.SessionUsageError (SQL.QueryError _ _ e)) = code e code SQL.AcquisitionTimeoutUsageError = "PGRST003" message (SQL.ConnectionUsageError _) = "Database connection error. Retrying the connection." + message (SQL.SessionUsageError (SQL.PipelineError e)) = message e message (SQL.SessionUsageError (SQL.QueryError _ _ e)) = message e message SQL.AcquisitionTimeoutUsageError = "Timed out acquiring connection from connection pool." details (SQL.ConnectionUsageError e) = JSON.String . T.decodeUtf8 <$> e + details (SQL.SessionUsageError (SQL.PipelineError e)) = details e details (SQL.SessionUsageError (SQL.QueryError _ _ e)) = details e details SQL.AcquisitionTimeoutUsageError = Nothing hint (SQL.ConnectionUsageError _) = Nothing + hint (SQL.SessionUsageError (SQL.PipelineError e)) = hint e hint (SQL.SessionUsageError (SQL.QueryError _ _ e)) = hint e hint SQL.AcquisitionTimeoutUsageError = Nothing @@ -586,8 +590,13 @@ instance ErrorBody SQL.CommandError where pgErrorStatus :: Bool -> SQL.UsageError -> HTTP.Status pgErrorStatus _ (SQL.ConnectionUsageError _) = HTTP.status503 pgErrorStatus _ SQL.AcquisitionTimeoutUsageError = HTTP.status504 +pgErrorStatus _ (SQL.SessionUsageError (SQL.PipelineError (SQL.ClientError _))) = HTTP.status503 pgErrorStatus _ (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ClientError _))) = HTTP.status503 -pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError rError))) = +pgErrorStatus authed (SQL.SessionUsageError (SQL.PipelineError (SQL.ResultError rError))) = mapSQLtoHTTP authed rError +pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError rError))) = mapSQLtoHTTP authed rError + +mapSQLtoHTTP :: Bool -> SQL.ResultError -> HTTP.Status +mapSQLtoHTTP authed rError = case rError of (SQL.ServerError c m d _ _) -> case BS.unpack c of diff --git a/src/PostgREST/Metrics.hs b/src/PostgREST/Metrics.hs index 7a39557751..26ca017408 100644 --- a/src/PostgREST/Metrics.hs +++ b/src/PostgREST/Metrics.hs @@ -53,7 +53,7 @@ observationMetrics MetricsState{..} obs = case obs of (PoolAcqTimeoutObs _) -> do incCounter poolTimeouts (HasqlPoolObs (SQL.ConnectionObservation _ status)) -> case status of - SQL.ReadyForUseConnectionStatus -> do + SQL.ReadyForUseConnectionStatus _ -> do incGauge poolAvailable SQL.InUseConnectionStatus -> do decGauge poolAvailable diff --git a/src/PostgREST/Observation.hs b/src/PostgREST/Observation.hs index 296de0f486..52cd36517f 100644 --- a/src/PostgREST/Observation.hs +++ b/src/PostgREST/Observation.hs @@ -141,13 +141,17 @@ observationMessage = \case "Connection " <> show uuid <> ( case status of SQL.ConnectingConnectionStatus -> " is being established" - SQL.ReadyForUseConnectionStatus -> " is available" + SQL.ReadyForUseConnectionStatus reason -> " is available due to " <> case reason of + SQL.EstablishedConnectionReadyForUseReason -> "connection establishment" + SQL.SessionFailedConnectionReadyForUseReason _ -> "session failure" + SQL.SessionSucceededConnectionReadyForUseReason -> "session success" SQL.InUseConnectionStatus -> " is used" SQL.TerminatedConnectionStatus reason -> " is terminated due to " <> case reason of SQL.AgingConnectionTerminationReason -> "max lifetime" SQL.IdlenessConnectionTerminationReason -> "max idletime" SQL.ReleaseConnectionTerminationReason -> "release" SQL.NetworkErrorConnectionTerminationReason _ -> "network error" -- usage error is already logged, no need to repeat the same message. + SQL.InitializationErrorTerminationReason _ -> "init failure" ) PoolRequest -> "Trying to borrow a connection from pool" diff --git a/stack.yaml b/stack.yaml index 20de8d453e..856dde4b46 100644 --- a/stack.yaml +++ b/stack.yaml @@ -10,6 +10,13 @@ nix: extra-deps: - fuzzyset-0.2.4 - - hasql-pool-1.0.1 + - hasql-1.8.1.4 + - hasql-dynamic-statements-0.3.1.8 + - hasql-implicits-0.2.0.1 + - hasql-notifications-0.2.3.2 + - hasql-pool-1.2.0.3 + - hasql-transaction-1.1.1.2 - jose-jwt-0.10.0 - - postgresql-libpq-0.10.1.0 + - postgresql-binary-0.14.0.1 + - postgresql-libpq-0.11.0.0 + - postgresql-libpq-configure-0.11 diff --git a/stack.yaml.lock b/stack.yaml.lock index 32956204fc..fce7ce1329 100644 --- a/stack.yaml.lock +++ b/stack.yaml.lock @@ -12,12 +12,47 @@ packages: original: hackage: fuzzyset-0.2.4 - completed: - hackage: hasql-pool-1.0.1@sha256:3cfb4c7153a6c536ac7e126c17723e6d26ee03794954deed2d72bcc826d05a40,2302 + hackage: hasql-1.8.1.4@sha256:9f8b036b1485994da1fc804cead2bdf0eccebcb31fe109e8c0aaa49dea6ad72b,5696 pantry-tree: - sha256: d98e1269bdd60989b0eb0b84e1d5357eaa9f92821439d9f206663b7251ee95b2 - size: 799 + sha256: a429280972f8c8f4624b2e84ea550a05e1d9a9cd7d8b2cdbcf27253feb6f98a1 + size: 3438 original: - hackage: hasql-pool-1.0.1 + hackage: hasql-1.8.1.4 +- completed: + hackage: hasql-dynamic-statements-0.3.1.8@sha256:7665dae003849430980b835f864c571f1a7aa8c8a2640876c94576955c98444e,2537 + pantry-tree: + sha256: 68c29c16f70bf1450926c8e511a54755cebd25002c06d0664202b8913b523c15 + size: 595 + original: + hackage: hasql-dynamic-statements-0.3.1.8 +- completed: + hackage: hasql-implicits-0.2.0.1@sha256:63ca855a4b857e762d48757f6a9562a2cb9fd895c3d38c941260768278c4923c,1336 + pantry-tree: + sha256: cb9e593c1579dd8de430cb587643896c39eff09e6c45a1432324e9d6af2ca876 + size: 264 + original: + hackage: hasql-implicits-0.2.0.1 +- completed: + hackage: hasql-notifications-0.2.3.2@sha256:547c1c677227b3063042f3af4d150dd224c1219545e936428da6d6b05e56c5fb,1998 + pantry-tree: + sha256: 6e0427de378fe97da347ec977b0353001940e8f8ff772a310eb2acac95ef7b12 + size: 452 + original: + hackage: hasql-notifications-0.2.3.2 +- completed: + hackage: hasql-pool-1.2.0.3@sha256:27cfef3f4921c9cdaf4cae095f802c9977976a434842792d2b073537681b16c8,2389 + pantry-tree: + sha256: f380573da2665b994fa9fb31e55c272f3757598f57f850fdb31d1b7fbe184a5e + size: 982 + original: + hackage: hasql-pool-1.2.0.3 +- completed: + hackage: hasql-transaction-1.1.1.2@sha256:be4fb0e49da04f55c9bfbd5828f7025f1cf3165340ecf79b57ec286f4bde2368,2592 + pantry-tree: + sha256: 41a8a96841e612787f2a744c49a57b5d5f4aa3fbaec3efe5ed1d37441f54c3b5 + size: 1027 + original: + hackage: hasql-transaction-1.1.1.2 - completed: hackage: jose-jwt-0.10.0@sha256:6ed175a01c721e317ceea15eb251a81de145c03711a977517935633a5cdec1d4,3546 pantry-tree: @@ -26,12 +61,26 @@ packages: original: hackage: jose-jwt-0.10.0 - completed: - hackage: postgresql-libpq-0.10.1.0@sha256:6b580c9d5068e78eecc13e655b2885c8e79cdacfca513c5d1e5a6b9dc61d9758,3166 + hackage: postgresql-binary-0.14.0.1@sha256:06366fd82deda89f9237b885e9493eafe0b9903d05c19761288b048dcc9a99ee,4027 + pantry-tree: + sha256: ca58c715b5e5ad2abbfc1d87e3a620f27b93007591168acd2e60617880be35fa + size: 1661 + original: + hackage: postgresql-binary-0.14.0.1 +- completed: + hackage: postgresql-libpq-0.11.0.0@sha256:ca7facdf755f7ad3950e75eee4a388f52179b027ca983be362c400ab0a37a4c4,2702 + pantry-tree: + sha256: d401e0dd176fcbb8badb3ea9fd57614bb70c1a486cbe202c271a911f91d1556c + size: 1048 + original: + hackage: postgresql-libpq-0.11.0.0 +- completed: + hackage: postgresql-libpq-configure-0.11@sha256:019c5d83da0b4dc0b4487e0e868f2eed5efa25e0688f5cfc2ba8191a80e527aa,1309 pantry-tree: - sha256: ae81e7628a8f3d1ef33ace71fa0845c073c003ca7f1150cc9d9ba1e55fc84236 - size: 1096 + sha256: 04a70f52cfacce71901efb1eafe3ae3103d133c1c4ee89ba7a51877c67503041 + size: 353 original: - hackage: postgresql-libpq-0.10.1.0 + hackage: postgresql-libpq-configure-0.11 snapshots: - completed: sha256: 238fa745b64f91184f9aa518fe04bdde6552533d169b0da5256670df83a0f1a9 From 9390e10328897560db178ad7c5f1c96ee4385687 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 10 Jul 2025 12:27:39 +0200 Subject: [PATCH 2/6] nix: pin hsie at GHC 9.4 This needs more involved changes in the hsie source code, which can be deferred to later. --- default.nix | 4 +--- nix/hsie/default.nix | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/default.nix b/default.nix index 6d1245c4db..7a03645d11 100644 --- a/default.nix +++ b/default.nix @@ -104,9 +104,7 @@ rec { # Tooling for analyzing Haskell imports and exports. hsie = - pkgs.callPackage nix/hsie { - inherit (pkgs.haskell.packages."${compiler}") ghcWithPackages; - }; + pkgs.callPackage nix/hsie { }; ### Tools diff --git a/nix/hsie/default.nix b/nix/hsie/default.nix index 706c43848c..5b272051cc 100644 --- a/nix/hsie/default.nix +++ b/nix/hsie/default.nix @@ -1,4 +1,4 @@ -{ ghcWithPackages +{ haskell , runCommand }: let @@ -14,7 +14,7 @@ let ps.ghc-paths ps.optparse-applicative ]; - ghc = ghcWithPackages modules; + ghc = haskell.packages.ghc94.ghcWithPackages modules; hsie = runCommand "haskellimports" { inherit name src; } '' From cd9c613902d304cc9769050b6d6769ff6893bf98 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 10 Jul 2025 12:30:02 +0200 Subject: [PATCH 3/6] nix: build with GHC 9.6.7 --- default.nix | 2 +- postgrest.cabal | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 7a03645d11..e46900415a 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { system ? builtins.currentSystem -, compiler ? "ghc948" +, compiler ? "ghc967" , # Commit of the Nixpkgs repository that we want to use. # It defaults to reading the inputs from flake.lock, which serves diff --git a/postgrest.cabal b/postgrest.cabal index 74f946d683..582c5a039e 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -16,11 +16,10 @@ extra-source-files: CHANGELOG.md cabal-version: >= 1.10 tested-with: - -- nix - GHC == 9.4.8 -- cabal on Ubuntu + -- nix -- stack on FreeBSD, MacOS, Ubuntu, Windows - , GHC == 9.6.7 + GHC == 9.6.7 -- cabal on Ubuntu , GHC == 9.8.4 From 52001b9b4b8eb227f52780905aebcb41b2ff5842 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Fri, 8 Aug 2025 22:20:21 -0500 Subject: [PATCH 4/6] chore(deps): upgrade hasql to 1.9.1.2 --- nix/overlays/haskell-packages.nix | 14 +++++++++++ postgrest.cabal | 13 +++++----- src/PostgREST/AppState.hs | 40 +++++++++++++++++-------------- src/PostgREST/CLI.hs | 5 +--- src/PostgREST/Config/Database.hs | 6 ++--- src/PostgREST/Listener.hs | 18 ++++++++------ src/PostgREST/Query.hs | 4 +--- stack.yaml | 10 ++++---- test/spec/Main.hs | 10 ++++---- 9 files changed, 70 insertions(+), 50 deletions(-) diff --git a/nix/overlays/haskell-packages.nix b/nix/overlays/haskell-packages.nix index 6d4b2343c1..4ddbcb9b55 100644 --- a/nix/overlays/haskell-packages.nix +++ b/nix/overlays/haskell-packages.nix @@ -49,6 +49,20 @@ let # Before upgrading fuzzyset to 0.3, check: https://github.com/PostgREST/postgrest/issues/3329 # jailbreak, because hspec limit for tests fuzzyset = prev.fuzzyset_0_2_4; + + hasql = lib.dontCheck prev.hasql_1_9_1_2; + hasql-pool = lib.dontCheck prev.hasql-pool_1_3_0_1; + hasql-notifications = lib.dontCheck (prev.callHackageDirect + { + pkg = "hasql-notifications"; + ver = "0.2.4.0"; + sha256 = "sha256-5NsF0WyiZuqkZemlQfA/J7rAJttkE56oPJK4zgqMbZ4="; + } + { }); + hasql-transaction = lib.dontCheck prev.hasql-transaction_1_2_0_1; + + # Needed for hasql 1.9 + text-builder = lib.dontCheck prev.text-builder_1_0_0_3; }; in { diff --git a/postgrest.cabal b/postgrest.cabal index 582c5a039e..61069d2045 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -110,11 +110,11 @@ library , either >= 4.4.1 && < 5.1 , extra >= 1.7.0 && < 2.0 , fuzzyset >= 0.2.4 && < 0.3 - , hasql >= 1.7 && < 1.9 + , hasql >= 1.9.1.2 && < 1.10 , hasql-dynamic-statements >= 0.3.1 && < 0.4 - , hasql-notifications >= 0.2.2.0 && < 0.3 - , hasql-pool >= 1.1 && < 1.3 - , hasql-transaction >= 1.0.1 && < 1.2 + , hasql-notifications >= 0.2.4.0 && < 0.3 + , hasql-pool >= 1.3.0.1 && < 1.4 + , hasql-transaction >= 1.2.0.1 && < 1.3 , heredoc >= 0.2 && < 0.3 , http-types >= 0.12.2 && < 0.13 , insert-ordered-containers >= 0.2.2 && < 0.3 @@ -263,8 +263,9 @@ test-suite spec , bytestring >= 0.10.8 && < 0.13 , case-insensitive >= 1.2 && < 1.3 , containers >= 0.5.7 && < 0.7 - , hasql-pool >= 1.0.1 && < 1.3 - , hasql-transaction >= 1.0.1 && < 1.2 + , hasql >= 1.9.1.2 && < 1.10 + , hasql-pool >= 1.3.0.1 && < 1.4 + , hasql-transaction >= 1.2.0.1 && < 1.3 , heredoc >= 0.2 && < 0.3 , hspec >= 2.3 && < 2.12 , hspec-expectations >= 0.8.4 && < 0.9 diff --git a/src/PostgREST/AppState.hs b/src/PostgREST/AppState.hs index 8ee2e963ac..516ad16be6 100644 --- a/src/PostgREST/AppState.hs +++ b/src/PostgREST/AppState.hs @@ -30,22 +30,24 @@ module PostgREST.AppState , isPending ) where -import qualified Data.ByteString.Char8 as BS -import Data.Either.Combinators (whenLeft) -import qualified Data.Text as T (unpack) -import qualified Hasql.Pool as SQL -import qualified Hasql.Pool.Config as SQL -import qualified Hasql.Session as SQL -import qualified Hasql.Transaction.Sessions as SQL -import qualified Network.HTTP.Types.Status as HTTP -import qualified Network.Socket as NS -import qualified PostgREST.Auth.JwtCache as JwtCache -import qualified PostgREST.Error as Error -import qualified PostgREST.Logger as Logger -import qualified PostgREST.Metrics as Metrics +import qualified Data.ByteString.Char8 as BS +import Data.Either.Combinators (whenLeft) +import qualified Data.Text as T (unpack) +import qualified Hasql.Connection.Setting as SQL +import qualified Hasql.Connection.Setting.Connection as SQL +import qualified Hasql.Pool as SQL +import qualified Hasql.Pool.Config as SQL +import qualified Hasql.Session as SQL +import qualified Hasql.Transaction.Sessions as SQL +import qualified Network.HTTP.Types.Status as HTTP +import qualified Network.Socket as NS +import qualified PostgREST.Auth.JwtCache as JwtCache +import qualified PostgREST.Error as Error +import qualified PostgREST.Logger as Logger +import qualified PostgREST.Metrics as Metrics import PostgREST.Observation -import PostgREST.Version (prettyVersion) -import System.TimeIt (timeItT) +import PostgREST.Version (prettyVersion) +import System.TimeIt (timeItT) import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate, updateAction) @@ -207,7 +209,10 @@ initPool AppConfig{..} observer = do , SQL.acquisitionTimeout $ fromIntegral configDbPoolAcquisitionTimeout , SQL.agingTimeout $ fromIntegral configDbPoolMaxLifetime , SQL.idlenessTimeout $ fromIntegral configDbPoolMaxIdletime - , SQL.staticConnectionSettings (toUtf8 $ addFallbackAppName prettyVersion configDbUri) + , SQL.staticConnectionSettings [ + SQL.connection $ SQL.string (addFallbackAppName prettyVersion configDbUri), + SQL.usePreparedStatements configDbPreparedStatements + ] , SQL.observationHandler $ observer . HasqlPoolObs ] @@ -403,8 +408,7 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea qSchemaCache = do conf@AppConfig{..} <- getConfig appState (resultTime, result) <- - let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in - timeItT $ usePool appState (transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) + timeItT $ usePool appState (SQL.transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) case result of Left e -> do putSCacheStatus appState SCPending diff --git a/src/PostgREST/CLI.hs b/src/PostgREST/CLI.hs index d42ed594f4..4a58b2425b 100644 --- a/src/PostgREST/CLI.hs +++ b/src/PostgREST/CLI.hs @@ -53,10 +53,7 @@ main CLI{cliCommand, cliPath} = do dumpSchema :: AppState -> IO LBS.ByteString dumpSchema appState = do conf@AppConfig{..} <- AppState.getConfig appState - result <- - let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in - AppState.usePool appState - (transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) + result <- AppState.usePool appState (SQL.transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) case result of Left e -> do let observer = AppState.getObserver appState diff --git a/src/PostgREST/Config/Database.hs b/src/PostgREST/Config/Database.hs index aff4b5b8a3..865d2fbd71 100644 --- a/src/PostgREST/Config/Database.hs +++ b/src/PostgREST/Config/Database.hs @@ -92,8 +92,7 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow -- A setting on the database only will have no effect: ALTER DATABASE postgres SET jwt_aud = 'xx' queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)] queryDbSettings preConfFunc prepared = - let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in - transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared + SQL.transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared where sql = encodeUtf8 [trimming| WITH @@ -133,8 +132,7 @@ queryDbSettings preConfFunc prepared = queryRoleSettings :: PgVersion -> Bool -> Session (RoleSettings, RoleIsolationLvl) queryRoleSettings pgVer prepared = - let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in - transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared + SQL.transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared where sql = encodeUtf8 [trimming| with diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs index 0c5c42d4eb..a6bf14bca9 100644 --- a/src/PostgREST/Listener.hs +++ b/src/PostgREST/Listener.hs @@ -5,12 +5,15 @@ module PostgREST.Listener (runListener) where import qualified Data.ByteString.Char8 as BS -import qualified Hasql.Connection as SQL -import qualified Hasql.Notifications as SQL -import PostgREST.AppState (AppState, getConfig) -import PostgREST.Config (AppConfig (..)) -import PostgREST.Observation (Observation (..)) -import PostgREST.Version (prettyVersion) +import qualified Hasql.Connection as SQL +import qualified Hasql.Connection.Setting as SQL +import qualified Hasql.Connection.Setting.Connection as SQL +import qualified Hasql.Notifications as SQL +import PostgREST.AppState (AppState, + getConfig) +import PostgREST.Config (AppConfig (..)) +import PostgREST.Observation (Observation (..)) +import PostgREST.Version (prettyVersion) import qualified PostgREST.AppState as AppState import qualified PostgREST.Config as Config @@ -29,6 +32,7 @@ retryingListen :: AppState -> IO () retryingListen appState = do AppConfig{..} <- AppState.getConfig appState let + connectionString = Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri dbChannel = toS configDbChannel handleFinally err = do AppState.putIsListenerOn appState False @@ -46,7 +50,7 @@ retryingListen appState = do -- forkFinally allows to detect if the thread dies void . flip forkFinally handleFinally $ do - dbOrError <- SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri) + dbOrError <- SQL.acquire [ SQL.connection $ SQL.string connectionString ] case dbOrError of Right db -> do SQL.listen db $ SQL.toPgIdentifier dbChannel diff --git a/src/PostgREST/Query.hs b/src/PostgREST/Query.hs index 1d0d12511d..a460e36c9c 100644 --- a/src/PostgREST/Query.hs +++ b/src/PostgREST/Query.hs @@ -76,10 +76,8 @@ data QueryResult query :: AppConfig -> AuthResult -> ApiRequest -> ActionPlan -> SchemaCache -> Query query _ _ _ (NoDb x) _ = NoDbQuery $ NoDbResult x query config AuthResult{..} apiReq (Db plan) sCache = - DbQuery isoLvl txMode dbHandler transaction mainSQLQuery + DbQuery isoLvl txMode dbHandler SQL.transaction mainSQLQuery where - transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction - prepared = configDbPreparedStatements config isoLvl = planIsoLvl config authRole plan txMode = planTxMode plan (mainActionQuery, mainSQLQuery) = actionQuery plan config apiReq sCache diff --git a/stack.yaml b/stack.yaml index 856dde4b46..d54ca81d48 100644 --- a/stack.yaml +++ b/stack.yaml @@ -10,13 +10,15 @@ nix: extra-deps: - fuzzyset-0.2.4 - - hasql-1.8.1.4 + - hasql-1.9.1.2 - hasql-dynamic-statements-0.3.1.8 - hasql-implicits-0.2.0.1 - - hasql-notifications-0.2.3.2 - - hasql-pool-1.2.0.3 - - hasql-transaction-1.1.1.2 + - hasql-notifications-0.2.4.0 + - hasql-pool-1.3.0.1 + - hasql-transaction-1.2.0.1 - jose-jwt-0.10.0 - postgresql-binary-0.14.0.1 - postgresql-libpq-0.11.0.0 - postgresql-libpq-configure-0.11 + - text-builder-1.0.0.4 + - witherable-0.4.2 diff --git a/test/spec/Main.hs b/test/spec/Main.hs index e847926b6f..3045aa09a4 100644 --- a/test/spec/Main.hs +++ b/test/spec/Main.hs @@ -1,8 +1,10 @@ module Main where -import qualified Hasql.Pool as P -import qualified Hasql.Pool.Config as P -import qualified Hasql.Transaction.Sessions as HT +import qualified Hasql.Connection.Setting as C +import qualified Hasql.Connection.Setting.Connection as C +import qualified Hasql.Pool as P +import qualified Hasql.Pool.Config as P +import qualified Hasql.Transaction.Sessions as HT import Data.Function (id) @@ -77,7 +79,7 @@ main = do , P.acquisitionTimeout 10 , P.agingTimeout 60 , P.idlenessTimeout 60 - , P.staticConnectionSettings (toUtf8 $ configDbUri testCfg) + , P.staticConnectionSettings [ C.connection $ C.string $ configDbUri testCfg ] ] actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False) From b3569e577ee04f0d4c661882546d7deda4f37b5a Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Tue, 12 Aug 2025 20:47:35 -0500 Subject: [PATCH 5/6] chore(deps): upgrade to hasql-transactions 1.2.1 --- nix/overlays/haskell-packages.nix | 8 +++++++- postgrest.cabal | 4 ++-- stack.yaml | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/nix/overlays/haskell-packages.nix b/nix/overlays/haskell-packages.nix index 4ddbcb9b55..90609380ce 100644 --- a/nix/overlays/haskell-packages.nix +++ b/nix/overlays/haskell-packages.nix @@ -59,7 +59,13 @@ let sha256 = "sha256-5NsF0WyiZuqkZemlQfA/J7rAJttkE56oPJK4zgqMbZ4="; } { }); - hasql-transaction = lib.dontCheck prev.hasql-transaction_1_2_0_1; + hasql-transaction = lib.dontCheck (prev.callHackageDirect + { + pkg = "hasql-transaction"; + ver = "1.2.1"; + sha256 = "sha256-7Q7gt5ts4OoGU58dp6PJFZmVjfwjozANHNg2u1PJf6Q="; + } + { }); # Needed for hasql 1.9 text-builder = lib.dontCheck prev.text-builder_1_0_0_3; diff --git a/postgrest.cabal b/postgrest.cabal index 61069d2045..f3465392e1 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -114,7 +114,7 @@ library , hasql-dynamic-statements >= 0.3.1 && < 0.4 , hasql-notifications >= 0.2.4.0 && < 0.3 , hasql-pool >= 1.3.0.1 && < 1.4 - , hasql-transaction >= 1.2.0.1 && < 1.3 + , hasql-transaction >= 1.2.1 && < 1.3 , heredoc >= 0.2 && < 0.3 , http-types >= 0.12.2 && < 0.13 , insert-ordered-containers >= 0.2.2 && < 0.3 @@ -265,7 +265,7 @@ test-suite spec , containers >= 0.5.7 && < 0.7 , hasql >= 1.9.1.2 && < 1.10 , hasql-pool >= 1.3.0.1 && < 1.4 - , hasql-transaction >= 1.2.0.1 && < 1.3 + , hasql-transaction >= 1.2.1 && < 1.3 , heredoc >= 0.2 && < 0.3 , hspec >= 2.3 && < 2.12 , hspec-expectations >= 0.8.4 && < 0.9 diff --git a/stack.yaml b/stack.yaml index d54ca81d48..f8d44d614f 100644 --- a/stack.yaml +++ b/stack.yaml @@ -15,7 +15,7 @@ extra-deps: - hasql-implicits-0.2.0.1 - hasql-notifications-0.2.4.0 - hasql-pool-1.3.0.1 - - hasql-transaction-1.2.0.1 + - hasql-transaction-1.2.1 - jose-jwt-0.10.0 - postgresql-binary-0.14.0.1 - postgresql-libpq-0.11.0.0 From 23426d2e2f210127a9e3d331edce563885bec1f7 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Tue, 12 Aug 2025 20:48:30 -0500 Subject: [PATCH 6/6] fix: retrying transaction on 40001 errors --- nix/tools/withTools.nix | 4 ++-- src/PostgREST/AppState.hs | 2 +- src/PostgREST/CLI.hs | 2 +- src/PostgREST/Config/Database.hs | 4 ++-- src/PostgREST/Query.hs | 2 +- test/io/replica.sql | 4 ++++ test/io/test_replica.py | 32 ++++++++++++++++++++++++++++++++ 7 files changed, 43 insertions(+), 7 deletions(-) diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 82de1e4d6e..c68d7ec4aa 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -85,7 +85,7 @@ let >&2 echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log" if test "$_arg_replica" = "on"; then - replica_slot="replica_$RANDOM" + replica_slot="rr_$RANDOM" replica_dir="$tmpdir/$replica_slot" replica_host="$tmpdir/socket_$replica_slot" @@ -100,7 +100,7 @@ let log "Starting replica on $replica_host" - pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" " \ + pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\"" \ >> "$setuplog" >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres" diff --git a/src/PostgREST/AppState.hs b/src/PostgREST/AppState.hs index 516ad16be6..d1cf442bf8 100644 --- a/src/PostgREST/AppState.hs +++ b/src/PostgREST/AppState.hs @@ -408,7 +408,7 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea qSchemaCache = do conf@AppConfig{..} <- getConfig appState (resultTime, result) <- - timeItT $ usePool appState (SQL.transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) + timeItT $ usePool appState (SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ querySchemaCache conf) case result of Left e -> do putSCacheStatus appState SCPending diff --git a/src/PostgREST/CLI.hs b/src/PostgREST/CLI.hs index 4a58b2425b..0338ea8a21 100644 --- a/src/PostgREST/CLI.hs +++ b/src/PostgREST/CLI.hs @@ -53,7 +53,7 @@ main CLI{cliCommand, cliPath} = do dumpSchema :: AppState -> IO LBS.ByteString dumpSchema appState = do conf@AppConfig{..} <- AppState.getConfig appState - result <- AppState.usePool appState (SQL.transaction SQL.ReadCommitted SQL.Read $ querySchemaCache conf) + result <- AppState.usePool appState (SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ querySchemaCache conf) case result of Left e -> do let observer = AppState.getObserver appState diff --git a/src/PostgREST/Config/Database.hs b/src/PostgREST/Config/Database.hs index 865d2fbd71..712baeaf1d 100644 --- a/src/PostgREST/Config/Database.hs +++ b/src/PostgREST/Config/Database.hs @@ -92,7 +92,7 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow -- A setting on the database only will have no effect: ALTER DATABASE postgres SET jwt_aud = 'xx' queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)] queryDbSettings preConfFunc prepared = - SQL.transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared + SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared where sql = encodeUtf8 [trimming| WITH @@ -132,7 +132,7 @@ queryDbSettings preConfFunc prepared = queryRoleSettings :: PgVersion -> Bool -> Session (RoleSettings, RoleIsolationLvl) queryRoleSettings pgVer prepared = - SQL.transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared + SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared where sql = encodeUtf8 [trimming| with diff --git a/src/PostgREST/Query.hs b/src/PostgREST/Query.hs index a460e36c9c..4dc4f447f7 100644 --- a/src/PostgREST/Query.hs +++ b/src/PostgREST/Query.hs @@ -76,7 +76,7 @@ data QueryResult query :: AppConfig -> AuthResult -> ApiRequest -> ActionPlan -> SchemaCache -> Query query _ _ _ (NoDb x) _ = NoDbQuery $ NoDbResult x query config AuthResult{..} apiReq (Db plan) sCache = - DbQuery isoLvl txMode dbHandler SQL.transaction mainSQLQuery + DbQuery isoLvl txMode dbHandler SQL.transactionNoRetry mainSQLQuery where isoLvl = planIsoLvl config authRole plan txMode = planTxMode plan diff --git a/test/io/replica.sql b/test/io/replica.sql index deffbec069..595f1df033 100644 --- a/test/io/replica.sql +++ b/test/io/replica.sql @@ -10,6 +10,10 @@ $$ language sql; create table replica.items as select x as id from generate_series(1, 10) x; +create table replica.conflict as select x as id from generate_series(1, 1000000) x; + +create view replica.conflict_view as select * from replica.conflict where (pg_sleep(0.01) is not null); + DROP ROLE IF EXISTS postgrest_test_anonymous; CREATE ROLE postgrest_test_anonymous; diff --git a/test/io/test_replica.py b/test/io/test_replica.py index 24b996253e..c0dedd75e8 100644 --- a/test/io/test_replica.py +++ b/test/io/test_replica.py @@ -1,6 +1,8 @@ "IO tests for PostgREST started on replicas" +import os import pytest +import time from config import * from util import * @@ -26,3 +28,33 @@ def test_sanity_replica(replicaenv): response = postgrest.session.get("/items?select=count") assert response.text == '[{"count":10}]' + + +def test_conflict_replica(replicaenv): + "Test that PostgREST does not retry the transaction on conflict with recovery (PG error code 40001)" + + with run(env=replicaenv["replica"]) as postgrest: + + def conflict(): + response = postgrest.session.get("/conflict_view") + # Checks that the transaction stops and returns the 40001 error instead of retrying + assert response.json()["code"] == "40001" + assert response.status_code == 500 + + t = Thread(target=conflict) + t.start() + + # make sure the request has started + time.sleep(0.1) + + prienv = replicaenv["primary"] + connopts = f'-d {prienv["PGDATABASE"]} -U postgres -h {prienv["PGHOST"]}' + + # Delete the table data while the request with the lock is running to trigger the recovery conflict + os.system( + f'psql {connopts} --set ON_ERROR_STOP=1 -a -c "DELETE FROM replica.conflict;"' + ) + # Vacuum the table to accelerate the process + os.system(f"vacuumdb {connopts} -t replica.conflict") + + t.join()