fmt

clippy
Add checkout_failure_limit config/feature
2026-03-23 01:16:30 +00:00 · 2025-02-27 07:26:17 -06:00 · 2025-02-27 07:24:23 -06:00 · 2025-02-27 07:17:10 -06:00 · 2025-01-09 11:59:42 -08:00 · 2024-12-22 05:23:57 -06:00
46 changed files with 4609 additions and 770 deletions
--- a/.circleci/pgcat.toml
+++ b/.circleci/pgcat.toml
@@ -59,6 +59,7 @@ admin_password = "admin_pass"
 # session: one server connection per connected client
 # transaction: one server connection per client transaction
 pool_mode = "transaction"
+prepared_statements_cache_size = 500

 # If the client doesn't specify, route traffic to
 # this role by default.
@@ -141,6 +142,7 @@ query_parser_enabled = true
 query_parser_read_write_splitting = true
 primary_reads_enabled = true
 sharding_function = "pg_bigint_hash"
+prepared_statements_cache_size = 500

 [pools.simple_db.users.0]
 username = "simple_user"
--- a/.circleci/run_tests.sh
+++ b/.circleci/run_tests.sh
@@ -26,6 +26,7 @@ PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i
 PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i

 # Start Toxiproxy
+kill -9 $(pgrep toxiproxy) || true
 LOG_LEVEL=error toxiproxy-server &
 sleep 1

@@ -106,7 +107,7 @@ cd ../..
 # These tests will start and stop the pgcat server so it will need to be restarted after the tests
 #
 pip3 install -r tests/python/requirements.txt
-python3 tests/python/tests.py || exit 1
+pytest || exit 1


 #
@@ -177,3 +178,6 @@ killall pgcat -s SIGINT

 # Allow for graceful shutdown
 sleep 1
+
+kill -9 $(pgrep toxiproxy)
+sleep 1
--- a/.github/workflows/build-and-push.yaml
+++ b/.github/workflows/build-and-push.yaml
@@ -23,14 +23,17 @@ jobs:

    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3

      - name: Determine tags
        id: metadata
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
        with:
          images: ${{ env.registry }}/${{ env.image-name }}
          tags: |
@@ -42,15 +45,18 @@ jobs:
            type=raw,value=latest,enable={{ is_default_branch }}

      - name: Log in to the Container registry
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v3
        with:
          registry: ${{ env.registry }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push ${{ env.image-name }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          provenance: false
          push: true
          tags: ${{ steps.metadata.outputs.tags }}
          labels: ${{ steps.metadata.outputs.labels }}
--- a/.github/workflows/chart-lint-test.yaml
+++ b/.github/workflows/chart-lint-test.yaml
@@ -22,7 +22,7 @@ jobs:
      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
      # yamllint (https://github.com/adrienverge/yamllint) which require Python
      - name: Set up Python
-        uses: actions/setup-python@v4.1.0
+        uses: actions/setup-python@v5.1.0
        with:
          python-version: 3.7

@@ -43,7 +43,7 @@ jobs:
        run: ct lint --config ct.yaml

      - name: Create kind cluster
-        uses: helm/kind-action@v1.7.0
+        uses: helm/kind-action@v1.10.0
        if: steps.list-changed.outputs.changed == 'true'

      - name: Run chart-testing (install)
--- a/.github/workflows/chart-release.yaml
+++ b/.github/workflows/chart-release.yaml
@@ -32,7 +32,7 @@ jobs:
          version: v3.13.0

      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@be16258da8010256c6e82849661221415f031968 # v1.5.0
+        uses: helm/chart-releaser-action@a917fd15b20e8b64b94d9158ad54cd6345335584 # v1.6.0
        with:
          charts_dir: charts
          config: cr.yaml
--- a/.github/workflows/publish-deb-package.yml
+++ b/.github/workflows/publish-deb-package.yml
@@ -1,6 +1,9 @@
 name: pgcat package (deb)

 on:
+  push:
+    tags:
+      - v*
  workflow_dispatch:
    inputs:
      packageVersion:
@@ -16,6 +19,14 @@ jobs:
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
+    - name: Set package version
+      if: github.event_name == 'push'  # For push event
+      run: |
+        TAG=${{ github.ref_name }}
+        echo "packageVersion=${TAG#v}" >> "$GITHUB_ENV"
+    - name: Set package version (manual dispatch)
+      if: github.event_name == 'workflow_dispatch'  # For manual dispatch
+      run: echo "packageVersion=${{ github.event.inputs.packageVersion }}" >> "$GITHUB_ENV"
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
@@ -39,10 +50,10 @@ jobs:
          export ARCH=arm64
        fi

-        bash utilities/deb.sh ${{ inputs.packageVersion }}
+        bash utilities/deb.sh ${{ env.packageVersion }}

        deb-s3 upload \
          --lock \
          --bucket apt.postgresml.org \
-          pgcat-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \
+          pgcat-${{ env.packageVersion }}-ubuntu22.04-${ARCH}.deb \
          --codename $(lsb_release -cs)
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,6 @@ lcov.info
 dev/.bash_history
 dev/cache
 !dev/cache/.keepme
-.venv
+.venv
+**/__pycache__
+.bundle
--- a/CONFIG.md
+++ b/CONFIG.md
@@ -36,10 +36,11 @@ Port at which prometheus exporter listens on.
 ### connect_timeout
 ```
 path: general.connect_timeout
-default: 5000 # milliseconds
+default: 1000 # milliseconds
 ```

-How long to wait before aborting a server connection (ms).
+How long the client waits to obtain a server connection before aborting (ms).
+This is similar to PgBouncer's `query_wait_timeout`.

 ### idle_timeout
 ```
@@ -297,6 +298,19 @@ Load balancing mode
 `random` selects the server at random
 `loc` selects the server with the least outstanding busy connections

+### checkout_failure_limit
+```
+path: pools.<pool_name>.checkout_failure_limit
+default: 0 (disabled)
+```
+
+`Maximum number of checkout failures a client is allowed before it
+gets disconnected. This is needed to prevent persistent client/server
+imbalance in high availability setups where multiple PgCat instances are placed
+behind a single load balancer. If for any reason a client lands on a PgCat instance that has 
+a large number of connected clients, it might get stuck in perpetual checkout failure loop especially
+in session mode
+`
 ### default_role
 ```
 path: pools.<pool_name>.default_role
@@ -308,6 +322,45 @@ If the client doesn't specify, PgCat routes traffic to this role by default.
 `replica` round-robin between replicas only without touching the primary,
 `primary` all queries go to the primary unless otherwise specified.

+### db_activity_based_routing
+```
+path: pools.<pool_name>.db_activity_based_routing
+default: false
+```
+
+If enabled, PgCat will route queries to the primary if the queried table was recently written to.
+Only relevant when `query_parser_enabled` *and* `query_parser_read_write_splitting` is enabled.
+
+##### Considerations:
+- *This feature is experimental and may not work as expected.*
+- This feature only works when the same PgCat instance is used for both reads and writes to the database.
+- This feature is not relevant when the primary is not part of the pool of databases used for load balancing of read queries.
+- If more than one PgCat instance is used for HA purposes, this feature will not work as expected. A way to still make it work is by using sticky sessions.
+
+### db_activity_based_ms_init_delay
+```
+path: pools.<pool_name>.db_activity_based_ms_init_delay
+default: 100
+```
+
+The delay in milliseconds before the first activity-based routing check is performed.
+
+### db_activity_ttl
+```
+path: pools.<pool_name>.db_activity_ttl
+default: 900
+```
+
+The time in seconds after which a DB is considered inactive when no queries/updates are performed to it.
+
+### table_mutation_cache_ms_ttl
+```
+path: pools.<pool_name>.table_mutation_cache_ms_ttl
+default: 50
+```
+
+The time in milliseconds after a write to a table that all queries to that table will be routed to the primary.
+
 ### prepared_statements_cache_size
 ```
 path: general.prepared_statements_cache_size
@@ -462,10 +515,18 @@ path: pools.<pool_name>.users.<user_index>.pool_size
 default: 9
 ```

-Maximum number of server connections that can be established for this user
+Maximum number of server connections that can be established for this user.
 The maximum number of connection from a single Pgcat process to any database in the cluster
 is the sum of pool_size across all users.

+### min_pool_size
+```
+path: pools.<pool_name>.users.<user_index>.min_pool_size
+default: 0
+```
+
+Minimum number of idle server connections to retain for this pool.
+
 ### statement_timeout
 ```
 path: pools.<pool_name>.users.<user_index>.statement_timeout
@@ -475,6 +536,16 @@ default: 0
 Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way.
 0 means it is disabled.

+### connect_timeout
+```
+path: pools.<pool_name>.users.<user_index>.connect_timeout
+default: <UNSET> # milliseconds
+```
+
+How long the client waits to obtain a server connection before aborting (ms).
+This is similar to PgBouncer's `query_wait_timeout`.
+If unset, uses the `connect_timeout` defined globally.
+
 ## `pools.<pool_name>.shards.<shard_index>` Section

 ### servers
@@ -502,4 +573,3 @@ default: "shard0"
 ```

 Database name (e.g. "postgres")
-
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,6 +6,32 @@ Thank you for contributing! Just a few tips here:
 2. Run the test suite (e.g. `pgbench`) to make sure everything still works. The tests are in `.circleci/run_tests.sh`.
 3. Performance is important, make sure there are no regressions in your branch vs. `main`.

+## How to run the integration tests locally and iterate on them
+We have integration tests written in Ruby, Python, Go and Rust.
+Below are the steps to run them in a developer-friendly way that allows iterating and quick turnaround.
+Hear me out, this should be easy, it will involve opening a shell into a container with all the necessary dependancies available for you and you can modify the test code and immediately rerun your test in the interactive shell.
+
+
+Quite simply, make sure you have docker installed and then run
+`./start_test_env.sh`
+
+That is it!
+
+Within this test environment you can modify the file in your favorite IDE and rerun the tests without having to bootstrap the entire environment again.
+
+Once the environment is ready, you can run the tests by running
+Ruby:   `cd /app/tests/ruby && bundle exec ruby <test_name>.rb --format documentation`
+Python: `cd /app/ && pytest`
+Rust:   `cd /app/tests/rust && cargo run`
+Go:     `cd /app/tests/go && /usr/local/go/bin/go test`
+
+You can also rebuild PgCat directly within the environment and the tests will run against the newly built binary
+To rebuild PgCat, just run `cargo build` within the container under `/app`
+
+![Animated gif showing how to run tests](https://github.com/user-attachments/assets/2258fde3-2aed-4efb-bdc5-e4f12dcd4d33)
+
+
+
 Happy hacking!

 ## TODOs
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -132,7 +132,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -143,9 +143,15 @@ checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
 [[package]]
 name = "atomic_enum"
 version = "0.2.0"
@@ -186,12 +192,11 @@ checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"

 [[package]]
 name = "bb8"
-version = "0.8.1"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2"
+checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8"
 dependencies = [
 "async-trait",
- "futures-channel",
 "futures-util",
 "parking_lot",
 "tokio",
@@ -224,6 +229,12 @@ version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"

+[[package]]
+name = "bytecount"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -236,6 +247,37 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"

+[[package]]
+name = "camino"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -295,7 +337,7 @@ dependencies = [
 "heck",
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -325,6 +367,21 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "crossbeam-channel"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -335,6 +392,19 @@ dependencies = [
 "typenum",
 ]

+[[package]]
+name = "dashmap"
+version = "5.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.4.0"
@@ -397,6 +467,15 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "error-chain"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "exitcode"
 version = "1.1.2"
@@ -409,6 +488,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"

+[[package]]
+name = "fastrand"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -480,7 +565,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -541,30 +626,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"

 [[package]]
-name = "h2"
-version = "0.3.20"
+name = "glob"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
+[[package]]
+name = "h2"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
 dependencies = [
+ "atomic-waker",
 "bytes",
 "fnv",
 "futures-core",
 "futures-sink",
- "futures-util",
 "http",
- "indexmap 1.9.3",
+ "indexmap",
 "slab",
 "tokio",
 "tokio-util",
 "tracing",
 ]

-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
 [[package]]
 name = "hashbrown"
 version = "0.14.0"
@@ -609,9 +694,9 @@ dependencies = [

 [[package]]
 name = "http"
-version = "0.2.9"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
 dependencies = [
 "bytes",
 "fnv",
@@ -620,12 +705,24 @@ dependencies = [

 [[package]]
 name = "http-body"
-version = "0.4.5"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
 "bytes",
 "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
 "pin-project-lite",
 ]

@@ -643,13 +740,12 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"

 [[package]]
 name = "hyper"
-version = "0.14.27"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
 dependencies = [
 "bytes",
 "futures-channel",
- "futures-core",
 "futures-util",
 "h2",
 "http",
@@ -658,13 +754,26 @@ dependencies = [
 "httpdate",
 "itoa",
 "pin-project-lite",
- "socket2 0.4.9",
+ "smallvec",
 "tokio",
- "tower-service",
- "tracing",
 "want",
 ]

+[[package]]
+name = "hyper-util"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.57"
@@ -709,16 +818,6 @@ dependencies = [
 "unicode-normalization",
 ]

-[[package]]
-name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
-dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
-]
-
 [[package]]
 name = "indexmap"
 version = "2.0.0"
@@ -726,7 +825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
 "equivalent",
- "hashbrown 0.14.0",
+ "hashbrown",
 ]

 [[package]]
@@ -810,9 +909,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"

 [[package]]
 name = "linked-hash-map"
@@ -848,7 +947,7 @@ version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60"
 dependencies = [
- "hashbrown 0.14.0",
+ "hashbrown",
 ]

 [[package]]
@@ -905,6 +1004,21 @@ dependencies = [
 "autocfg",
 ]

+[[package]]
+name = "mini-moka"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c325dfab65f261f386debee8b0969da215b3fa0037e74c8a1234db7ba986d803"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "dashmap",
+ "skeptic",
+ "smallvec",
+ "tagptr",
+ "triomphe",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -916,9 +1030,9 @@ dependencies = [

 [[package]]
 name = "mio"
-version = "0.8.11"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
 dependencies = [
 "libc",
 "wasi 0.11.0+wasi-snapshot-preview1",
@@ -979,9 +1093,9 @@ dependencies = [

 [[package]]
 name = "once_cell"
-version = "1.18.0"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"

 [[package]]
 name = "overload"
@@ -1020,7 +1134,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"

 [[package]]
 name = "pgcat"
-version = "1.1.2-dev4"
+version = "1.3.0"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1034,12 +1148,15 @@ dependencies = [
 "fallible-iterator",
 "futures",
 "hmac",
+ "http-body-util",
 "hyper",
+ "hyper-util",
 "itertools",
 "jemallocator",
 "log",
 "lru",
 "md-5",
+ "mini-moka",
 "nix",
 "num_cpus",
 "once_cell",
@@ -1054,6 +1171,7 @@ dependencies = [
 "serde",
 "serde_derive",
 "serde_json",
+ "serial_test",
 "sha-1",
 "sha2",
 "socket2 0.4.9",
@@ -1099,7 +1217,7 @@ dependencies = [
 "phf_shared",
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1128,7 +1246,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1169,13 +1287,24 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"

 [[package]]
 name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
 dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "pulldown-cmark"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
+dependencies = [
+ "bitflags 2.3.3",
+ "memchr",
+ "unicase",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -1184,9 +1313,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"

 [[package]]
 name = "quote"
-version = "1.0.31"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
 "proc-macro2",
 ]
@@ -1365,6 +1494,24 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"

+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "scc"
+version = "2.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d25269dd3a12467afe2e510f69fb0b46b698e5afb296b59f2145259deaf8e8"
+dependencies = [
+ "sdd",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -1382,23 +1529,38 @@ dependencies = [
 ]

 [[package]]
-name = "serde"
-version = "1.0.171"
+name = "sdd"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95"
+
+[[package]]
+name = "semver"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.214"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.214"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1421,6 +1583,31 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "serial_test"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d"
+dependencies = [
+ "futures",
+ "log",
+ "once_cell",
+ "parking_lot",
+ "scc",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.10.1"
@@ -1467,6 +1654,21 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"

+[[package]]
+name = "skeptic"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
+dependencies = [
+ "bytecount",
+ "cargo_metadata",
+ "error-chain",
+ "glob",
+ "pulldown-cmark",
+ "tempfile",
+ "walkdir",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.8"
@@ -1478,9 +1680,9 @@ dependencies = [

 [[package]]
 name = "smallvec"
-version = "1.11.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"

 [[package]]
 name = "socket2"
@@ -1510,9 +1712,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"

 [[package]]
 name = "sqlparser"
-version = "0.34.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59"
+checksum = "9a875d8cd437cc8a97e9aeaeea352ec9a19aea99c23e9effb17757291de80b08"
 dependencies = [
 "log",
 "sqlparser_derive",
@@ -1520,13 +1722,13 @@ dependencies = [

 [[package]]
 name = "sqlparser_derive"
-version = "0.1.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e"
+checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 1.0.109",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1570,15 +1772,34 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.26"
+version = "2.0.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970"
+checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
 dependencies = [
 "proc-macro2",
 "quote",
 "unicode-ident",
 ]

+[[package]]
+name = "tagptr"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
+
+[[package]]
+name = "tempfile"
+version = "3.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "redox_syscall",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.43"
@@ -1596,7 +1817,7 @@ checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1663,7 +1884,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1741,19 +1962,13 @@ version = "0.19.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap",
 "serde",
 "serde_spanned",
 "toml_datetime",
 "winnow",
 ]

-[[package]]
-name = "tower-service"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
-
 [[package]]
 name = "tracing"
 version = "0.1.37"
@@ -1774,7 +1989,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 ]

 [[package]]
@@ -1829,6 +2044,12 @@ dependencies = [
 "tracing-serde",
 ]

+[[package]]
+name = "triomphe"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3"
+
 [[package]]
 name = "trust-dns-proto"
 version = "0.22.0"
@@ -1886,6 +2107,12 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"

+[[package]]
+name = "unicase"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df"
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.13"
@@ -1942,6 +2169,16 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -1984,7 +2221,7 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 "wasm-bindgen-shared",
 ]

@@ -2006,7 +2243,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.26",
+ "syn 2.0.87",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
@@ -2058,6 +2295,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"

+[[package]]
+name = "winapi-util"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pgcat"
-version = "1.1.2-dev4"
+version = "1.3.0"
 edition = "2021"

 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -8,7 +8,7 @@ edition = "2021"
 tokio = { version = "1", features = ["full"] }
 bytes = "1"
 md-5 = "0.10"
-bb8 = "0.8.1"
+bb8 = "=0.8.6"
 async-trait = "0.1"
 rand = "0.8"
 chrono = "0.4"
@@ -19,7 +19,7 @@ serde_derive = "1"
 regex = "1"
 num_cpus = "1"
 once_cell = "1"
-sqlparser = {version = "0.34", features = ["visitor"] }
+sqlparser = { version = "0.52", features = ["visitor"] }
 log = "0.4"
 arc-swap = "1"
 parking_lot = "0.12.1"
@@ -29,7 +29,9 @@ base64 = "0.21"
 stringprep = "0.1"
 tokio-rustls = "0.24"
 rustls-pemfile = "1"
-hyper = { version = "0.14", features = ["full"] }
+http-body-util = "0.1.2"
+hyper = { version = "1.4.1", features = ["full"] }
+hyper-util = { version = "0.1.7", features = ["tokio"] }
 phf = { version = "0.11.1", features = ["macros"] }
 exitcode = "1.1.2"
 futures = "0.3"
@@ -47,9 +49,16 @@ serde_json = "1"
 itertools = "0.10"
 clap = { version = "4.3.1", features = ["derive", "env"] }
 tracing = "0.1.37"
-tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter", "std"]}
+tracing-subscriber = { version = "0.3.17", features = [
+    "json",
+    "env-filter",
+    "std",
+] }
 lru = "0.12.0"
+mini-moka = "0.10.3"

 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 jemallocator = "0.5.0"

+[dev-dependencies]
+serial_test = "*"
--- a/3
+++ b/3
@@ -1,4 +1,4 @@
-FROM rust:1-slim-bookworm AS builder
+FROM rust:1.79.0-slim-bookworm AS builder

 RUN apt-get update && \
    apt-get install -y build-essential
@@ -19,3 +19,4 @@ COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml
 WORKDIR /etc/pgcat
 ENV RUST_LOG=info
 CMD ["pgcat"]
+STOPSIGNAL SIGINT
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1,4 +1,4 @@
-FROM cimg/rust:1.67.1
+FROM cimg/rust:1.79.0
 COPY --from=sclevine/yj /bin/yj /bin/yj
 RUN /bin/yj -h
 RUN sudo apt-get update && \
--- a/README.md
+++ b/README.md
@@ -231,7 +231,7 @@ User.find_by_email("test@example.com")
 ```sql
 -- Grab a bunch of users from shard 1
 SET SHARD TO '1';
-SELECT * FROM users LIMT 10;
+SELECT * FROM users LIMIT 10;

 -- Find by id
 SET SHARDING KEY TO '1234';
@@ -268,6 +268,8 @@ psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES'

 Additionally, Prometheus statistics are available at `/metrics` via HTTP.

+We also have a [basic Grafana dashboard](https://github.com/postgresml/pgcat/blob/main/grafana_dashboard.json) based on Prometheus metrics that you can import into Grafana and build on it or use it for monitoring.
+
 ### Live configuration reloading

 The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. All settings except the `host` and `port` can be reloaded without restarting the pooler, including sharding and replicas configurations.
--- a/charts/pgcat/Chart.yaml
+++ b/charts/pgcat/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: pgcat
 description: A Helm chart for PgCat a PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring.
 maintainers:
-  - name: Wildcard
-    email: support@w6d.io
-appVersion: "1.1.1"
-version: 0.1.0
+  - name: PostgresML
+    email: team@postgresml.org
+appVersion: "1.3.0"
+version: 0.2.5
--- a/charts/pgcat/templates/secret.yaml
+++ b/charts/pgcat/templates/secret.yaml
@@ -15,6 +15,7 @@ stringData:
    connect_timeout = {{ .Values.configuration.general.connect_timeout }}
    idle_timeout = {{ .Values.configuration.general.idle_timeout | int }}
    server_lifetime = {{ .Values.configuration.general.server_lifetime | int }}
+    server_tls = {{ .Values.configuration.general.server_tls }}
    idle_client_in_transaction_timeout = {{ .Values.configuration.general.idle_client_in_transaction_timeout | int }}
    healthcheck_timeout = {{ .Values.configuration.general.healthcheck_timeout }}
    healthcheck_delay = {{ .Values.configuration.general.healthcheck_delay }}
@@ -50,6 +51,10 @@ stringData:
    query_parser_enabled = {{ default true $pool.query_parser_enabled }}
    query_parser_read_write_splitting = {{ default true $pool.query_parser_read_write_splitting }}
    primary_reads_enabled = {{ default true $pool.primary_reads_enabled }}
+    db_activity_based_routing = {{ default false $pool.db_activity_based_routing }}
+    db_activity_based_ms_init_delay = {{ default 100 $pool.db_activity_based_ms_init_delay }}
+    db_activity_ttl = {{ default 900 $pool.db_activity_ttl }}
+    table_mutation_cache_ttl = {{ default 50 $pool.table_mutation_cache_ttl }}
    sharding_function = {{ default "pg_bigint_hash" $pool.sharding_function | quote }}

    {{-   range $index, $user := $pool.users }}
@@ -58,11 +63,21 @@ stringData:
    ##
    [pools.{{ $pool.name | quote }}.users.{{ $index }}]
    username = {{ $user.username | quote }}
+    {{- if $user.password }}
    password = {{ $user.password | quote }}
+    {{- else if and $user.passwordSecret.name $user.passwordSecret.key }}
+    {{- $secret := (lookup "v1" "Secret" $.Release.Namespace $user.passwordSecret.name) }}
+    {{- if $secret }}
+    {{- $password := index $secret.data $user.passwordSecret.key | b64dec }}
+    password = {{ $password | quote }}
+    {{- end }}
+    {{- end }}
    pool_size = {{ $user.pool_size }}
-    statement_timeout = {{ $user.statement_timeout }}
-    min_pool_size = 3
-    server_lifetime = 60000
+    statement_timeout = {{ default 0 $user.statement_timeout }}
+    min_pool_size = {{ default 3 $user.min_pool_size }}
+    {{- if $user.server_lifetime }}
+    server_lifetime = {{ $user.server_lifetime }}
+    {{- end }}
    {{-     if and $user.server_username $user.server_password }}
    server_username = {{ $user.server_username | quote }}
    server_password = {{ $user.server_password | quote }}
--- a/charts/pgcat/values.yaml
+++ b/charts/pgcat/values.yaml
@@ -170,13 +170,16 @@ configuration:
    connect_timeout: 5000

    # How long an idle connection with a server is left open (ms).
-    idle_timeout: 30000 # milliseconds
+    idle_timeout: 30000  # milliseconds

    # Max connection lifetime before it's closed, even if actively used.
-    server_lifetime: 86400000 # 24 hours
+    server_lifetime: 86400000  # 24 hours
+
+    # Whether to use TLS for server connections or not.
+    server_tls: false

    # How long a client is allowed to be idle while in a transaction (ms).
-    idle_client_in_transaction_timeout: 0 # milliseconds
+    idle_client_in_transaction_timeout: 0  # milliseconds

    # @param configuration.general.healthcheck_timeout How much time to give `SELECT 1` health check query to return with a result (ms).
    healthcheck_timeout: 1000
@@ -240,7 +243,15 @@ configuration:
  ## the pool_name is what clients use as database name when connecting
  ## For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded"
  ## @param [object]
-  pools: []
+  pools:
+    [{
+      name: "simple", pool_mode: "transaction",
+      users: [{username: "user", password: "pass", pool_size: 5, statement_timeout: 0}],
+      shards: [{
+        servers: [{host: "postgres", port: 5432, role: "primary"}],
+        database: "postgres"
+      }]
+    }]
    # - ## default values
    #   ##
    #   ##
@@ -287,6 +298,22 @@ configuration:
    #  ## @param configuration.poolsPostgres.query_parser_read_write_splitting
    #  query_parser_read_write_splitting: true

+    # ## Db activity based routing. If enabled, we'll route queries to the primary if the table was recently mutated.
+    # ## @param configuration.poolsPostgres.db_activity_based_routing
+    # db_activity_based_routing: false
+
+    # ## DB activity based init delay. How long to wait before starting to route queries to the primary after a table mutation.
+    # ## @param configuration.poolsPostgres.db_activity_based_ms_init_delay
+    # db_activity_based_ms_init_delay: 100
+
+    # ## DB activity TTL. How long before marking the DB as inactive after no mutations or queries.
+    # ## @param configuration.poolsPostgres.db_activity_ttl
+    # db_activity_ttl: 900
+
+    # ## Table mutation cache TTL. How long to keep track of table mutations.
+    # ## @param configuration.poolsPostgres.table_mutation_cache_ttl
+    # table_mutation_cache_ttl: 50
+
    #  ## If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for
    #  ## load balancing of read queries. Otherwise, the primary will only be used for write
    #  ## queries. The primary can always be explicitly selected with our custom protocol.
@@ -307,7 +334,9 @@ configuration:
    #  ## Credentials for users that may connect to this cluster
    #  ## @param users [array]
    #  ## @param users[0].username Name of the env var (required)
-    #  ## @param users[0].password Value for the env var (required)
+    #  ## @param users[0].password Value for the env var (required) leave empty to use existing secret see passwordSecret.name and passwordSecret.key
+    #  ## @param users[0].passwordSecret.name Name of the secret containing the password
+    #  ## @param users[0].passwordSecret.key Key in the secret containing the password
    #  ## @param users[0].pool_size Maximum number of server connections that can be established for this user
    #  ## @param users[0].statement_timeout Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way.
    #  users: []
--- a/cr.yaml
+++ b/cr.yaml
@@ -1 +1,2 @@
 sign: false
+pages_branch: main
--- a/dev/Dockerfile
+++ b/dev/Dockerfile
@@ -1,4 +1,4 @@
-FROM rust:1.70-bullseye
+FROM rust:bullseye

 # Dependencies
 COPY --from=sclevine/yj /bin/yj /bin/yj
--- a/grafana_dashboard.json
+++ b/grafana_dashboard.json
--- a/pgcat.service
+++ b/pgcat.service
@@ -11,6 +11,7 @@ RestartSec=1
 Environment=RUST_LOG=info
 LimitNOFILE=65536
 ExecStart=/usr/bin/pgcat /etc/pgcat.toml
+ExecReload=/bin/kill -SIGHUP $MAINPID

 [Install]
 WantedBy=multi-user.target
--- a/pgcat.toml
+++ b/pgcat.toml
@@ -179,7 +179,7 @@ primary_reads_enabled = true
 # `random`: picks a shard at random
 # `random_healthy`: picks a shard at random favoring shards with the least number of recent errors
 # `shard_<number>`: e.g. shard_0, shard_4, etc. picks a specific shard, everytime
-# no_shard_specified_behavior = "shard_0"
+# default_shard = "shard_0"

 # So what if you wanted to implement a different hashing function,
 # or you've already built one and you want this pooler to use it?
--- a/4
+++ b/4
@@ -7,3 +7,7 @@ systemctl enable pgcat
 if ! id pgcat 2> /dev/null; then
 	useradd -s /usr/bin/false pgcat
 fi
+
+if [ -f /etc/pgcat.toml ]; then
+	systemctl start pgcat
+fi
--- a/src/admin.rs
+++ b/src/admin.rs
@@ -55,7 +55,12 @@ where

    let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect();

-    match query_parts[0].to_ascii_uppercase().as_str() {
+    match query_parts
+        .first()
+        .unwrap_or(&"")
+        .to_ascii_uppercase()
+        .as_str()
+    {
        "BAN" => {
            trace!("BAN");
            ban(stream, query_parts).await
@@ -84,7 +89,12 @@ where
            trace!("SHUTDOWN");
            shutdown(stream).await
        }
-        "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() {
+        "SHOW" => match query_parts
+            .get(1)
+            .unwrap_or(&"")
+            .to_ascii_uppercase()
+            .as_str()
+        {
            "HELP" => {
                trace!("SHOW HELP");
                show_help(stream).await
--- a/src/auth_passthrough.rs
+++ b/src/auth_passthrough.rs
@@ -1,3 +1,4 @@
+use crate::config::AuthType;
 use crate::errors::Error;
 use crate::pool::ConnectionPool;
 use crate::server::Server;
@@ -71,6 +72,7 @@ impl AuthPassthrough {
    pub async fn fetch_hash(&self, address: &crate::config::Address) -> Result<String, Error> {
        let auth_user = crate::config::User {
            username: self.user.clone(),
+            auth_type: AuthType::MD5,
            password: Some(self.password.clone()),
            server_username: None,
            server_password: None,
--- a/src/client.rs
+++ b/src/client.rs
@@ -14,7 +14,9 @@ use tokio::sync::mpsc::Sender;

 use crate::admin::{generate_server_parameters_for_admin, handle_admin};
 use crate::auth_passthrough::refetch_auth_hash;
-use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode};
+use crate::config::{
+    get_config, get_idle_client_in_transaction_timeout, Address, AuthType, PoolMode,
+};
 use crate::constants::*;
 use crate::messages::*;
 use crate::plugins::PluginOutput;
@@ -463,8 +465,8 @@ where
            .count()
            == 1;

-        // Kick any client that's not admin while we're in admin-only mode.
        if !admin && admin_only {
+            // Kick any client that's not admin while we're in admin-only mode.
            debug!(
                "Rejecting non-admin connection to {} when in admin only mode",
                pool_name
@@ -481,72 +483,76 @@ where
        let process_id: i32 = rand::random();
        let secret_key: i32 = rand::random();

-        // Perform MD5 authentication.
-        // TODO: Add SASL support.
-        let salt = md5_challenge(&mut write).await?;
-
-        let code = match read.read_u8().await {
-            Ok(p) => p,
-            Err(_) => {
-                return Err(Error::ClientSocketError(
-                    "password code".into(),
-                    client_identifier,
-                ))
-            }
-        };
-
-        // PasswordMessage
-        if code as char != 'p' {
-            return Err(Error::ProtocolSyncError(format!(
-                "Expected p, got {}",
-                code as char
-            )));
-        }
-
-        let len = match read.read_i32().await {
-            Ok(len) => len,
-            Err(_) => {
-                return Err(Error::ClientSocketError(
-                    "password message length".into(),
-                    client_identifier,
-                ))
-            }
-        };
-
-        let mut password_response = vec![0u8; (len - 4) as usize];
-
-        match read.read_exact(&mut password_response).await {
-            Ok(_) => (),
-            Err(_) => {
-                return Err(Error::ClientSocketError(
-                    "password message".into(),
-                    client_identifier,
-                ))
-            }
-        };
-
        let mut prepared_statements_enabled = false;

        // Authenticate admin user.
        let (transaction_mode, mut server_parameters) = if admin {
            let config = get_config();
+            // TODO: Add SASL support.
+            // Perform MD5 authentication.
+            match config.general.admin_auth_type {
+                AuthType::Trust => (),
+                AuthType::MD5 => {
+                    let salt = md5_challenge(&mut write).await?;

-            // Compare server and client hashes.
-            let password_hash = md5_hash_password(
-                &config.general.admin_username,
-                &config.general.admin_password,
-                &salt,
-            );
+                    let code = match read.read_u8().await {
+                        Ok(p) => p,
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password code".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };

-            if password_hash != password_response {
-                let error = Error::ClientGeneralError("Invalid password".into(), client_identifier);
+                    // PasswordMessage
+                    if code as char != 'p' {
+                        return Err(Error::ProtocolSyncError(format!(
+                            "Expected p, got {}",
+                            code as char
+                        )));
+                    }

-                warn!("{}", error);
-                wrong_password(&mut write, username).await?;
+                    let len = match read.read_i32().await {
+                        Ok(len) => len,
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password message length".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };

-                return Err(error);
+                    let mut password_response = vec![0u8; (len - 4) as usize];
+
+                    match read.read_exact(&mut password_response).await {
+                        Ok(_) => (),
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password message".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };
+
+                    // Compare server and client hashes.
+                    let password_hash = md5_hash_password(
+                        &config.general.admin_username,
+                        &config.general.admin_password,
+                        &salt,
+                    );
+
+                    if password_hash != password_response {
+                        let error =
+                            Error::ClientGeneralError("Invalid password".into(), client_identifier);
+
+                        warn!("{}", error);
+                        wrong_password(&mut write, username).await?;
+
+                        return Err(error);
+                    }
+                }
            }
-
            (false, generate_server_parameters_for_admin())
        }
        // Authenticate normal user.
@@ -573,92 +579,143 @@ where
            // Obtain the hash to compare, we give preference to that written in cleartext in config
            // if there is nothing set in cleartext and auth passthrough (auth_query) is configured, we use the hash obtained
            // when the pool was created. If there is no hash there, we try to fetch it one more time.
-            let password_hash = if let Some(password) = &pool.settings.user.password {
-                Some(md5_hash_password(username, password, &salt))
-            } else {
-                if !get_config().is_auth_query_configured() {
-                    wrong_password(&mut write, username).await?;
-                    return Err(Error::ClientAuthImpossible(username.into()));
-                }
+            match pool.settings.user.auth_type {
+                AuthType::Trust => (),
+                AuthType::MD5 => {
+                    // Perform MD5 authentication.
+                    // TODO: Add SASL support.
+                    let salt = md5_challenge(&mut write).await?;

-                let mut hash = (*pool.auth_hash.read()).clone();
+                    let code = match read.read_u8().await {
+                        Ok(p) => p,
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password code".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };

-                if hash.is_none() {
-                    warn!(
-                        "Query auth configured \
-                          but no hash password found \
-                          for pool {}. Will try to refetch it.",
-                        pool_name
-                    );
+                    // PasswordMessage
+                    if code as char != 'p' {
+                        return Err(Error::ProtocolSyncError(format!(
+                            "Expected p, got {}",
+                            code as char
+                        )));
+                    }

-                    match refetch_auth_hash(&pool).await {
-                        Ok(fetched_hash) => {
-                            warn!("Password for {}, obtained. Updating.", client_identifier);
+                    let len = match read.read_i32().await {
+                        Ok(len) => len,
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password message length".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };
+
+                    let mut password_response = vec![0u8; (len - 4) as usize];
+
+                    match read.read_exact(&mut password_response).await {
+                        Ok(_) => (),
+                        Err(_) => {
+                            return Err(Error::ClientSocketError(
+                                "password message".into(),
+                                client_identifier,
+                            ))
+                        }
+                    };
+
+                    let password_hash = if let Some(password) = &pool.settings.user.password {
+                        Some(md5_hash_password(username, password, &salt))
+                    } else {
+                        if !get_config().is_auth_query_configured() {
+                            wrong_password(&mut write, username).await?;
+                            return Err(Error::ClientAuthImpossible(username.into()));
+                        }
+
+                        let mut hash = (*pool.auth_hash.read()).clone();
+
+                        if hash.is_none() {
+                            warn!(
+                                "Query auth configured \
+                                  but no hash password found \
+                                  for pool {}. Will try to refetch it.",
+                                pool_name
+                            );
+
+                            match refetch_auth_hash(&pool).await {
+                                Ok(fetched_hash) => {
+                                    warn!(
+                                        "Password for {}, obtained. Updating.",
+                                        client_identifier
+                                    );
+
+                                    {
+                                        let mut pool_auth_hash = pool.auth_hash.write();
+                                        *pool_auth_hash = Some(fetched_hash.clone());
+                                    }
+
+                                    hash = Some(fetched_hash);
+                                }
+
+                                Err(err) => {
+                                    wrong_password(&mut write, username).await?;
+
+                                    return Err(Error::ClientAuthPassthroughError(
+                                        err.to_string(),
+                                        client_identifier,
+                                    ));
+                                }
+                            }
+                        };
+
+                        Some(md5_hash_second_pass(&hash.unwrap(), &salt))
+                    };
+
+                    // Once we have the resulting hash, we compare with what the client gave us.
+                    // If they do not match and auth query is set up, we try to refetch the hash one more time
+                    // to see if the password has changed since the pool was created.
+                    //
+                    // @TODO: we could end up fetching again the same password twice (see above).
+                    if password_hash.unwrap() != password_response {
+                        warn!(
+                            "Invalid password {}, will try to refetch it.",
+                            client_identifier
+                        );
+
+                        let fetched_hash = match refetch_auth_hash(&pool).await {
+                            Ok(fetched_hash) => fetched_hash,
+                            Err(err) => {
+                                wrong_password(&mut write, username).await?;
+
+                                return Err(err);
+                            }
+                        };
+
+                        let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt);
+
+                        // Ok password changed in server an auth is possible.
+                        if new_password_hash == password_response {
+                            warn!(
+                                "Password for {}, changed in server. Updating.",
+                                client_identifier
+                            );

                            {
                                let mut pool_auth_hash = pool.auth_hash.write();
-                                *pool_auth_hash = Some(fetched_hash.clone());
+                                *pool_auth_hash = Some(fetched_hash);
                            }
-
-                            hash = Some(fetched_hash);
-                        }
-
-                        Err(err) => {
+                        } else {
                            wrong_password(&mut write, username).await?;
-
-                            return Err(Error::ClientAuthPassthroughError(
-                                err.to_string(),
+                            return Err(Error::ClientGeneralError(
+                                "Invalid password".into(),
                                client_identifier,
                            ));
                        }
                    }
-                };
-
-                Some(md5_hash_second_pass(&hash.unwrap(), &salt))
-            };
-
-            // Once we have the resulting hash, we compare with what the client gave us.
-            // If they do not match and auth query is set up, we try to refetch the hash one more time
-            // to see if the password has changed since the pool was created.
-            //
-            // @TODO: we could end up fetching again the same password twice (see above).
-            if password_hash.unwrap() != password_response {
-                warn!(
-                    "Invalid password {}, will try to refetch it.",
-                    client_identifier
-                );
-
-                let fetched_hash = match refetch_auth_hash(&pool).await {
-                    Ok(fetched_hash) => fetched_hash,
-                    Err(err) => {
-                        wrong_password(&mut write, username).await?;
-
-                        return Err(err);
-                    }
-                };
-
-                let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt);
-
-                // Ok password changed in server an auth is possible.
-                if new_password_hash == password_response {
-                    warn!(
-                        "Password for {}, changed in server. Updating.",
-                        client_identifier
-                    );
-
-                    {
-                        let mut pool_auth_hash = pool.auth_hash.write();
-                        *pool_auth_hash = Some(fetched_hash);
-                    }
-                } else {
-                    wrong_password(&mut write, username).await?;
-                    return Err(Error::ClientGeneralError(
-                        "Invalid password".into(),
-                        client_identifier,
-                    ));
                }
            }
-
            let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction;
            prepared_statements_enabled =
                transaction_mode && pool.prepared_statement_cache.is_some();
@@ -802,6 +859,8 @@ where
        // e.g. primary, replica, which shard.
        let mut query_router = QueryRouter::new();

+        let mut checkout_failure_count: u64 = 0;
+
        self.stats.register(self.stats.clone());

        // Result returned by one of the plugins.
@@ -824,6 +883,7 @@ where
        };

        query_router.update_pool_settings(&pool.settings);
+        query_router.set_default_role();

        // Our custom protocol loop.
        // We expect the client to either start a transaction with regular queries
@@ -1050,7 +1110,25 @@ where
                        query_router.role(),
                        err
                    );
-
+                    checkout_failure_count += 1;
+                    if let Some(limit) = pool.settings.checkout_failure_limit {
+                        if checkout_failure_count >= limit {
+                            error!(
+                                "Checkout failure limit reached ({} / {}) - disconnecting client",
+                                checkout_failure_count, limit
+                            );
+                            error_response_terminal(
+                                &mut self.write,
+                                &format!(
+                                    "checkout failure limit reached ({} / {})",
+                                    checkout_failure_count, limit
+                                ),
+                            )
+                            .await?;
+                            self.stats.disconnect();
+                            return Ok(());
+                        }
+                    }
                    continue;
                }
            };
--- a/src/config.rs
+++ b/src/config.rs
@@ -38,12 +38,12 @@ pub enum Role {
    Mirror,
 }

-impl ToString for Role {
-    fn to_string(&self) -> String {
-        match *self {
-            Role::Primary => "primary".to_string(),
-            Role::Replica => "replica".to_string(),
-            Role::Mirror => "mirror".to_string(),
+impl std::fmt::Display for Role {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Role::Primary => write!(f, "primary"),
+            Role::Replica => write!(f, "replica"),
+            Role::Mirror => write!(f, "mirror"),
        }
    }
 }
@@ -208,6 +208,9 @@ impl Address {
 pub struct User {
    pub username: String,
    pub password: Option<String>,
+
+    #[serde(default = "User::default_auth_type")]
+    pub auth_type: AuthType,
    pub server_username: Option<String>,
    pub server_password: Option<String>,
    pub pool_size: u32,
@@ -225,6 +228,7 @@ impl Default for User {
        User {
            username: String::from("postgres"),
            password: None,
+            auth_type: AuthType::MD5,
            server_username: None,
            server_password: None,
            pool_size: 15,
@@ -239,6 +243,10 @@ impl Default for User {
 }

 impl User {
+    pub fn default_auth_type() -> AuthType {
+        AuthType::MD5
+    }
+
    fn validate(&self) -> Result<(), Error> {
        if let Some(min_pool_size) = self.min_pool_size {
            if min_pool_size > self.pool_size {
@@ -334,6 +342,9 @@ pub struct General {
    pub admin_username: String,
    pub admin_password: String,

+    #[serde(default = "General::default_admin_auth_type")]
+    pub admin_auth_type: AuthType,
+
    #[serde(default = "General::default_validate_config")]
    pub validate_config: bool,

@@ -348,6 +359,10 @@ impl General {
        "0.0.0.0".into()
    }

+    pub fn default_admin_auth_type() -> AuthType {
+        AuthType::MD5
+    }
+
    pub fn default_port() -> u16 {
        5432
    }
@@ -456,6 +471,7 @@ impl Default for General {
            verify_server_certificate: false,
            admin_username: String::from("admin"),
            admin_password: String::from("admin"),
+            admin_auth_type: AuthType::MD5,
            validate_config: true,
            auth_query: None,
            auth_query_user: None,
@@ -476,11 +492,20 @@ pub enum PoolMode {
    Session,
 }

-impl ToString for PoolMode {
-    fn to_string(&self) -> String {
-        match *self {
-            PoolMode::Transaction => "transaction".to_string(),
-            PoolMode::Session => "session".to_string(),
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Copy, Hash)]
+pub enum AuthType {
+    #[serde(alias = "trust", alias = "Trust")]
+    Trust,
+
+    #[serde(alias = "md5", alias = "MD5")]
+    MD5,
+}
+
+impl std::fmt::Display for PoolMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            PoolMode::Transaction => write!(f, "transaction"),
+            PoolMode::Session => write!(f, "session"),
        }
    }
 }
@@ -493,12 +518,13 @@ pub enum LoadBalancingMode {
    #[serde(alias = "loc", alias = "LOC", alias = "least_outstanding_connections")]
    LeastOutstandingConnections,
 }
-impl ToString for LoadBalancingMode {
-    fn to_string(&self) -> String {
-        match *self {
-            LoadBalancingMode::Random => "random".to_string(),
+
+impl std::fmt::Display for LoadBalancingMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            LoadBalancingMode::Random => write!(f, "random"),
            LoadBalancingMode::LeastOutstandingConnections => {
-                "least_outstanding_connections".to_string()
+                write!(f, "least_outstanding_connections")
            }
        }
    }
@@ -532,6 +558,14 @@ pub struct Pool {
    /// Close idle connections that have been opened for longer than this.
    pub idle_timeout: Option<u64>,

+    /// Maximum number of checkout failures a client is allowed before it
+    /// gets disconnected. This is needed to prevent persistent client/server
+    /// imbalance in high availability setups where multiple PgCat instances are placed
+    /// behind a single load balancer. If for any reason a client lands on a PgCat instance that has
+    /// a large number of connected clients, it might get stuck in perpetual checkout failure loop especially
+    /// in session mode
+    pub checkout_failure_limit: Option<u64>,
+
    /// Close server connections that have been opened for longer than this.
    /// Only applied to idle connections. If the connection is actively used for
    /// longer than this period, the pool will not interrupt it.
@@ -563,6 +597,19 @@ pub struct Pool {
    #[serde(default = "Pool::default_prepared_statements_cache_size")]
    pub prepared_statements_cache_size: usize,

+    // Support for query routing based on database activity
+    #[serde(default = "Pool::default_db_activity_based_routing")]
+    pub db_activity_based_routing: bool,
+
+    #[serde(default = "Pool::default_db_activity_init_delay")]
+    pub db_activity_init_delay: u64,
+
+    #[serde(default = "Pool::default_db_activity_ttl")]
+    pub db_activity_ttl: u64,
+
+    #[serde(default = "Pool::default_table_mutation_cache_ms_ttl")]
+    pub table_mutation_cache_ms_ttl: u64,
+
    pub plugins: Option<Plugins>,
    pub shards: BTreeMap<String, Shard>,
    pub users: BTreeMap<String, User>,
@@ -616,6 +663,25 @@ impl Pool {
        0
    }

+    pub fn default_db_activity_based_routing() -> bool {
+        false
+    }
+
+    pub fn default_db_activity_init_delay() -> u64 {
+        // 100 milliseconds
+        100
+    }
+
+    pub fn default_db_activity_ttl() -> u64 {
+        // 15 minutes
+        15 * 60
+    }
+
+    pub fn default_table_mutation_cache_ms_ttl() -> u64 {
+        // 50 milliseconds
+        50
+    }
+
    pub fn validate(&mut self) -> Result<(), Error> {
        match self.default_role.as_ref() {
            "any" => (),
@@ -698,6 +764,23 @@ impl Pool {
            user.validate()?;
        }

+        if self.db_activity_based_routing {
+            if self.db_activity_init_delay == 0 {
+                error!("db_activity_init_delay must be greater than 0");
+                return Err(Error::BadConfig);
+            }
+
+            if self.table_mutation_cache_ms_ttl == 0 {
+                error!("table_mutation_cache_ms_ttl must be greater than 0");
+                return Err(Error::BadConfig);
+            }
+
+            if self.db_activity_ttl == 0 {
+                error!("db_activity_ttl must be greater than 0");
+                return Err(Error::BadConfig);
+            }
+        }
+
        Ok(())
    }
 }
@@ -707,6 +790,7 @@ impl Default for Pool {
        Pool {
            pool_mode: Self::default_pool_mode(),
            load_balancing_mode: Self::default_load_balancing_mode(),
+            checkout_failure_limit: None,
            default_role: String::from("any"),
            query_parser_enabled: false,
            query_parser_max_length: None,
@@ -727,6 +811,10 @@ impl Default for Pool {
            cleanup_server_connections: true,
            log_client_parameter_status_changes: false,
            prepared_statements_cache_size: Self::default_prepared_statements_cache_size(),
+            db_activity_based_routing: Self::default_db_activity_based_routing(),
+            db_activity_init_delay: Self::default_db_activity_init_delay(),
+            db_activity_ttl: Self::default_db_activity_ttl(),
+            table_mutation_cache_ms_ttl: Self::default_table_mutation_cache_ms_ttl(),
            plugins: None,
            shards: BTreeMap::from([(String::from("1"), Shard::default())]),
            users: BTreeMap::default(),
@@ -999,15 +1087,17 @@ impl Config {
    pub fn fill_up_auth_query_config(&mut self) {
        for (_name, pool) in self.pools.iter_mut() {
            if pool.auth_query.is_none() {
-                pool.auth_query = self.general.auth_query.clone();
+                pool.auth_query.clone_from(&self.general.auth_query);
            }

            if pool.auth_query_user.is_none() {
-                pool.auth_query_user = self.general.auth_query_user.clone();
+                pool.auth_query_user
+                    .clone_from(&self.general.auth_query_user);
            }

            if pool.auth_query_password.is_none() {
-                pool.auth_query_password = self.general.auth_query_password.clone();
+                pool.auth_query_password
+                    .clone_from(&self.general.auth_query_password);
            }
        }
    }
@@ -1155,7 +1245,7 @@ impl Config {
            "Default max server lifetime: {}ms",
            self.general.server_lifetime
        );
-        info!("Sever round robin: {}", self.general.server_round_robin);
+        info!("Server round robin: {}", self.general.server_round_robin);
        match self.general.tls_certificate.clone() {
            Some(tls_certificate) => {
                info!("TLS certificate: {}", tls_certificate);
@@ -1217,6 +1307,17 @@ impl Config {
                None => self.general.idle_timeout,
            };
            info!("[pool: {}] Idle timeout: {}ms", pool_name, idle_timeout);
+            match pool_config.checkout_failure_limit {
+                Some(checkout_failure_limit) => {
+                    info!(
+                        "[pool: {}] Checkout failure limit: {}",
+                        pool_name, checkout_failure_limit
+                    );
+                }
+                None => {
+                    info!("[pool: {}] Checkout failure limit: not set", pool_name);
+                }
+            };
            info!(
                "[pool: {}] Sharding function: {}",
                pool_name,
@@ -1261,6 +1362,22 @@ impl Config {
                "[pool: {}] Cleanup server connections: {}",
                pool_name, pool_config.cleanup_server_connections
            );
+            info!(
+                "[pool: {}] DB activity based routing: {}",
+                pool_name, pool_config.db_activity_based_routing
+            );
+            info!(
+                "[pool: {}] DB activity init delay: {}",
+                pool_name, pool_config.db_activity_init_delay
+            );
+            info!(
+                "[pool: {}] DB activity TTL: {}",
+                pool_name, pool_config.db_activity_ttl
+            );
+            info!(
+                "[pool: {}] Table mutation cache TTL: {}",
+                pool_name, pool_config.table_mutation_cache_ms_ttl
+            );
            info!(
                "[pool: {}] Log client parameter status changes: {}",
                pool_name, pool_config.log_client_parameter_status_changes
--- a/src/logger.rs
+++ b/src/logger.rs
@@ -3,7 +3,7 @@ use tracing_subscriber;
 use tracing_subscriber::EnvFilter;

 pub fn init(args: &Args) {
-    // Iniitalize a default filter, and then override the builtin default "warning" with our
+    // Initialize a default filter, and then override the builtin default "warning" with our
    // commandline, (default: "info")
    let filter = EnvFilter::from_default_env().add_directive(args.log_level.into());

--- a/src/messages.rs
+++ b/src/messages.rs
@@ -733,6 +733,10 @@ pub fn configure_socket(stream: &TcpStream) {
        }
        Err(err) => error!("Could not configure socket: {}", err),
    }
+    match sock_ref.set_nodelay(true) {
+        Ok(_) => (),
+        Err(err) => error!("Could not configure TCP_NODELAY for socket: {}", err),
+    }
 }

 pub trait BytesMutReader {
--- a/src/pool.rs
+++ b/src/pool.rs
@@ -152,6 +152,14 @@ pub struct PoolSettings {
    /// Random or LeastOutstandingConnections.
    pub load_balancing_mode: LoadBalancingMode,

+    /// Maximum number of checkout failures a client is allowed before it
+    /// gets disconnected. This is needed to prevent persistent client/server
+    /// imbalance in high availability setups where multiple PgCat instances are placed
+    /// behind a single load balancer. If for any reason a client lands on a PgCat instance that has
+    /// a large number of connected clients, it might get stuck in perpetual checkout failure loop especially
+    /// in session mode
+    pub checkout_failure_limit: Option<u64>,
+
    // Number of shards.
    pub shards: usize,

@@ -174,6 +182,18 @@ pub struct PoolSettings {
    // Read from the primary as well or not.
    pub primary_reads_enabled: bool,

+    // Automatic primary/replica selection based on recent activity.
+    pub db_activity_based_routing: bool,
+
+    // DB activity init delay
+    pub db_activity_init_delay: u64,
+
+    // DB activity TTL
+    pub db_activity_ttl: u64,
+
+    // Table mutation cache TTL
+    pub table_mutation_cache_ms_ttl: u64,
+
    // Sharding function.
    pub sharding_function: ShardingFunction,

@@ -215,6 +235,7 @@ impl Default for PoolSettings {
        PoolSettings {
            pool_mode: PoolMode::Transaction,
            load_balancing_mode: LoadBalancingMode::Random,
+            checkout_failure_limit: None,
            shards: 1,
            user: User::default(),
            db: String::default(),
@@ -223,6 +244,10 @@ impl Default for PoolSettings {
            query_parser_max_length: None,
            query_parser_read_write_splitting: false,
            primary_reads_enabled: true,
+            db_activity_based_routing: false,
+            db_activity_init_delay: 100,
+            db_activity_ttl: 15 * 60,
+            table_mutation_cache_ms_ttl: 50,
            sharding_function: ShardingFunction::PgBigintHash,
            automatic_sharding_key: None,
            healthcheck_delay: General::default_healthcheck_delay(),
@@ -521,6 +546,7 @@ impl ConnectionPool {
                            None => pool_config.pool_mode,
                        },
                        load_balancing_mode: pool_config.load_balancing_mode,
+                        checkout_failure_limit: pool_config.checkout_failure_limit,
                        // shards: pool_config.shards.clone(),
                        shards: shard_ids.len(),
                        user: user.clone(),
@@ -537,6 +563,10 @@ impl ConnectionPool {
                            .query_parser_read_write_splitting,
                        primary_reads_enabled: pool_config.primary_reads_enabled,
                        sharding_function: pool_config.sharding_function,
+                        db_activity_based_routing: pool_config.db_activity_based_routing,
+                        db_activity_init_delay: pool_config.db_activity_init_delay,
+                        db_activity_ttl: pool_config.db_activity_ttl,
+                        table_mutation_cache_ms_ttl: pool_config.table_mutation_cache_ms_ttl,
                        automatic_sharding_key: pool_config.automatic_sharding_key.clone(),
                        healthcheck_delay: config.general.healthcheck_delay,
                        healthcheck_timeout: config.general.healthcheck_timeout,
@@ -813,7 +843,7 @@ impl ConnectionPool {
            }
        }

-        client_stats.checkout_success();
+        client_stats.checkout_error();

        Err(Error::AllServersDown)
    }
--- a/src/prometheus.rs
+++ b/src/prometheus.rs
@@ -1,23 +1,41 @@
-use hyper::service::{make_service_fn, service_fn};
-use hyper::{Body, Method, Request, Response, Server, StatusCode};
+use http_body_util::Full;
+use hyper::body;
+use hyper::body::Bytes;
+
+use hyper::server::conn::http1;
+use hyper::service::service_fn;
+use hyper::{Method, Request, Response, StatusCode};
+use hyper_util::rt::TokioIo;
 use log::{debug, error, info};
 use phf::phf_map;
 use std::collections::HashMap;
 use std::fmt;
 use std::net::SocketAddr;
 use std::sync::atomic::Ordering;
-use std::sync::Arc;
+use tokio::net::TcpListener;

 use crate::config::Address;
 use crate::pool::{get_all_pools, PoolIdentifier};
+use crate::stats::get_server_stats;
 use crate::stats::pool::PoolStats;
-use crate::stats::{get_server_stats, ServerStats};

 struct MetricHelpType {
    help: &'static str,
    ty: &'static str,
 }

+struct ServerPrometheusStats {
+    bytes_received: u64,
+    bytes_sent: u64,
+    transaction_count: u64,
+    query_count: u64,
+    error_count: u64,
+    active_count: u64,
+    idle_count: u64,
+    login_count: u64,
+    tested_count: u64,
+}
+
 // reference for metric types: https://prometheus.io/docs/concepts/metric_types/
 // counters only increase
 // gauges can arbitrarily increase or decrease
@@ -120,22 +138,46 @@ static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = ph
    },
    "servers_bytes_received" => MetricHelpType {
        help: "Volume in bytes of network traffic received by server",
-        ty: "gauge",
+        ty: "counter",
    },
    "servers_bytes_sent" => MetricHelpType {
        help: "Volume in bytes of network traffic sent by server",
-        ty: "gauge",
+        ty: "counter",
    },
    "servers_transaction_count" => MetricHelpType {
        help: "Number of transactions executed by server",
-        ty: "gauge",
+        ty: "counter",
    },
    "servers_query_count" => MetricHelpType {
        help: "Number of queries executed by server",
-        ty: "gauge",
+        ty: "counter",
    },
    "servers_error_count" => MetricHelpType {
        help: "Number of errors",
+        ty: "counter",
+    },
+    "servers_idle_count" => MetricHelpType {
+        help: "Number of server connection in idle state",
+        ty: "gauge",
+    },
+    "servers_active_count" => MetricHelpType {
+        help: "Number of server connection in active state",
+        ty: "gauge",
+    },
+    "servers_tested_count" => MetricHelpType {
+        help: "Number of server connection in tested state",
+        ty: "gauge",
+    },
+    "servers_login_count" => MetricHelpType {
+        help: "Number of server connection in login state",
+        ty: "gauge",
+    },
+    "servers_is_banned" => MetricHelpType {
+        help: "0 if server is not banned, 1 if server is banned",
+        ty: "gauge",
+    },
+    "servers_is_paused" => MetricHelpType {
+        help: "0 if server is not paused, 1 if server is paused",
        ty: "gauge",
    },
    "databases_pool_size" => MetricHelpType {
@@ -158,18 +200,17 @@ struct PrometheusMetric<Value: fmt::Display> {

 impl<Value: fmt::Display> fmt::Display for PrometheusMetric<Value> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let formatted_labels = self
-            .labels
+        let mut sorted_labels: Vec<_> = self.labels.iter().collect();
+        sorted_labels.sort_by_key(|&(key, _)| key);
+        let formatted_labels = sorted_labels
            .iter()
            .map(|(key, value)| format!("{}=\"{}\"", key, value))
            .collect::<Vec<_>>()
            .join(",");
        write!(
            f,
-            "# HELP {name} {help}\n# TYPE {name} {ty}\n{name}{{{formatted_labels}}} {value}\n",
+            "{name}{{{formatted_labels}}} {value}",
            name = format_args!("pgcat_{}", self.name),
-            help = self.help,
-            ty = self.ty,
            formatted_labels = formatted_labels,
            value = self.value
        )
@@ -203,7 +244,9 @@ impl<Value: fmt::Display> PrometheusMetric<Value> {
        labels.insert("shard", address.shard.to_string());
        labels.insert("role", address.role.to_string());
        labels.insert("pool", address.pool_name.clone());
+        labels.insert("index", address.address_index.to_string());
        labels.insert("database", address.database.to_string());
+        labels.insert("username", address.username.clone());

        Self::from_name(&format!("databases_{}", name), value, labels)
    }
@@ -218,7 +261,9 @@ impl<Value: fmt::Display> PrometheusMetric<Value> {
        labels.insert("shard", address.shard.to_string());
        labels.insert("role", address.role.to_string());
        labels.insert("pool", address.pool_name.clone());
+        labels.insert("index", address.address_index.to_string());
        labels.insert("database", address.database.to_string());
+        labels.insert("username", address.username.clone());

        Self::from_name(&format!("servers_{}", name), value, labels)
    }
@@ -229,7 +274,9 @@ impl<Value: fmt::Display> PrometheusMetric<Value> {
        labels.insert("shard", address.shard.to_string());
        labels.insert("pool", address.pool_name.clone());
        labels.insert("role", address.role.to_string());
+        labels.insert("index", address.address_index.to_string());
        labels.insert("database", address.database.to_string());
+        labels.insert("username", address.username.clone());

        Self::from_name(&format!("stats_{}", name), value, labels)
    }
@@ -241,9 +288,20 @@ impl<Value: fmt::Display> PrometheusMetric<Value> {

        Self::from_name(&format!("pools_{}", name), value, labels)
    }
+
+    fn get_header(&self) -> String {
+        format!(
+            "\n# HELP {name} {help}\n# TYPE {name} {ty}",
+            name = format_args!("pgcat_{}", self.name),
+            help = self.help,
+            ty = self.ty,
+        )
+    }
 }

-async fn prometheus_stats(request: Request<Body>) -> Result<Response<Body>, hyper::http::Error> {
+async fn prometheus_stats(
+    request: Request<body::Incoming>,
+) -> Result<Response<Full<Bytes>>, hyper::http::Error> {
    match (request.method(), request.uri().path()) {
        (&Method::GET, "/metrics") => {
            let mut lines = Vec::new();
@@ -251,6 +309,7 @@ async fn prometheus_stats(request: Request<Body>) -> Result<Response<Body>, hype
            push_pool_stats(&mut lines);
            push_server_stats(&mut lines);
            push_database_stats(&mut lines);
+            lines.push("".to_string()); // Ensure to end the stats with a line terminator as required by the specification.

            Response::builder()
                .header("content-type", "text/plain; version=0.0.4")
@@ -264,6 +323,7 @@ async fn prometheus_stats(request: Request<Body>) -> Result<Response<Body>, hype

 // Adds metrics shown in a SHOW STATS admin command.
 fn push_address_stats(lines: &mut Vec<String>) {
+    let mut grouped_metrics: HashMap<String, Vec<PrometheusMetric<u64>>> = HashMap::new();
    for (_, pool) in get_all_pools() {
        for shard in 0..pool.shards() {
            for server in 0..pool.servers(shard) {
@@ -273,7 +333,10 @@ fn push_address_stats(lines: &mut Vec<String>) {
                    if let Some(prometheus_metric) =
                        PrometheusMetric::<u64>::from_address(address, &key, value)
                    {
-                        lines.push(prometheus_metric.to_string());
+                        grouped_metrics
+                            .entry(key)
+                            .or_default()
+                            .push(prometheus_metric);
                    } else {
                        debug!("Metric {} not implemented for {}", key, address.name());
                    }
@@ -281,33 +344,53 @@ fn push_address_stats(lines: &mut Vec<String>) {
            }
        }
    }
+    for (_key, metrics) in grouped_metrics {
+        if !metrics.is_empty() {
+            lines.push(metrics[0].get_header());
+            for metric in metrics {
+                lines.push(metric.to_string());
+            }
+        }
+    }
 }

 // Adds relevant metrics shown in a SHOW POOLS admin command.
 fn push_pool_stats(lines: &mut Vec<String>) {
+    let mut grouped_metrics: HashMap<String, Vec<PrometheusMetric<u64>>> = HashMap::new();
    let pool_stats = PoolStats::construct_pool_lookup();
    for (pool_id, stats) in pool_stats.iter() {
        for (name, value) in stats.clone() {
            if let Some(prometheus_metric) =
                PrometheusMetric::<u64>::from_pool(pool_id.clone(), &name, value)
            {
-                lines.push(prometheus_metric.to_string());
+                grouped_metrics
+                    .entry(name)
+                    .or_default()
+                    .push(prometheus_metric);
            } else {
                debug!("Metric {} not implemented for ({})", name, *pool_id);
            }
        }
    }
+    for (_key, metrics) in grouped_metrics {
+        if !metrics.is_empty() {
+            lines.push(metrics[0].get_header());
+            for metric in metrics {
+                lines.push(metric.to_string());
+            }
+        }
+    }
 }

 // Adds relevant metrics shown in a SHOW DATABASES admin command.
 fn push_database_stats(lines: &mut Vec<String>) {
+    let mut grouped_metrics: HashMap<String, Vec<PrometheusMetric<u32>>> = HashMap::new();
    for (_, pool) in get_all_pools() {
        let pool_config = pool.settings.clone();
        for shard in 0..pool.shards() {
            for server in 0..pool.servers(shard) {
                let address = pool.address(shard, server);
                let pool_state = pool.pool_state(shard, server);
-
                let metrics = vec![
                    ("pool_size", pool_config.user.pool_size),
                    ("current_connections", pool_state.connections),
@@ -316,7 +399,10 @@ fn push_database_stats(lines: &mut Vec<String>) {
                    if let Some(prometheus_metric) =
                        PrometheusMetric::<u32>::from_database_info(address, key, value)
                    {
-                        lines.push(prometheus_metric.to_string());
+                        grouped_metrics
+                            .entry(key.to_string())
+                            .or_default()
+                            .push(prometheus_metric);
                    } else {
                        debug!("Metric {} not implemented for {}", key, address.name());
                    }
@@ -324,45 +410,73 @@ fn push_database_stats(lines: &mut Vec<String>) {
            }
        }
    }
+    for (_key, metrics) in grouped_metrics {
+        if !metrics.is_empty() {
+            lines.push(metrics[0].get_header());
+            for metric in metrics {
+                lines.push(metric.to_string());
+            }
+        }
+    }
 }

 // Adds relevant metrics shown in a SHOW SERVERS admin command.
 fn push_server_stats(lines: &mut Vec<String>) {
    let server_stats = get_server_stats();
-    let mut server_stats_by_addresses = HashMap::<String, Arc<ServerStats>>::new();
+    let mut prom_stats = HashMap::<String, ServerPrometheusStats>::new();
    for (_, stats) in server_stats {
-        server_stats_by_addresses.insert(stats.address_name(), stats);
+        let entry = prom_stats
+            .entry(stats.address_name())
+            .or_insert(ServerPrometheusStats {
+                bytes_received: 0,
+                bytes_sent: 0,
+                transaction_count: 0,
+                query_count: 0,
+                error_count: 0,
+                active_count: 0,
+                idle_count: 0,
+                login_count: 0,
+                tested_count: 0,
+            });
+        entry.bytes_received += stats.bytes_received.load(Ordering::Relaxed);
+        entry.bytes_sent += stats.bytes_sent.load(Ordering::Relaxed);
+        entry.transaction_count += stats.transaction_count.load(Ordering::Relaxed);
+        entry.query_count += stats.query_count.load(Ordering::Relaxed);
+        entry.error_count += stats.error_count.load(Ordering::Relaxed);
+        match stats.state.load(Ordering::Relaxed) {
+            crate::stats::ServerState::Login => entry.login_count += 1,
+            crate::stats::ServerState::Active => entry.active_count += 1,
+            crate::stats::ServerState::Tested => entry.tested_count += 1,
+            crate::stats::ServerState::Idle => entry.idle_count += 1,
+        }
    }
-
+    let mut grouped_metrics: HashMap<String, Vec<PrometheusMetric<u64>>> = HashMap::new();
    for (_, pool) in get_all_pools() {
        for shard in 0..pool.shards() {
            for server in 0..pool.servers(shard) {
                let address = pool.address(shard, server);
-                if let Some(server_info) = server_stats_by_addresses.get(&address.name()) {
+                if let Some(server_info) = prom_stats.get(&address.name()) {
                    let metrics = [
-                        (
-                            "bytes_received",
-                            server_info.bytes_received.load(Ordering::Relaxed),
-                        ),
-                        ("bytes_sent", server_info.bytes_sent.load(Ordering::Relaxed)),
-                        (
-                            "transaction_count",
-                            server_info.transaction_count.load(Ordering::Relaxed),
-                        ),
-                        (
-                            "query_count",
-                            server_info.query_count.load(Ordering::Relaxed),
-                        ),
-                        (
-                            "error_count",
-                            server_info.error_count.load(Ordering::Relaxed),
-                        ),
+                        ("bytes_received", server_info.bytes_received),
+                        ("bytes_sent", server_info.bytes_sent),
+                        ("transaction_count", server_info.transaction_count),
+                        ("query_count", server_info.query_count),
+                        ("error_count", server_info.error_count),
+                        ("idle_count", server_info.idle_count),
+                        ("active_count", server_info.active_count),
+                        ("login_count", server_info.login_count),
+                        ("tested_count", server_info.tested_count),
+                        ("is_banned", if pool.is_banned(address) { 1 } else { 0 }),
+                        ("is_paused", if pool.paused() { 1 } else { 0 }),
                    ];
                    for (key, value) in metrics {
                        if let Some(prometheus_metric) =
                            PrometheusMetric::<u64>::from_server_info(address, key, value)
                        {
-                            lines.push(prometheus_metric.to_string());
+                            grouped_metrics
+                                .entry(key.to_string())
+                                .or_default()
+                                .push(prometheus_metric);
                        } else {
                            debug!("Metric {} not implemented for {}", key, address.name());
                        }
@@ -371,17 +485,46 @@ fn push_server_stats(lines: &mut Vec<String>) {
            }
        }
    }
+    for (_key, metrics) in grouped_metrics {
+        if !metrics.is_empty() {
+            lines.push(metrics[0].get_header());
+            for metric in metrics {
+                lines.push(metric.to_string());
+            }
+        }
+    }
 }

 pub async fn start_metric_server(http_addr: SocketAddr) {
-    let http_service_factory =
-        make_service_fn(|_conn| async { Ok::<_, hyper::Error>(service_fn(prometheus_stats)) });
-    let server = Server::bind(&http_addr).serve(http_service_factory);
+    let listener = TcpListener::bind(http_addr);
+    let listener = match listener.await {
+        Ok(listener) => listener,
+        Err(e) => {
+            error!("Failed to bind prometheus server to HTTP address: {}.", e);
+            return;
+        }
+    };
    info!(
        "Exposing prometheus metrics on http://{}/metrics.",
        http_addr
    );
-    if let Err(e) = server.await {
-        error!("Failed to run HTTP server: {}.", e);
+    loop {
+        let stream = match listener.accept().await {
+            Ok((stream, _)) => stream,
+            Err(e) => {
+                error!("Error accepting connection: {}", e);
+                continue;
+            }
+        };
+        let io = TokioIo::new(stream);
+
+        tokio::task::spawn(async move {
+            if let Err(err) = http1::Builder::new()
+                .serve_connection(io, service_fn(prometheus_stats))
+                .await
+            {
+                eprintln!("Error serving HTTP connection for metrics: {:?}", err);
+            }
+        });
    }
 }
--- a/src/query_router.rs
+++ b/src/query_router.rs
@@ -2,6 +2,7 @@
 /// or implied query characteristics.
 use bytes::{Buf, BytesMut};
 use log::{debug, error};
+use mini_moka::sync::Cache;
 use once_cell::sync::OnceCell;
 use regex::{Regex, RegexSet};
 use sqlparser::ast::Statement::{Delete, Insert, Query, StartTransaction, Update};
@@ -11,6 +12,7 @@ use sqlparser::ast::{
 };
 use sqlparser::dialect::PostgreSqlDialect;
 use sqlparser::parser::Parser;
+use std::sync::OnceLock;

 use crate::config::Role;
 use crate::errors::Error;
@@ -21,6 +23,7 @@ use crate::sharding::Sharder;

 use std::collections::BTreeSet;
 use std::io::Cursor;
+use std::time::Duration;
 use std::{cmp, mem};

 /// Regexes used to parse custom commands.
@@ -66,6 +69,18 @@ static CUSTOM_SQL_REGEX_SET: OnceCell<RegexSet> = OnceCell::new();
 // Get the value inside the custom command.
 static CUSTOM_SQL_REGEX_LIST: OnceCell<Vec<Regex>> = OnceCell::new();

+#[derive(Debug, Clone, PartialEq)]
+enum DatabaseActivityState {
+    Active,
+    Initializing,
+}
+
+// A moka cache for the databases
+// the key is the database name and the value is the database activity state
+static DATABASE_ACTIVITY_CACHE: OnceLock<Cache<String, DatabaseActivityState>> = OnceLock::new();
+// A moka cache for the tables, the key is the db_table.
+static TABLE_MUTATIONS_CACHE: OnceLock<Cache<String, bool>> = OnceLock::new();
+
 /// The query router.
 pub struct QueryRouter {
    /// Which shard we should be talking to right now.
@@ -87,6 +102,12 @@ pub struct QueryRouter {
    placeholders: Vec<i16>,
 }

+struct ExtractedExprsAndTables<'a> {
+    exprs: Vec<Expr>,
+    table_names: Vec<Vec<Ident>>,
+    assignments_opt: Option<&'a Vec<Assignment>>,
+}
+
 impl QueryRouter {
    /// One-time initialization of regexes
    /// that parse our custom SQL protocol.
@@ -386,6 +407,53 @@ impl QueryRouter {
        }
    }

+    /// Determines if a query is a mutation or not.
+    fn is_mutation_query(q: &sqlparser::ast::Query) -> bool {
+        use sqlparser::ast::*;
+
+        match q.body.as_ref() {
+            SetExpr::Insert(_) => true,
+            SetExpr::Update(_) => true,
+            SetExpr::Query(q) => Self::is_mutation_query(q),
+            _ => false,
+        }
+    }
+
+    fn database_activity_cache(&self) -> Cache<String, DatabaseActivityState> {
+        DATABASE_ACTIVITY_CACHE
+            .get_or_init(|| {
+                Cache::builder()
+                    .time_to_idle(Duration::from_secs(self.pool_settings.db_activity_ttl))
+                    .build()
+            })
+            .clone()
+    }
+
+    /// Check database activity state and reset it if necessary
+    fn database_activity_state(&self, db: &String) -> DatabaseActivityState {
+        let cache = self.database_activity_cache();
+
+        // Exists in cache
+        if cache.contains_key(db) {
+            return cache.get(db).unwrap();
+        }
+
+        // Not in cache
+        debug!("Adding database to cache: {}", db);
+
+        cache.insert(db.to_string(), DatabaseActivityState::Initializing);
+
+        // Set a timer to update the cache
+        let db = db.clone();
+        let db_activity_init_delay = self.pool_settings.db_activity_init_delay;
+        tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_millis(db_activity_init_delay)).await;
+            cache.insert(db, DatabaseActivityState::Active);
+        });
+
+        DatabaseActivityState::Initializing
+    }
+
    /// Try to infer which server to connect to based on the contents of the query.
    pub fn infer(&mut self, ast: &Vec<sqlparser::ast::Statement>) -> Result<(), Error> {
        if !self.pool_settings.query_parser_read_write_splitting {
@@ -400,9 +468,23 @@ impl QueryRouter {
            return Err(Error::QueryRouterParserError("empty query".into()));
        }

+        let mut primary_set_based_on_activity = false;
        let mut visited_write_statement = false;
        let mut prev_inferred_shard = None;

+        if self.pool_settings.db_activity_based_routing {
+            let db = self.pool_settings.db.clone();
+            let state = self.database_activity_state(&db);
+            debug!("Database activity state: {:?}", state);
+
+            if let DatabaseActivityState::Initializing = state {
+                debug!("Database is initializing, going to primary");
+
+                self.active_role = Some(Role::Primary);
+                primary_set_based_on_activity = true;
+            }
+        }
+
        for q in ast {
            match q {
                // All transactions go to the primary, probably a write.
@@ -413,6 +495,22 @@ impl QueryRouter {

                // Likely a read-only query
                Query(query) => {
+                    if primary_set_based_on_activity {
+                        // If we already set the role based on activity, we don't need to do it again
+                        continue;
+                    }
+
+                    if self.pool_settings.db_activity_based_routing {
+                        // Check if the tables in the query have been written to recently
+                        if self.query_handles_tables_in_mutation_cache(query) {
+                            debug!("Query handles tables in mutation cache, going to primary");
+
+                            self.active_role = Some(Role::Primary);
+                            primary_set_based_on_activity = true;
+                            continue;
+                        }
+                    }
+
                    match &self.pool_settings.automatic_sharding_key {
                        Some(_) => {
                            // TODO: if we have multiple queries in the same message,
@@ -427,8 +525,13 @@ impl QueryRouter {
                        None => (),
                    };

-                    // If we already visited a write statement, we should be going to the primary.
-                    if !visited_write_statement {
+                    let has_locks = !query.locks.is_empty();
+                    let has_mutation = Self::is_mutation_query(query);
+
+                    if has_locks || has_mutation {
+                        self.active_role = Some(Role::Primary);
+                    } else if !visited_write_statement {
+                        // If we already visited a write statement, we should be going to the primary.
                        self.active_role = match self.primary_reads_enabled() {
                            false => Some(Role::Replica), // If primary should not be receiving reads, use a replica.
                            true => None,                 // Any server role is fine in this case.
@@ -438,6 +541,13 @@ impl QueryRouter {

                // Likely a write
                _ => {
+                    debug!("Write statement found, going to primary");
+
+                    if self.pool_settings.db_activity_based_routing {
+                        // add all of the query tables to the mutation cache
+                        self.update_mutation_cache_on_write(q);
+                    }
+
                    match &self.pool_settings.automatic_sharding_key {
                        Some(_) => {
                            // TODO: similar to the above, if we have multiple queries in the
@@ -480,57 +590,69 @@ impl QueryRouter {
        Ok(())
    }

-    fn infer_shard_on_write(&mut self, q: &Statement) -> Result<Option<usize>, Error> {
-        let mut exprs = Vec::new();
+    fn table_mutations_cache(&self) -> Cache<String, bool> {
+        TABLE_MUTATIONS_CACHE
+            .get_or_init(|| {
+                Cache::builder()
+                    .time_to_live(Duration::from_millis(
+                        self.pool_settings.table_mutation_cache_ms_ttl,
+                    ))
+                    .build()
+            })
+            .clone()
+    }

-        // Collect all table names from the query.
+    fn query_handles_tables_in_mutation_cache(&self, query: &sqlparser::ast::Query) -> bool {
+        let table_mutations_cache = self.table_mutations_cache();
+        debug!("Checking if query handles tables in mutation cache");
+        debug!("Table mutations cache: {:?}", table_mutations_cache);
+
+        for tables in self.table_names(query) {
+            for table in tables {
+                if table_mutations_cache.contains_key(&self.table_mutation_cache_key(table)) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+    fn extract_exprs_and_table_names<'a>(
+        &'a self,
+        q: &'a Statement,
+    ) -> Option<ExtractedExprsAndTables<'a>> {
+        let mut exprs = Vec::new();
        let mut table_names = Vec::new();
+        let mut assignments_opt = None;

        match q {
-            Insert {
-                or,
-                into: _,
-                table_name,
-                columns,
-                overwrite: _,
-                source,
-                partitioned,
-                after_columns,
-                table: _,
-                on: _,
-                returning: _,
-            } => {
+            Insert(i) => {
                // Not supported in postgres.
-                assert!(or.is_none());
-                assert!(partitioned.is_none());
-                assert!(after_columns.is_empty());
+                assert!(i.or.is_none());
+                assert!(i.partitioned.is_none());
+                assert!(i.after_columns.is_empty());

-                Self::process_table(table_name, &mut table_names);
-                Self::process_query(source, &mut exprs, &mut table_names, &Some(columns));
+                Self::process_table(&i.table_name, &mut table_names);
+                if let Some(source) = &i.source {
+                    Self::process_query(source, &mut exprs, &mut table_names, &Some(&i.columns));
+                }
            }
-            Delete {
-                tables,
-                from,
-                using,
-                selection,
-                returning: _,
-            } => {
-                if let Some(expr) = selection {
+            Delete(d) => {
+                if let Some(expr) = &d.selection {
                    exprs.push(expr.clone());
                }

-                // Multi tables delete are not supported in postgres.
-                assert!(tables.is_empty());
+                // Multi-tables delete are not supported in postgres.
+                assert!(d.tables.is_empty());

-                Self::process_tables_with_join(from, &mut exprs, &mut table_names);
-                if let Some(using_tbl_with_join) = using {
+                if let Some(using_tbl_with_join) = &d.using {
                    Self::process_tables_with_join(
                        using_tbl_with_join,
                        &mut exprs,
                        &mut table_names,
                    );
                }
-                Self::process_selection(selection, &mut exprs);
+                Self::process_selection(&d.selection, &mut exprs);
            }
            Update {
                table,
@@ -544,14 +666,55 @@ impl QueryRouter {
                    Self::process_table_with_join(from_tbl, &mut exprs, &mut table_names);
                }
                Self::process_selection(selection, &mut exprs);
-                self.assignment_parser(assignments)?;
-            }
-            _ => {
-                return Ok(None);
+
+                assignments_opt = Some(assignments);
            }
+            _ => return None,
        };

-        Ok(self.infer_shard_from_exprs(exprs, table_names))
+        Some(ExtractedExprsAndTables {
+            exprs,
+            table_names,
+            assignments_opt,
+        })
+    }
+
+    fn infer_shard_on_write(&mut self, q: &Statement) -> Result<Option<usize>, Error> {
+        if let Some(extracted) = self.extract_exprs_and_table_names(q) {
+            let exprs = extracted.exprs;
+            let table_names = extracted.table_names;
+            let assignments_opt = extracted.assignments_opt;
+
+            if let Some(assignments) = assignments_opt {
+                self.assignment_parser(assignments)?;
+            }
+
+            Ok(self.infer_shard_from_exprs(exprs, table_names))
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn update_mutation_cache_on_write(&self, q: &Statement) {
+        if let Some(extracted) = self.extract_exprs_and_table_names(q) {
+            debug!("Updating mutation cache on write");
+
+            let table_names = extracted.table_names;
+            debug!("Table names in mutation query: {:?}", table_names);
+            let table_mutations_cache = self.table_mutations_cache();
+            for tables in table_names {
+                for table in tables {
+                    table_mutations_cache.insert(self.table_mutation_cache_key(table), true);
+                }
+            }
+        }
+    }
+
+    // combines the database name and table name into a single string
+    // to be used as the key in the table mutation cache
+    // e.g. "mydb.mytable"
+    fn table_mutation_cache_key(&self, table: Ident) -> String {
+        format!("{}.{}", self.pool_settings.db, table.value)
    }

    fn process_query(
@@ -800,7 +963,13 @@ impl QueryRouter {

        for a in assignments {
            if sharding_key[0].value == "*"
-                && sharding_key[1].value == a.id.last().unwrap().value.to_lowercase()
+                && sharding_key[1].value
+                    == a.target
+                        .to_string()
+                        .split('.')
+                        .last()
+                        .unwrap()
+                        .to_lowercase()
            {
                return Err(Error::QueryRouterParserError(
                    "Sharding key cannot be updated.".into(),
@@ -933,6 +1102,18 @@ impl QueryRouter {
        self.infer_shard_from_exprs(exprs, table_names)
    }

+    /// get table names from query
+    fn table_names(&self, query: &sqlparser::ast::Query) -> Vec<Vec<Ident>> {
+        let mut exprs = Vec::new();
+
+        let mut table_names = Vec::new();
+        Self::process_query(query, &mut exprs, &mut table_names, &None);
+
+        debug!("Table names in query: {:?}", table_names);
+
+        table_names
+    }
+
    fn infer_shard_from_exprs(
        &mut self,
        exprs: Vec<Expr>,
@@ -1039,6 +1220,11 @@ impl QueryRouter {
        self.active_shard
    }

+    /// Set active_role as the default_role specified in the pool.
+    pub fn set_default_role(&mut self) {
+        self.active_role = self.pool_settings.default_role;
+    }
+
    /// Get the current desired server role we should be talking to.
    pub fn role(&self) -> Option<Role> {
        self.active_role
@@ -1095,6 +1281,7 @@ mod test {
    use crate::messages::simple_query;
    use crate::sharding::ShardingFunction;
    use bytes::BufMut;
+    use serial_test::serial;

    #[test]
    fn test_defaults() {
@@ -1104,6 +1291,26 @@ mod test {
        assert_eq!(qr.role(), None);
    }

+    #[test]
+    fn test_split_cte_queries() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.query_parser_read_write_splitting = true;
+        qr.pool_settings.query_parser_enabled = true;
+
+        let query = simple_query(
+            "WITH t AS (
+                SELECT id FROM users WHERE name ILIKE '%ja%'
+            )
+            UPDATE user_languages
+               SET settings = '{}'
+              FROM t WHERE t.id = user_id;",
+        );
+        let ast = qr.parse(&query).unwrap();
+        assert!(qr.infer(&ast).is_ok());
+        assert_eq!(qr.role(), Some(Role::Primary));
+    }
+
    #[test]
    fn test_infer_replica() {
        QueryRouter::setup();
@@ -1153,6 +1360,29 @@ mod test {
        }
    }

+    #[test]
+    fn test_select_for_update() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.query_parser_read_write_splitting = true;
+
+        let queries_in_primary_role = vec![
+            simple_query("BEGIN"), // Transaction start
+            simple_query("SELECT * FROM items WHERE id = 5 FOR UPDATE"),
+            simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"),
+        ];
+
+        for query in queries_in_primary_role {
+            assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok());
+            assert_eq!(qr.role(), Some(Role::Primary));
+        }
+
+        // query without lock do not change role
+        let query = simple_query("SELECT * FROM items WHERE id = 5");
+        assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok());
+        assert_eq!(qr.role(), None);
+    }
+
    #[test]
    fn test_infer_primary_reads_enabled() {
        QueryRouter::setup();
@@ -1367,6 +1597,19 @@ mod test {
        assert!(!qr.query_parser_enabled());
    }

+    #[test]
+    fn test_query_parser() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.query_parser_read_write_splitting = true;
+
+        let query = simple_query("SELECT req_tab_0.*  FROM validation req_tab_0  WHERE  array['http://www.w3.org/ns/shacl#ValidationResult'] && req_tab_0.type::text[] AND ( (  (req_tab_0.focusnode = 'DataSource_Credilogic_DataSourceAddress_144959227') )  )");
+        assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok());
+
+        let query = simple_query("WITH EmployeeSalaries AS (SELECT Department, Salary FROM Employees) SELECT Department, AVG(Salary) AS AverageSalary FROM EmployeeSalaries GROUP BY Department;");
+        assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok());
+    }
+
    #[test]
    fn test_update_from_pool_settings() {
        QueryRouter::setup();
@@ -1374,6 +1617,7 @@ mod test {
        let pool_settings = PoolSettings {
            pool_mode: PoolMode::Transaction,
            load_balancing_mode: crate::config::LoadBalancingMode::Random,
+            checkout_failure_limit: None,
            shards: 2,
            user: crate::config::User::default(),
            default_role: Some(Role::Replica),
@@ -1394,6 +1638,10 @@ mod test {
            auth_query_password: None,
            auth_query_user: None,
            db: "test".to_string(),
+            db_activity_based_routing: PoolSettings::default().db_activity_based_routing,
+            db_activity_init_delay: PoolSettings::default().db_activity_init_delay,
+            db_activity_ttl: PoolSettings::default().db_activity_ttl,
+            table_mutation_cache_ms_ttl: PoolSettings::default().table_mutation_cache_ms_ttl,
            plugins: None,
        };
        let mut qr = QueryRouter::new();
@@ -1452,6 +1700,7 @@ mod test {
        let pool_settings = PoolSettings {
            pool_mode: PoolMode::Transaction,
            load_balancing_mode: crate::config::LoadBalancingMode::Random,
+            checkout_failure_limit: Some(10),
            shards: 5,
            user: crate::config::User::default(),
            default_role: Some(Role::Replica),
@@ -1472,6 +1721,10 @@ mod test {
            auth_query_password: None,
            auth_query_user: None,
            db: "test".to_string(),
+            db_activity_based_routing: PoolSettings::default().db_activity_based_routing,
+            db_activity_init_delay: PoolSettings::default().db_activity_init_delay,
+            db_activity_ttl: PoolSettings::default().db_activity_ttl,
+            table_mutation_cache_ms_ttl: PoolSettings::default().table_mutation_cache_ms_ttl,
            plugins: None,
        };

@@ -1887,4 +2140,150 @@ mod test {

        assert_eq!(res, Ok(PluginOutput::Allow));
    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_db_activity_based_routing_initializing_state() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.db_activity_based_routing = true;
+        qr.pool_settings.query_parser_read_write_splitting = true;
+        qr.pool_settings.query_parser_enabled = true;
+        qr.pool_settings.db = "test_table_mutation_cache".to_string();
+
+        qr.database_activity_cache()
+            .invalidate(&qr.pool_settings.db.clone());
+
+        let query = simple_query("SELECT * FROM some_table");
+        let ast = qr.parse(&query).unwrap();
+
+        // Initially, the database activity should be in the "Initializing" state
+        let state = qr.database_activity_state(&qr.pool_settings.db.clone());
+        assert_eq!(state, DatabaseActivityState::Initializing);
+
+        // Check that the router chooses the primary role due to "Initializing" state
+        assert!(qr.infer(&ast).is_ok());
+        assert_eq!(qr.role(), Some(Role::Primary));
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_db_activity_based_routing_active_state() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.db_activity_based_routing = true;
+        qr.pool_settings.query_parser_read_write_splitting = true;
+        qr.pool_settings.query_parser_enabled = true;
+        qr.pool_settings.db = "test_table_mutation_cache".to_string();
+
+        let db_name = qr.pool_settings.db.clone();
+        let cache = qr.database_activity_cache();
+        cache.insert(db_name.clone(), DatabaseActivityState::Active);
+
+        let query = simple_query("SELECT * FROM some_table");
+        let ast = qr.parse(&query).unwrap();
+
+        // Check that the router can choose a replica role when in "Active" state
+        assert!(qr.infer(&ast).is_ok());
+        assert_eq!(qr.role(), None); // Default should allow replica due to active state
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_table_mutation_cache_on_write() {
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+        qr.pool_settings.db_activity_based_routing = true;
+        qr.pool_settings.table_mutation_cache_ms_ttl = 20_000; // 20 seconds in milliseconds
+        qr.pool_settings.query_parser_enabled = true;
+        qr.pool_settings.query_parser_read_write_splitting = true;
+        qr.pool_settings.db = "test_table_mutation_cache".to_string();
+
+        qr.database_activity_cache()
+            .invalidate(&qr.pool_settings.db.clone());
+
+        let query = simple_query("UPDATE some_table SET col1 = 'value' WHERE col2 = 1");
+        let ast = qr.parse(&query).unwrap();
+
+        // Simulate the mutation query which should populate the mutation cache
+        assert!(qr.infer(&ast).is_ok());
+        assert_eq!(qr.role(), Some(Role::Primary));
+
+        let table_cache_key = qr.table_mutation_cache_key(Ident::new("some_table"));
+        let cache = qr.table_mutations_cache();
+
+        // Ensure the table mutation cache contains the table with recent write
+        assert!(cache.contains_key(&table_cache_key));
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_db_activity_based_routing_multi_query() {
+        use super::*;
+        use crate::messages::simple_query;
+        use tokio::time::Duration;
+
+        QueryRouter::setup();
+        let mut qr = QueryRouter::new();
+
+        // Configure the pool settings for db_activity_based_routing
+        qr.pool_settings.query_parser_read_write_splitting = true;
+        qr.pool_settings.query_parser_enabled = true;
+        qr.pool_settings.db_activity_based_routing = true;
+        qr.pool_settings.db = "test_db_activity_routing".to_string();
+
+        qr.database_activity_cache()
+            .invalidate(&qr.pool_settings.db.clone());
+
+        // First query when database is initializing
+        let query = simple_query("SELECT * FROM test_table");
+        let ast = qr.parse(&query).unwrap();
+        assert!(qr.infer(&ast).is_ok());
+        // Should route to primary because database is initializing
+        assert_eq!(qr.role(), Some(Role::Primary));
+
+        // Wait for the initialization delay to pass
+        tokio::time::sleep(Duration::from_millis(
+            qr.pool_settings.db_activity_init_delay * 2,
+        ))
+        .await;
+
+        // Next query after database is active
+        let query = simple_query("SELECT * FROM test_table");
+        let ast = qr.parse(&query).unwrap();
+        qr.active_role = None; // Reset the active_role
+        assert!(qr.infer(&ast).is_ok());
+        // Should route to replica because database is active and no recent mutations
+        assert_eq!(qr.role(), None);
+
+        // Simulate a write query to update the mutation cache
+        let query = simple_query("INSERT INTO test_table (id, name) VALUES (1, 'test')");
+        let ast = qr.parse(&query).unwrap();
+        qr.active_role = None; // Reset the active_role
+        assert!(qr.infer(&ast).is_ok());
+        // Should route to primary because it's a write operation
+        assert_eq!(qr.role(), Some(Role::Primary));
+
+        // Immediately run a read query on the same table
+        let query = simple_query("SELECT * FROM test_table WHERE id = 1");
+        let ast = qr.parse(&query).unwrap();
+        qr.active_role = None; // Reset the active_role
+        assert!(qr.infer(&ast).is_ok());
+        // Should route to primary because the table was recently mutated
+        assert_eq!(qr.role(), Some(Role::Primary));
+
+        // Wait for the mutation cache TTL to expire
+        tokio::time::sleep(Duration::from_millis(
+            qr.pool_settings.table_mutation_cache_ms_ttl * 2,
+        ))
+        .await;
+
+        // Run the read query again after cache expiration
+        let query = simple_query("SELECT * FROM test_table WHERE id = 1");
+        let ast = qr.parse(&query).unwrap();
+        qr.active_role = None; // Reset the active_role
+        assert!(qr.infer(&ast).is_ok());
+        // Should route to replica because mutation cache has expired
+        assert_eq!(qr.role(), None);
+    }
 }
--- a/src/sharding.rs
+++ b/src/sharding.rs
@@ -14,11 +14,11 @@ pub enum ShardingFunction {
    Sha1,
 }

-impl ToString for ShardingFunction {
-    fn to_string(&self) -> String {
-        match *self {
-            ShardingFunction::PgBigintHash => "pg_bigint_hash".to_string(),
-            ShardingFunction::Sha1 => "sha1".to_string(),
+impl std::fmt::Display for ShardingFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ShardingFunction::PgBigintHash => write!(f, "pg_bigint_hash"),
+            ShardingFunction::Sha1 => write!(f, "sha1"),
        }
    }
 }
--- a/start_test_env.sh
+++ b/start_test_env.sh
@@ -0,0 +1,34 @@
+GREEN="\033[0;32m"
+RED="\033[0;31m"
+BLUE="\033[0;34m"
+RESET="\033[0m"
+
+
+cd tests/docker/
+docker compose kill main || true
+docker compose build main
+docker compose down
+docker compose up -d
+# wait for the container to start
+while ! docker compose exec main ls; do
+    echo "Waiting for test environment to start"
+    sleep 1
+done
+echo "==================================="
+docker compose exec -e LOG_LEVEL=error -d main toxiproxy-server
+docker compose exec --workdir /app main cargo build
+docker compose exec -d --workdir /app main ./target/debug/pgcat ./.circleci/pgcat.toml
+docker compose exec --workdir /app/tests/ruby main bundle install
+docker compose exec --workdir /app/tests/python main pip3 install -r requirements.txt
+echo "Interactive test environment ready"
+echo "To run integration tests, you can use the following commands:"
+echo -e "   ${BLUE}Ruby:   ${RED}cd /app/tests/ruby && bundle exec ruby tests.rb --format documentation${RESET}"
+echo -e "   ${BLUE}Python: ${RED}cd /app/ && pytest ${RESET}"
+echo -e "   ${BLUE}Rust:   ${RED}cd /app/tests/rust && cargo run ${RESET}"
+echo -e "   ${BLUE}Go:     ${RED}cd /app/tests/go && /usr/local/go/bin/go test${RESET}"
+echo "the source code for tests are directly linked to the source code in the container so you can modify the code and run the tests again"
+echo "You can rebuild PgCat from within the container by running"
+echo -e "  ${GREEN}cargo build${RESET}"
+echo "and then run the tests again"
+echo "==================================="
+docker compose exec --workdir /app/tests main bash
--- a/tests/docker/docker-compose.yml
+++ b/tests/docker/docker-compose.yml
@@ -1,4 +1,3 @@
-version: "3"
 services:
  pg1:
    image: postgres:14
@@ -48,6 +47,8 @@ services:
  main:
    build: .
    command: ["bash", "/app/tests/docker/run.sh"]
+    environment:
+      - INTERACTIVE_TEST_ENVIRONMENT=true
    volumes:
      - ../../:/app/
      - /app/target/
--- a/tests/docker/run.sh
+++ b/tests/docker/run.sh
@@ -5,6 +5,38 @@ rm /app/*.profraw || true
 rm /app/pgcat.profdata || true
 rm -rf /app/cov || true

+# Prepares the interactive test environment
+# 
+if [ -n "$INTERACTIVE_TEST_ENVIRONMENT" ]; then
+    ports=(5432 7432 8432 9432 10432)
+    for port in "${ports[@]}"; do
+        is_it_up=0
+        attempts=0
+        while [ $is_it_up -eq 0 ]; do
+            PGPASSWORD=postgres psql -h 127.0.0.1 -p $port -U postgres -c '\q' > /dev/null 2>&1
+            if [ $? -eq 0 ]; then
+                echo "PostgreSQL on port $port is up."
+                is_it_up=1
+            else
+                attempts=$((attempts+1))
+                if [ $attempts -gt 10 ]; then
+                    echo "PostgreSQL on port $port is down, giving up."
+                    exit 1
+                fi
+                echo "PostgreSQL on port $port is down, waiting for it to start."
+                sleep 1
+            fi
+        done
+    done
+    PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432  -U postgres -f /app/tests/sharding/query_routing_setup.sql
+    PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 7432  -U postgres -f /app/tests/sharding/query_routing_setup.sql
+    PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 8432  -U postgres -f /app/tests/sharding/query_routing_setup.sql
+    PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 9432  -U postgres -f /app/tests/sharding/query_routing_setup.sql
+    PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 10432 -U postgres -f /app/tests/sharding/query_routing_setup.sql
+    sleep 100000000000000000
+    exit 0
+fi
+
 export LLVM_PROFILE_FILE="/app/pgcat-%m-%p.profraw"
 export RUSTC_BOOTSTRAP=1
 export CARGO_INCREMENTAL=0
--- a/tests/python/requirements.txt
+++ b/tests/python/requirements.txt
@@ -1,2 +1,3 @@
+pytest
 psycopg2==2.9.3
-psutil==5.9.1
+psutil==5.9.1
--- a/tests/python/test_auth.py
+++ b/tests/python/test_auth.py
@@ -0,0 +1,71 @@
+import utils
+import signal
+
+class TestTrustAuth:
+    @classmethod
+    def setup_method(cls):
+        config= """
+        [general]
+        host = "0.0.0.0"
+        port = 6432
+        admin_username = "admin_user"
+        admin_password = ""
+        admin_auth_type = "trust"
+
+        [pools.sharded_db.users.0]
+        username = "sharding_user"
+        password = "sharding_user"
+        auth_type = "trust"
+        pool_size = 10
+        min_pool_size = 1
+        pool_mode = "transaction"
+
+        [pools.sharded_db.shards.0]
+        servers = [
+          [ "127.0.0.1", 5432, "primary" ],
+        ]
+        database = "shard0"
+        """
+        utils.pgcat_generic_start(config)
+
+    @classmethod
+    def teardown_method(self):
+        utils.pg_cat_send_signal(signal.SIGTERM)
+
+    def test_admin_trust_auth(self):
+        conn, cur = utils.connect_db_trust(admin=True)
+        cur.execute("SHOW POOLS")
+        res = cur.fetchall()
+        print(res)
+        utils.cleanup_conn(conn, cur)
+
+    def test_normal_trust_auth(self):
+        conn, cur = utils.connect_db_trust(autocommit=False)
+        cur.execute("SELECT 1")
+        res = cur.fetchall()
+        print(res)
+        utils.cleanup_conn(conn, cur)
+
+class TestMD5Auth:
+    @classmethod
+    def setup_method(cls):
+        utils.pgcat_start()
+
+    @classmethod
+    def teardown_method(self):
+        utils.pg_cat_send_signal(signal.SIGTERM)
+
+    def test_normal_db_access(self):
+        conn, cur = utils.connect_db(autocommit=False)
+        cur.execute("SELECT 1")
+        res = cur.fetchall()
+        print(res)
+        utils.cleanup_conn(conn, cur)
+
+    def test_admin_db_access(self):
+        conn, cur = utils.connect_db(admin=True)
+
+        cur.execute("SHOW POOLS")
+        res = cur.fetchall()
+        print(res)
+        utils.cleanup_conn(conn, cur)
--- a/tests/python/test_pgcat.py
+++ b/tests/python/test_pgcat.py
@@ -1,84 +1,12 @@
-from typing import Tuple
-import psycopg2
-import psutil
-import os
+
 import signal
 import time

+import psycopg2
+import utils
+
 SHUTDOWN_TIMEOUT = 5

-PGCAT_HOST = "127.0.0.1"
-PGCAT_PORT = "6432"
-
-
-def pgcat_start():
-    pg_cat_send_signal(signal.SIGTERM)
-    os.system("./target/debug/pgcat .circleci/pgcat.toml &")
-    time.sleep(2)
-
-
-def pg_cat_send_signal(signal: signal.Signals):
-    try:
-        for proc in psutil.process_iter(["pid", "name"]):
-            if "pgcat" == proc.name():
-                os.kill(proc.pid, signal)
-    except Exception as e:
-        # The process can be gone when we send this signal
-        print(e)
-
-    if signal == signal.SIGTERM:
-        # Returns 0 if pgcat process exists
-        time.sleep(2)
-        if not os.system('pgrep pgcat'):
-            raise Exception("pgcat not closed after SIGTERM")
-
-
-def connect_db(
-    autocommit: bool = True,
-    admin: bool = False,
-) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]:
-
-    if admin:
-        user = "admin_user"
-        password = "admin_pass"
-        db = "pgcat"
-    else:
-        user = "sharding_user"
-        password = "sharding_user"
-        db = "sharded_db"
-
-    conn = psycopg2.connect(
-        f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat",
-        connect_timeout=2,
-    )
-    conn.autocommit = autocommit
-    cur = conn.cursor()
-
-    return (conn, cur)
-
-
-def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor):
-    cur.close()
-    conn.close()
-
-
-def test_normal_db_access():
-    pgcat_start()
-    conn, cur = connect_db(autocommit=False)
-    cur.execute("SELECT 1")
-    res = cur.fetchall()
-    print(res)
-    cleanup_conn(conn, cur)
-
-
-def test_admin_db_access():
-    conn, cur = connect_db(admin=True)
-
-    cur.execute("SHOW POOLS")
-    res = cur.fetchall()
-    print(res)
-    cleanup_conn(conn, cur)
-

 def test_shutdown_logic():

@@ -86,17 +14,17 @@ def test_shutdown_logic():
    # NO ACTIVE QUERIES SIGINT HANDLING

    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and send query (not in transaction)
-    conn, cur = connect_db()
+    conn, cur = utils.connect_db()

    cur.execute("BEGIN;")
    cur.execute("SELECT 1;")
    cur.execute("COMMIT;")

    # Send sigint to pgcat
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)
    time.sleep(1)

    # Check that any new queries fail after sigint since server should close with no active transactions
@@ -108,18 +36,18 @@ def test_shutdown_logic():
        # Fail if query execution succeeded
        raise Exception("Server not closed after sigint")

-    cleanup_conn(conn, cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(conn, cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # NO ACTIVE QUERIES ADMIN SHUTDOWN COMMAND

    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    conn, cur = connect_db()
-    admin_conn, admin_cur = connect_db(admin=True)
+    conn, cur = utils.connect_db()
+    admin_conn, admin_cur = utils.connect_db(admin=True)

    cur.execute("BEGIN;")
    cur.execute("SELECT 1;")
@@ -138,24 +66,24 @@ def test_shutdown_logic():
        # Fail if query execution succeeded
        raise Exception("Server not closed after sigint")

-    cleanup_conn(conn, cur)
-    cleanup_conn(admin_conn, admin_cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(conn, cur)
+    utils.cleanup_conn(admin_conn, admin_cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # HANDLE TRANSACTION WITH SIGINT

    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    conn, cur = connect_db()
+    conn, cur = utils.connect_db()

    cur.execute("BEGIN;")
    cur.execute("SELECT 1;")

    # Send sigint to pgcat while still in transaction
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)
    time.sleep(1)

    # Check that any new queries succeed after sigint since server should still allow transaction to complete
@@ -165,18 +93,18 @@ def test_shutdown_logic():
        # Fail if query fails since server closed
        raise Exception("Server closed while in transaction", e.pgerror)

-    cleanup_conn(conn, cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(conn, cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # HANDLE TRANSACTION WITH ADMIN SHUTDOWN COMMAND

    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    conn, cur = connect_db()
-    admin_conn, admin_cur = connect_db(admin=True)
+    conn, cur = utils.connect_db()
+    admin_conn, admin_cur = utils.connect_db(admin=True)

    cur.execute("BEGIN;")
    cur.execute("SELECT 1;")
@@ -194,30 +122,30 @@ def test_shutdown_logic():
        # Fail if query fails since server closed
        raise Exception("Server closed while in transaction", e.pgerror)

-    cleanup_conn(conn, cur)
-    cleanup_conn(admin_conn, admin_cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(conn, cur)
+    utils.cleanup_conn(admin_conn, admin_cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # NO NEW NON-ADMIN CONNECTIONS DURING SHUTDOWN
    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    transaction_conn, transaction_cur = connect_db()
+    transaction_conn, transaction_cur = utils.connect_db()

    transaction_cur.execute("BEGIN;")
    transaction_cur.execute("SELECT 1;")

    # Send sigint to pgcat while still in transaction
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)
    time.sleep(1)

    start = time.perf_counter()
    try:
-        conn, cur = connect_db()
+        conn, cur = utils.connect_db()
        cur.execute("SELECT 1;")
-        cleanup_conn(conn, cur)
+        utils.cleanup_conn(conn, cur)
    except psycopg2.OperationalError as e:
        time_taken = time.perf_counter() - start
        if time_taken > 0.1:
@@ -227,49 +155,49 @@ def test_shutdown_logic():
    else:
        raise Exception("Able connect to database during shutdown")

-    cleanup_conn(transaction_conn, transaction_cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(transaction_conn, transaction_cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # ALLOW NEW ADMIN CONNECTIONS DURING SHUTDOWN
    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    transaction_conn, transaction_cur = connect_db()
+    transaction_conn, transaction_cur = utils.connect_db()

    transaction_cur.execute("BEGIN;")
    transaction_cur.execute("SELECT 1;")

    # Send sigint to pgcat while still in transaction
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)
    time.sleep(1)

    try:
-        conn, cur = connect_db(admin=True)
+        conn, cur = utils.connect_db(admin=True)
        cur.execute("SHOW DATABASES;")
-        cleanup_conn(conn, cur)
+        utils.cleanup_conn(conn, cur)
    except psycopg2.OperationalError as e:
        raise Exception(e)

-    cleanup_conn(transaction_conn, transaction_cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(transaction_conn, transaction_cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # ADMIN CONNECTIONS CONTINUING TO WORK AFTER SHUTDOWN
    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction
-    transaction_conn, transaction_cur = connect_db()
+    transaction_conn, transaction_cur = utils.connect_db()
    transaction_cur.execute("BEGIN;")
    transaction_cur.execute("SELECT 1;")

-    admin_conn, admin_cur = connect_db(admin=True)
+    admin_conn, admin_cur = utils.connect_db(admin=True)
    admin_cur.execute("SHOW DATABASES;")

    # Send sigint to pgcat while still in transaction
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)
    time.sleep(1)

    try:
@@ -277,24 +205,24 @@ def test_shutdown_logic():
    except psycopg2.OperationalError as e:
        raise Exception("Could not execute admin command:", e)

-    cleanup_conn(transaction_conn, transaction_cur)
-    cleanup_conn(admin_conn, admin_cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(transaction_conn, transaction_cur)
+    utils.cleanup_conn(admin_conn, admin_cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
    # HANDLE SHUTDOWN TIMEOUT WITH SIGINT

    # Start pgcat
-    pgcat_start()
+    utils.pgcat_start()

    # Create client connection and begin transaction, which should prevent server shutdown unless shutdown timeout is reached
-    conn, cur = connect_db()
+    conn, cur = utils.connect_db()

    cur.execute("BEGIN;")
    cur.execute("SELECT 1;")

    # Send sigint to pgcat while still in transaction
-    pg_cat_send_signal(signal.SIGINT)
+    utils.pg_cat_send_signal(signal.SIGINT)

    # pgcat shutdown timeout is set to SHUTDOWN_TIMEOUT seconds, so we sleep for SHUTDOWN_TIMEOUT + 1 seconds
    time.sleep(SHUTDOWN_TIMEOUT + 1)
@@ -308,12 +236,7 @@ def test_shutdown_logic():
        # Fail if query execution succeeded
        raise Exception("Server not closed after sigint and expected timeout")

-    cleanup_conn(conn, cur)
-    pg_cat_send_signal(signal.SIGTERM)
+    utils.cleanup_conn(conn, cur)
+    utils.pg_cat_send_signal(signal.SIGTERM)

    # - - - - - - - - - - - - - - - - - -
-
-
-test_normal_db_access()
-test_admin_db_access()
-test_shutdown_logic()
--- a/tests/python/utils.py
+++ b/tests/python/utils.py
@@ -0,0 +1,110 @@
+import os
+import signal
+import time
+from typing import Tuple
+import tempfile
+
+import psutil
+import psycopg2
+
+PGCAT_HOST = "127.0.0.1"
+PGCAT_PORT = "6432"
+
+
+def _pgcat_start(config_path: str):
+    pg_cat_send_signal(signal.SIGTERM)
+    os.system(f"./target/debug/pgcat {config_path} &")
+    time.sleep(2)
+
+
+def pgcat_start():
+    _pgcat_start(config_path='.circleci/pgcat.toml')
+
+
+def pgcat_generic_start(config: str):
+    tmp = tempfile.NamedTemporaryFile()
+    with open(tmp.name, 'w') as f:
+        f.write(config)
+    _pgcat_start(config_path=tmp.name)
+
+
+def glauth_send_signal(signal: signal.Signals):
+    try:
+        for proc in psutil.process_iter(["pid", "name"]):
+            if proc.name() == "glauth":
+                os.kill(proc.pid, signal)
+    except Exception as e:
+        # The process can be gone when we send this signal
+        print(e)
+
+    if signal == signal.SIGTERM:
+        # Returns 0 if pgcat process exists
+        time.sleep(2)
+        if not os.system('pgrep glauth'):
+            raise Exception("glauth not closed after SIGTERM")
+
+
+def pg_cat_send_signal(signal: signal.Signals):
+    try:
+        for proc in psutil.process_iter(["pid", "name"]):
+            if "pgcat" == proc.name():
+                os.kill(proc.pid, signal)
+    except Exception as e:
+        # The process can be gone when we send this signal
+        print(e)
+
+    if signal == signal.SIGTERM:
+        # Returns 0 if pgcat process exists
+        time.sleep(2)
+        if not os.system('pgrep pgcat'):
+            raise Exception("pgcat not closed after SIGTERM")
+
+
+def connect_db(
+    autocommit: bool = True,
+    admin: bool = False,
+) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]:
+
+    if admin:
+        user = "admin_user"
+        password = "admin_pass"
+        db = "pgcat"
+    else:
+        user = "sharding_user"
+        password = "sharding_user"
+        db = "sharded_db"
+
+    conn = psycopg2.connect(
+        f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat",
+        connect_timeout=2,
+    )
+    conn.autocommit = autocommit
+    cur = conn.cursor()
+
+    return (conn, cur)
+
+def connect_db_trust(
+    autocommit: bool = True,
+    admin: bool = False,
+) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]:
+
+    if admin:
+        user = "admin_user"
+        db = "pgcat"
+    else:
+        user = "sharding_user"
+        db = "sharded_db"
+
+    conn = psycopg2.connect(
+        f"postgres://{user}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat",
+        connect_timeout=2,
+    )
+    conn.autocommit = autocommit
+    cur = conn.cursor()
+
+    return (conn, cur)
+
+
+def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor):
+    cur.close()
+    conn.close()
--- a/tests/ruby/Gemfile.lock
+++ b/tests/ruby/Gemfile.lock
@@ -1,22 +1,33 @@
 GEM
  remote: https://rubygems.org/
  specs:
-    activemodel (7.0.4.1)
-      activesupport (= 7.0.4.1)
-    activerecord (7.0.4.1)
-      activemodel (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
-    activesupport (7.0.4.1)
+    activemodel (7.1.4)
+      activesupport (= 7.1.4)
+    activerecord (7.1.4)
+      activemodel (= 7.1.4)
+      activesupport (= 7.1.4)
+      timeout (>= 0.4.0)
+    activesupport (7.1.4)
+      base64
+      bigdecimal
      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
+      mutex_m
      tzinfo (~> 2.0)
    ast (2.4.2)
-    concurrent-ruby (1.1.10)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
    diff-lcs (1.5.0)
-    i18n (1.12.0)
+    drb (2.2.1)
+    i18n (1.14.5)
      concurrent-ruby (~> 1.0)
-    minitest (5.17.0)
+    minitest (5.25.1)
+    mutex_m (0.2.0)
    parallel (1.22.1)
    parser (3.1.2.0)
      ast (~> 2.4.1)
@@ -24,7 +35,8 @@ GEM
    pg (1.3.2)
    rainbow (3.1.1)
    regexp_parser (2.3.1)
-    rexml (3.2.5)
+    rexml (3.3.6)
+      strscan
    rspec (3.11.0)
      rspec-core (~> 3.11.0)
      rspec-expectations (~> 3.11.0)
@@ -50,10 +62,12 @@ GEM
    rubocop-ast (1.17.0)
      parser (>= 3.1.1.0)
    ruby-progressbar (1.11.0)
+    strscan (3.1.0)
+    timeout (0.4.1)
    toml (0.3.0)
      parslet (>= 1.8.0, < 3.0.0)
    toxiproxy (2.0.1)
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
      concurrent-ruby (~> 1.0)
    unicode-display_width (2.1.0)

--- a/tests/ruby/admin_spec.rb
+++ b/tests/ruby/admin_spec.rb
@@ -91,6 +91,27 @@ describe "Admin" do
    end
  end

+  [
+    "SHOW ME THE MONEY", 
+    "SHOW ME THE WAY", 
+    "SHOW UP", 
+    "SHOWTIME",
+    "HAMMER TIME",
+    "SHOWN TO BE TRUE",
+    "SHOW ",
+    "SHOW     ",
+    "SHOW 1",
+    ";;;;;"
+  ].each do |cmd|
+    describe "Bad command #{cmd}" do
+      it "does not panic and responds with PG::SystemError" do
+        admin_conn = PG::connect(processes.pgcat.admin_connection_string)
+        expect { admin_conn.async_exec(cmd) }.to raise_error(PG::SystemError).with_message(/Unsupported/)
+        admin_conn.close
+      end
+    end
+  end
+
  describe "PAUSE" do
    it "pauses all pools" do
      admin_conn = PG::connect(processes.pgcat.admin_connection_string)
--- a/tests/ruby/load_balancing_spec.rb
+++ b/tests/ruby/load_balancing_spec.rb
@@ -56,6 +56,41 @@ describe "Random Load Balancing" do
      end
    end
  end
+
+  context "when all replicas are down " do
+    let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction", "random", "debug", {"default_role" => "replica"}) }
+
+    it "unbans them automatically to prevent false positives in health checks that could make all replicas unavailable" do
+      conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+      failed_count = 0
+      number_of_replicas = processes[:replicas].length
+
+      # Take down all replicas
+      processes[:replicas].each(&:take_down)
+
+      (number_of_replicas + 1).times do |n|
+        conn.async_exec("SELECT 1 + 2")
+      rescue
+        conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+        failed_count += 1
+      end
+
+      expect(failed_count).to eq(number_of_replicas + 1)
+      failed_count = 0
+
+      # Ban_time is configured to 60 so this reset will only work
+      # if the replicas are unbanned automatically
+      processes[:replicas].each(&:reset)
+
+      number_of_replicas.times do
+        conn.async_exec("SELECT 1 + 2")
+      rescue
+        conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+        failed_count += 1
+      end
+      expect(failed_count).to eq(0)
+    end
+  end
 end

 describe "Least Outstanding Queries Load Balancing" do
@@ -161,4 +196,3 @@ describe "Least Outstanding Queries Load Balancing" do
    end
  end
 end
-
--- a/tests/ruby/misc_spec.rb
+++ b/tests/ruby/misc_spec.rb
@@ -188,6 +188,102 @@ describe "Miscellaneous" do
    end
  end

+  describe "Checkout failure limit" do
+    context "when no checkout failure limit is set" do
+      before do 
+        new_configs = processes.pgcat.current_config
+        new_configs["general"]["connect_timeout"] = 200
+        new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1
+        processes.pgcat.update_config(new_configs)
+        processes.pgcat.reload_config
+        sleep 0.5
+      end
+  
+      it "does not disconnect client" do
+        Array.new(5) do
+          Thread.new do
+            conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+            for i in 0..4
+              begin
+                conn.async_exec("SELECT pg_sleep(0.5);")
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              rescue PG::SystemError
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              end
+            end
+            conn.close
+          end
+        end.each(&:join)
+      end
+    end
+
+    context "when checkout failure limit is set high" do
+      before do 
+        new_configs = processes.pgcat.current_config
+        new_configs["general"]["connect_timeout"] = 200
+        new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1
+        new_configs["pools"]["sharded_db"]["checkout_failure_limit"] = 10000
+        processes.pgcat.update_config(new_configs)
+        processes.pgcat.reload_config
+        sleep 0.5
+      end
+  
+      it "does not disconnect client" do
+        Array.new(5) do
+          Thread.new do
+            conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+            for i in 0..4
+              begin
+                conn.async_exec("SELECT pg_sleep(0.5);")
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              rescue PG::SystemError
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              end
+            end
+            conn.close
+          end
+        end.each(&:join)
+      end
+    end
+
+    context "when checkout failure limit is set low" do
+      before do 
+        new_configs = processes.pgcat.current_config
+        new_configs["general"]["connect_timeout"] = 200
+        new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1
+        new_configs["pools"]["sharded_db"]["checkout_failure_limit"] = 2
+        processes.pgcat.update_config(new_configs)
+        processes.pgcat.reload_config
+        sleep 0.5
+      end
+  
+      it "disconnects client after reaching limit" do
+        Array.new(5) do
+          Thread.new do
+            conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
+            checkout_failure_count = 0
+            for i in 0..4
+              begin
+                conn.async_exec("SELECT pg_sleep(1);")
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              rescue PG::SystemError
+                checkout_failure_count += 1
+                expect(conn.status).to eq(PG::CONNECTION_OK)
+              rescue PG::ConnectionBad
+                expect(checkout_failure_count).to eq(2)
+                expect(conn.status).to eq(PG::CONNECTION_BAD)
+                break
+              end
+            end
+            conn.close
+          end
+        end.each(&:join)
+        puts processes.pgcat.logs
+
+      end
+    end
+  end
+  
  describe "Server version reporting" do
    it "reports correct version for normal and admin databases" do
      server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
--- a/tests/rust/Cargo.lock
+++ b/tests/rust/Cargo.lock
--- a/tests/rust/src/main.rs
+++ b/tests/rust/src/main.rs
@@ -15,13 +15,11 @@ async fn test_prepared_statements() {
    for _ in 0..5 {
        let pool = pool.clone();
        let handle = tokio::task::spawn(async move {
-            for _ in 0..1000 {
-                match sqlx::query("SELECT one").fetch_all(&pool).await {
+            for i in 0..1000 {
+                match sqlx::query(&format!("SELECT {:?}", i % 5)).fetch_all(&pool).await {
                    Ok(_) => (),
                    Err(err) => {
-                        if err.to_string().contains("prepared statement") {
-                            panic!("prepared statement error: {}", err);
-                        }
+                        panic!("prepared statement error: {}", err);
                    }
                }
            }
Author	SHA1	Message	Date
Mostafa	e299a2e71a	fmt	2025-02-27 07:26:17 -06:00
Mostafa	00ac44427f	clippy	2025-02-27 07:24:23 -06:00
Mostafa	147eba52c0	Add checkout_failure_limit config/feature	2025-02-27 07:17:10 -06:00
Alex Kesling	f8e2fcd0ed	s/Iniitalize/Initialize/ (#897 )	2025-01-09 11:59:42 -08:00
Nadav Shatz	3202f5685b	Add DB activity based routing (#864 )	2024-12-22 05:23:57 -06:00
Gabriel Simmer	b37d105184	chore(deps): bump sqlparser from 0.41.0 to 0.52.0 (#870 ) * chore(deps): bump sqlparser from 0.41.0 to 0.52.0 Bumps [sqlparser](https://github.com/apache/datafusion-sqlparser-rs) from 0.41.0 to 0.52.0. - [Changelog](https://github.com/apache/datafusion-sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion-sqlparser-rs/commits) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * bump * Update to latest sqlparser version --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mostafa <mostafa.mohmmed@gmail.com>	2024-11-23 07:25:37 -06:00
Vitalii Tverdokhlib	32f4752daf	DOCS: small typo LIMT (Update README.md) (#866 )	2024-11-17 06:51:05 -06:00
Mostafa	3796e26402	Fix contact info for Helm chart (#861 ) * Fix contact info for Helm chart	2024-11-11 09:24:17 -06:00
Mostafa	0ee59c0c40	Another no-op helm release (#853 )	2024-11-08 06:07:12 -06:00
Mostafa	b61d2cc6f0	Use main branch for helm chart releases (#852 )	2024-11-08 06:04:42 -06:00
Jose Fernández	c11418c083	Revert "Do not unban replicas if a primary is available" (#850 ) Revert "Do not unban replicas if a primary is available (#843)" This reverts commit `cdcfa99fb9`.	2024-11-07 22:00:43 +01:00
Jose Fernández	c9544bdff2	Fix `default_role` being ignored when `query_parser_enabled` was false (#847 ) Fix default_role being ignored when query_parser_enabled was false	2024-11-07 11:11:49 -06:00
Jose Fernández	cdcfa99fb9	Do not unban replicas if a primary is available (#843 ) Add `unban_replicas_when_all_banned` to control unbanning replicas behavior.	2024-11-07 11:11:11 -06:00
Víťa Tauer	f27dc6b483	Fixing invalid setting name in pgcat.toml (#849 )	2024-11-07 06:17:09 -06:00
Mostafa	326efc22b3	Another no-op release for helm (#845 ) Another no-op release	2024-11-02 18:05:41 -05:00
Mostafa	01c6afb2e5	Attempt a helm chart release (#844 ) Attempt a release Co-authored-by: Mostafa <no_reply@github.com>	2024-11-02 11:55:18 -05:00
Nicolas Vanelslande	a68071dd28	Bump bb8 from 0.8.1 to 0.8.6 (#709 ) * Update bb8 to 0.8.6 To get https://github.com/djc/bb8/pull/186 and https://github.com/djc/bb8/pull/189 which fix potential deadlocks (https://github.com/djc/bb8/issues/154). Also, this (https://github.com/djc/bb8/pull/225) was needed to prevent a connection leak which was conveniently spotted in our integration tests. * Ignore ./.bundle (created by dev console) --------- Co-authored-by: Jose Fernandez (magec) <joseferper@gmail.com>	2024-10-28 06:49:36 -05:00
Mostafa	c27d801abf	Rename a couple of variables (#839 )	2024-10-23 06:38:07 -05:00
Javier Goday	186e72298f	#829 : read/write splitting on CTE mutable statements (#835 )	2024-10-23 06:20:04 -05:00
Sebastian Serth	3935366d86	End Prometheus stats with a new line separator (#826 ) End prometheus stats with a new line separator According to the [OpenMetrics specification](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#overall-structure), each line MUST end with `\n`. Previously, the last line was not ending with `\n`, so that strict parsers had issues reading the Prometheus stats.	2024-09-22 17:14:04 -05:00
Sean McGivern	b575935b1d	Improve documentation for connect_timeout and add min_pool_size (#822 ) Currently, `connect_timeout` sounds like it should be for connections to the Postgres server. It's actually used for obtaining a connection from the pool.	2024-09-18 06:56:17 -05:00
Shijun Wang	efbab1c333	Helm chart improvements including allowing user password to be pulled from K8s secret (#753 ) * Make user min_pool_size configurable * Set user server_lifetime only if specified * Increment chart version * Use default instea of or * Allow enabling server_tls * statement_timeout default value * Allow pulling password from existing secret --------- Co-authored-by: Mostafa Abdelraouf <mostafa.mohmmed@gmail.com>	2024-09-14 09:57:17 -05:00
Mostafa Abdelraouf	9f12d7958e	Fix Ruby tests (#819 ) Build is failing with this error Downloading activerecord-3.2.14 revealed dependencies not in the API or the lockfile (activesupport (= 3.2.14), activemodel (= 3.2.14), arel (~> 3.0.2), tzinfo (~> 0.3.29)). Either installing with `--full-index` or running `bundle update activerecord` should fix the problem. After ActiveSupport was updated. This PR fixes that	2024-09-13 20:02:38 -05:00
dependabot[bot]	e6634ef461	chore(deps): bump activesupport from 7.0.4.1 to 7.0.7.1 in /tests/ruby (#804 ) Bumps [activesupport](https://github.com/rails/rails) from 7.0.4.1 to 7.0.7.1. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v7.2.1/activesupport/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v7.0.4.1...v7.0.7.1) --- updated-dependencies: - dependency-name: activesupport dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-13 19:43:26 -05:00
dependabot[bot]	dab2e58647	chore(deps): bump helm/chart-releaser-action from 1.5.0 to 1.6.0 (#812 ) Bumps [helm/chart-releaser-action](https://github.com/helm/chart-releaser-action) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/helm/chart-releaser-action/releases) - [Commits](`be16258da8...a917fd15b2`) --- updated-dependencies: - dependency-name: helm/chart-releaser-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-13 19:41:25 -05:00
dependabot[bot]	4aaa4378cf	chore(deps): bump rexml from 3.2.8 to 3.3.6 in /tests/ruby (#803 ) Bumps [rexml](https://github.com/ruby/rexml) from 3.2.8 to 3.3.6. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.8...v3.3.6) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-13 19:19:30 -05:00
Andrew Jackson	670311daf9	Implement Trust Authentication (#805 ) * Implement Trust Authentication * Remove remaining LDAP stuff * Reverted LDAP changes, Cleaned up tests --------- Co-authored-by: Andrew Jackson <andrewjackson2988@gmail.com> Co-authored-by: CommanderKeynes <andrewjackson947@gmail.coma>	2024-09-10 09:29:45 -05:00
dependabot[bot]	b9ec7f8036	chore(deps): bump actions/setup-python from 4.1.0 to 5.1.0 (#715 ) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.1.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.1.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-07 12:21:21 -05:00
dependabot[bot]	d91d23848b	chore(deps): bump helm/kind-action from 1.7.0 to 1.10.0 (#732 ) Bumps [helm/kind-action](https://github.com/helm/kind-action) from 1.7.0 to 1.10.0. - [Release notes](https://github.com/helm/kind-action/releases) - [Commits](https://github.com/helm/kind-action/compare/v1.7.0...v1.10.0) --- updated-dependencies: - dependency-name: helm/kind-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-07 12:20:38 -05:00
dependabot[bot]	bbbc01a467	chore(deps): bump rexml from 3.2.5 to 3.2.8 in /tests/ruby (#743 ) Bumps [rexml](https://github.com/ruby/rexml) from 3.2.5 to 3.2.8. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.5...v3.2.8) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-07 12:20:01 -05:00
Sebastian Serth	9bb71ede9d	Automatically build deb package on a new version tag (#801 ) In #796, I noticed that the deb package was not build since an automation was missing. With this PR, I add the missing automation. I tested the workflow in my repo... when starting the workflow manually: https://github.com/MrSerth/pgcat/actions/runs/10737879151/job/29780286094 when drafting a new release: https://github.com/MrSerth/pgcat/actions/runs/10737835796/job/29780146212 Obviously, both workflows failed since I cannot upload to the APT repo. However, the version substitution for the workflow is working correctly (as shown when collapsing the first line of the "Build and release package" step).	2024-09-06 09:11:52 -05:00
Sebastian Serth	88b2afb19b	Automatically start systemd service if config file is present (#800 ) Previously, upgrading the deb package stopped the service but didn't reenable it after a successful upgrade. This made upgrading the package more difficult and required a second step to restart the service. With this commit, the systemd service is automatically started when the default config file is present.	2024-09-06 09:07:01 -05:00
Mostafa Abdelraouf	f0865ca616	Improve Prometheus exporter output (#795 ) * Prometheus metrics updates: * Add username label to deconflict metrics that would otherwise have duplicate labels across different pools. * Group metrics by name and only print HELP and TYPE once per metric name. * Sort labels for a deterministic output. --------- Co-authored-by: Curtis Myzie <curtis.myzie@gmail.com> Co-authored-by: Towhid Khan	2024-09-05 08:58:18 -05:00
Andrew Jackson	7d047c6c19	Implemented python tests with pytest (#790 ) Currently the python tests act as scripts. A lot of output is generated to stdout which makes it very hard to figure out where problems were. Also if you want to run only a single test you basically need to comment out code in order to accomplish this. This PR modifies the python tests to us the pytest python testing framework. This framework allows individual tests to be targeted via the command line, without touching the source code. It also suppressed stdout by default making the test output much easier to read. Also after the tests run it will provide a summary of what failed, what succeded, etc. Co-authored-by: CommanderKeynes <andrewjackson947@gmail.coma> Co-authored-by: Andrew Jackson <andrewjackson2988@gmail.com>	2024-09-05 08:16:45 -05:00
Andrew Jackson	f73d15f82c	Fix CI script to allow consecutive runs locally (#793 ) Co-authored-by: CommanderKeynes <andrewjackson947@gmail.coma>	2024-09-05 08:01:33 -05:00
Mostafa Abdelraouf	69af6cc5e5	Make iterating on integration tests easier (#789 ) Writing and iterating on integration tests are cumbersome, having to wait 10 minutes for the test-suite to run just to see if your test works or not is unacceptable. In this PR, I added a detailed workflow for writing tests that should shorten the feedback cycle of modifying tests to be as low as a few seconds. It will involve opening a shell into a long-lived container that has all the setup and dependencies necessary and then running your desired tests directly there. I added a convenience script that bootstraps the environment and then opens an interactive shell into the container and you can then run tests immediately in an environment that is more or less identical to what we have running in CircleCI	2024-09-03 11:15:53 -05:00
Mostafa Abdelraouf	ca34597002	Fix broken integration test #740 (#787 )	2024-08-31 17:15:13 -05:00
Mostafa Abdelraouf	2def40ea6a	Add test case for issue 776 (#786 ) I am adding a tiny test that uses the SQL statement that was reported to break an older version of SQL parser library #776	2024-08-31 10:52:33 -05:00
Mostafa Abdelraouf	c05129018d	Improve Prometheus stats + Add Grafana dashboard (#785 ) We were missing some labels on metrics generated by the Prometheus exporter so I fixed that. There are still some gaps that I want to address with respect to the metrics we track but this seems like a good start. I also created a Grafana Dashboard and exported it to JSON. It is designed with the same metric names the Prometheus exporter uses.	2024-08-31 08:18:57 -05:00
Mostafa Abdelraouf	4a7a6a8e7a	Cut 1.2.0 release (#783 )	2024-08-30 08:30:16 -05:00
Mostafa Abdelraouf	29a476e190	QueryRouter: route to primary when locks exists (select for update) (#782 ) Authored-by: Javier Goday <jgoday@gmail.com>	2024-08-30 04:26:36 -05:00
KwongTN	81933b918d	Add linux/arm64 docker image build support (#774 )	2024-08-29 13:50:38 -05:00
Saraj Munjal	7cbc9178d8	Bump the hyper crate to v1.4.1 and rework prometheus server handling (#778 ) Bump hyper to v1.4.1 and rework prometheus server handling	2024-08-29 09:47:58 -05:00
Mostafa Abdelraouf	2c8b2f0776	Fix CI image build step (#780 ) The docker CI build image is failing due to this error 249.5 Finished release [optimized] target(s) in 2m 49s 249.5 Installing /home/circleci/.cargo/bin/rustfilt 249.5 Installed package `rustfilt v0.2.1` (executable `rustfilt`) 249.5 error: failed to compile `cargo-binutils v0.3.6`, intermediate artifacts can be found at `/tmp/cargo-installrWENQG` 249.5 249.5 Caused by: 249.5 package `cargo-platform v0.1.8` cannot be built because it requires rustc 1.73 or newer, while the currently active rustc version is 1.67.1 249.5 Try re-running cargo install with `--locked` 249.5 Summary Successfully installed rustfilt! Failed to install cargo-binutils (see error(s) above). 249.5 error: some crates failed to install So I am bumping the version up	2024-08-29 08:37:13 -05:00
Mostafa Abdelraouf	8f9a2b8e6f	Fix a Panic in admin commands (#779 ) We have a panic when we send SHOW or ;;;;;;;;;;;;;;;;; to admin database. This PR fixes these panics and adds a couple of tests	2024-08-28 21:29:40 -05:00
brandonpike	cbf4d58144	Fix lint warnings for rust-1.79 (#769 ) 2 things that are recommended by rust-lang - implementing `std::fmt::Display` rather than ToString (1) and using clone_from (2). [1] https://rust-lang.github.io/rust-clippy/master/index.html#/to_string_trait_impl [2] https://rust-lang.github.io/rust-clippy/master/index.html#assigning_clones Signed-off-by: Brandon Pike <pikebrandon@att.net>	2024-07-15 20:30:26 -07:00
Олег Дулецкий	731aa047ba	Add ExecReload option to pgcat.service for configuration reloads (#760 )	2024-06-24 08:57:58 -07:00
Adrian Garcia Badaracco	88dbcc21d1	update rust version in docker image (#762 )	2024-06-24 08:51:38 -07:00
Adrian Garcia Badaracco	c34b15bddc	Add STOPSIGNAL to Dockerfile (#758 )	2024-06-20 23:23:41 -07:00
Andrey Stikheev	0b034a6831	Add TCP_NODELAY option to improve performance for large response queries (#749 ) This commit adds the TCP_NODELAY option to the socket configuration in `configure_socket` function. Without this option, we observed significant performance issues when executing SELECT queries with large responses. Before the fix: postgres=> SELECT repeat('a', 1); SELECT repeat('a', 8153); Time: 1.368 ms Time: 41.364 ms After the fix: postgres=> SELECT repeat('a', 1); SELECT repeat('a', 8153); Time: 1.332 ms Time: 1.528 ms By setting TCP_NODELAY, we eliminate the Nagle's algorithm delay, which results in a substantial improvement in response times for large queries. This problem was discussed in https://github.com/postgresml/pgcat/issues/616.	2024-05-26 14:47:21 -07:00
Mostafa Abdelraouf	966b8e093c	Report checkout error when all servers are down (#736 ) We shouldn't report checkout_success when we are going to return Error.	2024-05-08 12:18:27 -05:00
Horacio	c9270a47d4	Use rust:bullseye as base image (#725 ) Use rust:bullseye base image With the original rust:1.70-bullseye image, the container cannot be built: 17.06 Installing /usr/local/cargo/bin/rustfilt 17.06 Installed package `rustfilt v0.2.1` (executable `rustfilt`) 17.06 error: failed to compile `cargo-binutils v0.3.6`, intermediate artifacts can be found at `/tmp/cargo-installrc6mPb` 17.06 17.06 Caused by: 17.06 package `cargo-platform v0.1.8` cannot be built because it requires rustc 1.73 or newer, while the currently active rustc version is 1.70.0 17.06 Try re-running cargo install with `--locked` 17.06 Summary Successfully installed rustfilt! Failed to install cargo-binutils (see error(s) above). 17.06 error: some crates failed to install This is the same base image used on tests/docker/Dockerfile	2024-04-19 09:12:57 -07:00
Toby Hede	0d94d0b90a	Update sqlparser to 0.41 (#666 )	2024-04-12 22:12:37 -07:00