docker/pgcat
1
0
Fork 0
mirror of https://github.com/postgresml/pgcat.git synced 2026-03-23 09:26:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: docker:v0.3.0
Branches Tags
docker:main docker:mostafa_test-helm docker:mostafa_test-new-ci-image docker:mostafa_bump-rust-version docker:circleci_JaLnYn_jalnyn/update-rust-17-18 docker:mostafa_checkout-limit docker:mostafa_avoid-holding-on-to-old-pool docker:circleci_TailorBrands_enh/db-activity-routing docker:dependabot/cargo/chrono-0.4.39 docker:dependabot/cargo/tokio-rustls-0.26.1 docker:dependabot/cargo/webpki-roots-0.26.7 docker:circleci_gmemstr_sqlparser-0.52.0 docker:dependabot/cargo/sqlparser-0.52.0 docker:circleci_OneSignal_replica-with-failover docker:gh-pages docker:gh-page-verified2 docker:mostafa_rebuild-charts docker:gh-pages-update docker:circleci_nothrow_fail-mode-for-default-shard docker:mostafa_another-release-helm docker:mostafa_update-helm-hook docker:circleci_OneSignal_fix-default-role docker:mostafa_another-helm-release docker:mostafa_attempt-to-cut-helm-release docker:circleci_OneSignal_don-t-unban-all-replicas docker:dependabot/bundler/tests/ruby/rexml-3.3.9 docker:dependabot/github_actions/actions/setup-python-5.3.0 docker:mostafa_rename-some-methods docker:circleci_jgoday_splitting_cte docker:circleci_MrSerth_prometheus-line-separator docker:dependabot/cargo/once_cell-1.20.0 docker:mostafa_fix-build-for-rails docker:mostafa_webpki-up docker:mostafa_sqlparse-up docker:circleci_AndrewJackson2020_trust docker:dependabot/github_actions/docker/login-action-3 docker:circleci_smcgivern_track-transaction-time docker:circleci_AndrewJackson2020_kerberos_fix docker:circleci_sarajmunjal_saraj/upgrade-tls docker:circleci_pgEdge_fix/duplicate-prometheus-metrics docker:circleci_AndrewJackson2020_pytest docker:mostafa_fix_prepared_stmts docker:mostafa_development-guide docker:mostafa_fix-broken-test docker:mostafa_add-test-for-sqlparser docker:mostafa_stats-grafana docker:mostafa_cut-1-2-0-release docker:circleci_jgoday_fix_select_for_update docker:dependabot/cargo/itertools-0.13.0 docker:dependabot/cargo/base64-0.22.1 docker:dependabot/cargo/rustls-0.21.12 docker:dependabot/cargo/rustls-0.21.11 docker:dependabot/cargo/h2-0.3.26 docker:dependabot/cargo/tests/rust/whoami-1.5.1 docker:dependabot/cargo/tests/rust/mio-0.8.11 docker:dependabot/cargo/mio-0.8.11 docker:dependabot/github_actions/docker/metadata-action-5 docker:dependabot/github_actions/docker/build-push-action-5 docker:dependabot/github_actions/helm/chart-testing-action-2.6.1 docker:dependabot/cargo/clap-4.4.18 docker:levkk-max-wait-time-for-now docker:levkk-bump-version docker:levkk-tls-dont-stop docker:levkk-deb-package docker:kczimm-meager-mods docker:levkk-v1.1 docker:levkk-fix-lint-5 docker:dependabot/cargo/fallible-iterator-0.3.0 docker:dependabot/cargo/socket2-0.5.3 docker:levkk-lowercase docker:levkk-more-plugins docker:levkk-start-server-tls docker:levkk-async docker:levkk-auth-mod-rebased docker:mostafa_seamless-retry docker:levkk-auto-shard-key docker:levkk-distributed-copy docker:levkk-search-path docker:levkk-drop-dirty-conn docker:levkk-log-events-more docker:levkk-fix-debug-log docker:levkk-fix-sven docker:sven_md5_auth docker:query-router-docs
docker:pgcat-0.2.5 docker:pgcat-0.2.4 docker:pgcat-0.2.3 docker:pgcat-0.2.2 docker:pgcat-0.2.1 docker:v1.2.0 docker:pgcat-0.2.0 docker:pgcat-0.1.0 docker:v1.1.1 docker:v1.1.0 docker:v1.0.0 docker:v0.3.0 docker:v0.2.1-beta1 docker:v0.2.0-beta1 docker:v0.1.0-beta3 docker:v0.1.0-beta2 docker:v0.1.0-beta docker:v0.1.0-alpha docker:sharded docker:simple
..
compare: docker:levkk-fix-debug-log
Branches Tags
docker:mostafa_test-helm docker:mostafa_test-new-ci-image docker:main docker:mostafa_bump-rust-version docker:circleci_JaLnYn_jalnyn/update-rust-17-18 docker:mostafa_checkout-limit docker:mostafa_avoid-holding-on-to-old-pool docker:circleci_TailorBrands_enh/db-activity-routing docker:dependabot/cargo/chrono-0.4.39 docker:dependabot/cargo/tokio-rustls-0.26.1 docker:dependabot/cargo/webpki-roots-0.26.7 docker:circleci_gmemstr_sqlparser-0.52.0 docker:dependabot/cargo/sqlparser-0.52.0 docker:circleci_OneSignal_replica-with-failover docker:gh-pages docker:gh-page-verified2 docker:mostafa_rebuild-charts docker:gh-pages-update docker:circleci_nothrow_fail-mode-for-default-shard docker:mostafa_another-release-helm docker:mostafa_update-helm-hook docker:circleci_OneSignal_fix-default-role docker:mostafa_another-helm-release docker:mostafa_attempt-to-cut-helm-release docker:circleci_OneSignal_don-t-unban-all-replicas docker:dependabot/bundler/tests/ruby/rexml-3.3.9 docker:dependabot/github_actions/actions/setup-python-5.3.0 docker:mostafa_rename-some-methods docker:circleci_jgoday_splitting_cte docker:circleci_MrSerth_prometheus-line-separator docker:dependabot/cargo/once_cell-1.20.0 docker:mostafa_fix-build-for-rails docker:mostafa_webpki-up docker:mostafa_sqlparse-up docker:circleci_AndrewJackson2020_trust docker:dependabot/github_actions/docker/login-action-3 docker:circleci_smcgivern_track-transaction-time docker:circleci_AndrewJackson2020_kerberos_fix docker:circleci_sarajmunjal_saraj/upgrade-tls docker:circleci_pgEdge_fix/duplicate-prometheus-metrics docker:circleci_AndrewJackson2020_pytest docker:mostafa_fix_prepared_stmts docker:mostafa_development-guide docker:mostafa_fix-broken-test docker:mostafa_add-test-for-sqlparser docker:mostafa_stats-grafana docker:mostafa_cut-1-2-0-release docker:circleci_jgoday_fix_select_for_update docker:dependabot/cargo/itertools-0.13.0 docker:dependabot/cargo/base64-0.22.1 docker:dependabot/cargo/rustls-0.21.12 docker:dependabot/cargo/rustls-0.21.11 docker:dependabot/cargo/h2-0.3.26 docker:dependabot/cargo/tests/rust/whoami-1.5.1 docker:dependabot/cargo/tests/rust/mio-0.8.11 docker:dependabot/cargo/mio-0.8.11 docker:dependabot/github_actions/docker/metadata-action-5 docker:dependabot/github_actions/docker/build-push-action-5 docker:dependabot/github_actions/helm/chart-testing-action-2.6.1 docker:dependabot/cargo/clap-4.4.18 docker:levkk-max-wait-time-for-now docker:levkk-bump-version docker:levkk-tls-dont-stop docker:levkk-deb-package docker:kczimm-meager-mods docker:levkk-v1.1 docker:levkk-fix-lint-5 docker:dependabot/cargo/fallible-iterator-0.3.0 docker:dependabot/cargo/socket2-0.5.3 docker:levkk-lowercase docker:levkk-more-plugins docker:levkk-start-server-tls docker:levkk-async docker:levkk-auth-mod-rebased docker:mostafa_seamless-retry docker:levkk-auto-shard-key docker:levkk-distributed-copy docker:levkk-search-path docker:levkk-drop-dirty-conn docker:levkk-log-events-more docker:levkk-fix-debug-log docker:levkk-fix-sven docker:sven_md5_auth docker:query-router-docs
docker:pgcat-0.2.5 docker:pgcat-0.2.4 docker:pgcat-0.2.3 docker:pgcat-0.2.2 docker:pgcat-0.2.1 docker:v1.2.0 docker:pgcat-0.2.0 docker:pgcat-0.1.0 docker:v1.1.1 docker:v1.1.0 docker:v1.0.0 docker:v0.3.0 docker:v0.2.1-beta1 docker:v0.2.0-beta1 docker:v0.1.0-beta3 docker:v0.1.0-beta2 docker:v0.1.0-beta docker:v0.1.0-alpha docker:sharded docker:simple

1 Commits
v0.3.0 ... levkk-fix-

Author SHA1 Message Date
Lev Kokotov
28172cc1d5 Fix debug log 2022-08-11 22:47:22 -07:00
38 changed files with 1365 additions and 3639 deletions
Expand all files Collapse all files

26
.circleci/config.yml
View File

@@ -15,34 +15,14 @@ jobs:
RUSTFLAGS: "-C instrument-coverage"
LLVM_PROFILE_FILE: "pgcat-%m.profraw"
- image: postgres:14
command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements"]
# auth:
# username: mydockerhub-user
# password: $DOCKERHUB_PASSWORD
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
- image: postgres:14
command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements"]
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
- image: postgres:14
command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements"]
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
- image: postgres:14
command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements"]
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
# Add steps to the job
# See: https://circleci.com/docs/2.0/configuration-reference/#steps
steps:

8
.circleci/pgcat.toml
View File

@@ -11,12 +11,9 @@ host = "0.0.0.0"
# Port to run on, same as PgBouncer used in this example.
port = 6432
# Whether to enable prometheus exporter or not.
# enable prometheus exporter on port 9930
enable_prometheus_exporter = true
# Port at which prometheus exporter listens on.
prometheus_exporter_port = 9930
# How long to wait before aborting a server connection (ms).
connect_timeout = 100
@@ -91,13 +88,11 @@ password = "sharding_user"
# The maximum number of connection from a single Pgcat process to any database in the cluster
# is the sum of pool_size across all users.
pool_size = 9
statement_timeout = 0
[pools.sharded_db.users.1]
username = "other_user"
password = "other_user"
pool_size = 21
statement_timeout = 30000
# Shard 0
[pools.sharded_db.shards.0]
@@ -135,7 +130,6 @@ sharding_function = "pg_bigint_hash"
username = "simple_user"
password = "simple_user"
pool_size = 5
statement_timeout = 30000
[pools.simple_db.shards.0]
servers = [

40
.circleci/run_tests.sh
View File

@@ -3,9 +3,6 @@
set -e
set -o xtrace
# non-zero exit code if we provide bad configs
(! ./target/debug/pgcat "fake_configs" 2>/dev/null)
# Start PgCat with a particular log level
# for inspection.
function start_pgcat() {
@@ -16,20 +13,17 @@ function start_pgcat() {
# Setup the database with shards and user
PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql
PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 7432 -U postgres -f tests/sharding/query_routing_setup.sql
PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 8432 -U postgres -f tests/sharding/query_routing_setup.sql
PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 9432 -U postgres -f tests/sharding/query_routing_setup.sql
PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i
PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i
PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i
# Install Toxiproxy to simulate a downed/slow database
wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb
sudo dpkg -i toxiproxy-2.4.0.deb
wget -O toxiproxy-2.1.4.deb https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy_2.1.4_amd64.deb
sudo dpkg -i toxiproxy-2.1.4.deb
# Start Toxiproxy
LOG_LEVEL=error toxiproxy-server &
toxiproxy-server &
sleep 1
# Create a database at port 5433, forward it to Postgres
@@ -72,26 +66,13 @@ psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_te
# Replica/primary selection & more sharding tests
psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null
# Statement timeout tests
sed -i 's/statement_timeout = 0/statement_timeout = 100/' .circleci/pgcat.toml
kill -SIGHUP $(pgrep pgcat) # Reload config
sleep 0.2
# This should timeout
(! psql -U sharding_user -e -h 127.0.0.1 -p 6432 -c 'select pg_sleep(0.5)')
# Disable statement timeout
sed -i 's/statement_timeout = 100/statement_timeout = 0/' .circleci/pgcat.toml
kill -SIGHUP $(pgrep pgcat) # Reload config again
#
# ActiveRecord tests
#
cd tests/ruby
sudo gem install bundler
bundle install
bundle exec ruby tests.rb || exit 1
bundle exec rspec *_spec.rb || exit 1
ruby tests.rb
cd ../..
#
@@ -99,7 +80,7 @@ cd ../..
# These tests will start and stop the pgcat server so it will need to be restarted after the tests
#
pip3 install -r tests/python/requirements.txt
python3 tests/python/tests.py || exit 1
python3 tests/python/tests.py
start_pgcat "info"
@@ -109,9 +90,9 @@ psql -U admin_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/n
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW LISTS' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW POOLS' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW VERSION' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null
psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore
(! psql -U admin_user -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null)
export PGPASSWORD=sharding_user
@@ -136,14 +117,11 @@ toxiproxy-cli toxic remove --toxicName latency_downstream postgres_replica
start_pgcat "info"
# Test session mode (and config reload)
sed -i '0,/simple_db/s/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml
sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml
# Reload config test
kill -SIGHUP $(pgrep pgcat)
# Revert settings after reload. Makes test runs idempotent
sed -i '0,/simple_db/s/pool_mode = "session"/pool_mode = "transaction"/' .circleci/pgcat.toml
sleep 1
# Prepared statements that will only work in session mode

113
Cargo.lock generated
View File

@@ -53,14 +53,14 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bb8"
version = "0.8.0"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582"
checksum = "2e9f4fa9768efd269499d8fba693260cfc670891cf6de3adc935588447a77cc8"
dependencies = [
"async-trait",
"futures-channel",
"futures-util",
"parking_lot 0.12.1",
"parking_lot",
"tokio",
]
@@ -159,12 +159,6 @@ dependencies = [
"termcolor",
]
[[package]]
name = "exitcode"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
[[package]]
name = "fnv"
version = "1.0.7"
@@ -493,17 +487,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.5",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.3",
"parking_lot_core",
]
[[package]]
@@ -520,19 +504,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "parking_lot_core"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-sys",
]
[[package]]
name = "pgcat"
version = "0.6.0-alpha1"
@@ -544,14 +515,13 @@ dependencies = [
"bytes",
"chrono",
"env_logger",
"exitcode",
"hmac",
"hyper",
"log",
"md-5",
"num_cpus",
"once_cell",
"parking_lot 0.12.1",
"parking_lot",
"phf",
"rand",
"regex",
@@ -569,19 +539,20 @@ dependencies = [
[[package]]
name = "phf"
version = "0.11.1"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c"
checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
dependencies = [
"phf_macros",
"phf_shared",
"proc-macro-hack",
]
[[package]]
name = "phf_generator"
version = "0.11.1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf"
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
dependencies = [
"phf_shared",
"rand",
@@ -589,12 +560,13 @@ dependencies = [
[[package]]
name = "phf_macros"
version = "0.11.1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66"
checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro-hack",
"proc-macro2",
"quote",
"syn",
@@ -602,9 +574,9 @@ dependencies = [
[[package]]
name = "phf_shared"
version = "0.11.1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
@@ -627,6 +599,12 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro2"
version = "1.0.36"
@@ -847,9 +825,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "sqlparser"
version = "0.23.0"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25"
checksum = "b8f192f29f4aa49e57bebd0aa05858e0a1f32dd270af36efe49edb82cbfffab6"
dependencies = [
"log",
]
@@ -928,7 +906,7 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
"parking_lot 0.11.2",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"tokio-macros",
@@ -1183,46 +1161,3 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
dependencies = [
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
[[package]]
name = "windows_i686_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
[[package]]
name = "windows_i686_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
[[package]]
name = "windows_x86_64_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"

9
Cargo.toml
View File

@@ -9,7 +9,7 @@ edition = "2021"
tokio = { version = "1", features = ["full"] }
bytes = "1"
md-5 = "0.10"
bb8 = "0.8.0"
bb8 = "0.7"
async-trait = "0.1"
rand = "0.8"
chrono = "0.4"
@@ -20,11 +20,11 @@ serde_derive = "1"
regex = "1"
num_cpus = "1"
once_cell = "1"
sqlparser = "0.23.0"
sqlparser = "0.14"
log = "0.4"
arc-swap = "1"
env_logger = "0.9"
parking_lot = "0.12.1"
parking_lot = "0.11"
hmac = "0.12"
sha2 = "0.10"
base64 = "0.13"
@@ -32,5 +32,4 @@ stringprep = "0.1"
tokio-rustls = "0.23"
rustls-pemfile = "1"
hyper = { version = "0.14", features = ["full"] }
phf = { version = "0.11.1", features = ["macros"] }
exitcode = "1.1.2"
phf = { version = "0.10", features = ["macros"] }

100
README.md
View File

@@ -1,11 +1,8 @@
![PgCat](./pgcat3.png)
##### PgCat: PostgreSQL at petabyte scale
# PgCat
[![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main)
<a href="https://discord.gg/DmyJP3qJ7U" target="_blank">
<img src="https://img.shields.io/discord/1013868243036930099" alt="Join our Discord!" />
</a>
![PgCat](./pgcat3.png)
PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover support.
@@ -18,7 +15,7 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su
| Session pooling | :white_check_mark: | Identical to PgBouncer. |
| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. |
| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. |
| Load balancing of read queries | :white_check_mark: | Using random between replicas. Primary is included when `primary_reads_enabled` is enabled (default). |
| Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). |
| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. |
| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. |
| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. |
@@ -41,35 +38,30 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1'
### Config
| **Name** | **Description** | **Examples** |
|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|
| **`general`** | | |
| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` |
| `port` | The pooler will run on this port. | `6432` |
| `enable_prometheus_exporter` | Enable prometheus exporter which will export metrics in prometheus exposition format. | `true` |
| `prometheus_exporter_port` | Port at which prometheus exporter listens on. | `9930` |
| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` |
| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` |
| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` |
| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` |
| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` |
| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` |
| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` |
| `autoreload` | Enable auto-reload of config after fixed time-interval. | `false` |
| | | |
| **`user`** | | |
| `name` | The user name. | `sharding_user` |
| `password` | The user password in plaintext. | `hunter2` |
| `statement_timeout` | Timeout in milliseconds for how long a query takes to execute | `0 (disabled)`
| | | |
| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` |
| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` |
| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | |
| | | |
| **`query_router`** | | |
| `default_role` | Traffic is routed to this role by default (random), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` |
| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` |
| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` |
| **Name** | **Description** | **Examples** |
|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|
| **`general`** | | |
| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` |
| `port` | The pooler will run on this port. | `6432` |
| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` |
| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` |
| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` |
| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` |
| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` |
| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` |
| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` |
| | | |
| **`user`** | | |
| `name` | The user name. | `sharding_user` |
| `password` | The user password in plaintext. | `hunter2` |
| | | |
| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` |
| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` |
| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | |
| **`query_router`** | | |
| `default_role` | Traffic is routed to this role by default (round-robin), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` |
| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` |
| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` |
## Local development
@@ -91,14 +83,6 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended
See [sharding README](./tests/sharding/README.md) for sharding logic testing.
Run `cargo test` to run Rust tests.
Run the following commands to run Integration tests locally.
```
cd tests/docker/
docker compose up --exit-code-from main # This will also produce coverage report under ./cov/
```
| **Feature** | **Tested in CI** | **Tested manually** | **Comments** |
|-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------|
| Transaction pooling | :white_check_mark: | :white_check_mark: | Used by default for all tests. |
@@ -124,7 +108,7 @@ In transaction mode, a client talks to one server for the duration of a single t
This mode is enabled by default.
### Load balancing of read queries
All queries are load balanced against the configured servers using the random algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries.
All queries are load balanced against the configured servers using the round-robin algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries.
If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary.
@@ -163,18 +147,18 @@ Failover behavior can get pretty interesting (read complex) when multiple config
| **Query** | **`SET SERVER ROLE TO`** | **`query_parser_enabled`** | **`primary_reads_enabled`** | **Target state** | **Outcome** |
|---------------------------|--------------------------|----------------------------|-----------------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the random loop. |
| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the random loop. |
| Read query | unset (any) | true | true | up | Query is routed to the first instance in the random loop. |
| Read query | replica | false | false | up | Query is routed to the first replica instance in the random loop. |
| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the round-robin loop. |
| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the round-robin loop. |
| Read query | unset (any) | true | true | up | Query is routed to the first instance in the round-robin loop. |
| Read query | replica | false | false | up | Query is routed to the first replica instance in the round-robin loop. |
| Read query | primary | false | false | up | Query is routed to the primary. |
| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the random loop is attempted. |
| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. |
| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the random loop. |
| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. |
| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the round-robin loop is attempted. |
| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. |
| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the round-robin loop. |
| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. |
| Read query | primary | false | false | down | The query is attempted against the primary and fails. The client receives an error. |
| | | | | | |
| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the random loop. If the instance is a replica, the query fails and the client receives an error. |
| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the round-robin loop. If the instance is a replica, the query fails and the client receives an error. |
| Write query | unset (any) | true | false | up | The query is routed to the primary. |
| Write query | unset (any) | true | true | up | The query is routed to the primary. |
| Write query | primary | false | false | up | The query is routed to the primary. |
@@ -459,7 +443,7 @@ Always good to have a base line.
```
$ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0
Password:
Password:
starting vacuum...end.
transaction type: <builtin: select only>
scaling factor: 1
@@ -473,7 +457,7 @@ tps = 139443.955722 (including connections establishing)
tps = 142314.859075 (excluding connections establishing)
$ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0
Password:
Password:
starting vacuum...end.
transaction type: <builtin: select only>
scaling factor: 1
@@ -487,7 +471,7 @@ tps = 150644.840891 (including connections establishing)
tps = 152218.499430 (excluding connections establishing)
$ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0
Password:
Password:
starting vacuum...end.
transaction type: <builtin: select only>
scaling factor: 1
@@ -501,7 +485,7 @@ tps = 152517.663404 (including connections establishing)
tps = 153319.188482 (excluding connections establishing)
$ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0
Password:
Password:
starting vacuum...end.
transaction type: <builtin: select only>
scaling factor: 1

2
docker-compose.yml
View File

@@ -1,7 +1,7 @@
version: "3"
services:
postgres:
image: postgres:14
image: postgres:13
environment:
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: md5

10
examples/docker/pgcat.toml
View File

@@ -11,12 +11,9 @@ host = "0.0.0.0"
# Port to run on, same as PgBouncer used in this example.
port = 6432
# Whether to enable prometheus exporter or not.
# enable prometheus exporter on port 9930
enable_prometheus_exporter = true
# Port at which prometheus exporter listens on.
prometheus_exporter_port = 9930
# How long to wait before aborting a server connection (ms).
connect_timeout = 5000
@@ -92,14 +89,10 @@ password = "postgres"
# is the sum of pool_size across all users.
pool_size = 9
# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way.
statement_timeout = 0
[pools.sharded.users.1]
username = "postgres"
password = "postgres"
pool_size = 21
statement_timeout = 15000
# Shard 0
[pools.sharded.shards.0]
@@ -137,7 +130,6 @@ sharding_function = "pg_bigint_hash"
username = "postgres"
password = "postgres"
pool_size = 5
statement_timeout = 0
[pools.simple_db.shards.0]
servers = [

10
pgcat.toml
View File

@@ -11,12 +11,9 @@ host = "0.0.0.0"
# Port to run on, same as PgBouncer used in this example.
port = 6432
# Whether to enable prometheus exporter or not.
# enable prometheus exporter on port 9930
enable_prometheus_exporter = true
# Port at which prometheus exporter listens on.
prometheus_exporter_port = 9930
# How long to wait before aborting a server connection (ms).
connect_timeout = 5000
@@ -92,14 +89,10 @@ password = "sharding_user"
# is the sum of pool_size across all users.
pool_size = 9
# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way.
statement_timeout = 0
[pools.sharded_db.users.1]
username = "other_user"
password = "other_user"
pool_size = 21
statement_timeout = 15000
# Shard 0
[pools.sharded_db.shards.0]
@@ -137,7 +130,6 @@ sharding_function = "pg_bigint_hash"
username = "simple_user"
password = "simple_user"
pool_size = 5
statement_timeout = 0
[pools.simple_db.shards.0]
servers = [

281
src/admin.rs
View File

@@ -2,25 +2,22 @@
use bytes::{Buf, BufMut, BytesMut};
use log::{info, trace};
use std::collections::HashMap;
use tokio::time::Instant;
use crate::config::{get_config, reload_config, VERSION};
use crate::errors::Error;
use crate::messages::*;
use crate::pool::get_all_pools;
use crate::stats::{
get_address_stats, get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState,
};
use crate::stats::get_stats;
use crate::ClientServerMap;
pub fn generate_server_info_for_admin() -> BytesMut {
let mut server_info = BytesMut::new();
server_info.put(server_parameter_message("application_name", ""));
server_info.put(server_parameter_message("client_encoding", "UTF8"));
server_info.put(server_parameter_message("server_encoding", "UTF8"));
server_info.put(server_parameter_message("server_version", VERSION));
server_info.put(server_parameter_message("DateStyle", "ISO, MDY"));
server_info.put(server_paramater_message("application_name", ""));
server_info.put(server_paramater_message("client_encoding", "UTF8"));
server_info.put(server_paramater_message("server_encoding", "UTF8"));
server_info.put(server_paramater_message("server_version", VERSION));
server_info.put(server_paramater_message("DateStyle", "ISO, MDY"));
return server_info;
}
@@ -47,53 +44,32 @@ where
trace!("Admin query: {}", query);
let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect();
match query_parts[0] {
"RELOAD" => {
trace!("RELOAD");
reload(stream, client_server_map).await
}
"SET" => {
trace!("SET");
ignore_set(stream).await
}
"SHOW" => match query_parts[1] {
"CONFIG" => {
trace!("SHOW CONFIG");
show_config(stream).await
}
"DATABASES" => {
trace!("SHOW DATABASES");
show_databases(stream).await
}
"LISTS" => {
trace!("SHOW LISTS");
show_lists(stream).await
}
"POOLS" => {
trace!("SHOW POOLS");
show_pools(stream).await
}
"CLIENTS" => {
trace!("SHOW CLIENTS");
show_clients(stream).await
}
"SERVERS" => {
trace!("SHOW SERVERS");
show_servers(stream).await
}
"STATS" => {
trace!("SHOW STATS");
show_stats(stream).await
}
"VERSION" => {
trace!("SHOW VERSION");
show_version(stream).await
}
_ => error_response(stream, "Unsupported SHOW query against the admin database").await,
},
_ => error_response(stream, "Unsupported query against the admin database").await,
if query.starts_with("SHOW STATS") {
trace!("SHOW STATS");
show_stats(stream).await
} else if query.starts_with("RELOAD") {
trace!("RELOAD");
reload(stream, client_server_map).await
} else if query.starts_with("SHOW CONFIG") {
trace!("SHOW CONFIG");
show_config(stream).await
} else if query.starts_with("SHOW DATABASES") {
trace!("SHOW DATABASES");
show_databases(stream).await
} else if query.starts_with("SHOW POOLS") {
trace!("SHOW POOLS");
show_pools(stream).await
} else if query.starts_with("SHOW LISTS") {
trace!("SHOW LISTS");
show_lists(stream).await
} else if query.starts_with("SHOW VERSION") {
trace!("SHOW VERSION");
show_version(stream).await
} else if query.starts_with("SET ") {
trace!("SET");
ignore_set(stream).await
} else {
error_response(stream, "Unsupported query against the admin database").await
}
}
@@ -102,8 +78,7 @@ async fn show_lists<T>(stream: &mut T) -> Result<(), Error>
where
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
let client_stats = get_client_stats();
let server_stats = get_server_stats();
let stats = get_stats();
let columns = vec![("list", DataType::Text), ("items", DataType::Int4)];
@@ -123,18 +98,18 @@ where
res.put(data_row(&vec!["pools".to_string(), databases.to_string()]));
res.put(data_row(&vec![
"free_clients".to_string(),
client_stats
stats
.keys()
.filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Idle)
.count()
.map(|address_id| stats[&address_id]["cl_idle"])
.sum::<i64>()
.to_string(),
]));
res.put(data_row(&vec![
"used_clients".to_string(),
client_stats
stats
.keys()
.filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Active)
.count()
.map(|address_id| stats[&address_id]["cl_active"])
.sum::<i64>()
.to_string(),
]));
res.put(data_row(&vec![
@@ -143,18 +118,18 @@ where
]));
res.put(data_row(&vec![
"free_servers".to_string(),
server_stats
stats
.keys()
.filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Idle)
.count()
.map(|address_id| stats[&address_id]["sv_idle"])
.sum::<i64>()
.to_string(),
]));
res.put(data_row(&vec![
"used_servers".to_string(),
server_stats
stats
.keys()
.filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Active)
.count()
.map(|address_id| stats[&address_id]["sv_active"])
.sum::<i64>()
.to_string(),
]));
res.put(data_row(&vec!["dns_names".to_string(), "0".to_string()]));
@@ -194,13 +169,11 @@ async fn show_pools<T>(stream: &mut T) -> Result<(), Error>
where
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
let all_pool_stats = get_pool_stats();
let stats = get_stats();
let columns = vec![
("database", DataType::Text),
("user", DataType::Text),
("pool_mode", DataType::Text),
("cl_idle", DataType::Numeric),
("cl_active", DataType::Numeric),
("cl_waiting", DataType::Numeric),
("cl_cancel_req", DataType::Numeric),
@@ -211,33 +184,32 @@ where
("sv_login", DataType::Numeric),
("maxwait", DataType::Numeric),
("maxwait_us", DataType::Numeric),
("pool_mode", DataType::Text),
];
let mut res = BytesMut::new();
res.put(row_description(&columns));
for (user_pool, pool) in get_all_pools() {
let def = HashMap::default();
let pool_stats = all_pool_stats
.get(&(user_pool.db.clone(), user_pool.user.clone()))
.unwrap_or(&def);
for (_, pool) in get_all_pools() {
let pool_config = &pool.settings;
let mut row = vec![
user_pool.db.clone(),
user_pool.user.clone(),
pool_config.pool_mode.to_string(),
];
for column in &columns[3..columns.len()] {
let value = match column.0 {
"maxwait" => (pool_stats.get("maxwait_us").unwrap_or(&0) / 1_000_000).to_string(),
"maxwait_us" => {
(pool_stats.get("maxwait_us").unwrap_or(&0) % 1_000_000).to_string()
for shard in 0..pool.shards() {
for server in 0..pool.servers(shard) {
let address = pool.address(shard, server);
let stats = match stats.get(&address.id) {
Some(stats) => stats.clone(),
None => HashMap::new(),
};
let mut row = vec![address.name(), pool_config.user.username.clone()];
for column in &columns[2..columns.len() - 1] {
let value = stats.get(column.0).unwrap_or(&0).to_string();
row.push(value);
}
_other_values => pool_stats.get(column.0).unwrap_or(&0).to_string(),
};
row.push(value);
row.push(pool_config.pool_mode.to_string());
res.put(data_row(&row));
}
}
res.put(data_row(&row));
}
res.put(command_complete("SHOW"));
@@ -279,11 +251,11 @@ where
for (_, pool) in get_all_pools() {
let pool_config = pool.settings.clone();
for shard in 0..pool.shards() {
let database_name = &pool.address(shard, 0).database;
let database_name = &pool_config.shards[&shard.to_string()].database;
for server in 0..pool.servers(shard) {
let address = pool.address(shard, server);
let pool_state = pool.pool_state(shard, server);
let banned = pool.is_banned(address, Some(address.role));
let banned = pool.is_banned(address, shard, Some(address.role));
res.put(data_row(&vec![
address.name(), // name
@@ -401,7 +373,6 @@ where
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
let columns = vec![
("instance", DataType::Text),
("database", DataType::Text),
("user", DataType::Text),
("total_xact_count", DataType::Numeric),
@@ -411,32 +382,32 @@ where
("total_xact_time", DataType::Numeric),
("total_query_time", DataType::Numeric),
("total_wait_time", DataType::Numeric),
("total_errors", DataType::Numeric),
("avg_xact_count", DataType::Numeric),
("avg_query_count", DataType::Numeric),
("avg_recv", DataType::Numeric),
("avg_sent", DataType::Numeric),
("avg_errors", DataType::Numeric),
("avg_xact_time", DataType::Numeric),
("avg_query_time", DataType::Numeric),
("avg_wait_time", DataType::Numeric),
];
let all_stats = get_address_stats();
let stats = get_stats();
let mut res = BytesMut::new();
res.put(row_description(&columns));
for (user_pool, pool) in get_all_pools() {
for ((_db_name, username), pool) in get_all_pools() {
for shard in 0..pool.shards() {
for server in 0..pool.servers(shard) {
let address = pool.address(shard, server);
let stats = match all_stats.get(&address.id) {
let stats = match stats.get(&address.id) {
Some(stats) => stats.clone(),
None => HashMap::new(),
};
let mut row = vec![address.name(), user_pool.db.clone(), user_pool.user.clone()];
for column in &columns[3..] {
let mut row = vec![address.name()];
row.push(username.clone());
for column in &columns[2..] {
row.push(stats.get(column.0).unwrap_or(&0).to_string());
}
@@ -454,107 +425,3 @@ where
write_all_half(stream, res).await
}
/// Show currently connected clients
async fn show_clients<T>(stream: &mut T) -> Result<(), Error>
where
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
let columns = vec![
("client_id", DataType::Text),
("database", DataType::Text),
("user", DataType::Text),
("application_name", DataType::Text),
("state", DataType::Text),
("transaction_count", DataType::Numeric),
("query_count", DataType::Numeric),
("error_count", DataType::Numeric),
("age_seconds", DataType::Numeric),
];
let new_map = get_client_stats();
let mut res = BytesMut::new();
res.put(row_description(&columns));
for (_, client) in new_map {
let row = vec![
format!("{:#010X}", client.client_id),
client.pool_name,
client.username,
client.application_name.clone(),
client.state.to_string(),
client.transaction_count.to_string(),
client.query_count.to_string(),
client.error_count.to_string(),
Instant::now()
.duration_since(client.connect_time)
.as_secs()
.to_string(),
];
res.put(data_row(&row));
}
res.put(command_complete("SHOW"));
// ReadyForQuery
res.put_u8(b'Z');
res.put_i32(5);
res.put_u8(b'I');
write_all_half(stream, res).await
}
/// Show currently connected servers
async fn show_servers<T>(stream: &mut T) -> Result<(), Error>
where
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
let columns = vec![
("server_id", DataType::Text),
("database_name", DataType::Text),
("user", DataType::Text),
("address_id", DataType::Text),
("application_name", DataType::Text),
("state", DataType::Text),
("transaction_count", DataType::Numeric),
("query_count", DataType::Numeric),
("bytes_sent", DataType::Numeric),
("bytes_received", DataType::Numeric),
("age_seconds", DataType::Numeric),
];
let new_map = get_server_stats();
let mut res = BytesMut::new();
res.put(row_description(&columns));
for (_, server) in new_map {
let row = vec![
format!("{:#010X}", server.server_id),
server.pool_name,
server.username,
server.address_name,
server.application_name,
server.state.to_string(),
server.transaction_count.to_string(),
server.query_count.to_string(),
server.bytes_sent.to_string(),
server.bytes_received.to_string(),
Instant::now()
.duration_since(server.connect_time)
.as_secs()
.to_string(),
];
res.put(data_row(&row));
}
res.put(command_complete("SHOW"));
// ReadyForQuery
res.put_u8(b'Z');
res.put_i32(5);
res.put_u8(b'I');
write_all_half(stream, res).await
}

562
src/client.rs
View File

@@ -1,15 +1,13 @@
/// Handle clients by pretending to be a PostgreSQL server.
use bytes::{Buf, BufMut, BytesMut};
use log::{debug, error, info, trace, warn};
use log::{debug, error, info, trace};
use std::collections::HashMap;
use std::time::Instant;
use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf};
use tokio::net::TcpStream;
use tokio::sync::broadcast::Receiver;
use tokio::sync::mpsc::Sender;
use crate::admin::{generate_server_info_for_admin, handle_admin};
use crate::config::{get_config, Address, PoolMode};
use crate::config::{get_config, Address};
use crate::constants::*;
use crate::errors::Error;
use crate::messages::*;
@@ -75,29 +73,21 @@ pub struct Client<S, T> {
/// Last server process id we talked to.
last_server_id: Option<i32>,
/// Connected to server
connected_to_server: bool,
/// Name of the server pool for this client (This comes from the database name in the connection string)
pool_name: String,
target_pool_name: String,
/// Postgres user for this client (This comes from the user in the connection string)
username: String,
/// Application name for this client (defaults to pgcat)
application_name: String,
target_user_name: String,
/// Used to notify clients about an impending shutdown
shutdown: Receiver<()>,
shutdown_event_receiver: Receiver<()>,
}
/// Client entrypoint.
pub async fn client_entrypoint(
mut stream: TcpStream,
client_server_map: ClientServerMap,
shutdown: Receiver<()>,
drain: Sender<i32>,
admin_only: bool,
shutdown_event_receiver: Receiver<()>,
) -> Result<(), Error> {
// Figure out if the client wants TLS or not.
let addr = stream.peer_addr().unwrap();
@@ -116,21 +106,11 @@ pub async fn client_entrypoint(
write_all(&mut stream, yes).await?;
// Negotiate TLS.
match startup_tls(stream, client_server_map, shutdown, admin_only).await {
match startup_tls(stream, client_server_map, shutdown_event_receiver).await {
Ok(mut client) => {
info!("Client {:?} connected (TLS)", addr);
if !client.is_admin() {
let _ = drain.send(1).await;
}
let result = client.handle().await;
if !client.is_admin() {
let _ = drain.send(-1).await;
}
result
client.handle().await
}
Err(err) => Err(err),
}
@@ -156,25 +136,14 @@ pub async fn client_entrypoint(
addr,
bytes,
client_server_map,
shutdown,
admin_only,
shutdown_event_receiver,
)
.await
{
Ok(mut client) => {
info!("Client {:?} connected (plain)", addr);
if !client.is_admin() {
let _ = drain.send(1).await;
}
let result = client.handle().await;
if !client.is_admin() {
let _ = drain.send(-1).await;
}
result
client.handle().await
}
Err(err) => Err(err),
}
@@ -197,25 +166,14 @@ pub async fn client_entrypoint(
addr,
bytes,
client_server_map,
shutdown,
admin_only,
shutdown_event_receiver,
)
.await
{
Ok(mut client) => {
info!("Client {:?} connected (plain)", addr);
if !client.is_admin() {
let _ = drain.send(1).await;
}
let result = client.handle().await;
if !client.is_admin() {
let _ = drain.send(-1).await;
}
result
client.handle().await
}
Err(err) => Err(err),
}
@@ -226,21 +184,20 @@ pub async fn client_entrypoint(
let (read, write) = split(stream);
// Continue with cancel query request.
match Client::cancel(read, write, addr, bytes, client_server_map, shutdown).await {
match Client::cancel(
read,
write,
addr,
bytes,
client_server_map,
shutdown_event_receiver,
)
.await
{
Ok(mut client) => {
info!("Client {:?} issued a cancel query request", addr);
if !client.is_admin() {
let _ = drain.send(1).await;
}
let result = client.handle().await;
if !client.is_admin() {
let _ = drain.send(-1).await;
}
result
client.handle().await
}
Err(err) => Err(err),
@@ -293,8 +250,7 @@ where
pub async fn startup_tls(
stream: TcpStream,
client_server_map: ClientServerMap,
shutdown: Receiver<()>,
admin_only: bool,
shutdown_event_receiver: Receiver<()>,
) -> Result<Client<ReadHalf<TlsStream<TcpStream>>, WriteHalf<TlsStream<TcpStream>>>, Error> {
// Negotiate TLS.
let tls = Tls::new()?;
@@ -324,8 +280,7 @@ pub async fn startup_tls(
addr,
bytes,
client_server_map,
shutdown,
admin_only,
shutdown_event_receiver,
)
.await
}
@@ -340,10 +295,6 @@ where
S: tokio::io::AsyncRead + std::marker::Unpin,
T: tokio::io::AsyncWrite + std::marker::Unpin,
{
pub fn is_admin(&self) -> bool {
self.admin
}
/// Handle Postgres client startup after TLS negotiation is complete
/// or over plain text.
pub async fn startup(
@@ -352,49 +303,29 @@ where
addr: std::net::SocketAddr,
bytes: BytesMut, // The rest of the startup message.
client_server_map: ClientServerMap,
shutdown: Receiver<()>,
admin_only: bool,
shutdown_event_receiver: Receiver<()>,
) -> Result<Client<S, T>, Error> {
let config = get_config();
let stats = get_reporter();
let parameters = parse_startup(bytes.clone())?;
// These two parameters are mandatory by the protocol.
let pool_name = match parameters.get("database") {
trace!("Got StartupMessage");
let parameters = parse_startup(bytes.clone())?;
let target_pool_name = match parameters.get("database") {
Some(db) => db,
None => return Err(Error::ClientError),
};
let username = match parameters.get("user") {
let target_user_name = match parameters.get("user") {
Some(user) => user,
None => return Err(Error::ClientError),
};
let application_name = match parameters.get("application_name") {
Some(application_name) => application_name,
None => "pgcat",
};
let admin = ["pgcat", "pgbouncer"]
.iter()
.filter(|db| *db == &pool_name)
.filter(|db| *db == &target_pool_name)
.count()
== 1;
// Kick any client that's not admin while we're in admin-only mode.
if !admin && admin_only {
debug!(
"Rejecting non-admin connection to {} when in admin only mode",
pool_name
);
error_response_terminal(
&mut write,
&format!("terminating connection due to administrator command"),
)
.await?;
return Err(Error::ShuttingDown);
}
// Generate random backend ID and secret key
let process_id: i32 = rand::random();
let secret_key: i32 = rand::random();
@@ -426,56 +357,46 @@ where
Err(_) => return Err(Error::SocketError),
};
// Authenticate admin user.
let (transaction_mode, server_info) = if admin {
let correct_user = config.general.admin_username.as_str();
let correct_password = config.general.admin_password.as_str();
// Compare server and client hashes.
let password_hash = md5_hash_password(
&config.general.admin_username,
&config.general.admin_password,
&salt,
);
let password_hash = md5_hash_password(correct_user, correct_password, &salt);
if password_hash != password_response {
warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name);
wrong_password(&mut write, username).await?;
debug!("Password authentication failed");
wrong_password(&mut write, target_user_name).await?;
return Err(Error::ClientError);
}
(false, generate_server_info_for_admin())
}
// Authenticate normal user.
else {
let pool = match get_pool(&pool_name, &username) {
} else {
let target_pool = match get_pool(target_pool_name.clone(), target_user_name.clone()) {
Some(pool) => pool,
None => {
error_response(
&mut write,
&format!(
"No pool configured for database: {:?}, user: {:?}",
pool_name, username
target_pool_name, target_user_name
),
)
.await?;
warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name);
return Err(Error::ClientError);
}
};
let transaction_mode = target_pool.settings.pool_mode == "transaction";
let server_info = target_pool.server_info();
// Compare server and client hashes.
let password_hash = md5_hash_password(&username, &pool.settings.user.password, &salt);
let correct_password = target_pool.settings.user.password.as_str();
let password_hash = md5_hash_password(&target_user_name, correct_password, &salt);
if password_hash != password_response {
warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name);
wrong_password(&mut write, username).await?;
debug!("Password authentication failed");
wrong_password(&mut write, &target_user_name).await?;
return Err(Error::ClientError);
}
let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction;
(transaction_mode, pool.server_info())
(transaction_mode, server_info)
};
debug!("Password authentication successful");
@@ -487,26 +408,27 @@ where
trace!("Startup OK");
// Split the read and write streams
// so we can control buffering.
return Ok(Client {
read: BufReader::new(read),
write: write,
addr,
buffer: BytesMut::with_capacity(8196),
cancel_mode: false,
transaction_mode,
process_id,
secret_key,
client_server_map,
transaction_mode: transaction_mode,
process_id: process_id,
secret_key: secret_key,
client_server_map: client_server_map,
parameters: parameters.clone(),
stats: stats,
admin: admin,
last_address_id: None,
last_server_id: None,
pool_name: pool_name.clone(),
username: username.clone(),
application_name: application_name.to_string(),
shutdown,
connected_to_server: false,
target_pool_name: target_pool_name.clone(),
target_user_name: target_user_name.clone(),
shutdown_event_receiver: shutdown_event_receiver,
});
}
@@ -517,7 +439,7 @@ where
addr: std::net::SocketAddr,
mut bytes: BytesMut, // The rest of the startup message.
client_server_map: ClientServerMap,
shutdown: Receiver<()>,
shutdown_event_receiver: Receiver<()>,
) -> Result<Client<S, T>, Error> {
let process_id = bytes.get_i32();
let secret_key = bytes.get_i32();
@@ -528,19 +450,17 @@ where
buffer: BytesMut::with_capacity(8196),
cancel_mode: true,
transaction_mode: false,
process_id,
secret_key,
client_server_map,
process_id: process_id,
secret_key: secret_key,
client_server_map: client_server_map,
parameters: HashMap::new(),
stats: get_reporter(),
admin: false,
last_address_id: None,
last_server_id: None,
pool_name: String::from("undefined"),
username: String::from("undefined"),
application_name: String::from("undefined"),
shutdown,
connected_to_server: false,
target_pool_name: String::from("undefined"),
target_user_name: String::from("undefined"),
shutdown_event_receiver: shutdown_event_receiver,
});
}
@@ -561,7 +481,7 @@ where
process_id.clone(),
secret_key.clone(),
address.clone(),
*port,
port.clone(),
),
// The client doesn't know / got the wrong server,
@@ -573,18 +493,13 @@ where
// Opens a new separate connection to the server, sends the backend_id
// and secret_key and then closes it for security reasons. No other interactions
// take place.
return Ok(Server::cancel(&address, port, process_id, secret_key).await?);
return Ok(Server::cancel(&address, &port, process_id, secret_key).await?);
}
// The query router determines where the query is going to go,
// e.g. primary, replica, which shard.
let mut query_router = QueryRouter::new();
self.stats.client_register(
self.process_id,
self.pool_name.clone(),
self.username.clone(),
self.application_name.clone(),
);
let mut round_robin = 0;
// Our custom protocol loop.
// We expect the client to either start a transaction with regular queries
@@ -602,40 +517,17 @@ where
// SET SHARDING KEY TO 'bigint';
let mut message = tokio::select! {
_ = self.shutdown.recv() => {
if !self.admin {
error_response_terminal(
&mut self.write,
&format!("terminating connection due to administrator command")
).await?;
return Ok(())
}
// Admin clients ignore shutdown.
else {
read_message(&mut self.read).await?
}
_ = self.shutdown_event_receiver.recv() => {
error_response_terminal(&mut self.write, &format!("terminating connection due to administrator command")).await?;
return Ok(())
},
message_result = read_message(&mut self.read) => message_result?
};
match message[0] as char {
// Buffer extended protocol messages even if we do not have
// a server connection yet. Hopefully, when we get the S message
// we'll be able to allocate a connection. Also, clients do not expect
// the server to respond to these messages so even if we were not able to
// allocate a connection, we wouldn't be able to send back an error message
// to the client so we buffer them and defer the decision to error out or not
// to when we get the S message
'P' | 'B' | 'D' | 'E' => {
self.buffer.put(&message[..]);
continue;
}
'X' => {
debug!("Client disconnecting");
return Ok(());
}
_ => (),
// Avoid taking a server if the client just wants to disconnect.
if message[0] as char == 'X' {
debug!("Client disconnecting");
return Ok(());
}
// Handle admin database queries.
@@ -648,19 +540,18 @@ where
// Get a pool instance referenced by the most up-to-date
// pointer. This ensures we always read the latest config
// when starting a query.
let pool = match get_pool(&self.pool_name, &self.username) {
let pool = match get_pool(self.target_pool_name.clone(), self.target_user_name.clone())
{
Some(pool) => pool,
None => {
error_response(
&mut self.write,
&format!(
"No pool configured for database: {:?}, user: {:?}",
self.pool_name, self.username
self.target_pool_name, self.target_user_name
),
)
.await?;
warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", self.pool_name, self.username, self.application_name);
return Err(Error::ClientError);
}
};
@@ -740,7 +631,12 @@ where
// Grab a server from the pool.
let connection = match pool
.get(query_router.shard(), query_router.role(), self.process_id)
.get(
query_router.shard(),
query_router.role(),
self.process_id,
round_robin,
)
.await
{
Ok(conn) => {
@@ -748,19 +644,9 @@ where
conn
}
Err(err) => {
// Client is attempting to get results from the server,
// but we were unable to grab a connection from the pool
// We'll send back an error message and clean the extended
// protocol buffer
if message[0] as char == 'S' {
error!("Got Sync message but failed to get a connection from the pool");
self.buffer.clear();
}
error!("Could not get connection from pool: {:?}", err);
error_response(&mut self.write, "could not get connection from the pool")
.await?;
error!("Could not get connection from pool: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\", error: \"{:?}\" }}",
self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role(), err);
continue;
}
};
@@ -769,17 +655,22 @@ where
let address = connection.1;
let server = &mut *reference;
round_robin += 1;
// Server is assigned to the client in case the client wants to
// cancel a query later.
server.claim(self.process_id, self.secret_key);
self.connected_to_server = true;
// Update statistics
self.stats
.client_active(self.process_id, server.server_id());
// Update statistics.
if let Some(last_address_id) = self.last_address_id {
self.stats
.client_disconnecting(self.process_id, last_address_id);
}
self.stats.client_active(self.process_id, address.id);
self.stats.server_active(server.process_id(), address.id);
self.last_address_id = Some(address.id);
self.last_server_id = Some(server.server_id());
self.last_server_id = Some(server.process_id());
debug!(
"Client {:?} talking to server {:?}",
@@ -787,10 +678,13 @@ where
server.address()
);
// Set application_name if any.
// TODO: investigate other parameters and set them too.
// Set application_name.
server.set_name(&self.application_name).await?;
if self.parameters.contains_key("application_name") {
server
.set_name(&self.parameters["application_name"])
.await?;
}
// Transaction loop. Multiple queries can be issued by the client here.
// The connection belongs to the client until the transaction is over,
@@ -807,7 +701,12 @@ where
Err(err) => {
// Client disconnected inside a transaction.
// Clean up the server and re-use it.
server.checkin_cleanup().await?;
// This prevents connection thrashing by bad clients.
if server.in_transaction() {
server.query("ROLLBACK").await?;
server.query("DISCARD ALL").await?;
server.set_name("pgcat").await?;
}
return Err(err);
}
@@ -832,16 +731,52 @@ where
'Q' => {
debug!("Sending query to server");
self.send_and_receive_loop(code, original, server, &address, &pool)
.await?;
self.send_server_message(
server,
original,
&address,
query_router.shard(),
&pool,
)
.await?;
// Read all data the server has to offer, which can be multiple messages
// buffered in 8196 bytes chunks.
loop {
let response = self
.receive_server_message(
server,
&address,
query_router.shard(),
&pool,
)
.await?;
// Send server reply to the client.
match write_all_half(&mut self.write, response).await {
Ok(_) => (),
Err(err) => {
server.mark_bad();
return Err(err);
}
};
if !server.is_data_available() {
break;
}
}
// Report query executed statistics.
self.stats.query(self.process_id, address.id);
if !server.in_transaction() {
// Report transaction executed statistics.
self.stats.transaction(self.process_id, server.server_id());
self.stats.transaction(self.process_id, address.id);
// Release server back to the pool if we are in transaction mode.
// If we are in session mode, we keep the server until the client disconnects.
if self.transaction_mode {
self.stats.server_idle(server.process_id(), address.id);
break;
}
}
@@ -849,7 +784,16 @@ where
// Terminate
'X' => {
server.checkin_cleanup().await?;
// Client closing. Rollback and clean up
// connection before releasing into the pool.
// Pgbouncer closes the connection which leads to
// connection thrashing when clients misbehave.
if server.in_transaction() {
server.query("ROLLBACK").await?;
server.query("DISCARD ALL").await?;
server.set_name("pgcat").await?;
}
self.release();
return Ok(());
@@ -886,40 +830,52 @@ where
self.buffer.put(&original[..]);
// Clone after freeze does not allocate
let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char;
// Almost certainly true
if first_message_code == 'P' {
// Message layout
// P followed by 32 int followed by null-terminated statement name
// So message code should be in offset 0 of the buffer, first character
// in prepared statement name would be index 5
let first_char_in_name = *self.buffer.get(5).unwrap_or(&0);
if first_char_in_name != 0 {
// This is a named prepared statement
// Server connection state will need to be cleared at checkin
server.mark_dirty();
}
}
self.send_and_receive_loop(
code,
self.buffer.clone(),
self.send_server_message(
server,
self.buffer.clone(),
&address,
query_router.shard(),
&pool,
)
.await?;
self.buffer.clear();
// Read all data the server has to offer, which can be multiple messages
// buffered in 8196 bytes chunks.
loop {
let response = self
.receive_server_message(
server,
&address,
query_router.shard(),
&pool,
)
.await?;
match write_all_half(&mut self.write, response).await {
Ok(_) => (),
Err(err) => {
server.mark_bad();
return Err(err);
}
};
if !server.is_data_available() {
break;
}
}
// Report query executed statistics.
self.stats.query(self.process_id, address.id);
if !server.in_transaction() {
self.stats.transaction(self.process_id, server.server_id());
self.stats.transaction(self.process_id, address.id);
// Release server back to the pool if we are in transaction mode.
// If we are in session mode, we keep the server until the client disconnects.
if self.transaction_mode {
self.stats.server_idle(server.process_id(), address.id);
break;
}
}
@@ -929,17 +885,31 @@ where
'd' => {
// Forward the data to the server,
// don't buffer it since it can be rather large.
self.send_server_message(server, original, &address, &pool)
.await?;
self.send_server_message(
server,
original,
&address,
query_router.shard(),
&pool,
)
.await?;
}
// CopyDone or CopyFail
// Copy is done, successfully or not.
'c' | 'f' => {
self.send_server_message(server, original, &address, &pool)
.await?;
self.send_server_message(
server,
original,
&address,
query_router.shard(),
&pool,
)
.await?;
let response = self.receive_server_message(server, &address, &pool).await?;
let response = self
.receive_server_message(server, &address, query_router.shard(), &pool)
.await?;
match write_all_half(&mut self.write, response).await {
Ok(_) => (),
@@ -950,11 +920,12 @@ where
};
if !server.in_transaction() {
self.stats.transaction(self.process_id, server.server_id());
self.stats.transaction(self.process_id, address.id);
// Release server back to the pool if we are in transaction mode.
// If we are in session mode, we keep the server until the client disconnects.
if self.transaction_mode {
self.stats.server_idle(server.process_id(), address.id);
break;
}
}
@@ -970,12 +941,8 @@ where
// The server is no longer bound to us, we can't cancel it's queries anymore.
debug!("Releasing server back into the pool");
server.checkin_cleanup().await?;
self.stats.server_idle(server.server_id());
self.connected_to_server = false;
self.release();
self.stats.client_idle(self.process_id);
self.stats.client_idle(self.process_id, address.id);
}
}
@@ -985,112 +952,35 @@ where
guard.remove(&(self.process_id, self.secret_key));
}
async fn send_and_receive_loop(
&mut self,
code: char,
message: BytesMut,
server: &mut Server,
address: &Address,
pool: &ConnectionPool,
) -> Result<(), Error> {
debug!("Sending {} to server", code);
self.send_server_message(server, message, &address, &pool)
.await?;
let query_start = Instant::now();
// Read all data the server has to offer, which can be multiple messages
// buffered in 8196 bytes chunks.
loop {
let response = self.receive_server_message(server, &address, &pool).await?;
match write_all_half(&mut self.write, response).await {
Ok(_) => (),
Err(err) => {
server.mark_bad();
return Err(err);
}
};
if !server.is_data_available() {
break;
}
}
// Report query executed statistics.
self.stats.query(
self.process_id,
server.server_id(),
Instant::now().duration_since(query_start).as_millis(),
);
Ok(())
}
async fn send_server_message(
&self,
server: &mut Server,
message: BytesMut,
address: &Address,
shard: usize,
pool: &ConnectionPool,
) -> Result<(), Error> {
match server.send(message).await {
Ok(_) => Ok(()),
Err(err) => {
pool.ban(address, self.process_id);
pool.ban(address, shard, self.process_id);
Err(err)
}
}
}
async fn receive_server_message(
&mut self,
&self,
server: &mut Server,
address: &Address,
shard: usize,
pool: &ConnectionPool,
) -> Result<BytesMut, Error> {
if pool.settings.user.statement_timeout > 0 {
match tokio::time::timeout(
tokio::time::Duration::from_millis(pool.settings.user.statement_timeout),
server.recv(),
)
.await
{
Ok(result) => match result {
Ok(message) => Ok(message),
Err(err) => {
pool.ban(address, self.process_id);
error_response_terminal(
&mut self.write,
&format!("error receiving data from server: {:?}", err),
)
.await?;
Err(err)
}
},
Err(_) => {
error!(
"Statement timeout while talking to {:?} with user {}",
address, pool.settings.user.username
);
server.mark_bad();
pool.ban(address, self.process_id);
error_response_terminal(&mut self.write, "pool statement timeout").await?;
Err(Error::StatementTimeout)
}
}
} else {
match server.recv().await {
Ok(message) => Ok(message),
Err(err) => {
pool.ban(address, self.process_id);
error_response_terminal(
&mut self.write,
&format!("error receiving data from server: {:?}", err),
)
.await?;
Err(err)
}
match server.recv().await {
Ok(message) => Ok(message),
Err(err) => {
pool.ban(address, shard, self.process_id);
Err(err)
}
}
}
@@ -1101,11 +991,15 @@ impl<S, T> Drop for Client<S, T> {
let mut guard = self.client_server_map.lock();
guard.remove(&(self.process_id, self.secret_key));
// Dirty shutdown
// TODO: refactor, this is not the best way to handle state management.
self.stats.client_disconnecting(self.process_id);
if self.connected_to_server && self.last_server_id.is_some() {
self.stats.server_idle(self.last_server_id.unwrap());
// Update statistics.
if let Some(address_id) = self.last_address_id {
self.stats.client_disconnecting(self.process_id, address_id);
if let Some(process_id) = self.last_server_id {
self.stats.server_idle(process_id, address_id);
}
}
// self.release();
}
}

508
src/config.rs
View File

@@ -3,7 +3,7 @@ use arc_swap::ArcSwap;
use log::{error, info};
use once_cell::sync::Lazy;
use serde_derive::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::{HashMap, HashSet};
use std::hash::Hash;
use std::path::Path;
use std::sync::Arc;
@@ -12,9 +12,8 @@ use tokio::io::AsyncReadExt;
use toml;
use crate::errors::Error;
use crate::pool::{ClientServerMap, ConnectionPool};
use crate::sharding::ShardingFunction;
use crate::tls::{load_certs, load_keys};
use crate::{ClientServerMap, ConnectionPool};
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -24,9 +23,7 @@ static CONFIG: Lazy<ArcSwap<Config>> = Lazy::new(|| ArcSwap::from_pointee(Config
/// Server role: primary or replica.
#[derive(Clone, PartialEq, Serialize, Deserialize, Hash, std::cmp::Eq, Debug, Copy)]
pub enum Role {
#[serde(alias = "primary", alias = "Primary")]
Primary,
#[serde(alias = "replica", alias = "Replica")]
Replica,
}
@@ -60,35 +57,13 @@ impl PartialEq<Role> for Option<Role> {
/// Address identifying a PostgreSQL server uniquely.
#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)]
pub struct Address {
/// Unique ID per addressable Postgres server.
pub id: usize,
/// Server host.
pub host: String,
/// Server port.
pub port: u16,
/// Shard number of this Postgres server.
pub port: String,
pub shard: usize,
/// The name of the Postgres database.
pub database: String,
/// Server role: replica, primary.
pub role: Role,
/// If it's a replica, number it for reference and failover.
pub replica_number: usize,
/// Position of the server in the pool for failover.
pub address_index: usize,
/// The name of the user configured to use this pool.
pub username: String,
/// The name of this pool (i.e. database name visible to the client).
pub pool_name: String,
}
impl Default for Address {
@@ -96,14 +71,11 @@ impl Default for Address {
Address {
id: 0,
host: String::from("127.0.0.1"),
port: 5432,
port: String::from("5432"),
shard: 0,
address_index: 0,
replica_number: 0,
database: String::from("database"),
role: Role::Replica,
username: String::from("username"),
pool_name: String::from("pool_name"),
}
}
}
@@ -112,24 +84,22 @@ impl Address {
/// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`.
pub fn name(&self) -> String {
match self.role {
Role::Primary => format!("{}_shard_{}_primary", self.pool_name, self.shard),
Role::Primary => format!("{}_shard_{}_primary", self.database, self.shard),
Role::Replica => format!(
"{}_shard_{}_replica_{}",
self.pool_name, self.shard, self.replica_number
self.database, self.shard, self.replica_number
),
}
}
}
/// PostgreSQL user.
#[derive(Clone, PartialEq, Hash, Eq, Serialize, Deserialize, Debug)]
#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Serialize, Deserialize, Debug)]
pub struct User {
pub username: String,
pub password: String,
pub pool_size: u32,
#[serde(default)] // 0
pub statement_timeout: u64,
}
impl Default for User {
@@ -138,7 +108,6 @@ impl Default for User {
username: String::from("postgres"),
password: String::new(),
pool_size: 15,
statement_timeout: 0,
}
}
}
@@ -146,81 +115,32 @@ impl Default for User {
/// General configuration.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct General {
#[serde(default = "General::default_host")]
pub host: String,
#[serde(default = "General::default_port")]
pub port: i16,
pub enable_prometheus_exporter: Option<bool>,
pub prometheus_exporter_port: i16,
#[serde(default = "General::default_connect_timeout")]
pub connect_timeout: u64,
#[serde(default = "General::default_shutdown_timeout")]
pub shutdown_timeout: u64,
#[serde(default = "General::default_healthcheck_timeout")]
pub healthcheck_timeout: u64,
#[serde(default = "General::default_healthcheck_delay")]
pub shutdown_timeout: u64,
pub healthcheck_delay: u64,
#[serde(default = "General::default_ban_time")]
pub ban_time: i64,
#[serde(default)] // False
pub autoreload: bool,
pub tls_certificate: Option<String>,
pub tls_private_key: Option<String>,
pub admin_username: String,
pub admin_password: String,
}
impl General {
pub fn default_host() -> String {
"0.0.0.0".into()
}
pub fn default_port() -> i16 {
5432
}
pub fn default_connect_timeout() -> u64 {
1000
}
pub fn default_shutdown_timeout() -> u64 {
60000
}
pub fn default_healthcheck_timeout() -> u64 {
1000
}
pub fn default_healthcheck_delay() -> u64 {
30000
}
pub fn default_ban_time() -> i64 {
60
}
}
impl Default for General {
fn default() -> General {
General {
host: Self::default_host(),
port: Self::default_port(),
host: String::from("localhost"),
port: 5432,
enable_prometheus_exporter: Some(false),
prometheus_exporter_port: 9930,
connect_timeout: General::default_connect_timeout(),
shutdown_timeout: Self::default_shutdown_timeout(),
healthcheck_timeout: Self::default_healthcheck_timeout(),
healthcheck_delay: Self::default_healthcheck_delay(),
ban_time: Self::default_ban_time(),
connect_timeout: 5000,
healthcheck_timeout: 1000,
shutdown_timeout: 60000,
healthcheck_delay: 30000,
ban_time: 60,
autoreload: false,
tls_certificate: None,
tls_private_key: None,
@@ -229,166 +149,50 @@ impl Default for General {
}
}
}
/// Pool mode:
/// - transaction: server serves one transaction,
/// - session: server is attached to the client.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Copy, Hash)]
pub enum PoolMode {
#[serde(alias = "transaction", alias = "Transaction")]
Transaction,
#[serde(alias = "session", alias = "Session")]
Session,
}
impl ToString for PoolMode {
fn to_string(&self) -> String {
match *self {
PoolMode::Transaction => "transaction".to_string(),
PoolMode::Session => "session".to_string(),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct Pool {
#[serde(default = "Pool::default_pool_mode")]
pub pool_mode: PoolMode,
pub pool_mode: String,
pub default_role: String,
#[serde(default)] // False
pub query_parser_enabled: bool,
#[serde(default)] // False
pub primary_reads_enabled: bool,
pub connect_timeout: Option<u64>,
pub sharding_function: ShardingFunction,
pub shards: BTreeMap<String, Shard>,
pub users: BTreeMap<String, User>,
pub sharding_function: String,
pub shards: HashMap<String, Shard>,
pub users: HashMap<String, User>,
}
impl Pool {
pub fn default_pool_mode() -> PoolMode {
PoolMode::Transaction
}
pub fn validate(&self) -> Result<(), Error> {
match self.default_role.as_ref() {
"any" => (),
"primary" => (),
"replica" => (),
other => {
error!(
"Query router default_role must be 'primary', 'replica', or 'any', got: '{}'",
other
);
return Err(Error::BadConfig);
}
};
for (shard_idx, shard) in &self.shards {
match shard_idx.parse::<usize>() {
Ok(_) => (),
Err(_) => {
error!(
"Shard '{}' is not a valid number, shards must be numbered starting at 0",
shard_idx
);
return Err(Error::BadConfig);
}
};
shard.validate()?;
}
Ok(())
}
}
impl Default for Pool {
fn default() -> Pool {
Pool {
pool_mode: Self::default_pool_mode(),
shards: BTreeMap::from([(String::from("1"), Shard::default())]),
users: BTreeMap::default(),
pool_mode: String::from("transaction"),
shards: HashMap::from([(String::from("1"), Shard::default())]),
users: HashMap::default(),
default_role: String::from("any"),
query_parser_enabled: false,
primary_reads_enabled: false,
sharding_function: ShardingFunction::PgBigintHash,
connect_timeout: None,
primary_reads_enabled: true,
sharding_function: "pg_bigint_hash".to_string(),
}
}
}
#[derive(Clone, PartialEq, Serialize, Deserialize, Debug, Hash, Eq)]
pub struct ServerConfig {
pub host: String,
pub port: u16,
pub role: Role,
}
/// Shard configuration.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct Shard {
pub database: String,
pub servers: Vec<ServerConfig>,
}
impl Shard {
pub fn validate(&self) -> Result<(), Error> {
// We use addresses as unique identifiers,
// let's make sure they are unique in the config as well.
let mut dup_check = HashSet::new();
let mut primary_count = 0;
if self.servers.len() == 0 {
error!("Shard {} has no servers configured", self.database);
return Err(Error::BadConfig);
}
for server in &self.servers {
dup_check.insert(server);
// Check that we define only zero or one primary.
match server.role {
Role::Primary => primary_count += 1,
_ => (),
};
}
if primary_count > 1 {
error!(
"Shard {} has more than on primary configured",
self.database
);
return Err(Error::BadConfig);
}
if dup_check.len() != self.servers.len() {
error!("Shard {} contains duplicate server configs", self.database);
return Err(Error::BadConfig);
}
Ok(())
}
pub servers: Vec<(String, u16, String)>,
}
impl Default for Shard {
fn default() -> Shard {
Shard {
servers: vec![ServerConfig {
host: String::from("localhost"),
port: 5432,
role: Role::Primary,
}],
servers: vec![(String::from("localhost"), 5432, String::from("primary"))],
database: String::from("postgres"),
}
}
}
fn default_path() -> String {
String::from("pgcat.toml")
}
/// Configuration wrapper.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct Config {
@@ -396,30 +200,24 @@ pub struct Config {
// so we should always put simple fields before nested fields
// in all serializable structs to avoid ValueAfterTable errors
// These errors occur when the toml serializer is about to produce
// ambiguous toml structure like the one below
// ambigous toml structure like the one below
// [main]
// field1_under_main = 1
// field2_under_main = 2
// [main.subconf]
// field1_under_subconf = 1
// field3_under_main = 3 # This field will be interpreted as being under subconf and not under main
#[serde(default = "Config::default_path")]
#[serde(default = "default_path")]
pub path: String,
pub general: General,
pub pools: HashMap<String, Pool>,
}
impl Config {
pub fn default_path() -> String {
String::from("pgcat.toml")
}
}
impl Default for Config {
fn default() -> Config {
Config {
path: Self::default_path(),
path: String::from("pgcat.toml"),
general: General::default(),
pools: HashMap::default(),
}
@@ -435,7 +233,7 @@ impl From<&Config> for std::collections::HashMap<String, String> {
[
(
format!("pools.{}.pool_mode", pool_name),
pool.pool_mode.to_string(),
pool.pool_mode.clone(),
),
(
format!("pools.{}.primary_reads_enabled", pool_name),
@@ -451,7 +249,7 @@ impl From<&Config> for std::collections::HashMap<String, String> {
),
(
format!("pools.{}.sharding_function", pool_name),
pool.sharding_function.to_string(),
pool.sharding_function.clone(),
),
(
format!("pools.{:?}.shard_count", pool_name),
@@ -473,10 +271,6 @@ impl From<&Config> for std::collections::HashMap<String, String> {
let mut static_settings = vec![
("host".to_string(), config.general.host.to_string()),
("port".to_string(), config.general.port.to_string()),
(
"prometheus_exporter_port".to_string(),
config.general.prometheus_exporter_port.to_string(),
),
(
"connect_timeout".to_string(),
config.general.connect_timeout.to_string(),
@@ -532,10 +326,9 @@ impl Config {
};
for (pool_name, pool_config) in &self.pools {
// TODO: Make this output prettier (maybe a table?)
info!("--- Settings for pool {} ---", pool_name);
info!(
"[pool: {}] Maximum user connections: {}",
pool_name,
"Pool size from all users: {}",
pool_config
.users
.iter()
@@ -543,98 +336,14 @@ impl Config {
.sum::<u32>()
.to_string()
);
info!(
"[pool: {}] Pool mode: {:?}",
pool_name, pool_config.pool_mode
);
let connect_timeout = match pool_config.connect_timeout {
Some(connect_timeout) => connect_timeout,
None => self.general.connect_timeout,
};
info!(
"[pool: {}] Connection timeout: {}ms",
pool_name, connect_timeout
);
info!(
"[pool: {}] Sharding function: {}",
pool_name,
pool_config.sharding_function.to_string()
);
info!(
"[pool: {}] Primary reads: {}",
pool_name, pool_config.primary_reads_enabled
);
info!(
"[pool: {}] Query router: {}",
pool_name, pool_config.query_parser_enabled
);
info!(
"[pool: {}] Number of shards: {}",
pool_name,
pool_config.shards.len()
);
info!(
"[pool: {}] Number of users: {}",
pool_name,
pool_config.users.len()
);
for user in &pool_config.users {
info!(
"[pool: {}][user: {}] Pool size: {}",
pool_name, user.1.username, user.1.pool_size,
);
info!(
"[pool: {}][user: {}] Statement timeout: {}",
pool_name, user.1.username, user.1.statement_timeout
)
}
info!("Pool mode: {}", pool_config.pool_mode);
info!("Sharding function: {}", pool_config.sharding_function);
info!("Primary reads: {}", pool_config.primary_reads_enabled);
info!("Query router: {}", pool_config.query_parser_enabled);
info!("Number of shards: {}", pool_config.shards.len());
info!("Number of users: {}", pool_config.users.len());
}
}
pub fn validate(&mut self) -> Result<(), Error> {
// Validate TLS!
match self.general.tls_certificate.clone() {
Some(tls_certificate) => {
match load_certs(&Path::new(&tls_certificate)) {
Ok(_) => {
// Cert is okay, but what about the private key?
match self.general.tls_private_key.clone() {
Some(tls_private_key) => {
match load_keys(&Path::new(&tls_private_key)) {
Ok(_) => (),
Err(err) => {
error!(
"tls_private_key is incorrectly configured: {:?}",
err
);
return Err(Error::BadConfig);
}
}
}
None => {
error!("tls_certificate is set, but the tls_private_key is not");
return Err(Error::BadConfig);
}
};
}
Err(err) => {
error!("tls_certificate is incorrectly configured: {:?}", err);
return Err(Error::BadConfig);
}
}
}
None => (),
};
for (_, pool) in &mut self.pools {
pool.validate()?;
}
Ok(())
}
}
/// Get a read-only instance of the configuration
@@ -671,7 +380,120 @@ pub async fn parse(path: &str) -> Result<(), Error> {
}
};
config.validate()?;
// Validate TLS!
match config.general.tls_certificate.clone() {
Some(tls_certificate) => {
match load_certs(&Path::new(&tls_certificate)) {
Ok(_) => {
// Cert is okay, but what about the private key?
match config.general.tls_private_key.clone() {
Some(tls_private_key) => match load_keys(&Path::new(&tls_private_key)) {
Ok(_) => (),
Err(err) => {
error!("tls_private_key is incorrectly configured: {:?}", err);
return Err(Error::BadConfig);
}
},
None => {
error!("tls_certificate is set, but the tls_private_key is not");
return Err(Error::BadConfig);
}
};
}
Err(err) => {
error!("tls_certificate is incorrectly configured: {:?}", err);
return Err(Error::BadConfig);
}
}
}
None => (),
};
for (pool_name, pool) in &config.pools {
match pool.sharding_function.as_ref() {
"pg_bigint_hash" => (),
"sha1" => (),
_ => {
error!(
"Supported sharding functions are: 'pg_bigint_hash', 'sha1', got: '{}' in pool {} settings",
pool.sharding_function,
pool_name
);
return Err(Error::BadConfig);
}
};
match pool.default_role.as_ref() {
"any" => (),
"primary" => (),
"replica" => (),
other => {
error!(
"Query router default_role must be 'primary', 'replica', or 'any', got: '{}'",
other
);
return Err(Error::BadConfig);
}
};
for shard in &pool.shards {
// We use addresses as unique identifiers,
// let's make sure they are unique in the config as well.
let mut dup_check = HashSet::new();
let mut primary_count = 0;
match shard.0.parse::<usize>() {
Ok(_) => (),
Err(_) => {
error!(
"Shard '{}' is not a valid number, shards must be numbered starting at 0",
shard.0
);
return Err(Error::BadConfig);
}
};
if shard.1.servers.len() == 0 {
error!("Shard {} has no servers configured", shard.0);
return Err(Error::BadConfig);
}
for server in &shard.1.servers {
dup_check.insert(server);
// Check that we define only zero or one primary.
match server.2.as_ref() {
"primary" => primary_count += 1,
_ => (),
};
// Check role spelling.
match server.2.as_ref() {
"primary" => (),
"replica" => (),
_ => {
error!(
"Shard {} server role must be either 'primary' or 'replica', got: '{}'",
shard.0, server.2
);
return Err(Error::BadConfig);
}
};
}
if primary_count > 1 {
error!("Shard {} has more than on primary configured", &shard.0);
return Err(Error::BadConfig);
}
if dup_check.len() != shard.1.servers.len() {
error!("Shard {} contains duplicate server configs", &shard.0);
return Err(Error::BadConfig);
}
}
}
config.path = path.to_string();
@@ -693,7 +515,7 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result<bool, E
let new_config = get_config();
if old_config.pools != new_config.pools {
info!("Pool configuration changed");
info!("Pool configuration changed, re-creating server pools");
ConnectionPool::from_config(client_server_map).await?;
Ok(true)
} else if old_config != new_config {
@@ -721,12 +543,12 @@ mod test {
assert_eq!(get_config().pools["simple_db"].users.len(), 1);
assert_eq!(
get_config().pools["sharded_db"].shards["0"].servers[0].host,
get_config().pools["sharded_db"].shards["0"].servers[0].0,
"127.0.0.1"
);
assert_eq!(
get_config().pools["sharded_db"].shards["1"].servers[0].role,
Role::Primary
get_config().pools["sharded_db"].shards["1"].servers[0].2,
"primary"
);
assert_eq!(
get_config().pools["sharded_db"].shards["1"].database,
@@ -744,11 +566,11 @@ mod test {
assert_eq!(get_config().pools["sharded_db"].default_role, "any");
assert_eq!(
get_config().pools["simple_db"].shards["0"].servers[0].host,
get_config().pools["simple_db"].shards["0"].servers[0].0,
"127.0.0.1"
);
assert_eq!(
get_config().pools["simple_db"].shards["0"].servers[0].port,
get_config().pools["simple_db"].shards["0"].servers[0].1,
5432
);
assert_eq!(

2
src/errors.rs
View File

@@ -11,6 +11,4 @@ pub enum Error {
AllServersDown,
ClientError,
TlsError,
StatementTimeout,
ShuttingDown,
}

32
src/lib.rs
View File

@@ -1,32 +0,0 @@
pub mod config;
pub mod constants;
pub mod errors;
pub mod messages;
pub mod pool;
pub mod scram;
pub mod server;
pub mod sharding;
pub mod stats;
pub mod tls;
/// Format chrono::Duration to be more human-friendly.
///
/// # Arguments
///
/// * `duration` - A duration of time
pub fn format_duration(duration: &chrono::Duration) -> String {
let milliseconds = format!("{:0>3}", duration.num_milliseconds() % 1000);
let seconds = format!("{:0>2}", duration.num_seconds() % 60);
let minutes = format!("{:0>2}", duration.num_minutes() % 60);
let hours = format!("{:0>2}", duration.num_hours() % 24);
let days = duration.num_days().to_string();
format!(
"{}d {}:{}:{}.{}",
days, hours, minutes, seconds, milliseconds
)
}

315
src/main.rs
View File

@@ -24,7 +24,6 @@ extern crate async_trait;
extern crate bb8;
extern crate bytes;
extern crate env_logger;
extern crate exitcode;
extern crate log;
extern crate md5;
extern crate num_cpus;
@@ -37,9 +36,8 @@ extern crate tokio;
extern crate tokio_rustls;
extern crate toml;
use log::{error, info, warn};
use log::{debug, error, info};
use parking_lot::Mutex;
use pgcat::format_duration;
use tokio::net::TcpListener;
use tokio::{
signal::unix::{signal as unix_signal, SignalKind},
@@ -68,20 +66,18 @@ mod stats;
mod tls;
use crate::config::{get_config, reload_config, VERSION};
use crate::errors::Error;
use crate::pool::{ClientServerMap, ConnectionPool};
use crate::prometheus::start_metric_server;
use crate::stats::{Collector, Reporter, REPORTER};
#[tokio::main(worker_threads = 4)]
async fn main() {
env_logger::builder().format_timestamp_micros().init();
env_logger::init();
info!("Welcome to PgCat! Meow. (Version {})", VERSION);
if !query_router::QueryRouter::setup() {
error!("Could not setup query router");
std::process::exit(exitcode::CONFIG);
return;
}
let args = std::env::args().collect::<Vec<String>>();
@@ -96,22 +92,19 @@ async fn main() {
Ok(_) => (),
Err(err) => {
error!("Config parse error: {:?}", err);
std::process::exit(exitcode::CONFIG);
return;
}
};
let config = get_config();
if let Some(true) = config.general.enable_prometheus_exporter {
let http_addr_str = format!(
"{}:{}",
config.general.host, config.general.prometheus_exporter_port
);
let http_addr_str = format!("{}:{}", config.general.host, crate::prometheus::HTTP_PORT);
let http_addr = match SocketAddr::from_str(&http_addr_str) {
Ok(addr) => addr,
Err(err) => {
error!("Invalid http address: {}", err);
std::process::exit(exitcode::CONFIG);
return;
}
};
tokio::task::spawn(async move {
@@ -125,7 +118,7 @@ async fn main() {
Ok(sock) => sock,
Err(err) => {
error!("Listener socket error: {:?}", err);
std::process::exit(exitcode::CONFIG);
return;
}
};
@@ -137,33 +130,125 @@ async fn main() {
let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new()));
// Statistics reporting.
let (stats_tx, stats_rx) = mpsc::channel(100_000);
REPORTER.store(Arc::new(Reporter::new(stats_tx.clone())));
let (tx, rx) = mpsc::channel(100);
REPORTER.store(Arc::new(Reporter::new(tx.clone())));
// Connection pool that allows to query all shards and replicas.
match ConnectionPool::from_config(client_server_map.clone()).await {
Ok(_) => (),
Err(err) => {
error!("Pool error: {:?}", err);
std::process::exit(exitcode::CONFIG);
return;
}
};
// Statistics collector task.
let collector_tx = tx.clone();
// Save these for reloading
let reload_client_server_map = client_server_map.clone();
let autoreload_client_server_map = client_server_map.clone();
tokio::task::spawn(async move {
let mut stats_collector = Collector::new(stats_rx, stats_tx.clone());
let mut stats_collector = Collector::new(rx, collector_tx);
stats_collector.collect().await;
});
info!("Config autoreloader: {}", config.general.autoreload);
info!("Waiting for clients");
let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000));
let autoreload_client_server_map = client_server_map.clone();
let (shutdown_event_tx, mut shutdown_event_rx) = broadcast::channel::<()>(1);
let shutdown_event_tx_clone = shutdown_event_tx.clone();
// Client connection loop.
tokio::task::spawn(async move {
// Creates event subscriber for shutdown event, this is dropped when shutdown event is broadcast
let mut listener_shutdown_event_rx = shutdown_event_tx_clone.subscribe();
loop {
autoreload_interval.tick().await;
if config.general.autoreload {
info!("Automatically reloading config");
let client_server_map = client_server_map.clone();
// Listen for shutdown event and client connection at the same time
let (socket, addr) = tokio::select! {
_ = listener_shutdown_event_rx.recv() => {
// Exits client connection loop which drops listener, listener_shutdown_event_rx and shutdown_event_tx_clone
break;
}
listener_response = listener.accept() => {
match listener_response {
Ok((socket, addr)) => (socket, addr),
Err(err) => {
error!("{:?}", err);
continue;
}
}
}
};
// Used to signal shutdown
let client_shutdown_handler_rx = shutdown_event_tx_clone.subscribe();
// Used to signal that the task has completed
let dummy_tx = shutdown_event_tx_clone.clone();
// Handle client.
tokio::task::spawn(async move {
let start = chrono::offset::Utc::now().naive_utc();
match client::client_entrypoint(
socket,
client_server_map,
client_shutdown_handler_rx,
)
.await
{
Ok(_) => {
let duration = chrono::offset::Utc::now().naive_utc() - start;
info!(
"Client {:?} disconnected, session duration: {}",
addr,
format_duration(&duration)
);
}
Err(err) => {
debug!("Client disconnected with error {:?}", err);
}
};
// Drop this transmitter so receiver knows that the task is completed
drop(dummy_tx);
});
}
});
// Reload config:
// kill -SIGHUP $(pgrep pgcat)
tokio::task::spawn(async move {
let mut stream = unix_signal(SignalKind::hangup()).unwrap();
loop {
stream.recv().await;
info!("Reloading config");
match reload_config(reload_client_server_map.clone()).await {
Ok(_) => (),
Err(_) => continue,
};
get_config().show();
}
});
if config.general.autoreload {
let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000));
tokio::task::spawn(async move {
info!("Config autoreloader started");
loop {
interval.tick().await;
match reload_config(autoreload_client_server_map.clone()).await {
Ok(changed) => {
if changed {
@@ -173,132 +258,82 @@ async fn main() {
Err(_) => (),
};
}
}
});
});
}
let mut term_signal = unix_signal(SignalKind::terminate()).unwrap();
let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap();
let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap();
let (shutdown_tx, _) = broadcast::channel::<()>(1);
let (drain_tx, mut drain_rx) = mpsc::channel::<i32>(2048);
let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1);
info!("Waiting for clients");
tokio::select! {
// Initiate graceful shutdown sequence on sig int
_ = interrupt_signal.recv() => {
info!("Got SIGINT, waiting for client connection drain now");
let mut admin_only = false;
let mut total_clients = 0;
// Broadcast that client tasks need to finish
shutdown_event_tx.send(()).unwrap();
// Closes transmitter
drop(shutdown_event_tx);
loop {
tokio::select! {
// Reload config:
// kill -SIGHUP $(pgrep pgcat)
_ = sighup_signal.recv() => {
info!("Reloading config");
match reload_config(client_server_map.clone()).await {
Ok(_) => (),
Err(_) => (),
};
get_config().show();
},
// Initiate graceful shutdown sequence on sig int
_ = interrupt_signal.recv() => {
info!("Got SIGINT, waiting for client connection drain now");
admin_only = true;
// Broadcast that client tasks need to finish
let _ = shutdown_tx.send(());
let exit_tx = exit_tx.clone();
let _ = drain_tx.send(0).await;
tokio::task::spawn(async move {
let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout));
// First tick fires immediately.
interval.tick().await;
// Second one in the interval time.
interval.tick().await;
// We're done waiting.
error!("Graceful shutdown timed out. {} active clients being closed", total_clients);
let _ = exit_tx.send(()).await;
});
},
_ = term_signal.recv() => {
info!("Got SIGTERM, closing with {} clients active", total_clients);
break;
},
new_client = listener.accept() => {
let (socket, addr) = match new_client {
Ok((socket, addr)) => (socket, addr),
Err(err) => {
error!("{:?}", err);
continue;
// This is in a loop because the first event that the receiver receives will be the shutdown event
// This is not what we are waiting for instead, we want the receiver to send an error once all senders are closed which is reached after the shutdown event is received
loop {
match tokio::time::timeout(
tokio::time::Duration::from_millis(config.general.shutdown_timeout),
shutdown_event_rx.recv(),
)
.await
{
Ok(res) => match res {
Ok(_) => {}
Err(_) => break,
},
Err(_) => {
info!("Timed out while waiting for clients to shutdown");
break;
}
};
let shutdown_rx = shutdown_tx.subscribe();
let drain_tx = drain_tx.clone();
let client_server_map = client_server_map.clone();
tokio::task::spawn(async move {
let start = chrono::offset::Utc::now().naive_utc();
match client::client_entrypoint(
socket,
client_server_map,
shutdown_rx,
drain_tx,
admin_only,
)
.await
{
Ok(()) => {
let duration = chrono::offset::Utc::now().naive_utc() - start;
info!(
"Client {:?} disconnected, session duration: {}",
addr,
format_duration(&duration)
);
}
Err(err) => {
match err {
// Don't count the clients we rejected.
Error::ShuttingDown => (),
_ => {
// drain_tx.send(-1).await.unwrap();
}
}
warn!("Client disconnected with error {:?}", err);
}
};
});
}
_ = exit_rx.recv() => {
break;
}
client_ping = drain_rx.recv() => {
let client_ping = client_ping.unwrap();
total_clients += client_ping;
if total_clients == 0 && admin_only {
let _ = exit_tx.send(()).await;
}
}
}
},
_ = term_signal.recv() => (),
}
info!("Shutting down...");
}
/// Format chrono::Duration to be more human-friendly.
///
/// # Arguments
///
/// * `duration` - A duration of time
fn format_duration(duration: &chrono::Duration) -> String {
let seconds = {
let seconds = duration.num_seconds() % 60;
if seconds < 10 {
format!("0{}", seconds)
} else {
format!("{}", seconds)
}
};
let minutes = {
let minutes = duration.num_minutes() % 60;
if minutes < 10 {
format!("0{}", minutes)
} else {
format!("{}", minutes)
}
};
let hours = {
let hours = duration.num_hours() % 24;
if hours < 10 {
format!("0{}", hours)
} else {
format!("{}", hours)
}
};
let days = duration.num_days().to_string();
format!("{}d {}:{}:{}", days, hours, minutes, seconds)
}

2
src/messages.rs
View File

@@ -496,7 +496,7 @@ where
Ok(bytes)
}
pub fn server_parameter_message(key: &str, value: &str) -> BytesMut {
pub fn server_paramater_message(key: &str, value: &str) -> BytesMut {
let mut server_info = BytesMut::new();
let null_byte_size = 1;

375
src/pool.rs
View File

@@ -6,91 +6,44 @@ use chrono::naive::NaiveDateTime;
use log::{debug, error, info, warn};
use once_cell::sync::Lazy;
use parking_lot::{Mutex, RwLock};
use rand::seq::SliceRandom;
use rand::thread_rng;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
use crate::config::{get_config, Address, PoolMode, Role, User};
use crate::config::{get_config, Address, Role, Shard, User};
use crate::errors::Error;
use crate::server::Server;
use crate::sharding::ShardingFunction;
use crate::stats::{get_reporter, Reporter};
pub type ProcessId = i32;
pub type SecretKey = i32;
pub type ServerHost = String;
pub type ServerPort = u16;
pub type BanList = Arc<RwLock<Vec<HashMap<Address, NaiveDateTime>>>>;
pub type ClientServerMap =
Arc<Mutex<HashMap<(ProcessId, SecretKey), (ProcessId, SecretKey, ServerHost, ServerPort)>>>;
pub type PoolMap = HashMap<PoolIdentifier, ConnectionPool>;
pub type ClientServerMap = Arc<Mutex<HashMap<(i32, i32), (i32, i32, String, String)>>>;
pub type PoolMap = HashMap<(String, String), ConnectionPool>;
/// The connection pool, globally available.
/// This is atomic and safe and read-optimized.
/// The pool is recreated dynamically when the config is reloaded.
pub static POOLS: Lazy<ArcSwap<PoolMap>> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default()));
static POOLS_HASH: Lazy<ArcSwap<HashSet<crate::config::Pool>>> =
Lazy::new(|| ArcSwap::from_pointee(HashSet::default()));
/// An identifier for a PgCat pool,
/// a database visible to clients.
#[derive(Hash, Debug, Clone, PartialEq, Eq)]
pub struct PoolIdentifier {
// The name of the database clients want to connect to.
pub db: String,
/// The username the client connects with. Each user gets its own pool.
pub user: String,
}
impl PoolIdentifier {
/// Create a new user/pool identifier.
pub fn new(db: &str, user: &str) -> PoolIdentifier {
PoolIdentifier {
db: db.to_string(),
user: user.to_string(),
}
}
}
/// Pool settings.
#[derive(Clone, Debug)]
pub struct PoolSettings {
/// Transaction or Session.
pub pool_mode: PoolMode,
// Number of shards.
pub shards: usize,
// Connecting user.
pub pool_mode: String,
pub shards: HashMap<String, Shard>,
pub user: User,
// Default server role to connect to.
pub default_role: Option<Role>,
// Enable/disable query parser.
pub default_role: String,
pub query_parser_enabled: bool,
// Read from the primary as well or not.
pub primary_reads_enabled: bool,
// Sharding function.
pub sharding_function: ShardingFunction,
pub sharding_function: String,
}
impl Default for PoolSettings {
fn default() -> PoolSettings {
PoolSettings {
pool_mode: PoolMode::Transaction,
shards: 1,
pool_mode: String::from("transaction"),
shards: HashMap::from([(String::from("1"), Shard::default())]),
user: User::default(),
default_role: None,
default_role: String::from("any"),
query_parser_enabled: false,
primary_reads_enabled: true,
sharding_function: ShardingFunction::PgBigintHash,
sharding_function: "pg_bigint_hash".to_string(),
}
}
}
@@ -118,7 +71,6 @@ pub struct ConnectionPool {
/// on pool creation and save the K messages here.
server_info: BytesMut,
/// Pool configuration.
pub settings: PoolSettings,
}
@@ -126,41 +78,11 @@ impl ConnectionPool {
/// Construct the connection pool from the configuration.
pub async fn from_config(client_server_map: ClientServerMap) -> Result<(), Error> {
let config = get_config();
let mut new_pools = PoolMap::default();
let mut new_pools = HashMap::new();
let mut address_id = 0;
let mut pools_hash = (*(*POOLS_HASH.load())).clone();
for (pool_name, pool_config) in &config.pools {
let changed = pools_hash.insert(pool_config.clone());
// There is one pool per database/user pair.
for (_, user) in &pool_config.users {
// If the pool hasn't changed, get existing reference and insert it into the new_pools.
// We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8).
if !changed {
match get_pool(&pool_name, &user.username) {
Some(pool) => {
info!(
"[pool: {}][user: {}] has not changed",
pool_name, user.username
);
new_pools.insert(
PoolIdentifier::new(&pool_name, &user.username),
pool.clone(),
);
continue;
}
None => (),
}
}
info!(
"[pool: {}][user: {}] creating new pool",
pool_name, user.username
);
for (_user_index, user_info) in &pool_config.users {
let mut shards = Vec::new();
let mut addresses = Vec::new();
let mut banlist = Vec::new();
@@ -174,50 +96,51 @@ impl ConnectionPool {
// Sort by shard number to ensure consistency.
shard_ids.sort_by_key(|k| k.parse::<i64>().unwrap());
for shard_idx in &shard_ids {
let shard = &pool_config.shards[shard_idx];
for shard_idx in shard_ids {
let shard = &pool_config.shards[&shard_idx];
let mut pools = Vec::new();
let mut servers = Vec::new();
let mut address_index = 0;
let mut replica_number = 0;
for server in shard.servers.iter() {
let role = match server.2.as_ref() {
"primary" => Role::Primary,
"replica" => Role::Replica,
_ => {
error!("Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2);
Role::Replica
}
};
let address = Address {
id: address_id,
database: shard.database.clone(),
host: server.host.clone(),
port: server.port,
role: server.role,
address_index,
database: pool_name.clone(),
host: server.0.clone(),
port: server.1.to_string(),
role: role,
replica_number,
shard: shard_idx.parse::<usize>().unwrap(),
username: user.username.clone(),
pool_name: pool_name.clone(),
};
address_id += 1;
address_index += 1;
if server.role == Role::Replica {
if role == Role::Replica {
replica_number += 1;
}
let manager = ServerPool::new(
address.clone(),
user.clone(),
user_info.clone(),
&shard.database,
client_server_map.clone(),
get_reporter(),
);
let connect_timeout = match pool_config.connect_timeout {
Some(connect_timeout) => connect_timeout,
None => config.general.connect_timeout,
};
let pool = Pool::builder()
.max_size(user.pool_size)
.connection_timeout(std::time::Duration::from_millis(connect_timeout))
.max_size(user_info.pool_size)
.connection_timeout(std::time::Duration::from_millis(
config.general.connect_timeout,
))
.test_on_check_out(false)
.build(manager)
.await
@@ -241,19 +164,13 @@ impl ConnectionPool {
stats: get_reporter(),
server_info: BytesMut::new(),
settings: PoolSettings {
pool_mode: pool_config.pool_mode,
// shards: pool_config.shards.clone(),
shards: shard_ids.len(),
user: user.clone(),
default_role: match pool_config.default_role.as_str() {
"any" => None,
"replica" => Some(Role::Replica),
"primary" => Some(Role::Primary),
_ => unreachable!(),
},
pool_mode: pool_config.pool_mode.clone(),
shards: pool_config.shards.clone(),
user: user_info.clone(),
default_role: pool_config.default_role.clone(),
query_parser_enabled: pool_config.query_parser_enabled.clone(),
primary_reads_enabled: pool_config.primary_reads_enabled,
sharding_function: pool_config.sharding_function,
sharding_function: pool_config.sharding_function.clone(),
},
};
@@ -266,14 +183,11 @@ impl ConnectionPool {
return Err(err);
}
};
// There is one pool per database/user pair.
new_pools.insert(PoolIdentifier::new(&pool_name, &user.username), pool);
new_pools.insert((pool_name.clone(), user_info.username.clone()), pool);
}
}
POOLS.store(Arc::new(new_pools.clone()));
POOLS_HASH.store(Arc::new(pools_hash.clone()));
Ok(())
}
@@ -285,9 +199,16 @@ impl ConnectionPool {
/// the pooler starts up.
async fn validate(&mut self) -> Result<(), Error> {
let mut server_infos = Vec::new();
let stats = self.stats.clone();
for shard in 0..self.shards() {
for server in 0..self.servers(shard) {
let connection = match self.databases[shard][server].get().await {
let mut round_robin = 0;
for _ in 0..self.servers(shard) {
// To keep stats consistent.
let fake_process_id = 0;
let connection = match self.get(shard, None, fake_process_id, round_robin).await {
Ok(conn) => conn,
Err(err) => {
error!("Shard {} down or misconfigured: {:?}", shard, err);
@@ -295,21 +216,25 @@ impl ConnectionPool {
}
};
let proxy = connection;
let proxy = connection.0;
let address = connection.1;
let server = &*proxy;
let server_info = server.server_info();
stats.client_disconnecting(fake_process_id, address.id);
if server_infos.len() > 0 {
// Compare against the last server checked.
if server_info != server_infos[server_infos.len() - 1] {
warn!(
"{:?} has different server configuration than the last server",
proxy.address()
address
);
}
}
server_infos.push(server_info);
round_robin += 1;
}
}
@@ -319,8 +244,6 @@ impl ConnectionPool {
return Err(Error::AllServersDown);
}
// We're assuming all servers are identical.
// TODO: not true.
self.server_info = server_infos[0].clone();
Ok(())
@@ -329,47 +252,73 @@ impl ConnectionPool {
/// Get a connection from the pool.
pub async fn get(
&self,
shard: usize, // shard number
role: Option<Role>, // primary or replica
process_id: i32, // client id
shard: usize, // shard number
role: Option<Role>, // primary or replica
process_id: i32, // client id
mut round_robin: usize, // round robin offset
) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> {
let mut candidates: Vec<&Address> = self.addresses[shard]
.iter()
.filter(|address| address.role == role)
.collect();
let now = Instant::now();
let addresses = &self.addresses[shard];
// Random load balancing
candidates.shuffle(&mut thread_rng());
let mut allowed_attempts = match role {
// Primary-specific queries get one attempt, if the primary is down,
// nothing we should do about it I think. It's dangerous to retry
// write queries.
Some(Role::Primary) => 1,
// Replicas get to try as many times as there are replicas
// and connections in the pool.
_ => addresses.len(),
};
debug!("Allowed attempts for {:?}: {}", role, allowed_attempts);
let exists = match role {
Some(role) => addresses.iter().filter(|addr| addr.role == role).count() > 0,
None => true,
};
if !exists {
error!("Requested role {:?}, but none are configured", role);
return Err(Error::BadConfig);
}
let healthcheck_timeout = get_config().general.healthcheck_timeout;
let healthcheck_delay = get_config().general.healthcheck_delay as u128;
while !candidates.is_empty() {
// Get the next candidate
let address = match candidates.pop() {
Some(address) => address,
None => break,
};
while allowed_attempts > 0 {
// Round-robin replicas.
round_robin += 1;
if self.is_banned(&address, role) {
debug!("Address {:?} is banned", address);
let index = round_robin % addresses.len();
let address = &addresses[index];
// Make sure you're getting a primary or a replica
// as per request. If no specific role is requested, the first
// available will be chosen.
if address.role != role {
continue;
}
allowed_attempts -= 1;
// Don't attempt to connect to banned servers.
if self.is_banned(address, shard, role) {
continue;
}
// Indicate we're waiting on a server connection from a pool.
let now = Instant::now();
self.stats.client_waiting(process_id);
self.stats.client_waiting(process_id, address.id);
// Check if we can connect
let mut conn = match self.databases[address.shard][address.address_index]
.get()
.await
{
let mut conn = match self.databases[shard][index].get().await {
Ok(conn) => conn,
Err(err) => {
error!("Banning instance {:?}, error: {:?}", address, err);
self.ban(&address, process_id);
self.stats.client_checkout_error(process_id, address.id);
error!("Banning replica {}, error: {:?}", index, err);
self.ban(address, shard, process_id);
self.stats.client_disconnecting(process_id, address.id);
self.stats
.checkout_time(now.elapsed().as_micros(), process_id, address.id);
continue;
}
};
@@ -381,99 +330,89 @@ impl ConnectionPool {
let require_healthcheck =
server.last_activity().elapsed().unwrap().as_millis() > healthcheck_delay;
// Do not issue a health check unless it's been a little while
// since we last checked the server is ok.
// Health checks are pretty expensive.
if !require_healthcheck {
self.stats
.checkout_time(now.elapsed().as_micros(), process_id, server.server_id());
self.stats.server_active(process_id, server.server_id());
.checkout_time(now.elapsed().as_micros(), process_id, address.id);
self.stats.server_idle(conn.process_id(), address.id);
return Ok((conn, address.clone()));
}
debug!("Running health check on server {:?}", address);
self.stats.server_tested(server.server_id());
self.stats.server_tested(server.process_id(), address.id);
match tokio::time::timeout(
tokio::time::Duration::from_millis(healthcheck_timeout),
server.query(";"), // Cheap query as it skips the query planner
server.query(";"),
)
.await
{
// Check if health check succeeded.
Ok(res) => match res {
Ok(_) => {
self.stats.checkout_time(
now.elapsed().as_micros(),
process_id,
conn.server_id(),
);
self.stats.server_active(process_id, conn.server_id());
self.stats
.checkout_time(now.elapsed().as_micros(), process_id, address.id);
self.stats.server_idle(conn.process_id(), address.id);
return Ok((conn, address.clone()));
}
// Health check failed.
Err(err) => {
error!(
"Banning instance {:?} because of failed health check, {:?}",
address, err
);
Err(_) => {
error!("Banning replica {} because of failed health check", index);
// Don't leave a bad connection in the pool.
server.mark_bad();
self.ban(&address, process_id);
self.ban(address, shard, process_id);
continue;
}
},
// Health check timed out.
Err(err) => {
error!(
"Banning instance {:?} because of health check timeout, {:?}",
address, err
);
Err(_) => {
error!("Banning replica {} because of health check timeout", index);
// Don't leave a bad connection in the pool.
server.mark_bad();
self.ban(&address, process_id);
self.ban(address, shard, process_id);
continue;
}
}
}
Err(Error::AllServersDown)
return Err(Error::AllServersDown);
}
/// Ban an address (i.e. replica). It no longer will serve
/// traffic for any new transactions. Existing transactions on that replica
/// will finish successfully or error out to the clients.
pub fn ban(&self, address: &Address, client_id: i32) {
error!("Banning {:?}", address);
self.stats.client_ban_error(client_id, address.id);
pub fn ban(&self, address: &Address, shard: usize, process_id: i32) {
self.stats.client_disconnecting(process_id, address.id);
self.stats
.checkout_time(Instant::now().elapsed().as_micros(), process_id, address.id);
error!("Banning {:?}", address);
let now = chrono::offset::Utc::now().naive_utc();
let mut guard = self.banlist.write();
guard[address.shard].insert(address.clone(), now);
guard[shard].insert(address.clone(), now);
}
/// Clear the replica to receive traffic again. Takes effect immediately
/// for all new transactions.
pub fn _unban(&self, address: &Address) {
pub fn _unban(&self, address: &Address, shard: usize) {
let mut guard = self.banlist.write();
guard[address.shard].remove(address);
guard[shard].remove(address);
}
/// Check if a replica can serve traffic. If all replicas are banned,
/// we unban all of them. Better to try then not to.
pub fn is_banned(&self, address: &Address, role: Option<Role>) -> bool {
pub fn is_banned(&self, address: &Address, shard: usize, role: Option<Role>) -> bool {
let replicas_available = match role {
Some(Role::Replica) => self.addresses[address.shard]
Some(Role::Replica) => self.addresses[shard]
.iter()
.filter(|addr| addr.role == Role::Replica)
.count(),
None => self.addresses[address.shard].len(),
None => self.addresses[shard].len(),
Some(Role::Primary) => return false, // Primary cannot be banned.
};
@@ -482,17 +421,17 @@ impl ConnectionPool {
let guard = self.banlist.read();
// Everything is banned = nothing is banned.
if guard[address.shard].len() == replicas_available {
if guard[shard].len() == replicas_available {
drop(guard);
let mut guard = self.banlist.write();
guard[address.shard].clear();
guard[shard].clear();
drop(guard);
warn!("Unbanning all replicas.");
return false;
}
// I expect this to miss 99.9999% of the time.
match guard[address.shard].get(address) {
match guard[shard].get(address) {
Some(timestamp) => {
let now = chrono::offset::Utc::now().naive_utc();
let config = get_config();
@@ -502,7 +441,7 @@ impl ConnectionPool {
drop(guard);
warn!("Unbanning {:?}", address);
let mut guard = self.banlist.write();
guard[address.shard].remove(address);
guard[shard].remove(address);
false
} else {
debug!("{:?} is banned", address);
@@ -586,21 +525,19 @@ impl ManageConnection for ServerPool {
/// Attempts to create a new connection.
async fn connect(&self) -> Result<Self::Connection, Self::Error> {
info!("Creating a new server connection {:?}", self.address);
let server_id = rand::random::<i32>();
self.stats.server_register(
server_id,
self.address.id,
info!(
"Creating a new connection to {:?} using user {:?}",
self.address.name(),
self.address.pool_name.clone(),
self.address.username.clone(),
self.user.username
);
self.stats.server_login(server_id);
// Put a temporary process_id into the stats
// for server login.
let process_id = rand::random::<i32>();
self.stats.server_login(process_id, self.address.id);
// Connect to the PostgreSQL server.
match Server::startup(
server_id,
&self.address,
&self.user,
&self.database,
@@ -610,18 +547,20 @@ impl ManageConnection for ServerPool {
.await
{
Ok(conn) => {
self.stats.server_idle(server_id);
// Remove the temporary process_id from the stats.
self.stats.server_disconnecting(process_id, self.address.id);
Ok(conn)
}
Err(err) => {
self.stats.server_disconnecting(server_id);
// Remove the temporary process_id from the stats.
self.stats.server_disconnecting(process_id, self.address.id);
Err(err)
}
}
}
/// Determines if the connection is still connected to the database.
async fn is_valid(&self, _conn: &mut Self::Connection) -> Result<(), Self::Error> {
async fn is_valid(&self, _conn: &mut PooledConnection<'_, Self>) -> Result<(), Self::Error> {
Ok(())
}
@@ -632,22 +571,20 @@ impl ManageConnection for ServerPool {
}
/// Get the connection pool
pub fn get_pool(db: &str, user: &str) -> Option<ConnectionPool> {
match get_all_pools().get(&PoolIdentifier::new(&db, &user)) {
pub fn get_pool(db: String, user: String) -> Option<ConnectionPool> {
match get_all_pools().get(&(db, user)) {
Some(pool) => Some(pool.clone()),
None => None,
}
}
/// Get a pointer to all configured pools.
pub fn get_all_pools() -> HashMap<PoolIdentifier, ConnectionPool> {
return (*(*POOLS.load())).clone();
}
/// How many total servers we have in the config.
pub fn get_number_of_addresses() -> usize {
get_all_pools()
.iter()
.map(|(_, pool)| pool.databases())
.sum()
}
pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> {
return (*(*POOLS.load())).clone();
}

6
src/prometheus.rs
View File

@@ -8,7 +8,9 @@ use std::net::SocketAddr;
use crate::config::Address;
use crate::pool::get_all_pools;
use crate::stats::get_address_stats;
use crate::stats::get_stats;
pub const HTTP_PORT: usize = 9930;
struct MetricHelpType {
help: &'static str,
@@ -164,7 +166,7 @@ impl PrometheusMetric {
async fn prometheus_stats(request: Request<Body>) -> Result<Response<Body>, hyper::http::Error> {
match (request.method(), request.uri().path()) {
(&Method::GET, "/metrics") => {
let stats: HashMap<usize, HashMap<String, i64>> = get_address_stats();
let stats = get_stats();
let mut lines = Vec::new();
for (_, pool) in get_all_pools() {

58
src/query_router.rs
View File

@@ -10,7 +10,7 @@ use sqlparser::parser::Parser;
use crate::config::Role;
use crate::pool::PoolSettings;
use crate::sharding::Sharder;
use crate::sharding::{Sharder, ShardingFunction};
/// Regexes used to parse custom commands.
const CUSTOM_SQL_REGEXES: [&str; 7] = [
@@ -55,13 +55,11 @@ pub struct QueryRouter {
/// Include the primary into the replica pool for reads.
primary_reads_enabled: bool,
/// Pool configuration.
pool_settings: PoolSettings,
}
impl QueryRouter {
/// One-time initialization of regexes
/// that parse our custom SQL protocol.
/// One-time initialization of regexes.
pub fn setup() -> bool {
let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) {
Ok(rgx) => rgx,
@@ -76,7 +74,10 @@ impl QueryRouter {
.map(|rgx| Regex::new(rgx).unwrap())
.collect();
assert_eq!(list.len(), set.len());
// Impossible
if list.len() != set.len() {
return false;
}
match CUSTOM_SQL_REGEX_LIST.set(list) {
Ok(_) => true,
@@ -89,8 +90,7 @@ impl QueryRouter {
}
}
/// Create a new instance of the query router.
/// Each client gets its own.
/// Create a new instance of the query router. Each client gets its own.
pub fn new() -> QueryRouter {
QueryRouter {
active_shard: None,
@@ -101,7 +101,6 @@ impl QueryRouter {
}
}
/// Pool settings can change because of a config reload.
pub fn update_pool_settings(&mut self, pool_settings: PoolSettings) {
self.pool_settings = pool_settings;
}
@@ -137,6 +136,19 @@ impl QueryRouter {
return None;
}
let sharding_function = match self.pool_settings.sharding_function.as_ref() {
"pg_bigint_hash" => ShardingFunction::PgBigintHash,
"sha1" => ShardingFunction::Sha1,
_ => unreachable!(),
};
let default_server_role = match self.pool_settings.default_role.as_ref() {
"any" => None,
"primary" => Some(Role::Primary),
"replica" => Some(Role::Replica),
_ => unreachable!(),
};
let command = match matches[0] {
0 => Command::SetShardingKey,
1 => Command::SetShard,
@@ -169,8 +181,8 @@ impl QueryRouter {
Command::ShowShard => self.shard().to_string(),
Command::ShowServerRole => match self.active_role {
Some(Role::Primary) => Role::Primary.to_string(),
Some(Role::Replica) => Role::Replica.to_string(),
Some(Role::Primary) => String::from("primary"),
Some(Role::Replica) => String::from("replica"),
None => {
if self.query_parser_enabled {
String::from("auto")
@@ -188,10 +200,7 @@ impl QueryRouter {
match command {
Command::SetShardingKey => {
let sharder = Sharder::new(
self.pool_settings.shards,
self.pool_settings.sharding_function,
);
let sharder = Sharder::new(self.pool_settings.shards.len(), sharding_function);
let shard = sharder.shard(value.parse::<i64>().unwrap());
self.active_shard = Some(shard);
value = shard.to_string();
@@ -199,7 +208,7 @@ impl QueryRouter {
Command::SetShard => {
self.active_shard = match value.to_ascii_uppercase().as_ref() {
"ANY" => Some(rand::random::<usize>() % self.pool_settings.shards),
"ANY" => Some(rand::random::<usize>() % self.pool_settings.shards.len()),
_ => Some(value.parse::<usize>().unwrap()),
};
}
@@ -227,7 +236,7 @@ impl QueryRouter {
}
"default" => {
self.active_role = self.pool_settings.default_role;
self.active_role = default_server_role;
self.query_parser_enabled = self.query_parser_enabled;
self.active_role
}
@@ -358,10 +367,10 @@ impl QueryRouter {
#[cfg(test)]
mod test {
use std::collections::HashMap;
use super::*;
use crate::config::PoolMode;
use crate::messages::simple_query;
use crate::sharding::ShardingFunction;
use bytes::BufMut;
#[test]
@@ -624,13 +633,13 @@ mod test {
QueryRouter::setup();
let pool_settings = PoolSettings {
pool_mode: PoolMode::Transaction,
shards: 0,
pool_mode: "transaction".to_string(),
shards: HashMap::default(),
user: crate::config::User::default(),
default_role: Some(Role::Replica),
default_role: Role::Replica.to_string(),
query_parser_enabled: true,
primary_reads_enabled: false,
sharding_function: ShardingFunction::PgBigintHash,
sharding_function: "pg_bigint_hash".to_string(),
};
let mut qr = QueryRouter::new();
assert_eq!(qr.active_role, None);
@@ -652,6 +661,9 @@ mod test {
let q2 = simple_query("SET SERVER ROLE TO 'default'");
assert!(qr.try_execute_command(q2) != None);
assert_eq!(qr.active_role.unwrap(), pool_settings.clone().default_role);
assert_eq!(
qr.active_role.unwrap().to_string(),
pool_settings.clone().default_role
);
}
}

103
src/server.rs
View File

@@ -1,8 +1,7 @@
/// Implementation of the PostgreSQL server (database) protocol.
/// Here we are pretending to the a Postgres client.
use bytes::{Buf, BufMut, BytesMut};
use log::{debug, error, info, trace, warn};
use std::io::Read;
use log::{debug, error, info, trace};
use std::time::SystemTime;
use tokio::io::{AsyncReadExt, BufReader};
use tokio::net::{
@@ -14,14 +13,12 @@ use crate::config::{Address, User};
use crate::constants::*;
use crate::errors::Error;
use crate::messages::*;
use crate::pool::ClientServerMap;
use crate::scram::ScramSha256;
use crate::stats::Reporter;
use crate::ClientServerMap;
/// Server state.
pub struct Server {
server_id: i32,
/// Server host, e.g. localhost,
/// port, e.g. 5432, and role, e.g. primary or replica.
address: Address,
@@ -51,9 +48,6 @@ pub struct Server {
/// Is the server broken? We'll remote it from the pool if so.
bad: bool,
/// If server connection requires a DISCARD ALL before checkin
needs_cleanup: bool,
/// Mapping of clients and servers used for query cancellation.
client_server_map: ClientServerMap,
@@ -74,7 +68,6 @@ impl Server {
/// Pretend to be the Postgres client and connect to the server given host, port and credentials.
/// Perform the authentication and return the server in a ready for query state.
pub async fn startup(
server_id: i32,
address: &Address,
user: &User,
database: &str,
@@ -82,7 +75,7 @@ impl Server {
stats: Reporter,
) -> Result<Server, Error> {
let mut stream =
match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await {
match TcpStream::connect(&format!("{}:{}", &address.host, &address.port)).await {
Ok(stream) => stream,
Err(err) => {
error!("Could not connect to server: {}", err);
@@ -318,13 +311,11 @@ impl Server {
write: write,
buffer: BytesMut::with_capacity(8196),
server_info: server_info,
server_id: server_id,
process_id: process_id,
secret_key: secret_key,
in_transaction: false,
data_available: false,
bad: false,
needs_cleanup: false,
client_server_map: client_server_map,
connected_at: chrono::offset::Utc::now().naive_utc(),
stats: stats,
@@ -351,7 +342,7 @@ impl Server {
/// Uses a separate connection that's not part of the connection pool.
pub async fn cancel(
host: &str,
port: u16,
port: &str,
process_id: i32,
secret_key: i32,
) -> Result<(), Error> {
@@ -376,7 +367,8 @@ impl Server {
/// Send messages to the server from the client.
pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> {
self.stats.data_sent(messages.len(), self.server_id);
self.stats
.data_sent(messages.len(), self.process_id, self.address.id);
match write_all_half(&mut self.write, messages).await {
Ok(_) => {
@@ -448,29 +440,6 @@ impl Server {
break;
}
// CommandComplete
'C' => {
let mut command_tag = String::new();
match message.reader().read_to_string(&mut command_tag) {
Ok(_) => {
// Non-exhaustive list of commands that are likely to change session variables/resources
// which can leak between clients. This is a best effort to block bad clients
// from poisoning a transaction-mode pool by setting inappropriate session variables
match command_tag.as_str() {
"SET\0" | "PREPARE\0" => {
debug!("Server connection marked for clean up");
self.needs_cleanup = true;
}
_ => (),
}
}
Err(err) => {
warn!("Encountered an error while parsing CommandTag {}", err);
}
}
}
// DataRow
'D' => {
// More data is available after this message, this is not the end of the reply.
@@ -508,7 +477,8 @@ impl Server {
let bytes = self.buffer.clone();
// Keep track of how much data we got from the server for stats.
self.stats.data_received(bytes.len(), self.server_id);
self.stats
.data_received(bytes.len(), self.process_id, self.address.id);
// Clear the buffer for next query.
self.buffer.clear();
@@ -559,7 +529,7 @@ impl Server {
self.process_id,
self.secret_key,
self.address.host.clone(),
self.address.port,
self.address.port.clone(),
),
);
}
@@ -583,45 +553,14 @@ impl Server {
Ok(())
}
/// Perform any necessary cleanup before putting the server
/// connection back in the pool
pub async fn checkin_cleanup(&mut self) -> Result<(), Error> {
// Client disconnected with an open transaction on the server connection.
// Pgbouncer behavior is to close the server connection but that can cause
// server connection thrashing if clients repeatedly do this.
// Instead, we ROLLBACK that transaction before putting the connection back in the pool
if self.in_transaction() {
warn!("Server returned while still in transaction, rolling back transaction");
self.query("ROLLBACK").await?;
}
// Client disconnected but it perfromed session-altering operations such as
// SET statement_timeout to 1 or create a prepared statement. We clear that
// to avoid leaking state between clients. For performance reasons we only
// send `DISCARD ALL` if we think the session is altered instead of just sending
// it before each checkin.
if self.needs_cleanup {
warn!("Server returned with session state altered, discarding state");
self.query("DISCARD ALL").await?;
self.needs_cleanup = false;
}
return Ok(());
}
/// A shorthand for `SET application_name = $1`.
#[allow(dead_code)]
pub async fn set_name(&mut self, name: &str) -> Result<(), Error> {
if self.application_name != name {
self.application_name = name.to_string();
// We don't want `SET application_name` to mark the server connection
// as needing cleanup
let needs_cleanup_before = self.needs_cleanup;
let result = Ok(self
Ok(self
.query(&format!("SET application_name = '{}'", name))
.await?);
self.needs_cleanup = needs_cleanup_before;
return result;
.await?)
} else {
Ok(())
}
@@ -633,21 +572,15 @@ impl Server {
self.address.clone()
}
/// Get the server connection identifier
/// Used to uniquely identify connection in statistics
pub fn server_id(&self) -> i32 {
self.server_id
/// Get the server's unique identifier.
pub fn process_id(&self) -> i32 {
self.process_id
}
// Get server's latest response timestamp
pub fn last_activity(&self) -> SystemTime {
self.last_activity
}
// Marks a connection as needing DISCARD ALL at checkin
pub fn mark_dirty(&mut self) {
self.needs_cleanup = true;
}
}
impl Drop for Server {
@@ -655,7 +588,8 @@ impl Drop for Server {
/// the socket is in non-blocking mode, so it may not be ready
/// for a write.
fn drop(&mut self) {
self.stats.server_disconnecting(self.server_id);
self.stats
.server_disconnecting(self.process_id(), self.address.id);
let mut bytes = BytesMut::with_capacity(4);
bytes.put_u8(b'X');
@@ -673,8 +607,7 @@ impl Drop for Server {
let duration = now - self.connected_at;
info!(
"Server connection closed {:?}, session duration: {}",
self.address,
"Server connection closed, session duration: {}",
crate::format_duration(&duration)
);
}

14
src/sharding.rs
View File

@@ -1,4 +1,3 @@
use serde_derive::{Deserialize, Serialize};
/// Implements various sharding functions.
use sha1::{Digest, Sha1};
@@ -6,23 +5,12 @@ use sha1::{Digest, Sha1};
const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD;
/// The sharding functions we support.
#[derive(Debug, PartialEq, Copy, Clone, Serialize, Deserialize, Hash, std::cmp::Eq)]
#[derive(Debug, PartialEq, Copy, Clone)]
pub enum ShardingFunction {
#[serde(alias = "pg_bigint_hash", alias = "PgBigintHash")]
PgBigintHash,
#[serde(alias = "sha1", alias = "Sha1")]
Sha1,
}
impl ToString for ShardingFunction {
fn to_string(&self) -> String {
match *self {
ShardingFunction::PgBigintHash => "pg_bigint_hash".to_string(),
ShardingFunction::Sha1 => "sha1".to_string(),
}
}
}
/// The sharder.
pub struct Sharder {
/// Number of shards in the cluster.

1086
src/stats.rs
View File

File diff suppressed because it is too large Load Diff

5
tests/docker/Dockerfile
View File

@@ -1,5 +0,0 @@
FROM rust:bullseye
RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov sudo curl -y
RUN cargo install cargo-binutils rustfilt
RUN rustup component add llvm-tools-preview

47
tests/docker/docker-compose.yml
View File

@@ -1,47 +0,0 @@
version: "3"
services:
pg1:
image: postgres:14
network_mode: "service:main"
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"]
pg2:
image: postgres:14
network_mode: "service:main"
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"]
pg3:
image: postgres:14
network_mode: "service:main"
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"]
pg4:
image: postgres:14
network_mode: "service:main"
environment:
POSTGRES_USER: postgres
POSTGRES_DB: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: scram-sha-256
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"]
main:
build: .
command: ["bash", "/app/tests/docker/run.sh"]
environment:
RUSTFLAGS: "-C instrument-coverage"
LLVM_PROFILE_FILE: "pgcat-%m.profraw"
volumes:
- ../../:/app/
- /app/target/

21
tests/docker/run.sh
View File

@@ -1,21 +0,0 @@
#!/bin/bash
rm /app/*.profraw || true
rm /app/pgcat.profdata || true
rm -rf /app/cov || true
cd /app/
cargo build
cargo test --tests
bash .circleci/run_tests.sh
rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata
rust-cov export -ignore-filename-regex="rustc|registry" -Xdemangler=rustfilt -instr-profile=pgcat.profdata --object ./target/debug/pgcat --format lcov > ./lcov.info
genhtml lcov.info --output-directory cov --prefix $(pwd)
rm /app/*.profraw
rm /app/pgcat.profdata

150
tests/python/tests.py
View File

@@ -14,18 +14,12 @@ PGCAT_PORT = "6432"
def pgcat_start():
pg_cat_send_signal(signal.SIGTERM)
os.system("./target/debug/pgcat .circleci/pgcat.toml &")
time.sleep(2)
def pg_cat_send_signal(signal: signal.Signals):
try:
for proc in psutil.process_iter(["pid", "name"]):
if "pgcat" == proc.name():
os.kill(proc.pid, signal)
except Exception as e:
# The process can be gone when we send this signal
print(e)
for proc in psutil.process_iter(["pid", "name"]):
if "pgcat" == proc.name():
os.kill(proc.pid, signal)
if signal == signal.SIGTERM:
# Returns 0 if pgcat process exists
time.sleep(2)
@@ -33,23 +27,11 @@ def pg_cat_send_signal(signal: signal.Signals):
raise Exception("pgcat not closed after SIGTERM")
def connect_db(
autocommit: bool = True,
admin: bool = False,
def connect_normal_db(
autocommit: bool = False,
) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]:
if admin:
user = "admin_user"
password = "admin_pass"
db = "pgcat"
else:
user = "sharding_user"
password = "sharding_user"
db = "sharded_db"
conn = psycopg2.connect(
f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat",
connect_timeout=2,
f"postgres://sharding_user:sharding_user@{PGCAT_HOST}:{PGCAT_PORT}/sharded_db?application_name=testing_pgcat"
)
conn.autocommit = autocommit
cur = conn.cursor()
@@ -63,7 +45,7 @@ def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.
def test_normal_db_access():
conn, cur = connect_db(autocommit=False)
conn, cur = connect_normal_db()
cur.execute("SELECT 1")
res = cur.fetchall()
print(res)
@@ -71,7 +53,11 @@ def test_normal_db_access():
def test_admin_db_access():
conn, cur = connect_db(admin=True)
conn = psycopg2.connect(
f"postgres://admin_user:admin_pass@{PGCAT_HOST}:{PGCAT_PORT}/pgcat"
)
conn.autocommit = True # BEGIN/COMMIT is not supported by admin db
cur = conn.cursor()
cur.execute("SHOW POOLS")
res = cur.fetchall()
@@ -81,14 +67,15 @@ def test_admin_db_access():
def test_shutdown_logic():
# - - - - - - - - - - - - - - - - - -
# NO ACTIVE QUERIES SIGINT HANDLING
##### NO ACTIVE QUERIES SIGINT HANDLING #####
# Start pgcat
pgcat_start()
# Wait for server to fully start up
time.sleep(2)
# Create client connection and send query (not in transaction)
conn, cur = connect_db()
conn, cur = connect_normal_db(True)
cur.execute("BEGIN;")
cur.execute("SELECT 1;")
@@ -110,14 +97,17 @@ def test_shutdown_logic():
cleanup_conn(conn, cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
# HANDLE TRANSACTION WITH SIGINT
##### END #####
##### HANDLE TRANSACTION WITH SIGINT #####
# Start pgcat
pgcat_start()
# Wait for server to fully start up
time.sleep(2)
# Create client connection and begin transaction
conn, cur = connect_db()
conn, cur = connect_normal_db(True)
cur.execute("BEGIN;")
cur.execute("SELECT 1;")
@@ -136,97 +126,17 @@ def test_shutdown_logic():
cleanup_conn(conn, cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
# NO NEW NON-ADMIN CONNECTIONS DURING SHUTDOWN
##### END #####
##### HANDLE SHUTDOWN TIMEOUT WITH SIGINT #####
# Start pgcat
pgcat_start()
# Create client connection and begin transaction
transaction_conn, transaction_cur = connect_db()
transaction_cur.execute("BEGIN;")
transaction_cur.execute("SELECT 1;")
# Send sigint to pgcat while still in transaction
pg_cat_send_signal(signal.SIGINT)
time.sleep(1)
start = time.perf_counter()
try:
conn, cur = connect_db()
cur.execute("SELECT 1;")
cleanup_conn(conn, cur)
except psycopg2.OperationalError as e:
time_taken = time.perf_counter() - start
if time_taken > 0.1:
raise Exception(
"Failed to reject connection within 0.1 seconds, got", time_taken, "seconds")
pass
else:
raise Exception("Able connect to database during shutdown")
cleanup_conn(transaction_conn, transaction_cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
# ALLOW NEW ADMIN CONNECTIONS DURING SHUTDOWN
# Start pgcat
pgcat_start()
# Create client connection and begin transaction
transaction_conn, transaction_cur = connect_db()
transaction_cur.execute("BEGIN;")
transaction_cur.execute("SELECT 1;")
# Send sigint to pgcat while still in transaction
pg_cat_send_signal(signal.SIGINT)
time.sleep(1)
try:
conn, cur = connect_db(admin=True)
cur.execute("SHOW DATABASES;")
cleanup_conn(conn, cur)
except psycopg2.OperationalError as e:
raise Exception(e)
cleanup_conn(transaction_conn, transaction_cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
# ADMIN CONNECTIONS CONTINUING TO WORK AFTER SHUTDOWN
# Start pgcat
pgcat_start()
# Create client connection and begin transaction
transaction_conn, transaction_cur = connect_db()
transaction_cur.execute("BEGIN;")
transaction_cur.execute("SELECT 1;")
admin_conn, admin_cur = connect_db(admin=True)
admin_cur.execute("SHOW DATABASES;")
# Send sigint to pgcat while still in transaction
pg_cat_send_signal(signal.SIGINT)
time.sleep(1)
try:
admin_cur.execute("SHOW DATABASES;")
except psycopg2.OperationalError as e:
raise Exception("Could not execute admin command:", e)
cleanup_conn(transaction_conn, transaction_cur)
cleanup_conn(admin_conn, admin_cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
# HANDLE SHUTDOWN TIMEOUT WITH SIGINT
# Start pgcat
pgcat_start()
# Wait for server to fully start up
time.sleep(3)
# Create client connection and begin transaction, which should prevent server shutdown unless shutdown timeout is reached
conn, cur = connect_db()
conn, cur = connect_normal_db(True)
cur.execute("BEGIN;")
cur.execute("SELECT 1;")
@@ -249,7 +159,7 @@ def test_shutdown_logic():
cleanup_conn(conn, cur)
pg_cat_send_signal(signal.SIGTERM)
# - - - - - - - - - - - - - - - - - -
##### END #####
test_normal_db_access()

6
tests/ruby/Gemfile
View File

@@ -1,8 +1,6 @@
source "https://rubygems.org"
gem "pg"
gem "toml"
gem "rspec"
gem "rubocop"
gem "toxiproxy"
gem "activerecord"
gem "rubocop"
gem "toml", "~> 0.3.0"

22
tests/ruby/Gemfile.lock
View File

@@ -13,7 +13,6 @@ GEM
tzinfo (~> 2.0)
ast (2.4.2)
concurrent-ruby (1.1.10)
diff-lcs (1.5.0)
i18n (1.11.0)
concurrent-ruby (~> 1.0)
minitest (5.16.2)
@@ -25,19 +24,6 @@ GEM
rainbow (3.1.1)
regexp_parser (2.3.1)
rexml (3.2.5)
rspec (3.11.0)
rspec-core (~> 3.11.0)
rspec-expectations (~> 3.11.0)
rspec-mocks (~> 3.11.0)
rspec-core (3.11.0)
rspec-support (~> 3.11.0)
rspec-expectations (3.11.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.11.0)
rspec-mocks (3.11.1)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.11.0)
rspec-support (3.11.0)
rubocop (1.29.0)
parallel (~> 1.10)
parser (>= 3.1.0.0)
@@ -52,23 +38,19 @@ GEM
ruby-progressbar (1.11.0)
toml (0.3.0)
parslet (>= 1.8.0, < 3.0.0)
toxiproxy (2.0.1)
tzinfo (2.0.4)
concurrent-ruby (~> 1.0)
unicode-display_width (2.1.0)
PLATFORMS
aarch64-linux
arm64-darwin-21
x86_64-linux
DEPENDENCIES
activerecord
pg
rspec
rubocop
toml
toxiproxy
toml (~> 0.3.0)
BUNDLED WITH
2.3.21
2.3.7

289
tests/ruby/admin_spec.rb
View File

@@ -1,289 +0,0 @@
# frozen_string_literal: true
require 'uri'
require_relative 'spec_helper'
describe "Admin" do
let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) }
let(:pgcat_conn_str) { processes.pgcat.connection_string("sharded_db", "sharding_user") }
after do
processes.all_databases.map(&:reset)
processes.pgcat.shutdown
end
describe "SHOW STATS" do
context "clients connect and make one query" do
it "updates *_query_time and *_wait_time" do
connection = PG::connect("#{pgcat_conn_str}?application_name=one_query")
connection.async_exec("SELECT pg_sleep(0.25)")
connection.async_exec("SELECT pg_sleep(0.25)")
connection.async_exec("SELECT pg_sleep(0.25)")
connection.close
# wait for averages to be calculated, we shouldn't do this too often
sleep(15.5)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW STATS")[0]
admin_conn.close
expect(results["total_query_time"].to_i).to be_within(200).of(750)
expect(results["avg_query_time"].to_i).to_not eq(0)
expect(results["total_wait_time"].to_i).to_not eq(0)
expect(results["avg_wait_time"].to_i).to_not eq(0)
end
end
end
describe "SHOW POOLS" do
context "bad credentials" do
it "does not change any stats" do
bad_passsword_url = URI(pgcat_conn_str)
bad_passsword_url.password = "wrong"
expect { PG::connect("#{bad_passsword_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad)
sleep(1)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["sv_idle"]).to eq("1")
end
end
context "bad database name" do
it "does not change any stats" do
bad_db_url = URI(pgcat_conn_str)
bad_db_url.path = "/wrong_db"
expect { PG::connect("#{bad_db_url.to_s}?application_name=bad_db") }.to raise_error(PG::ConnectionBad)
sleep(1)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["sv_idle"]).to eq("1")
end
end
context "client connects but issues no queries" do
it "only affects cl_idle stats" do
connections = Array.new(20) { PG::connect(pgcat_conn_str) }
sleep(1)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_idle"]).to eq("20")
expect(results["sv_idle"]).to eq("1")
connections.map(&:close)
sleep(1.1)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_active cl_idle cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["sv_idle"]).to eq("1")
end
end
context "clients connect and make one query" do
it "only affects cl_idle, sv_idle stats" do
connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") }
connections.each do |c|
Thread.new { c.async_exec("SELECT pg_sleep(2.5)") }
end
sleep(1.1)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_waiting cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_active"]).to eq("5")
expect(results["sv_active"]).to eq("5")
sleep(3)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_idle"]).to eq("5")
expect(results["sv_idle"]).to eq("5")
connections.map(&:close)
sleep(1)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["sv_idle"]).to eq("5")
end
end
context "client connects and opens a transaction and closes connection uncleanly" do
it "produces correct statistics" do
connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") }
connections.each do |c|
Thread.new do
c.async_exec("BEGIN")
c.async_exec("SELECT pg_sleep(0.01)")
c.close
end
end
sleep(1.1)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["sv_idle"]).to eq("5")
end
end
context "client fail to checkout connection from the pool" do
it "counts clients as idle" do
new_configs = processes.pgcat.current_config
new_configs["general"]["connect_timeout"] = 500
new_configs["general"]["ban_time"] = 1
new_configs["general"]["shutdown_timeout"] = 1
new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1
processes.pgcat.update_config(new_configs)
processes.pgcat.reload_config
threads = []
connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") }
connections.each do |c|
threads << Thread.new { c.async_exec("SELECT pg_sleep(1)") rescue PG::SystemError }
end
sleep(2)
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_idle"]).to eq("5")
expect(results["sv_idle"]).to eq("1")
threads.map(&:join)
connections.map(&:close)
end
end
context "clients overwhelm server pools" do
let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) }
it "cl_waiting is updated to show it" do
threads = []
connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") }
connections.each do |c|
threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") }
end
sleep(1.1) # Allow time for stats to update
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_waiting"]).to eq("2")
expect(results["cl_active"]).to eq("2")
expect(results["sv_active"]).to eq("2")
sleep(2.5) # Allow time for stats to update
results = admin_conn.async_exec("SHOW POOLS")[0]
%w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s|
raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0"
end
expect(results["cl_idle"]).to eq("4")
expect(results["sv_idle"]).to eq("2")
threads.map(&:join)
connections.map(&:close)
end
it "show correct max_wait" do
threads = []
connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") }
connections.each do |c|
threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") }
end
sleep(2.5) # Allow time for stats to update
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
results = admin_conn.async_exec("SHOW POOLS")[0]
expect(results["maxwait"]).to eq("1")
expect(results["maxwait_us"].to_i).to be_within(100_000).of(500_000)
sleep(4.5) # Allow time for stats to update
results = admin_conn.async_exec("SHOW POOLS")[0]
expect(results["maxwait"]).to eq("0")
threads.map(&:join)
connections.map(&:close)
end
end
end
describe "SHOW CLIENTS" do
it "reports correct number and application names" do
conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user")
connections = Array.new(20) { |i| PG::connect("#{conn_str}?application_name=app#{i % 5}") }
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
sleep(1) # Wait for stats to be updated
results = admin_conn.async_exec("SHOW CLIENTS")
expect(results.count).to eq(21) # count admin clients
expect(results.select { |c| c["application_name"] == "app3" || c["application_name"] == "app4" }.count).to eq(8)
expect(results.select { |c| c["database"] == "pgcat" }.count).to eq(1)
connections[0..5].map(&:close)
sleep(1) # Wait for stats to be updated
results = admin_conn.async_exec("SHOW CLIENTS")
expect(results.count).to eq(15)
connections[6..].map(&:close)
sleep(1) # Wait for stats to be updated
expect(admin_conn.async_exec("SHOW CLIENTS").count).to eq(1)
admin_conn.close
end
it "reports correct number of queries and transactions" do
conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user")
connections = Array.new(2) { |i| PG::connect("#{conn_str}?application_name=app#{i}") }
connections.each do |c|
c.async_exec("SELECT 1")
c.async_exec("SELECT 2")
c.async_exec("SELECT 3")
c.async_exec("BEGIN")
c.async_exec("SELECT 4")
c.async_exec("SELECT 5")
c.async_exec("COMMIT")
end
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
sleep(1) # Wait for stats to be updated
results = admin_conn.async_exec("SHOW CLIENTS")
expect(results.count).to eq(3)
normal_client_results = results.reject { |r| r["database"] == "pgcat" }
expect(normal_client_results[0]["transaction_count"]).to eq("4")
expect(normal_client_results[1]["transaction_count"]).to eq("4")
expect(normal_client_results[0]["query_count"]).to eq("7")
expect(normal_client_results[1]["query_count"]).to eq("7")
admin_conn.close
connections.map(&:close)
end
end
end

82
tests/ruby/helpers/pg_instance.rb
View File

@@ -1,82 +0,0 @@
require 'pg'
require 'toxiproxy'
class PgInstance
attr_reader :port
attr_reader :username
attr_reader :password
attr_reader :database_name
def initialize(port, username, password, database_name)
@original_port = port
@toxiproxy_port = 10000 + port.to_i
@port = @toxiproxy_port
@username = username
@password = password
@database_name = database_name
@toxiproxy_name = "database_#{@original_port}"
Toxiproxy.populate([{
name: @toxiproxy_name,
listen: "0.0.0.0:#{@toxiproxy_port}",
upstream: "localhost:#{@original_port}",
}])
# Toxiproxy server will outlive our PgInstance objects
# so we want to destroy our proxies before exiting
# Ruby finalizer is ideal for doing this
ObjectSpace.define_finalizer(@toxiproxy_name, proc { Toxiproxy[@toxiproxy_name].destroy })
end
def with_connection
conn = PG.connect("postgres://#{@username}:#{@password}@localhost:#{port}/#{database_name}")
yield conn
ensure
conn&.close
end
def reset
reset_toxics
reset_stats
end
def toxiproxy
Toxiproxy[@toxiproxy_name]
end
def take_down
if block_given?
Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).apply { yield }
else
Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).toxics.each(&:save)
end
end
def add_latency(latency)
if block_given?
Toxiproxy[@toxiproxy_name].toxic(:latency, latency: latency).apply { yield }
else
Toxiproxy[@toxiproxy_name].toxic(:latency, latency: latency).toxics.each(&:save)
end
end
def delete_proxy
Toxiproxy[@toxiproxy_name].delete
end
def reset_toxics
Toxiproxy[@toxiproxy_name].toxics.each(&:destroy)
end
def reset_stats
with_connection { |c| c.async_exec("SELECT pg_stat_statements_reset()") }
end
def count_query(query)
with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = '#{query}'")[0]["sum"].to_i }
end
def count_select_1_plus_2
with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = 'SELECT $1 + $2'")[0]["sum"].to_i }
end
end

144
tests/ruby/helpers/pgcat_helper.rb
View File

@@ -1,144 +0,0 @@
require 'json'
require 'ostruct'
require_relative 'pgcat_process'
require_relative 'pg_instance'
module Helpers
module Pgcat
def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction")
user = {
"password" => "sharding_user",
"pool_size" => pool_size,
"statement_timeout" => 0,
"username" => "sharding_user"
}
pgcat = PgcatProcess.new("info")
primary0 = PgInstance.new(5432, user["username"], user["password"], "shard0")
primary1 = PgInstance.new(7432, user["username"], user["password"], "shard1")
primary2 = PgInstance.new(8432, user["username"], user["password"], "shard2")
pgcat_cfg = pgcat.current_config
pgcat_cfg["pools"] = {
"#{pool_name}" => {
"default_role" => "any",
"pool_mode" => pool_mode,
"primary_reads_enabled" => false,
"query_parser_enabled" => false,
"sharding_function" => "pg_bigint_hash",
"shards" => {
"0" => { "database" => "shard0", "servers" => [["localhost", primary0.port.to_s, "primary"]] },
"1" => { "database" => "shard1", "servers" => [["localhost", primary1.port.to_s, "primary"]] },
"2" => { "database" => "shard2", "servers" => [["localhost", primary2.port.to_s, "primary"]] },
},
"users" => { "0" => user }
}
}
pgcat.update_config(pgcat_cfg)
pgcat.start
pgcat.wait_until_ready
OpenStruct.new.tap do |struct|
struct.pgcat = pgcat
struct.shards = [primary0, primary1, primary2]
struct.all_databases = [primary0, primary1, primary2]
end
end
def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction")
user = {
"password" => "sharding_user",
"pool_size" => pool_size,
"statement_timeout" => 0,
"username" => "sharding_user"
}
pgcat = PgcatProcess.new("trace")
pgcat_cfg = pgcat.current_config
primary = PgInstance.new(5432, user["username"], user["password"], "shard0")
# Main proxy configs
pgcat_cfg["pools"] = {
"#{pool_name}" => {
"default_role" => "primary",
"pool_mode" => pool_mode,
"primary_reads_enabled" => false,
"query_parser_enabled" => false,
"sharding_function" => "pg_bigint_hash",
"shards" => {
"0" => {
"database" => "shard0",
"servers" => [
["localhost", primary.port.to_s, "primary"]
]
},
},
"users" => { "0" => user }
}
}
pgcat_cfg["general"]["port"] = pgcat.port
pgcat.update_config(pgcat_cfg)
pgcat.start
pgcat.wait_until_ready
OpenStruct.new.tap do |struct|
struct.pgcat = pgcat
struct.primary = primary
struct.all_databases = [primary]
end
end
def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction")
user = {
"password" => "sharding_user",
"pool_size" => pool_size,
"statement_timeout" => 0,
"username" => "sharding_user"
}
pgcat = PgcatProcess.new("info")
pgcat_cfg = pgcat.current_config
primary = PgInstance.new(5432, user["username"], user["password"], "shard0")
replica0 = PgInstance.new(7432, user["username"], user["password"], "shard0")
replica1 = PgInstance.new(8432, user["username"], user["password"], "shard0")
replica2 = PgInstance.new(9432, user["username"], user["password"], "shard0")
# Main proxy configs
pgcat_cfg["pools"] = {
"#{pool_name}" => {
"default_role" => "any",
"pool_mode" => pool_mode,
"primary_reads_enabled" => false,
"query_parser_enabled" => false,
"sharding_function" => "pg_bigint_hash",
"shards" => {
"0" => {
"database" => "shard0",
"servers" => [
["localhost", primary.port.to_s, "primary"],
["localhost", replica0.port.to_s, "replica"],
["localhost", replica1.port.to_s, "replica"],
["localhost", replica2.port.to_s, "replica"]
]
},
},
"users" => { "0" => user }
}
}
pgcat_cfg["general"]["port"] = pgcat.port
pgcat.update_config(pgcat_cfg)
pgcat.start
pgcat.wait_until_ready
OpenStruct.new.tap do |struct|
struct.pgcat = pgcat
struct.primary = primary
struct.replicas = [replica0, replica1, replica2]
struct.all_databases = [primary, replica0, replica1, replica2]
end
end
end
end

116
tests/ruby/helpers/pgcat_process.rb
View File

@@ -1,116 +0,0 @@
require 'pg'
require 'toml'
require 'fileutils'
require 'securerandom'
class PgcatProcess
attr_reader :port
attr_reader :pid
def self.finalize(pid, log_filename, config_filename)
`kill #{pid}`
File.delete(config_filename) if File.exist?(config_filename)
File.delete(log_filename) if File.exist?(log_filename)
end
def initialize(log_level)
@env = {"RUST_LOG" => log_level}
@port = rand(20000..32760)
@log_level = log_level
@log_filename = "/tmp/pgcat_log_#{SecureRandom.urlsafe_base64}.log"
@config_filename = "/tmp/pgcat_cfg_#{SecureRandom.urlsafe_base64}.toml"
@command = "../../target/debug/pgcat #{@config_filename}"
FileUtils.cp("../../pgcat.toml", @config_filename)
cfg = current_config
cfg["general"]["port"] = @port.to_i
cfg["general"]["enable_prometheus_exporter"] = false
update_config(cfg)
end
def logs
File.read(@log_filename)
end
def update_config(config_hash)
@original_config = current_config
output_to_write = TOML::Generator.new(config_hash).body
output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*,/, ',\1,')
File.write(@config_filename, output_to_write)
end
def current_config
old_cfg = File.read(@config_filename)
loadable_string = old_cfg.gsub(/,\s*(\d+)\s*,/, ', "\1",')
TOML.load(loadable_string)
end
def reload_config
`kill -s HUP #{@pid}`
sleep 0.1
end
def start
raise StandardError, "Process is already started" unless @pid.nil?
@pid = Process.spawn(@env, @command, err: @log_filename, out: @log_filename)
ObjectSpace.define_finalizer(@log_filename, proc { PgcatProcess.finalize(@pid, @log_filename, @config_filename) })
return self
end
def wait_until_ready
exc = nil
10.times do
PG::connect(example_connection_string).close
return self
rescue => e
exc = e
sleep(0.5)
end
puts exc
raise StandardError, "Process #{@pid} never became ready. Logs #{logs}"
end
def stop
`kill #{@pid}`
sleep 0.1
end
def shutdown
stop
File.delete(@config_filename) if File.exist?(@config_filename)
File.delete(@log_filename) if File.exist?(@log_filename)
end
def admin_connection_string
cfg = current_config
username = cfg["general"]["admin_username"]
password = cfg["general"]["admin_password"]
"postgresql://#{username}:#{password}@0.0.0.0:#{@port}/pgcat"
end
def connection_string(pool_name, username)
cfg = current_config
user_idx, user_obj = cfg["pools"][pool_name]["users"].detect { |k, user| user["username"] == username }
password = user_obj["password"]
"postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{pool_name}"
end
def example_connection_string
cfg = current_config
first_pool_name = cfg["pools"].keys[0]
db_name = first_pool_name
username = cfg["pools"][first_pool_name]["users"]["0"]["username"]
password = cfg["pools"][first_pool_name]["users"]["0"]["password"]
"postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{db_name}?application_name=example_app"
end
end

61
tests/ruby/load_balancing_spec.rb
View File

@@ -1,61 +0,0 @@
# frozen_string_literal: true
require_relative 'spec_helper'
describe "Load Balancing" do
let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) }
after do
processes.all_databases.map(&:reset)
processes.pgcat.shutdown
end
context "under regular circumstances" do
it "balances query volume between all instances" do
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
query_count = QUERY_COUNT
expected_share = query_count / processes.all_databases.count
failed_count = 0
query_count.times do
conn.async_exec("SELECT 1 + 2")
rescue
failed_count += 1
end
expect(failed_count).to eq(0)
processes.all_databases.map(&:count_select_1_plus_2).each do |instance_share|
expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share)
end
end
end
context "when some replicas are down" do
it "balances query volume between working instances" do
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
expected_share = QUERY_COUNT / (processes.all_databases.count - 2)
failed_count = 0
processes[:replicas][0].take_down do
processes[:replicas][1].take_down do
QUERY_COUNT.times do
conn.async_exec("SELECT 1 + 2")
rescue
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
failed_count += 1
end
end
end
expect(failed_count).to eq(2)
processes.all_databases.each do |instance|
queries_routed = instance.count_select_1_plus_2
if processes.replicas[0..1].include?(instance)
expect(queries_routed).to eq(0)
else
expect(queries_routed).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share)
end
end
end
end
end

193
tests/ruby/misc_spec.rb
View File

@@ -1,193 +0,0 @@
# frozen_string_literal: true
require_relative 'spec_helper'
describe "Miscellaneous" do
let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) }
after do
processes.all_databases.map(&:reset)
processes.pgcat.shutdown
end
describe "Extended Protocol handling" do
it "does not send packets that client does not expect during extended protocol sequence" do
new_configs = processes.pgcat.current_config
new_configs["general"]["connect_timeout"] = 500
new_configs["general"]["ban_time"] = 1
new_configs["general"]["shutdown_timeout"] = 1
new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1
processes.pgcat.update_config(new_configs)
processes.pgcat.reload_config
25.times do
Thread.new do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SELECT pg_sleep(5)") rescue PG::SystemError
ensure
conn&.close
end
end
sleep(0.5)
conn_under_test = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
stdout, stderr = with_captured_stdout_stderr do
15.times do |i|
conn_under_test.async_exec("SELECT 1") rescue PG::SystemError
conn_under_test.exec_params("SELECT #{i} + $1", [i]) rescue PG::SystemError
sleep 1
end
end
raise StandardError, "Libpq got unexpected messages while idle" if stderr.include?("arrived from server while idle")
end
end
describe "Pool recycling after config reload" do
let(:processes) { Helpers::Pgcat.three_shard_setup("sharded_db", 5) }
it "should update pools for new clients and clients that are no longer in transaction" do
server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
server_conn.async_exec("BEGIN")
# No config change yet, client should set old configs
current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"]
expect(current_datebase_from_pg).to eq('shard0')
# Swap shards
new_config = processes.pgcat.current_config
shard0 = new_config["pools"]["sharded_db"]["shards"]["0"]
shard1 = new_config["pools"]["sharded_db"]["shards"]["1"]
new_config["pools"]["sharded_db"]["shards"]["0"] = shard1
new_config["pools"]["sharded_db"]["shards"]["1"] = shard0
# Reload config
processes.pgcat.update_config(new_config)
processes.pgcat.reload_config
sleep 0.5
# Config changed but transaction is in progress, client should set old configs
current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"]
expect(current_datebase_from_pg).to eq('shard0')
server_conn.async_exec("COMMIT")
# Transaction finished, client should get new configs
current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"]
expect(current_datebase_from_pg).to eq('shard1')
# New connection should get new configs
server_conn.close()
server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"]
expect(current_datebase_from_pg).to eq('shard1')
end
end
describe "Clients closing connection in the middle of transaction" do
it "sends a rollback to the server" do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
conn.async_exec("BEGIN")
conn.close
expect(processes.primary.count_query("ROLLBACK")).to eq(1)
end
end
describe "Server version reporting" do
it "reports correct version for normal and admin databases" do
server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
expect(server_conn.server_version).not_to eq(0)
server_conn.close
admin_conn = PG::connect(processes.pgcat.admin_connection_string)
expect(admin_conn.server_version).not_to eq(0)
admin_conn.close
end
end
describe "State clearance" do
context "session mode" do
let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "session") }
it "Clears state before connection checkin" do
# Both modes of operation should not raise
# ERROR: prepared statement "prepared_q" already exists
15.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("PREPARE prepared_q (int) AS SELECT $1")
conn.close
end
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
initial_value = conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]
conn.async_exec("SET statement_timeout to 1000")
current_value = conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]
expect(conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]).to eq("1s")
conn.close
end
it "Does not send DISCARD ALL unless necessary" do
10.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
conn.async_exec("SELECT 1")
conn.close
end
expect(processes.primary.count_query("DISCARD ALL")).to eq(0)
10.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
conn.async_exec("SELECT 1")
conn.async_exec("SET statement_timeout to 5000")
conn.close
end
expect(processes.primary.count_query("DISCARD ALL")).to eq(10)
end
end
context "transaction mode" do
let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction") }
it "Clears state before connection checkin" do
# Both modes of operation should not raise
# ERROR: prepared statement "prepared_q" already exists
15.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("PREPARE prepared_q (int) AS SELECT $1")
conn.close
end
15.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.prepare("prepared_q", "SELECT $1")
conn.close
end
end
it "Does not send DISCARD ALL unless necessary" do
10.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
conn.async_exec("SELECT 1")
conn.exec_params("SELECT $1", [1])
conn.close
end
expect(processes.primary.count_query("DISCARD ALL")).to eq(0)
10.times do
conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
conn.async_exec("SELECT 1")
conn.async_exec("SET statement_timeout to 5000")
conn.close
end
expect(processes.primary.count_query("DISCARD ALL")).to eq(10)
end
end
end
end

81
tests/ruby/routing_spec.rb
View File

@@ -1,81 +0,0 @@
# frozen_string_literal: true
require_relative 'spec_helper'
describe "Routing" do
let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) }
after do
processes.all_databases.map(&:reset)
processes.pgcat.shutdown
end
describe "SET ROLE" do
context "primary" do
it "routes queries only to primary" do
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'primary'")
query_count = 30
failed_count = 0
query_count.times do
conn.async_exec("SELECT 1 + 2")
rescue
failed_count += 1
end
expect(failed_count).to eq(0)
processes.replicas.map(&:count_select_1_plus_2).each do |instance_share|
expect(instance_share).to eq(0)
end
expect(processes.primary.count_select_1_plus_2).to eq(query_count)
end
end
context "replica" do
it "routes queries only to replicas" do
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'replica'")
expected_share = QUERY_COUNT / processes.replicas.count
failed_count = 0
QUERY_COUNT.times do
conn.async_exec("SELECT 1 + 2")
rescue
failed_count += 1
end
expect(failed_count).to eq(0)
processes.replicas.map(&:count_select_1_plus_2).each do |instance_share|
expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share)
end
expect(processes.primary.count_select_1_plus_2).to eq(0)
end
end
context "any" do
it "routes queries to all instances" do
conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user"))
conn.async_exec("SET SERVER ROLE to 'any'")
expected_share = QUERY_COUNT / processes.all_databases.count
failed_count = 0
QUERY_COUNT.times do
conn.async_exec("SELECT 1 + 2")
rescue
failed_count += 1
end
expect(failed_count).to eq(0)
processes.all_databases.map(&:count_select_1_plus_2).each do |instance_share|
expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share)
end
end
end
end
end

21
tests/ruby/spec_helper.rb
View File

@@ -1,21 +0,0 @@
# frozen_string_literal: true
require 'pg'
require_relative 'helpers/pgcat_helper'
QUERY_COUNT = 300
MARGIN_OF_ERROR = 0.30
def with_captured_stdout_stderr
sout = STDOUT.clone
serr = STDERR.clone
STDOUT.reopen("/tmp/out.txt", "w+")
STDERR.reopen("/tmp/err.txt", "w+")
STDOUT.sync = true
STDERR.sync = true
yield
return File.read('/tmp/out.txt'), File.read('/tmp/err.txt')
ensure
STDOUT.reopen(sout)
STDERR.reopen(serr)
end

92
tests/ruby/tests.rb
View File

@@ -1,6 +1,10 @@
# frozen_string_literal: true
require 'pg'
require 'active_record'
require 'pg'
require 'toml'
$stdout.sync = true
# Uncomment these two to see all queries.
# ActiveRecord.verbose_query_logs = true
@@ -111,3 +115,89 @@ begin
rescue ActiveRecord::StatementInvalid
puts 'OK'
end
# Test evil clients
def poorly_behaved_client
conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat")
conn.async_exec 'BEGIN'
conn.async_exec 'SELECT 1'
conn.close
puts 'Bad client ok'
end
25.times do
poorly_behaved_client
end
def test_server_parameters
server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat")
raise StandardError, "Bad server version" if server_conn.server_version == 0
server_conn.close
admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat")
raise StandardError, "Bad server version" if admin_conn.server_version == 0
admin_conn.close
puts 'Server parameters ok'
end
class ConfigEditor
def initialize
@original_config_text = File.read('../../.circleci/pgcat.toml')
text_to_load = @original_config_text.gsub("5432", "\"5432\"")
@original_configs = TOML.load(text_to_load)
end
def original_configs
TOML.load(TOML::Generator.new(@original_configs).body)
end
def with_modified_configs(new_configs)
text_to_write = TOML::Generator.new(new_configs).body
text_to_write = text_to_write.gsub("\"5432\"", "5432")
File.write('../../.circleci/pgcat.toml', text_to_write)
yield
ensure
File.write('../../.circleci/pgcat.toml', @original_config_text)
end
end
def test_reload_pool_recycling
admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat")
server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat")
server_conn.async_exec("BEGIN")
conf_editor = ConfigEditor.new
new_configs = conf_editor.original_configs
# swap shards
new_configs["pools"]["sharded_db"]["shards"]["0"]["database"] = "shard1"
new_configs["pools"]["sharded_db"]["shards"]["1"]["database"] = "shard0"
raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0'
conf_editor.with_modified_configs(new_configs) { admin_conn.async_exec("RELOAD") }
raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0'
server_conn.async_exec("COMMIT;")
# Transaction finished, client should get new configs
raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1'
server_conn.close()
# New connection should get new configs
server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat")
raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1'
ensure
admin_conn.async_exec("RELOAD") # Go back to old state
admin_conn.close
server_conn.close
puts "Pool Recycling okay!"
end
test_reload_pool_recycling

12
tests/sharding/query_routing_setup.sql
View File

@@ -70,35 +70,23 @@ GRANT CONNECT ON DATABASE shard2 TO other_user;
GRANT CONNECT ON DATABASE some_db TO simple_user;
\c shard0
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user;
GRANT ALL ON SCHEMA public TO sharding_user;
GRANT ALL ON TABLE data TO sharding_user;
GRANT ALL ON SCHEMA public TO other_user;
GRANT ALL ON TABLE data TO other_user;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user;
\c shard1
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user;
GRANT ALL ON SCHEMA public TO sharding_user;
GRANT ALL ON TABLE data TO sharding_user;
GRANT ALL ON SCHEMA public TO other_user;
GRANT ALL ON TABLE data TO other_user;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user;
\c shard2
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user;
GRANT ALL ON SCHEMA public TO sharding_user;
GRANT ALL ON TABLE data TO sharding_user;
GRANT ALL ON SCHEMA public TO other_user;
GRANT ALL ON TABLE data TO other_user;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user;
\c some_db
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO simple_user;
GRANT ALL ON SCHEMA public TO simple_user;
GRANT ALL ON TABLE data TO simple_user;