mirror of
https://github.com/postgresml/pgcat.git
synced 2026-03-23 09:26:30 +00:00
Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
9fe50c48e8 | ||
|
|
3cc41ab6c8 | ||
|
|
bb1474f175 | ||
|
|
64b6cde2a4 | ||
|
|
bde4f214cc | ||
|
|
8f2de5bdfc | ||
|
|
abe3291780 | ||
|
|
75537736e9 | ||
|
|
dfc05c3dca | ||
|
|
edf6a69ca4 | ||
|
|
95c2d593cc | ||
|
|
9657256adf | ||
|
|
eb682db52e | ||
|
|
75dd88a099 | ||
|
|
a80f118a8d | ||
|
|
f807e4b425 | ||
|
|
3e66aba62b | ||
|
|
1f4a2e53e2 | ||
|
|
af6f770271 | ||
|
|
74dadcef30 | ||
|
|
5f9f2c21fc | ||
|
|
39a9dff26b | ||
|
|
27c05cfd53 | ||
|
|
2557c4408e | ||
|
|
b2c7d6bdab | ||
|
|
c27a7d30dc | ||
|
|
ef2aab3c61 | ||
|
|
290015957b | ||
|
|
f8d6bd8d89 | ||
|
|
99d3e0b803 | ||
|
|
be79f3446b | ||
|
|
b943ff3fa6 | ||
|
|
e0ca175129 | ||
|
|
9ac5614d50 | ||
|
|
7133d049c6 | ||
|
|
0e29d5e0ed | ||
|
|
20ad693e10 | ||
|
|
a6d3545082 | ||
|
|
7d895b9e37 | ||
|
|
e13795d762 | ||
|
|
f9bfae365f | ||
|
|
5931b6142e | ||
|
|
2407f9acc8 | ||
|
|
ee8627ba9f | ||
|
|
20a0bd272d | ||
|
|
9010819ac4 | ||
|
|
84c24ac0b9 | ||
|
|
de800b8a10 | ||
|
|
f41874c249 | ||
|
|
8479c74354 | ||
|
|
a6574acbc3 | ||
|
|
17b690f759 | ||
|
|
52c4cac175 | ||
|
|
fa46e15318 | ||
|
|
fbc3777769 | ||
|
|
8e88c47f76 | ||
|
|
381f06d46f | ||
|
|
77507e7343 | ||
|
|
05250d9374 | ||
|
|
6e83556867 | ||
|
|
83daaf92d1 | ||
|
|
baf2852f03 | ||
|
|
4146245f15 | ||
|
|
00823287da | ||
|
|
2114cb2a97 | ||
|
|
89043ef12f | ||
|
|
6deb7b1162 | ||
|
|
abb71b6c4d | ||
|
|
042aed9dc6 | ||
|
|
1bdd546fbb | ||
|
|
e3ec5036d7 | ||
|
|
f921966e2b |
37
.circleci/config.yml
Normal file
37
.circleci/config.yml
Normal file
@@ -0,0 +1,37 @@
|
||||
# Use the latest 2.1 version of CircleCI pipeline process engine.
|
||||
# See: https://circleci.com/docs/2.0/configuration-reference
|
||||
version: 2.1
|
||||
|
||||
# Define a job to be invoked later in a workflow.
|
||||
# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
|
||||
jobs:
|
||||
build:
|
||||
# Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
|
||||
# See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
|
||||
docker:
|
||||
- image: cimg/rust:1.58.1
|
||||
# Add steps to the job
|
||||
# See: https://circleci.com/docs/2.0/configuration-reference/#steps
|
||||
steps:
|
||||
- checkout
|
||||
- restore_cache:
|
||||
key: cargo-lock-2-{{ checksum "Cargo.lock" }}
|
||||
- run:
|
||||
name: "Build"
|
||||
command: "cargo build"
|
||||
- run:
|
||||
name: "Test"
|
||||
command: "cargo test"
|
||||
- save_cache:
|
||||
key: cargo-lock-2-{{ checksum "Cargo.lock" }}
|
||||
paths:
|
||||
- target
|
||||
- ~/.cargo
|
||||
|
||||
|
||||
# Invoke jobs via workflows
|
||||
# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
|
||||
workflows:
|
||||
build:
|
||||
jobs:
|
||||
- build
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1 +1,2 @@
|
||||
/target
|
||||
*.deb
|
||||
|
||||
6
CONTRIBUTING.md
Normal file
6
CONTRIBUTING.md
Normal file
@@ -0,0 +1,6 @@
|
||||
Thank you for contributing! Just a few tips here:
|
||||
|
||||
1. `cargo fmt` your code before opening up a PR
|
||||
2. Run the "test suite" (i.e. PgBench) to make sure everything still works.
|
||||
|
||||
Happy hacking!
|
||||
274
Cargo.lock
generated
274
Cargo.lock
generated
@@ -2,6 +2,45 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
|
||||
[[package]]
|
||||
name = "bb8"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e9f4fa9768efd269499d8fba693260cfc670891cf6de3adc935588447a77cc8"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"parking_lot",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -29,6 +68,28 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"time",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.1"
|
||||
@@ -49,6 +110,41 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.5"
|
||||
@@ -59,6 +155,17 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
@@ -147,6 +254,25 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.1"
|
||||
@@ -188,12 +314,42 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pgcat"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bb8",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"md-5",
|
||||
"rand",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"sha-1",
|
||||
"tokio",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.36"
|
||||
@@ -213,12 +369,43 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rabbit"
|
||||
version = "0.1.0"
|
||||
name = "rand"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"md-5",
|
||||
"tokio",
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -230,12 +417,57 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha-1"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
@@ -245,6 +477,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.8.0"
|
||||
@@ -262,6 +500,17 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.16.1"
|
||||
@@ -292,6 +541,15 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.15.0"
|
||||
@@ -310,6 +568,12 @@ version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
||||
13
Cargo.toml
13
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[package]
|
||||
name = "rabbit"
|
||||
name = "pgcat"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
@@ -8,4 +8,13 @@ edition = "2021"
|
||||
[dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
bytes = "1"
|
||||
md-5 = "*"
|
||||
md-5 = "0.10"
|
||||
bb8 = "0.7"
|
||||
async-trait = "0.1"
|
||||
rand = "0.8"
|
||||
chrono = "0.4"
|
||||
sha-1 = "0.10"
|
||||
toml = "0.5"
|
||||
serde = "1"
|
||||
serde_derive = "1"
|
||||
regex = "1"
|
||||
|
||||
674
LICENSE
Normal file
674
LICENSE
Normal file
@@ -0,0 +1,674 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short
|
||||
notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, your program's commands
|
||||
might be different; for a GUI interface, you would use an "about box".
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU GPL, see
|
||||
<http://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program
|
||||
into proprietary programs. If your program is a subroutine library, you
|
||||
may consider it more useful to permit linking proprietary applications with
|
||||
the library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License. But first, please read
|
||||
<http://www.gnu.org/philosophy/why-not-lgpl.html>.
|
||||
208
README.md
Normal file
208
README.md
Normal file
@@ -0,0 +1,208 @@
|
||||
# PgCat
|
||||
|
||||
[](https://circleci.com/gh/levkk/pgcat/tree/main)
|
||||
|
||||
|
||||
|
||||
Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support.
|
||||
|
||||
**Alpha**: don't use in production just yet.
|
||||
|
||||
## Local development
|
||||
|
||||
1. Install Rust (latest stable will work great).
|
||||
2. `cargo run --release` (to get better benchmarks).
|
||||
3. Change the config in `pgcat.toml` to fit your setup (optional given next step).
|
||||
4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql`
|
||||
|
||||
### Tests
|
||||
|
||||
You can just PgBench to test your changes:
|
||||
|
||||
```
|
||||
pgbench -i -h 127.0.0.1 -p 6432 && \
|
||||
pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && \
|
||||
pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended
|
||||
```
|
||||
|
||||
See [sharding README](./tests/sharding/README.md) for sharding logic testing.
|
||||
|
||||
## Features
|
||||
|
||||
1. Session mode.
|
||||
2. Transaction mode.
|
||||
3. `COPY` protocol support.
|
||||
4. Query cancellation.
|
||||
5. Round-robin load balancing of replicas.
|
||||
6. Banlist & failover
|
||||
7. Sharding!
|
||||
|
||||
### Session mode
|
||||
Each client owns its own server for the duration of the session. Commands like `SET` are allowed.
|
||||
This is identical to PgBouncer session mode.
|
||||
|
||||
### Transaction mode
|
||||
The connection is attached to the server for the duration of the transaction. `SET` will pollute the connection,
|
||||
but `SET LOCAL` works great. Identical to PgBouncer transaction mode.
|
||||
|
||||
### COPY protocol
|
||||
That one isn't particularly special, but good to mention that you can `COPY` data in and from the server
|
||||
using this pooler.
|
||||
|
||||
### Query cancellation
|
||||
Okay, this is just basic stuff, but we support cancelling queries. If you know the Postgres protocol,
|
||||
this might be relevant given than this is a transactional pooler but if you're new to Pg, don't worry about it, it works.
|
||||
|
||||
### Round-robin load balancing
|
||||
This is the novel part. PgBouncer doesn't support it and suggests we use DNS or a TCP proxy instead.
|
||||
We prefer to have everything as part of one package; arguably, it's easier to understand and optimize.
|
||||
This pooler will round-robin between multiple replicas keeping load reasonably even.
|
||||
|
||||
### Banlist & failover
|
||||
This is where it gets even more interesting. If we fail to connect to one of the replicas or it fails a health check,
|
||||
we add it to a ban list. No more new transactions will be served by that replica for, in our case, 60 seconds. This
|
||||
gives it the opportunity to recover while clients are happily served by the remaining replicas.
|
||||
|
||||
This decreases error rates substantially! Worth noting here that on busy systems, if the replicas are running too hot,
|
||||
failing over could bring even more load and tip over the remaining healthy-ish replicas. In this case, a decision should be made:
|
||||
either lose 1/x of your traffic or risk losing it all eventually. Ideally you overprovision your system, so you don't necessarily need
|
||||
to make this choice :-).
|
||||
|
||||
### Sharding
|
||||
We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` fields. This works well for tables that use `BIGSERIAL` primary key which I think is common enough these days. We can also add many more functions here, but this is a good start. See `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql` for more details on the implementation.
|
||||
|
||||
The biggest advantage of using this sharding function is that anyone can shard the dataset using Postgres partitions
|
||||
while also access it for both reads and writes using this pooler. No custom obscure sharding function is needed and database sharding can be done entirely in Postgres.
|
||||
|
||||
To select the shard we want to talk to, we introduced special syntax:
|
||||
|
||||
```sql
|
||||
SET SHARDING KEY TO '1234';
|
||||
```
|
||||
|
||||
This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key will be used until it's set again or the client disconnects.
|
||||
|
||||
|
||||
## Missing
|
||||
|
||||
1. Authentication, ehem, this proxy is letting anyone in at the moment.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
You can setup PgBench locally through PgCat:
|
||||
|
||||
```
|
||||
pgbench -h 127.0.0.1 -p 6432 -i
|
||||
```
|
||||
|
||||
Coincidenly, this uses `COPY` so you can test if that works.
|
||||
|
||||
### PgBouncer
|
||||
|
||||
```
|
||||
$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended
|
||||
dropping old tables...
|
||||
creating tables...
|
||||
generating data...
|
||||
100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s)
|
||||
vacuuming...
|
||||
creating primary keys...
|
||||
done.
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: simple
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 1.089 ms
|
||||
tps = 918.687098 (including connections establishing)
|
||||
tps = 918.847790 (excluding connections establishing)
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: extended
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 1.136 ms
|
||||
tps = 880.622009 (including connections establishing)
|
||||
tps = 880.769550 (excluding connections establishing)
|
||||
```
|
||||
|
||||
### PgCat
|
||||
|
||||
|
||||
```
|
||||
$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended
|
||||
dropping old tables...
|
||||
creating tables...
|
||||
generating data...
|
||||
100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s)
|
||||
vacuuming...
|
||||
creating primary keys...
|
||||
done.
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: simple
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 1.142 ms
|
||||
tps = 875.645437 (including connections establishing)
|
||||
tps = 875.799995 (excluding connections establishing)
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: extended
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 1.181 ms
|
||||
tps = 846.539176 (including connections establishing)
|
||||
tps = 846.713636 (excluding connections establishing)
|
||||
```
|
||||
|
||||
### Direct Postgres
|
||||
|
||||
```
|
||||
$ pgbench -i -h 127.0.0.1 -p 5432 && pgbench -t 1000 -p 5432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p
|
||||
5432 -h 127.0.0.1 --protocol extended
|
||||
Password:
|
||||
dropping old tables...
|
||||
creating tables...
|
||||
generating data...
|
||||
100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s)
|
||||
vacuuming...
|
||||
creating primary keys...
|
||||
done.
|
||||
Password:
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: simple
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 0.902 ms
|
||||
tps = 1109.014867 (including connections establishing)
|
||||
tps = 1112.318595 (excluding connections establishing)
|
||||
Password:
|
||||
starting vacuum...end.
|
||||
transaction type: <builtin: TPC-B (sort of)>
|
||||
scaling factor: 1
|
||||
query mode: extended
|
||||
number of clients: 1
|
||||
number of threads: 1
|
||||
number of transactions per client: 1000
|
||||
number of transactions actually processed: 1000/1000
|
||||
latency average = 0.931 ms
|
||||
tps = 1074.017747 (including connections establishing)
|
||||
tps = 1077.121752 (excluding connections establishing)
|
||||
```
|
||||
68
pgcat.toml
Normal file
68
pgcat.toml
Normal file
@@ -0,0 +1,68 @@
|
||||
#
|
||||
# PgCat config example.
|
||||
#
|
||||
|
||||
#
|
||||
# General pooler settings
|
||||
[general]
|
||||
|
||||
# What IP to run on, 0.0.0.0 means accessible from everywhere.
|
||||
host = "0.0.0.0"
|
||||
|
||||
# Port to run on, same as PgBouncer used in this example.
|
||||
port = 6432
|
||||
|
||||
# How many connections to allocate per server.
|
||||
pool_size = 15
|
||||
|
||||
# Pool mode (see PgBouncer docs for more).
|
||||
# session: one server connection per connected client
|
||||
# transaction: one server connection per client transaction
|
||||
pool_mode = "transaction"
|
||||
|
||||
# How long to wait before aborting a server connection (ms).
|
||||
connect_timeout = 5000
|
||||
|
||||
# How much time to give `SELECT 1` health check query to return with a result (ms).
|
||||
healthcheck_timeout = 1000
|
||||
|
||||
# For how long to ban a server if it fails a health check (seconds).
|
||||
ban_time = 60 # Seconds
|
||||
|
||||
#
|
||||
# User to use for authentication against the server.
|
||||
[user]
|
||||
name = "sharding_user"
|
||||
password = "sharding_user"
|
||||
|
||||
|
||||
#
|
||||
# Shards in the cluster
|
||||
[shards]
|
||||
|
||||
# Shard 0
|
||||
[shards.0]
|
||||
|
||||
# [ host, port ]
|
||||
servers = [
|
||||
[ "127.0.0.1", 5432 ],
|
||||
[ "localhost", 5432 ],
|
||||
]
|
||||
# Database name (e.g. "postgres")
|
||||
database = "shard0"
|
||||
|
||||
[shards.1]
|
||||
# [ host, port ]
|
||||
servers = [
|
||||
[ "127.0.0.1", 5432 ],
|
||||
[ "localhost", 5432 ],
|
||||
]
|
||||
database = "shard1"
|
||||
|
||||
[shards.2]
|
||||
# [ host, port ]
|
||||
servers = [
|
||||
[ "127.0.0.1", 5432 ],
|
||||
[ "localhost", 5432 ],
|
||||
]
|
||||
database = "shard2"
|
||||
BIN
pgcat3.png
Normal file
BIN
pgcat3.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
334
src/client.rs
334
src/client.rs
@@ -1,22 +1,63 @@
|
||||
/// Implementation of the PostgreSQL client.
|
||||
/// We are pretending to the server in this scenario,
|
||||
/// and this module implements that.
|
||||
use bytes::{Buf, BufMut, BytesMut};
|
||||
use regex::Regex;
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
|
||||
/// PostgreSQL client (frontend).
|
||||
/// We are pretending to be the backend.
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
use bytes::{Buf, BufMut, BytesMut};
|
||||
|
||||
use crate::errors::Error;
|
||||
use crate::messages::*;
|
||||
use crate::pool::{ClientServerMap, ConnectionPool};
|
||||
use crate::server::Server;
|
||||
use crate::sharding::Sharder;
|
||||
|
||||
const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';";
|
||||
|
||||
/// The client state. One of these is created per client.
|
||||
pub struct Client {
|
||||
// The reads are buffered (8K by default).
|
||||
read: BufReader<OwnedReadHalf>,
|
||||
|
||||
// We buffer the writes ourselves because we know the protocol
|
||||
// better than a stock buffer.
|
||||
write: OwnedWriteHalf,
|
||||
|
||||
// Internal buffer, where we place messages until we have to flush
|
||||
// them to the backend.
|
||||
buffer: BytesMut,
|
||||
|
||||
// The client was started with the sole reason to cancel another running query.
|
||||
cancel_mode: bool,
|
||||
|
||||
// In transaction mode, the connection is released after each transaction.
|
||||
// Session mode has slightly higher throughput per client, but lower capacity.
|
||||
transaction_mode: bool,
|
||||
|
||||
// For query cancellation, the client is given a random process ID and secret on startup.
|
||||
process_id: i32,
|
||||
secret_key: i32,
|
||||
|
||||
// Clients are mapped to servers while they use them. This allows a client
|
||||
// to connect and cancel a query.
|
||||
client_server_map: ClientServerMap,
|
||||
|
||||
// sharding regex
|
||||
sharding_regex: Regex,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
pub async fn startup(mut stream: TcpStream) -> Result<Client, Error> {
|
||||
/// Given a TCP socket, trick the client into thinking we are
|
||||
/// the Postgres server. Perform the authentication and place
|
||||
/// the client in query-ready mode.
|
||||
pub async fn startup(
|
||||
mut stream: TcpStream,
|
||||
client_server_map: ClientServerMap,
|
||||
transaction_mode: bool,
|
||||
) -> Result<Client, Error> {
|
||||
let sharding_regex = Regex::new(SHARDING_REGEX).unwrap();
|
||||
|
||||
loop {
|
||||
// Could be StartupMessage or SSLRequest
|
||||
// which makes this variable length.
|
||||
@@ -49,14 +90,50 @@ impl Client {
|
||||
196608 => {
|
||||
// TODO: perform actual auth.
|
||||
// TODO: record startup parameters client sends over.
|
||||
|
||||
// Generate random backend ID and secret key
|
||||
let process_id: i32 = rand::random();
|
||||
let secret_key: i32 = rand::random();
|
||||
|
||||
auth_ok(&mut stream).await?;
|
||||
server_parameters(&mut stream).await?;
|
||||
backend_key_data(&mut stream, process_id, secret_key).await?;
|
||||
ready_for_query(&mut stream).await?;
|
||||
|
||||
// Split the read and write streams
|
||||
// so we can control buffering.
|
||||
let (read, write) = stream.into_split();
|
||||
|
||||
return Ok(Client {
|
||||
read: BufReader::new(read),
|
||||
write: write,
|
||||
buffer: BytesMut::with_capacity(8196),
|
||||
cancel_mode: false,
|
||||
transaction_mode: transaction_mode,
|
||||
process_id: process_id,
|
||||
secret_key: secret_key,
|
||||
client_server_map: client_server_map,
|
||||
sharding_regex: sharding_regex,
|
||||
});
|
||||
}
|
||||
|
||||
// Query cancel request.
|
||||
80877102 => {
|
||||
let (read, write) = stream.into_split();
|
||||
|
||||
let process_id = bytes.get_i32();
|
||||
let secret_key = bytes.get_i32();
|
||||
|
||||
return Ok(Client {
|
||||
read: BufReader::new(read),
|
||||
write: write,
|
||||
buffer: BytesMut::with_capacity(8196),
|
||||
cancel_mode: true,
|
||||
transaction_mode: transaction_mode,
|
||||
process_id: process_id,
|
||||
secret_key: secret_key,
|
||||
client_server_map: client_server_map,
|
||||
sharding_regex: sharding_regex,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -67,29 +144,246 @@ impl Client {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&mut self, mut server: Server) -> Result<(), Error> {
|
||||
/// Client loop. We handle all messages between the client and the database here.
|
||||
pub async fn handle(&mut self, pool: ConnectionPool) -> Result<(), Error> {
|
||||
// Special: cancelling existing running query
|
||||
if self.cancel_mode {
|
||||
let (process_id, secret_key, address, port) = {
|
||||
let guard = self.client_server_map.lock().unwrap();
|
||||
match guard.get(&(self.process_id, self.secret_key)) {
|
||||
// Drop the mutex as soon as possible.
|
||||
Some((process_id, secret_key, address, port)) => (
|
||||
process_id.clone(),
|
||||
secret_key.clone(),
|
||||
address.clone(),
|
||||
port.clone(),
|
||||
),
|
||||
None => return Ok(()),
|
||||
}
|
||||
};
|
||||
|
||||
// TODO: pass actual server host and port somewhere.
|
||||
return Ok(Server::cancel(&address, &port, process_id, secret_key).await?);
|
||||
}
|
||||
|
||||
// Active shard we're talking to.
|
||||
// The lifetime of this depends on the pool mode:
|
||||
// - if in session mode, this lives until client disconnects or changes it,
|
||||
// - if in transaction mode, this lives for the duration of one transaction.
|
||||
let mut shard: Option<usize> = None;
|
||||
|
||||
loop {
|
||||
// Read a complete message from the client, which normally would be
|
||||
// either a `Q` (query) or `P` (prepare, extended protocol).
|
||||
// We can parse it here before grabbing a server from the pool,
|
||||
// in case the client is sending some control messages, e.g.
|
||||
// SET sharding_context.key = '1234';
|
||||
let mut message = read_message(&mut self.read).await?;
|
||||
let original = message.clone(); // To be forwarded to the server
|
||||
let code = message.get_u8() as char;
|
||||
let _len = message.get_i32() as usize;
|
||||
|
||||
match code {
|
||||
'Q' => {
|
||||
server.send(original).await?;
|
||||
let response = server.recv().await?;
|
||||
write_all_half(&mut self.write, response).await?;
|
||||
// Parse for special select shard command.
|
||||
// SET SHARDING KEY TO 'bigint';
|
||||
match self.select_shard(message.clone(), pool.shards()).await {
|
||||
Some(s) => {
|
||||
set_sharding_key(&mut self.write).await?;
|
||||
shard = Some(s);
|
||||
continue;
|
||||
}
|
||||
None => (),
|
||||
};
|
||||
|
||||
'X' => {
|
||||
// Client closing
|
||||
return Ok(());
|
||||
}
|
||||
// The message is part of the regular protocol.
|
||||
// self.buffer.put(message);
|
||||
|
||||
_ => {
|
||||
println!(">>> Unexpected code: {}", code);
|
||||
// Grab a server from the pool.
|
||||
// None = any shard
|
||||
let connection = pool.get(shard).await.unwrap();
|
||||
let mut proxy = connection.0;
|
||||
let _address = connection.1;
|
||||
let server = &mut *proxy;
|
||||
|
||||
// Claim this server as mine for query cancellation.
|
||||
server.claim(self.process_id, self.secret_key);
|
||||
|
||||
loop {
|
||||
// No messages in the buffer, read one.
|
||||
let mut message = if message.len() == 0 {
|
||||
match read_message(&mut self.read).await {
|
||||
Ok(message) => message,
|
||||
Err(err) => {
|
||||
// Client disconnected without warning.
|
||||
if server.in_transaction() {
|
||||
// TODO: this is what PgBouncer does
|
||||
// which leads to connection thrashing.
|
||||
//
|
||||
// I think we could issue a ROLLBACK here instead.
|
||||
// server.mark_bad();
|
||||
server.query("ROLLBACK; DISCARD ALL;").await?;
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let msg = message.clone();
|
||||
message.clear();
|
||||
msg
|
||||
};
|
||||
|
||||
let original = message.clone(); // To be forwarded to the server
|
||||
let code = message.get_u8() as char;
|
||||
let _len = message.get_i32() as usize;
|
||||
|
||||
match code {
|
||||
'Q' => {
|
||||
server.send(original).await?;
|
||||
|
||||
loop {
|
||||
let response = server.recv().await?;
|
||||
match write_all_half(&mut self.write, response).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
server.mark_bad();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
if !server.is_data_available() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Release server
|
||||
if !server.in_transaction() && self.transaction_mode {
|
||||
shard = None;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
'X' => {
|
||||
// Client closing. Rollback and clean up
|
||||
// connection before releasing into the pool.
|
||||
// Pgbouncer closes the connection which leads to
|
||||
// connection thrashing when clients misbehave.
|
||||
// This pool will protect the database. :salute:
|
||||
if server.in_transaction() {
|
||||
server.query("ROLLBACK; DISCARD ALL;").await?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
'P' => {
|
||||
// Extended protocol, let's buffer most of it
|
||||
self.buffer.put(&original[..]);
|
||||
}
|
||||
|
||||
'B' => {
|
||||
self.buffer.put(&original[..]);
|
||||
}
|
||||
|
||||
// Describe
|
||||
'D' => {
|
||||
self.buffer.put(&original[..]);
|
||||
}
|
||||
|
||||
'E' => {
|
||||
self.buffer.put(&original[..]);
|
||||
}
|
||||
|
||||
'S' => {
|
||||
// Extended protocol, client requests sync
|
||||
self.buffer.put(&original[..]);
|
||||
server.send(self.buffer.clone()).await?;
|
||||
self.buffer.clear();
|
||||
|
||||
loop {
|
||||
let response = server.recv().await?;
|
||||
match write_all_half(&mut self.write, response).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
server.mark_bad();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
if !server.is_data_available() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Release server
|
||||
if !server.in_transaction() && self.transaction_mode {
|
||||
shard = None;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// CopyData
|
||||
'd' => {
|
||||
// Forward the data to the server,
|
||||
// don't buffer it since it can be rather large.
|
||||
server.send(original).await?;
|
||||
}
|
||||
|
||||
'c' | 'f' => {
|
||||
// Copy is done.
|
||||
server.send(original).await?;
|
||||
let response = server.recv().await?;
|
||||
match write_all_half(&mut self.write, response).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
server.mark_bad();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
// Release the server
|
||||
if !server.in_transaction() && self.transaction_mode {
|
||||
println!("Releasing after copy done");
|
||||
shard = None;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_ => {
|
||||
println!(">>> Unexpected code: {}", code);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the server from being mine. I can't cancel its queries anymore.
|
||||
pub fn release(&mut self) {
|
||||
let mut guard = self.client_server_map.lock().unwrap();
|
||||
guard.remove(&(self.process_id, self.secret_key));
|
||||
}
|
||||
|
||||
async fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option<usize> {
|
||||
let code = buf.get_u8() as char;
|
||||
|
||||
match code {
|
||||
'Q' => (),
|
||||
// 'P' => (),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let len = buf.get_i32();
|
||||
let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); // Don't read the ternminating null
|
||||
|
||||
if self.sharding_regex.is_match(&query) {
|
||||
let shard = query.split("'").collect::<Vec<&str>>()[1];
|
||||
match shard.parse::<i64>() {
|
||||
Ok(shard) => {
|
||||
let sharder = Sharder::new(shards);
|
||||
Some(sharder.pg_bigint_hash(shard))
|
||||
}
|
||||
Err(_) => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
87
src/config.rs
Normal file
87
src/config.rs
Normal file
@@ -0,0 +1,87 @@
|
||||
use serde_derive::Deserialize;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use toml;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::errors::Error;
|
||||
|
||||
#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)]
|
||||
pub struct Address {
|
||||
pub host: String,
|
||||
pub port: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)]
|
||||
pub struct User {
|
||||
pub name: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct General {
|
||||
pub host: String,
|
||||
pub port: i16,
|
||||
pub pool_size: u32,
|
||||
pub pool_mode: String,
|
||||
pub connect_timeout: u64,
|
||||
pub healthcheck_timeout: u64,
|
||||
pub ban_time: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Shard {
|
||||
pub servers: Vec<(String, u16)>,
|
||||
pub database: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Config {
|
||||
pub general: General,
|
||||
pub user: User,
|
||||
pub shards: HashMap<String, Shard>,
|
||||
}
|
||||
|
||||
pub async fn parse(path: &str) -> Result<Config, Error> {
|
||||
// let path = Path::new(path);
|
||||
let mut contents = String::new();
|
||||
let mut file = match File::open(path).await {
|
||||
Ok(file) => file,
|
||||
Err(err) => {
|
||||
println!("> Config error: {:?}", err);
|
||||
return Err(Error::BadConfig);
|
||||
}
|
||||
};
|
||||
|
||||
match file.read_to_string(&mut contents).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
println!("> Config error: {:?}", err);
|
||||
return Err(Error::BadConfig);
|
||||
}
|
||||
};
|
||||
|
||||
let config: Config = match toml::from_str(&contents) {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
println!("> Config error: {:?}", err);
|
||||
return Err(Error::BadConfig);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_config() {
|
||||
let config = parse("pgcat.toml").await.unwrap();
|
||||
assert_eq!(config.general.pool_size, 15);
|
||||
assert_eq!(config.shards.len(), 3);
|
||||
assert_eq!(config.shards["1"].servers[0].0, "127.0.0.1");
|
||||
}
|
||||
}
|
||||
@@ -5,4 +5,7 @@ pub enum Error {
|
||||
ClientBadStartup,
|
||||
ProtocolSyncError,
|
||||
ServerError,
|
||||
// ServerTimeout,
|
||||
// DirtyServer,
|
||||
BadConfig,
|
||||
}
|
||||
|
||||
82
src/main.rs
82
src/main.rs
@@ -1,19 +1,59 @@
|
||||
// PgCat, a PostgreSQL pooler with load balancing, failover, and sharding support.
|
||||
// Copyright (C) 2022 Lev Kokotov <lev@levthe.dev>
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
extern crate async_trait;
|
||||
extern crate bb8;
|
||||
extern crate bytes;
|
||||
extern crate md5;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate tokio;
|
||||
extern crate toml;
|
||||
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
mod client;
|
||||
mod config;
|
||||
mod errors;
|
||||
mod messages;
|
||||
mod pool;
|
||||
mod server;
|
||||
mod sharding;
|
||||
|
||||
// Support for query cancellation: this maps our process_ids and
|
||||
// secret keys to the backend's.
|
||||
use pool::{ClientServerMap, ConnectionPool};
|
||||
|
||||
/// Main!
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("> Welcome to PgRabbit");
|
||||
println!("> Welcome to PgCat! Meow.");
|
||||
|
||||
let listener = match TcpListener::bind("0.0.0.0:5433").await {
|
||||
let config = match config::parse("pgcat.toml").await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
println!("> Config parse error: {:?}", err);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let addr = format!("{}:{}", config.general.host, config.general.port);
|
||||
let listener = match TcpListener::bind(&addr).await {
|
||||
Ok(sock) => sock,
|
||||
Err(err) => {
|
||||
println!("> Error: {:?}", err);
|
||||
@@ -21,7 +61,28 @@ async fn main() {
|
||||
}
|
||||
};
|
||||
|
||||
println!("> Running on {}", addr);
|
||||
|
||||
// Tracks which client is connected to which server for query cancellation.
|
||||
let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new()));
|
||||
|
||||
println!("> Pool size: {}", config.general.pool_size);
|
||||
println!("> Pool mode: {}", config.general.pool_mode);
|
||||
println!("> Ban time: {}s", config.general.ban_time);
|
||||
println!(
|
||||
"> Healthcheck timeout: {}ms",
|
||||
config.general.healthcheck_timeout
|
||||
);
|
||||
|
||||
let pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await;
|
||||
let transaction_mode = config.general.pool_mode == "transaction";
|
||||
|
||||
println!("> Waiting for clients...");
|
||||
|
||||
loop {
|
||||
let pool = pool.clone();
|
||||
let client_server_map = client_server_map.clone();
|
||||
|
||||
let (socket, addr) = match listener.accept().await {
|
||||
Ok((socket, addr)) => (socket, addr),
|
||||
Err(err) => {
|
||||
@@ -32,26 +93,23 @@ async fn main() {
|
||||
|
||||
// Client goes to another thread, bye.
|
||||
tokio::task::spawn(async move {
|
||||
println!(">> Client {:?} connected.", addr);
|
||||
println!(
|
||||
">> Client {:?} connected, transaction pooling: {}",
|
||||
addr, transaction_mode
|
||||
);
|
||||
|
||||
match client::Client::startup(socket).await {
|
||||
match client::Client::startup(socket, client_server_map, transaction_mode).await {
|
||||
Ok(mut client) => {
|
||||
println!(">> Client {:?} authenticated successfully!", addr);
|
||||
let server =
|
||||
match server::Server::startup("127.0.0.1", "5432", "lev", "lev", "lev")
|
||||
.await
|
||||
{
|
||||
Ok(server) => server,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
match client.handle(server).await {
|
||||
match client.handle(pool).await {
|
||||
Ok(()) => {
|
||||
println!(">> Client {:?} disconnected.", addr);
|
||||
}
|
||||
|
||||
Err(err) => {
|
||||
println!(">> Client disconnected with error: {:?}", err);
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,14 @@ use tokio::net::TcpStream;
|
||||
|
||||
use crate::errors::Error;
|
||||
|
||||
// This is a funny one. `psql` parses this to figure out which
|
||||
// queries to send when using shortcuts, e.g. \d+.
|
||||
//
|
||||
// TODO: Actually get the version from the server itself.
|
||||
//
|
||||
const SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)";
|
||||
|
||||
/// Tell the client that authentication handshake completed successfully.
|
||||
pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> {
|
||||
let mut auth_ok = BytesMut::with_capacity(9);
|
||||
|
||||
@@ -16,6 +24,45 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> {
|
||||
Ok(write_all(stream, auth_ok).await?)
|
||||
}
|
||||
|
||||
/// Send server parameters to the client. This will tell the client
|
||||
/// what server version and what's the encoding we're using.
|
||||
pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> {
|
||||
let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]);
|
||||
let server_version =
|
||||
BytesMut::from(&format!("server_version\0{}\0", SERVER_VESION).as_bytes()[..]);
|
||||
|
||||
// Client encoding
|
||||
let len = client_encoding.len() as i32 + 4; // TODO: add more parameters here
|
||||
let mut res = BytesMut::with_capacity(64);
|
||||
|
||||
res.put_u8(b'S');
|
||||
res.put_i32(len);
|
||||
res.put_slice(&client_encoding[..]);
|
||||
|
||||
let len = server_version.len() as i32 + 4;
|
||||
res.put_u8(b'S');
|
||||
res.put_i32(len);
|
||||
res.put_slice(&server_version[..]);
|
||||
|
||||
Ok(write_all(stream, res).await?)
|
||||
}
|
||||
|
||||
/// Give the client the process_id and secret we generated
|
||||
/// used in query cancellation.
|
||||
pub async fn backend_key_data(
|
||||
stream: &mut TcpStream,
|
||||
backend_id: i32,
|
||||
secret_key: i32,
|
||||
) -> Result<(), Error> {
|
||||
let mut key_data = BytesMut::from(&b"K"[..]);
|
||||
key_data.put_i32(12);
|
||||
key_data.put_i32(backend_id);
|
||||
key_data.put_i32(secret_key);
|
||||
|
||||
Ok(write_all(stream, key_data).await?)
|
||||
}
|
||||
|
||||
/// Tell the client we're ready for another query.
|
||||
pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> {
|
||||
let mut bytes = BytesMut::with_capacity(5);
|
||||
|
||||
@@ -26,6 +73,8 @@ pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> {
|
||||
Ok(write_all(stream, bytes).await?)
|
||||
}
|
||||
|
||||
/// Send the startup packet the server. We're pretending we're a Pg client.
|
||||
/// This tells the server which user we are and what database we want.
|
||||
pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Result<(), Error> {
|
||||
let mut bytes = BytesMut::with_capacity(25);
|
||||
|
||||
@@ -55,6 +104,8 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu
|
||||
}
|
||||
}
|
||||
|
||||
/// Send password challenge response to the server.
|
||||
/// This is the MD5 challenge.
|
||||
pub async fn md5_password(
|
||||
stream: &mut TcpStream,
|
||||
user: &str,
|
||||
@@ -87,6 +138,24 @@ pub async fn md5_password(
|
||||
Ok(write_all(stream, message).await?)
|
||||
}
|
||||
|
||||
pub async fn set_sharding_key(stream: &mut OwnedWriteHalf) -> Result<(), Error> {
|
||||
let mut res = BytesMut::with_capacity(25);
|
||||
|
||||
let set_complete = BytesMut::from(&"SET SHARDING KEY\0"[..]);
|
||||
let len = (set_complete.len() + 4) as i32;
|
||||
|
||||
res.put_u8(b'C');
|
||||
res.put_i32(len);
|
||||
res.put_slice(&set_complete[..]);
|
||||
|
||||
res.put_u8(b'Z');
|
||||
res.put_i32(5);
|
||||
res.put_u8(b'I');
|
||||
|
||||
write_all_half(stream, res).await
|
||||
}
|
||||
|
||||
/// Write all data in the buffer to the TcpStream.
|
||||
pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> {
|
||||
match stream.write_all(&buf).await {
|
||||
Ok(_) => Ok(()),
|
||||
@@ -94,6 +163,7 @@ pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Erro
|
||||
}
|
||||
}
|
||||
|
||||
/// Write all the data in the buffer to the TcpStream, write owned half (see mpsc).
|
||||
pub async fn write_all_half(stream: &mut OwnedWriteHalf, buf: BytesMut) -> Result<(), Error> {
|
||||
match stream.write_all(&buf).await {
|
||||
Ok(_) => Ok(()),
|
||||
|
||||
267
src/pool.rs
Normal file
267
src/pool.rs
Normal file
@@ -0,0 +1,267 @@
|
||||
/// Pooling and failover and banlist.
|
||||
use async_trait::async_trait;
|
||||
use bb8::{ManageConnection, Pool, PooledConnection};
|
||||
use chrono::naive::NaiveDateTime;
|
||||
|
||||
use crate::config::{Address, Config, User};
|
||||
use crate::errors::Error;
|
||||
use crate::server::Server;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc, Mutex,
|
||||
};
|
||||
|
||||
// Banlist: bad servers go in here.
|
||||
pub type BanList = Arc<Mutex<Vec<HashMap<Address, NaiveDateTime>>>>;
|
||||
pub type Counter = Arc<AtomicUsize>;
|
||||
pub type ClientServerMap = Arc<Mutex<HashMap<(i32, i32), (i32, i32, String, String)>>>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ConnectionPool {
|
||||
databases: Vec<Vec<Pool<ServerPool>>>,
|
||||
addresses: Vec<Vec<Address>>,
|
||||
round_robin: Counter,
|
||||
banlist: BanList,
|
||||
healthcheck_timeout: u64,
|
||||
ban_time: i64,
|
||||
}
|
||||
|
||||
impl ConnectionPool {
|
||||
/// Construct the connection pool from a config file.
|
||||
pub async fn from_config(config: Config, client_server_map: ClientServerMap) -> ConnectionPool {
|
||||
let mut shards = Vec::new();
|
||||
let mut addresses = Vec::new();
|
||||
let mut banlist = Vec::new();
|
||||
let mut shard_ids = config
|
||||
.shards
|
||||
.clone()
|
||||
.into_keys()
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<String>>();
|
||||
shard_ids.sort_by_key(|k| k.parse::<i64>().unwrap());
|
||||
|
||||
for shard in shard_ids {
|
||||
let shard = &config.shards[&shard];
|
||||
let mut pools = Vec::new();
|
||||
let mut replica_addresses = Vec::new();
|
||||
|
||||
for server in &shard.servers {
|
||||
let address = Address {
|
||||
host: server.0.clone(),
|
||||
port: server.1.to_string(),
|
||||
};
|
||||
|
||||
let manager = ServerPool::new(
|
||||
address.clone(),
|
||||
config.user.clone(),
|
||||
&shard.database,
|
||||
client_server_map.clone(),
|
||||
);
|
||||
|
||||
let pool = Pool::builder()
|
||||
.max_size(config.general.pool_size)
|
||||
.connection_timeout(std::time::Duration::from_millis(
|
||||
config.general.connect_timeout,
|
||||
))
|
||||
.test_on_check_out(false)
|
||||
.build(manager)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
pools.push(pool);
|
||||
replica_addresses.push(address);
|
||||
}
|
||||
|
||||
shards.push(pools);
|
||||
addresses.push(replica_addresses);
|
||||
banlist.push(HashMap::new());
|
||||
}
|
||||
|
||||
ConnectionPool {
|
||||
databases: shards,
|
||||
addresses: addresses,
|
||||
round_robin: Arc::new(AtomicUsize::new(0)),
|
||||
banlist: Arc::new(Mutex::new(banlist)),
|
||||
healthcheck_timeout: config.general.healthcheck_timeout,
|
||||
ban_time: config.general.ban_time,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a connection from the pool.
|
||||
pub async fn get(
|
||||
&self,
|
||||
shard: Option<usize>,
|
||||
) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> {
|
||||
// Set this to false to gain ~3-4% speed.
|
||||
let with_health_check = true;
|
||||
|
||||
let shard = match shard {
|
||||
Some(shard) => shard,
|
||||
None => 0, // TODO: pick a shard at random
|
||||
};
|
||||
|
||||
loop {
|
||||
let index =
|
||||
self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len();
|
||||
let address = self.addresses[shard][index].clone();
|
||||
|
||||
if self.is_banned(&address, shard) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if we can connect
|
||||
// TODO: implement query wait timeout, i.e. time to get a conn from the pool
|
||||
let mut conn = match self.databases[shard][index].get().await {
|
||||
Ok(conn) => conn,
|
||||
Err(err) => {
|
||||
println!(">> Banning replica {}, error: {:?}", index, err);
|
||||
self.ban(&address, shard);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !with_health_check {
|
||||
return Ok((conn, address));
|
||||
}
|
||||
|
||||
// // Check if this server is alive with a health check
|
||||
let server = &mut *conn;
|
||||
|
||||
match tokio::time::timeout(
|
||||
tokio::time::Duration::from_millis(self.healthcheck_timeout),
|
||||
server.query("SELECT 1"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Check if health check succeeded
|
||||
Ok(res) => match res {
|
||||
Ok(_) => return Ok((conn, address)),
|
||||
Err(_) => {
|
||||
println!(
|
||||
">> Banning replica {} because of failed health check",
|
||||
index
|
||||
);
|
||||
self.ban(&address, shard);
|
||||
continue;
|
||||
}
|
||||
},
|
||||
// Health check never came back, database is really really down
|
||||
Err(_) => {
|
||||
println!(
|
||||
">> Banning replica {} because of health check timeout",
|
||||
index
|
||||
);
|
||||
self.ban(&address, shard);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ban an address (i.e. replica). It no longer will serve
|
||||
/// traffic for any new transactions. Existing transactions on that replica
|
||||
/// will finish successfully or error out to the clients.
|
||||
pub fn ban(&self, address: &Address, shard: usize) {
|
||||
println!(">> Banning {:?}", address);
|
||||
let now = chrono::offset::Utc::now().naive_utc();
|
||||
let mut guard = self.banlist.lock().unwrap();
|
||||
guard[shard].insert(address.clone(), now);
|
||||
}
|
||||
|
||||
/// Clear the replica to receive traffic again. Takes effect immediately
|
||||
/// for all new transactions.
|
||||
pub fn _unban(&self, address: &Address, shard: usize) {
|
||||
let mut guard = self.banlist.lock().unwrap();
|
||||
guard[shard].remove(address);
|
||||
}
|
||||
|
||||
/// Check if a replica can serve traffic. If all replicas are banned,
|
||||
/// we unban all of them. Better to try then not to.
|
||||
pub fn is_banned(&self, address: &Address, shard: usize) -> bool {
|
||||
let mut guard = self.banlist.lock().unwrap();
|
||||
|
||||
// Everything is banned, nothig is banned
|
||||
if guard[shard].len() == self.databases[shard].len() {
|
||||
guard[shard].clear();
|
||||
drop(guard);
|
||||
println!(">> Unbanning all replicas.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// I expect this to miss 99.9999% of the time.
|
||||
match guard[shard].get(address) {
|
||||
Some(timestamp) => {
|
||||
let now = chrono::offset::Utc::now().naive_utc();
|
||||
if now.timestamp() - timestamp.timestamp() > self.ban_time {
|
||||
// 1 minute
|
||||
guard[shard].remove(address);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn shards(&self) -> usize {
|
||||
self.databases.len()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ServerPool {
|
||||
address: Address,
|
||||
user: User,
|
||||
database: String,
|
||||
client_server_map: ClientServerMap,
|
||||
}
|
||||
|
||||
impl ServerPool {
|
||||
pub fn new(
|
||||
address: Address,
|
||||
user: User,
|
||||
database: &str,
|
||||
client_server_map: ClientServerMap,
|
||||
) -> ServerPool {
|
||||
ServerPool {
|
||||
address: address,
|
||||
user: user,
|
||||
database: database.to_string(),
|
||||
client_server_map: client_server_map,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ManageConnection for ServerPool {
|
||||
type Connection = Server;
|
||||
type Error = Error;
|
||||
|
||||
/// Attempts to create a new connection.
|
||||
async fn connect(&self) -> Result<Self::Connection, Self::Error> {
|
||||
println!(">> Creating a new connection for the pool");
|
||||
|
||||
Server::startup(
|
||||
&self.address.host,
|
||||
&self.address.port,
|
||||
&self.user.name,
|
||||
&self.user.password,
|
||||
&self.database,
|
||||
self.client_server_map.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Determines if the connection is still connected to the database.
|
||||
async fn is_valid(&self, _conn: &mut PooledConnection<'_, Self>) -> Result<(), Self::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Synchronously determine if the connection is no longer usable, if possible.
|
||||
fn has_broken(&self, conn: &mut Self::Connection) -> bool {
|
||||
conn.is_bad()
|
||||
}
|
||||
}
|
||||
285
src/server.rs
285
src/server.rs
@@ -1,24 +1,65 @@
|
||||
#![allow(dead_code)]
|
||||
#![allow(unused_variables)]
|
||||
|
||||
///! Implementation of the PostgreSQL server (database) protocol.
|
||||
///! Here we are pretending to the a Postgres client.
|
||||
use bytes::{Buf, BufMut, BytesMut};
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
use crate::config::Address;
|
||||
use crate::errors::Error;
|
||||
use crate::messages::*;
|
||||
use crate::ClientServerMap;
|
||||
|
||||
/// Server state.
|
||||
pub struct Server {
|
||||
// Server host, e.g. localhost
|
||||
host: String,
|
||||
|
||||
// Server port: e.g. 5432
|
||||
port: String,
|
||||
|
||||
// Buffered read socket
|
||||
read: BufReader<OwnedReadHalf>,
|
||||
|
||||
// Unbuffered write socket (our client code buffers)
|
||||
write: OwnedWriteHalf,
|
||||
|
||||
// Our server response buffer
|
||||
buffer: BytesMut,
|
||||
|
||||
// Server information the server sent us over on startup
|
||||
server_info: BytesMut,
|
||||
|
||||
// Backend id and secret key used for query cancellation.
|
||||
backend_id: i32,
|
||||
secret_key: i32,
|
||||
|
||||
// Is the server inside a transaction at the moment.
|
||||
in_transaction: bool,
|
||||
|
||||
// Is there more data for the client to read.
|
||||
data_available: bool,
|
||||
|
||||
// Is the server broken? We'll remote it from the pool if so.
|
||||
bad: bool,
|
||||
|
||||
// Mapping of clients and servers used for query cancellation.
|
||||
client_server_map: ClientServerMap,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Pretend to be the Postgres client and connect to the server given host, port and credentials.
|
||||
/// Perform the authentication and return the server in a ready-for-query mode.
|
||||
pub async fn startup(
|
||||
host: &str,
|
||||
port: &str,
|
||||
user: &str,
|
||||
password: &str,
|
||||
database: &str,
|
||||
client_server_map: ClientServerMap,
|
||||
) -> Result<Server, Error> {
|
||||
let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await {
|
||||
Ok(stream) => stream,
|
||||
@@ -28,8 +69,13 @@ impl Server {
|
||||
}
|
||||
};
|
||||
|
||||
// Send the startup packet.
|
||||
startup(&mut stream, user, database).await?;
|
||||
|
||||
let mut server_info = BytesMut::with_capacity(25);
|
||||
let mut backend_id: i32 = 0;
|
||||
let mut secret_key: i32 = 0;
|
||||
|
||||
loop {
|
||||
let code = match stream.read_u8().await {
|
||||
Ok(code) => code as char,
|
||||
@@ -62,10 +108,8 @@ impl Server {
|
||||
md5_password(&mut stream, user, password, &salt[..]).await?;
|
||||
}
|
||||
|
||||
// We're in!
|
||||
0 => {
|
||||
println!(">> Server authentication successful!");
|
||||
}
|
||||
// Authentication handshake complete.
|
||||
0 => (),
|
||||
|
||||
_ => {
|
||||
println!(">> Unsupported authentication mechanism: {}", code);
|
||||
@@ -75,7 +119,23 @@ impl Server {
|
||||
}
|
||||
|
||||
'E' => {
|
||||
println!(">> Database error");
|
||||
let error_code = match stream.read_u8().await {
|
||||
Ok(error_code) => error_code,
|
||||
Err(_) => return Err(Error::SocketError),
|
||||
};
|
||||
|
||||
match error_code {
|
||||
0 => (), // Terminator
|
||||
_ => {
|
||||
let mut error = vec![0u8; len as usize - 4 - 1];
|
||||
match stream.read_exact(&mut error).await {
|
||||
Ok(_) => (),
|
||||
Err(_) => return Err(Error::SocketError),
|
||||
};
|
||||
|
||||
println!(">> Server error: {}", String::from_utf8_lossy(&error));
|
||||
}
|
||||
};
|
||||
return Err(Error::ServerError);
|
||||
}
|
||||
|
||||
@@ -86,14 +146,22 @@ impl Server {
|
||||
Ok(_) => (),
|
||||
Err(_) => return Err(Error::SocketError),
|
||||
};
|
||||
|
||||
server_info.put_u8(b'S');
|
||||
server_info.put_i32(len);
|
||||
server_info.put_slice(¶m[..]);
|
||||
}
|
||||
|
||||
'K' => {
|
||||
// TODO: save cancellation secret
|
||||
let mut cancel_secret = vec![0u8; len as usize - 4];
|
||||
match stream.read_exact(&mut cancel_secret).await {
|
||||
Ok(_) => (),
|
||||
Err(_) => return Err(Error::SocketError),
|
||||
// Query cancellation data.
|
||||
backend_id = match stream.read_i32().await {
|
||||
Ok(id) => id,
|
||||
Err(err) => return Err(Error::SocketError),
|
||||
};
|
||||
|
||||
secret_key = match stream.read_i32().await {
|
||||
Ok(id) => id,
|
||||
Err(err) => return Err(Error::SocketError),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -109,9 +177,18 @@ impl Server {
|
||||
let (read, write) = stream.into_split();
|
||||
|
||||
return Ok(Server {
|
||||
host: host.to_string(),
|
||||
port: port.to_string(),
|
||||
read: BufReader::new(read),
|
||||
write: write,
|
||||
buffer: BytesMut::with_capacity(8196),
|
||||
server_info: server_info,
|
||||
backend_id: backend_id,
|
||||
secret_key: secret_key,
|
||||
in_transaction: false,
|
||||
data_available: false,
|
||||
bad: false,
|
||||
client_server_map: client_server_map,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -123,24 +200,122 @@ impl Server {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> {
|
||||
Ok(write_all_half(&mut self.write, messages).await?)
|
||||
/// Issue a cancellation request to the server.
|
||||
/// Uses a separate connection that's not part of the connection pool.
|
||||
pub async fn cancel(
|
||||
host: &str,
|
||||
port: &str,
|
||||
process_id: i32,
|
||||
secret_key: i32,
|
||||
) -> Result<(), Error> {
|
||||
let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await {
|
||||
Ok(stream) => stream,
|
||||
Err(err) => {
|
||||
println!(">> Could not connect to server: {}", err);
|
||||
return Err(Error::SocketError);
|
||||
}
|
||||
};
|
||||
|
||||
let mut bytes = BytesMut::with_capacity(16);
|
||||
bytes.put_i32(16);
|
||||
bytes.put_i32(80877102);
|
||||
bytes.put_i32(process_id);
|
||||
bytes.put_i32(secret_key);
|
||||
|
||||
Ok(write_all(&mut stream, bytes).await?)
|
||||
}
|
||||
|
||||
/// Send data to the server from the client.
|
||||
pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> {
|
||||
match write_all_half(&mut self.write, messages).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
println!(">> Terminating server because of: {:?}", err);
|
||||
self.bad = true;
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive data from the server in response to a client request sent previously.
|
||||
/// This method must be called multiple times while `self.is_data_available()` is true
|
||||
/// in order to receive all data the server has to offer.
|
||||
pub async fn recv(&mut self) -> Result<BytesMut, Error> {
|
||||
loop {
|
||||
let mut message = read_message(&mut self.read).await?;
|
||||
let mut message = match read_message(&mut self.read).await {
|
||||
Ok(message) => message,
|
||||
Err(err) => {
|
||||
println!(">> Terminating server because of: {:?}", err);
|
||||
self.bad = true;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
// Buffer the message we'll forward to the client in a bit.
|
||||
self.buffer.put(&message[..]);
|
||||
|
||||
let code = message.get_u8() as char;
|
||||
let _len = message.get_i32();
|
||||
|
||||
match code {
|
||||
'Z' => {
|
||||
// Ready for query, time to forward buffer to client.
|
||||
let transaction_state = message.get_u8() as char;
|
||||
|
||||
match transaction_state {
|
||||
'T' => {
|
||||
self.in_transaction = true;
|
||||
}
|
||||
|
||||
'I' => {
|
||||
self.in_transaction = false;
|
||||
}
|
||||
|
||||
// Error client didn't clean up!
|
||||
// We shuold drop this server
|
||||
'E' => {
|
||||
self.in_transaction = true;
|
||||
}
|
||||
|
||||
_ => {
|
||||
self.bad = true;
|
||||
return Err(Error::ProtocolSyncError);
|
||||
}
|
||||
};
|
||||
|
||||
self.data_available = false;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
'D' => {
|
||||
self.data_available = true;
|
||||
|
||||
// Don't flush yet, the more we buffer, the faster this goes.
|
||||
// Up to a limit of course.
|
||||
if self.buffer.len() >= 8196 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// CopyInResponse: copy is starting from client to server
|
||||
'G' => break,
|
||||
|
||||
// CopyOutResponse: copy is starting from the server to the client
|
||||
'H' => {
|
||||
self.data_available = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// CopyData
|
||||
'd' => break,
|
||||
|
||||
// CopyDone
|
||||
'c' => {
|
||||
self.data_available = false;
|
||||
// Buffer until ReadyForQuery shows up
|
||||
}
|
||||
|
||||
_ => {
|
||||
// Keep buffering,
|
||||
}
|
||||
@@ -152,4 +327,88 @@ impl Server {
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// If the server is still inside a transaction.
|
||||
/// If the client disconnects while the server is in a transaction, we will clean it up.
|
||||
pub fn in_transaction(&self) -> bool {
|
||||
self.in_transaction
|
||||
}
|
||||
|
||||
/// We don't buffer all of server responses, e.g. COPY OUT produces too much data.
|
||||
/// The client is responsible to call `self.recv()` while this method returns true.
|
||||
pub fn is_data_available(&self) -> bool {
|
||||
self.data_available
|
||||
}
|
||||
|
||||
/// Server & client are out of sync, we must discard this connection.
|
||||
/// This happens with clients that misbehave.
|
||||
pub fn is_bad(&self) -> bool {
|
||||
self.bad
|
||||
}
|
||||
|
||||
/// Get server startup information to forward it to the client.
|
||||
/// Not used at the moment.
|
||||
pub fn server_info(&self) -> BytesMut {
|
||||
self.server_info.clone()
|
||||
}
|
||||
|
||||
/// Indicate that this server connection cannot be re-used and must be discarded.
|
||||
pub fn mark_bad(&mut self) {
|
||||
println!(">> Server marked bad");
|
||||
self.bad = true;
|
||||
}
|
||||
|
||||
/// Claim this server as mine for the purposes of query cancellation.
|
||||
pub fn claim(&mut self, process_id: i32, secret_key: i32) {
|
||||
let mut guard = self.client_server_map.lock().unwrap();
|
||||
guard.insert(
|
||||
(process_id, secret_key),
|
||||
(
|
||||
self.backend_id,
|
||||
self.secret_key,
|
||||
self.host.clone(),
|
||||
self.port.clone(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// Execute an arbitrary query against the server.
|
||||
/// It will use the Simple query protocol.
|
||||
/// Result will not be returned, so this is useful for things like `SET` or `ROLLBACK`.
|
||||
pub async fn query(&mut self, query: &str) -> Result<(), Error> {
|
||||
let mut query = BytesMut::from(&query.as_bytes()[..]);
|
||||
query.put_u8(0);
|
||||
|
||||
let len = query.len() as i32 + 4;
|
||||
|
||||
let mut msg = BytesMut::with_capacity(len as usize + 1);
|
||||
|
||||
msg.put_u8(b'Q');
|
||||
msg.put_i32(len);
|
||||
msg.put_slice(&query[..]);
|
||||
|
||||
self.send(msg).await?;
|
||||
loop {
|
||||
let _ = self.recv().await?;
|
||||
if !self.data_available {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A shorthand for `SET application_name = $1`.
|
||||
pub async fn set_name(&mut self, name: &str) -> Result<(), Error> {
|
||||
Ok(self
|
||||
.query(&format!("SET application_name = '{}'", name))
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub fn address(&self) -> Address {
|
||||
Address {
|
||||
host: self.host.to_string(),
|
||||
port: self.port.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
144
src/sharding.rs
Normal file
144
src/sharding.rs
Normal file
@@ -0,0 +1,144 @@
|
||||
// https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/catalog/partition.h#L20
|
||||
const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD;
|
||||
|
||||
pub struct Sharder {
|
||||
shards: usize,
|
||||
}
|
||||
|
||||
impl Sharder {
|
||||
pub fn new(shards: usize) -> Sharder {
|
||||
Sharder { shards: shards }
|
||||
}
|
||||
|
||||
/// Hash function used by Postgres to determine which partition
|
||||
/// to put the row in when using HASH(column) partitioning.
|
||||
/// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631
|
||||
/// Supports only 1 bigint at the moment, but we can add more later.
|
||||
pub fn pg_bigint_hash(&self, key: i64) -> usize {
|
||||
let mut lohalf = key as u32;
|
||||
let hihalf = (key >> 32) as u32;
|
||||
lohalf ^= if key >= 0 { hihalf } else { !hihalf };
|
||||
Self::combine(0, Self::pg_u32_hash(lohalf)) as usize % self.shards as usize
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn rot(x: u32, k: u32) -> u32 {
|
||||
(x << k) | (x >> (32 - k))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mix(mut a: u32, mut b: u32, mut c: u32) -> (u32, u32, u32) {
|
||||
a = a.wrapping_sub(c);
|
||||
a ^= Self::rot(c, 4);
|
||||
c = c.wrapping_add(b);
|
||||
|
||||
b = b.wrapping_sub(a);
|
||||
b ^= Self::rot(a, 6);
|
||||
a = a.wrapping_add(c);
|
||||
|
||||
c = c.wrapping_sub(b);
|
||||
c ^= Self::rot(b, 8);
|
||||
b = b.wrapping_add(a);
|
||||
|
||||
a = a.wrapping_sub(c);
|
||||
a ^= Self::rot(c, 16);
|
||||
c = c.wrapping_add(b);
|
||||
|
||||
b = b.wrapping_sub(a);
|
||||
b ^= Self::rot(a, 19);
|
||||
a = a.wrapping_add(c);
|
||||
|
||||
c = c.wrapping_sub(b);
|
||||
c ^= Self::rot(b, 4);
|
||||
b = b.wrapping_add(a);
|
||||
|
||||
(a, b, c)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn _final(mut a: u32, mut b: u32, mut c: u32) -> (u32, u32, u32) {
|
||||
c ^= b;
|
||||
c = c.wrapping_sub(Self::rot(b, 14));
|
||||
a ^= c;
|
||||
a = a.wrapping_sub(Self::rot(c, 11));
|
||||
b ^= a;
|
||||
b = b.wrapping_sub(Self::rot(a, 25));
|
||||
c ^= b;
|
||||
c = c.wrapping_sub(Self::rot(b, 16));
|
||||
a ^= c;
|
||||
a = a.wrapping_sub(Self::rot(c, 4));
|
||||
b ^= a;
|
||||
b = b.wrapping_sub(Self::rot(a, 14));
|
||||
c ^= b;
|
||||
c = c.wrapping_sub(Self::rot(b, 24));
|
||||
(a, b, c)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn combine(mut a: u64, b: u64) -> u64 {
|
||||
a ^= b
|
||||
.wrapping_add(0x49a0f4dd15e5a8e3 as u64)
|
||||
.wrapping_add(a << 54)
|
||||
.wrapping_add(a >> 7);
|
||||
a
|
||||
}
|
||||
|
||||
fn pg_u32_hash(k: u32) -> u64 {
|
||||
let mut a: u32 = 0x9e3779b9 as u32 + std::mem::size_of::<u32>() as u32 + 3923095 as u32;
|
||||
let mut b = a;
|
||||
let c = a;
|
||||
|
||||
a = a.wrapping_add((PARTITION_HASH_SEED >> 32) as u32);
|
||||
b = b.wrapping_add(PARTITION_HASH_SEED as u32);
|
||||
let (mut a, b, c) = Self::mix(a, b, c);
|
||||
|
||||
a = a.wrapping_add(k);
|
||||
|
||||
let (_a, b, c) = Self::_final(a, b, c);
|
||||
|
||||
((b as u64) << 32) | (c as u64)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
// See tests/sharding/partition_hash_test_setup.sql
|
||||
// The output of those SELECT statements will match this test,
|
||||
// confirming that we implemented Postgres BIGINT hashing correctly.
|
||||
#[test]
|
||||
fn test_pg_bigint_hash() {
|
||||
let sharder = Sharder::new(5);
|
||||
|
||||
let shard_0 = vec![1, 4, 5, 14, 19, 39, 40, 46, 47, 53];
|
||||
|
||||
for v in shard_0 {
|
||||
assert_eq!(sharder.pg_bigint_hash(v), 0);
|
||||
}
|
||||
|
||||
let shard_1 = vec![2, 3, 11, 17, 21, 23, 30, 49, 51, 54];
|
||||
|
||||
for v in shard_1 {
|
||||
assert_eq!(sharder.pg_bigint_hash(v), 1);
|
||||
}
|
||||
|
||||
let shard_2 = vec![6, 7, 15, 16, 18, 20, 25, 28, 34, 35];
|
||||
|
||||
for v in shard_2 {
|
||||
assert_eq!(sharder.pg_bigint_hash(v), 2);
|
||||
}
|
||||
|
||||
let shard_3 = vec![8, 12, 13, 22, 29, 31, 33, 36, 41, 43];
|
||||
|
||||
for v in shard_3 {
|
||||
assert_eq!(sharder.pg_bigint_hash(v), 3);
|
||||
}
|
||||
|
||||
let shard_4 = vec![9, 10, 24, 26, 27, 32, 37, 38, 42, 45];
|
||||
|
||||
for v in shard_4 {
|
||||
assert_eq!(sharder.pg_bigint_hash(v), 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
1
tests/python/.gitignore
vendored
Normal file
1
tests/python/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
venv/
|
||||
1
tests/python/requirements.txt
Normal file
1
tests/python/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
psycopg2==2.9.3
|
||||
11
tests/python/tests.py
Normal file
11
tests/python/tests.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import psycopg2
|
||||
|
||||
conn = psycopg2.connect("postgres://random:password@127.0.0.1:6432/db")
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT 1");
|
||||
res = cur.fetchall()
|
||||
|
||||
print(res)
|
||||
|
||||
# conn.commit()
|
||||
11
tests/ruby/tests.rb
Normal file
11
tests/ruby/tests.rb
Normal file
@@ -0,0 +1,11 @@
|
||||
require 'pg'
|
||||
|
||||
conn = PG.connect(host: '127.0.0.1', port: 5433, dbname: 'test')
|
||||
|
||||
conn.exec( "SELECT * FROM pg_stat_activity" ) do |result|
|
||||
puts " PID | User | Query"
|
||||
result.each do |row|
|
||||
puts " %7d | %-16s | %s " %
|
||||
row.values_at('pid', 'usename', 'query')
|
||||
end
|
||||
end
|
||||
35
tests/sharding/README.md
Normal file
35
tests/sharding/README.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Sharding tests
|
||||
|
||||
This helps us test the sharding algorithm we implemented.
|
||||
|
||||
|
||||
## Setup
|
||||
|
||||
We setup 3 Postgres DBs, `shard0`, `shard1`, and `shard2`. In each database, we create a partitioned table called `data`. The table is partitioned by hash, and each database will only have _one_ partition, `shard0` will satisfy `modulus 3, remainder 0`, `shard1` will satisfy `modulus 3, remainder 1`, etc.
|
||||
|
||||
To set this up, you can just run:
|
||||
|
||||
```bash
|
||||
psql -f query_routing_setup.sql
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
|
||||
Start up PgCat by running `cargo run --release` in the root of the repo. In a different tab, run this:
|
||||
|
||||
```bash
|
||||
psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql
|
||||
psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql
|
||||
```
|
||||
|
||||
Note that no errors should take place. If our sharding logic was incorrect, we would get some errors
|
||||
about unsatisfiable partition bounds. We don't because the pooler picked the correct databases
|
||||
given the sharding keys.
|
||||
|
||||
Finally, you can validate the result again by running
|
||||
|
||||
```bash
|
||||
psql -f query_routing_test_validate.sql
|
||||
```
|
||||
|
||||
## That's it!
|
||||
26
tests/sharding/partition_hash_test_setup.sql
Normal file
26
tests/sharding/partition_hash_test_setup.sql
Normal file
@@ -0,0 +1,26 @@
|
||||
DROP TABLE IF EXISTS shards CASCADE;
|
||||
|
||||
CREATE TABLE shards (
|
||||
id BIGINT,
|
||||
value VARCHAR
|
||||
) PARTITION BY HASH (id);
|
||||
|
||||
-- DROP TABLE IF EXISTS shard_0;
|
||||
CREATE TABLE shard_0 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 0);
|
||||
-- DROP TABLE IF EXISTS shard_1;
|
||||
CREATE TABLE shard_1 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 1);
|
||||
-- DROP TABLE IF EXISTS shard_2;
|
||||
CREATE TABLE shard_2 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 2);
|
||||
-- DROP TABLE IF EXISTS shard_3;
|
||||
CREATE TABLE shard_3 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 3);
|
||||
-- DROP TABLE IF EXISTS shard_4;
|
||||
CREATE TABLE shard_4 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 4);
|
||||
|
||||
|
||||
INSERT INTO shards SELECT generate_series(1, 500), 'value';
|
||||
|
||||
SELECT * FROM shard_0 ORDER BY id LIMIT 10;
|
||||
SELECT * FROM shard_1 ORDER BY id LIMIT 10;
|
||||
SELECT * FROM shard_2 ORDER BY id LIMIT 10;
|
||||
SELECT * FROM shard_3 ORDER BY id LIMIT 10;
|
||||
SELECT * FROM shard_4 ORDER BY id LIMIT 10;
|
||||
12
tests/sharding/query_routing.sh
Normal file
12
tests/sharding/query_routing.sh
Normal file
@@ -0,0 +1,12 @@
|
||||
#/bin/bash
|
||||
|
||||
# Setup all the shards.
|
||||
sudo service postgresql restart
|
||||
|
||||
psql -f query_routing_setup.sql
|
||||
|
||||
psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql
|
||||
|
||||
psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql
|
||||
|
||||
psql -f query_routing_test_validate.sql
|
||||
61
tests/sharding/query_routing_setup.sql
Normal file
61
tests/sharding/query_routing_setup.sql
Normal file
@@ -0,0 +1,61 @@
|
||||
|
||||
DROP DATABASE IF EXISTS shard0;
|
||||
DROP DATABASE IF EXISTS shard1;
|
||||
DROP DATABASE IF EXISTS shard2;
|
||||
|
||||
CREATE DATABASE shard0;
|
||||
CREATE DATABASE shard1;
|
||||
CREATE DATABASE shard2;
|
||||
|
||||
\c shard0
|
||||
|
||||
DROP TABLE IF EXISTS data CASCADE;
|
||||
|
||||
CREATE TABLE data (
|
||||
id BIGINT,
|
||||
value VARCHAR
|
||||
) PARTITION BY HASH (id);
|
||||
|
||||
CREATE TABLE data_shard_0 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 0);
|
||||
|
||||
\c shard1
|
||||
|
||||
DROP TABLE IF EXISTS data CASCADE;
|
||||
|
||||
CREATE TABLE data (
|
||||
id BIGINT,
|
||||
value VARCHAR
|
||||
) PARTITION BY HASH (id);
|
||||
|
||||
CREATE TABLE data_shard_1 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 1);
|
||||
|
||||
|
||||
\c shard2
|
||||
|
||||
DROP TABLE IF EXISTS data CASCADE;
|
||||
|
||||
CREATE TABLE data (
|
||||
id BIGINT,
|
||||
value VARCHAR
|
||||
) PARTITION BY HASH (id);
|
||||
|
||||
CREATE TABLE data_shard_2 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 2);
|
||||
|
||||
DROP ROLE IF EXISTS sharding_user;
|
||||
CREATE ROLE sharding_user ENCRYPTED PASSWORD 'sharding_user' LOGIN;
|
||||
|
||||
GRANT CONNECT ON DATABASE shard0 TO sharding_user;
|
||||
GRANT CONNECT ON DATABASE shard1 TO sharding_user;
|
||||
GRANT CONNECT ON DATABASE shard2 TO sharding_user;
|
||||
|
||||
\c shard0
|
||||
GRANT ALL ON SCHEMA public TO sharding_user;
|
||||
GRANT ALL ON TABLE data TO sharding_user;
|
||||
|
||||
\c shard1
|
||||
GRANT ALL ON SCHEMA public TO sharding_user;
|
||||
GRANT ALL ON TABLE data TO sharding_user;
|
||||
|
||||
\c shard2
|
||||
GRANT ALL ON SCHEMA public TO sharding_user;
|
||||
GRANT ALL ON TABLE data TO sharding_user;
|
||||
47
tests/sharding/query_routing_test_insert.sql
Normal file
47
tests/sharding/query_routing_test_insert.sql
Normal file
@@ -0,0 +1,47 @@
|
||||
SET SHARDING KEY TO '1';
|
||||
INSERT INTO data (id, value) VALUES (1, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '2';
|
||||
INSERT INTO data (id, value) VALUES (2, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '3';
|
||||
INSERT INTO data (id, value) VALUES (3, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '4';
|
||||
INSERT INTO data (id, value) VALUES (4, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '5';
|
||||
INSERT INTO data (id, value) VALUES (5, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '6';
|
||||
INSERT INTO data (id, value) VALUES (6, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '7';
|
||||
INSERT INTO data (id, value) VALUES (7, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '8';
|
||||
INSERT INTO data (id, value) VALUES (8, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '9';
|
||||
INSERT INTO data (id, value) VALUES (9, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '10';
|
||||
INSERT INTO data (id, value) VALUES (10, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '11';
|
||||
INSERT INTO data (id, value) VALUES (11, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '12';
|
||||
INSERT INTO data (id, value) VALUES (12, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '13';
|
||||
INSERT INTO data (id, value) VALUES (13, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '14';
|
||||
INSERT INTO data (id, value) VALUES (14, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '15';
|
||||
INSERT INTO data (id, value) VALUES (15, 'value_1');
|
||||
|
||||
SET SHARDING KEY TO '16';
|
||||
INSERT INTO data (id, value) VALUES (16, 'value_1');
|
||||
47
tests/sharding/query_routing_test_select.sql
Normal file
47
tests/sharding/query_routing_test_select.sql
Normal file
@@ -0,0 +1,47 @@
|
||||
SET SHARDING KEY TO '1';
|
||||
SELECT * FROM data WHERE id = 1;
|
||||
|
||||
SET SHARDING KEY TO '2';
|
||||
SELECT * FROM data WHERE id = 2;
|
||||
|
||||
SET SHARDING KEY TO '3';
|
||||
SELECT * FROM data WHERE id = 3;
|
||||
|
||||
SET SHARDING KEY TO '4';
|
||||
SELECT * FROM data WHERE id = 4;
|
||||
|
||||
SET SHARDING KEY TO '5';
|
||||
SELECT * FROM data WHERE id = 5;
|
||||
|
||||
SET SHARDING KEY TO '6';
|
||||
SELECT * FROM data WHERE id = 6;
|
||||
|
||||
SET SHARDING KEY TO '7';
|
||||
SELECT * FROM data WHERE id = 7;
|
||||
|
||||
SET SHARDING KEY TO '8';
|
||||
SELECT * FROM data WHERE id = 8;
|
||||
|
||||
SET SHARDING KEY TO '9';
|
||||
SELECT * FROM data WHERE id = 9;
|
||||
|
||||
SET SHARDING KEY TO '10';
|
||||
SELECT * FROM data WHERE id = 10;
|
||||
|
||||
SET SHARDING KEY TO '11';
|
||||
SELECT * FROM data WHERE id = 11;
|
||||
|
||||
SET SHARDING KEY TO '12';
|
||||
SELECT * FROM data WHERE id = 12;
|
||||
|
||||
SET SHARDING KEY TO '13';
|
||||
SELECT * FROM data WHERE id = 13;
|
||||
|
||||
SET SHARDING KEY TO '14';
|
||||
SELECT * FROM data WHERE id = 14;
|
||||
|
||||
SET SHARDING KEY TO '15';
|
||||
SELECT * FROM data WHERE id = 15;
|
||||
|
||||
SET SHARDING KEY TO '16';
|
||||
SELECT * FROM data WHERE id = 16;
|
||||
11
tests/sharding/query_routing_test_validate.sql
Normal file
11
tests/sharding/query_routing_test_validate.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
\c shard0
|
||||
|
||||
SELECT * FROM data;
|
||||
|
||||
\c shard1
|
||||
|
||||
SELECT * FROM data;
|
||||
|
||||
\c shard2
|
||||
|
||||
SELECT * FROM data;
|
||||
Reference in New Issue
Block a user