Refactor stats to use atomics (#375)

* Refactor stats to use atomics

When we are dealing with a high number of connections, generated
stats cannot be consumed fast enough by the stats collector loop.
This makes the stats subsystem inconsistent and a log of
warning messages are thrown due to unregistered server/clients.

This change refactors the stats subsystem so it uses atomics:

- Now counters are handled using U64 atomics
- Event system is dropped and averages are calculated using a loop
  every 15 seconds.
- Now, instead of snapshots being generated ever second we keep track of servers/clients
  that have registered. Each pool/server/client has its own instance of the counter and
  makes changes directly, instead of adding an event that gets processed later.

* Manually mplement Hash/Eq in `config::Address` ignoring stats

* Add tests for client connection counters

* Allow connecting to dockerized dev pgcat from the host

* stats: Decrease cl_idle when idle socket disconnects
This commit is contained in:
Jose Fernández
2023-03-28 17:19:37 +02:00
committed by GitHub
parent 9a2076a9eb
commit 58ce76d9b9
19 changed files with 1303 additions and 1182 deletions

View File

@@ -3,6 +3,7 @@
use bytes::{Buf, BufMut, BytesMut};
use log::{debug, error, info, trace, warn};
use std::io::Read;
use std::sync::Arc;
use std::time::SystemTime;
use tokio::io::{AsyncReadExt, BufReader};
use tokio::net::{
@@ -17,12 +18,10 @@ use crate::messages::*;
use crate::mirrors::MirroringManager;
use crate::pool::ClientServerMap;
use crate::scram::ScramSha256;
use crate::stats::Reporter;
use crate::stats::ServerStats;
/// Server state.
pub struct Server {
server_id: i32,
/// Server host, e.g. localhost,
/// port, e.g. 5432, and role, e.g. primary or replica.
address: Address,
@@ -62,7 +61,7 @@ pub struct Server {
connected_at: chrono::naive::NaiveDateTime,
/// Reports various metrics, e.g. data sent & received.
stats: Reporter,
stats: Arc<ServerStats>,
/// Application name using the server at the moment.
application_name: String,
@@ -77,12 +76,11 @@ impl Server {
/// Pretend to be the Postgres client and connect to the server given host, port and credentials.
/// Perform the authentication and return the server in a ready for query state.
pub async fn startup(
server_id: i32,
address: &Address,
user: &User,
database: &str,
client_server_map: ClientServerMap,
stats: Reporter,
stats: Arc<ServerStats>,
) -> Result<Server, Error> {
let mut stream =
match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await {
@@ -325,7 +323,6 @@ impl Server {
write,
buffer: BytesMut::with_capacity(8196),
server_info,
server_id,
process_id,
secret_key,
in_transaction: false,
@@ -396,7 +393,7 @@ impl Server {
/// Send messages to the server from the client.
pub async fn send(&mut self, messages: &BytesMut) -> Result<(), Error> {
self.mirror_send(messages);
self.stats.data_sent(messages.len(), self.server_id);
self.stats().data_sent(messages.len());
match write_all_half(&mut self.write, messages).await {
Ok(_) => {
@@ -545,7 +542,7 @@ impl Server {
let bytes = self.buffer.clone();
// Keep track of how much data we got from the server for stats.
self.stats.data_received(bytes.len(), self.server_id);
self.stats().data_received(bytes.len());
// Clear the buffer for next query.
self.buffer.clear();
@@ -665,18 +662,17 @@ impl Server {
}
}
/// get Server stats
pub fn stats(&self) -> Arc<ServerStats> {
self.stats.clone()
}
/// Get the servers address.
#[allow(dead_code)]
pub fn address(&self) -> Address {
self.address.clone()
}
/// Get the server connection identifier
/// Used to uniquely identify connection in statistics
pub fn server_id(&self) -> i32 {
self.server_id
}
// Get server's latest response timestamp
pub fn last_activity(&self) -> SystemTime {
self.last_activity
@@ -708,7 +704,9 @@ impl Drop for Server {
/// for a write.
fn drop(&mut self) {
self.mirror_disconnect();
self.stats.server_disconnecting(self.server_id);
// Update statistics
self.stats.disconnect();
let mut bytes = BytesMut::with_capacity(4);
bytes.put_u8(b'X');