From 8a0da10a876ac434b54c6f5008192b8edfb19f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 2 Mar 2023 18:14:10 +0100 Subject: [PATCH] Dev environment (#338) Add dev env --- .gitignore | 5 ++ README.md | 11 +++ dev/Dockerfile | 33 ++++++++ dev/dev_bashrc | 120 ++++++++++++++++++++++++++++ dev/docker-compose.yaml | 84 +++++++++++++++++++ dev/script/console | 6 ++ tests/ruby/helpers/pgcat_process.rb | 8 +- 7 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 dev/Dockerfile create mode 100644 dev/dev_bashrc create mode 100644 dev/docker-compose.yaml create mode 100755 dev/script/console diff --git a/.gitignore b/.gitignore index cf28bff..b3ca013 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ .profraw cov/ lcov.info + +# Dev +dev/.bash_history +dev/cache +!dev/cache/.keepme diff --git a/README.md b/README.md index 2ba7ba8..a8723b6 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,17 @@ docker compose up --exit-code-from main # This will also produce coverage report | Statistics | :white_check_mark: | :white_check_mark: | Query the admin database with `psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS'`. | | Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | +### Dev + +Also, you can open a 'dev' environment where you can debug tests easier by running the following command: + +``` +./dev/script/console +``` + +This will open a terminal in an environment similar to that used in tests. In there you can compile, run tests, do some debugging with the test environment, etc. Objects +compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have in your host. + ## Usage ### Session mode diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000..f2054ef --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,33 @@ +FROM rust:bullseye + +# Dependencies +RUN apt-get update -y \ + && apt-get install -y \ + llvm-11 psmisc postgresql-contrib postgresql-client \ + ruby ruby-dev libpq-dev python3 python3-pip lcov curl sudo iproute2 \ + strace ngrep iproute2 dnsutils lsof net-tools telnet + +# Rust +RUN cargo install cargo-binutils rustfilt +RUN rustup component add llvm-tools-preview + +# Ruby +RUN sudo gem install bundler + +# Toxyproxy +RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo dpkg -i toxiproxy-2.4.0.deb + +# Config +ENV APP_ROOT=/app +ARG APP_USER=pgcat +COPY dev_bashrc /etc/bash.bashrc + +RUN useradd -m -o -u 999 ${APP_USER} || exit 0 && mkdir ${APP_ROOT} && chown ${APP_USER} ${APP_ROOT} +RUN adduser ${APP_USER} sudo \ + && echo "${APP_USER} ALL=NOPASSWD: ALL" > /etc/sudoers.d/${APP_USER} \ + && chmod ugo+s /usr/sbin/usermod /usr/sbin/groupmod +ENV HOME=${APP_ROOT} +WORKDIR ${APP_ROOT} + +ENTRYPOINT ["/bin/bash"] diff --git a/dev/dev_bashrc b/dev/dev_bashrc new file mode 100644 index 0000000..25d6a03 --- /dev/null +++ b/dev/dev_bashrc @@ -0,0 +1,120 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# FIX USER NEEDED SO WE CAN SHARE UID BETWEEN HOST AND DEV ENV +usermod -o -u $(id -u) pgcat +groupmod -o -g $(id -g) pgcat + +# We fix the setuid in those commands as we now have sudo +sudo chmod ugo-s /usr/sbin/usermod /usr/sbin/groupmod + +# Environment customization +export DEV_ROOT="${APP_ROOT}/dev" +export HISTFILE="${DEV_ROOT}/.bash_history" +export CARGO_TARGET_DIR="${DEV_ROOT}/cache/target" +export CARGO_HOME="${DEV_ROOT}/cache/target/.cargo" +export BUNDLE_PATH="${DEV_ROOT}/cache/bundle" + +# Regular bashrc +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +PS1='\[\e]0;pgcat@dev-container\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]pgcat\[\033[00m\]@\[\033[01;32m\]dev-container\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\[\033[01;31m\]$(git branch &>/dev/null; if [ $? -eq 0 ]; then echo " ($(git branch | grep ^* |sed s/\*\ //))"; fi)\[\033[00m\]\$ ' + +unset color_prompt force_color_prompt + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' + +# Add an "alert" alias for long running commands. Use like so: +# sleep 10; alert +alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml new file mode 100644 index 0000000..ee609e0 --- /dev/null +++ b/dev/docker-compose.yaml @@ -0,0 +1,84 @@ +version: "3" + +x-common-definition-pg: + &common-definition-pg + image: postgres:14 + network_mode: "service:main" + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres -d postgres" ] + interval: 5s + timeout: 5s + retries: 5 + volumes: + - type: bind + source: ../tests/sharding/query_routing_setup.sql + target: /docker-entrypoint-initdb.d/query_routing_setup.sql + - type: bind + source: ../tests/sharding/partition_hash_test_setup.sql + target: /docker-entrypoint-initdb.d/partition_hash_test_setup.sql + +x-common-env-pg: + &common-env-pg + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + +services: + main: + image: kubernetes/pause + + pg1: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 + PGPORT: 5432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] + + pg2: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 7432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] + pg3: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 8432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] + pg4: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 9432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] + + toxiproxy: + build: . + network_mode: "service:main" + container_name: toxiproxy + environment: + LOG_LEVEL: info + entrypoint: toxiproxy-server + depends_on: + - pg1 + - pg2 + - pg3 + - pg4 + + pgcat-shell: + stdin_open: true + user: "${HOST_UID}:${HOST_GID}" + build: . + network_mode: "service:main" + depends_on: + - toxiproxy + volumes: + - ../:/app/ + entrypoint: + - /bin/bash + - -i diff --git a/dev/script/console b/dev/script/console new file mode 100755 index 0000000..f2a12d6 --- /dev/null +++ b/dev/script/console @@ -0,0 +1,6 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export HOST_UID="$(id -u)" +export HOST_GID="$(id -g)" +docker-compose -f "${DIR}/../docker-compose.yaml" run --rm pgcat-shell diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index b67c4a8..2108eaf 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -24,7 +24,13 @@ class PgcatProcess @log_filename = "/tmp/pgcat_log_#{SecureRandom.urlsafe_base64}.log" @config_filename = "/tmp/pgcat_cfg_#{SecureRandom.urlsafe_base64}.toml" - @command = "../../target/debug/pgcat #{@config_filename}" + command_path = if ENV['CARGO_TARGET_DIR'] then + "#{ENV['CARGO_TARGET_DIR']}/debug/pgcat" + else + '../../target/debug/pgcat' + end + + @command = "#{command_path} #{@config_filename}" FileUtils.cp("../../pgcat.toml", @config_filename) cfg = current_config