1 使用pgaudt
1.1 常见SQL
-- 1 ddl
CREATE USER u1 PASSWORD 'u1.password';
CREATE TABLE t1(c1 INT, c2 TEXT);
CREATE TABLE t2 (c1 INT, c2 TEXT);
CREATE INDEX i1 ON t1(c2);
-- 2 dml
INSERT INTO t1 VALUES (1, 'aaa'), (2, 'bbb'), (3, 'ccc');
INSERT INTO t2 VALUES (1, 'aaa'), (5, 'bbb');
UPDATE t1 SET c2 = 'ddd' WHERE c1 < 3;
DELETE FROM t1 WHERE c1 = 2;
-- 3 select
SELECT * FROM t1;
-- 4 function
SELECT c2 || '_oper' FROM t1;
SELECT concat(c2, '_func') FROM t1;
INSERT INTO t1 VALUES (5, concat('aaa', '_func')), (6, concat('aaa', '_func'));
-- 5 multi
INSERT INTO t1 SELECT * FROM t2;
SELECT * FROM t1 JOIN t2 ON t1.c1 = t2.c1;
CREATE TABLE t3 AS SELECT * FROM t2;
DROP TABLE t1,t2,t3;
1 安装pgaudit
1.1 安装postgresql
# 1 配置环境变量
echo 'export BUILD_ROOT=`pwd`' > pgenv
echo 'export PG_HOME=$BUILD_ROOT/install' >> pgenv
echo 'export PATH="$PG_HOME/bin:$PATH"' >> pgenv
source pgenv
# 2 下载源码
wget https://ftp.postgresql.org/pub/source/v14.18/postgresql-14.18.tar.gz
tar -zxvf postgresql-14.18.tar.gz
cd postgresql-14.18.tar.gz
# 3 编译源码 (不开启debug模式,提高性能)
#./configure --prefix=$PG_HOME --enable-debug=yes
./configure --prefix=$PG_HOME --enable-debug=no --enable-cassert=no
make -sj
make install -sj
# ==========================================
cd $BUILD_ROOT/pg14/contrib/pgaudit
make install USE_PGXS=1 PG_CONFIG=$PG_HOME/bin/pg_config
cd -
# ==========================================
# 4 初始化集群
initdb -D $PG_HOME/data
# 4 配置集群
# echo "port = 54000" >> $PG_HOME/data/postgresql.conf
# echo "max_connections = 1000" >> $PG_HOME/data/postgresql.conf
# ==========================================
echo "logging_collector = on" >> $PG_HOME/data/postgresql.conf
echo "shared_preload_libraries = 'pgaudit'" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log = 'ALL'" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_catalog = true" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_client = true" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_parameter = true" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_relation = true" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_rows = true" >> $PG_HOME/data/postgresql.conf
echo "pgaudit.log_statement = true" >> $PG_HOME/data/postgresql.conf
# ==========================================
# 5 启动集群
pg_ctl start -D $PG_HOME/data
# ==========================================
DROP EXTENSION pgaudit;
CREATE EXTENSION pgaudit;
CREATE UNLOGGED TABLE adt(c1 TEXT);
CREATE TABLE t1 (c1 INT);
INSERT INTO t1 VALUES (1);
\! ls $PG_HOME/data/log
\! tail $PG_HOME/data/log/
# ==========================================
1.2 tpcc优化
echo "
shared_buffers = 200GB
work_mem = 1GB
maintenance_work_mem = 4GB
effective_cache_size = 500GB
wal_buffers = 1GB
checkpoint_timeout = 55min
" >> $PG_HOME/data/postgresql.conf
show shared_buffers;
show work_mem;
show maintenance_work_mem;
show effective_cache_size;
show wal_buffers;
show checkpoint_timeout;
1.3 安装pgaudit
# 1 下载源码
wget https://codeload.github.com/pgaudit/pgaudit/tar.gz/refs/tags/1.6.2 -O pgaudit-1.6.2.tar.gz
tar -zxvf pgaudit-1.6.2.tar.gz
# 2 编译
cd pgaudit-1.6.2
make install USE_PGXS=1 PG_CONFIG=$PG_HOME/bin/pg_config
# 3 验证编译成功
ll $PG_HOME/share/extension
cat $PG_HOME/data/postgresql.conf | grep shared_preload_libraries
# 4 开启日志功能
echo "logging_collector = on" >> $PG_HOME/data/postgresql.conf
echo "log_statement = all" >> $PG_HOME/data/postgresql.conf
# 5 开启 pgaudit
echo "shared_preload_libraries = 'pgaudit'" >> $PG_HOME/data/postgresql.conf
echo " pgaudit.log = 'ALL'" >> $PG_HOME/data/postgresql.conf
-- 1 安装pgaudit
CREATE EXTENSION pgaudit;
-- 2 查看配置参数
SHOW pgaudit.log;
-- ALL
-- READ: SELECT
-- WRITE: INSERT, UPDATE, DELETE
-- ROLE
-- DDL
-- FUNCTION
-- MISC
-- MISC_SET
SHOW pgaudit.role;
SHOW pgaudit.log_level;
-- 查看审计文件
SHOW log_directory;
SHOW log_filename;
- 审计策略:客户
优先3.0
2 pgaudit工作原理
E100
pgaudit性能测试(pg-14)
- 配置:arm 128核,内存700G,1000仓,调整内存相关guc参数,其他guc参数默认(导数据约1h)
- 测试(不开审计):cpu 60%,30min,500并发,tpmc = 35.68w
- 测试(开pgaudit全审计):cpu 100%,30min,500并发,tpmc = 1.02w
- 测试(开pgaudit全审计,优化:不通过ereport记录日志,记录到unlogged table中):cpu 60%,30min,500并发,tmpc = 预计20w (有1点点提升空间)
E100客户要求:
- 功能:E100审计所有SQL语句
- 性能:性能劣化不高(内部认为20-30%可能能接收)
- 交付:尽量6月上旬,内核开发约2周
E100审计方案:
4种方案:
- 迁移G100审计功能(不建议)
- 功能:非常多
- 性能:开启1个审计线程,与pgaudit接近。开启48个审计线程,劣化20%
- 交付:工作量较大。G100与E100代码差异非常大,比如进程/线程架构。且G100审计代码:质量差、较分散、逻辑乱,不过2周应该可搞定
- 使用pgaudit(排除:性能非常低)
- 功能:支持DML,DDL,FUNCTION等,不支持用户登录、登出
- 性能:不开审计35.68w(cpu 60%),开审计1.05w (cpu 99%)
- 交付:工作量最小
- 优化pgaudit(日志存到TABLE,可花1天开发demo并验证性能)
- 功能:支持DML,DDL,FUNCTION等
- 日志收集:参考pgaudit
- 日志存储:UNLOGGED TABLE(备机是否需要审计日志?)
- 日志传输:Postgres进程 –> ShareBuffer –> BgWriter/CheckPointer进程
- 性能:不好评估,30%有点悬
- 交付:工作量适中
- 功能:支持DML,DDL,FUNCTION等
- 优化pgaudit(日志存到文件,可花2天开发demo并验证性能)
- 功能:支持DML,DDL,FUNCTION等
- 日志收集:参考pgaudit
- 日志存储:新增审计文件(发生故障时,比如突然断电,允许丢失小部分日志吗,比如8k)
- 日志传输:Postgres进程 –> 新增共享内存、信号量 –> 新增日志进程
- 性能:不好评估,比方案3好一些
- 交付:工作量适中,2周应该可以搞懂
- 功能:支持DML,DDL,FUNCTION等