feat: add pg_replication module for reactive WAL-based cache invalidation
Implements a PostgreSQL logical replication module that streams WAL changes, buffers them per XID for transaction atomicity, and publishes CacheKey invalidation events to the existing framework broadcast channel. Key design choices: - Transaction atomicity: changes are buffered until COMMIT so the fanout always receives a consistent view. A 1000-row bulk-insert emits one table-level CacheKey, not 1000 row events. - Native protocol: uses a self-contained raw TCP + postgres-protocol implementation for the replication connection (tokio-postgres 0.7 does not expose copy_both_simple publicly). - Corrected pgoutput v1 parser: original maybe_sql_integration code incorrectly read XID from DML messages; DML messages carry no XID in proto v1 — only Begin does. - Clean integration: publishes CacheKey::Channel events to store.events so the existing fanout task picks them up with zero changes to the fanout loop. - New cx.subscribe(key) API on RenderContext for manually registering subscription keys (needed when queries go through PgPool, not cx.run). New files: src/pg_replication/mod.rs — PgReplicationListener, key helpers src/pg_replication/wal_parser.rs — pgoutput v1 binary protocol parser src/pg_replication/emitter.rs — WalEmitter (tx-buffering + CacheKey emit) src/pg_replication/proto.rs — raw TCP PG wire-protocol connection examples/pg_replication/main.rs — live message board example docker-compose.yml — PG 16 container with wal_level=logical Run the example: docker compose up -d cargo run --example pg_replication --features pg_replication https://claude.ai/code/session_01SLGgXeqKV2o7KTmCaQZZPg
This commit is contained in:
Claude
Generated
+321
-14
@@ -2,6 +2,15 @@
|
|||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "1.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "allocator-api2"
|
name = "allocator-api2"
|
||||||
version = "0.2.21"
|
version = "0.2.21"
|
||||||
@@ -370,6 +379,12 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fallible-iterator"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.9"
|
version = "0.1.9"
|
||||||
@@ -519,7 +534,19 @@ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"wasi",
|
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.3.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"r-efi 5.3.0",
|
||||||
|
"wasip2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -530,7 +557,7 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"r-efi",
|
"r-efi 6.0.0",
|
||||||
"wasip2",
|
"wasip2",
|
||||||
"wasip3",
|
"wasip3",
|
||||||
]
|
]
|
||||||
@@ -909,16 +936,20 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"axum",
|
"axum",
|
||||||
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"dashmap",
|
"dashmap",
|
||||||
"futures",
|
"futures",
|
||||||
"maud",
|
"maud",
|
||||||
|
"postgres-protocol",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-postgres",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-subscriber",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -937,6 +968,15 @@ version = "0.4.29"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "matchers"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
|
||||||
|
dependencies = [
|
||||||
|
"regex-automata",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "matchit"
|
name = "matchit"
|
||||||
version = "0.7.3"
|
version = "0.7.3"
|
||||||
@@ -996,7 +1036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
|
checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"wasi",
|
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nu-ansi-term"
|
||||||
|
version = "0.50.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||||
|
dependencies = [
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1011,7 +1060,7 @@ dependencies = [
|
|||||||
"num-integer",
|
"num-integer",
|
||||||
"num-iter",
|
"num-iter",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
@@ -1046,6 +1095,24 @@ dependencies = [
|
|||||||
"libm",
|
"libm",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "objc2-core-foundation"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "objc2-system-configuration"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396"
|
||||||
|
dependencies = [
|
||||||
|
"objc2-core-foundation",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.21.3"
|
version = "1.21.3"
|
||||||
@@ -1096,6 +1163,25 @@ version = "2.3.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf"
|
||||||
|
version = "0.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||||
|
dependencies = [
|
||||||
|
"phf_shared",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "phf_shared"
|
||||||
|
version = "0.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
|
||||||
|
dependencies = [
|
||||||
|
"siphasher",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project-lite"
|
name = "pin-project-lite"
|
||||||
version = "0.2.17"
|
version = "0.2.17"
|
||||||
@@ -1141,6 +1227,35 @@ version = "0.2.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "postgres-protocol"
|
||||||
|
version = "0.6.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"byteorder",
|
||||||
|
"bytes",
|
||||||
|
"fallible-iterator",
|
||||||
|
"hmac",
|
||||||
|
"md-5",
|
||||||
|
"memchr",
|
||||||
|
"rand 0.9.2",
|
||||||
|
"sha2",
|
||||||
|
"stringprep",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "postgres-types"
|
||||||
|
version = "0.2.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"fallible-iterator",
|
||||||
|
"postgres-protocol",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "potential_utf"
|
name = "potential_utf"
|
||||||
version = "0.1.4"
|
version = "0.1.4"
|
||||||
@@ -1210,6 +1325,12 @@ dependencies = [
|
|||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "r-efi"
|
||||||
|
version = "5.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "r-efi"
|
name = "r-efi"
|
||||||
version = "6.0.0"
|
version = "6.0.0"
|
||||||
@@ -1223,8 +1344,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"rand_chacha",
|
"rand_chacha 0.3.1",
|
||||||
"rand_core",
|
"rand_core 0.6.4",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand"
|
||||||
|
version = "0.9.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||||
|
dependencies = [
|
||||||
|
"rand_chacha 0.9.0",
|
||||||
|
"rand_core 0.9.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1234,7 +1365,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ppv-lite86",
|
"ppv-lite86",
|
||||||
"rand_core",
|
"rand_core 0.6.4",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_chacha"
|
||||||
|
version = "0.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||||
|
dependencies = [
|
||||||
|
"ppv-lite86",
|
||||||
|
"rand_core 0.9.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1246,6 +1387,15 @@ dependencies = [
|
|||||||
"getrandom 0.2.17",
|
"getrandom 0.2.17",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_core"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redox_syscall"
|
name = "redox_syscall"
|
||||||
version = "0.5.18"
|
version = "0.5.18"
|
||||||
@@ -1264,6 +1414,23 @@ dependencies = [
|
|||||||
"bitflags",
|
"bitflags",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.4.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.8.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rsa"
|
name = "rsa"
|
||||||
version = "0.9.10"
|
version = "0.9.10"
|
||||||
@@ -1277,7 +1444,7 @@ dependencies = [
|
|||||||
"num-traits",
|
"num-traits",
|
||||||
"pkcs1",
|
"pkcs1",
|
||||||
"pkcs8",
|
"pkcs8",
|
||||||
"rand_core",
|
"rand_core 0.6.4",
|
||||||
"signature",
|
"signature",
|
||||||
"spki",
|
"spki",
|
||||||
"subtle",
|
"subtle",
|
||||||
@@ -1396,6 +1563,15 @@ dependencies = [
|
|||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sharded-slab"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "shlex"
|
name = "shlex"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
@@ -1419,9 +1595,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
|
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"digest",
|
"digest",
|
||||||
"rand_core",
|
"rand_core 0.6.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "siphasher"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "slab"
|
name = "slab"
|
||||||
version = "0.4.12"
|
version = "0.4.12"
|
||||||
@@ -1580,7 +1762,7 @@ dependencies = [
|
|||||||
"memchr",
|
"memchr",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"rsa",
|
"rsa",
|
||||||
"serde",
|
"serde",
|
||||||
"sha1",
|
"sha1",
|
||||||
@@ -1590,7 +1772,7 @@ dependencies = [
|
|||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tracing",
|
"tracing",
|
||||||
"whoami",
|
"whoami 1.6.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1618,7 +1800,7 @@ dependencies = [
|
|||||||
"md-5",
|
"md-5",
|
||||||
"memchr",
|
"memchr",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rand",
|
"rand 0.8.5",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
@@ -1627,7 +1809,7 @@ dependencies = [
|
|||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tracing",
|
"tracing",
|
||||||
"whoami",
|
"whoami 1.6.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1725,6 +1907,15 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thread_local"
|
||||||
|
version = "1.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinystr"
|
name = "tinystr"
|
||||||
version = "0.8.2"
|
version = "0.8.2"
|
||||||
@@ -1778,6 +1969,32 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-postgres"
|
||||||
|
version = "0.7.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
"byteorder",
|
||||||
|
"bytes",
|
||||||
|
"fallible-iterator",
|
||||||
|
"futures-channel",
|
||||||
|
"futures-util",
|
||||||
|
"log",
|
||||||
|
"parking_lot",
|
||||||
|
"percent-encoding",
|
||||||
|
"phf",
|
||||||
|
"pin-project-lite",
|
||||||
|
"postgres-protocol",
|
||||||
|
"postgres-types",
|
||||||
|
"rand 0.9.2",
|
||||||
|
"socket2",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"whoami 2.1.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-stream"
|
name = "tokio-stream"
|
||||||
version = "0.1.18"
|
version = "0.1.18"
|
||||||
@@ -1789,6 +2006,19 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-util"
|
||||||
|
version = "0.7.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"futures-core",
|
||||||
|
"futures-sink",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tower"
|
name = "tower"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
@@ -1847,6 +2077,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
"valuable",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-log"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
|
||||||
|
dependencies = [
|
||||||
|
"log",
|
||||||
|
"once_cell",
|
||||||
|
"tracing-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-subscriber"
|
||||||
|
version = "0.3.22"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
|
||||||
|
dependencies = [
|
||||||
|
"matchers",
|
||||||
|
"nu-ansi-term",
|
||||||
|
"once_cell",
|
||||||
|
"regex-automata",
|
||||||
|
"sharded-slab",
|
||||||
|
"smallvec",
|
||||||
|
"thread_local",
|
||||||
|
"tracing",
|
||||||
|
"tracing-core",
|
||||||
|
"tracing-log",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1918,6 +2178,12 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "valuable"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vcpkg"
|
name = "vcpkg"
|
||||||
version = "0.2.15"
|
version = "0.2.15"
|
||||||
@@ -1936,6 +2202,15 @@ version = "0.11.1+wasi-snapshot-preview1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasi"
|
||||||
|
version = "0.14.7+wasi-0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
|
||||||
|
dependencies = [
|
||||||
|
"wasip2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasip2"
|
name = "wasip2"
|
||||||
version = "1.0.2+wasi-0.2.9"
|
version = "1.0.2+wasi-0.2.9"
|
||||||
@@ -1960,6 +2235,15 @@ version = "0.1.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
|
checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasite"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42"
|
||||||
|
dependencies = [
|
||||||
|
"wasi 0.14.7+wasi-0.2.4",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasm-bindgen"
|
name = "wasm-bindgen"
|
||||||
version = "0.2.114"
|
version = "0.2.114"
|
||||||
@@ -2039,6 +2323,16 @@ dependencies = [
|
|||||||
"semver",
|
"semver",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "web-sys"
|
||||||
|
version = "0.3.91"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
|
||||||
|
dependencies = [
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "whoami"
|
name = "whoami"
|
||||||
version = "1.6.1"
|
version = "1.6.1"
|
||||||
@@ -2046,7 +2340,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d"
|
checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libredox",
|
"libredox",
|
||||||
"wasite",
|
"wasite 0.1.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "whoami"
|
||||||
|
version = "2.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"libredox",
|
||||||
|
"objc2-system-configuration",
|
||||||
|
"wasite 1.0.2",
|
||||||
|
"web-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+18
-2
@@ -3,14 +3,20 @@ name = "livevue-rs"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
# Run the todo example with: cargo run --example todo
|
# Run the todo example with: cargo run --example todo
|
||||||
|
# Run the pg_replication example with: cargo run --example pg_replication --features pg_replication
|
||||||
|
|
||||||
|
[features]
|
||||||
|
# Opt-in feature: PostgreSQL logical replication for reactive cache invalidation.
|
||||||
|
# Adds tokio-postgres and bytes as dependencies.
|
||||||
|
pg_replication = ["dep:tokio-postgres", "dep:postgres-protocol", "dep:bytes"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
axum = { version = "0.7", features = ["macros"] }
|
axum = { version = "0.7", features = ["macros"] }
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
|
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "postgres"] }
|
||||||
maud = { version = "0.26", features = ["axum"] }
|
maud = { version = "0.26", features = ["axum"] }
|
||||||
uuid = { version = "1", features = ["v4", "serde"] }
|
uuid = { version = "1", features = ["v4", "serde"] }
|
||||||
dashmap = "6"
|
dashmap = "6"
|
||||||
@@ -19,6 +25,16 @@ tokio-stream = "0.1"
|
|||||||
tracing = "0.1.44"
|
tracing = "0.1.44"
|
||||||
futures = "0.3.32"
|
futures = "0.3.32"
|
||||||
|
|
||||||
|
# Optional: PostgreSQL logical replication
|
||||||
|
tokio-postgres = { version = "0.7", optional = true }
|
||||||
|
postgres-protocol = { version = "0.6", optional = true }
|
||||||
|
bytes = { version = "1", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
chrono = { version = "0.4", features = ["clock"] }
|
chrono = { version = "0.4", features = ["clock"] }
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
|
[[example]]
|
||||||
|
name = "pg_replication"
|
||||||
|
path = "examples/pg_replication/main.rs"
|
||||||
|
required-features = ["pg_replication"]
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: livevue_postgres
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: postgres
|
||||||
|
POSTGRES_PASSWORD: postgres
|
||||||
|
POSTGRES_DB: livevue
|
||||||
|
ports:
|
||||||
|
- "5432:5432"
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
# Enable logical replication — required for the pg_replication example.
|
||||||
|
command:
|
||||||
|
- postgres
|
||||||
|
- -c
|
||||||
|
- wal_level=logical
|
||||||
|
- -c
|
||||||
|
- max_replication_slots=10
|
||||||
|
- -c
|
||||||
|
- max_wal_senders=10
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U postgres -d livevue"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
@@ -0,0 +1,387 @@
|
|||||||
|
//! # pg_replication example
|
||||||
|
//!
|
||||||
|
//! A live message board backed by PostgreSQL.
|
||||||
|
//!
|
||||||
|
//! Run:
|
||||||
|
//! ```sh
|
||||||
|
//! docker compose up -d # start the PG container (see docker-compose.yml)
|
||||||
|
//! cargo run --example pg_replication --features pg_replication
|
||||||
|
//! ```
|
||||||
|
//! Then open http://localhost:3001 in two browser tabs and watch them stay in sync.
|
||||||
|
//!
|
||||||
|
//! ## How it works
|
||||||
|
//!
|
||||||
|
//! 1. The render function queries the `messages` table directly via `sqlx::PgPool`
|
||||||
|
//! and calls `cx.subscribe(pg_table_key("public", "messages"))` to tell the
|
||||||
|
//! framework "re-render this connection whenever `pg:public.messages` is
|
||||||
|
//! invalidated".
|
||||||
|
//!
|
||||||
|
//! 2. `PgReplicationListener` connects to PostgreSQL over the logical replication
|
||||||
|
//! protocol, buffers changes per transaction, and on each COMMIT publishes
|
||||||
|
//! `CacheKey::Channel { key: "pg:public.messages" }` to the store's broadcast
|
||||||
|
//! channel.
|
||||||
|
//!
|
||||||
|
//! 3. The fanout task picks up the event, finds all connections subscribed to that
|
||||||
|
//! key, and triggers a re-render for each — sending fresh HTML over SSE.
|
||||||
|
//!
|
||||||
|
//! No polling. No websocket boilerplate. Just PostgreSQL WAL + SSE.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use axum::{
|
||||||
|
extract::{Json, State},
|
||||||
|
http::StatusCode,
|
||||||
|
response::{Html, IntoResponse},
|
||||||
|
routing::{get, post},
|
||||||
|
Router,
|
||||||
|
};
|
||||||
|
use maud::{html, Markup, DOCTYPE};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use sqlx::FromRow;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use livevue_rs::{
|
||||||
|
pg_replication::{pg_table_key, PgReplicationConfig, PgReplicationListener},
|
||||||
|
server::{ingest_signals, ActionParams, AppState, SharedState},
|
||||||
|
spawn_fanout, RenderContext, Store,
|
||||||
|
};
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Domain types
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, FromRow)]
|
||||||
|
struct Message {
|
||||||
|
id: i64,
|
||||||
|
body: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Database helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async fn setup_db(pool: &sqlx::PgPool) -> anyhow::Result<()> {
|
||||||
|
// Enable logical replication for this table (requires superuser or
|
||||||
|
// rds_superuser on managed PG).
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE TABLE IF NOT EXISTS messages (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
body TEXT NOT NULL
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// Create the publication if it doesn't exist.
|
||||||
|
// `IF NOT EXISTS` was added in PG 14 — fall back gracefully on older versions.
|
||||||
|
let exists: bool = sqlx::query_scalar(
|
||||||
|
"SELECT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'livevue_pub')",
|
||||||
|
)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
sqlx::query("CREATE PUBLICATION livevue_pub FOR TABLE messages")
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
println!("Created publication livevue_pub");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Rendering
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Shared render state — anything the render function needs beyond the store.
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct RenderState {
|
||||||
|
pg: sqlx::PgPool,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn render_page(cx: &mut RenderContext, rs: &RenderState) -> anyhow::Result<Markup> {
|
||||||
|
// Subscribe to PG-sourced invalidation events for the messages table.
|
||||||
|
// The fanout task will re-render this connection whenever the WAL emitter
|
||||||
|
// fires a `CacheKey::Channel { key: "pg:public.messages" }` event.
|
||||||
|
cx.subscribe(pg_table_key("public", "messages"));
|
||||||
|
|
||||||
|
let messages: Vec<Message> = sqlx::query_as("SELECT id, body FROM messages ORDER BY id DESC")
|
||||||
|
.fetch_all(&rs.pg)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let new_msg = cx.signal("newMsg", livevue_rs::global());
|
||||||
|
|
||||||
|
Ok(html! {
|
||||||
|
div #app {
|
||||||
|
h1 { "Live Message Board" }
|
||||||
|
p.subtitle {
|
||||||
|
"Messages are stored in PostgreSQL. "
|
||||||
|
"Add one and watch every open tab update in real time via WAL replication."
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── New message form ─────────────────────────────────────────
|
||||||
|
div.compose {
|
||||||
|
input
|
||||||
|
type="text"
|
||||||
|
placeholder="Type a message…"
|
||||||
|
data-bind=(new_msg.name)
|
||||||
|
data-on:keydown="{if (event.key === 'Enter') $post('/action/post_message')}"
|
||||||
|
;
|
||||||
|
button
|
||||||
|
data-on:click="$post('/action/post_message')"
|
||||||
|
data-attr=(format!("{{disabled: {}.trim() === ''}}", new_msg.val))
|
||||||
|
{ "Send" }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Message list ─────────────────────────────────────────────
|
||||||
|
@if messages.is_empty() {
|
||||||
|
p.empty { "No messages yet. Be the first!" }
|
||||||
|
} @else {
|
||||||
|
ul.messages {
|
||||||
|
@for msg in &messages {
|
||||||
|
li {
|
||||||
|
span.id { "#" (msg.id) }
|
||||||
|
span.body { (msg.body) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Full HTML shell for the initial page load.
|
||||||
|
fn page_shell(conn_id: Uuid, signals_json: &str, inner: &Markup) -> String {
|
||||||
|
html! {
|
||||||
|
(DOCTYPE)
|
||||||
|
html lang="en" {
|
||||||
|
head {
|
||||||
|
meta charset="utf-8";
|
||||||
|
meta name="viewport" content="width=device-width, initial-scale=1";
|
||||||
|
title { "pg_replication example — livevue-rs" }
|
||||||
|
style {
|
||||||
|
(r#"
|
||||||
|
body { font-family: system-ui, sans-serif; max-width: 640px; margin: 2rem auto; padding: 0 1rem; }
|
||||||
|
h1 { margin-bottom: 0.25rem; }
|
||||||
|
.subtitle { color: #666; font-size: 0.9rem; margin-top: 0; }
|
||||||
|
.compose { display: flex; gap: 0.5rem; margin: 1.5rem 0; }
|
||||||
|
.compose input { flex: 1; padding: 0.5rem; border: 1px solid #ccc; border-radius: 4px; font-size: 1rem; }
|
||||||
|
.compose button { padding: 0.5rem 1rem; background: #0070f3; color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 1rem; }
|
||||||
|
.compose button:disabled { background: #aaa; cursor: default; }
|
||||||
|
ul.messages { list-style: none; padding: 0; margin: 0; }
|
||||||
|
ul.messages li { padding: 0.75rem; border-bottom: 1px solid #eee; display: flex; gap: 0.75rem; }
|
||||||
|
.id { color: #999; font-size: 0.85rem; min-width: 2.5rem; }
|
||||||
|
.empty { color: #999; }
|
||||||
|
"#)
|
||||||
|
}
|
||||||
|
// DataStar SDK (loaded from CDN for the example)
|
||||||
|
script
|
||||||
|
type="module"
|
||||||
|
src="https://cdn.jsdelivr.net/gh/starfederation/datastar@v1.0.0-beta.11/bundles/datastar.js"
|
||||||
|
{}
|
||||||
|
}
|
||||||
|
body {
|
||||||
|
div
|
||||||
|
data-signals=(signals_json)
|
||||||
|
data-on:load=(format!("@get('/sse?conn={conn_id}')"))
|
||||||
|
{
|
||||||
|
(inner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.into_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// HTTP handlers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async fn index_handler(State(state): State<(SharedState, RenderState)>) -> impl IntoResponse {
|
||||||
|
let (app_state, rs) = &state;
|
||||||
|
let conn_id = Uuid::new_v4();
|
||||||
|
let signals_json = r#"{"newMsg":""}"#;
|
||||||
|
app_state.connections.insert(conn_id, {
|
||||||
|
// We need a sender for the connection, but the SSE handler will replace
|
||||||
|
// it. Use a dummy channel that immediately drops.
|
||||||
|
let (tx, _rx) = tokio::sync::mpsc::channel(1);
|
||||||
|
tx
|
||||||
|
});
|
||||||
|
app_state
|
||||||
|
.connections
|
||||||
|
.update_signals(&conn_id, signals_json.to_string());
|
||||||
|
|
||||||
|
let mut cx =
|
||||||
|
RenderContext::new(conn_id, livevue_rs::SignalStore::from_json(serde_json::json!({"newMsg": ""})), Default::default(), app_state.store.clone());
|
||||||
|
|
||||||
|
let inner = render_page(&mut cx, rs).await.unwrap_or_else(|e| {
|
||||||
|
html! { p { "Render error: " (e) } }
|
||||||
|
});
|
||||||
|
|
||||||
|
Html(page_shell(conn_id, signals_json, &inner))
|
||||||
|
}
|
||||||
|
|
||||||
|
// SSE handler is provided by the framework; it needs SharedState only.
|
||||||
|
// We wrap it to adapt the two-tuple state.
|
||||||
|
async fn sse_adapter(
|
||||||
|
State(state): State<(SharedState, RenderState)>,
|
||||||
|
query: axum::extract::Query<livevue_rs::server::SseParams>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
livevue_rs::sse_handler(State(state.0), query).await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct PostMessageBody {
|
||||||
|
#[serde(rename = "newMsg")]
|
||||||
|
new_msg: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn post_message_handler(
|
||||||
|
State(state): State<(SharedState, RenderState)>,
|
||||||
|
axum::extract::Query(params): axum::extract::Query<ActionParams>,
|
||||||
|
Json(body): Json<serde_json::Value>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
let signals = ingest_signals(&state.0, params.conn, body);
|
||||||
|
let msg: String = signals
|
||||||
|
.get("newMsg")
|
||||||
|
.and_then(|v| v.as_str().map(|s| s.trim().to_string()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
if msg.is_empty() {
|
||||||
|
return StatusCode::BAD_REQUEST;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = sqlx::query("INSERT INTO messages (body) VALUES ($1)")
|
||||||
|
.bind(&msg)
|
||||||
|
.execute(&state.1.pg)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::error!("failed to insert message: {e}");
|
||||||
|
return StatusCode::INTERNAL_SERVER_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note: we do NOT call store.publish() here — the PgReplicationListener
|
||||||
|
// will detect the INSERT through the WAL stream and fire the invalidation
|
||||||
|
// automatically. This demonstrates the key value of the module: mutations
|
||||||
|
// from *any* source (other services, psql, migrations) are reflected
|
||||||
|
// without any extra wiring.
|
||||||
|
|
||||||
|
StatusCode::OK
|
||||||
|
}
|
||||||
|
|
||||||
|
fn router(app_state: SharedState, render_state: RenderState) -> Router {
|
||||||
|
let combined = (app_state, render_state);
|
||||||
|
Router::new()
|
||||||
|
.route("/", get(index_handler))
|
||||||
|
.route("/sse", get(sse_adapter))
|
||||||
|
.route("/action/post_message", post(post_message_handler))
|
||||||
|
.with_state(combined)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// main
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(
|
||||||
|
tracing_subscriber::EnvFilter::from_default_env()
|
||||||
|
.add_directive("livevue_rs=debug".parse().unwrap())
|
||||||
|
.add_directive("pg_replication=debug".parse().unwrap()),
|
||||||
|
)
|
||||||
|
.init();
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Database setup
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
let database_url = std::env::var("DATABASE_URL")
|
||||||
|
.unwrap_or_else(|_| "postgres://postgres:postgres@localhost:5432/livevue".into());
|
||||||
|
|
||||||
|
let pg = sqlx::PgPool::connect(&database_url).await?;
|
||||||
|
setup_db(&pg).await?;
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Framework store (SQLite in-memory, unused for queries here)
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The Store provides the broadcast channel and KV store that the framework
|
||||||
|
// needs. The embedded SQLite pool is not used in this example — queries go
|
||||||
|
// directly to PostgreSQL via the captured `pg` pool in the render closure.
|
||||||
|
let sqlite = sqlx::SqlitePool::connect("sqlite::memory:").await?;
|
||||||
|
let store = Store::new(sqlite);
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Render function
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
let render_state = RenderState { pg: pg.clone() };
|
||||||
|
let rs_clone = render_state.clone();
|
||||||
|
|
||||||
|
let render_fn: livevue_rs::RenderFn = Arc::new(move |mut cx| {
|
||||||
|
let rs = rs_clone.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let signals_json = cx.signals.to_json_string();
|
||||||
|
let conn_id = cx.connection_id;
|
||||||
|
|
||||||
|
let inner = render_page(&mut cx, &rs).await?;
|
||||||
|
let keys = cx.take_subscription_keys();
|
||||||
|
|
||||||
|
// Wrap in the full page shell so DataStar can morph the entire document.
|
||||||
|
let html = page_shell(conn_id, &signals_json, &inner);
|
||||||
|
Ok((html, keys))
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Application state + fanout
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
let state = AppState::new(store.clone(), render_fn);
|
||||||
|
spawn_fanout(state.clone());
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// PG replication listener
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The replication protocol requires a dedicated connection.
|
||||||
|
|
||||||
|
let replication_store = store.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
// Rebuild the config each iteration — PgReplicationConfig is not
|
||||||
|
// Clone, and start() takes ownership, so we reconstruct from the
|
||||||
|
// env-var values cached in the outer scope.
|
||||||
|
let c = PgReplicationConfig {
|
||||||
|
host: std::env::var("PG_HOST").unwrap_or_else(|_| "localhost".into()),
|
||||||
|
port: std::env::var("PG_PORT").ok()
|
||||||
|
.and_then(|p| p.parse().ok()).unwrap_or(5432),
|
||||||
|
user: std::env::var("PG_USER").unwrap_or_else(|_| "postgres".into()),
|
||||||
|
password: std::env::var("PG_PASSWORD").unwrap_or_else(|_| "postgres".into()),
|
||||||
|
database: std::env::var("PG_DATABASE").unwrap_or_else(|_| "livevue".into()),
|
||||||
|
slot_name: "livevue_slot".into(),
|
||||||
|
publication_name: "livevue_pub".into(),
|
||||||
|
start_lsn: None,
|
||||||
|
};
|
||||||
|
if let Err(e) = PgReplicationListener::new(c, replication_store.clone()).start().await {
|
||||||
|
tracing::error!("pg replication listener error: {e:#}");
|
||||||
|
}
|
||||||
|
tracing::info!("pg replication listener reconnecting in 5 s…");
|
||||||
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// HTTP server
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
let app = router(state, render_state);
|
||||||
|
let listener_tcp = tokio::net::TcpListener::bind("0.0.0.0:3001").await?;
|
||||||
|
println!("pg_replication example listening on http://localhost:3001");
|
||||||
|
axum::serve(listener_tcp, app).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -197,6 +197,30 @@ impl RenderContext {
|
|||||||
self.store.kv.get(&key)
|
self.store.kv.get(&key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Manual subscriptions
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Manually register a subscription key without executing a query.
|
||||||
|
///
|
||||||
|
/// Use this to subscribe to externally-sourced invalidation events that
|
||||||
|
/// don't go through `cx.run()` — for example, events published by the
|
||||||
|
/// PostgreSQL replication module:
|
||||||
|
///
|
||||||
|
/// ```ignore
|
||||||
|
/// use livevue_rs::pg_replication::pg_table_key;
|
||||||
|
///
|
||||||
|
/// cx.subscribe(pg_table_key("public", "posts"));
|
||||||
|
/// let posts = sqlx::query_as::<_, Post>("SELECT id, body FROM posts")
|
||||||
|
/// .fetch_all(&pg_pool)
|
||||||
|
/// .await?;
|
||||||
|
/// ```
|
||||||
|
pub fn subscribe(&mut self, key: CacheKey) {
|
||||||
|
if key != CacheKey::None {
|
||||||
|
self.subscription_keys.push(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// Post-render access
|
// Post-render access
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
|
|||||||
@@ -6,6 +6,15 @@ pub mod connection;
|
|||||||
pub mod sse;
|
pub mod sse;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
|
|
||||||
|
/// PostgreSQL logical replication module.
|
||||||
|
///
|
||||||
|
/// Enabled via the `pg_replication` Cargo feature:
|
||||||
|
/// ```toml
|
||||||
|
/// livevue-rs = { features = ["pg_replication"] }
|
||||||
|
/// ```
|
||||||
|
#[cfg(feature = "pg_replication")]
|
||||||
|
pub mod pg_replication;
|
||||||
|
|
||||||
pub use signal::{Signal, SignalStore, SignalOpts, mangle, client_only, global, data_signals};
|
pub use signal::{Signal, SignalStore, SignalOpts, mangle, client_only, global, data_signals};
|
||||||
pub use query::{CacheKey, Query};
|
pub use query::{CacheKey, Query};
|
||||||
pub use store::{Store, KVStore};
|
pub use store::{Store, KVStore};
|
||||||
|
|||||||
@@ -0,0 +1,311 @@
|
|||||||
|
use super::wal_parser::{ColumnValue, LogicalMessage, MessageParser, RelationInfo, TupleData};
|
||||||
|
use crate::query::CacheKey;
|
||||||
|
use bytes::BytesMut;
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Per-transaction buffer
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Accumulates the set of invalidation keys for one PostgreSQL transaction.
|
||||||
|
///
|
||||||
|
/// We use a `HashSet` for table-level keys so that a transaction inserting
|
||||||
|
/// 10 000 rows into one table still produces exactly **one**
|
||||||
|
/// `CacheKey::Channel { key: "pg:public.todos" }` event. Row-level keys are
|
||||||
|
/// collected separately for callers that need fine-grained precision.
|
||||||
|
struct TxBuffer {
|
||||||
|
xid: u32,
|
||||||
|
/// Deduplicated `(schema, table)` pairs touched in this transaction.
|
||||||
|
table_keys: HashSet<(String, String)>,
|
||||||
|
/// `(schema, table, id)` tuples for changes with a detectable integer PK.
|
||||||
|
row_keys: Vec<(String, String, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TxBuffer {
|
||||||
|
fn new(xid: u32) -> Self {
|
||||||
|
Self {
|
||||||
|
xid,
|
||||||
|
table_keys: HashSet::new(),
|
||||||
|
row_keys: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_table(&mut self, schema: &str, table: &str) {
|
||||||
|
self.table_keys.insert((schema.to_owned(), table.to_owned()));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_row(&mut self, schema: &str, table: &str, id: i64) {
|
||||||
|
self.row_keys.push((schema.to_owned(), table.to_owned(), id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// WalEmitter
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Processes pgoutput logical replication messages and publishes
|
||||||
|
/// [`CacheKey`] invalidation events to the framework's broadcast channel.
|
||||||
|
///
|
||||||
|
/// ## Transaction atomicity
|
||||||
|
///
|
||||||
|
/// Changes are buffered per XID. On `COMMIT`, all invalidation events for
|
||||||
|
/// that transaction are published in a tight loop so the fanout task receives
|
||||||
|
/// them as a burst. The transaction's WAL data is **never** partially visible:
|
||||||
|
/// the emitter waits for the `Commit` message before publishing anything.
|
||||||
|
///
|
||||||
|
/// ## Event granularity
|
||||||
|
///
|
||||||
|
/// For every transaction two kinds of events are emitted:
|
||||||
|
///
|
||||||
|
/// * **Table-level** (`"pg:{schema}.{table}"`) — deduplicated; at most one per
|
||||||
|
/// distinct table touched. Components that track whole-table queries should
|
||||||
|
/// subscribe with [`pg_table_key`](super::pg_table_key).
|
||||||
|
///
|
||||||
|
/// * **Row-level** (`"pg:{schema}.{table}:{id}"`) — emitted when the change
|
||||||
|
/// has a single-column integer replica-identity key. Components that track
|
||||||
|
/// individual rows can subscribe with [`pg_row_key`](super::pg_row_key).
|
||||||
|
///
|
||||||
|
/// ## Usage
|
||||||
|
///
|
||||||
|
/// ```ignore
|
||||||
|
/// let emitter = WalEmitter::new(store.events.clone());
|
||||||
|
/// // Inside the replication loop:
|
||||||
|
/// emitter.process_raw(&wal_bytes).await?;
|
||||||
|
/// ```
|
||||||
|
pub struct WalEmitter {
|
||||||
|
parser: MessageParser,
|
||||||
|
events: broadcast::Sender<CacheKey>,
|
||||||
|
/// XID of the currently open transaction (pgoutput v1 is sequential).
|
||||||
|
current_xid: Option<u32>,
|
||||||
|
/// Buffer for the current transaction; `None` when outside a transaction.
|
||||||
|
buffer: Option<TxBuffer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WalEmitter {
|
||||||
|
/// Create a new emitter that publishes to `events`.
|
||||||
|
///
|
||||||
|
/// Pass `store.events.clone()` so the emitter integrates directly with the
|
||||||
|
/// framework's fanout task.
|
||||||
|
pub fn new(events: broadcast::Sender<CacheKey>) -> Self {
|
||||||
|
Self {
|
||||||
|
parser: MessageParser::new(),
|
||||||
|
events,
|
||||||
|
current_xid: None,
|
||||||
|
buffer: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Public entry point
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Parse and process all pgoutput messages in a raw XLogData payload.
|
||||||
|
///
|
||||||
|
/// Call this for every `XLogData` message received from the replication
|
||||||
|
/// stream. The method is `&mut self` because the parser maintains relation
|
||||||
|
/// metadata across calls.
|
||||||
|
pub async fn process_raw(&mut self, data: &[u8]) -> Result<(), String> {
|
||||||
|
let mut buf = BytesMut::from(data);
|
||||||
|
while !buf.is_empty() {
|
||||||
|
match self.parser.parse_message(&mut buf)? {
|
||||||
|
Some(msg) => self.process_message(msg)?,
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Message dispatch
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn process_message(&mut self, msg: LogicalMessage) -> Result<(), String> {
|
||||||
|
match msg {
|
||||||
|
LogicalMessage::Begin { xid, .. } => {
|
||||||
|
self.current_xid = Some(xid);
|
||||||
|
self.buffer = Some(TxBuffer::new(xid));
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicalMessage::Commit { .. } => {
|
||||||
|
if let Some(buf) = self.buffer.take() {
|
||||||
|
self.emit(buf);
|
||||||
|
}
|
||||||
|
self.current_xid = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicalMessage::Insert { rel_oid, new_tuple } => {
|
||||||
|
self.record_change(rel_oid, &new_tuple);
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicalMessage::Update { rel_oid, new_tuple, key_tuple, .. } => {
|
||||||
|
// Prefer the key_tuple for row-level id extraction if available
|
||||||
|
// (it holds the *old* key when the PK is being changed).
|
||||||
|
let id_source = key_tuple.as_ref().unwrap_or(&new_tuple);
|
||||||
|
let id = self.extract_pk_id(rel_oid, id_source);
|
||||||
|
self.record_change_with_id(rel_oid, &new_tuple, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicalMessage::Delete { rel_oid, key_tuple, old_tuple } => {
|
||||||
|
let tuple = key_tuple.as_ref().or(old_tuple.as_ref());
|
||||||
|
let id = tuple.and_then(|t| self.extract_pk_id(rel_oid, t));
|
||||||
|
// We still need the relation name; use an empty TupleData as placeholder.
|
||||||
|
if let Some(info) = self.relation_info(rel_oid) {
|
||||||
|
let schema = info.0;
|
||||||
|
let table = info.1;
|
||||||
|
if let Some(buf) = &mut self.buffer {
|
||||||
|
buf.record_table(&schema, &table);
|
||||||
|
if let Some(id) = id {
|
||||||
|
buf.record_row(&schema, &table, id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicalMessage::Truncate { rel_oids, .. } => {
|
||||||
|
let pairs: Vec<(String, String)> = rel_oids
|
||||||
|
.iter()
|
||||||
|
.filter_map(|&oid| self.relation_info(oid))
|
||||||
|
.collect();
|
||||||
|
if let Some(buf) = &mut self.buffer {
|
||||||
|
for (schema, table) in pairs {
|
||||||
|
buf.record_table(&schema, &table);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Relation messages are consumed by the parser (internal state).
|
||||||
|
// Type and Message are not relevant for cache invalidation.
|
||||||
|
LogicalMessage::Relation(_)
|
||||||
|
| LogicalMessage::Type { .. }
|
||||||
|
| LogicalMessage::Message { .. } => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Record a table + optional row key for a DML change.
|
||||||
|
fn record_change(&mut self, rel_oid: u32, tuple: &TupleData) {
|
||||||
|
let id = self.extract_pk_id(rel_oid, tuple);
|
||||||
|
self.record_change_with_id(rel_oid, tuple, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_change_with_id(&mut self, rel_oid: u32, _tuple: &TupleData, id: Option<i64>) {
|
||||||
|
if let Some(info) = self.relation_info(rel_oid) {
|
||||||
|
let (schema, table) = info;
|
||||||
|
if let Some(buf) = &mut self.buffer {
|
||||||
|
buf.record_table(&schema, &table);
|
||||||
|
if let Some(id) = id {
|
||||||
|
buf.record_row(&schema, &table, id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve a relation OID to `(schema, table)`. Clones so the parser
|
||||||
|
/// borrow is released before we mutably borrow `self.buffer`.
|
||||||
|
fn relation_info(&self, rel_oid: u32) -> Option<(String, String)> {
|
||||||
|
self.parser
|
||||||
|
.get_relation(rel_oid)
|
||||||
|
.map(|r| (r.namespace.clone(), r.name.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try to extract a single-column integer primary key from `tuple`.
|
||||||
|
///
|
||||||
|
/// Returns `None` for composite keys, non-integer keys, or when the
|
||||||
|
/// relation has no replica-identity columns flagged (`flags & 1`).
|
||||||
|
fn extract_pk_id(&self, rel_oid: u32, tuple: &TupleData) -> Option<i64> {
|
||||||
|
let rel = self.parser.get_relation(rel_oid)?;
|
||||||
|
extract_single_int_pk(rel, tuple)
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Emission
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Publish all invalidation events for a committed transaction.
|
||||||
|
fn emit(&self, buf: TxBuffer) {
|
||||||
|
tracing::debug!(
|
||||||
|
xid = buf.xid,
|
||||||
|
tables = buf.table_keys.len(),
|
||||||
|
rows = buf.row_keys.len(),
|
||||||
|
"pg_replication: emitting cache invalidation"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Table-level events (deduplicated by HashSet).
|
||||||
|
for (schema, table) in &buf.table_keys {
|
||||||
|
self.events
|
||||||
|
.send(CacheKey::Channel {
|
||||||
|
key: format!("pg:{schema}.{table}"),
|
||||||
|
})
|
||||||
|
.ok(); // ok() = ignore when no receivers are subscribed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Row-level events.
|
||||||
|
for (schema, table, id) in &buf.row_keys {
|
||||||
|
self.events
|
||||||
|
.send(CacheKey::Channel {
|
||||||
|
key: format!("pg:{schema}.{table}:{id}"),
|
||||||
|
})
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Row-level PK extraction
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Extract an integer PK only when the relation has exactly one column
|
||||||
|
/// flagged as part of the replica identity key and its value is parseable
|
||||||
|
/// as `i64`.
|
||||||
|
fn extract_single_int_pk(rel: &RelationInfo, tuple: &TupleData) -> Option<i64> {
|
||||||
|
let key_cols: Vec<usize> = rel
|
||||||
|
.columns
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter(|(_, col)| col.flags & 1 != 0)
|
||||||
|
.map(|(i, _)| i)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if key_cols.len() != 1 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let idx = key_cols[0];
|
||||||
|
match tuple.values.get(idx)? {
|
||||||
|
ColumnValue::Text(s) => s.parse::<i64>().ok(),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_receivers_does_not_panic() {
|
||||||
|
let (tx, _rx) = broadcast::channel(16);
|
||||||
|
drop(_rx); // no receivers
|
||||||
|
let mut emitter = WalEmitter::new(tx);
|
||||||
|
// Publishing to a channel with no receivers should silently succeed.
|
||||||
|
emitter
|
||||||
|
.emit(TxBuffer { xid: 1, table_keys: Default::default(), row_keys: vec![] });
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn outside_transaction_dml_is_ignored() {
|
||||||
|
let (tx, _rx) = broadcast::channel(16);
|
||||||
|
let mut emitter = WalEmitter::new(tx);
|
||||||
|
// No Begin sent → buffer is None → Insert is silently dropped.
|
||||||
|
assert!(emitter.buffer.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
//! PostgreSQL logical replication module for livevue-rs.
|
||||||
|
//!
|
||||||
|
//! Connects to a PostgreSQL `pgoutput` logical replication stream, buffers
|
||||||
|
//! changes per transaction, and publishes [`CacheKey`] invalidation events
|
||||||
|
//! to the framework's broadcast channel on every committed transaction.
|
||||||
|
//!
|
||||||
|
//! # Quick start
|
||||||
|
//!
|
||||||
|
//! ```ignore
|
||||||
|
//! use livevue_rs::pg_replication::{PgReplicationConfig, PgReplicationListener, pg_table_key};
|
||||||
|
//!
|
||||||
|
//! let config = PgReplicationConfig {
|
||||||
|
//! host: "localhost".into(),
|
||||||
|
//! port: 5432,
|
||||||
|
//! user: "postgres".into(),
|
||||||
|
//! password: "postgres".into(),
|
||||||
|
//! database: "mydb".into(),
|
||||||
|
//! slot_name: "livevue_slot".into(),
|
||||||
|
//! publication_name: "livevue_pub".into(),
|
||||||
|
//! start_lsn: None,
|
||||||
|
//! };
|
||||||
|
//!
|
||||||
|
//! let listener = PgReplicationListener::new(config, store.clone());
|
||||||
|
//! tokio::spawn(async move {
|
||||||
|
//! if let Err(e) = listener.start().await {
|
||||||
|
//! tracing::error!("replication listener exited: {e}");
|
||||||
|
//! }
|
||||||
|
//! });
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! # CacheKey convention
|
||||||
|
//!
|
||||||
|
//! | Granularity | Channel key format | Example |
|
||||||
|
//! |-------------|------------------------------|-----------------------------|
|
||||||
|
//! | Table-level | `"pg:{schema}.{table}"` | `"pg:public.posts"` |
|
||||||
|
//! | Row-level | `"pg:{schema}.{table}:{id}"` | `"pg:public.posts:42"` |
|
||||||
|
//!
|
||||||
|
//! Use [`pg_table_key`] / [`pg_row_key`] in `cx.subscribe()` in render functions.
|
||||||
|
//!
|
||||||
|
//! # Feature flag
|
||||||
|
//!
|
||||||
|
//! Enable with `features = ["pg_replication"]` in your `Cargo.toml`.
|
||||||
|
|
||||||
|
mod wal_parser;
|
||||||
|
mod emitter;
|
||||||
|
mod proto;
|
||||||
|
|
||||||
|
pub use wal_parser::{
|
||||||
|
ColumnInfo, ColumnValue, LogicalMessage, MessageParser, RelationInfo, TupleData,
|
||||||
|
};
|
||||||
|
pub use emitter::WalEmitter;
|
||||||
|
|
||||||
|
use crate::query::CacheKey;
|
||||||
|
use crate::store::Store;
|
||||||
|
use bytes::{BufMut, Bytes, BytesMut};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Key helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// [`CacheKey`] matching table-level events from the PG replication module.
|
||||||
|
///
|
||||||
|
/// Call `cx.subscribe(pg_table_key("public", "messages"))` in a render
|
||||||
|
/// function to re-render whenever any row in `messages` changes.
|
||||||
|
pub fn pg_table_key(schema: &str, table: &str) -> CacheKey {
|
||||||
|
CacheKey::Channel { key: format!("pg:{schema}.{table}") }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`CacheKey`] matching row-level events from the PG replication module.
|
||||||
|
///
|
||||||
|
/// Row-level events are only emitted when the row has a single-column
|
||||||
|
/// integer replica-identity key.
|
||||||
|
pub fn pg_row_key(schema: &str, table: &str, id: i64) -> CacheKey {
|
||||||
|
CacheKey::Channel { key: format!("pg:{schema}.{table}:{id}") }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Configuration
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Configuration for a [`PgReplicationListener`].
|
||||||
|
pub struct PgReplicationConfig {
|
||||||
|
pub host: String,
|
||||||
|
pub port: u16,
|
||||||
|
pub user: String,
|
||||||
|
pub password: String,
|
||||||
|
pub database: String,
|
||||||
|
|
||||||
|
/// Logical replication slot name. Auto-created with the `pgoutput`
|
||||||
|
/// plugin if it does not already exist.
|
||||||
|
pub slot_name: String,
|
||||||
|
|
||||||
|
/// Publication name. Must be created in PostgreSQL first:
|
||||||
|
/// ```sql
|
||||||
|
/// CREATE PUBLICATION livevue_pub FOR ALL TABLES;
|
||||||
|
/// ```
|
||||||
|
pub publication_name: String,
|
||||||
|
|
||||||
|
/// LSN to start from. `None` = start from the slot's confirmed LSN
|
||||||
|
/// (only future changes).
|
||||||
|
pub start_lsn: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Listener
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Streams PostgreSQL WAL changes and publishes [`CacheKey`] events to the
|
||||||
|
/// framework's broadcast channel.
|
||||||
|
///
|
||||||
|
/// Spawn as a long-running background task after calling `AppState::new` and
|
||||||
|
/// `spawn_fanout`. Wrap in a retry loop for automatic reconnection.
|
||||||
|
pub struct PgReplicationListener {
|
||||||
|
config: PgReplicationConfig,
|
||||||
|
store: Arc<Store>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PgReplicationListener {
|
||||||
|
pub fn new(config: PgReplicationConfig, store: Arc<Store>) -> Self {
|
||||||
|
Self { config, store }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Connect and stream WAL until an error or stream close.
|
||||||
|
pub async fn start(self) -> anyhow::Result<()> {
|
||||||
|
use anyhow::Context as _;
|
||||||
|
|
||||||
|
// --- connect in replication mode ---
|
||||||
|
let mut conn = proto::RawPgConn::connect(
|
||||||
|
&self.config.host,
|
||||||
|
self.config.port,
|
||||||
|
&self.config.user,
|
||||||
|
&self.config.password,
|
||||||
|
&self.config.database,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("connect to PostgreSQL (replication mode)")?;
|
||||||
|
|
||||||
|
// --- ensure replication slot exists ---
|
||||||
|
let slot = &self.config.slot_name;
|
||||||
|
let create_sql = format!(
|
||||||
|
"CREATE_REPLICATION_SLOT {slot} LOGICAL pgoutput NOEXPORT_SNAPSHOT"
|
||||||
|
);
|
||||||
|
match conn.simple_query(&create_sql).await {
|
||||||
|
Ok(_) => tracing::info!(slot, "pg_replication: created replication slot"),
|
||||||
|
Err(e) => {
|
||||||
|
let msg = e.to_string();
|
||||||
|
if msg.contains("42710") || msg.contains("already exists") {
|
||||||
|
tracing::debug!(slot, "pg_replication: slot already exists");
|
||||||
|
} else {
|
||||||
|
return Err(e).context("create replication slot");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- start logical replication ---
|
||||||
|
let lsn = self.config.start_lsn.unwrap_or(0);
|
||||||
|
let lsn_str = format!("{:X}/{:X}", (lsn >> 32) as u32, lsn as u32);
|
||||||
|
let pub_name = &self.config.publication_name;
|
||||||
|
let start_sql = format!(
|
||||||
|
"START_REPLICATION SLOT {slot} LOGICAL {lsn_str} \
|
||||||
|
(proto_version '1', publication_names '{pub_name}')"
|
||||||
|
);
|
||||||
|
|
||||||
|
conn.start_replication(&start_sql)
|
||||||
|
.await
|
||||||
|
.context("START_REPLICATION")?;
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
slot, publication = pub_name, lsn = lsn_str,
|
||||||
|
"pg_replication: replication stream started"
|
||||||
|
);
|
||||||
|
|
||||||
|
// --- stream processing loop ---
|
||||||
|
let mut emitter = WalEmitter::new(self.store.events.clone());
|
||||||
|
let mut last_lsn: u64 = lsn;
|
||||||
|
let mut last_status_at = Instant::now();
|
||||||
|
const STATUS_INTERVAL: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let frame = tokio::time::timeout(
|
||||||
|
Duration::from_secs(1),
|
||||||
|
conn.read_copy_data(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
match frame {
|
||||||
|
Err(_timeout) => {}
|
||||||
|
|
||||||
|
Ok(Err(e)) => return Err(e).context("replication stream read"),
|
||||||
|
|
||||||
|
Ok(Ok(None)) => {
|
||||||
|
tracing::info!("pg_replication: replication stream ended");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Ok(Some(data))) => {
|
||||||
|
if data.is_empty() { continue; }
|
||||||
|
match data[0] {
|
||||||
|
// XLogData: 'w' + u64 wal_start(8) + u64 wal_end(8) + i64 time(8) + WAL
|
||||||
|
b'w' if data.len() >= 25 => {
|
||||||
|
last_lsn = u64::from_be_bytes(
|
||||||
|
data[9..17].try_into().expect("8 byte slice"),
|
||||||
|
);
|
||||||
|
if let Err(e) = emitter.process_raw(&data[25..]).await {
|
||||||
|
tracing::warn!("pg_replication: WAL parse warning: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// PrimaryKeepalive: 'k' + u64 wal_end(8) + i64 time(8) + u8 reply_req
|
||||||
|
b'k' if data.len() >= 18 => {
|
||||||
|
if data[17] != 0 {
|
||||||
|
conn.send_standby_status(last_lsn)
|
||||||
|
.await
|
||||||
|
.context("send standby status (keepalive reply)")?;
|
||||||
|
last_status_at = Instant::now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
tracing::debug!(
|
||||||
|
"pg_replication: unknown replication msg 0x{:02x}", other
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if last_status_at.elapsed() >= STATUS_INTERVAL {
|
||||||
|
conn.send_standby_status(last_lsn)
|
||||||
|
.await
|
||||||
|
.context("send periodic standby status")?;
|
||||||
|
last_status_at = Instant::now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Internal helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// StandbyStatusUpdate payload for acknowledging processed WAL.
|
||||||
|
pub(crate) fn standby_status_payload(lsn: u64) -> Bytes {
|
||||||
|
let ack = lsn.saturating_add(1);
|
||||||
|
let mut buf = BytesMut::with_capacity(34);
|
||||||
|
buf.put_u8(b'r');
|
||||||
|
buf.put_u64(ack); // write LSN
|
||||||
|
buf.put_u64(ack); // flush LSN
|
||||||
|
buf.put_u64(ack); // apply LSN
|
||||||
|
buf.put_i64(pg_epoch_micros());
|
||||||
|
buf.put_u8(0); // no reply requested
|
||||||
|
buf.freeze()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current time as microseconds since the PostgreSQL epoch (2000-01-01 UTC).
|
||||||
|
pub(crate) fn pg_epoch_micros() -> i64 {
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
let micros = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_micros() as i64;
|
||||||
|
micros - 946_684_800_000_000
|
||||||
|
}
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
//! Raw PostgreSQL wire-protocol connection for logical replication.
|
||||||
|
//!
|
||||||
|
//! `tokio-postgres` 0.7 does not expose a `copy_both_simple` API suitable for
|
||||||
|
//! the replication stream. This module implements the minimum subset of the
|
||||||
|
//! PostgreSQL wire protocol needed to:
|
||||||
|
//!
|
||||||
|
//! 1. Connect with `replication=database` startup parameter.
|
||||||
|
//! 2. Perform MD5 or cleartext password authentication.
|
||||||
|
//! 3. Send simple queries (for `CREATE_REPLICATION_SLOT` and `START_REPLICATION`).
|
||||||
|
//! 4. Enter `COPY BOTH` mode and stream `CopyData` frames.
|
||||||
|
//! 5. Send `StandbyStatusUpdate` CopyData frames back to the server.
|
||||||
|
//!
|
||||||
|
//! `postgres-protocol` is used for building most frontend messages and for
|
||||||
|
//! MD5 hash computation. The backend parser is bypassed for `CopyBothResponse`
|
||||||
|
//! (tag `'W'` / 0x57) which is not present in that crate's public API.
|
||||||
|
|
||||||
|
use super::standby_status_payload;
|
||||||
|
use anyhow::{bail, Context as _};
|
||||||
|
use bytes::{Buf, Bytes, BytesMut};
|
||||||
|
use postgres_protocol::authentication::md5_hash;
|
||||||
|
use postgres_protocol::message::frontend;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// RawPgConn
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// A raw PostgreSQL connection suitable for replication streaming.
|
||||||
|
pub struct RawPgConn {
|
||||||
|
stream: TcpStream,
|
||||||
|
/// Accumulates bytes read from the server until a complete frame arrives.
|
||||||
|
read_buf: BytesMut,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawPgConn {
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Constructor
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Connect to a PostgreSQL server in **logical replication** mode.
|
||||||
|
///
|
||||||
|
/// Sends a startup message with `replication=database` so the backend
|
||||||
|
/// enters walsender mode. Handles MD5 and cleartext password auth.
|
||||||
|
pub async fn connect(
|
||||||
|
host: &str,
|
||||||
|
port: u16,
|
||||||
|
user: &str,
|
||||||
|
password: &str,
|
||||||
|
database: &str,
|
||||||
|
) -> anyhow::Result<Self> {
|
||||||
|
let addr = format!("{host}:{port}");
|
||||||
|
let stream = TcpStream::connect(&addr)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("TCP connect to {addr}"))?;
|
||||||
|
|
||||||
|
let mut conn = RawPgConn {
|
||||||
|
stream,
|
||||||
|
read_buf: BytesMut::with_capacity(4096),
|
||||||
|
};
|
||||||
|
|
||||||
|
// --- Startup message -----------------------------------------------
|
||||||
|
// Parameters include `replication=database` which instructs PostgreSQL
|
||||||
|
// to accept replication commands on this connection.
|
||||||
|
let mut startup_buf = BytesMut::new();
|
||||||
|
frontend::startup_message(
|
||||||
|
[
|
||||||
|
("user", user),
|
||||||
|
("database", database),
|
||||||
|
("replication", "database"),
|
||||||
|
("application_name", "livevue-rs"),
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.copied(),
|
||||||
|
&mut startup_buf,
|
||||||
|
)?;
|
||||||
|
conn.stream.write_all(&startup_buf).await?;
|
||||||
|
conn.stream.flush().await?;
|
||||||
|
|
||||||
|
// --- Authentication loop -------------------------------------------
|
||||||
|
loop {
|
||||||
|
let (tag, body) = conn.read_frame().await?;
|
||||||
|
match tag {
|
||||||
|
b'R' => {
|
||||||
|
// Authentication request
|
||||||
|
conn.handle_auth_request(&body, user, password).await?;
|
||||||
|
}
|
||||||
|
b'K' => {
|
||||||
|
// BackendKeyData — ignore
|
||||||
|
}
|
||||||
|
b'S' => {
|
||||||
|
// ParameterStatus — ignore
|
||||||
|
}
|
||||||
|
b'Z' => {
|
||||||
|
// ReadyForQuery — handshake complete
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
b'E' => {
|
||||||
|
let msg = parse_error_message(&body);
|
||||||
|
bail!("PostgreSQL startup error: {msg}");
|
||||||
|
}
|
||||||
|
b'N' => {
|
||||||
|
// NoticeResponse — ignore
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
tracing::debug!("startup: unexpected tag 0x{:02x}", other);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Simple query (used for CREATE_REPLICATION_SLOT etc.)
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Send a simple query and drain responses until `ReadyForQuery`.
|
||||||
|
///
|
||||||
|
/// Returns `Err` if the server sends an `ErrorResponse`.
|
||||||
|
pub async fn simple_query(&mut self, query: &str) -> anyhow::Result<()> {
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
frontend::query(query, &mut buf)?;
|
||||||
|
self.stream.write_all(&buf).await?;
|
||||||
|
self.stream.flush().await?;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let (tag, body) = self.read_frame().await?;
|
||||||
|
match tag {
|
||||||
|
b'Z' => break, // ReadyForQuery
|
||||||
|
b'E' => {
|
||||||
|
let msg = parse_error_message(&body);
|
||||||
|
bail!("query error: {msg}");
|
||||||
|
}
|
||||||
|
_ => {} // CommandComplete, RowDescription, DataRow, NoticeResponse …
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Enter COPY BOTH mode for replication streaming
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Send `START_REPLICATION` and absorb the `CopyBothResponse`.
|
||||||
|
///
|
||||||
|
/// After this call the connection is in COPY BOTH mode. Use
|
||||||
|
/// [`read_copy_data`] / [`send_standby_status`] to exchange messages.
|
||||||
|
pub async fn start_replication(&mut self, query: &str) -> anyhow::Result<()> {
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
frontend::query(query, &mut buf)?;
|
||||||
|
self.stream.write_all(&buf).await?;
|
||||||
|
self.stream.flush().await?;
|
||||||
|
|
||||||
|
// Expect CopyBothResponse ('W') followed by the replication stream.
|
||||||
|
// postgres-protocol doesn't parse 'W', so we handle it manually.
|
||||||
|
loop {
|
||||||
|
let (tag, _body) = self.read_frame().await?;
|
||||||
|
match tag {
|
||||||
|
b'W' => {
|
||||||
|
// CopyBothResponse — body contains column-format info we
|
||||||
|
// don't need for the replication stream. Just break out.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
b'E' => {
|
||||||
|
let msg = parse_error_message(&_body);
|
||||||
|
bail!("START_REPLICATION error: {msg}");
|
||||||
|
}
|
||||||
|
b'N' => {} // NoticeResponse — skip
|
||||||
|
other => {
|
||||||
|
tracing::debug!("start_replication: unexpected tag 0x{:02x}", other);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Replication stream I/O
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Read one `CopyData` payload from the replication stream.
|
||||||
|
///
|
||||||
|
/// Returns:
|
||||||
|
/// - `Ok(Some(bytes))` — a CopyData payload (XLogData or PrimaryKeepalive).
|
||||||
|
/// - `Ok(None)` — the server sent `CopyDone` or `ErrorResponse` (stream end).
|
||||||
|
/// - `Err(_)` — I/O or protocol error.
|
||||||
|
pub async fn read_copy_data(&mut self) -> anyhow::Result<Option<Bytes>> {
|
||||||
|
let (tag, body) = self.read_frame().await?;
|
||||||
|
match tag {
|
||||||
|
b'd' => Ok(Some(body)),
|
||||||
|
b'c' => Ok(None), // CopyDone
|
||||||
|
b'E' => {
|
||||||
|
let msg = parse_error_message(&body);
|
||||||
|
bail!("replication error: {msg}");
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
tracing::debug!("copy stream: unexpected tag 0x{:02x}", other);
|
||||||
|
Ok(Some(Bytes::new()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Send a `StandbyStatusUpdate` CopyData frame to acknowledge `lsn`.
|
||||||
|
pub async fn send_standby_status(&mut self, lsn: u64) -> anyhow::Result<()> {
|
||||||
|
let payload = standby_status_payload(lsn);
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
frontend::CopyData::new(&payload[..])?.write(&mut buf);
|
||||||
|
self.stream.write_all(&buf).await?;
|
||||||
|
self.stream.flush().await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Low-level frame I/O
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Read one complete PostgreSQL message frame.
|
||||||
|
///
|
||||||
|
/// Frame format: `tag (u8) | length (i32, BE, includes itself) | body`.
|
||||||
|
/// Returns `(tag, body_bytes)`.
|
||||||
|
async fn read_frame(&mut self) -> anyhow::Result<(u8, Bytes)> {
|
||||||
|
// Ensure we have at least 5 bytes (1 tag + 4 length).
|
||||||
|
while self.read_buf.len() < 5 {
|
||||||
|
let n = self.stream.read_buf(&mut self.read_buf).await?;
|
||||||
|
if n == 0 {
|
||||||
|
bail!("server closed connection unexpectedly");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let tag = self.read_buf[0];
|
||||||
|
// Length field is i32 big-endian, includes the 4-length bytes but NOT
|
||||||
|
// the tag byte. Total frame length = 1 (tag) + length.
|
||||||
|
let body_len = (i32::from_be_bytes(
|
||||||
|
self.read_buf[1..5].try_into().expect("4 byte slice"),
|
||||||
|
) as usize)
|
||||||
|
.saturating_sub(4); // subtract the 4 length bytes themselves
|
||||||
|
|
||||||
|
let total = 5 + body_len; // tag(1) + length(4) + body
|
||||||
|
while self.read_buf.len() < total {
|
||||||
|
let n = self.stream.read_buf(&mut self.read_buf).await?;
|
||||||
|
if n == 0 {
|
||||||
|
bail!("server closed connection while reading message body");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.read_buf.advance(5); // consume tag + length
|
||||||
|
let body = self.read_buf.split_to(body_len).freeze();
|
||||||
|
Ok((tag, body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Authentication
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
async fn handle_auth_request(
|
||||||
|
&mut self,
|
||||||
|
body: &[u8],
|
||||||
|
user: &str,
|
||||||
|
password: &str,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
if body.len() < 4 {
|
||||||
|
bail!("AuthenticationRequest body too short");
|
||||||
|
}
|
||||||
|
let auth_type = i32::from_be_bytes(body[0..4].try_into().expect("4 bytes"));
|
||||||
|
|
||||||
|
match auth_type {
|
||||||
|
0 => {
|
||||||
|
// AuthenticationOk — nothing to do
|
||||||
|
}
|
||||||
|
3 => {
|
||||||
|
// AuthenticationCleartextPassword
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
frontend::password_message(password.as_bytes(), &mut buf)?;
|
||||||
|
self.stream.write_all(&buf).await?;
|
||||||
|
self.stream.flush().await?;
|
||||||
|
}
|
||||||
|
5 => {
|
||||||
|
// AuthenticationMD5Password — salt is bytes 4..8
|
||||||
|
if body.len() < 8 {
|
||||||
|
bail!("MD5 auth: body too short for salt");
|
||||||
|
}
|
||||||
|
let salt: [u8; 4] = body[4..8].try_into().expect("4 byte salt");
|
||||||
|
let hashed = md5_hash(user.as_bytes(), password.as_bytes(), salt);
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
frontend::password_message(hashed.as_bytes(), &mut buf)?;
|
||||||
|
self.stream.write_all(&buf).await?;
|
||||||
|
self.stream.flush().await?;
|
||||||
|
}
|
||||||
|
10 => {
|
||||||
|
// AuthenticationSASL (SCRAM-SHA-256) — not implemented here.
|
||||||
|
// For Docker dev setups, use `md5` or `trust` in pg_hba.conf.
|
||||||
|
bail!(
|
||||||
|
"SCRAM-SHA-256 authentication is not supported by the \
|
||||||
|
replication module. Configure PostgreSQL to use `md5` \
|
||||||
|
or `trust` for this user in pg_hba.conf."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
bail!("unsupported authentication type: {other}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Extract a human-readable message from an `ErrorResponse` body.
|
||||||
|
///
|
||||||
|
/// Fields are null-terminated key-value pairs. We extract the 'M' (message)
|
||||||
|
/// and 'D' (detail) fields.
|
||||||
|
fn parse_error_message(body: &[u8]) -> String {
|
||||||
|
let mut parts = Vec::new();
|
||||||
|
let mut i = 0;
|
||||||
|
while i < body.len() {
|
||||||
|
let field_type = body[i];
|
||||||
|
i += 1;
|
||||||
|
if field_type == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// Find the null terminator of the field value.
|
||||||
|
let start = i;
|
||||||
|
while i < body.len() && body[i] != 0 {
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
let value = String::from_utf8_lossy(&body[start..i]).into_owned();
|
||||||
|
i += 1; // skip null terminator
|
||||||
|
if matches!(field_type, b'M' | b'D' | b'C') {
|
||||||
|
parts.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if parts.is_empty() {
|
||||||
|
"unknown PostgreSQL error".into()
|
||||||
|
} else {
|
||||||
|
parts.join(" — ")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,505 @@
|
|||||||
|
use bytes::{Buf, BytesMut};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
/// PostgreSQL logical replication binary protocol parser (pgoutput, version 1).
|
||||||
|
///
|
||||||
|
/// Parses the raw WAL bytes delivered by `START_REPLICATION … (proto_version '1')`.
|
||||||
|
/// In protocol version 1 the transaction ID (XID) is **only** present in `Begin`
|
||||||
|
/// messages; DML messages (Insert / Update / Delete / Truncate) carry no XID.
|
||||||
|
/// The [`WalEmitter`](super::emitter::WalEmitter) tracks the current XID via the
|
||||||
|
/// `Begin`/`Commit` boundary.
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Schema metadata
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ColumnInfo {
|
||||||
|
/// Bit 0: column is part of the replica identity key.
|
||||||
|
pub flags: u8,
|
||||||
|
pub name: String,
|
||||||
|
pub type_oid: u32,
|
||||||
|
pub type_modifier: i32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct RelationInfo {
|
||||||
|
pub oid: u32,
|
||||||
|
pub namespace: String,
|
||||||
|
pub name: String,
|
||||||
|
/// `d` = default (PK), `n` = nothing, `f` = full row, `i` = index.
|
||||||
|
pub replica_identity: u8,
|
||||||
|
pub columns: Vec<ColumnInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tuple data
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum ColumnValue {
|
||||||
|
Null,
|
||||||
|
UnchangedToasted,
|
||||||
|
Text(String),
|
||||||
|
Binary(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct TupleData {
|
||||||
|
pub values: Vec<ColumnValue>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Logical messages (pgoutput v1)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum LogicalMessage {
|
||||||
|
/// Transaction start. `xid` identifies the transaction; DML messages that
|
||||||
|
/// follow do not carry their own XID.
|
||||||
|
Begin {
|
||||||
|
/// Final LSN of the transaction (equals the Commit LSN).
|
||||||
|
final_lsn: u64,
|
||||||
|
/// Commit timestamp (microseconds since PG epoch 2000-01-01).
|
||||||
|
timestamp: i64,
|
||||||
|
/// Transaction ID.
|
||||||
|
xid: u32,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Schema descriptor sent before the first DML that references a relation
|
||||||
|
/// in a session (and after schema changes). Stored internally by the parser.
|
||||||
|
Relation(RelationInfo),
|
||||||
|
|
||||||
|
/// Row inserted into `rel_oid`.
|
||||||
|
Insert {
|
||||||
|
rel_oid: u32,
|
||||||
|
new_tuple: TupleData,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Row updated in `rel_oid`.
|
||||||
|
///
|
||||||
|
/// `key_tuple` is present when REPLICA IDENTITY is the primary key
|
||||||
|
/// (default). `old_tuple` is present for REPLICA IDENTITY FULL.
|
||||||
|
Update {
|
||||||
|
rel_oid: u32,
|
||||||
|
key_tuple: Option<TupleData>,
|
||||||
|
old_tuple: Option<TupleData>,
|
||||||
|
new_tuple: TupleData,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Row deleted from `rel_oid`.
|
||||||
|
Delete {
|
||||||
|
rel_oid: u32,
|
||||||
|
key_tuple: Option<TupleData>,
|
||||||
|
old_tuple: Option<TupleData>,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Transaction committed successfully.
|
||||||
|
Commit {
|
||||||
|
flags: u8,
|
||||||
|
commit_lsn: u64,
|
||||||
|
end_lsn: u64,
|
||||||
|
timestamp: i64,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// One or more tables truncated.
|
||||||
|
Truncate {
|
||||||
|
num_relations: u32,
|
||||||
|
options: u8,
|
||||||
|
rel_oids: Vec<u32>,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Custom data-type descriptor (pgoutput emits these before first use).
|
||||||
|
Type {
|
||||||
|
type_oid: u32,
|
||||||
|
namespace: String,
|
||||||
|
type_name: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Application-level logical message emitted via `pg_logical_emit_message`.
|
||||||
|
Message {
|
||||||
|
/// Bit 0: message is transactional.
|
||||||
|
flags: u8,
|
||||||
|
lsn: u64,
|
||||||
|
prefix: String,
|
||||||
|
content: Vec<u8>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Parser
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Stateful pgoutput v1 parser.
|
||||||
|
///
|
||||||
|
/// Maintains a cache of `RelationInfo` objects keyed by OID so that
|
||||||
|
/// Insert/Update/Delete messages can be decoded without extra round-trips.
|
||||||
|
pub struct MessageParser {
|
||||||
|
relations: HashMap<u32, RelationInfo>,
|
||||||
|
_types: HashMap<u32, (String, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MessageParser {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
relations: HashMap::new(),
|
||||||
|
_types: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Public entry point
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Parse the next logical message from `buf`.
|
||||||
|
///
|
||||||
|
/// `Relation` messages are absorbed into internal state and return `None`
|
||||||
|
/// so callers only see semantically meaningful messages.
|
||||||
|
pub fn parse_message(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
if buf.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
match buf[0] as char {
|
||||||
|
'B' => self.parse_begin(buf),
|
||||||
|
'C' => self.parse_commit(buf),
|
||||||
|
'R' => {
|
||||||
|
self.parse_relation(buf)?;
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
'I' => self.parse_insert(buf),
|
||||||
|
'U' => self.parse_update(buf),
|
||||||
|
'D' => self.parse_delete(buf),
|
||||||
|
'T' => self.parse_truncate(buf),
|
||||||
|
'Y' => self.parse_type(buf),
|
||||||
|
'M' => self.parse_message_msg(buf),
|
||||||
|
other => Err(format!(
|
||||||
|
"unknown pgoutput message type: '{}' (0x{:02x})",
|
||||||
|
other, buf[0]
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up a relation by OID (returns `None` if not yet seen).
|
||||||
|
pub fn get_relation(&self, oid: u32) -> Option<&RelationInfo> {
|
||||||
|
self.relations.get(&oid)
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Low-level readers
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn read_u8(buf: &mut BytesMut) -> Result<u8, String> {
|
||||||
|
if buf.is_empty() {
|
||||||
|
return Err("unexpected EOF reading u8".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_u8())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_i16(buf: &mut BytesMut) -> Result<i16, String> {
|
||||||
|
if buf.len() < 2 {
|
||||||
|
return Err("unexpected EOF reading i16".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_i16())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_u32(buf: &mut BytesMut) -> Result<u32, String> {
|
||||||
|
if buf.len() < 4 {
|
||||||
|
return Err("unexpected EOF reading u32".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_u32())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_i32(buf: &mut BytesMut) -> Result<i32, String> {
|
||||||
|
if buf.len() < 4 {
|
||||||
|
return Err("unexpected EOF reading i32".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_i32())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_u64(buf: &mut BytesMut) -> Result<u64, String> {
|
||||||
|
if buf.len() < 8 {
|
||||||
|
return Err("unexpected EOF reading u64".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_u64())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_i64(buf: &mut BytesMut) -> Result<i64, String> {
|
||||||
|
if buf.len() < 8 {
|
||||||
|
return Err("unexpected EOF reading i64".into());
|
||||||
|
}
|
||||||
|
Ok(buf.get_i64())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a null-terminated UTF-8 string.
|
||||||
|
fn read_string(buf: &mut BytesMut) -> Result<String, String> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
loop {
|
||||||
|
if buf.is_empty() {
|
||||||
|
return Err("unexpected EOF reading null-terminated string".into());
|
||||||
|
}
|
||||||
|
let b = buf.get_u8();
|
||||||
|
if b == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bytes.push(b);
|
||||||
|
}
|
||||||
|
String::from_utf8(bytes).map_err(|e| format!("invalid UTF-8 in string: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Message parsers
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn parse_begin(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'B'
|
||||||
|
let final_lsn = Self::read_u64(buf)?;
|
||||||
|
let timestamp = Self::read_i64(buf)?;
|
||||||
|
let xid = Self::read_u32(buf)?;
|
||||||
|
Ok(Some(LogicalMessage::Begin { final_lsn, timestamp, xid }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_commit(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'C'
|
||||||
|
let flags = Self::read_u8(buf)?;
|
||||||
|
let commit_lsn = Self::read_u64(buf)?;
|
||||||
|
let end_lsn = Self::read_u64(buf)?;
|
||||||
|
let timestamp = Self::read_i64(buf)?;
|
||||||
|
Ok(Some(LogicalMessage::Commit { flags, commit_lsn, end_lsn, timestamp }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_relation(&mut self, buf: &mut BytesMut) -> Result<(), String> {
|
||||||
|
buf.get_u8(); // 'R'
|
||||||
|
let rel_oid = Self::read_u32(buf)?;
|
||||||
|
let namespace = Self::read_string(buf)?;
|
||||||
|
let name = Self::read_string(buf)?;
|
||||||
|
let replica_identity = Self::read_u8(buf)?;
|
||||||
|
let num_columns = Self::read_i16(buf)? as usize;
|
||||||
|
|
||||||
|
let mut columns = Vec::with_capacity(num_columns);
|
||||||
|
for _ in 0..num_columns {
|
||||||
|
let flags = Self::read_u8(buf)?;
|
||||||
|
let col_name = Self::read_string(buf)?;
|
||||||
|
let type_oid = Self::read_u32(buf)?;
|
||||||
|
let type_modifier = Self::read_i32(buf)?;
|
||||||
|
columns.push(ColumnInfo { flags, name: col_name, type_oid, type_modifier });
|
||||||
|
}
|
||||||
|
|
||||||
|
self.relations.insert(
|
||||||
|
rel_oid,
|
||||||
|
RelationInfo { oid: rel_oid, namespace, name, replica_identity, columns },
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_insert(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'I'
|
||||||
|
// pgoutput v1: Relation OID comes directly after type byte (no XID).
|
||||||
|
let rel_oid = Self::read_u32(buf)?;
|
||||||
|
|
||||||
|
let marker = Self::read_u8(buf)?;
|
||||||
|
if marker != b'N' {
|
||||||
|
return Err(format!("INSERT: expected 'N' marker, got '{}'", marker as char));
|
||||||
|
}
|
||||||
|
|
||||||
|
let new_tuple = self.parse_tuple_data(buf, rel_oid)?;
|
||||||
|
Ok(Some(LogicalMessage::Insert { rel_oid, new_tuple }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_update(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'U'
|
||||||
|
// pgoutput v1: no XID in Update.
|
||||||
|
let rel_oid = Self::read_u32(buf)?;
|
||||||
|
|
||||||
|
let mut key_tuple = None;
|
||||||
|
let mut old_tuple = None;
|
||||||
|
|
||||||
|
if !buf.is_empty() {
|
||||||
|
match buf[0] as char {
|
||||||
|
'K' => {
|
||||||
|
buf.get_u8();
|
||||||
|
key_tuple = Some(self.parse_tuple_data(buf, rel_oid)?);
|
||||||
|
}
|
||||||
|
'O' => {
|
||||||
|
buf.get_u8();
|
||||||
|
old_tuple = Some(self.parse_tuple_data(buf, rel_oid)?);
|
||||||
|
}
|
||||||
|
_ => {} // no optional tuple, proceed to 'N'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let marker = Self::read_u8(buf)?;
|
||||||
|
if marker != b'N' {
|
||||||
|
return Err(format!("UPDATE: expected 'N' marker, got '{}'", marker as char));
|
||||||
|
}
|
||||||
|
let new_tuple = self.parse_tuple_data(buf, rel_oid)?;
|
||||||
|
|
||||||
|
Ok(Some(LogicalMessage::Update { rel_oid, key_tuple, old_tuple, new_tuple }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_delete(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'D'
|
||||||
|
// pgoutput v1: no XID in Delete.
|
||||||
|
let rel_oid = Self::read_u32(buf)?;
|
||||||
|
|
||||||
|
let mut key_tuple = None;
|
||||||
|
let mut old_tuple = None;
|
||||||
|
|
||||||
|
if !buf.is_empty() {
|
||||||
|
match buf[0] as char {
|
||||||
|
'K' => {
|
||||||
|
buf.get_u8();
|
||||||
|
key_tuple = Some(self.parse_tuple_data(buf, rel_oid)?);
|
||||||
|
}
|
||||||
|
'O' => {
|
||||||
|
buf.get_u8();
|
||||||
|
old_tuple = Some(self.parse_tuple_data(buf, rel_oid)?);
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
return Err(format!("DELETE: expected 'K' or 'O', got '{}'", other));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(LogicalMessage::Delete { rel_oid, key_tuple, old_tuple }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_truncate(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'T'
|
||||||
|
// pgoutput v1: no XID in Truncate.
|
||||||
|
let num_relations = Self::read_u32(buf)?;
|
||||||
|
let options = Self::read_u8(buf)?;
|
||||||
|
|
||||||
|
let mut rel_oids = Vec::with_capacity(num_relations as usize);
|
||||||
|
for _ in 0..num_relations {
|
||||||
|
rel_oids.push(Self::read_u32(buf)?);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(LogicalMessage::Truncate { num_relations, options, rel_oids }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_type(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'Y'
|
||||||
|
// pgoutput v1: no XID in Type.
|
||||||
|
let type_oid = Self::read_u32(buf)?;
|
||||||
|
let namespace = Self::read_string(buf)?;
|
||||||
|
let type_name = Self::read_string(buf)?;
|
||||||
|
|
||||||
|
self._types.insert(type_oid, (namespace.clone(), type_name.clone()));
|
||||||
|
|
||||||
|
Ok(Some(LogicalMessage::Type { type_oid, namespace, type_name }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_message_msg(&mut self, buf: &mut BytesMut) -> Result<Option<LogicalMessage>, String> {
|
||||||
|
buf.get_u8(); // 'M'
|
||||||
|
// pgoutput v1: no XID in Message.
|
||||||
|
let flags = Self::read_u8(buf)?;
|
||||||
|
let lsn = Self::read_u64(buf)?;
|
||||||
|
let prefix = Self::read_string(buf)?;
|
||||||
|
let content_len = Self::read_i32(buf)? as usize;
|
||||||
|
|
||||||
|
if buf.len() < content_len {
|
||||||
|
return Err(format!(
|
||||||
|
"Message: need {} bytes for content, have {}",
|
||||||
|
content_len,
|
||||||
|
buf.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let content = buf.split_to(content_len).to_vec();
|
||||||
|
|
||||||
|
Ok(Some(LogicalMessage::Message { flags, lsn, prefix, content }))
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Tuple decoder
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn parse_tuple_data(&self, buf: &mut BytesMut, rel_oid: u32) -> Result<TupleData, String> {
|
||||||
|
let num_columns = Self::read_i16(buf)? as usize;
|
||||||
|
|
||||||
|
let relation = self
|
||||||
|
.relations
|
||||||
|
.get(&rel_oid)
|
||||||
|
.ok_or_else(|| format!("unknown relation OID {rel_oid} — missing Relation message?"))?;
|
||||||
|
|
||||||
|
if num_columns != relation.columns.len() {
|
||||||
|
return Err(format!(
|
||||||
|
"tuple has {} columns but relation '{}' has {}",
|
||||||
|
num_columns,
|
||||||
|
relation.name,
|
||||||
|
relation.columns.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut values = Vec::with_capacity(num_columns);
|
||||||
|
for _ in 0..num_columns {
|
||||||
|
let marker = Self::read_u8(buf)?;
|
||||||
|
let value = match marker {
|
||||||
|
b'n' => ColumnValue::Null,
|
||||||
|
b'u' => ColumnValue::UnchangedToasted,
|
||||||
|
b't' => {
|
||||||
|
let len = Self::read_i32(buf)? as usize;
|
||||||
|
if buf.len() < len {
|
||||||
|
return Err(format!(
|
||||||
|
"text column: need {} bytes, have {}",
|
||||||
|
len,
|
||||||
|
buf.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let data = buf.split_to(len).to_vec();
|
||||||
|
let text = String::from_utf8(data)
|
||||||
|
.map_err(|e| format!("non-UTF-8 text column: {e}"))?;
|
||||||
|
ColumnValue::Text(text)
|
||||||
|
}
|
||||||
|
b'b' => {
|
||||||
|
let len = Self::read_i32(buf)? as usize;
|
||||||
|
if buf.len() < len {
|
||||||
|
return Err(format!(
|
||||||
|
"binary column: need {} bytes, have {}",
|
||||||
|
len,
|
||||||
|
buf.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
ColumnValue::Binary(buf.split_to(len).to_vec())
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
return Err(format!(
|
||||||
|
"unknown column marker '{}'",
|
||||||
|
other as char
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
values.push(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(TupleData { values })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for MessageParser {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parser_starts_empty() {
|
||||||
|
let p = MessageParser::new();
|
||||||
|
assert!(p.relations.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_buf_returns_none() {
|
||||||
|
let mut p = MessageParser::new();
|
||||||
|
let mut buf = BytesMut::new();
|
||||||
|
assert!(p.parse_message(&mut buf).unwrap().is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user