dump some extra files
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
/// Example integration of pgwire-replication with cache invalidation
|
||||
///
|
||||
/// This shows how to:
|
||||
/// 1. Connect to PostgreSQL logical replication stream
|
||||
/// 2. Parse logical messages
|
||||
/// 3. Buffer changes per transaction
|
||||
/// 4. Emit all changes atomically at commit via Tokio broadcast
|
||||
/// 5. Subscribe to events for client notification
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
use std::time::Duration;
|
||||
|
||||
// These would be your actual modules
|
||||
mod pg_wal_parser;
|
||||
mod cache_invalidation;
|
||||
|
||||
use cache_invalidation::{CacheInvalidationEvent, CacheInvalidationEmitter};
|
||||
|
||||
/// Example configuration
|
||||
pub struct ReplicationConfig {
|
||||
pub pg_host: String,
|
||||
pub pg_port: u16,
|
||||
pub pg_user: String,
|
||||
pub pg_password: String,
|
||||
pub pg_database: String,
|
||||
pub replication_slot: String,
|
||||
pub publication: String,
|
||||
pub start_lsn: String, // e.g. "0/0" or "0/16B6C50"
|
||||
}
|
||||
|
||||
/// Main replication stream handler
|
||||
pub struct WalStreamListener {
|
||||
config: ReplicationConfig,
|
||||
tx: broadcast::Sender<CacheInvalidationEvent>,
|
||||
}
|
||||
|
||||
impl WalStreamListener {
|
||||
pub fn new(
|
||||
config: ReplicationConfig,
|
||||
tx: broadcast::Sender<CacheInvalidationEvent>,
|
||||
) -> Self {
|
||||
Self { config, tx }
|
||||
}
|
||||
|
||||
/// Start the replication stream listener
|
||||
/// In production, this would use pgwire-replication::ReplicationClient
|
||||
pub async fn start(self) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// This is a sketch - actual implementation would use pgwire-replication
|
||||
|
||||
// 1. Create replication client
|
||||
// let mut client = pgwire_replication::ReplicationClient::connect(
|
||||
// pgwire_replication::ReplicationConfig {
|
||||
// host: self.config.pg_host.into(),
|
||||
// port: self.config.pg_port,
|
||||
// user: self.config.pg_user.into(),
|
||||
// password: self.config.pg_password.into(),
|
||||
// database: self.config.pg_database.into(),
|
||||
// slot: self.config.replication_slot.into(),
|
||||
// publication: self.config.publication.into(),
|
||||
// start_lsn: pgwire_replication::Lsn::parse(&self.config.start_lsn)?,
|
||||
// status_interval: Duration::from_secs(10),
|
||||
// idle_wakeup_interval: Duration::from_secs(10),
|
||||
// ..Default::default()
|
||||
// },
|
||||
// ).await?;
|
||||
|
||||
// 2. Create emitter with transaction buffering
|
||||
let mut emitter = CacheInvalidationEmitter::new(self.tx);
|
||||
|
||||
// 3. Process stream
|
||||
// loop {
|
||||
// match client.recv().await? {
|
||||
// Some(pgwire_replication::ReplicationEvent::XLogData { data, wal_end, .. }) => {
|
||||
// // Parse and buffer changes per transaction
|
||||
// emitter.process_raw(&data).await?;
|
||||
//
|
||||
// // Update LSN checkpoint after processing
|
||||
// client.update_applied_lsn(wal_end);
|
||||
// }
|
||||
// Some(pgwire_replication::ReplicationEvent::KeepAlive { .. }) => {
|
||||
// // Server keepalive, continue
|
||||
// }
|
||||
// Some(pgwire_replication::ReplicationEvent::StoppedAt { .. }) => {
|
||||
// // Stream ended gracefully
|
||||
// break;
|
||||
// }
|
||||
// Some(_) => {}
|
||||
// None => break,
|
||||
// }
|
||||
// }
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Example client that subscribes to cache invalidation events
|
||||
/// All changes in a transaction are emitted atomically on commit
|
||||
pub async fn example_subscriber(
|
||||
mut rx: broadcast::Receiver<CacheInvalidationEvent>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(event) => {
|
||||
println!(
|
||||
"Transaction committed: XID={} LSN={:016X}-{:016X} timestamp={} changes={}",
|
||||
event.xid, event.commit_lsn, event.end_lsn, event.timestamp, event.changes.len()
|
||||
);
|
||||
|
||||
// All changes in this transaction are now available atomically
|
||||
for change in &event.changes {
|
||||
match &change.operation {
|
||||
cache_invalidation::Operation::Insert => {
|
||||
println!(" INSERT {}.{}", change.schema, change.table);
|
||||
}
|
||||
cache_invalidation::Operation::Update => {
|
||||
println!(" UPDATE {}.{}", change.schema, change.table);
|
||||
}
|
||||
cache_invalidation::Operation::Delete => {
|
||||
println!(" DELETE {}.{}", change.schema, change.table);
|
||||
}
|
||||
cache_invalidation::Operation::Truncate => {
|
||||
println!(" TRUNCATE {}.{}", change.schema, change.table);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(row) = &change.row_data {
|
||||
if let Some(keys) = &row.key_values {
|
||||
print!(" Keys: ");
|
||||
for (k, v) in keys {
|
||||
print!("{}={:?} ", k, v);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// In a real system, you would:
|
||||
// 1. Invalidate all cache entries for this transaction atomically
|
||||
// 2. Notify WebSocket clients (single transaction message)
|
||||
// 3. Store event for audit/replication trail
|
||||
// 4. Update Prometheus metrics
|
||||
// 5. Persist LSN for crash recovery
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||
println!("Warning: Cache invalidation queue lagged, reloading cache from DB");
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
println!("Cache invalidation stream closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Example showing transaction atomicity
|
||||
/// Demonstrates that a multi-table transaction is delivered as one event
|
||||
async fn example_transaction_atomicity() {
|
||||
println!("Example: Multi-table transaction");
|
||||
println!();
|
||||
println!("BEGIN");
|
||||
println!(" INSERT INTO users (id, name) VALUES (1, 'alice')");
|
||||
println!(" INSERT INTO posts (id, user_id, title) VALUES (100, 1, 'Hello')");
|
||||
println!(" INSERT INTO comments (id, post_id, text) VALUES (1000, 100, 'Nice!')");
|
||||
println!("COMMIT LSN=0/1234567 XID=5000");
|
||||
println!();
|
||||
println!("Result: Single CacheInvalidationEvent with 3 changes");
|
||||
println!(" - Change[0]: users.INSERT");
|
||||
println!(" - Change[1]: posts.INSERT");
|
||||
println!(" - Change[2]: comments.INSERT");
|
||||
println!();
|
||||
println!("Subscribers receive all 3 changes atomically,");
|
||||
println!("so they can update cache without partial state.");
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// In a real application, you might:
|
||||
|
||||
// 1. Create broadcast channel for cache events (large capacity for bursty writes)
|
||||
let (tx, rx) = broadcast::channel::<CacheInvalidationEvent>(10000);
|
||||
|
||||
// 2. Spawn WAL stream listener
|
||||
let config = ReplicationConfig {
|
||||
pg_host: "127.0.0.1".to_string(),
|
||||
pg_port: 5432,
|
||||
pg_user: "postgres".to_string(),
|
||||
pg_password: "postgres".to_string(),
|
||||
pg_database: "mydb".to_string(),
|
||||
replication_slot: "my_slot".to_string(),
|
||||
publication: "my_publication".to_string(),
|
||||
start_lsn: "0/0".to_string(),
|
||||
};
|
||||
|
||||
let listener = WalStreamListener::new(config, tx);
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = listener.start().await {
|
||||
eprintln!("Replication stream error: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Spawn multiple subscribers (e.g., for different cache layers)
|
||||
let rx_cache = tx.subscribe();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = example_subscriber(rx_cache).await {
|
||||
eprintln!("Cache subscriber error: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
let rx_analytics = tx.subscribe();
|
||||
tokio::spawn(async move {
|
||||
// Another subscriber could log for analytics
|
||||
let mut rx = rx_analytics;
|
||||
loop {
|
||||
if let Ok(event) = rx.recv().await {
|
||||
// Track metrics, e.g., changes per table per second
|
||||
eprintln!("Metric: {} changes in XID {}", event.changes.len(), event.xid);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 4. Your application continues in the background
|
||||
// The cache invalidation events flow continuously
|
||||
|
||||
// Keep main alive
|
||||
example_transaction_atomicity().await;
|
||||
tokio::signal::ctrl_c().await?;
|
||||
println!("Shutting down");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_broadcast_channel() {
|
||||
let (tx, mut rx) = broadcast::channel::<CacheInvalidationEvent>(10);
|
||||
|
||||
let event = CacheInvalidationEvent {
|
||||
xid: 1000,
|
||||
commit_lsn: 0x123456,
|
||||
end_lsn: 0x654321,
|
||||
timestamp: 1_000_000,
|
||||
changes: vec![],
|
||||
};
|
||||
|
||||
tx.send(event.clone()).unwrap();
|
||||
|
||||
let received = rx.recv().await.unwrap();
|
||||
assert_eq!(received.xid, 1000);
|
||||
assert_eq!(received.changes.len(), 0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user