feat(1168): Track subscription query duration for each reducer

Closes #1168.

crates/core/src/host/wasm_common/module_host_actor.rs

@@ -27,6 +27,7 @@ use crate::module_host_context::ModuleCreationContext;
 use crate::sql;
 use crate::subscription::module_subscription_actor::ModuleSubscriptions;
 use crate::util::const_unwrap;
+use crate::util::prometheus_handle::HistogramExt;
 use crate::worker_metrics::WORKER_METRICS;
 use spacetimedb_sats::db::def::TableDef;
 
@@ -535,6 +536,11 @@ impl<T: WasmInstance> WasmModuleInstance<T> {
         };
 
         let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable));
+        let _guard = WORKER_METRICS
+            .reducer_plus_query_duration
+            .with_label_values(&address, op.name)
+            .with_timer(tx.timer);
+
         let mut tx_slot = self.instance.instance_env().tx.clone();
 
         let reducer_span = tracing::trace_span!(

crates/core/src/util/prometheus_handle.rs

@@ -1,4 +1,6 @@
-use prometheus::IntGauge;
+use std::time::Instant;
+
+use prometheus::{Histogram, IntGauge};
 
 /// Decrements the inner [`IntGauge`] on drop.
 pub struct GaugeInc {
@@ -27,3 +29,26 @@ impl IntGaugeExt for IntGauge {
         inc_scope(self)
     }
 }
+
+/// A scope guard for a timer,
+/// the total duration of which is written to a Histogram metric on drop.
+pub struct TimerGuard {
+    histogram: Histogram,
+    timer: Instant,
+}
+
+impl Drop for TimerGuard {
+    fn drop(&mut self) {
+        self.histogram.observe(self.timer.elapsed().as_secs_f64());
+    }
+}
+
+pub trait HistogramExt {
+    fn with_timer(self, timer: Instant) -> TimerGuard;
+}
+
+impl HistogramExt for Histogram {
+    fn with_timer(self, timer: Instant) -> TimerGuard {
+        TimerGuard { histogram: self, timer }
+    }
+}

crates/core/src/worker_metrics/mod.rs

@@ -70,6 +70,11 @@ metrics_group!(
         #[help = "The total time it takes for request to complete"]
         #[labels(txn_type: WorkloadType, database_address: Address, reducer_symbol: str)]
         pub request_round_trip: HistogramVec,
+
+        #[name = spacetime_reducer_plus_query_duration_sec]
+        #[help = "The time spent executing a reducer (in seconds), plus the time spent evaluating its subscription queries"]
+        #[labels(db: Address, reducer: str)]
+        pub reducer_plus_query_duration: HistogramVec,
     }
 );