From 1e1666fc0ba42f434474f57c9ee0f5dd44648b54 Mon Sep 17 00:00:00 2001 From: hshafqat-art Date: Thu, 20 Aug 2026 13:43:21 -0400 Subject: [PATCH] Adds a shared sql_utilities::ast_matching::pattern_rewrites module that detects ClickHouse-specific complex SQL shapes (CTE/window functions, tokenized subqueries, MOAS, multi-aggregate) and rewrites them into simplified queries. Both the planner and the query engine call into this implementation. Also adds ComputedLabelConfig/StatefulTransitionConfig as shared asap_types, wires ingest-time computed-label and stateful-transition execution, and adds the 200-query bgp_jan2024_rrc00 analyst workload. --- .../rs/asap_types/src/computed_label.rs | 37 + .../dependencies/rs/asap_types/src/lib.rs | 4 + .../rs/asap_types/src/stateful_transition.rs | 48 + .../rs/asap_types/src/streaming_config.rs | 58 +- .../rs/sql_utilities/src/ast_matching/mod.rs | 2 + .../src/ast_matching/pattern_rewrites.rs | 929 ++++++ .../src/ast_matching/sqlhelper.rs | 40 + .../src/ast_matching/sqlpattern_matcher.rs | 8 + .../src/ast_matching/sqlpattern_parser.rs | 491 ++- asap-planner-rs/src/planner/sql.rs | 865 +++++- asap-planner-rs/src/sql/generator.rs | 179 +- .../src/bin/precompute_engine.rs | 20 +- .../drivers/query/adapters/clickhouse_http.rs | 26 +- .../src/drivers/query/servers/http.rs | 14 +- asap-query-engine/src/engine_config.rs | 10 +- .../src/engines/simple_engine/sql.rs | 490 ++- asap-query-engine/src/main.rs | 61 +- .../precompute_engine/accumulator_factory.rs | 113 +- .../src/precompute_engine/computed_labels.rs | 84 + .../src/precompute_engine/csv_ingest.rs | 327 +- .../src/precompute_engine/ingest_source.rs | 216 +- .../src/precompute_engine/mod.rs | 3 + .../precompute_engine/stateful_transition.rs | 184 ++ .../src/precompute_engine/worker.rs | 58 +- .../bgp_jan2024_rrc00_200_query_workload.yaml | 2686 +++++++++++++++++ 25 files changed, 6777 insertions(+), 176 deletions(-) create mode 100644 asap-common/dependencies/rs/asap_types/src/computed_label.rs create mode 100644 asap-common/dependencies/rs/asap_types/src/stateful_transition.rs create mode 100644 asap-common/dependencies/rs/sql_utilities/src/ast_matching/pattern_rewrites.rs create mode 100644 asap-query-engine/src/precompute_engine/computed_labels.rs create mode 100644 asap-query-engine/src/precompute_engine/stateful_transition.rs create mode 100644 local_experiments/bgp_jan2024_rrc00_200_query_workload.yaml diff --git a/asap-common/dependencies/rs/asap_types/src/computed_label.rs b/asap-common/dependencies/rs/asap_types/src/computed_label.rs new file mode 100644 index 00000000..5a0f3d44 --- /dev/null +++ b/asap-common/dependencies/rs/asap_types/src/computed_label.rs @@ -0,0 +1,37 @@ +use serde::{Deserialize, Serialize}; + +/// Config for a computed label: a metadata column derived from another raw +/// column at ingest time rather than read directly off the row, e.g. +/// extracting the origin ASN (`select: last`) or every ASN (`select: all`, +/// via `type: token_explode`) out of a space-separated `as_path` string. +/// +/// Shared between asap-planner-rs (which detects a nested-subquery SQL shape +/// like `arrayFilter(...) AS as_path_array ... as_path_array[-1]` and emits +/// this config automatically) and asap-query-engine (which reads it back to +/// actually compute the label at ingest time in +/// precompute_engine::computed_labels::compute_label_values). Previously the +/// engine owned this type privately and nothing on the planner side could +/// construct it - a human had to notice the pattern and hand-write it. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(default, deny_unknown_fields)] +pub struct ComputedLabelConfig { + pub r#type: String, + pub source_col: String, + pub tokenizer: Option, + pub filter_regex: Option, + pub select: Option, + pub on_missing: Option, +} + +impl Default for ComputedLabelConfig { + fn default() -> Self { + Self { + r#type: "field_alias".to_string(), + source_col: String::new(), + tokenizer: None, + filter_regex: None, + select: None, + on_missing: None, + } + } +} diff --git a/asap-common/dependencies/rs/asap_types/src/lib.rs b/asap-common/dependencies/rs/asap_types/src/lib.rs index 2ae3471e..0bdea40c 100644 --- a/asap-common/dependencies/rs/asap_types/src/lib.rs +++ b/asap-common/dependencies/rs/asap_types/src/lib.rs @@ -5,7 +5,9 @@ pub mod enums; pub mod inference_config; pub mod promql_schema; pub mod query_config; +pub mod computed_label; pub mod query_requirements; +pub mod stateful_transition; pub mod streaming_config; pub mod traits; pub mod utils; @@ -13,9 +15,11 @@ pub mod utils; pub use aggregation_config::*; pub use aggregation_reference::*; pub use capability_matching::find_compatible_aggregation; +pub use computed_label::*; pub use enums::*; pub use inference_config::*; pub use promql_schema::*; pub use query_config::*; pub use query_requirements::*; +pub use stateful_transition::*; pub use streaming_config::*; diff --git a/asap-common/dependencies/rs/asap_types/src/stateful_transition.rs b/asap-common/dependencies/rs/asap_types/src/stateful_transition.rs new file mode 100644 index 00000000..b2b60d54 --- /dev/null +++ b/asap-common/dependencies/rs/asap_types/src/stateful_transition.rs @@ -0,0 +1,48 @@ +use serde::{Deserialize, Serialize}; + +/// Config for a stateful transition operator: remembers the last value of +/// `state_column` per `partition_by` key and emits a derived event into +/// `metric_name` when `predicate` (comparing the remembered previous value +/// to the current row) holds. This is how patterns like ClickHouse's +/// `lagInFrame(col) OVER (PARTITION BY ... ORDER BY timestamp)` get lowered +/// into something the precompute engine's ordinary aggregators can count, +/// without the planner or query engine ever needing to understand window +/// functions. +/// +/// Shared between asap-planner-rs (which detects the SQL pattern and emits +/// this config into streaming_config.yaml) and asap-query-engine (which +/// reads it back out to drive the ingest-time operator). Previously these +/// were two disconnected things: the engine had this struct privately and +/// nothing ever auto-populated it, so it had to be hand-written per query. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +#[serde(default, deny_unknown_fields)] +pub struct StatefulTransitionConfig { + /// Generated metric name for the derived event stream. + /// Example: derived_lag_transition_path_changes + pub metric_name: String, + + /// Columns that define independent state machines. + /// Example: [prefix, collector, peer_ip] + pub partition_by: Vec, + + /// Column whose previous value is remembered. + /// Example: as_path + pub state_column: String, + + /// Alias used by the SQL query for the previous value. + /// Example: previous_path + pub previous_alias: String, + + /// Raw predicate from countIf(...). + /// V0 supports AND of simple comparisons: + /// previous_alias != '' + /// previous_alias != state_column + /// previous_alias = 'literal' + /// previous_alias != 'literal' + pub predicate: String, + + /// Labels to put on emitted derived samples. + /// Usually this is the outer GROUP BY list. + /// Empty means global aggregate. + pub emit_labels: Vec, +} diff --git a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs index 6833f81d..1ddddff9 100644 --- a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs @@ -8,19 +8,59 @@ use std::ops::Index; use crate::aggregation_config::{AggregationConfig, AggregationIdInfo}; use crate::capability_matching::find_compatible_aggregation as common_find_compatible; +use crate::computed_label::ComputedLabelConfig; use crate::enums::QueryLanguage; use crate::inference_config::{InferenceConfig, SchemaConfig}; use crate::query_requirements::QueryRequirements; +use crate::stateful_transition::StatefulTransitionConfig; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StreamingConfig { pub aggregation_configs: HashMap, + /// Stateful-transition operators (e.g. lagInFrame-derived path-change + /// streams) the planner detected while building this config. The + /// precompute engine's CSV/HTTP ingest reads these to know what derived + /// event streams it needs to maintain before any aggregation in + /// `aggregation_configs` can be satisfied against them. + #[serde(default)] + pub stateful_transitions: Vec, + /// Computed labels (e.g. origin-ASN extraction from as_path) the planner + /// detected while building this config. Keyed by the label name the rest + /// of the plan (grouping labels, spatial filters) refers to as if it were + /// an ordinary column. + #[serde(default)] + pub computed_label_cols: HashMap, } impl StreamingConfig { pub fn new(aggregation_configs: HashMap) -> Self { Self { aggregation_configs, + stateful_transitions: Vec::new(), + computed_label_cols: HashMap::new(), + } + } + + pub fn with_stateful_transitions( + aggregation_configs: HashMap, + stateful_transitions: Vec, + ) -> Self { + Self { + aggregation_configs, + stateful_transitions, + computed_label_cols: HashMap::new(), + } + } + + pub fn with_extras( + aggregation_configs: HashMap, + stateful_transitions: Vec, + computed_label_cols: HashMap, + ) -> Self { + Self { + aggregation_configs, + stateful_transitions, + computed_label_cols, } } @@ -101,7 +141,23 @@ impl StreamingConfig { } } - Ok(Self::new(aggregation_configs)) + let stateful_transitions: Vec = data + .get("stateful_transitions") + .map(|v| serde_yaml::from_value(v.clone())) + .transpose()? + .unwrap_or_default(); + + let computed_label_cols: HashMap = data + .get("computed_label_cols") + .map(|v| serde_yaml::from_value(v.clone())) + .transpose()? + .unwrap_or_default(); + + Ok(Self::with_extras( + aggregation_configs, + stateful_transitions, + computed_label_cols, + )) } } diff --git a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs index 78206679..5d6026c4 100644 --- a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs +++ b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs @@ -1,8 +1,10 @@ +pub mod pattern_rewrites; pub mod sqlhelper; pub mod sqlparser_test; pub mod sqlpattern_matcher; pub mod sqlpattern_parser; +pub use pattern_rewrites::*; pub use sqlhelper::{detect_sql_topk, SQLSchema, SqlTopk, Table, TopkWeighting}; pub use sqlpattern_matcher::*; pub use sqlpattern_parser::*; diff --git a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/pattern_rewrites.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/pattern_rewrites.rs new file mode 100644 index 00000000..6c273cf1 --- /dev/null +++ b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/pattern_rewrites.rs @@ -0,0 +1,929 @@ +//! Shared "complex SQL shape -> simplified surrogate query" detectors. +//! +//! ASAP's planner and query engine both need to recognize the same +//! ClickHouse-specific SQL shapes (CTE + window function, nested subqueries +//! with array functions) and rewrite them into a plain query the classic +//! SQLPatternParser/SQLPatternMatcher machinery already understands - the +//! planner does this once, offline, to decide what to build; the query +//! engine does it on every incoming request, to know what to serve. Before +//! this module existed, each pattern's detection logic was either +//! duplicated across both crates (risking silent drift - this is exactly +//! how a `<=`-vs-`<` truncation bug ended up fixed in one copy and not the +//! other) or only implemented on one side, leaving the other unable to +//! recognize the pattern at all. This is the single implementation both +//! crates call. +//! +//! Three patterns, three building blocks: +//! - lag-transition: `lagInFrame(col) OVER (PARTITION BY ... ORDER BY ...)` +//! wrapped in a CTE with an outer countIf -> a derived event stream a +//! stateful-transition operator maintains at ingest time. +//! - token-select: a nested subquery tokenizing a column with +//! `arrayFilter(x -> match(x, regex), splitByWhitespace(col))`, then +//! indexing the last token (`[-1]`) -> a computed label (`token_select`). +//! - token-explode: the same tokenizer, but every token becomes its own +//! row via `arrayJoin(...)` instead of indexing one -> a computed label +//! (`token_explode`). + +/// Finds the index of the `)` matching the `(` at `open_idx`, accounting for +/// nesting. +pub fn find_matching_close_paren(s: &str, open_idx: usize) -> Option { + let bytes = s.as_bytes(); + if bytes.get(open_idx) != Some(&b'(') { + return None; + } + let mut depth = 0i32; + for (i, &b) in bytes.iter().enumerate().skip(open_idx) { + match b { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + } + None +} + +fn extract_paren_arg_after<'a>(sql: &'a str, marker_lower: &str) -> Option<&'a str> { + let lower = sql.to_lowercase(); + let marker_idx = lower.find(marker_lower)?; + let open_idx = marker_idx + marker_lower.len() - 1; // marker ends in "(" + let close_idx = find_matching_close_paren(sql, open_idx)?; + Some(sql[open_idx + 1..close_idx].trim()) +} + +/// Exact-operator-match timestamp bound extraction: a naive +/// `starts_with("<")` also matches "<=", silently truncating an inclusive +/// bound to an exclusive one. +pub fn extract_ts_bound(sql: &str, op: &str) -> Option { + let lower = sql.to_lowercase(); + let mut search_start = 0usize; + loop { + let rel_idx = lower[search_start..].find("timestamp")?; + let idx = search_start + rel_idx; + let after = lower[idx + "timestamp".len()..].trim_start(); + let exact = after.starts_with(op) && !after[op.len()..].starts_with('='); + if exact { + let after_original = &sql[idx..]; + let q1 = after_original.find('\'')?; + let rest = &after_original[q1 + 1..]; + let q2 = rest.find('\'')?; + return Some(rest[..q2].to_string()); + } + search_start = idx + "timestamp".len(); + } +} + +// --------------------------------------------------------------------------- +// Lag-transition pattern +// --------------------------------------------------------------------------- + +pub fn looks_like_lag_transition_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("laginframe(") + && q.contains("partition by") + && q.contains("countif(") + && q.contains("group by") +} + +fn extract_partition_by(sql: &str) -> Option> { + let lower = sql.to_lowercase(); + let start = lower.find("partition by")? + "partition by".len(); + let end_rel = lower[start..].find("order by")?; + let cols = &sql[start..start + end_rel]; + let out: Vec = cols + .split(',') + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty()) + .collect(); + if out.is_empty() { + None + } else { + Some(out) + } +} + +fn extract_laginframe_state_column(sql: &str) -> Option { + let arg = extract_paren_arg_after(sql, "laginframe(")?; + // lagInFrame(col) or lagInFrame(col, offset, default) - only the + // single-column, default-offset form is supported (matches the + // detection guard: offset isn't checked, so only bare `col` is safe). + let col = arg.split(',').next()?.trim(); + if col.is_empty() { + None + } else { + Some(col.to_string()) + } +} + +/// The alias immediately after the `OVER (...)` clause closes, e.g. +/// `lagInFrame(as_path) OVER (...) AS previous_path` -> "previous_path". +fn extract_over_alias(sql: &str) -> Option { + let lower = sql.to_lowercase(); + let over_idx = lower.find("over")?; + let paren_rel = lower[over_idx..].find('(')?; + let open_idx = over_idx + paren_rel; + let close_idx = find_matching_close_paren(sql, open_idx)?; + let after = &sql[close_idx + 1..]; + let after_lower = after.to_lowercase(); + let as_idx = after_lower.find("as ")?; + if !after[..as_idx].trim().is_empty() { + return None; + } + let rest = after[as_idx + 3..].trim_start(); + let alias: String = rest + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if alias.is_empty() { + None + } else { + Some(alias) + } +} + +/// The raw countIf(...) predicate in the OUTER query, e.g. +/// `countIf(previous_path != '' AND previous_path != as_path) AS path_changes` +/// -> "previous_path != '' AND previous_path != as_path". +fn extract_outer_countif_predicate(sql: &str) -> Option { + let lower = sql.to_lowercase(); + let idx = lower.rfind("countif(")?; + let open_idx = idx + "countif(".len() - 1; + let close_idx = find_matching_close_paren(sql, open_idx)?; + Some(sql[open_idx + 1..close_idx].trim().to_string()) +} + +#[derive(Debug, Clone)] +pub struct LagTransitionMatch { + pub partition_by: Vec, + pub state_column: String, + pub previous_alias: String, + pub predicate: String, + pub group_label: String, + pub alias: String, + pub start: String, + pub end: String, + pub limit: String, +} + +impl LagTransitionMatch { + pub fn derived_metric(&self) -> String { + format!("derived_lag_transition_{}", self.alias) + } +} + +pub fn parse_lag_transition_query(query: &str) -> Option { + let partition_by = extract_partition_by(query)?; + let state_column = extract_laginframe_state_column(query)?; + let previous_alias = extract_over_alias(query)?; + let predicate = extract_outer_countif_predicate(query)?; + + let lower = query.to_lowercase(); + let group_idx = lower.rfind("group by")? + "group by".len(); + let group_label = query[group_idx..] + .split_whitespace() + .next()? + .trim() + .trim_end_matches(',') + .to_string(); + + let countif_idx = lower.rfind("countif(")?; + let open_idx = countif_idx + "countif(".len() - 1; + let close_idx = find_matching_close_paren(query, open_idx)?; + let after_countif = &query[close_idx + 1..]; + let after_countif_lower = after_countif.to_lowercase(); + let as_idx = after_countif_lower.find(" as ")?; + let alias_part = &after_countif[as_idx + 4..]; + let alias = alias_part + .split(|c: char| c.is_whitespace() || c == ',' || c == '\n') + .find(|s| !s.trim().is_empty())? + .trim() + .to_string(); + + let start = extract_ts_bound(query, ">=")?; + let end = extract_ts_bound(query, "<")?; + + let limit = lower + .rfind("limit") + .map(|i| &query[i + "limit".len()..]) + .and_then(|s| s.split_whitespace().next()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "100".to_string()); + + Some(LagTransitionMatch { + partition_by, + state_column, + previous_alias, + predicate, + group_label, + alias, + start, + end, + limit, + }) +} + +/// The plain-aggregation surrogate this pattern lowers to. Must stay +/// byte-for-byte identical regardless of caller (planner or engine): the +/// query-time matcher parses whatever inference_config.yaml registered as +/// the template and compares its *structured* form against the structured +/// form of whatever the engine rewrites an incoming request to - an +/// ORDER BY tie-break present on one side and not the other is enough to +/// make that match fail even though both queries are equivalent. +pub fn build_lag_transition_surrogate(m: &LagTransitionMatch) -> String { + format!( + "SELECT\n {group_label},\n count() AS {alias}\nFROM {metric}\nWHERE timestamp >= '{start}'\n AND timestamp < '{end}'\nGROUP BY {group_label}\nORDER BY {alias} DESC, {group_label} ASC\nLIMIT {limit}", + group_label = m.group_label, + alias = m.alias, + metric = m.derived_metric(), + start = m.start, + end = m.end, + limit = m.limit, + ) +} + +/// Detects and rewrites in one call - what the query engine needs at serve +/// time. The planner needs the structured `LagTransitionMatch` too (to build +/// a StatefulTransitionConfig), so it calls `parse_lag_transition_query` + +/// `build_lag_transition_surrogate` directly instead of this. +pub fn rewrite_lag_transition_query(query: &str) -> Option { + if !looks_like_lag_transition_sql(query) { + return None; + } + let m = parse_lag_transition_query(query)?; + Some(build_lag_transition_surrogate(&m)) +} + +// --------------------------------------------------------------------------- +// Token-select / token-explode patterns: a nested subquery that tokenizes a +// space-separated column with `arrayFilter(x -> match(x, ''), +// splitByWhitespace(col))`, then either indexes the last token (`[-1]`, +// token-select) or explodes every token into its own row (`arrayJoin(...)`, +// token-explode). +// --------------------------------------------------------------------------- + +struct TokenExtraction { + source_col: String, + filter_regex: String, + inner_alias: String, +} + +/// Extracts just the tokenizer inputs from `arrayFilter(x -> match(x, +/// ''), splitByWhitespace(col))`, without requiring an alias +/// immediately after - arrayFilter is aliased directly in the token-select +/// shape (`... AS as_path_array`), but nested unaliased inside arrayJoin(...) +/// in the token-explode shape, so the alias step has to be optional here and +/// handled separately by each caller. +fn extract_regex_and_source_col(query: &str) -> Option<(String, String, usize)> { + let lower = query.to_lowercase(); + let af_idx = lower.find("arrayfilter(")?; + let open_idx = af_idx + "arrayfilter(".len() - 1; + let close_idx = find_matching_close_paren(query, open_idx)?; + let inner = &query[open_idx + 1..close_idx]; + let inner_lower = inner.to_lowercase(); + + let match_idx = inner_lower.find("match(")?; + let match_open = match_idx + "match(".len() - 1; + let match_close = find_matching_close_paren(inner, match_open)?; + let match_args = &inner[match_open + 1..match_close]; + let q1 = match_args.find('\'')?; + let rest = &match_args[q1 + 1..]; + let q2 = rest.find('\'')?; + let filter_regex = rest[..q2].to_string(); + + let source_col = extract_paren_arg_after(inner, "splitbywhitespace(")?.to_string(); + + Some((source_col, filter_regex, close_idx)) +} + +fn extract_token_filter(query: &str) -> Option { + let (source_col, filter_regex, close_idx) = extract_regex_and_source_col(query)?; + + let after = &query[close_idx + 1..]; + let after_lower = after.to_lowercase(); + let as_idx = after_lower.find("as ")?; + if !after[..as_idx].trim().is_empty() { + return None; + } + let rest = after[as_idx + 3..].trim_start(); + let inner_alias: String = rest + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if inner_alias.is_empty() { + return None; + } + + Some(TokenExtraction { + source_col, + filter_regex, + inner_alias, + }) +} + +/// The raw WHERE clause of the *inner* subquery (the real time/spatial +/// filters), stopping before the outer query's synthetic +/// `WHERE length(...) > 0` guard - that guard is exactly what +/// `on_missing: skip_sample` already means at ingest time. +fn extract_inner_where(query: &str, subquery_close_idx: usize) -> Option { + let inner_text = &query[..subquery_close_idx]; + let lower = inner_text.to_lowercase(); + let where_idx = lower.rfind("where")?; + let after_where = inner_text[where_idx + "where".len()..].trim(); + Some(after_where.trim_end().to_string()) +} + +/// The inner subquery's own FROM target (e.g. "bgp.bgp_updates"), so the +/// surrogate references the same base table rather than assuming a fixed +/// name. Scoped strictly to the subquery body - taking the first "from" +/// from the start of the whole query would find the *outer* query's +/// "FROM (" instead. +fn extract_inner_from(query: &str, subquery_open_idx: usize, subquery_close_idx: usize) -> Option { + let inner_text = &query[subquery_open_idx + 1..subquery_close_idx]; + let lower = inner_text.to_lowercase(); + let from_idx = lower.find("from")?; + let after_from = &inner_text[from_idx + "from".len()..]; + let where_idx = after_from.to_lowercase().find("where")?; + Some(after_from[..where_idx].trim().to_string()) +} + +pub fn looks_like_token_select_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("arrayfilter(") + && q.contains("splitbywhitespace(") + && q.contains("match(") + && q.contains("[-1]") + && q.contains("group by") +} + +#[derive(Debug, Clone)] +pub struct TokenSelectMatch { + pub label: String, + pub source_col: String, + pub filter_regex: String, + /// The outer SELECT's non-label item, e.g. "count() AS x". + pub select_expr: String, + pub from_target: String, + pub where_clause: String, + pub group_by: String, + pub order_by_and_limit: String, +} + +pub fn parse_token_select_query(query: &str) -> Option { + let tok = extract_token_filter(query)?; + + let lower = query.to_lowercase(); + let from_idx = lower.find("from")?; + let open_paren_rel = lower[from_idx..].find('(')?; + let subquery_open = from_idx + open_paren_rel; + let subquery_close = find_matching_close_paren(query, subquery_open)?; + + let outer_select = &query[..from_idx]; + let outer_tail = &query[subquery_close + 1..]; + let outer_tail_lower = outer_tail.to_lowercase(); + + let index_marker = format!("{}[-1]", tok.inner_alias); + let index_pos = outer_select.find(&index_marker)?; + let after_index = &outer_select[index_pos + index_marker.len()..]; + let after_index_lower = after_index.to_lowercase(); + let as_idx = after_index_lower.find("as ")?; + let after_as = after_index[as_idx + 3..].trim_start(); + let label: String = after_as + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if label.is_empty() { + return None; + } + let comma_idx = after_as.find(',')?; + let select_expr = after_as[comma_idx + 1..].trim().to_string(); + + let where_clause = extract_inner_where(query, subquery_close)?; + let from_target = extract_inner_from(query, subquery_open, subquery_close)?; + + let group_idx = outer_tail_lower.find("group by")? + "group by".len(); + let after_group = &outer_tail[group_idx..]; + let after_group_lower = after_group.to_lowercase(); + let group_end = after_group_lower + .find("order by") + .or_else(|| after_group_lower.find("limit")) + .unwrap_or(after_group.len()); + let group_by = after_group[..group_end].trim().trim_end_matches(',').to_string(); + + let order_start = after_group_lower.find("order by").unwrap_or(group_end); + let order_by_and_limit = after_group[order_start..].trim().to_string(); + + Some(TokenSelectMatch { + label, + source_col: tok.source_col, + filter_regex: tok.filter_regex, + select_expr, + from_target, + where_clause, + group_by, + order_by_and_limit, + }) +} + +pub fn build_token_select_surrogate(m: &TokenSelectMatch) -> String { + format!( + "SELECT {label}, {select_expr} FROM {from_target} WHERE {where_clause} GROUP BY {group_by} {order_by_and_limit}", + label = m.label, + select_expr = m.select_expr, + from_target = m.from_target, + where_clause = m.where_clause, + group_by = m.group_by, + order_by_and_limit = m.order_by_and_limit, + ) +} + +pub fn rewrite_token_select_query(query: &str) -> Option { + if !looks_like_token_select_sql(query) { + return None; + } + let m = parse_token_select_query(query)?; + Some(build_token_select_surrogate(&m)) +} + +pub fn looks_like_token_explode_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("arrayjoin(") + && q.contains("arrayfilter(") + && q.contains("splitbywhitespace(") + && q.contains("match(") + && q.contains("group by") +} + +#[derive(Debug, Clone)] +pub struct TokenExplodeMatch { + pub label: String, + pub source_col: String, + pub filter_regex: String, + pub select_expr: String, + pub from_target: String, + pub where_clause: String, + pub group_by: String, + pub order_by_and_limit: String, +} + +pub fn parse_token_explode_query(query: &str) -> Option { + let lower = query.to_lowercase(); + let aj_idx = lower.find("arrayjoin(")?; + let aj_open = aj_idx + "arrayjoin(".len() - 1; + let aj_close = find_matching_close_paren(query, aj_open)?; + + let aj_inner = &query[aj_open + 1..aj_close]; + let (source_col, filter_regex, _) = extract_regex_and_source_col(aj_inner)?; + + let after = &query[aj_close + 1..]; + let after_lower = after.to_lowercase(); + let as_idx = after_lower.find("as ")?; + if !after[..as_idx].trim().is_empty() { + return None; + } + let rest = after[as_idx + 3..].trim_start(); + let label: String = rest + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if label.is_empty() { + return None; + } + + let from_idx = lower.find("from")?; + let open_paren_rel = lower[from_idx..].find('(')?; + let subquery_open = from_idx + open_paren_rel; + let subquery_close = find_matching_close_paren(query, subquery_open)?; + if aj_close > subquery_close { + return None; + } + + let outer_select = &query[..from_idx]; + let outer_tail = &query[subquery_close + 1..]; + let outer_tail_lower = outer_tail.to_lowercase(); + + let label_pos = outer_select.find(label.as_str())?; + let after_label = &outer_select[label_pos + label.len()..]; + let comma_idx = after_label.find(',')?; + let select_expr = after_label[comma_idx + 1..].trim().to_string(); + + let where_clause = extract_inner_where(query, subquery_close)?; + let from_target = extract_inner_from(query, subquery_open, subquery_close)?; + + let group_idx = outer_tail_lower.find("group by")? + "group by".len(); + let after_group = &outer_tail[group_idx..]; + let after_group_lower = after_group.to_lowercase(); + let group_end = after_group_lower + .find("order by") + .or_else(|| after_group_lower.find("limit")) + .unwrap_or(after_group.len()); + let group_by = after_group[..group_end].trim().trim_end_matches(',').to_string(); + + let order_start = after_group_lower.find("order by").unwrap_or(group_end); + let order_by_and_limit = after_group[order_start..].trim().to_string(); + + Some(TokenExplodeMatch { + label, + source_col, + filter_regex, + select_expr, + from_target, + where_clause, + group_by, + order_by_and_limit, + }) +} + +pub fn build_token_explode_surrogate(m: &TokenExplodeMatch) -> String { + format!( + "SELECT {label}, {select_expr} FROM {from_target} WHERE {where_clause} GROUP BY {group_by} {order_by_and_limit}", + label = m.label, + select_expr = m.select_expr, + from_target = m.from_target, + where_clause = m.where_clause, + group_by = m.group_by, + order_by_and_limit = m.order_by_and_limit, + ) +} + +pub fn rewrite_token_explode_query(query: &str) -> Option { + if !looks_like_token_explode_sql(query) { + return None; + } + let m = parse_token_explode_query(query)?; + Some(build_token_explode_surrogate(&m)) +} + +// --------------------------------------------------------------------------- +// MOAS (multiple-origin-AS) pattern: group by an existing column (e.g. +// `prefix`), aggregate the *set* of distinct values of some other column +// (e.g. origin ASN), keeping only groups whose set has more than one +// member. Two raw-SQL shapes reach the same semantics: +// - literal: the origin value is already a plain column +// (`COUNT(DISTINCT origin_asn)` + `DISTINCT_SET(origin_asn)`). +// - tokenized: the origin value is derived by the same tokenizer building +// block token-select/token-explode use (`uniqExact(as_path_array[-1])` +// + `groupUniqArray(as_path_array[-1])`, fed by a nested subquery +// tokenizing `as_path`). This shape additionally needs a computed label +// emitted at ingest time, same as token-select. +// Both normalize to one canonical surrogate: +// SELECT {group_by}, COUNT(DISTINCT {label}) AS origin_count +// FROM {from_target} WHERE {where_clause} GROUP BY {group_by} +// ORDER BY origin_count DESC LIMIT 1000000 +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct MoasMatch { + pub group_by: String, + pub label: String, + pub from_target: String, + pub where_clause: String, + /// Set only when `label` is a computed value (the tokenized last + /// AS-path element) rather than an existing column - the caller must + /// emit a ComputedLabelConfig for `label` using these + /// (source_col, filter_regex). + pub computed_label: Option<(String, String)>, +} + +pub fn looks_like_moas_literal_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("prefix") + && q.contains("origin_asn") + && q.contains("count(distinct") + && q.contains("distinct_set") + && q.contains("group by") +} + +pub fn looks_like_moas_tokenized_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("arrayfilter(") + && q.contains("splitbywhitespace(") + && q.contains("match(") + && q.contains("groupuniqarray(") + && q.contains("group by") +} + +/// True for either MOAS raw-SQL shape - the one gate both the planner and +/// the query engine call to decide "try MOAS handling" before falling +/// through to other patterns. +pub fn looks_like_moas_sql(query: &str) -> bool { + looks_like_moas_literal_sql(query) || looks_like_moas_tokenized_sql(query) +} + +/// True for the canonical MOAS *surrogate* shape (what gets registered in +/// inference_config.yaml and what query-time lookup scans registered +/// queries for) - distinct from `looks_like_moas_sql`, which recognizes raw +/// user SQL. Deliberately loose: any registered query built by +/// `build_moas_surrogate` matches this, regardless of which raw shape it +/// came from. +pub fn looks_like_moas_registered_sql(query: &str) -> bool { + let q = query.to_lowercase(); + q.contains("count(distinct") && q.contains("origin_count") && q.contains("group by") +} + +fn parse_moas_literal_query(query: &str) -> Option { + let lower = query.to_lowercase(); + let from_idx = lower.find("from ")?; + let where_idx = lower.find("where ")?; + let group_idx = lower.find("group by ")?; + if !(from_idx < where_idx && where_idx < group_idx) { + return None; + } + let from_target = query[from_idx + "from ".len()..where_idx].trim().to_string(); + let where_clause = query[where_idx + "where ".len()..group_idx].trim().to_string(); + Some(MoasMatch { + group_by: "prefix".to_string(), + label: "origin_asn".to_string(), + from_target, + where_clause, + computed_label: None, + }) +} + +fn parse_moas_tokenized_query(query: &str) -> Option { + let tok = extract_token_filter(query)?; + + let lower = query.to_lowercase(); + let from_idx = lower.find("from")?; + let open_paren_rel = lower[from_idx..].find('(')?; + let subquery_open = from_idx + open_paren_rel; + let subquery_close = find_matching_close_paren(query, subquery_open)?; + + let outer_select = &query[..from_idx]; + let outer_select_lower = outer_select.to_lowercase(); + let outer_tail = &query[subquery_close + 1..]; + let outer_tail_lower = outer_tail.to_lowercase(); + + // Confirm the tokenized array is actually aggregated as a set: both the + // cardinality expression and groupUniqArray must reference + // `{inner_alias}[-1]` (the same "last token" indexing token-select + // uses). + let index_marker = format!("{}[-1]", tok.inner_alias.to_lowercase()); + if !outer_select_lower.contains(&index_marker) || !outer_select_lower.contains("groupuniqarray(") + { + return None; + } + + let where_clause = extract_inner_where(query, subquery_close)?; + let from_target = extract_inner_from(query, subquery_open, subquery_close)?; + + let group_idx = outer_tail_lower.find("group by")? + "group by".len(); + let after_group = &outer_tail[group_idx..]; + let after_group_lower = after_group.to_lowercase(); + let group_end = after_group_lower + .find("having") + .or_else(|| after_group_lower.find("order by")) + .or_else(|| after_group_lower.find("limit")) + .unwrap_or(after_group.len()); + let group_by = after_group[..group_end] + .trim() + .trim_end_matches(',') + .to_string(); + if group_by.is_empty() { + return None; + } + + // A HAVING clause, if present, must be asking for exactly "more than + // one" - the engine's MOAS handler hardcodes a >1 filter (that's the + // whole definition of MOAS), so silently matching a HAVING with a + // different threshold would misrepresent the query rather than serve + // it correctly. + if let Some(having_idx) = after_group_lower.find("having") { + let having_text = &after_group[having_idx + "having".len()..]; + let having_lower = having_text.to_lowercase(); + let having_end = having_lower + .find("order by") + .or_else(|| having_lower.find("limit")) + .unwrap_or(having_text.len()); + let normalized: String = having_text[..having_end] + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + let normalized_lower = normalized.to_lowercase(); + if !(normalized_lower.contains(">1") || normalized_lower.contains(">=2")) { + return None; + } + } + + let label = format!("{}_last_token", tok.source_col); + + Some(MoasMatch { + group_by, + label, + from_target, + where_clause, + computed_label: Some((tok.source_col, tok.filter_regex)), + }) +} + +pub fn parse_moas_query(query: &str) -> Option { + if looks_like_moas_tokenized_sql(query) { + return parse_moas_tokenized_query(query); + } + if looks_like_moas_literal_sql(query) { + return parse_moas_literal_query(query); + } + None +} + +pub fn build_moas_surrogate(m: &MoasMatch) -> String { + format!( + "SELECT {group_by}, COUNT(DISTINCT {label}) AS origin_count FROM {from_target} WHERE {where_clause} GROUP BY {group_by} ORDER BY origin_count DESC LIMIT 1000000", + group_by = m.group_by, + label = m.label, + from_target = m.from_target, + where_clause = m.where_clause, + ) +} + +pub fn rewrite_moas_query(query: &str) -> Option { + let m = parse_moas_query(query)?; + Some(build_moas_surrogate(&m)) +} + +/// True for a query with no aggregate function anywhere in its text and no +/// GROUP BY clause: a raw row scan (`SELECT DISTINCT ...`, or a bare column +/// listing). No precomputed summary can ever answer this - the whole point +/// of DISTINCT / raw-listing semantics is exact per-row output, which by +/// definition a lossy aggregate summary cannot provide. Callers should treat +/// a `true` result as "route this query straight to the source of truth," +/// not as "try harder to plan it." Intended as a last-resort check, run only +/// after every other pattern in this module has had a chance to claim the +/// query. +const AGGREGATE_FUNCTIONS: &[&str] = &[ + "COUNT(", + "SUM(", + "AVG(", + "MIN(", + "MAX(", + "UNIQ(", + "UNIQEXACT(", + "UNIQCOMBINED(", + "UNIQCOMBINED64(", + "GROUPUNIQARRAY(", + "GROUPARRAY(", + "TOPK(", + "QUANTILE(", + "ANY(", + "ANYLAST(", + "ARGMIN(", + "ARGMAX(", +]; + +pub fn looks_like_exact_only_sql(query: &str) -> bool { + let upper = query.to_uppercase(); + if upper.contains("GROUP BY") { + return false; + } + !AGGREGATE_FUNCTIONS.iter().any(|f| upper.contains(f)) +} + +// --------------------------------------------------------------------------- +// Multi-aggregate pattern: a flat (no subquery) query with one GROUP BY and +// 2+ independent aggregate expressions in the SELECT list, e.g. +// SELECT collector, peer_ip, count() AS updates, uniqExact(prefix) AS distinct_prefixes +// FROM ... GROUP BY collector, peer_ip +// The classic SQLQueryData model tracks exactly one aggregate per query, so +// this can't be planned/matched as-is. It splits cleanly into N independent +// single-aggregate queries sharing the same FROM/WHERE/GROUP BY - each one +// individually is exactly the shape the classic path already handles, so +// this building block only has to handle the split, not reimplement +// aggregation planning or matching. +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct MultiAggregateMatch { + /// The non-aggregate SELECT-list items (the GROUP BY columns), in + /// original order. + pub group_by_cols: Vec, + /// The aggregate SELECT-list items (e.g. "count() AS updates"), in + /// original order. Always 2 or more. + pub aggregate_exprs: Vec, + /// "FROM ... WHERE ..." (or just "FROM ..." with no WHERE), verbatim. + pub from_where: String, + /// Raw text of the GROUP BY clause (column list), verbatim. + pub group_by_clause: String, +} + +/// Paren- and quote-aware top-level comma split - a SELECT list item like +/// `count(distinct foo)` must not be split on the comma inside its own +/// argument list. +fn split_top_level_commas(s: &str) -> Vec { + let mut items = Vec::new(); + let mut current = String::new(); + let mut depth = 0i32; + let mut in_quotes = false; + for c in s.chars() { + if c == '\'' { + in_quotes = !in_quotes; + current.push(c); + continue; + } + if !in_quotes { + if c == '(' { + depth += 1; + current.push(c); + continue; + } + if c == ')' { + depth -= 1; + current.push(c); + continue; + } + if c == ',' && depth == 0 { + items.push(current.trim().to_string()); + current.clear(); + continue; + } + } + current.push(c); + } + if !current.trim().is_empty() { + items.push(current.trim().to_string()); + } + items +} + +pub fn parse_multi_aggregate_query(query: &str) -> Option { + let lower = query.to_lowercase(); + if lower.contains("having") { + // Not handled by this pattern yet - HAVING needs its own semantics + // per aggregate, not just a split. + return None; + } + + let select_kw_end = lower.find("select")? + "select".len(); + let from_idx = lower.find("from")?; + if select_kw_end > from_idx { + return None; + } + + // A nested-subquery FROM belongs to other patterns (MOAS/token-select/ + // token-explode), not this one. + let after_from = query[from_idx + "from".len()..].trim_start(); + if after_from.starts_with('(') { + return None; + } + + let select_list = &query[select_kw_end..from_idx]; + let items = split_top_level_commas(select_list); + + let mut group_by_cols = Vec::new(); + let mut aggregate_exprs = Vec::new(); + for item in items { + let item_upper = item.to_uppercase(); + if AGGREGATE_FUNCTIONS.iter().any(|f| item_upper.contains(f)) { + aggregate_exprs.push(item); + } else { + group_by_cols.push(item); + } + } + if aggregate_exprs.len() < 2 { + return None; + } + + let group_idx = lower.find("group by")?; + let from_where = query[from_idx..group_idx].trim_end().to_string(); + + let after_group = &query[group_idx + "group by".len()..]; + let after_group_lower = after_group.to_lowercase(); + let group_end = after_group_lower + .find("order by") + .or_else(|| after_group_lower.find("limit")) + .unwrap_or(after_group.len()); + let group_by_clause = after_group[..group_end].trim().to_string(); + if group_by_clause.is_empty() { + return None; + } + + Some(MultiAggregateMatch { + group_by_cols, + aggregate_exprs, + from_where, + group_by_clause, + }) +} + +pub fn looks_like_multi_aggregate_sql(query: &str) -> bool { + parse_multi_aggregate_query(query).is_some() +} + +/// One single-aggregate surrogate per aggregate expression, in the same +/// order as `m.aggregate_exprs` - each independently plannable/servable by +/// the existing classic single-aggregate machinery. +pub fn build_multi_aggregate_surrogates(m: &MultiAggregateMatch) -> Vec { + m.aggregate_exprs + .iter() + .map(|expr| { + format!( + "SELECT {group_by}, {expr} {from_where} GROUP BY {group_by}", + group_by = m.group_by_clause, + expr = expr, + from_where = m.from_where, + ) + }) + .collect() +} diff --git a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs index 7ec90f33..69c6a14c 100644 --- a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs +++ b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs @@ -89,6 +89,9 @@ impl SQLSchema { #[derive(Debug, Clone)] pub struct SQLQueryData { pub aggregation_info: AggregationInfo, + /// Metadata predicates from WHERE after removing the time predicate. + /// Example: collector = 'rrc00' or collector IN ('rrc00'). + pub spatial_filter: Option, /// Alias of the aggregate function in SELECT, e.g. `agg(v) AS p99` → `Some("p99")`. /// Captured separately from `aggregation_info` because it's presentational only: /// two queries that differ solely in alias must still match the same template. @@ -104,6 +107,43 @@ pub struct SQLQueryData { pub limit: Option, } +#[derive(Debug, Clone)] +pub struct SQLBucketedCountIfOutput { + pub alias: String, + /// Extra per-output filter extracted from countIf(...). + /// Example: operation = 'A' + pub filter: String, +} + +#[derive(Debug, Clone)] +pub struct SQLBucketedCountIfQueryData { + pub metric: String, + pub time_info: TimeInfo, + pub bucket_alias: String, + pub bucket_ms: u64, + /// WHERE predicates after removing the time predicate. + /// Example: collector = 'rrc00' + pub base_spatial_filter: Option, + pub outputs: Vec, + pub order_by: Vec, +} + +impl SQLBucketedCountIfQueryData { + /// Match reusable bucketed templates by structure, not by absolute timestamps. + pub fn matches_bucketed_pattern(&self, template: &SQLBucketedCountIfQueryData) -> bool { + self.metric == template.metric + && self.time_info.get_time_col_name() == template.time_info.get_time_col_name() + && self.bucket_ms == template.bucket_ms + && self.base_spatial_filter == template.base_spatial_filter + && self.outputs.len() == template.outputs.len() + && self + .outputs + .iter() + .zip(template.outputs.iter()) + .all(|(a, b)| a.alias == b.alias && a.filter == b.filter) + } +} + /// Single `ORDER BY` clause item: a column reference plus sort direction. /// `column` is either a GROUP BY identifier or the aggregate alias. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs index d24a22b9..f4d47b95 100644 --- a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs +++ b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs @@ -56,6 +56,7 @@ impl SQLQuery { let query_data = SQLQueryData { aggregation_info: aggregation, + spatial_filter: None, aggregation_alias: None, metric, labels, @@ -193,6 +194,13 @@ impl SQLPatternMatcher { .schema .get_metadata_columns(&query.metric) .is_some_and(|cols| cols.contains(value_column_name)) + } else if query.aggregation_info.get_name() == "COUNT" + && value_column_name == "__event_count__" + { + // COUNT() is an event-count aggregate. The parser represents it + // using a synthetic value column, but this column is not a real + // table column and should not be required in value_columns. + true } else { self.schema .is_valid_value_column(&query.metric, value_column_name) diff --git a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs index 659ac325..7914a1b0 100644 --- a/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs +++ b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs @@ -1,5 +1,8 @@ use crate::sqlhelper::SQLSchema; -use crate::sqlhelper::{AggregationInfo, OrderByItem, SQLQueryData, TimeInfo}; +use crate::sqlhelper::{ + AggregationInfo, OrderByItem, SQLBucketedCountIfOutput, SQLBucketedCountIfQueryData, + SQLQueryData, TimeInfo, +}; use sqlparser::ast::*; use std::collections::HashSet; @@ -42,6 +45,289 @@ impl SQLPatternParser { } } + /// Flatten an AND expression into a list of conjuncts. + /// Example: + /// time BETWEEN ... AND ... AND collector = 'rrc00' + /// becomes: + /// [time BETWEEN ... AND ..., collector = 'rrc00'] + fn flatten_and_conjuncts<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { + match expr { + Expr::BinaryOp { + left, + op: BinaryOperator::And, + right, + } => { + Self::flatten_and_conjuncts(left, out); + Self::flatten_and_conjuncts(right, out); + } + _ => out.push(expr), + } + } + + /// Try to parse one expression as a time predicate. + fn get_time_info_from_expr(&self, expr: &Expr) -> Option { + match expr { + Expr::Between { + expr, + negated, + low, + high, + } => { + if *negated { + return None; + } + + let col_name = match expr.as_ref() { + Expr::Identifier(ident) => ident.value.clone(), + _ => return None, + }; + + let start = self.get_timestamp_from_between_highlow(low)?; + let end = self.get_timestamp_from_between_highlow(high)?; + let duration = end - start; + + Some(TimeInfo::new(col_name, start, duration)) + } + + Expr::BinaryOp { + left, + op: BinaryOperator::And, + right, + } => self.get_time_info_from_half_open(left, right), + + _ => None, + } + } + + pub fn parse_bucketed_countif_query( + &self, + statements: &[Statement], + ) -> Option { + if statements.len() != 1 { + return None; + } + + let query = match &statements[0] { + Statement::Query(query) => query, + _ => return None, + }; + + let order_by_items = self.parse_order_by_items(query)?; + if query.limit_clause.is_some() { + return None; + } + + let query = self.cte_to_subquery(query); + + let select = match query.body.as_ref() { + SetExpr::Select(select) => select, + _ => return None, + }; + + self.parse_bucketed_countif_select(select, order_by_items) + } + + fn parse_bucketed_countif_select( + &self, + select: &Select, + order_by_items: Vec, + ) -> Option { + let (metric, has_subquery) = self.get_metric(select)?; + if has_subquery { + return None; + } + + if select.projection.len() < 2 { + return None; + } + + if select.distinct.is_some() + || select.top.is_some() + || select.into.is_some() + || !select.lateral_views.is_empty() + || select.prewhere.is_some() + || !select.cluster_by.is_empty() + || !select.distribute_by.is_empty() + || !select.sort_by.is_empty() + || select.having.is_some() + || !select.named_window.is_empty() + || select.window_before_qualify + { + return None; + } + + let time_info = self.get_time_info(select, &metric)?; + let base_spatial_filter = self.get_spatial_filter(select); + + let (bucket_time_col, bucket_ms, bucket_alias) = + self.parse_time_bucket_projection(&select.projection[0])?; + + if bucket_time_col != time_info.get_time_col_name() { + return None; + } + + let group_bys = self.get_groupbys(select)?; + if group_bys.len() != 1 || !group_bys.contains(&bucket_alias) { + return None; + } + + for item in &order_by_items { + if item.column != bucket_alias { + return None; + } + } + + let mut outputs = Vec::new(); + for item in select.projection.iter().skip(1) { + outputs.push(self.parse_countif_projection(item)?); + } + + if outputs.is_empty() { + return None; + } + + Some(SQLBucketedCountIfQueryData { + metric, + time_info, + bucket_alias, + bucket_ms, + base_spatial_filter, + outputs, + order_by: order_by_items, + }) + } + + fn parse_time_bucket_projection(&self, item: &SelectItem) -> Option<(String, u64, String)> { + let (expr, alias) = match item { + SelectItem::ExprWithAlias { expr, alias } => (expr, alias.value.clone()), + _ => return None, + }; + + let func = match expr { + Expr::Function(func) => func, + _ => return None, + }; + + if !func + .name + .to_string() + .eq_ignore_ascii_case("toStartOfInterval") + { + return None; + } + + let args = match &func.args { + FunctionArguments::List(args) => &args.args, + _ => return None, + }; + + if args.len() != 2 { + return None; + } + + let time_col = match &args[0] { + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident))) => { + ident.value.clone() + } + _ => return None, + }; + + let interval_func = match &args[1] { + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Function(f))) => f, + _ => return None, + }; + + if !interval_func + .name + .to_string() + .eq_ignore_ascii_case("toIntervalMinute") + { + return None; + } + + let interval_args = match &interval_func.args { + FunctionArguments::List(args) => &args.args, + _ => return None, + }; + + if interval_args.len() != 1 { + return None; + } + + let minutes = match &interval_args[0] { + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan { + value: Value::Number(n, _), + .. + }))) => n.parse::().ok()?, + _ => return None, + }; + + Some((time_col, minutes * 60_000, alias)) + } + + fn parse_countif_projection(&self, item: &SelectItem) -> Option { + let (expr, alias) = match item { + SelectItem::ExprWithAlias { expr, alias } => (expr, alias.value.clone()), + _ => return None, + }; + + let func = match expr { + Expr::Function(func) => func, + _ => return None, + }; + + if !func.name.to_string().eq_ignore_ascii_case("countIf") { + return None; + } + + let args = match &func.args { + FunctionArguments::List(args) => &args.args, + _ => return None, + }; + + if args.len() != 1 { + return None; + } + + let cond = match &args[0] { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => expr, + _ => return None, + }; + + let filter = self.parse_simple_equality_filter(cond)?; + + Some(SQLBucketedCountIfOutput { alias, filter }) + } + + fn parse_simple_equality_filter(&self, expr: &Expr) -> Option { + match expr { + Expr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } => { + let col = match left.as_ref() { + Expr::Identifier(ident) => ident.value.clone(), + _ => return None, + }; + + let lit = match right.as_ref() { + Expr::Value(ValueWithSpan { + value: Value::SingleQuotedString(s), + .. + }) => s.clone(), + _ => return None, + }; + + // sqlparser already unescapes SingleQuotedString content, so any + // embedded `'` must be re-escaped before we re-wrap it in quotes, + // or the resulting filter string is malformed. + Some(format!("{} = '{}'", col, lit.replace('\'', "''"))) + } + _ => None, + } + } + pub fn parse_query(&self, statements: &[Statement]) -> Option { if statements.len() != 1 { println!("illegal query length"); @@ -193,6 +479,75 @@ impl SQLPatternParser { query } + /// Find the single time predicate inside a flattened AND list. + /// + /// Supports both: + /// ts BETWEEN DATEADD(...) AND NOW() + /// and: + /// ts >= start AND ts < end + /// + /// The second form becomes two separate conjuncts after flattening, so we + /// must try pairs of conjuncts as a half-open time range. + fn find_time_info_in_conjuncts(&self, conjuncts: &[&Expr]) -> Option { + let mut matches = Vec::new(); + + // Single-expression time predicates, e.g. BETWEEN. + for expr in conjuncts { + if let Some(time_info) = self.get_time_info_from_expr(expr) { + matches.push(time_info); + } + } + + // Pair-expression half-open predicates: + // ts >= start AND ts < end + for i in 0..conjuncts.len() { + for j in (i + 1)..conjuncts.len() { + if let Some(time_info) = + self.get_time_info_from_half_open(conjuncts[i], conjuncts[j]) + { + matches.push(time_info); + } + } + } + + if matches.len() == 1 { + matches.into_iter().next() + } else { + None + } + } + + /// Return true if an expression is one side of a half-open time range. + fn is_time_comparison_side(&self, expr: &Expr) -> bool { + self.parse_time_comparison(expr).is_some() + } + + /// Return true if an expression can be parsed as the query's time predicate. + fn is_time_predicate(&self, expr: &Expr) -> bool { + self.get_time_info_from_expr(expr).is_some() + } + + /// Extract metadata predicates from WHERE by removing the time predicate. + /// The remaining predicates are returned as a SQL string for spatialFilter. + fn get_spatial_filter(&self, select: &Select) -> Option { + let selection = select.selection.as_ref()?; + + let mut conjuncts = Vec::new(); + Self::flatten_and_conjuncts(selection, &mut conjuncts); + + let filters: Vec = conjuncts + .into_iter() + .filter(|expr| !self.is_time_predicate(expr) && !self.is_time_comparison_side(expr)) + .map(|expr| expr.to_string()) + .collect(); + + if filters.is_empty() { + None + } else { + Some(filters.join(" AND ")) + } + } + fn parse_select(&self, select: &Select) -> Option { let (metric, has_subquery) = self.get_metric(select)?; @@ -206,6 +561,7 @@ impl SQLPatternParser { if !has_subquery { let time_info = self.get_time_info(select, &metric)?; + let spatial_filter = self.get_spatial_filter(select); // Check for unexpected fields if select.distinct.is_some() @@ -226,6 +582,7 @@ impl SQLPatternParser { Some(SQLQueryData { aggregation_info: aggregation, + spatial_filter, aggregation_alias, metric, labels: group_bys, @@ -247,8 +604,11 @@ impl SQLPatternParser { } let time_info = self.get_time_info(inner_select, &metric)?; + let spatial_filter = self.get_spatial_filter(inner_select); + Some(Box::new(SQLQueryData { aggregation_info: inner_aggregation, + spatial_filter, aggregation_alias: inner_alias, metric: metric.clone(), labels: inner_group_bys, @@ -265,6 +625,7 @@ impl SQLPatternParser { Some(SQLQueryData { aggregation_info: aggregation, + spatial_filter: None, aggregation_alias, metric, labels: group_bys, @@ -411,11 +772,40 @@ impl SQLPatternParser { } } + // ClickHouse's own distinct-count function family - same cardinality + // semantics as COUNT(DISTINCT col), just a different spelling. Without + // this, uniqExact(col) falls through to the generic "other aggregations" + // branch below, gets treated as a plain aggregation named "UNIQEXACT", + // and is rejected downstream as an illegal aggregation function - even + // though the CARDINALITY path it needs already exists and works. + let is_uniq_family = matches!(name.as_str(), "UNIQEXACT" | "UNIQ" | "UNIQCOMBINED"); + if is_uniq_family { + if let FunctionArguments::List(list) = &func.args { + if list.args.len() != 1 { + // Compound-key distinct (e.g. uniqExact(a, b)) isn't + // representable by the single-value-column model either - + // same limitation as COUNT(DISTINCT a, b) above. + return None; + } + } + } + let args = self.get_quantile_args(func); - // Get the column being aggregated + // Get the column being aggregated. + // + // ASAP's SQL planner originally required every aggregate to name a value + // column, e.g. COUNT(v) or SUM(v). BGP Q1 uses COUNT() as an event count. + // Treat COUNT() as a synthetic per-row count. The planner will map this + // to a count sketch where each matching row contributes weight 1. let col = match &func.args { - FunctionArguments::None => return None, + FunctionArguments::None => { + if name == "COUNT" { + "__event_count__".to_string() + } else { + return None; + } + } FunctionArguments::Subquery(_) => return None, FunctionArguments::List(func_args) => { if name == "QUANTILE" { @@ -456,15 +846,25 @@ impl SQLPatternParser { _ => return None, } } else { - // For other aggregations - column is first argument + // For other aggregations - column is first argument. + // Special case: COUNT() is parsed as an empty argument list. + // Treat COUNT() as an event count over a synthetic per-row value. if func_args.args.is_empty() { - return None; - } - match &func_args.args[0] { - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident))) => { - ident.value.clone() + if name == "COUNT" { + "__event_count__".to_string() + } else { + return None; + } + } else { + match &func_args.args[0] { + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier( + ident, + ))) => ident.value.clone(), + FunctionArg::Unnamed(FunctionArgExpr::Wildcard) if name == "COUNT" => { + "__event_count__".to_string() + } + _ => return None, } - _ => return None, } } } @@ -473,9 +873,10 @@ impl SQLPatternParser { // Normalisation: // - PERCENTILE → QUANTILE (legacy alias). // - COUNT(DISTINCT col) → CARDINALITY (validated above to be single-arg). + // - uniqExact/uniq/uniqCombined(col) → CARDINALITY (same, ClickHouse spelling). let normalized_name = if name == "PERCENTILE" { "QUANTILE".to_string() - } else if has_distinct { + } else if has_distinct || is_uniq_family { "CARDINALITY".to_string() } else { name @@ -514,12 +915,18 @@ impl SQLPatternParser { } fn get_timestamp_from_datetime_str(datetime_str: &str) -> Option { - // parse_datetime treats timezone-naive strings (e.g. "2025-10-01 00:00:00", - // "2025-10-01T00:00:00") as local server time, matching ClickHouse's behavior — - // but only when both run in the same timezone. Z-suffix strings (e.g. - // "2025-10-01T00:00:00Z") are interpreted as UTC here but rejected by ClickHouse. - // Use space-format datetime strings ("YYYY-MM-DD HH:MM:SS") for portability. - let parsed_datetime = parse_datetime(datetime_str).ok()?; + // Treat SQL timestamp literals as UTC. Internally append a Z suffix before + // parse_datetime so timezone-naive SQL literals match UTC-exported BGP data. + let trimmed = datetime_str.trim(); + let utc_datetime = if trimmed.ends_with('Z') { + trimmed.to_string() + } else if trimmed.contains('T') { + format!("{}Z", trimmed) + } else { + format!("{}Z", trimmed.replace(' ', "T")) + }; + + let parsed_datetime = parse_datetime(&utc_datetime).ok()?; Some(parsed_datetime.timestamp().as_second() as f64) } @@ -555,48 +962,10 @@ impl SQLPatternParser { fn get_time_info(&self, select: &Select, _table_name: &str) -> Option { let selection = select.selection.as_ref()?; - match selection { - Expr::Between { - expr, - negated, - low, - high, - } => { - if *negated { - return None; - } - - // Extract time column name - let col_name = match expr.as_ref() { - Expr::Identifier(ident) => ident.value.clone(), - _ => return None, - }; - - let start = self.get_timestamp_from_between_highlow(low)?; - let end = self.get_timestamp_from_between_highlow(high)?; - - let duration = end - start; - - Some(TimeInfo::new(col_name, start, duration)) - } - - // Half-open range: `time >= AND time < `. - // - // ClickHouse executes `>=`/`<` as a true half-open `[start, end)` - // scan, which is exactly how ASAP selects precompute windows — so a - // query written this way answers the same question on both backends - // (unlike inclusive `BETWEEN`). We accept STRICTLY `>=` for the lower - // bound and `<` for the upper bound; any other operator combination - // (`>`, `<=`) returns None and is treated as unmatched, to avoid a - // silent off-by-one against the ClickHouse baseline. - Expr::BinaryOp { - left, - op: BinaryOperator::And, - right, - } => self.get_time_info_from_half_open(left, right), + let mut conjuncts = Vec::new(); + Self::flatten_and_conjuncts(selection, &mut conjuncts); - _ => None, - } + self.find_time_info_in_conjuncts(&conjuncts) } /// Parse a `time >= A AND time < B` conjunction into `TimeInfo`. @@ -720,14 +1089,14 @@ impl SQLPatternParser { FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(ValueWithSpan { value: SingleQuotedString(datetime_str), span: _, - }))) => parse_datetime(datetime_str).ok()?.timestamp().as_second() as f64, + }))) => Self::get_timestamp_from_datetime_str(datetime_str)?, FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Cast { expr, .. })) => { match expr.as_ref() { Expr::Value(ValueWithSpan { value: SingleQuotedString(datetime_str), .. - }) => parse_datetime(datetime_str).ok()?.timestamp().as_second() as f64, + }) => Self::get_timestamp_from_datetime_str(datetime_str)?, _ => { println!("Unsupported CAST expression in DATEADD"); return None; @@ -809,7 +1178,7 @@ impl SQLPatternParser { // value: SingleQuotedString(datetime_str), // span: _, // }))) if start - // == (parse_datetime(datetime_str).ok()?.timestamp().as_second() as f64) => {} + // == (Self::get_timestamp_from_datetime_str(datetime_str)?) => {} // _ => { // println!("time upper bound not calculating from present"); diff --git a/asap-planner-rs/src/planner/sql.rs b/asap-planner-rs/src/planner/sql.rs index f7d13f2a..58fc8bc2 100644 --- a/asap-planner-rs/src/planner/sql.rs +++ b/asap-planner-rs/src/planner/sql.rs @@ -1,9 +1,20 @@ use std::collections::HashSet; +use asap_types::computed_label::ComputedLabelConfig; use asap_types::enums::{CleanupPolicy, WindowType}; +use asap_types::stateful_transition::StatefulTransitionConfig; use promql_utilities::data_model::KeyByLabelNames; use promql_utilities::query_logics::enums::{AggregationType, QueryTreatmentType, Statistic}; -use sql_utilities::ast_matching::sqlhelper::{detect_sql_topk, Table, TimeInfo}; +use sql_utilities::ast_matching::pattern_rewrites::{ + build_lag_transition_surrogate, build_moas_surrogate, build_token_explode_surrogate, + build_token_select_surrogate, looks_like_exact_only_sql, looks_like_lag_transition_sql, + looks_like_moas_sql, looks_like_token_explode_sql, looks_like_token_select_sql, + parse_lag_transition_query, parse_moas_query, parse_token_explode_query, + parse_token_select_query, +}; +use sql_utilities::ast_matching::sqlhelper::{ + detect_sql_topk, SQLBucketedCountIfQueryData, Table, TimeInfo, +}; use sql_utilities::ast_matching::sqlpattern_matcher::SQLPatternMatcher; use sql_utilities::ast_matching::sqlpattern_parser::SQLPatternParser; use sql_utilities::ast_matching::SQLSchema; @@ -51,25 +62,126 @@ impl SQLSingleQueryProcessor { } } + /// True when this query has no GROUP BY and no aggregate function - a raw + /// row scan or DISTINCT listing that no precomputed summary can ever + /// answer, no matter how the pattern matchers below are extended. + /// Callers should punt these rather than calling + /// `get_streaming_aggregation_configs`, which would otherwise fail with + /// a generic "Failed to parse SQL query" error indistinguishable from an + /// actual planner bug. See `looks_like_exact_only_sql`. + pub fn is_exact_only(&self) -> bool { + looks_like_exact_only_sql(&self.query_string) + } + pub fn get_streaming_aggregation_configs( &self, query_evaluation_time: f64, - ) -> Result<(Vec, Option), ControllerError> { + ) -> Result< + ( + Vec, + Option, + Option, + // Some(surrogate) when the raw query_string can't be parsed as a + // template by SQLPatternParser (e.g. it's a CTE/window-function + // query) and a simplified query must be registered in + // inference_config.yaml instead, or the query-time matcher will + // never find this aggregation (it can't parse the raw template + // either, so nothing would ever match against it). + Option, + // Computed labels (e.g. origin-ASN extraction) this query needs + // at ingest time, as (label_name, config) pairs. + Vec<(String, ComputedLabelConfig)>, + ), + ControllerError, + > { let schema = build_sql_schema(&self.table_definitions); // Parse SQL let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &self.query_string) .map_err(|e| ControllerError::SqlParse(e.to_string()))?; + let parser = SQLPatternParser::new(&schema, query_evaluation_time); + + // Native bucketed countIf time-series path. + // + // This is intentionally parallel to the classic SQLQueryData path because + // the classic model tracks exactly one aggregate. A bucketed countIf query + // has one time-bucket expression and multiple conditional count outputs. + if let Some(bucketed) = parser.parse_bucketed_countif_query(&stmts) { + let (configs, cleanup) = + self.get_bucketed_countif_streaming_aggregation_configs(&bucketed)?; + return Ok((configs, cleanup, None, None, Vec::new())); + } + + // Native MOAS path: + // prefix -> exact set(origin_asn) + // + // The classic SQLQueryData parser tracks one aggregate, but Q8 has both + // COUNT(DISTINCT origin_asn) and DISTINCT_SET(origin_asn). The planner + // lowers this query to one SetAggregator precompute. + if looks_like_moas_sql(&self.query_string) { + // Same reasoning as the lag-transition path below: the raw MOAS + // SQL (COUNT(DISTINCT ...) + DISTINCT_SET/groupUniqArray) isn't + // parseable by SQLPatternParser either, so the surrogate must be + // what's registered as the query-time matching template too. + return self.get_moas_streaming_aggregation_configs(&schema, query_evaluation_time); + } + + // Lag-transition path (e.g. ClickHouse `lagInFrame(...) OVER (PARTITION + // BY ...)`): the raw SQL is a CTE + window function the classic + // SQLQueryData model can't represent at all. Detect it, translate it + // into a plain aggregation over its derived event stream (reusing the + // exact surrogate-query approach the MOAS path above already uses), + // and additionally emit the StatefulTransitionConfig that tells the + // precompute engine how to actually maintain that derived stream — + // previously nothing auto-generated this; it had to be hand-written. + if looks_like_lag_transition_sql(&self.query_string) { + let (configs, cleanup, stateful_transition, surrogate) = self + .get_lag_transition_streaming_aggregation_configs( + &schema, + query_evaluation_time, + )?; + return Ok((configs, cleanup, stateful_transition, surrogate, Vec::new())); + } + + // Token-select path (e.g. origin-ASN extraction: a nested subquery + // that tokenizes as_path and indexes the last matching token). The + // classic SQLQueryData parser can't represent the nested subquery + // either, so - same shape as the two paths above - detect it, + // translate to a plain aggregation over the (now-computed) label, + // and emit the ComputedLabelConfig that tells ingest how to actually + // derive that label from the raw column. Previously nothing + // auto-generated this; a human had to hand-write the + // computed_label_cols entry. + if looks_like_token_select_sql(&self.query_string) { + return self + .get_token_select_streaming_aggregation_configs(&schema, query_evaluation_time); + } + + // Token-explode path: same tokenization building block as above, but + // every matching token becomes its own row (arrayJoin) instead of + // indexing just the last one. + if looks_like_token_explode_sql(&self.query_string) { + return self + .get_token_explode_streaming_aggregation_configs(&schema, query_evaluation_time); + } + + // Multi-aggregate queries (2+ aggregate expressions over one GROUP + // BY) are handled one level up, in generate_sql_plan: split into N + // independent single-aggregate surrogates, each registered and + // planned separately (a fresh SQLSingleQueryProcessor per surrogate, + // recursing into this same function). That keeps each surrogate + // individually findable by the query engine's ordinary + // find_query_config_sql structural matcher at serve time, rather + // than only through its capability-matching fallback (which doesn't + // carry the surrogate's spatial filter and so can't match a query + // with a WHERE clause) - see handle_multi_aggregate_sql in the query + // engine for the serve-time half of this split. + // Parse query into SQLQueryData - let qdata = SQLPatternParser::new(&schema, query_evaluation_time) - .parse_query(&stmts) - .ok_or_else(|| { - ControllerError::SqlParse(format!( - "Failed to parse SQL query: {}", - self.query_string - )) - })?; + let qdata = parser.parse_query(&stmts).ok_or_else(|| { + ControllerError::SqlParse(format!("Failed to parse SQL query: {}", self.query_string)) + })?; // Match query to pattern // SQLPatternMatcher.scrape_interval is in seconds (SQL timestamps are seconds-based). @@ -139,7 +251,7 @@ impl SQLSingleQueryProcessor { table_name, Some(table_name), Some(&value_column), - "", + qdata.spatial_filter.as_deref().unwrap_or(""), |agg_type: AggregationType, agg_sub_type: &str| { build_sketch_parameters( agg_type, @@ -179,10 +291,741 @@ impl SQLSingleQueryProcessor { ) }; + Ok((configs, cleanup_param, None, None, Vec::new())) + } + + fn get_moas_streaming_aggregation_configs( + &self, + schema: &SQLSchema, + query_evaluation_time: f64, + ) -> Result< + ( + Vec, + Option, + Option, + Option, + Vec<(String, ComputedLabelConfig)>, + ), + ControllerError, + > { + let m = parse_moas_query(&self.query_string).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse MOAS SQL query: {}", + self.query_string + )) + })?; + + let surrogate = build_moas_surrogate(&m); + + // As with token-select, a computed label doesn't exist as a real + // schema column - schema validation needs to know about it before + // planning this one query, or it fails even though ingest will + // genuinely produce it once the emitted ComputedLabelConfig below + // takes effect. Only needed for the tokenized MOAS shape; the + // literal shape's label is already a real column. + let augmented_tables; + let augmented_schema; + let schema = if m.computed_label.is_some() { + let mut tables = self.table_definitions.clone(); + for t in &mut tables { + if !t.metadata_columns.iter().any(|c| c == &m.label) { + t.metadata_columns.push(m.label.clone()); + } + } + augmented_tables = tables; + augmented_schema = build_sql_schema(&augmented_tables); + &augmented_schema + } else { + schema + }; + + let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &surrogate) + .map_err(|e| ControllerError::SqlParse(e.to_string()))?; + + let parser = SQLPatternParser::new(schema, query_evaluation_time); + let qdata = parser.parse_query(&stmts).ok_or_else(|| { + ControllerError::SqlParse(format!("Failed to parse MOAS surrogate query: {surrogate}")) + })?; + + let sql_query = SQLPatternMatcher::new( + schema.clone(), + self.data_ingestion_interval_ms as f64 / 1000.0, + ) + .query_info_to_pattern(&qdata); + + if !sql_query.is_valid() { + return Err(ControllerError::SqlParse(sql_query.msg.unwrap_or_default())); + } + + if sql_query.query_data.len() != 1 { + return Err(ControllerError::SqlParse(format!( + "MOAS surrogate must produce one query layer, got {}", + sql_query.query_data.len() + ))); + } + + let q = &sql_query.query_data[0]; + let agg_info = &q.aggregation_info; + let labels = &q.labels; + let table_name = &q.metric; + let value_column = agg_info.get_value_column_name().to_string(); + + if agg_info.get_name() != "CARDINALITY" || value_column != m.label { + return Err(ControllerError::SqlParse(format!( + "MOAS path expected COUNT(DISTINCT {}), got {}({})", + m.label, + agg_info.get_name(), + value_column + ))); + } + + let window_cfg = compute_sql_window( + &q.time_info, + self.data_ingestion_interval_ms, + self.t_repeat_ms, + )?; + + let spatial_output = KeyByLabelNames::new(labels.iter().cloned().collect::>()); + + let mut configs = build_agg_configs_for_statistics( + &[Statistic::Cardinality], + QueryTreatmentType::Approximate, + &spatial_output, + &KeyByLabelNames::empty(), + &window_cfg, + table_name, + Some(table_name), + Some(&value_column), + qdata.spatial_filter.as_deref().unwrap_or(""), + |agg_type: AggregationType, agg_sub_type: &str| { + build_sketch_parameters( + agg_type, + agg_sub_type, + None, + None, + self.sketch_parameters.as_ref(), + ) + }, + ) + .map_err(ControllerError::SqlParse)?; + + for cfg in &mut configs { + // Replace the default CARDINALITY/HLL lowering with an exact set-valued + // summary. MOAS needs the actual origin list, not only its cardinality. + cfg.aggregation_type = AggregationType::SetAggregator; + cfg.aggregation_sub_type = "".to_string(); + cfg.parameters.clear(); + cfg.grouping_labels = KeyByLabelNames::new(vec![m.group_by.clone()]); + cfg.aggregated_labels = KeyByLabelNames::new(vec![m.label.clone()]); + cfg.rollup_labels = KeyByLabelNames::empty(); + cfg.value_column = Some("__event_count__".to_string()); + } + + let t_lookback_ms = (q.time_info.get_duration() * 1000.0).round() as u64; + let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { + None + } else { + Some( + get_sql_cleanup_param(self.cleanup_policy, t_lookback_ms, self.t_repeat_ms) + .map_err(ControllerError::PlannerError)?, + ) + }; + + let computed_labels = match m.computed_label { + Some((source_col, filter_regex)) => vec![( + m.label.clone(), + ComputedLabelConfig { + r#type: "token_select".to_string(), + source_col, + tokenizer: Some("whitespace".to_string()), + filter_regex: Some(filter_regex), + select: Some("last".to_string()), + on_missing: Some("skip_sample".to_string()), + }, + )], + None => Vec::new(), + }; + + Ok((configs, cleanup_param, None, Some(surrogate), computed_labels)) + } + + fn get_lag_transition_streaming_aggregation_configs( + &self, + schema: &SQLSchema, + query_evaluation_time: f64, + ) -> Result< + ( + Vec, + Option, + Option, + Option, + ), + ControllerError, + > { + let m = parse_lag_transition_query(&self.query_string).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse lag-transition SQL query: {}", + self.query_string + )) + })?; + + let derived_metric = m.derived_metric(); + + // Same surrogate-query trick as MOAS above: translate the pattern into + // a plain query over its derived stream, then let the ordinary + // classic-path machinery (parse -> match -> build_agg_configs_for_statistics) + // do the rest, so top-k/ordinary-count selection logic isn't duplicated. + // build_lag_transition_surrogate is the same function the query engine + // calls at serve time (via rewrite_lag_transition_query) - using the + // shared builder instead of a local format! is what guarantees the + // registered template and the runtime rewrite can never drift apart. + let surrogate = build_lag_transition_surrogate(&m); + + let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &surrogate) + .map_err(|e| ControllerError::SqlParse(e.to_string()))?; + + let parser = SQLPatternParser::new(schema, query_evaluation_time); + let qdata = parser.parse_query(&stmts).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse lag-transition surrogate query: {surrogate}" + )) + })?; + + let sql_query = SQLPatternMatcher::new( + schema.clone(), + self.data_ingestion_interval_ms as f64 / 1000.0, + ) + .query_info_to_pattern(&qdata); + + if !sql_query.is_valid() { + return Err(ControllerError::SqlParse(sql_query.msg.unwrap_or_default())); + } + if sql_query.query_data.len() != 1 { + return Err(ControllerError::SqlParse(format!( + "Lag-transition surrogate must produce one query layer, got {}", + sql_query.query_data.len() + ))); + } + + let q = &sql_query.query_data[0]; + let agg_info = &q.aggregation_info; + let labels = &q.labels; + let table_name = &q.metric; + let value_column = agg_info.get_value_column_name().to_string(); + + let window_cfg = compute_sql_window( + &q.time_info, + self.data_ingestion_interval_ms, + self.t_repeat_ms, + )?; + let spatial_output = KeyByLabelNames::new(labels.iter().cloned().collect::>()); + + let sql_topk = detect_sql_topk(&qdata); + let treatment_type = get_sql_treatment_type(agg_info.get_name()); + let statistics = if sql_topk.is_some() { + vec![Statistic::Topk] + } else { + get_sql_statistics(agg_info.get_name())? + }; + let topk_k = sql_topk.map(|t| t.k); + let topk_count_events = sql_topk.map(|t| t.count_events()); + + let mut configs = build_agg_configs_for_statistics( + &statistics, + treatment_type, + &spatial_output, + &KeyByLabelNames::empty(), + &window_cfg, + table_name, + Some(table_name), + Some(&value_column), + qdata.spatial_filter.as_deref().unwrap_or(""), + |agg_type: AggregationType, agg_sub_type: &str| { + build_sketch_parameters( + agg_type, + agg_sub_type, + topk_k, + topk_count_events, + self.sketch_parameters.as_ref(), + ) + }, + ) + .map_err(ControllerError::SqlParse)?; + + if sql_topk.is_some() { + for cfg in &mut configs { + if cfg.aggregation_type == AggregationType::CountMinSketchWithHeap { + cfg.grouping_labels = KeyByLabelNames::empty(); + cfg.aggregated_labels = spatial_output.clone(); + } + } + } + + let t_lookback_ms = (q.time_info.get_duration() * 1000.0).round() as u64; + let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { + None + } else { + Some( + get_sql_cleanup_param(self.cleanup_policy, t_lookback_ms, self.t_repeat_ms) + .map_err(ControllerError::PlannerError)?, + ) + }; + + let stateful_transition = StatefulTransitionConfig { + metric_name: derived_metric, + partition_by: m.partition_by, + state_column: m.state_column, + previous_alias: m.previous_alias, + predicate: m.predicate, + emit_labels: vec![m.group_label], + }; + + Ok(( + configs, + cleanup_param, + Some(stateful_transition), + Some(surrogate), + )) + } + + fn get_token_select_streaming_aggregation_configs( + &self, + _schema: &SQLSchema, + query_evaluation_time: f64, + ) -> Result< + ( + Vec, + Option, + Option, + Option, + Vec<(String, ComputedLabelConfig)>, + ), + ControllerError, + > { + let m = parse_token_select_query(&self.query_string).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse token-select SQL query: {}", + self.query_string + )) + })?; + + // The computed label (e.g. origin_asn) doesn't exist as a real column + // in any table definition - it's synthesized at ingest time by the + // ComputedLabelConfig this function also emits. Schema validation + // needs to already know about it, or the surrogate below fails with + // "attempt to aggregate by columns {label}, which are not present + // for metric X" even though the label will genuinely exist by the + // time ingest runs. Register it on a cloned schema used only for + // planning this one query. + let mut augmented_tables = self.table_definitions.clone(); + for t in &mut augmented_tables { + if !t.metadata_columns.iter().any(|c| c == &m.label) { + t.metadata_columns.push(m.label.clone()); + } + } + let schema = &build_sql_schema(&augmented_tables); + + // Treat the computed label as an ordinary column of the base metric: + // the label itself replaces the nested subquery entirely, and + // {where_clause} is the inner subquery's real filter (the outer + // `WHERE length(...) > 0` guard is dropped - that's exactly what + // on_missing: skip_sample already means at ingest time). Shared + // builder, same reasoning as the lag-transition path above. + let surrogate = build_token_select_surrogate(&m); + + let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &surrogate) + .map_err(|e| ControllerError::SqlParse(e.to_string()))?; + + let parser = SQLPatternParser::new(schema, query_evaluation_time); + let qdata = parser.parse_query(&stmts).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse token-select surrogate query: {surrogate}" + )) + })?; + + let sql_query = SQLPatternMatcher::new( + schema.clone(), + self.data_ingestion_interval_ms as f64 / 1000.0, + ) + .query_info_to_pattern(&qdata); + + if !sql_query.is_valid() { + return Err(ControllerError::SqlParse(sql_query.msg.unwrap_or_default())); + } + if sql_query.query_data.len() != 1 { + return Err(ControllerError::SqlParse(format!( + "Token-select surrogate must produce one query layer, got {}", + sql_query.query_data.len() + ))); + } + + let q = &sql_query.query_data[0]; + let agg_info = &q.aggregation_info; + let labels = &q.labels; + let table_name = &q.metric; + let value_column = agg_info.get_value_column_name().to_string(); + + let window_cfg = compute_sql_window( + &q.time_info, + self.data_ingestion_interval_ms, + self.t_repeat_ms, + )?; + let spatial_output = KeyByLabelNames::new(labels.iter().cloned().collect::>()); + + let sql_topk = detect_sql_topk(&qdata); + let treatment_type = get_sql_treatment_type(agg_info.get_name()); + let statistics = if sql_topk.is_some() { + vec![Statistic::Topk] + } else { + get_sql_statistics(agg_info.get_name())? + }; + let rollup = if statistics.contains(&Statistic::Cardinality) { + KeyByLabelNames::empty() + } else { + get_all_metadata_columns(&augmented_tables, table_name)?.difference(&spatial_output) + }; + let topk_k = sql_topk.map(|t| t.k); + let topk_count_events = sql_topk.map(|t| t.count_events()); + + let mut configs = build_agg_configs_for_statistics( + &statistics, + treatment_type, + &spatial_output, + &rollup, + &window_cfg, + table_name, + Some(table_name), + Some(&value_column), + qdata.spatial_filter.as_deref().unwrap_or(""), + |agg_type: AggregationType, agg_sub_type: &str| { + build_sketch_parameters( + agg_type, + agg_sub_type, + topk_k, + topk_count_events, + self.sketch_parameters.as_ref(), + ) + }, + ) + .map_err(ControllerError::SqlParse)?; + + if sql_topk.is_some() { + for cfg in &mut configs { + if cfg.aggregation_type == AggregationType::CountMinSketchWithHeap { + cfg.grouping_labels = KeyByLabelNames::empty(); + cfg.aggregated_labels = spatial_output.clone(); + } + } + } + + let t_lookback_ms = (q.time_info.get_duration() * 1000.0).round() as u64; + let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { + None + } else { + Some( + get_sql_cleanup_param(self.cleanup_policy, t_lookback_ms, self.t_repeat_ms) + .map_err(ControllerError::PlannerError)?, + ) + }; + + let computed_label = ComputedLabelConfig { + r#type: "token_select".to_string(), + source_col: m.source_col, + tokenizer: Some("whitespace".to_string()), + filter_regex: Some(m.filter_regex), + select: Some("last".to_string()), + on_missing: Some("skip_sample".to_string()), + }; + + Ok(( + configs, + cleanup_param, + None, + Some(surrogate), + vec![(m.label, computed_label)], + )) + } + + fn get_token_explode_streaming_aggregation_configs( + &self, + _schema: &SQLSchema, + query_evaluation_time: f64, + ) -> Result< + ( + Vec, + Option, + Option, + Option, + Vec<(String, ComputedLabelConfig)>, + ), + ControllerError, + > { + let m = parse_token_explode_query(&self.query_string).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse token-explode SQL query: {}", + self.query_string + )) + })?; + + let mut augmented_tables = self.table_definitions.clone(); + for t in &mut augmented_tables { + if !t.metadata_columns.iter().any(|c| c == &m.label) { + t.metadata_columns.push(m.label.clone()); + } + } + let schema = &build_sql_schema(&augmented_tables); + + // Shared builder - same function the query engine calls at serve time. + let surrogate = build_token_explode_surrogate(&m); + + let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &surrogate) + .map_err(|e| ControllerError::SqlParse(e.to_string()))?; + + let parser = SQLPatternParser::new(schema, query_evaluation_time); + let qdata = parser.parse_query(&stmts).ok_or_else(|| { + ControllerError::SqlParse(format!( + "Failed to parse token-explode surrogate query: {surrogate}" + )) + })?; + + let sql_query = SQLPatternMatcher::new( + schema.clone(), + self.data_ingestion_interval_ms as f64 / 1000.0, + ) + .query_info_to_pattern(&qdata); + + if !sql_query.is_valid() { + return Err(ControllerError::SqlParse(sql_query.msg.unwrap_or_default())); + } + if sql_query.query_data.len() != 1 { + return Err(ControllerError::SqlParse(format!( + "Token-explode surrogate must produce one query layer, got {}", + sql_query.query_data.len() + ))); + } + + let q = &sql_query.query_data[0]; + let agg_info = &q.aggregation_info; + let labels = &q.labels; + let table_name = &q.metric; + let value_column = agg_info.get_value_column_name().to_string(); + + let window_cfg = compute_sql_window( + &q.time_info, + self.data_ingestion_interval_ms, + self.t_repeat_ms, + )?; + let spatial_output = KeyByLabelNames::new(labels.iter().cloned().collect::>()); + + let sql_topk = detect_sql_topk(&qdata); + let treatment_type = get_sql_treatment_type(agg_info.get_name()); + let statistics = if sql_topk.is_some() { + vec![Statistic::Topk] + } else { + get_sql_statistics(agg_info.get_name())? + }; + let rollup = if statistics.contains(&Statistic::Cardinality) { + KeyByLabelNames::empty() + } else { + get_all_metadata_columns(&augmented_tables, table_name)?.difference(&spatial_output) + }; + let topk_k = sql_topk.map(|t| t.k); + let topk_count_events = sql_topk.map(|t| t.count_events()); + + let mut configs = build_agg_configs_for_statistics( + &statistics, + treatment_type, + &spatial_output, + &rollup, + &window_cfg, + table_name, + Some(table_name), + Some(&value_column), + qdata.spatial_filter.as_deref().unwrap_or(""), + |agg_type: AggregationType, agg_sub_type: &str| { + build_sketch_parameters( + agg_type, + agg_sub_type, + topk_k, + topk_count_events, + self.sketch_parameters.as_ref(), + ) + }, + ) + .map_err(ControllerError::SqlParse)?; + + if sql_topk.is_some() { + for cfg in &mut configs { + if cfg.aggregation_type == AggregationType::CountMinSketchWithHeap { + cfg.grouping_labels = KeyByLabelNames::empty(); + cfg.aggregated_labels = spatial_output.clone(); + } + } + } + + let t_lookback_ms = (q.time_info.get_duration() * 1000.0).round() as u64; + let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { + None + } else { + Some( + get_sql_cleanup_param(self.cleanup_policy, t_lookback_ms, self.t_repeat_ms) + .map_err(ControllerError::PlannerError)?, + ) + }; + + let computed_label = ComputedLabelConfig { + r#type: "token_explode".to_string(), + source_col: m.source_col, + tokenizer: Some("whitespace".to_string()), + filter_regex: Some(m.filter_regex), + select: None, + on_missing: Some("skip_sample".to_string()), + }; + + Ok(( + configs, + cleanup_param, + None, + Some(surrogate), + vec![(m.label, computed_label)], + )) + } + + + fn get_bucketed_countif_streaming_aggregation_configs( + &self, + bucketed: &SQLBucketedCountIfQueryData, + ) -> Result<(Vec, Option), ControllerError> { + if bucketed.bucket_ms == 0 { + return Err(ControllerError::PlannerError( + "bucket size must be positive".to_string(), + )); + } + + if bucketed.bucket_ms < self.data_ingestion_interval_ms { + return Err(ControllerError::PlannerError(format!( + "bucket size ({}ms) must be >= data_ingestion_interval_ms ({}ms)", + bucketed.bucket_ms, self.data_ingestion_interval_ms + ))); + } + + if bucketed.bucket_ms % self.data_ingestion_interval_ms != 0 { + return Err(ControllerError::PlannerError(format!( + "bucket size ({}ms) must be a multiple of data_ingestion_interval_ms ({}ms)", + bucketed.bucket_ms, self.data_ingestion_interval_ms + ))); + } + + if bucketed.outputs.is_empty() { + return Err(ControllerError::SqlParse( + "bucketed countIf query has no outputs".to_string(), + )); + } + + let table_name = &bucketed.metric; + + // Bucketed countIf produces scalar counts per bucket. The bucket dimension + // is time, handled by window/range execution, not a metadata label. + // + // Do not roll up over all metadata columns here; otherwise the planner + // creates key-enumeration aggregations for a query whose output is just + // one scalar value per bucket per countIf output. + let spatial_output = KeyByLabelNames::empty(); + let rollup = KeyByLabelNames::empty(); + + let window_cfg = IntermediateWindowConfig { + window_size_ms: bucketed.bucket_ms, + slide_interval_ms: bucketed.bucket_ms, + window_type: WindowType::Tumbling, + }; + + let mut configs = Vec::new(); + + for output in &bucketed.outputs { + let spatial_filter = + combine_spatial_filters(bucketed.base_spatial_filter.as_deref(), &output.filter); + + let mut output_configs = build_agg_configs_for_statistics( + &[Statistic::Count], + get_sql_treatment_type("COUNT"), + &spatial_output, + &rollup, + &window_cfg, + table_name, + Some(table_name), + Some("__event_count__"), + &spatial_filter, + |agg_type: AggregationType, agg_sub_type: &str| { + build_sketch_parameters( + agg_type, + agg_sub_type, + None, + None, + self.sketch_parameters.as_ref(), + ) + }, + ) + .map_err(ControllerError::SqlParse)?; + + // The generic Count path may include a key-enumeration aggregation + // so grouped queries can discover candidate keys. Bucketed countIf + // outputs are scalar counts per time bucket, so no key-discovery + // aggregation is needed. + output_configs.retain(|cfg| { + !matches!( + cfg.aggregation_type, + AggregationType::DeltaSetAggregator | AggregationType::SetAggregator + ) + }); + + configs.append(&mut output_configs); + } + + let t_lookback_ms = (bucketed.time_info.get_duration() * 1000.0).round() as u64; + let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { + None + } else { + Some( + get_sql_cleanup_param(self.cleanup_policy, t_lookback_ms, self.t_repeat_ms) + .map_err(ControllerError::PlannerError)?, + ) + }; + Ok((configs, cleanup_param)) } } +// --------------------------------------------------------------------------- +// Lag-transition pattern (e.g. ClickHouse `lagInFrame(...) OVER (PARTITION BY +// ... ORDER BY ...)` wrapped in a CTE with an outer countIf) → derived event +// stream + StatefulTransitionConfig. +// +// This mirrors the shape the query engine's rewrite_lag_transition_query +// (engines/simple_engine/sql.rs) already produces at serve time - but +// previously nothing on the planning side auto-generated the +// StatefulTransitionConfig that makes that rewrite valid; a human had to +// notice the pattern and hand-write it into streaming_config.yaml. This is +// the automated version: the planner detects the pattern from raw SQL and +// emits both a normal aggregation config (via the same surrogate-query path +// MOAS already uses) and the StatefulTransitionConfig, so the CTE/window- +// function complexity never has to be understood by the rest of the planner. +// --------------------------------------------------------------------------- + +// The lag-transition / token-select / token-explode detectors used to be +// defined here directly; they're now shared with the query engine (which +// needs the identical detection+rewrite logic at serve time) via +// sql_utilities::ast_matching::pattern_rewrites, imported above. + +fn combine_spatial_filters(base: Option<&str>, extra: &str) -> String { + match (base, extra.trim()) { + (Some(b), e) if !b.trim().is_empty() && !e.is_empty() => { + format!("{} AND {}", b.trim(), e) + } + (Some(b), _) if !b.trim().is_empty() => b.trim().to_string(), + (_, e) => e.to_string(), + } +} + fn build_sql_schema(tables: &[TableDefinition]) -> SQLSchema { let table_vec: Vec = tables .iter() diff --git a/asap-planner-rs/src/sql/generator.rs b/asap-planner-rs/src/sql/generator.rs index 560f35f1..db381ca4 100644 --- a/asap-planner-rs/src/sql/generator.rs +++ b/asap-planner-rs/src/sql/generator.rs @@ -1,4 +1,6 @@ +use asap_types::computed_label::ComputedLabelConfig; use asap_types::enums::CleanupPolicy; +use asap_types::stateful_transition::StatefulTransitionConfig; use indexmap::IndexMap; use serde_yaml::Value as YamlValue; use std::collections::HashMap; @@ -7,13 +9,16 @@ use std::time::{SystemTime, UNIX_EPOCH}; use crate::config::input::SQLControllerConfig; use crate::error::ControllerError; use crate::generator::{ - build_aggregation_entry, build_queries_yaml, GeneratorOutput, KEY_AGGREGATIONS, + build_aggregation_entry, build_queries_yaml, GeneratorOutput, PuntedQuery, KEY_AGGREGATIONS, KEY_CLEANUP_POLICY, KEY_METADATA_COLUMNS, KEY_NAME, KEY_QUERIES, KEY_TABLES, KEY_TIME_COLUMN, KEY_VALUE_COLUMNS, }; use crate::planner::agg_config::IntermediateAggConfig; use crate::planner::sql::SQLSingleQueryProcessor; use crate::StreamingEngine; +use sql_utilities::ast_matching::pattern_rewrites::{ + build_multi_aggregate_surrogates, parse_multi_aggregate_query, +}; pub struct SQLRuntimeOptions { pub streaming_engine: StreamingEngine, @@ -74,9 +79,69 @@ pub fn generate_sql_plan( let mut dedup_map: IndexMap = IndexMap::new(); // query_string -> Vec<(key, cleanup_param)> let mut query_keys_map: IndexMap)>> = IndexMap::new(); + // Stateful transitions the planner auto-detected (e.g. lagInFrame queries), + // deduped by derived metric_name - multiple queries referencing the same + // derived stream only need one operator maintaining it. + let mut stateful_transitions: IndexMap = IndexMap::new(); + // Computed labels the planner auto-detected (e.g. origin-ASN extraction), + // deduped by label name. + let mut computed_label_cols: IndexMap = IndexMap::new(); + // Queries with no GROUP BY and no aggregate function - raw/DISTINCT row + // scans no precomputed summary can ever answer. Left out of both + // dedup_map and query_keys_map entirely, so they never appear in + // inference_config.yaml; at query time the local engine simply won't + // recognize them and (if forward_unsupported_queries is enabled) they + // fall through to the ClickHouse fallback for an exact answer. + let mut punted_queries: Vec = Vec::new(); for qg in &config.query_groups { for query_string in &qg.queries { + // Multi-aggregate queries (2+ aggregate expressions over one + // GROUP BY, e.g. `count()` and `uniqExact(...)` side by side) + // can't be represented as a single SQLQueryData - split into N + // independent single-aggregate surrogates and plan + register + // each one separately, so the query engine's ordinary + // structural matcher (find_query_config_sql) can find each + // surrogate directly at serve time. The engine reconstructs the + // identical split from the raw incoming query + // (handle_multi_aggregate_sql) and merges the N results back + // into one row per group. + if let Some(mm) = parse_multi_aggregate_query(query_string) { + for surrogate in build_multi_aggregate_surrogates(&mm) { + let sub_processor = SQLSingleQueryProcessor::new( + surrogate.clone(), + qg.repetition_delay_ms, + opts.data_ingestion_interval_ms, + config.tables.clone(), + opts.streaming_engine, + config.sketch_parameters.clone(), + cleanup_policy, + ); + + let (configs, cleanup_param, stateful_transition, template_override, labels) = + sub_processor.get_streaming_aggregation_configs(eval_time)?; + + if let Some(st) = stateful_transition { + stateful_transitions + .entry(st.metric_name.clone()) + .or_insert(st); + } + for (label_name, cfg) in labels { + computed_label_cols.entry(label_name).or_insert(cfg); + } + + let mut keys_for_query = Vec::new(); + for config_item in configs { + let key = config_item.identifying_key(); + keys_for_query.push((key.clone(), cleanup_param)); + dedup_map.entry(key).or_insert(config_item); + } + let registered_query = template_override.unwrap_or(surrogate); + query_keys_map.insert(registered_query, keys_for_query); + } + continue; + } + let processor = SQLSingleQueryProcessor::new( query_string.clone(), qg.repetition_delay_ms, @@ -87,16 +152,45 @@ pub fn generate_sql_plan( cleanup_policy, ); - let (configs, cleanup_param) = + if processor.is_exact_only() { + tracing::warn!( + query = %query_string, + "punting query: no aggregate function and no GROUP BY, so no \ + precomputed summary can answer it; relying on the ClickHouse \ + fallback (forward_unsupported_queries) for an exact answer" + ); + punted_queries.push(PuntedQuery { + query: query_string.clone(), + }); + continue; + } + + let (configs, cleanup_param, stateful_transition, template_override, labels) = processor.get_streaming_aggregation_configs(eval_time)?; + if let Some(st) = stateful_transition { + stateful_transitions + .entry(st.metric_name.clone()) + .or_insert(st); + } + for (label_name, cfg) in labels { + computed_label_cols.entry(label_name).or_insert(cfg); + } + let mut keys_for_query = Vec::new(); for config_item in configs { let key = config_item.identifying_key(); keys_for_query.push((key.clone(), cleanup_param)); dedup_map.entry(key).or_insert(config_item); } - query_keys_map.insert(query_string.clone(), keys_for_query); + // Some query shapes (lagInFrame CTEs, MOAS's DISTINCT_SET) aren't + // parseable by SQLPatternParser at all, so the raw query can never + // be matched against at query time either - the surrogate that was + // actually planned against must be what's registered here, or the + // query-time matcher will never find this aggregation no matter + // how correctly the runtime rewrites the incoming query. + let registered_query = template_override.unwrap_or_else(|| query_string.clone()); + query_keys_map.insert(registered_query, keys_for_query); } } @@ -106,12 +200,25 @@ pub fn generate_sql_plan( id_map.insert(key.clone(), idx as u32 + 1); } - let streaming_yaml = build_sql_streaming_yaml(config, &dedup_map, &id_map)?; - let inference_yaml = - build_sql_inference_yaml(config, cleanup_policy, &query_keys_map, &id_map)?; + let streaming_yaml = + build_sql_streaming_yaml( + config, + &dedup_map, + &id_map, + &stateful_transitions, + &computed_label_cols, + )?; + let extra_metadata_columns: Vec = computed_label_cols.keys().cloned().collect(); + let inference_yaml = build_sql_inference_yaml( + config, + cleanup_policy, + &query_keys_map, + &id_map, + &extra_metadata_columns, + )?; Ok(GeneratorOutput { - punted_queries: Vec::new(), + punted_queries, streaming_yaml, inference_yaml, aggregation_count: dedup_map.len(), @@ -119,7 +226,20 @@ pub fn generate_sql_plan( }) } -fn build_tables_yaml(config: &SQLControllerConfig) -> Vec { +/// `extra_metadata_columns` is every computed-label name the planner +/// auto-detected (e.g. "origin_asn") across the whole workload. It has to +/// land in the *emitted* tables list, not just be used locally while +/// building one query's aggregation config: this is the schema the engine +/// itself rebuilds from streaming_config.yaml/inference_config.yaml to +/// parse and match incoming queries at serve time. A computed label that's +/// valid enough to plan against but never makes it into this list is +/// invisible to the engine's own schema validation, so even a perfectly +/// rewritten runtime query fails the same "not present for metric" check +/// the planner would have hit without its own local augmentation. +fn build_tables_yaml( + config: &SQLControllerConfig, + extra_metadata_columns: &[String], +) -> Vec { config .tables .iter() @@ -142,10 +262,16 @@ fn build_tables_yaml(config: &SQLControllerConfig) -> Vec { .collect(), ), ); + let mut metadata_columns: Vec = t.metadata_columns.clone(); + for extra in extra_metadata_columns { + if !metadata_columns.iter().any(|c| c == extra) { + metadata_columns.push(extra.clone()); + } + } map.insert( YamlValue::String(KEY_METADATA_COLUMNS.to_string()), YamlValue::Sequence( - t.metadata_columns + metadata_columns .iter() .map(|c| YamlValue::String(c.clone())) .collect(), @@ -160,6 +286,8 @@ fn build_sql_streaming_yaml( config: &SQLControllerConfig, dedup_map: &IndexMap, id_map: &HashMap, + stateful_transitions: &IndexMap, + computed_label_cols: &IndexMap, ) -> Result { let aggregations: Vec = dedup_map .iter() @@ -171,11 +299,39 @@ fn build_sql_streaming_yaml( YamlValue::String(KEY_AGGREGATIONS.to_string()), YamlValue::Sequence(aggregations), ); + let extra_metadata_columns: Vec = computed_label_cols.keys().cloned().collect(); root.insert( YamlValue::String(KEY_TABLES.to_string()), - YamlValue::Sequence(build_tables_yaml(config)), + YamlValue::Sequence(build_tables_yaml(config, &extra_metadata_columns)), ); + if !stateful_transitions.is_empty() { + let entries: Result, ControllerError> = stateful_transitions + .values() + .map(|st| { + serde_yaml::to_value(st) + .map_err(|e| ControllerError::PlannerError(e.to_string())) + }) + .collect(); + root.insert( + YamlValue::String("stateful_transitions".to_string()), + YamlValue::Sequence(entries?), + ); + } + + if !computed_label_cols.is_empty() { + let mut labels_map = serde_yaml::Mapping::new(); + for (label_name, cfg) in computed_label_cols.iter() { + let value = serde_yaml::to_value(cfg) + .map_err(|e| ControllerError::PlannerError(e.to_string()))?; + labels_map.insert(YamlValue::String(label_name.clone()), value); + } + root.insert( + YamlValue::String("computed_label_cols".to_string()), + YamlValue::Mapping(labels_map), + ); + } + Ok(YamlValue::Mapping(root)) } @@ -184,6 +340,7 @@ fn build_sql_inference_yaml( cleanup_policy: CleanupPolicy, query_keys_map: &IndexMap)>>, id_map: &HashMap, + extra_metadata_columns: &[String], ) -> Result { let mut cleanup_map = serde_yaml::Mapping::new(); cleanup_map.insert( @@ -202,7 +359,7 @@ fn build_sql_inference_yaml( ); root.insert( YamlValue::String(KEY_TABLES.to_string()), - YamlValue::Sequence(build_tables_yaml(config)), + YamlValue::Sequence(build_tables_yaml(config, extra_metadata_columns)), ); Ok(YamlValue::Mapping(root)) diff --git a/asap-query-engine/src/bin/precompute_engine.rs b/asap-query-engine/src/bin/precompute_engine.rs index 8f32d080..09925f44 100644 --- a/asap-query-engine/src/bin/precompute_engine.rs +++ b/asap-query-engine/src/bin/precompute_engine.rs @@ -186,6 +186,8 @@ async fn main() -> Result<(), Box> { Arc::new(StoreOutputSink::new(store)) }; + let file_ingest_mode = args.input_file.is_some(); + let sources: Vec> = if let Some(path) = args.input_file { let metric_name = args @@ -213,9 +215,11 @@ async fn main() -> Result<(), Box> { vec![Box::new(CsvFileIngestSource::new(CsvFileIngestConfig { path, metric_name, - value_col, + value_col: Some(value_col), label_cols, timestamp_col: args.csv_timestamp_col, + computed_label_cols: std::collections::HashMap::new(), + stateful_transitions: Vec::new(), start_ts_ms: args.csv_start_ts_ms, ts_step_ms, batch_size: args.csv_batch_size, @@ -229,8 +233,22 @@ async fn main() -> Result<(), Box> { // Build and run the engine let engine = PrecomputeEngine::new(engine_config, streaming_config, output_sink, sources); + // In file-ingest mode, CSV ingestion is finite. The CSV source flushes + // active windows and gracefully shuts down the precompute workers when + // EOF is reached. Keep the process alive afterward so the HTTP query + // server can continue serving the materialized precomputed store. + let keep_query_server_alive = file_ingest_mode && args.query_port != 0; + info!("Starting precompute engine..."); engine.run().await?; + if keep_query_server_alive { + info!( + "File ingest/precompute complete; query server remains available on port {}", + args.query_port + ); + std::future::pending::<()>().await; + } + Ok(()) } diff --git a/asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs b/asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs index 19c3bc5d..d20059e4 100644 --- a/asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs +++ b/asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs @@ -123,9 +123,29 @@ impl QueryResponseAdapter for ClickHouseHttpAdapter { output.push('\n'); } } - QueryResult::Matrix(_) => { - // ClickHouse adapter doesn't support Matrix results - return Err(StatusCode::NOT_IMPLEMENTED); + QueryResult::Matrix(range_vector) => { + // Format range-vector results as TabSeparated rows: + // + // output_labeltimestamp_msvalue + // + // For bucketed SQL queries, each range-vector element is one output + // column such as "announcements" or "withdrawals", and each sample + // timestamp is the bucket timestamp. + for element in &range_vector.values { + for sample in &element.samples { + for (i, _label_name) in label_names.iter().enumerate() { + let label_value = + element.labels.get(i).map(|s| s.as_str()).unwrap_or(""); + output.push_str(label_value); + output.push('\t'); + } + + output.push_str(&sample.timestamp.to_string()); + output.push('\t'); + output.push_str(&sample.value.to_string()); + output.push('\n'); + } + } } }; diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs index 7fe38a39..d551ad58 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/asap-query-engine/src/drivers/query/servers/http.rs @@ -178,20 +178,28 @@ async fn process_query_request( } } + let local_engine_query = SimpleEngine::rewrite_recognized_pattern(&parsed_request.query); + if local_engine_query != parsed_request.query { + tracing::warn!( + "HTTP pattern rewrite produced local SQL: {}", + local_engine_query + ); + } + // Record query for passive auto-discovery (if tracker is enabled) if let Some(tracker) = &state.query_tracker { - tracker.record_instant(&parsed_request.query, parsed_request.time); + tracker.record_instant(&local_engine_query, parsed_request.time); } // Step 2: Execute query with engine (using parsed request) let query_start_time = Instant::now(); debug!( "About to call query_engine.handle_query with query='{}' and time={}", - parsed_request.query, parsed_request.time + local_engine_query, parsed_request.time ); match state .query_engine - .handle_query(parsed_request.query.clone(), parsed_request.time) + .handle_query(local_engine_query.clone(), parsed_request.time) { Some((query_output_labels, query_result)) => { let query_duration = query_start_time.elapsed(); diff --git a/asap-query-engine/src/engine_config.rs b/asap-query-engine/src/engine_config.rs index 8ef1ef65..e33c6c98 100644 --- a/asap-query-engine/src/engine_config.rs +++ b/asap-query-engine/src/engine_config.rs @@ -1,5 +1,8 @@ use asap_types::enums::QueryLanguage; use query_engine_rust::data_model::enums::{InputFormat, LockStrategy, StreamingEngine}; +use query_engine_rust::precompute_engine::computed_labels::ComputedLabelConfig; +use query_engine_rust::precompute_engine::stateful_transition::StatefulTransitionConfig; +use std::collections::HashMap; pub fn check_config(config: &EngineConfig) -> Result<(), String> { match (&config.ingest, &config.streaming_engine) { @@ -235,9 +238,14 @@ pub enum IngestConfig { Csv { path: String, metric_name: String, - value_col: String, + #[serde(default)] + value_col: Option, #[serde(default)] label_cols: Vec, + #[serde(default)] + computed_label_cols: HashMap, + #[serde(default)] + stateful_transitions: Vec, timestamp_col: Option, #[serde(default)] start_ts_ms: i64, diff --git a/asap-query-engine/src/engines/simple_engine/sql.rs b/asap-query-engine/src/engines/simple_engine/sql.rs index 24c08c84..c43c2456 100644 --- a/asap-query-engine/src/engines/simple_engine/sql.rs +++ b/asap-query-engine/src/engines/simple_engine/sql.rs @@ -3,9 +3,11 @@ //! Contains all SQL-specific context building, pattern matching, and query dispatch. use super::SimpleEngine; -use super::{QueryExecutionContext, QueryMetadata, QueryTimestamps}; -use crate::data_model::{AggregationIdInfo, QueryConfig, SchemaConfig}; -use crate::engines::query_result::{InstantVector, InstantVectorElement, QueryResult}; +use super::{QueryExecutionContext, QueryMetadata, QueryTimestamps, StoreQueryParams}; +use crate::data_model::{AggregationIdInfo, KeyByLabelValues, QueryConfig, SchemaConfig}; +use crate::engines::query_result::{ + InstantVector, InstantVectorElement, QueryResult, RangeVectorElement, +}; use asap_types::query_requirements::QueryRequirements; use asap_types::utils::normalize_spatial_filter; use promql_utilities::data_model::KeyByLabelNames; @@ -13,10 +15,15 @@ use promql_utilities::query_logics::enums::Statistic; use sql_utilities::ast_matching::{ detect_sql_topk, SQLPatternMatcher, SQLPatternParser, SQLQuery, SqlTopk, TopkWeighting, }; -use sql_utilities::sqlhelper::{OrderByItem, SQLQueryData}; +use sql_utilities::ast_matching::pattern_rewrites::{ + build_moas_surrogate, build_multi_aggregate_surrogates, looks_like_moas_registered_sql, + looks_like_moas_sql, parse_moas_query, parse_multi_aggregate_query, + rewrite_lag_transition_query, rewrite_token_explode_query, rewrite_token_select_query, +}; +use sql_utilities::sqlhelper::{OrderByItem, SQLBucketedCountIfQueryData, SQLQueryData}; use sqlparser::dialect::*; use sqlparser::parser::Parser as parser; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use tracing::{debug, warn}; /// SQL-only post-processing produced alongside a `QueryExecutionContext`: @@ -276,6 +283,20 @@ impl SimpleEngine { query: String, time: f64, ) -> Option<(KeyByLabelNames, QueryResult)> { + let query = Self::rewrite_recognized_pattern(&query); + + if let Some(result) = self.handle_moas_sql(&query, time) { + return Some(result); + } + + if let Some(result) = self.handle_bucketed_countif_sql(&query, time) { + return Some(result); + } + + if let Some(result) = self.handle_multi_aggregate_sql(&query, time) { + return Some(result); + } + let (context, post) = self.build_query_execution_context_sql_with_post_processing(query, time)?; let is_topk = context.metadata.statistic_to_compute == Statistic::Topk; @@ -292,6 +313,40 @@ impl SimpleEngine { Some((output_labels, result)) } + /// Tries each recognized complex-SQL-shape rewrite in turn (lag- + /// transition, token-select, token-explode) and returns the first one + /// that fires, or the original query unchanged if none do. These are + /// the same detectors asap-planner-rs uses to decide what to build - + /// shared via sql_utilities::ast_matching::pattern_rewrites so the two + /// can never silently disagree about what a given raw query means. + pub(crate) fn rewrite_recognized_pattern(sql: &str) -> String { + // MOAS has its own detection + surrogate-building (parse_moas_query / + // handle_moas_sql) that must run on the *raw* query text. The + // tokenized MOAS shape also satisfies looks_like_token_select_sql + // (both tokenize as_path and index [-1]), so without this early + // return the token-select rewrite below would mangle a MOAS query + // into a broken surrogate before MOAS ever saw the original text - + // and every caller of this function (not just handle_query_sql) + // needs that guarantee, since this is the single shared rewrite + // entry point. + if looks_like_moas_sql(sql) { + return sql.to_string(); + } + if let Some(rewritten) = rewrite_lag_transition_query(sql) { + warn!("lag-transition rewrite produced SQL: {}", rewritten); + return rewritten; + } + if let Some(rewritten) = rewrite_token_select_query(sql) { + warn!("token-select rewrite produced SQL: {}", rewritten); + return rewritten; + } + if let Some(rewritten) = rewrite_token_explode_query(sql) { + warn!("token-explode rewrite produced SQL: {}", rewritten); + return rewritten; + } + sql.to_string() + } + /// Public entry point retained for tests that only need the execution /// context (e.g. assertions on `agg_info` or `metadata`). Discards the /// SQL post-processing side-channel since it isn't applied without a @@ -305,6 +360,428 @@ impl SimpleEngine { .map(|(ctx, _)| ctx) } + fn find_query_config_sql_moas(&self) -> Option { + let ic = self.inference_config.read().unwrap(); + + ic.query_configs + .iter() + .find(|config| looks_like_moas_registered_sql(&config.query)) + .cloned() + } + + fn handle_moas_sql(&self, query: &str, time: f64) -> Option<(KeyByLabelNames, QueryResult)> { + if !looks_like_moas_sql(query) { + return None; + } + + warn!("MOAS handler matched query"); + + let schema = match &self.inference_config.read().unwrap().schema { + SchemaConfig::SQL(sql_schema) => sql_schema.clone(), + SchemaConfig::PromQL(_) => { + warn!("MOAS handler: non-SQL schema"); + return None; + } + &SchemaConfig::ElasticQueryDSL(_) => { + warn!("MOAS handler: ElasticQueryDSL schema"); + return None; + } + SchemaConfig::ElasticSQL(sql_schema) => sql_schema.clone(), + }; + + let m = match parse_moas_query(query) { + Some(m) => m, + None => { + warn!("MOAS handler: could not parse MOAS query"); + return None; + } + }; + let surrogate = build_moas_surrogate(&m); + warn!("MOAS surrogate query: {}", surrogate); + + let statements = match parser::parse_sql(&GenericDialect {}, surrogate.as_str()) { + Ok(statements) => statements, + Err(e) => { + warn!("MOAS handler: could not parse surrogate query: {}", e); + return None; + } + }; + + let query_data = match SQLPatternParser::new(&schema, time).parse_query(&statements) { + Some(qd) => qd, + None => { + warn!("MOAS handler: SQLPatternParser rejected surrogate query"); + return None; + } + }; + + let query_config = match self.find_query_config_sql_moas() { + Some(config) => config, + None => { + warn!("MOAS handler: no MOAS query_config found"); + return None; + } + }; + + let aggregation_id = match query_config.aggregations.first() { + Some(agg) => agg.aggregation_id, + None => { + warn!("MOAS handler: query_config has no aggregations"); + return None; + } + }; + + warn!( + "MOAS handler: using aggregation_id={} metric={} duration_s={}", + aggregation_id, + query_data.metric, + query_data.time_info.get_duration() + ); + + let end_timestamp = self.align_end_timestamp_sql(Self::convert_query_time_to_data_time( + query_data.time_info.get_start() + query_data.time_info.get_duration(), + )); + let duration_ms = (query_data.time_info.get_duration() * 1000.0).round() as u64; + let start_timestamp = match end_timestamp.checked_sub(duration_ms) { + Some(ts) => ts, + None => { + warn!("MOAS handler: invalid start/end timestamps"); + return None; + } + }; + + let params = StoreQueryParams { + metric: query_data.metric.clone(), + aggregation_id, + start_timestamp, + end_timestamp, + is_exact_query: false, + }; + + let timestamped_map = match self.execute_store_query(¶ms) { + Ok(map) => map, + Err(e) => { + warn!("MOAS handler: store query failed: {}", e); + return None; + } + }; + + warn!( + "MOAS handler: store returned {} {} groups", + timestamped_map.len(), + m.group_by + ); + + let mut rows: Vec<(String, Vec)> = Vec::new(); + + for (group_key, timestamped_buckets) in timestamped_map { + let Some(group_key_values) = group_key else { + continue; + }; + + let group_value = group_key_values + .get(0) + .map(|s| s.to_string()) + .unwrap_or_default(); + if group_value.is_empty() { + continue; + } + + let mut origins: HashSet = HashSet::new(); + + for (_bucket, precompute) in timestamped_buckets { + if let Some(keys) = precompute.get_keys() { + for key in keys { + if let Some(origin) = key.get(0) { + if !origin.is_empty() { + origins.insert(origin.to_string()); + } + } + } + } + } + + if origins.len() > 1 { + let mut origins_vec: Vec = origins.into_iter().collect(); + origins_vec.sort(); + rows.push((group_value, origins_vec)); + } + } + + warn!("MOAS handler: produced {} MOAS rows", rows.len()); + + rows.sort_by(|a, b| b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0))); + + let values: Vec = rows + .into_iter() + .map(|(group_value, origins)| { + let origin_count = origins.len() as f64; + let origins_joined = origins.join(","); + InstantVectorElement::new( + KeyByLabelValues::new_with_labels(vec![group_value, origins_joined]), + origin_count, + ) + }) + .collect(); + + let output_labels = KeyByLabelNames::new(vec![m.group_by.clone(), "origins".to_string()]); + + Some((output_labels, QueryResult::vector(values, end_timestamp))) + } + + /// Finds the query configuration for a bucketed countIf SQL query. + /// + /// This is parallel to `find_query_config_sql`: the classic SQL path matches + /// one aggregate, while bucketed countIf has one time-bucket expression and + /// multiple independent count outputs. + fn find_query_config_sql_bucketed( + &self, + query_data: &SQLBucketedCountIfQueryData, + ) -> Option { + let ic = self.inference_config.read().unwrap(); + let schema = match &ic.schema { + SchemaConfig::SQL(sql_schema) => sql_schema.clone(), + SchemaConfig::ElasticSQL(sql_schema) => sql_schema.clone(), + _ => return None, + }; + + ic.query_configs + .iter() + .find(|config| { + let template_statements = + match parser::parse_sql(&GenericDialect {}, config.query.as_str()) { + Ok(stmts) => stmts, + Err(_) => return false, + }; + + let template_data = match SQLPatternParser::new(&schema, 0.0) + .parse_bucketed_countif_query(&template_statements) + { + Some(data) => data, + None => return false, + }; + + query_data.matches_bucketed_pattern(&template_data) + }) + .cloned() + } + + fn handle_bucketed_countif_sql( + &self, + query: &str, + time: f64, + ) -> Option<(KeyByLabelNames, QueryResult)> { + let schema = match &self.inference_config.read().unwrap().schema { + SchemaConfig::SQL(sql_schema) => sql_schema.clone(), + SchemaConfig::PromQL(_) => return None, + &SchemaConfig::ElasticQueryDSL(_) => return None, + SchemaConfig::ElasticSQL(sql_schema) => sql_schema.clone(), + }; + + let statements = match parser::parse_sql(&GenericDialect {}, query) { + Ok(statements) => statements, + Err(_) => return None, + }; + + let bucketed = + SQLPatternParser::new(&schema, time).parse_bucketed_countif_query(&statements)?; + + let query_config = self.find_query_config_sql_bucketed(&bucketed)?; + + if query_config.aggregations.len() < bucketed.outputs.len() { + warn!( + "Bucketed countIf query has {} outputs but query_config only has {} aggregation refs", + bucketed.outputs.len(), + query_config.aggregations.len() + ); + return None; + } + + let end_timestamp = self.align_end_timestamp_sql(Self::convert_query_time_to_data_time( + bucketed.time_info.get_start() + bucketed.time_info.get_duration(), + )); + let duration_ms = (bucketed.time_info.get_duration() * 1000.0).round() as u64; + let raw_start_timestamp = end_timestamp.checked_sub(duration_ms)?; + + // The precompute store keys tumbling windows by bucket_ms-aligned + // absolute timestamps (bucket_start_ts = floor(t / bucket_ms) * bucket_ms). + // raw_start_timestamp is only guaranteed aligned to + // data_ingestion_interval_ms via align_end_timestamp_sql above, which can + // differ from bucket_ms. Without this, the read loop below probes + // timestamps that never land on a real stored key and every sample + // silently reads back as 0.0. + let start_timestamp = if bucketed.bucket_ms == 0 { + raw_start_timestamp + } else { + (raw_start_timestamp / bucketed.bucket_ms) * bucketed.bucket_ms + }; + + let mut range_elements = Vec::new(); + + for (idx, output) in bucketed.outputs.iter().enumerate() { + let aggregation_id = query_config.aggregations[idx].aggregation_id; + + let params = StoreQueryParams { + metric: bucketed.metric.clone(), + aggregation_id, + start_timestamp, + end_timestamp, + is_exact_query: false, + }; + + let timestamped_map = self + .execute_store_query(¶ms) + .map_err(|e| { + warn!( + "Failed to execute bucketed countIf store query for '{}': {}", + output.alias, e + ); + e + }) + .ok()?; + + let mut values_by_bucket: HashMap = HashMap::new(); + for (_key, timestamped_buckets) in timestamped_map { + for ((bucket_start_ts, _bucket_end_ts), precompute) in timestamped_buckets { + let scalar_key = Some(KeyByLabelValues::new_with_labels(vec![])); + + let value = self + .query_precompute_for_statistic( + precompute.as_ref(), + &Statistic::Count, + &scalar_key, + &HashMap::new(), + ) + .map_err(|e| { + warn!( + "Failed to query bucketed countIf precompute for '{}': {}", + output.alias, e + ); + e + }) + .ok()?; + + values_by_bucket.insert(bucket_start_ts, value); + } + } + + let mut element = + RangeVectorElement::new(KeyByLabelValues::new_with_labels(vec![output + .alias + .clone()])); + + let mut ts = start_timestamp; + while ts < end_timestamp { + let value = values_by_bucket.get(&ts).copied().unwrap_or(0.0); + element.add_sample(ts, value); + ts += bucketed.bucket_ms; + } + + range_elements.push(element); + } + + let output_labels = KeyByLabelNames::new(vec!["output".to_string()]); + Some((output_labels, QueryResult::matrix(range_elements))) + } + + /// Alias of a ` AS ` SELECT-list item, e.g. + /// "uniqExact(prefix) AS distinct_prefixes" -> "distinct_prefixes". + fn alias_of(expr: &str) -> Option { + let lower = expr.to_lowercase(); + let as_idx = lower.rfind(" as ")?; + Some(expr[as_idx + 4..].trim().to_string()) + } + + /// Serves a multi-aggregate classic query (e.g. `SELECT k1, k2, count() + /// AS a, uniqExact(v) AS b FROM ... GROUP BY k1, k2`) by splitting it + /// into N independent single-aggregate surrogates - the same split the + /// planner uses to plan and separately register each one (see + /// generate_sql_plan in the planner crate) - and running each through + /// the existing single-aggregate execution path unchanged. Each + /// surrogate's own text is what's registered in inference_config.yaml, + /// so the ordinary structural matcher (find_query_config_sql) finds it + /// directly. Results are merged back into one row per group key: the + /// first aggregate's value stays the primary numeric value, and every + /// other aggregate's value is appended to the output labels (matching + /// the format handle_moas_sql already established for "more than one + /// output value per row"). + fn handle_multi_aggregate_sql( + &self, + query: &str, + time: f64, + ) -> Option<(KeyByLabelNames, QueryResult)> { + let m = parse_multi_aggregate_query(query)?; + + warn!( + "multi-aggregate handler matched query with {} aggregates", + m.aggregate_exprs.len() + ); + + let surrogates = build_multi_aggregate_surrogates(&m); + + let mut per_aggregate: Vec<(KeyByLabelNames, InstantVector)> = Vec::new(); + for surrogate in &surrogates { + let Some((context, post)) = + self.build_query_execution_context_sql_with_post_processing(surrogate.clone(), time) + else { + warn!("multi-aggregate handler: failed to build execution context for surrogate"); + return None; + }; + let Some((output_labels, result)) = self.execute_context(context, false, false) else { + warn!("multi-aggregate handler: failed to execute context for surrogate"); + return None; + }; + let result = post.apply(&output_labels, result); + let QueryResult::Vector(vector) = result else { + warn!("multi-aggregate handler: expected instant vector result"); + return None; + }; + per_aggregate.push((output_labels, vector)); + } + + let mut iter = per_aggregate.into_iter(); + // The grouping-label *order* here comes from whichever surrogate's + // own execution context produced it (KeyByLabelNames may reorder + // relative to the SELECT list), not from `m.group_by_cols` - the + // element values below are keyed by that same order, so the output + // label names must track it exactly or labels and values misalign. + let (first_labels, first_vector) = iter.next()?; + let end_timestamp = first_vector.timestamp; + + let extra_maps: Vec, f64>> = iter + .map(|(_labels, vector)| { + vector + .values + .into_iter() + .map(|element| (element.labels.labels, element.value)) + .collect() + }) + .collect(); + + let mut output_label_names = first_labels.labels.clone(); + for expr in m.aggregate_exprs.iter().skip(1) { + output_label_names.push(Self::alias_of(expr).unwrap_or_else(|| expr.clone())); + } + let output_labels = KeyByLabelNames::new(output_label_names); + + let values: Vec = first_vector + .values + .into_iter() + .map(|element| { + let mut label_values = element.labels.labels.clone(); + for map in &extra_maps { + let v = map.get(&element.labels.labels).copied().unwrap_or(0.0); + label_values.push(v.to_string()); + } + InstantVectorElement::new(KeyByLabelValues::new_with_labels(label_values), element.value) + }) + .collect(); + + warn!("multi-aggregate handler: produced {} rows", values.len()); + + Some((output_labels, QueryResult::vector(values, end_timestamp))) + } + /// Internal: parses + plans a SQL query and returns both the execution /// context (shared with PromQL/Elastic engines) and the SQL-only /// post-processing rules (ORDER BY / LIMIT / alias resolution). @@ -313,6 +790,8 @@ impl SimpleEngine { query: String, time: f64, ) -> Option<(QueryExecutionContext, SqlPostProcessing)> { + let query = Self::rewrite_recognized_pattern(&query); + // Get SQL schema from inference config let schema = match &self.inference_config.read().unwrap().schema { SchemaConfig::SQL(sql_schema) => sql_schema.clone(), @@ -596,6 +1075,7 @@ mod detect_topk_tests { ), aggregation_alias: Some("transfer_events".to_string()), metric: "netflow_table".to_string(), + spatial_filter: None, labels: HashSet::from(["srcip".to_string()]), time_info: TimeInfo::new("time".to_string(), 0.0, 1.0), subquery: None, diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index f00a1135..497c6f01 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -218,10 +218,12 @@ async fn main() -> Result<()> { metric_name, value_col, label_cols, + computed_label_cols, timestamp_col, start_ts_ms, ts_step_ms, batch_size, + stateful_transitions, } => { // ts_step_ms is only used for timestamp synthesis (when timestamp_col is absent). // check_config ensures it is present in that case. @@ -231,11 +233,68 @@ async fn main() -> Result<()> { 0 }; info!("File ingest mode: {}", path); + // Merge stateful transitions the planner auto-detected (carried in + // streaming_config.yaml) with any hand-written in this engine + // config's ingest section, deduped by metric_name so a manual + // override still wins if both specify the same derived stream. + // Previously stateful_transitions only ever came from the + // hand-written side - the planner had no way to contribute here. + let mut merged_stateful_transitions = stateful_transitions.clone(); + let manual_metric_names: std::collections::HashSet<&str> = + stateful_transitions.iter().map(|t| t.metric_name.as_str()).collect(); + for auto_detected in &streaming_config.stateful_transitions { + if !manual_metric_names.contains(auto_detected.metric_name.as_str()) { + merged_stateful_transitions.push(auto_detected.clone()); + } + } + if !streaming_config.stateful_transitions.is_empty() { + info!( + "{} stateful transition(s) from streaming_config (planner-detected), \ + {} total after merging with engine config", + streaming_config.stateful_transitions.len(), + merged_stateful_transitions.len() + ); + } + + // Same merge as stateful_transitions above, for computed labels + // (e.g. origin-ASN extraction) the planner auto-detected. + let mut merged_computed_label_cols = computed_label_cols.clone(); + for (label_name, cfg) in &streaming_config.computed_label_cols { + merged_computed_label_cols + .entry(label_name.clone()) + .or_insert_with(|| cfg.clone()); + } + if !streaming_config.computed_label_cols.is_empty() { + info!( + "{} computed label(s) from streaming_config (planner-detected), \ + {} total after merging with engine config", + streaming_config.computed_label_cols.len(), + merged_computed_label_cols.len() + ); + } + + // csv_ingest.rs only ever computes a label for columns that + // also appear in label_cols (it iterates label_cols and + // *then* checks computed_label_cols for a rule) - a computed + // label absent from label_cols is silently never computed at + // all, which is exactly the sort of "config split across two + // fields that must be kept in sync by hand" gap the rest of + // this merge exists to close. Every merged computed label + // must be in label_cols too. + let mut merged_label_cols = label_cols.clone(); + for label_name in merged_computed_label_cols.keys() { + if !merged_label_cols.iter().any(|c| c == label_name) { + merged_label_cols.push(label_name.clone()); + } + } + vec![Box::new(CsvFileIngestSource::new(CsvFileIngestConfig { path: path.clone(), metric_name: metric_name.clone(), value_col: value_col.clone(), - label_cols: label_cols.clone(), + label_cols: merged_label_cols, + computed_label_cols: merged_computed_label_cols, + stateful_transitions: merged_stateful_transitions, timestamp_col: timestamp_col.clone(), start_ts_ms: *start_ts_ms, ts_step_ms: ts_step, diff --git a/asap-query-engine/src/precompute_engine/accumulator_factory.rs b/asap-query-engine/src/precompute_engine/accumulator_factory.rs index 7044cd25..ee8f2791 100644 --- a/asap-query-engine/src/precompute_engine/accumulator_factory.rs +++ b/asap-query-engine/src/precompute_engine/accumulator_factory.rs @@ -1,9 +1,9 @@ use crate::data_model::{AggregateCore, AggregationType, KeyByLabelValues, Measurement}; use crate::precompute_operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, DatasketchesKLLAccumulator, - HllAccumulator, HydraKllSketchAccumulator, IncreaseAccumulator, MinMaxAccumulator, - MultipleIncreaseAccumulator, MultipleMinMaxAccumulator, MultipleSumAccumulator, SumAccumulator, - DEFAULT_HLL_PRECISION, + DeltaSetAggregatorAccumulator, HllAccumulator, HydraKllSketchAccumulator, IncreaseAccumulator, + MinMaxAccumulator, MultipleIncreaseAccumulator, MultipleMinMaxAccumulator, + MultipleSumAccumulator, SetAggregatorAccumulator, SumAccumulator, DEFAULT_HLL_PRECISION, }; use asap_types::aggregation_config::AggregationConfig; @@ -675,6 +675,107 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { } } +// --------------------------------------------------------------------------- +// SetAggregatorAccumulatorUpdater +// --------------------------------------------------------------------------- + +pub struct SetAggregatorAccumulatorUpdater { + acc: SetAggregatorAccumulator, +} + +impl SetAggregatorAccumulatorUpdater { + pub fn new() -> Self { + Self { + acc: SetAggregatorAccumulator::new(), + } + } +} + +impl Default for SetAggregatorAccumulatorUpdater { + fn default() -> Self { + Self::new() + } +} + +impl AccumulatorUpdater for SetAggregatorAccumulatorUpdater { + fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { + debug_assert!( + false, + "update_single called on SetAggregator; use update_keyed" + ); + } + + fn update_keyed(&mut self, key: &KeyByLabelValues, _value: f64, _timestamp_ms: i64) { + self.acc.add_key(key.clone()); + } + + impl_accumulator_methods!(acc); + + fn reset(&mut self) { + self.acc = SetAggregatorAccumulator::new(); + } + + fn is_keyed(&self) -> bool { + true + } + + fn memory_usage_bytes(&self) -> usize { + std::mem::size_of::() + + self.acc.added.len() * std::mem::size_of::() + } +} + +// --------------------------------------------------------------------------- +// DeltaSetAggregatorAccumulatorUpdater +// --------------------------------------------------------------------------- + +pub struct DeltaSetAggregatorAccumulatorUpdater { + acc: DeltaSetAggregatorAccumulator, +} + +impl DeltaSetAggregatorAccumulatorUpdater { + pub fn new() -> Self { + Self { + acc: DeltaSetAggregatorAccumulator::new(), + } + } +} + +impl Default for DeltaSetAggregatorAccumulatorUpdater { + fn default() -> Self { + Self::new() + } +} + +impl AccumulatorUpdater for DeltaSetAggregatorAccumulatorUpdater { + fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { + debug_assert!( + false, + "update_single called on DeltaSetAggregator; use update_keyed" + ); + } + + fn update_keyed(&mut self, key: &KeyByLabelValues, _value: f64, _timestamp_ms: i64) { + self.acc.add_key(key.clone()); + } + + impl_accumulator_methods!(acc); + + fn reset(&mut self) { + self.acc = DeltaSetAggregatorAccumulator::new(); + } + + fn is_keyed(&self) -> bool { + true + } + + fn memory_usage_bytes(&self) -> usize { + std::mem::size_of::() + + (self.acc.added.len() + self.acc.removed.len()) + * std::mem::size_of::() + } +} + // --------------------------------------------------------------------------- // Config helpers // --------------------------------------------------------------------------- @@ -695,6 +796,8 @@ pub fn config_is_keyed(config: &AggregationConfig) -> bool { | AggregationType::CountMinSketch | AggregationType::CountMinSketchWithHeap | AggregationType::HydraKLL + | AggregationType::SetAggregator + | AggregationType::DeltaSetAggregator ) } @@ -887,6 +990,10 @@ pub fn create_accumulator_updater( AggregationType::HLL => Ok(Box::new(HllAccumulatorUpdater::new(hll_precision_param( config, )))), + AggregationType::SetAggregator => Ok(Box::new(SetAggregatorAccumulatorUpdater::new())), + AggregationType::DeltaSetAggregator => { + Ok(Box::new(DeltaSetAggregatorAccumulatorUpdater::new())) + } other => { tracing::warn!( "Unknown aggregation_type '{:?}', defaulting to SingleSubpopulation Sum", diff --git a/asap-query-engine/src/precompute_engine/computed_labels.rs b/asap-query-engine/src/precompute_engine/computed_labels.rs new file mode 100644 index 00000000..8152bb7e --- /dev/null +++ b/asap-query-engine/src/precompute_engine/computed_labels.rs @@ -0,0 +1,84 @@ +use regex::Regex; + +// ComputedLabelConfig now lives in asap_types so asap-planner-rs can +// construct it (from detecting a nested-subquery SQL shape) and the engine +// can consume it (to actually compute the label at ingest time) without +// either crate depending on the other - see asap_types::computed_label for +// the shared definition and design note. +pub use asap_types::computed_label::ComputedLabelConfig; + +pub fn should_skip_on_missing(rule: &ComputedLabelConfig) -> bool { + rule.on_missing.as_deref() == Some("skip_sample") +} + +fn tokenize(rule: &ComputedLabelConfig, raw_value: &str) -> Result, String> { + let tokenizer = rule.tokenizer.as_deref().unwrap_or("whitespace"); + + let mut tokens: Vec = match tokenizer { + "whitespace" => raw_value + .split_whitespace() + .filter(|x| !x.is_empty()) + .map(|x| x.to_string()) + .collect(), + other => { + return Err(format!( + "unsupported computed-label tokenizer {:?}; only whitespace is supported", + other + )); + } + }; + + if let Some(pat) = rule.filter_regex.as_deref() { + let re = Regex::new(pat) + .map_err(|e| format!("invalid computed-label filter_regex {:?}: {}", pat, e))?; + tokens.retain(|x| re.is_match(x)); + } + + Ok(tokens) +} + +pub fn compute_label_values( + rule: &ComputedLabelConfig, + raw_value: &str, +) -> Result, String> { + match rule.r#type.as_str() { + "field_alias" => { + if raw_value.is_empty() && should_skip_on_missing(rule) { + Ok(vec![]) + } else { + Ok(vec![raw_value.to_string()]) + } + } + + "token_select" => { + let tokens = tokenize(rule, raw_value)?; + if tokens.is_empty() { + return Ok(vec![]); + } + + let selected = match rule.select.as_deref().unwrap_or("last") { + "first" => tokens.first().cloned(), + "last" => tokens.last().cloned(), + sel if sel.starts_with("nth:") => { + let idx: usize = sel["nth:".len()..].parse().map_err(|e| { + format!("invalid token_select index in select={:?}: {}", sel, e) + })?; + tokens.get(idx).cloned() + } + other => { + return Err(format!( + "unsupported token_select selector {:?}; use first, last, or nth:N", + other + )); + } + }; + + Ok(selected.into_iter().collect()) + } + + "token_explode" => tokenize(rule, raw_value), + + other => Err(format!("unsupported computed-label type {:?}", other)), + } +} + diff --git a/asap-query-engine/src/precompute_engine/csv_ingest.rs b/asap-query-engine/src/precompute_engine/csv_ingest.rs index 459572f3..bd83ca6b 100644 --- a/asap-query-engine/src/precompute_engine/csv_ingest.rs +++ b/asap-query-engine/src/precompute_engine/csv_ingest.rs @@ -1,15 +1,24 @@ +use super::stateful_transition::{StatefulTransitionConfig, StatefulTransitionOperator}; use crate::drivers::ingest::prometheus_remote_write::DecodedSample; +use crate::precompute_engine::computed_labels::{ + compute_label_values, should_skip_on_missing, ComputedLabelConfig, +}; use crate::precompute_engine::ingest_source::{route_decoded_samples, IngestContext, IngestSource}; +use std::collections::HashMap; use std::time::Instant; -use tracing::info; +use tracing::{info, warn}; pub struct CsvFileIngestConfig { pub path: String, pub metric_name: String, - pub value_col: String, + pub value_col: Option, /// Label columns. Will be sorted alphabetically in the labels string. pub label_cols: Vec, - /// If Some, parse this column as the timestamp in milliseconds. + /// Computed labels. Each key is the logical label name; the rule says how to compute it. + pub computed_label_cols: HashMap, + pub stateful_transitions: Vec, + /// If Some, parse this column as the timestamp. + /// Accepts Unix milliseconds or ClickHouse DateTime strings like YYYY-MM-DD HH:MM:SS. /// If None, synthesize timestamps using start_ts_ms + row_index * ts_step_ms. pub timestamp_col: Option, pub start_ts_ms: i64, @@ -18,6 +27,19 @@ pub struct CsvFileIngestConfig { pub batch_size: usize, } +#[derive(Clone)] +enum LabelSource { + Physical { + name: String, + idx: usize, + }, + Computed { + name: String, + source_idx: usize, + rule: ComputedLabelConfig, + }, +} + pub struct CsvFileIngestSource { config: CsvFileIngestConfig, } @@ -28,6 +50,50 @@ impl CsvFileIngestSource { } } +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let y = year - if month <= 2 { 1 } else { 0 }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let mp = month + if month > 2 { -3 } else { 9 }; + let doy = (153 * mp + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +fn parse_timestamp_ms(raw: &str) -> Result> { + let trimmed = raw.trim(); + + // Existing behavior: Unix milliseconds. + if let Ok(v) = trimmed.parse::() { + return Ok(v); + } + + // ClickHouse DateTime commonly exports as "YYYY-MM-DD HH:MM:SS". + // Also tolerate "YYYY-MM-DDTHH:MM:SS" and trailing fractional seconds. + let normalized = trimmed.replace('T', " "); + let base = normalized + .split('.') + .next() + .unwrap_or(normalized.as_str()) + .trim_end_matches('Z'); + + if base.len() < 19 { + return Err(std::io::Error::other(format!("unsupported timestamp format: {}", raw)).into()); + } + + let dt = &base[..19]; + + let year: i64 = dt[0..4].parse()?; + let month: i64 = dt[5..7].parse()?; + let day: i64 = dt[8..10].parse()?; + let hour: i64 = dt[11..13].parse()?; + let minute: i64 = dt[14..16].parse()?; + let second: i64 = dt[17..19].parse()?; + + let days = days_from_civil(year, month, day); + Ok((days * 86_400 + hour * 3_600 + minute * 60 + second) * 1000) +} + #[async_trait::async_trait] impl IngestSource for CsvFileIngestSource { async fn run( @@ -42,13 +108,25 @@ impl IngestSource for CsvFileIngestSource { let mut rdr = csv::Reader::from_path(&config.path)?; let headers = rdr.headers()?.clone(); - let value_idx = headers - .iter() - .position(|h| h == config.value_col) - .ok_or_else(|| format!("value column '{}' not found in CSV", config.value_col)) - .map_err(|e| -> Box { - std::io::Error::other(e).into() - })?; + let value_idx = match config.value_col.as_deref() { + Some(value_col) => Some( + headers + .iter() + .position(|h| h == value_col) + .ok_or_else(|| format!("value column '{}' not found in CSV", value_col)) + .map_err(|e| -> Box { + std::io::Error::other(e).into() + })?, + ), + None => { + warn!( + "CSV ingest config has no value_col; every row will be treated as \ + an event count of 1.0. If a real metric column was intended, set \ + value_col explicitly — this is otherwise silent." + ); + None + } + }; let ts_idx = config .timestamp_col @@ -67,83 +145,210 @@ impl IngestSource for CsvFileIngestSource { let mut sorted_label_cols = config.label_cols.clone(); sorted_label_cols.sort(); - let mut label_idxs: Vec<(String, usize)> = Vec::new(); + let mut label_sources: Vec = Vec::new(); for col in &sorted_label_cols { - let idx = headers - .iter() - .position(|h| h == col.as_str()) - .ok_or_else(|| format!("label column '{}' not found in CSV", col)) - .map_err(|e| -> Box { - std::io::Error::other(e).into() - })?; - label_idxs.push((col.clone(), idx)); + if let Some(idx) = headers.iter().position(|h| h == col.as_str()) { + label_sources.push(LabelSource::Physical { + name: col.clone(), + idx, + }); + continue; + } + + if let Some(rule) = config.computed_label_cols.get(col) { + let source_idx = headers + .iter() + .position(|h| h == rule.source_col.as_str()) + .ok_or_else(|| { + format!( + "source column '{}' for computed label '{}' not found in CSV", + rule.source_col, col + ) + }) + .map_err(|e| -> Box { + std::io::Error::other(e).into() + })?; + + label_sources.push(LabelSource::Computed { + name: col.clone(), + source_idx, + rule: rule.clone(), + }); + continue; + } + + return Err(std::io::Error::other(format!( + "label column '{}' not found in CSV and no computed_label_cols rule was provided", + col + )) + .into()); } let mut batch: Vec = Vec::with_capacity(config.batch_size); let mut row_count: u64 = 0; + let mut stateful_ops: Vec = config + .stateful_transitions + .iter() + .cloned() + .map(StatefulTransitionOperator::new) + .collect(); for result in rdr.records() { let record = result?; - let labels = if label_idxs.is_empty() { - config.metric_name.clone() + let row_map: HashMap = headers + .iter() + .enumerate() + .map(|(idx, name)| { + (name.to_string(), record.get(idx).unwrap_or("").to_string()) + }) + .collect(); + + let label_strings: Vec = if label_sources.is_empty() { + vec![config.metric_name.clone()] } else { - let mut s = String::with_capacity(64); - s.push_str(&config.metric_name); - s.push('{'); - for (i, (col, idx)) in label_idxs.iter().enumerate() { - if i > 0 { - s.push(','); + let mut expanded: Vec> = vec![Vec::new()]; + let mut skip_sample = false; + + for source in &label_sources { + match source { + LabelSource::Physical { name, idx } => { + let value = record.get(*idx).unwrap_or("").to_string(); + for labels in &mut expanded { + labels.push((name.clone(), value.clone())); + } + } + + LabelSource::Computed { + name, + source_idx, + rule, + } => { + let raw_value = record.get(*source_idx).unwrap_or(""); + let mut values = compute_label_values(rule, raw_value) + .map_err(|e| { + let err: Box = + std::io::Error::other(e).into(); + err + })?; + + if values.is_empty() { + if should_skip_on_missing(rule) { + skip_sample = true; + break; + } + values.push(String::new()); + } + + let mut next = + Vec::with_capacity(expanded.len() * values.len()); + for labels in expanded.into_iter() { + for value in &values { + let mut labels2 = labels.clone(); + labels2.push((name.clone(), value.clone())); + next.push(labels2); + } + } + expanded = next; + } } - s.push_str(col); - s.push_str("=\""); - s.push_str(record.get(*idx).unwrap_or("")); - s.push('"'); } - s.push('}'); - s - }; - let value: f64 = record - .get(value_idx) - .ok_or("missing value field") - .map_err(|e| -> Box { - std::io::Error::other(e).into() - })? - .parse() - .map_err(|e| -> Box { - std::io::Error::other(format!("failed to parse value: {}", e)).into() - })?; + if skip_sample { + row_count += 1; + continue; + } + + expanded + .into_iter() + .map(|pairs| { + let mut s = String::with_capacity(64); + s.push_str(&config.metric_name); + s.push('{'); - let timestamp_ms = match ts_idx { + for (i, (name, value)) in pairs.into_iter().enumerate() { + if i > 0 { + s.push(','); + } + s.push_str(&name); + s.push_str("=\""); + s.push_str(&value); + s.push('"'); + } + + s.push('}'); + s + }) + .collect() + }; + + let value: f64 = match value_idx { Some(idx) => record .get(idx) - .ok_or("missing timestamp field") + .ok_or("missing value field") .map_err(|e| -> Box { std::io::Error::other(e).into() })? - .parse::() + .parse() .map_err(|e| -> Box { - std::io::Error::other(format!("failed to parse timestamp: {}", e)) + std::io::Error::other(format!("failed to parse value: {}", e)) .into() })?, + None => 1.0, + }; + + let timestamp_ms = match ts_idx { + Some(idx) => { + let raw_ts = record.get(idx).ok_or("missing timestamp field").map_err( + |e| -> Box { + std::io::Error::other(e).into() + }, + )?; + parse_timestamp_ms(raw_ts)? + } None => config.start_ts_ms + (row_count as i64) * config.ts_step_ms, }; - batch.push(DecodedSample { - labels, - timestamp_ms, - value, - }); - row_count += 1; + for labels in label_strings { + batch.push(DecodedSample { + labels, + timestamp_ms, + value, + }); - if batch.len() >= config.batch_size { - let send_batch = - std::mem::replace(&mut batch, Vec::with_capacity(config.batch_size)); - if tx.blocking_send(send_batch).is_err() { - break; + if batch.len() >= config.batch_size { + let send_batch = std::mem::replace( + &mut batch, + Vec::with_capacity(config.batch_size), + ); + if tx.blocking_send(send_batch).is_err() { + return Ok(row_count); + } } } + + for op in &mut stateful_ops { + if let Some(labels) = op.process_row(&row_map) { + batch.push(DecodedSample { + labels, + timestamp_ms, + value: 1.0, + }); + + if batch.len() >= config.batch_size { + let send_batch = std::mem::replace( + &mut batch, + Vec::with_capacity(config.batch_size), + ); + + if tx.blocking_send(send_batch).is_err() { + return Ok(row_count); + } + } + } + } + + row_count += 1; } if !batch.is_empty() { @@ -166,6 +371,10 @@ impl IngestSource for CsvFileIngestSource { rows, total_samples ); + // CSV precompute must explicitly flush after all batches are routed. + // Otherwise the final active windows may not be materialized before + // worker shutdown, causing "No precomputed outputs found" at query time. + ctx.router.broadcast_flush().await?; ctx.router.broadcast_shutdown().await?; Ok(()) } diff --git a/asap-query-engine/src/precompute_engine/ingest_source.rs b/asap-query-engine/src/precompute_engine/ingest_source.rs index 384b6403..6472b17d 100644 --- a/asap-query-engine/src/precompute_engine/ingest_source.rs +++ b/asap-query-engine/src/precompute_engine/ingest_source.rs @@ -3,12 +3,22 @@ use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::precompute_engine::worker::{extract_metric_name, parse_labels_from_series_key}; use arc_swap::ArcSwap; use asap_types::aggregation_config::AggregationConfig; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::Instant; use tracing::{debug, warn}; +/// Distinct unrecognized spatial-filter clauses already warned about. This +/// check runs per (sample, config) in the ingest hot path, so warning +/// unconditionally would mean one log line per matching CSV row - up to +/// millions of times for one unrecognized clause. Warn once per distinct +/// clause per process instead. +fn warned_unsupported_clauses() -> &'static Mutex> { + static CACHE: OnceLock>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashSet::new())) +} + /// Everything a source needs to push decoded samples into the worker pool. #[derive(Clone)] pub struct IngestContext { @@ -47,6 +57,204 @@ pub(crate) fn extract_group_key(series_key: &str, config: &AggregationConfig) -> values.join(";") } +/// Evaluate the simple label-filter subset emitted by the SQL planner for +/// precompute routing, e.g.: +/// +/// collector = 'rrc00' AND operation = 'A' +/// +/// This intentionally supports conjunctions of equality predicates. If a +/// clause is not understood, preserve the previous permissive behavior for +/// that clause instead of rejecting the sample. +fn sample_matches_spatial_filter(series_key: &str, config: &AggregationConfig) -> bool { + let filter = config.spatial_filter.trim(); + + if filter.is_empty() { + return true; + } + + let metric_name = extract_metric_name(series_key); + + // Preserve compatibility with older configs that used spatial_filter as a + // metric-like matcher rather than a label predicate. + if filter == metric_name || config.spatial_filter_normalized == metric_name { + return true; + } + + let labels = parse_labels_from_series_key(series_key); + + for clause in split_spatial_filter_clauses(filter) { + let Some(parsed) = parse_spatial_clause(&clause) else { + // Preserve the previous permissive behavior for clause shapes we + // genuinely don't recognize, but make it loud: a silent `debug!` + // here is indistinguishable from "this filter is enforced" in + // normal operation, and has already let an unsupported `IN (...)` + // clause through unfiltered in practice. + let is_new = warned_unsupported_clauses() + .lock() + .unwrap() + .insert(clause.clone()); + if is_new { + warn!( + "Ignoring unsupported spatial filter clause during ingest routing \ + (samples that should be filtered by it may pass through unfiltered); \ + further occurrences of this exact clause are suppressed: {}", + clause + ); + } + continue; + }; + + let matches = match &parsed { + SpatialClause::Eq(label, expected) => { + matches!(labels.get(label.as_str()), Some(actual) if *actual == expected.as_str()) + } + SpatialClause::Ne(label, excluded) => { + !matches!(labels.get(label.as_str()), Some(actual) if *actual == excluded.as_str()) + } + SpatialClause::In(label, allowed) => match labels.get(label.as_str()) { + Some(actual) => allowed.iter().any(|v| v == actual), + None => false, + }, + }; + + if !matches { + return false; + } + } + + true +} + +enum SpatialClause { + Eq(String, String), + Ne(String, String), + In(String, Vec), +} + +/// Splits a spatial-filter string on top-level `AND`/`,` separators, without +/// splitting on separators that appear inside a single-quoted literal (e.g. +/// `operation = 'read,write'` must stay one clause, not two) or inside +/// parentheses (e.g. `peer_asn IN ('174', '3356')` has commas between the +/// quoted values that are themselves outside any quotes, but they belong to +/// the IN-list, not to the top-level clause separator). +fn split_spatial_filter_clauses(filter: &str) -> Vec { + let cleaned = filter + .trim() + .trim_start_matches('{') + .trim_end_matches('}') + .trim(); + + let mut clauses = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + let mut paren_depth: i32 = 0; + let chars: Vec = cleaned.chars().collect(); + let mut i = 0; + + while i < chars.len() { + let c = chars[i]; + + if c == '\'' { + in_quotes = !in_quotes; + current.push(c); + i += 1; + continue; + } + + if !in_quotes { + if c == '(' { + paren_depth += 1; + current.push(c); + i += 1; + continue; + } + if c == ')' { + paren_depth = (paren_depth - 1).max(0); + current.push(c); + i += 1; + continue; + } + + if paren_depth == 0 { + if c == ',' { + clauses.push(current.trim().to_string()); + current.clear(); + i += 1; + continue; + } + + // Match " AND " / " and " as a whole-word separator. + let rest: String = chars[i..].iter().collect(); + let rest_upper = rest.to_uppercase(); + if rest_upper.starts_with(" AND ") { + clauses.push(current.trim().to_string()); + current.clear(); + i += 5; + continue; + } + } + } + + current.push(c); + i += 1; + } + + clauses.push(current.trim().to_string()); + clauses.into_iter().filter(|c| !c.is_empty()).collect() +} + +fn strip_quotes(s: &str) -> String { + s.trim() + .trim_matches('`') + .trim_matches('"') + .trim_matches('\'') + .to_string() +} + +fn parse_spatial_clause(clause: &str) -> Option { + if clause.contains("=~") || clause.contains("!~") { + return None; + } + + let upper = clause.to_uppercase(); + if let Some(in_idx) = upper.find(" IN ") { + let label = strip_quotes(&clause[..in_idx]); + if label.is_empty() { + return None; + } + let rest = clause[in_idx + 4..].trim(); + let inner = rest.strip_prefix('(')?.trim_end().strip_suffix(')')?; + let values: Vec = inner + .split(',') + .map(|v| strip_quotes(v)) + .filter(|v| !v.is_empty()) + .collect(); + if values.is_empty() { + return None; + } + return Some(SpatialClause::In(label, values)); + } + + // SQLPatternParser's extracted spatial_filter strings come from + // sqlparser's own Display impl, which renders a parsed `!=` back out as + // `<>` - so a query the analyst wrote with `!=` shows up here as `<>`. + // Recognize both spellings of the same operator. + if let Some((lhs, rhs)) = clause.split_once("!=").or_else(|| clause.split_once("<>")) { + let label = strip_quotes(lhs); + if label.is_empty() { + return None; + } + return Some(SpatialClause::Ne(label, strip_quotes(rhs))); + } + + let (lhs, rhs) = clause.split_once('=')?; + let label = strip_quotes(lhs); + if label.is_empty() { + return None; + } + Some(SpatialClause::Eq(label, strip_quotes(rhs))) +} + /// Group decoded samples by (agg_id, group_key) and route them to workers. /// /// Returns an error if the router fails to deliver any message. @@ -126,6 +334,10 @@ pub(crate) async fn route_decoded_samples( { continue; } + if !sample_matches_spatial_filter(&s.labels, config) { + continue; + } + matched_samples += 1; let group_key = extract_group_key(&s.labels, config); by_group diff --git a/asap-query-engine/src/precompute_engine/mod.rs b/asap-query-engine/src/precompute_engine/mod.rs index 702ed8e7..2914cd41 100644 --- a/asap-query-engine/src/precompute_engine/mod.rs +++ b/asap-query-engine/src/precompute_engine/mod.rs @@ -1,4 +1,5 @@ pub mod accumulator_factory; +pub mod computed_labels; pub mod config; pub mod csv_ingest; mod engine; @@ -16,3 +17,5 @@ pub use engine::{PrecomputeEngine, PrecomputeEngineHandle, PrecomputeWorkerDiagn pub use ingest_handler::{HttpIngestConfig, HttpIngestSource}; pub use ingest_source::{IngestContext, IngestSource}; pub use json_ingest::{JsonFileIngestConfig, JsonFileIngestSource, TimestampUnit}; + +pub mod stateful_transition; diff --git a/asap-query-engine/src/precompute_engine/stateful_transition.rs b/asap-query-engine/src/precompute_engine/stateful_transition.rs new file mode 100644 index 00000000..72139370 --- /dev/null +++ b/asap-query-engine/src/precompute_engine/stateful_transition.rs @@ -0,0 +1,184 @@ +use std::collections::HashMap; + +// StatefulTransitionConfig now lives in asap_types so asap-planner-rs can +// construct it (from detecting the SQL pattern) and the engine can consume +// it (to drive ingest-time state tracking) without either crate depending +// on the other - see asap_types::stateful_transition for the shared +// definition and design note. +pub use asap_types::stateful_transition::StatefulTransitionConfig; + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct PartitionKey(Vec); + +#[derive(Debug, Default)] +pub struct StatefulTransitionOperator { + cfg: StatefulTransitionConfig, + last_value: HashMap, +} + +impl StatefulTransitionOperator { + pub fn new(cfg: StatefulTransitionConfig) -> Self { + Self { + cfg, + last_value: HashMap::new(), + } + } + + pub fn process_row(&mut self, row: &HashMap) -> Option { + let key = PartitionKey( + self.cfg + .partition_by + .iter() + .map(|col| row.get(col).cloned().unwrap_or_default()) + .collect(), + ); + + let curr = row.get(&self.cfg.state_column).cloned().unwrap_or_default(); + + let prev = self.last_value.get(&key).cloned(); + self.last_value.insert(key, curr.clone()); + + let prev = prev?; + + if !eval_transition_predicate( + &self.cfg.predicate, + &self.cfg.previous_alias, + &self.cfg.state_column, + &prev, + &curr, + row, + ) { + return None; + } + + Some(build_label_string( + &self.cfg.metric_name, + &self.cfg.emit_labels, + row, + )) + } +} + +fn build_label_string(metric: &str, labels: &[String], row: &HashMap) -> String { + if labels.is_empty() { + return metric.to_string(); + } + + let mut out = String::with_capacity(64); + out.push_str(metric); + out.push('{'); + + for (i, label) in labels.iter().enumerate() { + if i > 0 { + out.push(','); + } + let value = row.get(label).map(String::as_str).unwrap_or(""); + out.push_str(label); + out.push_str("=\""); + out.push_str(value); + out.push('"'); + } + + out.push('}'); + out +} + +fn eval_transition_predicate( + raw: &str, + previous_alias: &str, + state_column: &str, + prev: &str, + curr: &str, + row: &HashMap, +) -> bool { + // V0: support conjunctions of simple equality/inequality expressions. + // This is generic: no BGP-specific column names or metric names. + raw.split(" AND ") + .map(str::trim) + .filter(|s| !s.is_empty()) + .all(|clause| eval_simple_clause(clause, previous_alias, state_column, prev, curr, row)) +} + +fn eval_simple_clause( + clause: &str, + previous_alias: &str, + state_column: &str, + prev: &str, + curr: &str, + row: &HashMap, +) -> bool { + if let Some((lhs, rhs)) = clause.split_once("!=") { + return resolve_value(lhs.trim(), previous_alias, state_column, prev, curr, row) + != resolve_value(rhs.trim(), previous_alias, state_column, prev, curr, row); + } + + if let Some((lhs, rhs)) = clause.split_once("=") { + return resolve_value(lhs.trim(), previous_alias, state_column, prev, curr, row) + == resolve_value(rhs.trim(), previous_alias, state_column, prev, curr, row); + } + + false +} + +fn resolve_value( + expr: &str, + previous_alias: &str, + state_column: &str, + prev: &str, + curr: &str, + row: &HashMap, +) -> String { + let expr = expr.trim(); + + if expr == previous_alias { + return prev.to_string(); + } + + if expr == state_column { + return curr.to_string(); + } + + if expr.len() >= 2 && expr.starts_with('\'') && expr.ends_with('\'') { + return expr[1..expr.len() - 1].to_string(); + } + + row.get(expr).cloned().unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_generic_transition() { + let cfg = StatefulTransitionConfig { + metric_name: "__derived_q11_path_changes".to_string(), + partition_by: vec![ + "prefix".to_string(), + "collector".to_string(), + "peer_ip".to_string(), + ], + state_column: "as_path".to_string(), + previous_alias: "previous_path".to_string(), + predicate: "previous_path != '' AND previous_path != as_path".to_string(), + emit_labels: vec!["prefix".to_string()], + }; + + let mut op = StatefulTransitionOperator::new(cfg); + + let mut row1 = HashMap::new(); + row1.insert("prefix".to_string(), "1.2.3.0/24".to_string()); + row1.insert("collector".to_string(), "rrc00".to_string()); + row1.insert("peer_ip".to_string(), "peer1".to_string()); + row1.insert("as_path".to_string(), "1 2 3".to_string()); + + let mut row2 = row1.clone(); + row2.insert("as_path".to_string(), "1 2 4".to_string()); + + assert!(op.process_row(&row1).is_none()); + assert_eq!( + op.process_row(&row2), + Some("__derived_q11_path_changes{prefix=\"1.2.3.0/24\"}".to_string()) + ); + } +} diff --git a/asap-query-engine/src/precompute_engine/worker.rs b/asap-query-engine/src/precompute_engine/worker.rs index d408c412..19c8cc0c 100644 --- a/asap-query-engine/src/precompute_engine/worker.rs +++ b/asap-query-engine/src/precompute_engine/worker.rs @@ -8,6 +8,7 @@ use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; use crate::precompute_operators::sum_accumulator::SumAccumulator; use asap_types::aggregation_config::AggregationConfig; +use asap_types::enums::AggregationType; use std::collections::{BTreeMap, HashMap}; use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; use std::sync::Arc; @@ -936,18 +937,36 @@ fn resolve_sample_value( match raw.parse::() { Ok(v) => v, - // Non-numeric distinct targets (e.g. COUNT(DISTINCT proto)) are not yet - // supported: silently falling back to the wire value would produce an - // INCORRECT aggregate, so fail loudly instead. This panic is a temporary - // measure — the longer-term fix is to make this path return a `Result` - // and propagate the error up to the caller. + Err(_) if config.aggregation_type == AggregationType::HLL => { + stable_string_hash_as_exact_f64(raw) + } Err(_) => panic!( - "value_column '{col}' label value {raw:?} is not numeric; non-numeric distinct \ - targets (e.g. COUNT(DISTINCT proto)) are not yet supported" + "value_column '{col}' label value {raw:?} is not numeric for aggregation type {:?}", + config.aggregation_type ), } } +/// Convert a string distinct target into a deterministic numeric surrogate. +/// +/// The precompute path currently passes one f64 scalar into accumulators. For +/// HLL/cardinality, the scalar only needs to distinguish distinct values before +/// the HLL hashes it. We use a stable FNV-1a hash and keep only 53 bits so the +/// integer is represented exactly as f64. +fn stable_string_hash_as_exact_f64(raw: &str) -> f64 { + const FNV_OFFSET: u64 = 0xcbf29ce484222325; + const FNV_PRIME: u64 = 0x100000001b3; + const F64_EXACT_INT_MASK: u64 = (1_u64 << 53) - 1; + + let mut hash = FNV_OFFSET; + for b in raw.as_bytes() { + hash ^= *b as u64; + hash = hash.wrapping_mul(FNV_PRIME); + } + + (hash & F64_EXACT_INT_MASK) as f64 +} + /// Extract aggregated label values from a series key string. /// These are the labels that form the key dimension *inside* keyed accumulators /// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. @@ -1104,10 +1123,7 @@ mod tests { } #[test] - #[should_panic(expected = "is not numeric")] - fn resolve_sample_value_non_numeric_label_panics() { - // Non-numeric distinct targets are a follow-up; for now we fail loudly - // rather than silently producing an incorrect aggregate. + fn resolve_sample_value_hashes_non_numeric_hll_label() { let mut config = make_agg_config( 4, "netflow_table", @@ -1118,9 +1134,23 @@ mod tests { vec!["srcip"], ); config.value_column = Some("proto".to_string()); - let series = "netflow_table{srcip=\"10\",proto=\"TCP\"}"; - let labels = parse_labels_from_series_key(series); - let _ = resolve_sample_value(&labels, 1400.0, &config); + + let labels_a = parse_labels_from_series_key("netflow_table{srcip=\"10\",proto=\"TCP\"}"); + let labels_b = parse_labels_from_series_key("netflow_table{srcip=\"10\",proto=\"UDP\"}"); + let labels_a2 = parse_labels_from_series_key("netflow_table{srcip=\"10\",proto=\"TCP\"}"); + + let tcp = resolve_sample_value(&labels_a, 1400.0, &config); + let udp = resolve_sample_value(&labels_b, 1400.0, &config); + let tcp_again = resolve_sample_value(&labels_a2, 7.0, &config); + + assert_eq!( + tcp, tcp_again, + "same string should hash to same f64 surrogate" + ); + assert_ne!( + tcp, udp, + "different strings should usually hash differently" + ); } #[test] diff --git a/local_experiments/bgp_jan2024_rrc00_200_query_workload.yaml b/local_experiments/bgp_jan2024_rrc00_200_query_workload.yaml new file mode 100644 index 00000000..aedadb85 --- /dev/null +++ b/local_experiments/bgp_jan2024_rrc00_200_query_workload.yaml @@ -0,0 +1,2686 @@ +workload_name: bgp_jan2024_rrc00_200_query_workload +month: 2024-01 +collector: rrc00 +table: bgp.bgp_updates +queries: + - id: q001 + title: "Total update volume on a single day" + analyst_question: "How many total BGP updates did rrc00 receive on Jan 3, 2024?" + window: 1 day + sql: | + SELECT count(*) AS total_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00'; + + - id: q002 + title: "Announcement vs withdrawal split (1 day)" + analyst_question: "What is the ratio of announcements to withdrawals on Jan 3, 2024?" + window: 1 day + sql: | + SELECT operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00' + GROUP BY operation + ORDER BY cnt DESC; + + - id: q003 + title: "Hourly update volume across a full day" + analyst_question: "How does update volume change hour by hour on Jan 5, 2024?" + window: 1 day + sql: | + SELECT toStartOfHour(timestamp) AS hour, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 00:00:00' AND timestamp < '2024-01-06 00:00:00' + GROUP BY hour + ORDER BY hour; + + - id: q004 + title: "Daily update volume for full month" + analyst_question: "What is the total daily update volume across all of January 2024?" + window: full month + sql: | + SELECT toDate(timestamp) AS day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q005 + title: "Update volume in a tight 5-minute window" + analyst_question: "How many updates arrived between 08:00 and 08:05 on Jan 15?" + window: 5 minutes + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 08:00:00' AND timestamp < '2024-01-15 08:05:00'; + + - id: q006 + title: "Update volume in a 15-minute window by operation" + analyst_question: "What was the announcement/withdrawal mix between 12:00 and 12:15 on Jan 10?" + window: 15 minutes + sql: | + SELECT operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-10 12:00:00' AND timestamp < '2024-01-10 12:15:00' + GROUP BY operation + ORDER BY cnt DESC; + + - id: q007 + title: "Top 20 most-updated prefixes (1 hour)" + analyst_question: "Which prefixes received the most updates between 09:00 and 10:00 on Jan 5?" + window: 1 hour + sql: | + SELECT prefix, count(*) AS updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00' + GROUP BY prefix + ORDER BY updates DESC + LIMIT 20; + + - id: q008 + title: "Top 20 most-updated prefixes over a full week" + analyst_question: "Which prefixes were updated most often during the first week of January?" + window: 1 week + sql: | + SELECT prefix, count(*) AS updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY prefix + ORDER BY updates DESC + LIMIT 20; + + - id: q009 + title: "Top origin ASNs by announcement count (1 day)" + analyst_question: "Which origin ASNs announced the most prefixes on Jan 12?" + window: 1 day + sql: | + SELECT origin, count(*) AS announcements + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-12 00:00:00' AND timestamp < '2024-01-13 00:00:00' + GROUP BY origin + ORDER BY announcements DESC + LIMIT 20; + + - id: q010 + title: "Top peer ASNs by total updates over the month" + analyst_question: "Which peer ASNs sent rrc00 the most updates over the entire month?" + window: full month + sql: | + SELECT peer_asn, count(*) AS updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_asn + ORDER BY updates DESC + LIMIT 25; + + - id: q011 + title: "Peer count active in a 6-hour window" + analyst_question: "How many distinct peers sent updates between midnight and 06:00 on Jan 8?" + window: 6 hours + sql: | + SELECT uniqExact(peer_ip) AS distinct_peers + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00'; + + - id: q012 + title: "Distinct prefixes seen per day (full month)" + analyst_question: "How many distinct prefixes were seen each day in January?" + window: full month + sql: | + SELECT toDate(timestamp) AS day, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q013 + title: "Average AS path length by day (1 week)" + analyst_question: "How does average AS path length trend across the first week of January?" + window: 1 week + sql: | + SELECT toDate(timestamp) AS day, + avg(length(splitByChar(' ', as_path))) AS avg_path_len + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q014 + title: "Longest AS paths observed (1 day)" + analyst_question: "What are the 15 longest AS paths seen on Jan 20?" + window: 1 day + sql: | + SELECT prefix, as_path, length(splitByChar(' ', as_path)) AS path_len + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-20 00:00:00' AND timestamp < '2024-01-21 00:00:00' + ORDER BY path_len DESC + LIMIT 15; + + - id: q015 + title: "Shortest (direct) AS paths (1 day)" + analyst_question: "Which announcements had single-hop AS paths on Jan 20?" + window: 1 day + sql: | + SELECT prefix, as_path, origin + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-20 00:00:00' AND timestamp < '2024-01-21 00:00:00' + AND length(splitByChar(' ', as_path)) = 1 + LIMIT 50; + + - id: q016 + title: "Paths transiting AS3356 (Lumen) in a 1-hour window" + analyst_question: "Which prefixes had AS3356 (Lumen) somewhere in the AS path between 14:00 and 15:00 on Jan 9?" + window: 1 hour + sql: | + SELECT prefix, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-09 14:00:00' AND timestamp < '2024-01-09 15:00:00' + AND has(splitByChar(' ', as_path), '3356') + LIMIT 100; + + - id: q017 + title: "Count of updates transiting AS174 (Cogent) over a day" + analyst_question: "How many updates on Jan 11 had AS174 (Cogent) in the path?" + window: 1 day + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-11 00:00:00' AND timestamp < '2024-01-12 00:00:00' + AND has(splitByChar(' ', as_path), '174'); + + - id: q018 + title: "Prefixes originated by AS15169 (Google) over a week" + analyst_question: "Which prefixes did Google (AS15169) originate during the first week of January?" + window: 1 week + sql: | + SELECT DISTINCT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND origin = '15169' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ORDER BY prefix + LIMIT 200; + + - id: q019 + title: "Update activity for AS13335 (Cloudflare) prefixes (1 day)" + analyst_question: "How many announcements and withdrawals involved Cloudflare-originated prefixes on Jan 18?" + window: 1 day + sql: | + SELECT operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '13335' + AND timestamp >= '2024-01-18 00:00:00' AND timestamp < '2024-01-19 00:00:00' + GROUP BY operation; + + - id: q020 + title: "Well-known prefix 8.8.8.0/24 update history (1 week)" + analyst_question: "What updates were seen for 8.8.8.0/24 during the first week of January?" + window: 1 week + sql: | + SELECT timestamp, operation, as_path, next_hop, local_pref, med + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '8.8.8.0/24' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ORDER BY timestamp; + + - id: q021 + title: "Well-known prefix 1.1.1.0/24 update history (1 day)" + analyst_question: "What updates touched Cloudflare's 1.1.1.0/24 anycast prefix on Jan 22?" + window: 1 day + sql: | + SELECT timestamp, operation, peer_asn, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '1.1.1.0/24' + AND timestamp >= '2024-01-22 00:00:00' AND timestamp < '2024-01-23 00:00:00' + ORDER BY timestamp; + + - id: q022 + title: "Prefixes with more than one distinct origin ASN (potential MOAS) in a day" + analyst_question: "Which prefixes were announced by more than one origin AS on Jan 14 (possible MOAS)?" + window: 1 day + sql: | + SELECT prefix, uniqExact(origin) AS distinct_origins + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-14 00:00:00' AND timestamp < '2024-01-15 00:00:00' + GROUP BY prefix + HAVING distinct_origins > 1 + ORDER BY distinct_origins DESC + LIMIT 50; + + - id: q023 + title: "MOAS detail listing for a specific prefix over a week" + analyst_question: "What origin ASNs and timestamps were involved for prefix 192.0.2.0/24 across the first week?" + window: 1 week + sql: | + SELECT timestamp, origin, as_path, peer_asn + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '192.0.2.0/24' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ORDER BY timestamp; + + - id: q024 + title: "Prefixes with origin AS changes using window functions (1 day)" + analyst_question: "Which prefixes changed origin AS between consecutive announcements on Jan 16?" + window: 1 day + sql: | + SELECT prefix, timestamp, origin, prev_origin + FROM ( + SELECT prefix, timestamp, origin, + lagInFrame(origin) OVER (PARTITION BY prefix ORDER BY timestamp) AS prev_origin + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-16 00:00:00' AND timestamp < '2024-01-17 00:00:00' + ) + WHERE prev_origin IS NOT NULL AND origin != prev_origin + ORDER BY prefix, timestamp + LIMIT 100; + + - id: q025 + title: "AS path change detection via window function (6 hours)" + analyst_question: "Which prefixes had their AS path change between consecutive updates between 00:00 and 06:00 on Jan 8?" + window: 6 hours + sql: | + SELECT prefix, timestamp, as_path, prev_path + FROM ( + SELECT prefix, timestamp, as_path, + lagInFrame(as_path) OVER (PARTITION BY prefix ORDER BY timestamp) AS prev_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00' + ) + WHERE prev_path IS NOT NULL AND as_path != prev_path + ORDER BY prefix, timestamp + LIMIT 200; + + - id: q026 + title: "Route flapping detection: prefixes with high update churn (1 day)" + analyst_question: "Which prefixes flapped (announce/withdraw repeatedly) the most on Jan 25?" + window: 1 day + sql: | + SELECT prefix, count(*) AS total_events, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-25 00:00:00' AND timestamp < '2024-01-26 00:00:00' + GROUP BY prefix + HAVING withdrawals > 5 AND announcements > 5 + ORDER BY total_events DESC + LIMIT 30; + + - id: q027 + title: "Flap rate per peer over a 3-day window" + analyst_question: "Which peers generated the most withdrawal churn between Jan 1 and Jan 4?" + window: 3 days + sql: | + SELECT peer_ip, peer_asn, + countIf(operation = 'W') AS withdrawals, + count(*) AS total + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-04 00:00:00' + GROUP BY peer_ip, peer_asn + ORDER BY withdrawals DESC + LIMIT 25; + + - id: q028 + title: "Prefixes withdrawn without a prior seen announcement in-window (1 hour)" + analyst_question: "Were there withdrawals between 09:00 and 10:00 on Jan 5 for prefixes not announced earlier that hour?" + window: 1 hour + sql: | + SELECT DISTINCT w.prefix + FROM bgp.bgp_updates AS w + WHERE w.collector = 'rrc00' + AND w.operation = 'W' + AND w.timestamp >= '2024-01-05 09:00:00' AND w.timestamp < '2024-01-05 10:00:00' + AND w.prefix NOT IN ( + SELECT prefix FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00' + ) + LIMIT 100; + + - id: q029 + title: "Community value frequency (1 day)" + analyst_question: "Which BGP communities appeared most frequently on Jan 6?" + window: 1 day + sql: | + SELECT communities, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + AND communities != '' + GROUP BY communities + ORDER BY cnt DESC + LIMIT 30; + + - id: q030 + title: "Updates tagged with no-export community (1 day)" + analyst_question: "How many updates on Jan 6 carried the well-known no-export community (65535:65281)?" + window: 1 day + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + AND positionCaseInsensitive(communities, '65535:65281') > 0; + + - id: q031 + title: "Updates tagged with no-advertise community (1 week)" + analyst_question: "How many updates in the first week carried the no-advertise community (65535:65282)?" + window: 1 week + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + AND positionCaseInsensitive(communities, '65535:65282') > 0; + + - id: q032 + title: "Distinct community values used by a specific peer ASN (full month)" + analyst_question: "What distinct community strings did peer ASN 3356 use during January?" + window: full month + sql: | + SELECT DISTINCT communities + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn = '3356' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + AND communities != '' + LIMIT 100; + + - id: q033 + title: "Number of communities attached per update (1 day)" + analyst_question: "What is the distribution of community-tag counts per update on Jan 9?" + window: 1 day + sql: | + SELECT length(splitByChar(' ', communities)) AS community_count, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-09 00:00:00' AND timestamp < '2024-01-10 00:00:00' + AND communities != '' + GROUP BY community_count + ORDER BY community_count; + + - id: q034 + title: "MED value distribution (1 day)" + analyst_question: "What does the MED value distribution look like on Jan 7?" + window: 1 day + sql: | + SELECT med, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-07 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY med + ORDER BY cnt DESC + LIMIT 30; + + - id: q035 + title: "Average MED per origin ASN (1 week)" + analyst_question: "What is the average MED value set by each origin AS during the first week?" + window: 1 week + sql: | + SELECT origin, avg(toFloat64OrZero(toString(med))) AS avg_med, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY origin + HAVING cnt > 10 + ORDER BY avg_med DESC + LIMIT 25; + + - id: q036 + title: "Local preference value distribution (1 day)" + analyst_question: "What local_pref values were observed on Jan 19 and how common is each?" + window: 1 day + sql: | + SELECT local_pref, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-19 00:00:00' AND timestamp < '2024-01-20 00:00:00' + GROUP BY local_pref + ORDER BY cnt DESC + LIMIT 30; + + - id: q037 + title: "Updates with non-default local preference (6 hours)" + analyst_question: "Which updates between 00:00 and 06:00 on Jan 8 had a local_pref other than 100?" + window: 6 hours + sql: | + SELECT prefix, peer_asn, local_pref, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00' + AND toString(local_pref) != '100' AND toString(local_pref) != '' + ORDER BY timestamp + LIMIT 200; + + - id: q038 + title: "Next-hop diversity per prefix (1 day)" + analyst_question: "Which prefixes were announced with multiple distinct next-hop addresses on Jan 21?" + window: 1 day + sql: | + SELECT prefix, uniqExact(next_hop) AS distinct_next_hops + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-21 00:00:00' AND timestamp < '2024-01-22 00:00:00' + GROUP BY prefix + HAVING distinct_next_hops > 1 + ORDER BY distinct_next_hops DESC + LIMIT 30; + + - id: q039 + title: "Top next-hop addresses by announcement count (1 week)" + analyst_question: "Which next-hop IPs were used most frequently during the first week?" + window: 1 week + sql: | + SELECT next_hop, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY next_hop + ORDER BY cnt DESC + LIMIT 25; + + - id: q040 + title: "Atomic aggregate flag frequency (1 day)" + analyst_question: "How many updates on Jan 13 were marked with the atomic aggregate flag?" + window: 1 day + sql: | + SELECT atomic, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-13 00:00:00' AND timestamp < '2024-01-14 00:00:00' + GROUP BY atomic; + + - id: q041 + title: "Prefixes with atomic aggregate and their aggregator info (1 day)" + analyst_question: "Which prefixes on Jan 13 had the atomic flag set, and who aggregated them?" + window: 1 day + sql: | + SELECT prefix, aggr_asn, aggr_ip, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-13 00:00:00' AND timestamp < '2024-01-14 00:00:00' + AND toString(atomic) IN ('1', 'true', 'True', 'TRUE') + LIMIT 100; + + - id: q042 + title: "Top aggregator ASNs (1 week)" + analyst_question: "Which ASNs appear most often as the aggregator (aggr_asn) during the first week?" + window: 1 week + sql: | + SELECT aggr_asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + AND aggr_asn != '' + GROUP BY aggr_asn + ORDER BY cnt DESC + LIMIT 20; + + - id: q043 + title: "Origin attribute distribution (IGP/EGP/incomplete) over a day" + analyst_question: "How are origin attribute types (IGP, EGP, INCOMPLETE) distributed on Jan 4?" + window: 1 day + sql: | + SELECT origin, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-04 00:00:00' AND timestamp < '2024-01-05 00:00:00' + GROUP BY origin + ORDER BY cnt DESC; + + - id: q044 + title: "Prefix length (mask) distribution (1 day)" + analyst_question: "What is the distribution of announced prefix lengths on Jan 17?" + window: 1 day + sql: | + SELECT splitByChar('/', prefix)[2] AS prefix_len, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-17 00:00:00' AND timestamp < '2024-01-18 00:00:00' + GROUP BY prefix_len + ORDER BY toUInt8OrZero(prefix_len); + + - id: q045 + title: "Count of highly specific /24-and-longer announcements (1 day)" + analyst_question: "How many /24 or longer prefixes were announced on Jan 17?" + window: 1 day + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-17 00:00:00' AND timestamp < '2024-01-18 00:00:00' + AND toUInt8OrZero(splitByChar('/', prefix)[2]) >= 24 + AND NOT match(prefix, ':'); + + - id: q046 + title: "IPv4 vs IPv6 update split (1 day)" + analyst_question: "What proportion of updates on Jan 23 were IPv4 versus IPv6?" + window: 1 day + sql: | + SELECT + countIf(NOT match(prefix, ':')) AS ipv4_updates, + countIf(match(prefix, ':')) AS ipv6_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-23 00:00:00' AND timestamp < '2024-01-24 00:00:00'; + + - id: q047 + title: "IPv6 prefix length distribution (1 week)" + analyst_question: "What IPv6 prefix lengths were seen during the first week of January?" + window: 1 week + sql: | + SELECT splitByChar('/', prefix)[2] AS prefix_len, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND match(prefix, ':') + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY prefix_len + ORDER BY toUInt8OrZero(prefix_len); + + - id: q048 + title: "Bogon/private ASN presence in AS paths (1 day)" + analyst_question: "Which updates on Jan 26 had a private-range ASN (64512-65534) in the AS path?" + window: 1 day + sql: | + SELECT prefix, as_path, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-26 00:00:00' AND timestamp < '2024-01-27 00:00:00' + AND arrayExists(x -> toUInt32OrZero(x) BETWEEN 64512 AND 65534, splitByChar(' ', as_path)) + LIMIT 100; + + - id: q049 + title: "32-bit private ASN range presence in origin (1 week)" + analyst_question: "Were any prefixes originated by 32-bit private ASNs (4200000000-4294967294) during the first week?" + window: 1 week + sql: | + SELECT prefix, origin, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + AND toUInt64OrZero(origin) BETWEEN 4200000000 AND 4294967294 + LIMIT 100; + + - id: q050 + title: "AS path prepending detection (1 day)" + analyst_question: "Which announcements on Jan 24 showed AS path prepending (same ASN repeated consecutively)?" + window: 1 day + sql: | + SELECT prefix, as_path, origin + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-24 00:00:00' AND timestamp < '2024-01-25 00:00:00' + AND length(splitByChar(' ', as_path)) > uniqArray(splitByChar(' ', as_path)) + LIMIT 100; + + - id: q051 + title: "Prefixes with the highest prepend counts (1 day)" + analyst_question: "Which prefixes on Jan 24 had the most AS path prepending?" + window: 1 day + sql: | + SELECT prefix, as_path, + (length(splitByChar(' ', as_path)) - uniqArray(splitByChar(' ', as_path))) AS prepend_count + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-24 00:00:00' AND timestamp < '2024-01-25 00:00:00' + ORDER BY prepend_count DESC + LIMIT 20; + + - id: q052 + title: "Raw sample of recent updates for a peer (15 minutes)" + analyst_question: "What do the raw update records for peer 192.0.2.1 look like between 12:00 and 12:15 on Jan 10?" + window: 15 minutes + sql: | + SELECT timestamp, operation, prefix, as_path, next_hop, local_pref, med + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_ip = '192.0.2.1' + AND timestamp >= '2024-01-10 12:00:00' AND timestamp < '2024-01-10 12:15:00' + ORDER BY timestamp + LIMIT 500; + + - id: q053 + title: "Raw sample of all updates in a 5-minute burst" + analyst_question: "What raw updates arrived between 08:00 and 08:05 on Jan 15?" + window: 5 minutes + sql: | + SELECT * + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 08:00:00' AND timestamp < '2024-01-15 08:05:00' + ORDER BY timestamp + LIMIT 500; + + - id: q054 + title: "Peer first-seen and last-seen timestamps (full month)" + analyst_question: "What is the first and last update timestamp for each peer during January?" + window: full month + sql: | + SELECT peer_ip, min(timestamp) AS first_seen, max(timestamp) AS last_seen, count(*) AS total_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_ip + ORDER BY total_updates DESC + LIMIT 30; + + - id: q055 + title: "New peers that appear mid-month (3 days)" + analyst_question: "Which peer IPs first appeared between Jan 15 and Jan 18?" + window: 3 days + sql: | + SELECT peer_ip, min(timestamp) AS first_seen + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 00:00:00' AND timestamp < '2024-01-18 00:00:00' + GROUP BY peer_ip + HAVING first_seen >= '2024-01-15 00:00:00' + ORDER BY first_seen + LIMIT 50; + + - id: q056 + title: "Peer session gap detection using window functions (1 day)" + analyst_question: "Were there any large gaps (>10 min) between consecutive updates from the same peer on Jan 27?" + window: 1 day + sql: | + SELECT peer_ip, timestamp, prev_ts, dateDiff('second', prev_ts, timestamp) AS gap_seconds + FROM ( + SELECT peer_ip, timestamp, + lagInFrame(timestamp) OVER (PARTITION BY peer_ip ORDER BY timestamp) AS prev_ts + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-27 00:00:00' AND timestamp < '2024-01-28 00:00:00' + ) + WHERE prev_ts IS NOT NULL AND dateDiff('second', prev_ts, timestamp) > 600 + ORDER BY gap_seconds DESC + LIMIT 50; + + - id: q057 + title: "Update rate per minute for a specific peer (1 hour)" + analyst_question: "What was the per-minute update rate for peer_asn 6939 between 09:00 and 10:00 on Jan 5?" + window: 1 hour + sql: | + SELECT toStartOfMinute(timestamp) AS minute, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn = '6939' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00' + GROUP BY minute + ORDER BY minute; + + - id: q058 + title: "Withdrawal spike detection per 5-minute bucket (1 day)" + analyst_question: "Are there any 5-minute windows on Jan 30 with an unusually high number of withdrawals?" + window: 1 day + sql: | + SELECT toStartOfFiveMinutes(timestamp) AS bucket, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-30 00:00:00' AND timestamp < '2024-01-31 00:00:00' + GROUP BY bucket + ORDER BY withdrawals DESC + LIMIT 20; + + - id: q059 + title: "Announcement spike detection per hour (1 week)" + analyst_question: "Which hours during the second week had the highest announcement volume?" + window: 1 week + sql: | + SELECT toStartOfHour(timestamp) AS hour, count(*) AS announcements + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-15 00:00:00' + GROUP BY hour + ORDER BY announcements DESC + LIMIT 20; + + - id: q060 + title: "Distinct source files ingested (1 day)" + analyst_question: "How many distinct source files contributed updates on Jan 2?" + window: 1 day + sql: | + SELECT source_file, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-02 00:00:00' AND timestamp < '2024-01-03 00:00:00' + GROUP BY source_file + ORDER BY cnt DESC + LIMIT 50; + + - id: q061 + title: "Update count per source file over the full month" + analyst_question: "How many updates did each ingested source file contribute during January?" + window: full month + sql: | + SELECT source_file, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY source_file + ORDER BY cnt DESC + LIMIT 60; + + - id: q062 + title: "Top prefixes by withdrawal count (1 day)" + analyst_question: "Which prefixes had the most withdrawals on Jan 29?" + window: 1 day + sql: | + SELECT prefix, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-29 00:00:00' AND timestamp < '2024-01-30 00:00:00' + GROUP BY prefix + ORDER BY withdrawals DESC + LIMIT 25; + + - id: q063 + title: "Prefixes announced then withdrawn within the same hour" + analyst_question: "Which prefixes were both announced and withdrawn within the same hour on Jan 11?" + window: 1 hour + sql: | + SELECT prefix, + countIf(operation = 'A') AS ann_cnt, + countIf(operation = 'W') AS with_cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-11 15:00:00' AND timestamp < '2024-01-11 16:00:00' + GROUP BY prefix + HAVING ann_cnt > 0 AND with_cnt > 0 + ORDER BY (ann_cnt + with_cnt) DESC + LIMIT 30; + + - id: q064 + title: "Update volume by peer_asn and operation over 6 hours" + analyst_question: "How does update volume split by peer ASN and operation type between 00:00 and 06:00 on Jan 8?" + window: 6 hours + sql: | + SELECT peer_asn, operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00' + GROUP BY peer_asn, operation + ORDER BY peer_asn, cnt DESC; + + - id: q065 + title: "Distinct AS paths seen for a given prefix (1 week)" + analyst_question: "How many distinct AS paths were used to reach prefix 9.9.9.0/24 during the first week?" + window: 1 week + sql: | + SELECT DISTINCT as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '9.9.9.0/24' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + LIMIT 100; + + - id: q066 + title: "Peer diversity per prefix (1 day)" + analyst_question: "Which prefixes were seen from the most distinct peers on Jan 3?" + window: 1 day + sql: | + SELECT prefix, uniqExact(peer_ip) AS distinct_peers + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00' + GROUP BY prefix + ORDER BY distinct_peers DESC + LIMIT 25; + + - id: q067 + title: "AS adjacency extraction (edges) for a 1-hour window" + analyst_question: "What AS-to-AS adjacencies appeared in AS paths between 09:00 and 10:00 on Jan 5?" + window: 1 hour + sql: | + SELECT arrayJoin(arrayZip( + arraySlice(splitByChar(' ', as_path), 1, length(splitByChar(' ', as_path)) - 1), + arraySlice(splitByChar(' ', as_path), 2, length(splitByChar(' ', as_path)) - 1) + )) AS as_edge, + count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00' + AND length(splitByChar(' ', as_path)) > 1 + GROUP BY as_edge + ORDER BY cnt DESC + LIMIT 50; + + - id: q068 + title: "First-hop AS (nearest peer AS) frequency (1 day)" + analyst_question: "Which ASNs appear most often as the first hop in AS paths on Jan 6?" + window: 1 day + sql: | + SELECT splitByChar(' ', as_path)[1] AS first_hop_asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + GROUP BY first_hop_asn + ORDER BY cnt DESC + LIMIT 25; + + - id: q069 + title: "Origin AS (last hop) frequency compared to origin field (1 day)" + analyst_question: "Does the last ASN in as_path always match the origin field on Jan 6?" + window: 1 day + sql: | + SELECT + countIf(splitByChar(' ', as_path)[-1] = origin) AS matching, + countIf(splitByChar(' ', as_path)[-1] != origin) AS mismatched + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00'; + + - id: q070 + title: "Mismatched origin vs last-path-hop listing (1 day)" + analyst_question: "Which specific updates on Jan 6 had a mismatch between origin and the last AS-path hop?" + window: 1 day + sql: | + SELECT prefix, as_path, origin, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + AND splitByChar(' ', as_path)[-1] != origin + LIMIT 100; + + - id: q071 + title: "Update count comparison across three consecutive days" + analyst_question: "How did total update volume compare across Jan 10, 11, and 12?" + window: 3 days + sql: | + SELECT toDate(timestamp) AS day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-10 00:00:00' AND timestamp < '2024-01-13 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q072 + title: "Percentile analysis of AS path length (1 week)" + analyst_question: "What are the median and 95th percentile AS path lengths during the first week?" + window: 1 week + sql: | + SELECT + quantile(0.5)(length(splitByChar(' ', as_path))) AS median_len, + quantile(0.95)(length(splitByChar(' ', as_path))) AS p95_len, + max(length(splitByChar(' ', as_path))) AS max_len + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00'; + + - id: q073 + title: "Peer with the largest routing table contribution (1 day)" + analyst_question: "Which peer announced the most distinct prefixes on Jan 15?" + window: 1 day + sql: | + SELECT peer_ip, peer_asn, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-15 00:00:00' AND timestamp < '2024-01-16 00:00:00' + GROUP BY peer_ip, peer_asn + ORDER BY distinct_prefixes DESC + LIMIT 20; + + - id: q074 + title: "Community-tagged traffic engineering events (1 day)" + analyst_question: "Which updates on Jan 28 carried communities suggesting traffic engineering (containing ':666' or ':777' style tags)?" + window: 1 day + sql: | + SELECT prefix, communities, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-28 00:00:00' AND timestamp < '2024-01-29 00:00:00' + AND (positionCaseInsensitive(communities, ':666') > 0 OR positionCaseInsensitive(communities, ':777') > 0) + LIMIT 100; + + - id: q075 + title: "Updates from a specific peer IP over a full week" + analyst_question: "What is the full update history for peer 198.51.100.1 during the first week?" + window: 1 week + sql: | + SELECT timestamp, operation, prefix, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_ip = '198.51.100.1' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ORDER BY timestamp + LIMIT 1000; + + - id: q076 + title: "Comparing morning vs evening update volume (1 day)" + analyst_question: "How does update volume in the morning (00:00-12:00) compare to the evening (12:00-24:00) on Jan 9?" + window: 1 day + sql: | + SELECT + countIf(toHour(timestamp) < 12) AS morning_updates, + countIf(toHour(timestamp) >= 12) AS evening_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-09 00:00:00' AND timestamp < '2024-01-10 00:00:00'; + + - id: q077 + title: "Top 10 busiest 15-minute intervals over a day" + analyst_question: "What were the busiest 15-minute intervals on Jan 31?" + window: 1 day + sql: | + SELECT toStartOfInterval(timestamp, INTERVAL 15 minute) AS bucket, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-31 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY bucket + ORDER BY cnt DESC + LIMIT 10; + + - id: q078 + title: "Prefix update count histogram (1 day)" + analyst_question: "What is the distribution of update counts per prefix on Jan 2 (histogram of churn levels)?" + window: 1 day + sql: | + SELECT update_count, count(*) AS num_prefixes + FROM ( + SELECT prefix, count(*) AS update_count + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-02 00:00:00' AND timestamp < '2024-01-03 00:00:00' + GROUP BY prefix + ) + GROUP BY update_count + ORDER BY update_count; + + - id: q079 + title: "Distinct AS path count per prefix over a week (path instability)" + analyst_question: "Which prefixes used the most distinct AS paths during the first week (instability indicator)?" + window: 1 week + sql: | + SELECT prefix, uniqExact(as_path) AS distinct_paths + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY prefix + ORDER BY distinct_paths DESC + LIMIT 30; + + - id: q080 + title: "Updates for AS32934 (Meta) prefixes over a day" + analyst_question: "What updates involved Meta (AS32934) originated prefixes on Jan 16?" + window: 1 day + sql: | + SELECT timestamp, operation, prefix, peer_asn + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '32934' + AND timestamp >= '2024-01-16 00:00:00' AND timestamp < '2024-01-17 00:00:00' + ORDER BY timestamp + LIMIT 300; + + - id: q081 + title: "Updates for AS16509 (Amazon) prefixes over 6 hours" + analyst_question: "What Amazon (AS16509) prefix activity occurred between 06:00 and 12:00 on Jan 19?" + window: 6 hours + sql: | + SELECT prefix, operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '16509' + AND timestamp >= '2024-01-19 06:00:00' AND timestamp < '2024-01-19 12:00:00' + GROUP BY prefix, operation + ORDER BY cnt DESC + LIMIT 50; + + - id: q082 + title: "Updates for AS8075 (Microsoft) prefixes over a 3-day window" + analyst_question: "What was Microsoft's (AS8075) announcement pattern between Jan 22 and Jan 25?" + window: 3 days + sql: | + SELECT toDate(timestamp) AS day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '8075' + AND timestamp >= '2024-01-22 00:00:00' AND timestamp < '2024-01-25 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q083 + title: "Prefix withdrawal burst for a specific origin AS (1 hour)" + analyst_question: "Did AS7018 (AT&T) show any withdrawal bursts between 20:00 and 21:00 on Jan 27?" + window: 1 hour + sql: | + SELECT toStartOfMinute(timestamp) AS minute, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '7018' + AND operation = 'W' + AND timestamp >= '2024-01-27 20:00:00' AND timestamp < '2024-01-27 21:00:00' + GROUP BY minute + ORDER BY minute; + + - id: q084 + title: "Peer ASN diversity feeding a specific prefix (full month)" + analyst_question: "How many distinct peer ASNs announced prefix 8.8.8.0/24 at any point during January?" + window: full month + sql: | + SELECT uniqExact(peer_asn) AS distinct_peer_asns + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '8.8.8.0/24' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q085 + title: "Communities containing a specific ASN prefix tag (1 day)" + analyst_question: "Which updates on Jan 20 carried a community tag starting with '13335:' (Cloudflare)?" + window: 1 day + sql: | + SELECT prefix, communities, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-20 00:00:00' AND timestamp < '2024-01-21 00:00:00' + AND positionCaseInsensitive(communities, '13335:') > 0 + LIMIT 100; + + - id: q086 + title: "Raw listing of withdrawals only for a peer (15 minutes)" + analyst_question: "What withdrawal messages came from peer_asn 6461 between 12:00 and 12:15 on Jan 10?" + window: 15 minutes + sql: | + SELECT timestamp, prefix, peer_ip + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn = '6461' + AND operation = 'W' + AND timestamp >= '2024-01-10 12:00:00' AND timestamp < '2024-01-10 12:15:00' + ORDER BY timestamp; + + - id: q087 + title: "Full-month total announcements vs withdrawals" + analyst_question: "What is the overall announcement-to-withdrawal ratio for all of January?" + window: full month + sql: | + SELECT operation, count(*) AS cnt, round(count(*) * 100.0 / sum(count(*)) OVER (), 2) AS pct + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY operation; + + - id: q088 + title: "Weekly update volume trend across January" + analyst_question: "How did weekly update totals trend across January 2024?" + window: full month + sql: | + SELECT toStartOfWeek(timestamp) AS week_start, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY week_start + ORDER BY week_start; + + - id: q089 + title: "Prefixes only ever withdrawn, never announced, in-window (1 day)" + analyst_question: "Were there prefixes withdrawn on Jan 31 that had no corresponding announcement that day?" + window: 1 day + sql: | + SELECT prefix, count(*) AS withdrawal_events + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-31 00:00:00' AND timestamp < '2024-02-01 00:00:00' + AND prefix NOT IN ( + SELECT prefix FROM bgp.bgp_updates + WHERE collector = 'rrc00' AND operation = 'A' + AND timestamp >= '2024-01-31 00:00:00' AND timestamp < '2024-02-01 00:00:00' + ) + GROUP BY prefix + ORDER BY withdrawal_events DESC + LIMIT 30; + + - id: q090 + title: "Peer update volume trend over a week using daily buckets" + analyst_question: "How did peer_asn 3356's daily update volume trend during the first week?" + window: 1 week + sql: | + SELECT toDate(timestamp) AS day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn = '3356' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q091 + title: "Correlation between path length and MED (1 day)" + analyst_question: "Is there a relationship between AS path length and MED value on Jan 14?" + window: 1 day + sql: | + SELECT length(splitByChar(' ', as_path)) AS path_len, + avg(toFloat64OrZero(toString(med))) AS avg_med, + count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-14 00:00:00' AND timestamp < '2024-01-15 00:00:00' + GROUP BY path_len + ORDER BY path_len; + + - id: q092 + title: "Top 15 most common full AS paths (1 day)" + analyst_question: "What were the most common complete AS paths observed on Jan 4?" + window: 1 day + sql: | + SELECT as_path, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-04 00:00:00' AND timestamp < '2024-01-05 00:00:00' + GROUP BY as_path + ORDER BY cnt DESC + LIMIT 15; + + - id: q093 + title: "Updates missing a next_hop value (1 day)" + analyst_question: "Were there any announcements on Jan 8 with an empty next_hop field?" + window: 1 day + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-09 00:00:00' + AND (next_hop = '' OR next_hop IS NULL); + + - id: q094 + title: "Updates with empty communities field vs populated (1 day)" + analyst_question: "What proportion of updates on Jan 8 had no community tags at all?" + window: 1 day + sql: | + SELECT + countIf(communities = '') AS no_communities, + countIf(communities != '') AS with_communities + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-09 00:00:00'; + + - id: q095 + title: "Prefix churn ranking with announcement/withdrawal breakdown (3 days)" + analyst_question: "Which prefixes churned the most between Jan 5 and Jan 8, broken down by operation type?" + window: 3 days + sql: | + SELECT prefix, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals, + count(*) AS total + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY prefix + ORDER BY total DESC + LIMIT 30; + + - id: q096 + title: "Distinct origin ASNs seen overall (full month)" + analyst_question: "How many distinct origin ASNs were observed announcing prefixes throughout January?" + window: full month + sql: | + SELECT uniqExact(origin) AS distinct_origin_asns + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q097 + title: "Distinct prefixes seen overall (full month)" + analyst_question: "How many distinct prefixes were observed in total throughout January?" + window: full month + sql: | + SELECT uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q098 + title: "Update volume by peer for a 6-hour evening window" + analyst_question: "How did update volume vary by peer between 18:00 and 24:00 on Jan 24?" + window: 6 hours + sql: | + SELECT peer_asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-24 18:00:00' AND timestamp < '2024-01-25 00:00:00' + GROUP BY peer_asn + ORDER BY cnt DESC + LIMIT 25; + + - id: q099 + title: "Prefixes announced exactly once in a day (stable routes)" + analyst_question: "Which prefixes were announced exactly once (no churn) on Jan 21, indicating stability?" + window: 1 day + sql: | + SELECT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-21 00:00:00' AND timestamp < '2024-01-22 00:00:00' + GROUP BY prefix + HAVING count(*) = 1 + LIMIT 100; + + - id: q100 + title: "Update volume comparison: weekday vs weekend (full month)" + analyst_question: "Was there a notable difference in update volume between weekdays and weekends in January?" + window: full month + sql: | + SELECT + countIf(toDayOfWeek(timestamp) IN (6,7)) AS weekend_updates, + countIf(toDayOfWeek(timestamp) NOT IN (6,7)) AS weekday_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q101 + title: "AS path array explosion for graph analysis (1 hour)" + analyst_question: "What is the full flattened list of ASNs appearing anywhere in AS paths between 09:00 and 10:00 on Jan 5?" + window: 1 hour + sql: | + SELECT arrayJoin(splitByChar(' ', as_path)) AS asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00' + GROUP BY asn + ORDER BY cnt DESC + LIMIT 40; + + - id: q102 + title: "Prefixes announced by AS6939 (Hurricane Electric) over a week" + analyst_question: "What prefixes did AS6939 (Hurricane Electric) originate during the first week?" + window: 1 week + sql: | + SELECT DISTINCT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '6939' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + LIMIT 200; + + - id: q103 + title: "Updates involving prefix 185.1.0.0/16 across the month" + analyst_question: "What is the full monthly history of updates for prefix 185.1.0.0/16?" + window: full month + sql: | + SELECT timestamp, operation, peer_asn, as_path, local_pref, med + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '185.1.0.0/16' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + ORDER BY timestamp + LIMIT 1000; + + - id: q104 + title: "Top 10 prefixes by peer diversity over the month" + analyst_question: "Which prefixes were visible from the widest set of distinct peers during January?" + window: full month + sql: | + SELECT prefix, uniqExact(peer_ip) AS peer_count + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY prefix + ORDER BY peer_count DESC + LIMIT 10; + + - id: q105 + title: "Local pref anomalies compared to peer's usual value (1 day)" + analyst_question: "Which peers used an unusual local_pref value compared to their most common value on Jan 12?" + window: 1 day + sql: | + SELECT peer_ip, local_pref, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-12 00:00:00' AND timestamp < '2024-01-13 00:00:00' + GROUP BY peer_ip, local_pref + ORDER BY peer_ip, cnt DESC + LIMIT 200; + + - id: q106 + title: "Aggregated route counts by aggr_ip (1 week)" + analyst_question: "Which aggregator IPs were used most often during the first week?" + window: 1 week + sql: | + SELECT aggr_ip, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND aggr_ip != '' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY aggr_ip + ORDER BY cnt DESC + LIMIT 20; + + - id: q107 + title: "Updates around a suspected outage window (15 minutes)" + analyst_question: "What was the update activity between 03:00 and 03:15 on Jan 18 (suspected outage window)?" + window: 15 minutes + sql: | + SELECT operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-18 03:00:00' AND timestamp < '2024-01-18 03:15:00' + GROUP BY operation; + + - id: q108 + title: "Prefixes affected during a suspected outage window (15 minutes)" + analyst_question: "Which specific prefixes were withdrawn between 03:00 and 03:15 on Jan 18?" + window: 15 minutes + sql: | + SELECT prefix, peer_asn, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-18 03:00:00' AND timestamp < '2024-01-18 03:15:00' + ORDER BY timestamp + LIMIT 200; + + - id: q109 + title: "Recovery pattern after outage window (1 hour)" + analyst_question: "How did announcements recover in the hour following the Jan 18 03:00 event?" + window: 1 hour + sql: | + SELECT toStartOfMinute(timestamp) AS minute, countIf(operation = 'A') AS anns, countIf(operation = 'W') AS withs + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-18 03:00:00' AND timestamp < '2024-01-18 04:00:00' + GROUP BY minute + ORDER BY minute; + + - id: q110 + title: "Distinct peer ASN count trend across the month (daily)" + analyst_question: "How many distinct peer ASNs were active on each day of January?" + window: full month + sql: | + SELECT toDate(timestamp) AS day, uniqExact(peer_asn) AS distinct_peer_asns + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q111 + title: "Updates with communities matching regexp pattern (1 day)" + analyst_question: "Which updates on Jan 13 had a community value matching a blackhole pattern like 'xxx:666'?" + window: 1 day + sql: | + SELECT prefix, communities, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-13 00:00:00' AND timestamp < '2024-01-14 00:00:00' + AND match(communities, '[0-9]+:666') + LIMIT 100; + + - id: q112 + title: "RTBH (blackhole) community usage trend over the month" + analyst_question: "How did use of blackhole-style communities (':666') trend across January?" + window: full month + sql: | + SELECT toDate(timestamp) AS day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + AND match(communities, '[0-9]+:666') + GROUP BY day + ORDER BY day; + + - id: q113 + title: "Peer withdrawal-only sessions (1 day)" + analyst_question: "Were there peers on Jan 29 that only sent withdrawals, no announcements (possible session reset)?" + window: 1 day + sql: | + SELECT peer_ip, count(*) AS withdrawal_cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-29 00:00:00' AND timestamp < '2024-01-30 00:00:00' + GROUP BY peer_ip + HAVING countIf(operation = 'A') = 0 AND countIf(operation = 'W') > 0 + ORDER BY withdrawal_cnt DESC + LIMIT 20; + + - id: q114 + title: "Median update inter-arrival time per peer (1 day)" + analyst_question: "What is the median time between consecutive updates from each peer on Jan 7?" + window: 1 day + sql: | + SELECT peer_ip, quantile(0.5)(gap) AS median_gap_seconds + FROM ( + SELECT peer_ip, timestamp, + dateDiff('second', lagInFrame(timestamp) OVER (PARTITION BY peer_ip ORDER BY timestamp), timestamp) AS gap + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-07 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ) + WHERE gap IS NOT NULL AND gap > 0 + GROUP BY peer_ip + ORDER BY median_gap_seconds + LIMIT 30; + + - id: q115 + title: "Announcement count per origin AS across the month (top 30)" + analyst_question: "Which origin ASNs announced the most total prefixes across all of January?" + window: full month + sql: | + SELECT origin, count(*) AS announcements, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY origin + ORDER BY announcements DESC + LIMIT 30; + + - id: q116 + title: "Prefixes with a suspiciously large number of distinct origins over the month" + analyst_question: "Which prefixes had 3 or more distinct origin ASNs across January (strong MOAS candidates)?" + window: full month + sql: | + SELECT prefix, uniqExact(origin) AS distinct_origins, groupArray(DISTINCT origin) AS origins + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY prefix + HAVING distinct_origins >= 3 + ORDER BY distinct_origins DESC + LIMIT 30; + + - id: q117 + title: "Time to first announcement of the month for select prefixes" + analyst_question: "When was prefix 203.0.113.0/24 first announced during January?" + window: full month + sql: | + SELECT min(timestamp) AS first_announcement + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '203.0.113.0/24' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q118 + title: "Last known state of a prefix before month end" + analyst_question: "What was the last recorded state (announce/withdraw) of prefix 203.0.113.0/24 before month end?" + window: full month + sql: | + SELECT timestamp, operation, as_path, next_hop + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '203.0.113.0/24' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + ORDER BY timestamp DESC + LIMIT 1; + + - id: q119 + title: "Community tag co-occurrence (1 day)" + analyst_question: "Which pairs of community values co-occurred most often on Jan 26?" + window: 1 day + sql: | + SELECT communities, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-26 00:00:00' AND timestamp < '2024-01-27 00:00:00' + AND length(splitByChar(' ', communities)) >= 2 + GROUP BY communities + ORDER BY cnt DESC + LIMIT 20; + + - id: q120 + title: "Updates per operation type over a 3-day rolling comparison" + analyst_question: "How did daily announcement and withdrawal totals compare over Jan 20-22?" + window: 3 days + sql: | + SELECT toDate(timestamp) AS day, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-20 00:00:00' AND timestamp < '2024-01-23 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q121 + title: "Distinct next_hop count for a specific origin AS (1 week)" + analyst_question: "How many distinct next-hop addresses did AS15169 (Google) use during the first week?" + window: 1 week + sql: | + SELECT uniqExact(next_hop) AS distinct_next_hops + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '15169' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00'; + + - id: q122 + title: "Rare AS path patterns (paths seen only once) in a day" + analyst_question: "Which AS paths appeared exactly once on Jan 3 (unusual/rare paths)?" + window: 1 day + sql: | + SELECT as_path, prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00' + GROUP BY as_path, prefix + HAVING count(*) = 1 + LIMIT 100; + + - id: q123 + title: "Updates arriving out of chronological order check (5 minutes)" + analyst_question: "Were there any updates between 08:00 and 08:05 on Jan 15 with timestamps that appear inconsistent with source_file ordering?" + window: 5 minutes + sql: | + SELECT source_file, min(timestamp) AS min_ts, max(timestamp) AS max_ts, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 08:00:00' AND timestamp < '2024-01-15 08:05:00' + GROUP BY source_file + ORDER BY min_ts; + + - id: q124 + title: "Peer ASN to prefix count matrix (top peers, 1 day)" + analyst_question: "For the top 10 busiest peers on Jan 6, how many distinct prefixes did each announce?" + window: 1 day + sql: | + SELECT peer_asn, uniqExact(prefix) AS distinct_prefixes, count(*) AS total_updates + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + GROUP BY peer_asn + ORDER BY total_updates DESC + LIMIT 10; + + - id: q125 + title: "Updates for a specific /8 supernet block over a day" + analyst_question: "What update activity occurred for prefixes within 10.0.0.0/8 on Jan 9 (private space leak check)?" + window: 1 day + sql: | + SELECT prefix, operation, peer_asn, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND startsWith(prefix, '10.') + AND timestamp >= '2024-01-09 00:00:00' AND timestamp < '2024-01-10 00:00:00' + LIMIT 100; + + - id: q126 + title: "RFC1918 private prefix leak check (1 week)" + analyst_question: "Were any RFC1918 private prefixes (192.168.x.x, 172.16-31.x.x) leaked into rrc00 during the first week?" + window: 1 week + sql: | + SELECT prefix, peer_asn, as_path, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + AND (startsWith(prefix, '192.168.') OR startsWith(prefix, '10.')) + LIMIT 100; + + - id: q127 + title: "Update volume grouped by 3-day totals across the month" + analyst_question: "What are the 3-day rolling totals of updates across January?" + window: full month + sql: | + SELECT toStartOfInterval(timestamp, INTERVAL 3 day) AS bucket, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY bucket + ORDER BY bucket; + + - id: q128 + title: "Peer with most withdrawals relative to announcements (1 day)" + analyst_question: "Which peer had the highest withdrawal-to-announcement ratio on Jan 15?" + window: 1 day + sql: | + SELECT peer_ip, + countIf(operation = 'W') AS withdrawals, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') / greatest(countIf(operation = 'A'), 1) AS ratio + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 00:00:00' AND timestamp < '2024-01-16 00:00:00' + GROUP BY peer_ip + HAVING announcements + withdrawals > 20 + ORDER BY ratio DESC + LIMIT 20; + + - id: q129 + title: "Community usage by top origin ASNs (1 day)" + analyst_question: "Which communities did the top 5 origin ASNs by volume use on Jan 10?" + window: 1 day + sql: | + SELECT origin, communities, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND communities != '' + AND timestamp >= '2024-01-10 00:00:00' AND timestamp < '2024-01-11 00:00:00' + AND origin IN ( + SELECT origin FROM bgp.bgp_updates + WHERE collector = 'rrc00' AND operation = 'A' + AND timestamp >= '2024-01-10 00:00:00' AND timestamp < '2024-01-11 00:00:00' + GROUP BY origin ORDER BY count(*) DESC LIMIT 5 + ) + GROUP BY origin, communities + ORDER BY origin, cnt DESC; + + - id: q130 + title: "Update event count for specific source file (1 day)" + analyst_question: "How many events came from source file 'rrc00.20240111.0800.bz2' if present?" + window: 1 day + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND source_file = 'rrc00.20240111.0800.bz2' + AND timestamp >= '2024-01-11 00:00:00' AND timestamp < '2024-01-12 00:00:00'; + + - id: q131 + title: "Announcement rate acceleration check (5-minute buckets, 1 hour)" + analyst_question: "Did announcement rates spike suspiciously in any 5-minute bucket between 14:00 and 15:00 on Jan 9?" + window: 1 hour + sql: | + SELECT toStartOfFiveMinutes(timestamp) AS bucket, countIf(operation = 'A') AS anns + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-09 14:00:00' AND timestamp < '2024-01-09 15:00:00' + GROUP BY bucket + ORDER BY bucket; + + - id: q132 + title: "Top 10 peer ASNs by distinct prefix coverage over the month" + analyst_question: "Which peer ASNs provided visibility into the largest number of distinct prefixes over all of January?" + window: full month + sql: | + SELECT peer_asn, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_asn + ORDER BY distinct_prefixes DESC + LIMIT 10; + + - id: q133 + title: "Origin AS churn: distinct origins per prefix trend by week" + analyst_question: "How did MOAS-prefix counts (2+ distinct origins) trend week over week in January?" + window: full month + sql: | + SELECT toStartOfWeek(timestamp) AS week_start, count(*) AS moas_prefix_count + FROM ( + SELECT toStartOfWeek(timestamp) AS timestamp, prefix, uniqExact(origin) AS origins + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY timestamp, prefix + HAVING origins > 1 + ) + GROUP BY week_start + ORDER BY week_start; + + - id: q134 + title: "Peer session churn: distinct peers per hour over a day" + analyst_question: "How did the number of distinct active peers vary hour by hour on Jan 30?" + window: 1 day + sql: | + SELECT toStartOfHour(timestamp) AS hour, uniqExact(peer_ip) AS distinct_peers + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-30 00:00:00' AND timestamp < '2024-01-31 00:00:00' + GROUP BY hour + ORDER BY hour; + + - id: q135 + title: "Longest single AS path of the month" + analyst_question: "What was the single longest AS path observed anywhere in January?" + window: full month + sql: | + SELECT prefix, as_path, length(splitByChar(' ', as_path)) AS path_len, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + ORDER BY path_len DESC + LIMIT 1; + + - id: q136 + title: "Updates with local_pref higher than typical default (1 day)" + analyst_question: "Which updates on Jan 17 had a local_pref greater than 200 (unusually high)?" + window: 1 day + sql: | + SELECT prefix, peer_asn, local_pref, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-17 00:00:00' AND timestamp < '2024-01-18 00:00:00' + AND toInt64OrZero(toString(local_pref)) > 200 + ORDER BY toInt64OrZero(toString(local_pref)) DESC + LIMIT 50; + + - id: q137 + title: "Updates with negative or zero MED (1 day)" + analyst_question: "Were there any updates on Jan 17 with a MED of zero or an invalid negative value?" + window: 1 day + sql: | + SELECT prefix, peer_asn, med, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-17 00:00:00' AND timestamp < '2024-01-18 00:00:00' + AND toInt64OrZero(toString(med)) <= 0 + LIMIT 50; + + - id: q138 + title: "Distinct communities count trend across the month (weekly)" + analyst_question: "How did the number of distinct community strings used trend week by week in January?" + window: full month + sql: | + SELECT toStartOfWeek(timestamp) AS week_start, uniqExact(communities) AS distinct_communities + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND communities != '' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY week_start + ORDER BY week_start; + + - id: q139 + title: "Prefix reachability status snapshot (argMax) at end of a day" + analyst_question: "What is the last known operation for each prefix as of end-of-day Jan 5?" + window: 1 day + sql: | + SELECT prefix, argMax(operation, timestamp) AS last_operation, max(timestamp) AS last_seen + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 00:00:00' AND timestamp < '2024-01-06 00:00:00' + GROUP BY prefix + LIMIT 200; + + - id: q140 + title: "Latest AS path per prefix snapshot (argMax) in a 6-hour window" + analyst_question: "What is the most recent AS path used for each prefix between 00:00 and 06:00 on Jan 8?" + window: 6 hours + sql: | + SELECT prefix, argMax(as_path, timestamp) AS latest_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00' + GROUP BY prefix + LIMIT 200; + + - id: q141 + title: "Peers that stopped sending updates mid-window (1 day)" + analyst_question: "Which peers were active in the first half of Jan 22 but silent in the second half?" + window: 1 day + sql: | + SELECT peer_ip + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-22 00:00:00' AND timestamp < '2024-01-22 12:00:00' + GROUP BY peer_ip + HAVING peer_ip NOT IN ( + SELECT DISTINCT peer_ip FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-22 12:00:00' AND timestamp < '2024-01-23 00:00:00' + ) + LIMIT 30; + + - id: q142 + title: "New prefixes first observed in a given week" + analyst_question: "Which prefixes were seen for the first time during the second week of January (not seen in week 1)?" + window: 1 week + sql: | + SELECT DISTINCT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-15 00:00:00' + AND prefix NOT IN ( + SELECT DISTINCT prefix FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + ) + LIMIT 100; + + - id: q143 + title: "Prefixes that disappeared after a given week (potential deaggregation/withdrawal)" + analyst_question: "Which prefixes seen in week 1 were never seen again during week 2?" + window: 1 week + sql: | + SELECT DISTINCT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + AND prefix NOT IN ( + SELECT DISTINCT prefix FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-15 00:00:00' + ) + LIMIT 100; + + - id: q144 + title: "Update count per collector sanity check (1 day)" + analyst_question: "As a sanity check, confirm all returned rows for Jan 1 are indeed from rrc00." + window: 1 day + sql: | + SELECT collector, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-02 00:00:00' + GROUP BY collector; + + - id: q145 + title: "Peer ASN with the shortest average AS path (1 day)" + analyst_question: "Which peer ASN reported the shortest average AS paths on Jan 19, suggesting close proximity?" + window: 1 day + sql: | + SELECT peer_asn, avg(length(splitByChar(' ', as_path))) AS avg_len, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-19 00:00:00' AND timestamp < '2024-01-20 00:00:00' + GROUP BY peer_asn + HAVING cnt > 50 + ORDER BY avg_len ASC + LIMIT 20; + + - id: q146 + title: "Update volume for /16 aggregate blocks (1 day)" + analyst_question: "Aggregating by /16 supernet, which blocks had the most update activity on Jan 25?" + window: 1 day + sql: | + SELECT splitByChar('.', prefix)[1] || '.' || splitByChar('.', prefix)[2] || '.0.0/16' AS supernet, + count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND NOT match(prefix, ':') + AND timestamp >= '2024-01-25 00:00:00' AND timestamp < '2024-01-26 00:00:00' + GROUP BY supernet + ORDER BY cnt DESC + LIMIT 25; + + - id: q147 + title: "Communities frequency ranked by distinct prefixes tagged (1 week)" + analyst_question: "Which communities were applied to the widest range of distinct prefixes during the first week?" + window: 1 week + sql: | + SELECT communities, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND communities != '' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY communities + ORDER BY distinct_prefixes DESC + LIMIT 20; + + - id: q148 + title: "Update volume around a known internet event (6 hours)" + analyst_question: "What update activity occurred between 12:00 and 18:00 on Jan 1, 2024 (New Year holiday traffic)?" + window: 6 hours + sql: | + SELECT toStartOfHour(timestamp) AS hour, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 12:00:00' AND timestamp < '2024-01-01 18:00:00' + GROUP BY hour + ORDER BY hour; + + - id: q149 + title: "Distinct as_path count for a single peer across the month" + analyst_question: "How many distinct AS paths did peer_ip 203.0.113.5 report throughout January?" + window: full month + sql: | + SELECT uniqExact(as_path) AS distinct_paths + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_ip = '203.0.113.5' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q150 + title: "Prefixes overlapping a target supernet (1 day)" + analyst_question: "Which more-specific prefixes overlap with 172.217.0.0/16 (Google range) on Jan 16?" + window: 1 day + sql: | + SELECT DISTINCT prefix + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND startsWith(prefix, '172.217.') + AND timestamp >= '2024-01-16 00:00:00' AND timestamp < '2024-01-17 00:00:00' + LIMIT 100; + + - id: q151 + title: "Update-type breakdown by hour for a specific origin AS (1 day)" + analyst_question: "How did announcement/withdrawal activity for AS701 (Verizon) vary hourly on Jan 2?" + window: 1 day + sql: | + SELECT toStartOfHour(timestamp) AS hour, + countIf(operation = 'A') AS anns, + countIf(operation = 'W') AS withs + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '701' + AND timestamp >= '2024-01-02 00:00:00' AND timestamp < '2024-01-03 00:00:00' + GROUP BY hour + ORDER BY hour; + + - id: q152 + title: "Cross-tabulation of operation by prefix length (1 day)" + analyst_question: "Are withdrawals more common for longer prefixes on Jan 23?" + window: 1 day + sql: | + SELECT toUInt8OrZero(splitByChar('/', prefix)[2]) AS prefix_len, operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND NOT match(prefix, ':') + AND timestamp >= '2024-01-23 00:00:00' AND timestamp < '2024-01-24 00:00:00' + GROUP BY prefix_len, operation + ORDER BY prefix_len, operation; + + - id: q153 + title: "Top 5 peer ASNs contributing withdrawals in a 3-day window" + analyst_question: "Which peer ASNs generated the most withdrawal messages between Jan 27 and Jan 30?" + window: 3 days + sql: | + SELECT peer_asn, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-27 00:00:00' AND timestamp < '2024-01-30 00:00:00' + GROUP BY peer_asn + ORDER BY withdrawals DESC + LIMIT 5; + + - id: q154 + title: "Update count for specific next_hop address over a week" + analyst_question: "How many updates used next_hop 198.51.100.254 during the first week?" + window: 1 week + sql: | + SELECT count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND next_hop = '198.51.100.254' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00'; + + - id: q155 + title: "Deduplicated distinct update signatures in a 5-minute window" + analyst_question: "How many unique (prefix, operation, as_path) combinations occurred between 08:00 and 08:05 on Jan 15?" + window: 5 minutes + sql: | + SELECT uniqExact(prefix, operation, as_path) AS distinct_signatures + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 08:00:00' AND timestamp < '2024-01-15 08:05:00'; + + - id: q156 + title: "Percent of updates from top 5 peers (1 day)" + analyst_question: "What percentage of Jan 6 updates came from the 5 busiest peers?" + window: 1 day + sql: | + SELECT peer_asn, count(*) AS cnt, round(count(*) * 100.0 / sum(count(*)) OVER (), 2) AS pct_of_day + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + GROUP BY peer_asn + ORDER BY cnt DESC + LIMIT 5; + + - id: q157 + title: "Updates for a large CDN prefix range over 3 days" + analyst_question: "What update activity did prefix 104.16.0.0/12-range (Cloudflare CDN) prefixes show between Jan 12 and Jan 15?" + window: 3 days + sql: | + SELECT prefix, operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND startsWith(prefix, '104.16.') + AND timestamp >= '2024-01-12 00:00:00' AND timestamp < '2024-01-15 00:00:00' + GROUP BY prefix, operation + ORDER BY cnt DESC + LIMIT 50; + + - id: q158 + title: "Communities value length distribution (1 day)" + analyst_question: "How long (in characters) are the community strings observed on Jan 5, and how are they distributed?" + window: 1 day + sql: | + SELECT length(communities) AS comm_len, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 00:00:00' AND timestamp < '2024-01-06 00:00:00' + AND communities != '' + GROUP BY comm_len + ORDER BY comm_len; + + - id: q159 + title: "Peer ASN update volume rank change over two weeks" + analyst_question: "How did the ranking of top peer ASNs by volume differ between week 1 and week 2 of January?" + window: full month + sql: | + SELECT peer_asn, + countIf(timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00') AS week1_cnt, + countIf(timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-15 00:00:00') AS week2_cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-15 00:00:00' + GROUP BY peer_asn + ORDER BY week1_cnt DESC + LIMIT 20; + + - id: q160 + title: "Prefix-level snapshot join comparing start-of-month vs mid-month state" + analyst_question: "For prefixes seen on Jan 1, what was their most recent AS path as of Jan 15?" + window: full month + sql: | + SELECT prefix, argMax(as_path, timestamp) AS latest_path_by_jan15 + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-15 00:00:00' + AND prefix IN ( + SELECT DISTINCT prefix FROM bgp.bgp_updates + WHERE collector = 'rrc00' AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-02 00:00:00' + ) + GROUP BY prefix + LIMIT 100; + + - id: q161 + title: "Update burst comparison: two adjacent 15-minute windows" + analyst_question: "How did update volume compare between 12:00-12:15 and 12:15-12:30 on Jan 10?" + window: 15 minutes + sql: | + SELECT + countIf(timestamp >= '2024-01-10 12:00:00' AND timestamp < '2024-01-10 12:15:00') AS window1, + countIf(timestamp >= '2024-01-10 12:15:00' AND timestamp < '2024-01-10 12:30:00') AS window2 + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-10 12:00:00' AND timestamp < '2024-01-10 12:30:00'; + + - id: q162 + title: "Origin AS with the most withdrawal activity (1 day)" + analyst_question: "Which origin AS had the highest withdrawal count on Jan 31?" + window: 1 day + sql: | + SELECT origin, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-31 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY origin + ORDER BY withdrawals DESC + LIMIT 20; + + - id: q163 + title: "AS path containing two specific transit ASNs in sequence (1 day)" + analyst_question: "Which updates on Jan 9 had AS3356 directly followed by AS174 in the path?" + window: 1 day + sql: | + SELECT prefix, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-09 00:00:00' AND timestamp < '2024-01-10 00:00:00' + AND positionCaseInsensitive(as_path, '3356 174') > 0 + LIMIT 50; + + - id: q164 + title: "Update rate normalized per peer per hour (6 hours)" + analyst_question: "What is the average updates-per-hour rate for each peer between 00:00 and 06:00 on Jan 8?" + window: 6 hours + sql: | + SELECT peer_asn, count(*) / 6.0 AS avg_updates_per_hour + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-08 00:00:00' AND timestamp < '2024-01-08 06:00:00' + GROUP BY peer_asn + ORDER BY avg_updates_per_hour DESC + LIMIT 25; + + - id: q165 + title: "Origin ASN stability check: same origin across full month for a prefix" + analyst_question: "Did prefix 8.8.8.0/24 keep a consistent origin AS throughout January?" + window: full month + sql: | + SELECT uniqExact(origin) AS distinct_origins, groupArray(DISTINCT origin) AS origins_list + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '8.8.8.0/24' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q166 + title: "Update volume vs distinct prefix count correlation (daily, full month)" + analyst_question: "Across January, how does total daily update volume compare to the number of distinct prefixes touched?" + window: full month + sql: | + SELECT toDate(timestamp) AS day, count(*) AS total_updates, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day + ORDER BY day; + + - id: q167 + title: "Peer ASNs seen only briefly (single-day presence) across the month" + analyst_question: "Which peer ASNs appeared on only one calendar day during all of January?" + window: full month + sql: | + SELECT peer_asn, uniqExact(toDate(timestamp)) AS active_days + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_asn + HAVING active_days = 1 + LIMIT 30; + + - id: q168 + title: "Consistently active peer ASNs across the month" + analyst_question: "Which peer ASNs sent updates on every single day of January?" + window: full month + sql: | + SELECT peer_asn, uniqExact(toDate(timestamp)) AS active_days + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_asn + HAVING active_days = 31 + ORDER BY peer_asn + LIMIT 30; + + - id: q169 + title: "Prefix count by first octet range (1 day)" + analyst_question: "How are announced IPv4 prefixes on Jan 27 distributed across first-octet ranges?" + window: 1 day + sql: | + SELECT toUInt16OrZero(splitByChar('.', prefix)[1]) AS first_octet, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND NOT match(prefix, ':') + AND timestamp >= '2024-01-27 00:00:00' AND timestamp < '2024-01-28 00:00:00' + GROUP BY first_octet + ORDER BY first_octet; + + - id: q170 + title: "Updates joined with their preceding state for local_pref changes (1 day)" + analyst_question: "Which prefixes had a local_pref change between consecutive announcements on Jan 3?" + window: 1 day + sql: | + SELECT prefix, timestamp, local_pref, prev_local_pref + FROM ( + SELECT prefix, timestamp, local_pref, + lagInFrame(local_pref) OVER (PARTITION BY prefix ORDER BY timestamp) AS prev_local_pref + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00' + ) + WHERE prev_local_pref IS NOT NULL AND toString(local_pref) != toString(prev_local_pref) + ORDER BY prefix, timestamp + LIMIT 100; + + - id: q171 + title: "Next-hop change detection via window function (1 day)" + analyst_question: "Which prefixes changed next_hop between consecutive announcements on Jan 3?" + window: 1 day + sql: | + SELECT prefix, timestamp, next_hop, prev_next_hop + FROM ( + SELECT prefix, timestamp, next_hop, + lagInFrame(next_hop) OVER (PARTITION BY prefix ORDER BY timestamp) AS prev_next_hop + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-03 00:00:00' AND timestamp < '2024-01-04 00:00:00' + ) + WHERE prev_next_hop IS NOT NULL AND next_hop != prev_next_hop + ORDER BY prefix, timestamp + LIMIT 100; + + - id: q172 + title: "Aggregate flag change over time for a prefix (full month)" + analyst_question: "Did prefix 172.217.0.0/16 ever toggle its atomic aggregate flag during January?" + window: full month + sql: | + SELECT timestamp, atomic, aggr_asn, aggr_ip + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND prefix = '172.217.0.0/16' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + ORDER BY timestamp; + + - id: q173 + title: "Update volume filtered to a specific origin and time-of-day window (1 hour)" + analyst_question: "What did AS13335 (Cloudflare) announcement activity look like between 03:00 and 04:00 on Jan 21?" + window: 1 hour + sql: | + SELECT timestamp, prefix, operation + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND origin = '13335' + AND timestamp >= '2024-01-21 03:00:00' AND timestamp < '2024-01-21 04:00:00' + ORDER BY timestamp + LIMIT 200; + + - id: q174 + title: "Count of updates by AS path first two hops (1 day)" + analyst_question: "What are the most common two-hop AS path prefixes (first two ASNs) seen on Jan 12?" + window: 1 day + sql: | + SELECT arraySlice(splitByChar(' ', as_path), 1, 2) AS first_two_hops, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-12 00:00:00' AND timestamp < '2024-01-13 00:00:00' + AND length(splitByChar(' ', as_path)) >= 2 + GROUP BY first_two_hops + ORDER BY cnt DESC + LIMIT 25; + + - id: q175 + title: "Update timestamp granularity check (distinct seconds with activity, 1 hour)" + analyst_question: "How many distinct seconds within 09:00-10:00 on Jan 5 had at least one update?" + window: 1 hour + sql: | + SELECT uniqExact(timestamp) AS distinct_seconds_with_activity + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-05 09:00:00' AND timestamp < '2024-01-05 10:00:00'; + + - id: q176 + title: "Updates grouped by peer_asn and prefix length category (1 day)" + analyst_question: "For each peer on Jan 24, what proportion of their announcements were /24 or longer vs shorter?" + window: 1 day + sql: | + SELECT peer_asn, + countIf(toUInt8OrZero(splitByChar('/', prefix)[2]) >= 24) AS long_prefixes, + countIf(toUInt8OrZero(splitByChar('/', prefix)[2]) < 24) AS short_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND NOT match(prefix, ':') + AND timestamp >= '2024-01-24 00:00:00' AND timestamp < '2024-01-25 00:00:00' + GROUP BY peer_asn + ORDER BY long_prefixes DESC + LIMIT 25; + + - id: q177 + title: "Update volume comparison across three different collectors' worth of rrc00 peers grouped by ASN family" + analyst_question: "How do updates from Tier-1 transit ASNs (174, 3356, 6939, 1299) compare in volume on Jan 6?" + window: 1 day + sql: | + SELECT peer_asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn IN ('174', '3356', '6939', '1299') + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + GROUP BY peer_asn + ORDER BY cnt DESC; + + - id: q178 + title: "Distinct prefix count for Tier-1 transit peers over a week" + analyst_question: "How many distinct prefixes did Tier-1 peers (174, 3356, 6939, 1299) collectively show during week 1?" + window: 1 week + sql: | + SELECT uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND peer_asn IN ('174', '3356', '6939', '1299') + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00'; + + - id: q179 + title: "Withdrawal count trend across the month by week" + analyst_question: "How did total weekly withdrawal counts trend across January?" + window: full month + sql: | + SELECT toStartOfWeek(timestamp) AS week_start, count(*) AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY week_start + ORDER BY week_start; + + - id: q180 + title: "Announcement count trend across the month by week" + analyst_question: "How did total weekly announcement counts trend across January?" + window: full month + sql: | + SELECT toStartOfWeek(timestamp) AS week_start, count(*) AS announcements + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY week_start + ORDER BY week_start; + + - id: q181 + title: "Prefixes with communities but no local_pref set (1 day)" + analyst_question: "Were there updates on Jan 14 that had communities attached but no local_pref value?" + window: 1 day + sql: | + SELECT prefix, communities, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-14 00:00:00' AND timestamp < '2024-01-15 00:00:00' + AND communities != '' + AND (toString(local_pref) = '' OR local_pref IS NULL) + LIMIT 50; + + - id: q182 + title: "Peer_ip vs peer_asn consistency check (1 day)" + analyst_question: "Did any peer_ip on Jan 18 report more than one distinct peer_asn (possible renumbering or data issue)?" + window: 1 day + sql: | + SELECT peer_ip, uniqExact(peer_asn) AS distinct_asns + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-18 00:00:00' AND timestamp < '2024-01-19 00:00:00' + GROUP BY peer_ip + HAVING distinct_asns > 1 + LIMIT 20; + + - id: q183 + title: "Update volume for a targeted incident window (5 minutes)" + analyst_question: "What exact updates occurred between 16:32 and 16:37 on Jan 29 during a reported incident?" + window: 5 minutes + sql: | + SELECT timestamp, operation, prefix, peer_asn, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-29 16:32:00' AND timestamp < '2024-01-29 16:37:00' + ORDER BY timestamp + LIMIT 500; + + - id: q184 + title: "Prefix deaggregation check: more-specifics of a supernet appearing suddenly (1 day)" + analyst_question: "Were there new /24 announcements within 203.0.113.0/24's parent block on Jan 29 suggesting deaggregation?" + window: 1 day + sql: | + SELECT prefix, operation, timestamp, as_path + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND startsWith(prefix, '203.0.113.') + AND operation = 'A' + AND timestamp >= '2024-01-29 00:00:00' AND timestamp < '2024-01-30 00:00:00' + ORDER BY timestamp + LIMIT 100; + + - id: q185 + title: "Community usage percentage by peer (1 day)" + analyst_question: "What percentage of each peer's announcements on Jan 20 carried at least one community?" + window: 1 day + sql: | + SELECT peer_asn, + countIf(communities != '') AS with_comm, + count(*) AS total, + round(countIf(communities != '') * 100.0 / count(*), 2) AS pct_with_comm + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-20 00:00:00' AND timestamp < '2024-01-21 00:00:00' + GROUP BY peer_asn + ORDER BY pct_with_comm DESC + LIMIT 25; + + - id: q186 + title: "Origin AS with widest next-hop diversity (1 week)" + analyst_question: "Which origin AS used the greatest number of distinct next-hop addresses during the first week?" + window: 1 week + sql: | + SELECT origin, uniqExact(next_hop) AS distinct_next_hops + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY origin + ORDER BY distinct_next_hops DESC + LIMIT 20; + + - id: q187 + title: "Updates sampled every 5-minute bucket showing avg path length trend (1 day)" + analyst_question: "How did average AS path length trend across 5-minute buckets on Jan 5?" + window: 1 day + sql: | + SELECT toStartOfFiveMinutes(timestamp) AS bucket, avg(length(splitByChar(' ', as_path))) AS avg_path_len + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-05 00:00:00' AND timestamp < '2024-01-06 00:00:00' + GROUP BY bucket + ORDER BY bucket + LIMIT 300; + + - id: q188 + title: "Ranking prefixes by update rate per hour using window function (1 day)" + analyst_question: "Using a running rank, which prefixes were consistently in the top 5 most-updated per hour on Jan 11?" + window: 1 day + sql: | + SELECT hour, prefix, cnt, rnk + FROM ( + SELECT toStartOfHour(timestamp) AS hour, prefix, count(*) AS cnt, + row_number() OVER (PARTITION BY toStartOfHour(timestamp) ORDER BY count(*) DESC) AS rnk + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-11 00:00:00' AND timestamp < '2024-01-12 00:00:00' + GROUP BY hour, prefix + ) + WHERE rnk <= 5 + ORDER BY hour, rnk; + + - id: q189 + title: "Detecting simultaneous withdrawals across many prefixes from one peer (5 minutes)" + analyst_question: "Did any single peer withdraw an unusually large batch of prefixes between 16:32 and 16:37 on Jan 29?" + window: 5 minutes + sql: | + SELECT peer_ip, count(*) AS withdrawal_count, uniqExact(prefix) AS distinct_prefixes + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'W' + AND timestamp >= '2024-01-29 16:32:00' AND timestamp < '2024-01-29 16:37:00' + GROUP BY peer_ip + ORDER BY withdrawal_count DESC + LIMIT 20; + + - id: q190 + title: "Cross-check: origin AS present in as_path array at all (1 day)" + analyst_question: "Are there announcements on Jan 6 where the declared origin AS does not appear anywhere in the AS path?" + window: 1 day + sql: | + SELECT prefix, origin, as_path, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-06 00:00:00' AND timestamp < '2024-01-07 00:00:00' + AND NOT has(splitByChar(' ', as_path), origin) + LIMIT 100; + + - id: q191 + title: "Update volume by peer for entire month, ranked with running total" + analyst_question: "What is the cumulative update contribution of each peer ranked over the full month?" + window: full month + sql: | + SELECT peer_asn, cnt, sum(cnt) OVER (ORDER BY cnt DESC) AS running_total + FROM ( + SELECT peer_asn, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY peer_asn + ) + ORDER BY cnt DESC + LIMIT 40; + + - id: q192 + title: "Update volume for a specific 3-day incident investigation window" + analyst_question: "What was the daily breakdown of updates during Jan 29-31 following a reported instability period?" + window: 3 days + sql: | + SELECT toDate(timestamp) AS day, operation, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-29 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day, operation + ORDER BY day, operation; + + - id: q193 + title: "Distinct AS path prefix (first ASN) diversity per collector day" + analyst_question: "How many distinct first-hop ASNs (immediate neighbors) were seen on Jan 13?" + window: 1 day + sql: | + SELECT uniqExact(splitByChar(' ', as_path)[1]) AS distinct_first_hops + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-13 00:00:00' AND timestamp < '2024-01-14 00:00:00'; + + - id: q194 + title: "Updates containing an empty AS path (possible iBGP or origin-only route) (1 day)" + analyst_question: "Were there any announcements on Jan 13 with a completely empty as_path?" + window: 1 day + sql: | + SELECT prefix, peer_asn, origin, timestamp + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND operation = 'A' + AND timestamp >= '2024-01-13 00:00:00' AND timestamp < '2024-01-14 00:00:00' + AND (as_path = '' OR as_path IS NULL) + LIMIT 50; + + - id: q195 + title: "Prefix-level summary table for a specific day (comprehensive)" + analyst_question: "Provide a full per-prefix summary (announcements, withdrawals, distinct origins, distinct peers) for Jan 15." + window: 1 day + sql: | + SELECT prefix, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals, + uniqExact(origin) AS distinct_origins, + uniqExact(peer_ip) AS distinct_peers + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-15 00:00:00' AND timestamp < '2024-01-16 00:00:00' + GROUP BY prefix + ORDER BY announcements + withdrawals DESC + LIMIT 50; + + - id: q196 + title: "Peer-level summary table for a full week" + analyst_question: "Provide a per-peer summary (total updates, distinct prefixes, announce/withdraw split) for the first week." + window: 1 week + sql: | + SELECT peer_ip, peer_asn, + count(*) AS total_updates, + uniqExact(prefix) AS distinct_prefixes, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-01-08 00:00:00' + GROUP BY peer_ip, peer_asn + ORDER BY total_updates DESC + LIMIT 50; + + - id: q197 + title: "Full-month executive summary of collector activity" + analyst_question: "Provide a single-row executive summary of rrc00 activity for all of January (totals and diversity metrics)." + window: full month + sql: | + SELECT count(*) AS total_updates, + countIf(operation = 'A') AS announcements, + countIf(operation = 'W') AS withdrawals, + uniqExact(prefix) AS distinct_prefixes, + uniqExact(origin) AS distinct_origin_asns, + uniqExact(peer_ip) AS distinct_peers + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; + + - id: q198 + title: "Hour-of-day seasonality across the full month" + analyst_question: "Is there a consistent hour-of-day pattern in update volume across all of January?" + window: full month + sql: | + SELECT toHour(timestamp) AS hour_of_day, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY hour_of_day + ORDER BY hour_of_day; + + - id: q199 + title: "Day-of-week seasonality across the full month" + analyst_question: "Does update volume vary systematically by day of week across January?" + window: full month + sql: | + SELECT toDayOfWeek(timestamp) AS day_of_week, count(*) AS cnt + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00' + GROUP BY day_of_week + ORDER BY day_of_week; + + - id: q200 + title: "Final data quality check: rows with unparseable numeric fields" + analyst_question: "How many rows in January have non-numeric or malformed local_pref or med values, indicating a data quality issue?" + window: full month + sql: | + SELECT + countIf(toString(local_pref) != '' AND toInt64OrNull(toString(local_pref)) IS NULL) AS bad_local_pref, + countIf(toString(med) != '' AND toInt64OrNull(toString(med)) IS NULL) AS bad_med + FROM bgp.bgp_updates + WHERE collector = 'rrc00' + AND timestamp >= '2024-01-01 00:00:00' AND timestamp < '2024-02-01 00:00:00'; \ No newline at end of file